HPCC-22038 Major ESDL transform script improvements

Add support for:

1. XPath functions for checking authorization
2. EXSLT xpath funcitons
3. Script parameters and variables
4. Scopes for variables.
5. Saving working cursor (xpathContext locations)
6. "for-each" operation.
7. relative path support (never worked prior)
8. "choose" with multiple "when" clauses.
9. "if" conditional
10. "assert" and "fail" operations.
11. Arbitrary namespace prefixes.
12. add-value (like ptree addProp)
13. Consistent operation names: AppendValue is now "append-to-value"
    and SetValue is now "set-value".
14. built target strings (use script level xpath to build target xpath)
15. unit tests

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexisrisk.com>

esp/esdllib/CMakeLists.txt

@@ -24,6 +24,7 @@ include_directories (
     ${HPCC_SOURCE_DIR}/esp/platform
     ${HPCC_SOURCE_DIR}/system/include
     ${HPCC_SOURCE_DIR}/esp/esdllib
+    ${HPCC_SOURCE_DIR}/esp/services/common
     ${HPCC_SOURCE_DIR}/system/jlib
     ${HPCC_SOURCE_DIR}/system/security/shared
     ${HPCC_SOURCE_DIR}/rtl/eclrtl
@@ -38,11 +39,22 @@ set ( SRCS
     esdl_def.cpp
     esdl_def_helper.cpp
     esdl_transformer2.cpp
+    esdl_script.cpp
     params2xml.cpp
     ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp/xpp/xpputils.cpp
     ${HPCC_SOURCE_DIR}/esp/services/common/wsexcept.cpp
 )
 
+if (USE_LIBXSLT)
+    list (APPEND SRCS
+        esdl_xpath_extensions_libxml.cpp
+    )
+else()
+    list (APPEND SRCS
+        esdl_xpath_extensions_unsupported.cpp
+    )
+endif()
+
 HPCC_ADD_LIBRARY( esdllib SHARED ${SRCS}
     ${HEADERS}
 )
@@ -56,3 +68,12 @@ target_link_libraries ( esdllib
     xmllib
     thorhelper
 )
+
+IF (USE_LIBXSLT)
+    include_directories (
+         ${LIBXML2_INCLUDE_DIR}
+    )
+    target_link_libraries ( esdl_svc_engine
+        ${LIBXML2_LIBRARIES}
+    )
+ENDIF()

esp/esdllib/esdl_script.cpp

@@ -0,0 +1,924 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "espcontext.hpp"
+#include "esdl_script.hpp"
+#include "wsexcept.hpp"
+
+interface IEsdlTransformOperation : public IInterface
+{
+    virtual const char *queryMergedTarget() = 0;
+    virtual bool process(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) = 0;
+    virtual void toDBGLog() = 0;
+};
+
+IEsdlTransformOperation *createEsdlTransformOperation(IPropertyTree *element, const StringBuffer &prefix);
+
+inline void esdlOperationError(int code, const char *op, const char *msg, const char *traceName, bool exception)
+{
+    StringBuffer s("ESDL Script: ");
+    if (!isEmptyString(traceName))
+        s.append(" '").append(traceName).append("' ");
+    if (!isEmptyString(op))
+        s.append(" ").append(op).append(" ");
+    s.append(msg);
+    if(exception)
+        throw MakeStringException(code, "%s", s.str());
+
+    IERRLOG("%s", s.str());
+}
+
+inline void esdlOperationError(int code, const char *op, const char *msg, bool exception)
+{
+    esdlOperationError(code, op, msg, "", exception);
+}
+
+static inline const char *checkSkipOpPrefix(const char *op, const StringBuffer &prefix)
+{
+    if (prefix.length())
+    {
+        if (!hasPrefix(op, prefix, true))
+        {
+            DBGLOG(1,"Unrecognized script operation: %s", op);
+            return nullptr;
+        }
+        return (op + prefix.length());
+    }
+    return op;
+}
+
+static inline StringBuffer &makeOperationTagName(StringBuffer &s, const StringBuffer &prefix, const char *op)
+{
+    return s.append(prefix).append(op);
+}
+
+class CEsdlTransformOperationBase : public CInterfaceOf<IEsdlTransformOperation>
+{
+protected:
+    StringAttr m_mergedTarget;
+    StringAttr m_tagname;
+    bool m_ignoreCodingErrors = false; //ideally used only for debugging
+
+public:
+    CEsdlTransformOperationBase(IPropertyTree *tree, const StringBuffer &prefix)
+    {
+        m_tagname.set(tree->queryName());
+        if (tree->hasProp("@_crtTarget"))
+            m_mergedTarget.set(tree->queryProp("@_crtTarget"));
+        m_ignoreCodingErrors = tree->getPropBool("@optional", false);
+    }
+
+    virtual const char *queryMergedTarget() override
+    {
+        return m_mergedTarget;
+    }
+};
+
+
+class CEsdlTransformOperationVariable : public CEsdlTransformOperationBase
+{
+protected:
+    StringAttr m_name;
+    Owned<ICompiledXpath> m_select;
+
+public:
+    CEsdlTransformOperationVariable(IPropertyTree *tree, const StringBuffer &prefix) : CEsdlTransformOperationBase(tree, prefix)
+    {
+        m_name.set(tree->queryProp("@name"));
+        if (m_name.isEmpty())
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, "without name", m_name, !m_ignoreCodingErrors);
+        const char *select = tree->queryProp("@select");
+        if (!isEmptyString(select))
+            m_select.setown(compileXpath(select));
+    }
+
+    virtual ~CEsdlTransformOperationVariable()
+    {
+    }
+
+    virtual bool process(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) override
+    {
+        if (m_select)
+            return xpathContext->addCompiledVariable(m_name, m_select);
+        return xpathContext->addVariable(m_name, "");
+    }
+
+    virtual void toDBGLog() override
+    {
+#if defined(_DEBUG)
+        DBGLOG(">%s> %s with select(%s)", m_name.str(), m_tagname.str(), m_select.get() ? m_select->getXpath() : "");
+#endif
+    }
+};
+
+class CEsdlTransformOperationParameter : public CEsdlTransformOperationVariable
+{
+public:
+    CEsdlTransformOperationParameter(IPropertyTree *tree, const StringBuffer &prefix) : CEsdlTransformOperationVariable(tree, prefix)
+    {
+    }
+
+    virtual ~CEsdlTransformOperationParameter()
+    {
+    }
+
+    virtual bool process(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) override
+    {
+        if (m_select)
+            return xpathContext->declareCompiledParameter(m_name, m_select);
+        return xpathContext->declareParameter(m_name, "");
+    }
+};
+
+class CEsdlTransformOperationSetValue : public CEsdlTransformOperationBase
+{
+protected:
+    Owned<ICompiledXpath> m_select;
+    Owned<ICompiledXpath> m_xpath_target;
+    StringAttr m_target;
+    StringAttr m_traceName;
+
+public:
+    CEsdlTransformOperationSetValue(IPropertyTree *tree, const StringBuffer &prefix) : CEsdlTransformOperationBase(tree, prefix)
+    {
+        m_traceName.set(tree->queryProp("@name"));
+
+        if (isEmptyString(tree->queryProp("@target")) && isEmptyString(tree->queryProp("@xpath_target")))
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname.str(), "without target", m_traceName.str(), !m_ignoreCodingErrors);
+
+        const char *select = tree->queryProp("@select");
+        if (isEmptyString(select))
+            select = tree->queryProp("@value");
+        if (isEmptyString(select))
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, "without select", m_traceName, !m_ignoreCodingErrors); //don't mention value, it's deprecated
+
+        m_select.setown(compileXpath(select));
+        if (tree->hasProp("@xpath_target"))
+            m_xpath_target.setown(compileXpath(tree->queryProp("@xpath_target")));
+        else if (tree->hasProp("@target"))
+            m_target.set(tree->queryProp("@target"));
+    }
+
+    virtual void toDBGLog() override
+    {
+#if defined(_DEBUG)
+        DBGLOG(">%s> %s(%s, select('%s'))", m_traceName.str(), m_tagname.str(), m_target.str(), m_select->getXpath());
+#endif
+    }
+
+    virtual ~CEsdlTransformOperationSetValue(){}
+
+    virtual bool process(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) override
+    {
+        if ((!m_xpath_target && m_target.isEmpty()) || !m_select)
+            return false; //only here if "optional" backward compatible support for now (optional syntax errors aren't actually helpful
+        try
+        {
+            StringBuffer value;
+            xpathContext->evaluateAsString(m_select, value);
+            return doSet(xpathContext, content, value);
+        }
+        catch (IException* e)
+        {
+            int code = e->errorCode();
+            StringBuffer msg;
+            e->errorMessage(msg);
+            e->Release();
+            esdlOperationError(code, m_tagname, msg, m_traceName, !m_ignoreCodingErrors);
+        }
+        catch (...)
+        {
+            esdlOperationError(ESDL_SCRIPT_Error, m_tagname, "unknown exception processing", m_traceName, !m_ignoreCodingErrors);
+        }
+        return false;
+    }
+
+    const char *getTargetPath(IXpathContext * xpathContext, StringBuffer &s)
+    {
+        if (m_xpath_target)
+        {
+            xpathContext->evaluateAsString(m_xpath_target, s);
+            return s;
+        }
+        return m_target.str();
+    }
+    virtual bool doSet(IXpathContext * xpathContext, IPropertyTree *tree, const char *value)
+    {
+        StringBuffer xpath;
+        const char *target = getTargetPath(xpathContext, xpath);
+        ensurePTree(tree, target);
+        tree->setProp(target, value);
+        return true;
+    }
+};
+
+class CEsdlTransformOperationAppendValue : public CEsdlTransformOperationSetValue
+{
+public:
+    CEsdlTransformOperationAppendValue(IPropertyTree *tree, const StringBuffer &prefix) : CEsdlTransformOperationSetValue(tree, prefix){}
+
+    virtual ~CEsdlTransformOperationAppendValue(){}
+
+    virtual bool doSet(IXpathContext * xpathContext, IPropertyTree *tree, const char *value)
+    {
+        StringBuffer xpath;
+        const char *target = getTargetPath(xpathContext, xpath);
+        ensurePTree(tree, target);
+        tree->appendProp(target, value);
+        return true;
+    }
+};
+
+class CEsdlTransformOperationAddValue : public CEsdlTransformOperationSetValue
+{
+public:
+    CEsdlTransformOperationAddValue(IPropertyTree *tree, const StringBuffer &prefix) : CEsdlTransformOperationSetValue(tree, prefix){}
+
+    virtual ~CEsdlTransformOperationAddValue(){}
+
+    virtual bool doSet(IXpathContext * xpathContext, IPropertyTree *tree, const char *value)
+    {
+        StringBuffer xpath;
+        const char *target = getTargetPath(xpathContext, xpath);
+        if (tree->getCount(target)==0)
+        {
+            ensurePTree(tree, target);
+            tree->setProp(target, value);
+        }
+        else
+            tree->addProp(target, value);
+        return true;
+    }
+};
+
+class CEsdlTransformOperationFail : public CEsdlTransformOperationBase
+{
+protected:
+    StringAttr m_traceName;
+    Owned<ICompiledXpath> m_message;
+    Owned<ICompiledXpath> m_code;
+
+public:
+    CEsdlTransformOperationFail(IPropertyTree *tree, const StringBuffer &prefix) : CEsdlTransformOperationBase(tree, prefix)
+    {
+        m_traceName.set(tree->queryProp("@name"));
+
+        if (isEmptyString(tree->queryProp("@code")))
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, "without code", m_traceName.str(), true);
+        if (isEmptyString(tree->queryProp("@message")))
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, "without message", m_traceName.str(), true);
+
+        m_code.setown(compileXpath(tree->queryProp("@code")));
+        m_message.setown(compileXpath(tree->queryProp("@message")));
+    }
+
+    virtual ~CEsdlTransformOperationFail()
+    {
+    }
+
+    virtual bool process(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) override
+    {
+        int code = m_code.get() ? (int) xpathContext->evaluateAsNumber(m_code) : ESDL_SCRIPT_Error;
+        StringBuffer msg;
+        if (m_message.get())
+            xpathContext->evaluateAsString(m_message, msg);
+        throw makeStringException(code, msg.str());
+        return true; //avoid compilation error
+    }
+
+    virtual void toDBGLog() override
+    {
+#if defined(_DEBUG)
+        DBGLOG(">%s> %s with message(%s)", m_traceName.str(), m_tagname.str(), m_message.get() ? m_message->getXpath() : "");
+#endif
+    }
+};
+
+class CEsdlTransformOperationAssert : public CEsdlTransformOperationFail
+{
+private:
+    Owned<ICompiledXpath> m_test; //assert is like a conditional fail
+
+public:
+    CEsdlTransformOperationAssert(IPropertyTree *tree, const StringBuffer &prefix) : CEsdlTransformOperationFail(tree, prefix)
+    {
+        if (isEmptyString(tree->queryProp("@test")))
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, "without test", m_traceName.str(), true);
+        m_test.setown(compileXpath(tree->queryProp("@test")));
+    }
+
+    virtual ~CEsdlTransformOperationAssert()
+    {
+    }
+
+    virtual bool process(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) override
+    {
+        if (m_test && xpathContext->evaluateAsBoolean(m_test))
+            return false;
+        return CEsdlTransformOperationFail::process(context, content, xpathContext);
+    }
+
+    virtual void toDBGLog() override
+    {
+#if defined(_DEBUG)
+        const char *testXpath = m_test.get() ? m_test->getXpath() : "SYNTAX ERROR IN test";
+        DBGLOG(">%s> %s if '%s' with message(%s)", m_traceName.str(), m_tagname.str(), testXpath, m_message.get() ? m_message->getXpath() : "");
+#endif
+    }
+};
+
+class CEsdlTransformOperationWithChildren : public CEsdlTransformOperationBase
+{
+protected:
+    IArrayOf<IEsdlTransformOperation> m_children;
+
+public:
+    CEsdlTransformOperationWithChildren(IPropertyTree *tree, const StringBuffer &prefix, bool withVariables) : CEsdlTransformOperationBase(tree, prefix)
+    {
+        if (tree)
+            loadChildren(tree, prefix, withVariables);
+    }
+
+    virtual ~CEsdlTransformOperationWithChildren(){}
+
+    virtual bool processChildren(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext)
+    {
+        bool ret = false;
+        ForEachItemIn(i, m_children)
+        {
+            if (m_children.item(i).process(context, content, xpathContext))
+                ret = true;
+        }
+        return ret;
+    }
+
+    virtual void toDBGLog () override
+    {
+    #if defined(_DEBUG)
+        ForEachItemIn(i, m_children)
+            m_children.item(i).toDBGLog();
+    #endif
+    }
+
+protected:
+    virtual void loadChildren(IPropertyTree * tree, const StringBuffer &prefix, bool withVariables)
+    {
+        if (withVariables)
+        {
+            StringBuffer xpath;
+            Owned<IPropertyTreeIterator> parameters = tree->getElements(makeOperationTagName(xpath, prefix, "param"));
+            ForEach(*parameters)
+                m_children.append(*new CEsdlTransformOperationParameter(&parameters->query(), prefix));
+
+            Owned<IPropertyTreeIterator> variables = tree->getElements(makeOperationTagName(xpath.clear(), prefix, "variable"));
+            ForEach(*variables)
+                m_children.append(*new CEsdlTransformOperationVariable(&variables->query(), prefix));
+        }
+        Owned<IPropertyTreeIterator> children = tree->getElements("*");
+        ForEach(*children)
+        {
+            Owned<IEsdlTransformOperation> operation = createEsdlTransformOperation(&children->query(), prefix);
+            if (operation)
+                m_children.append(*operation.getClear());
+        }
+    }
+};
+
+class CEsdlTransformOperationForEach : public CEsdlTransformOperationWithChildren
+{
+protected:
+    Owned<ICompiledXpath> m_select;
+
+public:
+    CEsdlTransformOperationForEach(IPropertyTree *tree, const StringBuffer &prefix) : CEsdlTransformOperationWithChildren(tree, prefix, true)
+    {
+        if (tree)
+        {
+            if (isEmptyString(tree->queryProp("@select")))
+                esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, "without select", !m_ignoreCodingErrors);
+            m_select.setown(compileXpath(tree->queryProp("@select")));
+        }
+    }
+
+    virtual ~CEsdlTransformOperationForEach(){}
+
+    bool process(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) override
+    {
+        Owned<IXpathContextIterator> contexts = evaluate(xpathContext);
+        if (!contexts)
+            return false;
+        if (!contexts->first())
+            return false;
+        CXpathContextScope scope(xpathContext, "for-each"); //new variables are scoped
+        ForEach(*contexts)
+            processChildren(context, content, &contexts->query());
+        return true;
+    }
+
+    virtual void toDBGLog () override
+    {
+    #if defined(_DEBUG)
+        DBGLOG (">>>>%s %s ", m_tagname.str(), m_select ? m_select->getXpath() : "");
+        CEsdlTransformOperationWithChildren::toDBGLog();
+        DBGLOG ("<<<<%s<<<<<", m_tagname.str());
+    #endif
+    }
+
+private:
+    IXpathContextIterator *evaluate(IXpathContext * xpathContext)
+    {
+        IXpathContextIterator *xpathset = nullptr;
+        try
+        {
+            xpathset = xpathContext->evaluateAsNodeSet(m_select);
+        }
+        catch (IException* e)
+        {
+            int code = e->errorCode();
+            StringBuffer msg;
+            e->errorMessage(msg);
+            e->Release();
+            esdlOperationError(code, m_tagname, msg, !m_ignoreCodingErrors);
+        }
+        catch (...)
+        {
+            VStringBuffer msg("unknown exception evaluating select '%s'", m_select.get() ? m_select->getXpath() : "undefined!");
+            esdlOperationError(ESDL_SCRIPT_Error, m_tagname, msg, !m_ignoreCodingErrors);
+        }
+        return xpathset;
+    }
+};
+
+class CEsdlTransformOperationConditional : public CEsdlTransformOperationWithChildren
+{
+private:
+    Owned<ICompiledXpath> m_test;
+    char m_op = 'i'; //'i'=if, 'w'=when, 'o'=otherwise
+
+public:
+    CEsdlTransformOperationConditional(IPropertyTree * tree, const StringBuffer &prefix) : CEsdlTransformOperationWithChildren(tree, prefix, true)
+    {
+        if (tree)
+        {
+            const char *op = checkSkipOpPrefix(tree->queryName(), prefix);
+            if (isEmptyString(op))
+                esdlOperationError(ESDL_SCRIPT_UnknownOperation, m_tagname, "unrecognized conditional", !m_ignoreCodingErrors);
+            //m_ignoreCodingErrors means op may still be null
+            if (!op || streq(op, "if"))
+                m_op = 'i';
+            else if (streq(op, "when"))
+                m_op = 'w';
+            else if (streq(op, "otherwise"))
+                m_op = 'o';
+            if (m_op!='o')
+            {
+                if (isEmptyString(tree->queryProp("@test")))
+                    esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, "without test", !m_ignoreCodingErrors);
+                m_test.setown(compileXpath(tree->queryProp("@test")));
+            }
+        }
+    }
+
+    virtual ~CEsdlTransformOperationConditional(){}
+
+    bool process(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) override
+    {
+        if (!evaluate(xpathContext))
+            return false;
+        CXpathContextScope scope(xpathContext, m_tagname); //child variables are scoped
+        processChildren(context, content, xpathContext);
+        return true; //just means that evaluation succeeded and attempted to process children
+    }
+
+    virtual void toDBGLog () override
+    {
+    #if defined(_DEBUG)
+        DBGLOG (">>>>%s %s ", m_tagname.str(), m_test ? m_test->getXpath() : "");
+        CEsdlTransformOperationWithChildren::toDBGLog();
+        DBGLOG ("<<<<%s<<<<<", m_tagname.str());
+    #endif
+    }
+
+private:
+    bool evaluate(IXpathContext * xpathContext)
+    {
+        if (m_op=='o')  //'o'/"otherwise" is unconditional
+            return true;
+        bool match = false;
+        try
+        {
+            match = xpathContext->evaluateAsBoolean(m_test);
+        }
+        catch (IException* e)
+        {
+            int code = e->errorCode();
+            StringBuffer msg;
+            e->errorMessage(msg);
+            e->Release();
+            esdlOperationError(code, m_tagname, msg, !m_ignoreCodingErrors);
+        }
+        catch (...)
+        {
+            VStringBuffer msg("unknown exception evaluating test '%s'", m_test.get() ? m_test->getXpath() : "undefined!");
+            esdlOperationError(ESDL_SCRIPT_Error, m_tagname, msg, !m_ignoreCodingErrors);
+        }
+        return match;
+    }
+};
+
+class CEsdlTransformOperationChoose : public CEsdlTransformOperationWithChildren
+{
+public:
+    CEsdlTransformOperationChoose(IPropertyTree * tree, const StringBuffer &prefix) : CEsdlTransformOperationWithChildren(tree, prefix, false)
+    {
+        if (tree)
+        {
+            loadWhens(tree, prefix);
+            loadOtherwise(tree, prefix);
+        }
+    }
+
+    virtual ~CEsdlTransformOperationChoose(){}
+
+    bool process(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) override
+    {
+        return processChildren(context, content, xpathContext);
+    }
+
+    virtual bool processChildren(IEspContext * context, IPropertyTree *content, IXpathContext * xpathContext) override
+    {
+        ForEachItemIn(i, m_children)
+        {
+            if (m_children.item(i).process(context, content, xpathContext))
+                return true;
+        }
+        return false;
+    }
+
+    virtual void toDBGLog () override
+    {
+    #if defined(_DEBUG)
+        DBGLOG (">>>>>>>>>>> %s >>>>>>>>>>", m_tagname.str());
+        CEsdlTransformOperationWithChildren::toDBGLog();
+        DBGLOG (">>>>>>>>>>> %s >>>>>>>>>>", m_tagname.str());
+    #endif
+    }
+
+protected:
+    void loadWhens(IPropertyTree * tree, const StringBuffer &prefix)
+    {
+        StringBuffer xpath;
+        Owned<IPropertyTreeIterator> children = tree->getElements(makeOperationTagName(xpath, prefix, "when"));
+        ForEach(*children)
+            m_children.append(*new CEsdlTransformOperationConditional(&children->query(), prefix));
+    }
+
+    void loadOtherwise(IPropertyTree * tree, const StringBuffer &prefix)
+    {
+        StringBuffer xpath;
+        IPropertyTree * otherwise = tree->queryPropTree(makeOperationTagName(xpath, prefix, "otherwise"));
+        if (!otherwise)
+            return;
+        m_children.append(*new CEsdlTransformOperationConditional(otherwise, prefix));
+    }
+    virtual void loadChildren(IPropertyTree * tree, const StringBuffer &prefix, bool withVariables) override
+    {
+        loadWhens(tree, prefix);
+        loadOtherwise(tree, prefix);
+    }
+};
+
+IEsdlTransformOperation *createEsdlTransformOperation(IPropertyTree *element, const StringBuffer &prefix)
+{
+    const char *op = checkSkipOpPrefix(element->queryName(), prefix);
+    if (isEmptyString(op))
+        return nullptr;
+    if (streq(op, "choose"))
+        return new CEsdlTransformOperationChoose(element, prefix);
+    if (streq(op, "for-each"))
+        return new CEsdlTransformOperationForEach(element, prefix);
+    if (streq(op, "if"))
+        return new CEsdlTransformOperationConditional(element, prefix);
+    if (streq(op, "set-value") || streq(op, "SetValue"))
+        return new CEsdlTransformOperationSetValue(element, prefix);
+    if (streq(op, "append-to-value") || streq(op, "AppendValue"))
+        return new CEsdlTransformOperationAppendValue(element, prefix);
+    if (streq(op, "add-value"))
+        return new CEsdlTransformOperationAddValue(element, prefix);
+    if (streq(op, "fail"))
+        return new CEsdlTransformOperationFail(element, prefix);
+    if (streq(op, "assert"))
+        return new CEsdlTransformOperationAssert(element, prefix);
+    return nullptr;
+}
+
+static IPropertyTree *getTargetPTree(IPropertyTree *tree, IXpathContext *xpathContext, const char *target)
+{
+    StringBuffer xpath(target);
+    if (xpath.length())
+    {
+        //we can use real xpath processing in the future, for now simple substitution is fine
+        StringBuffer variable;
+        xpath.replaceString("{$query}", xpathContext->getVariable("query", variable));
+        xpath.replaceString("{$method}", xpathContext->getVariable("method", variable.clear()));
+        xpath.replaceString("{$service}", xpathContext->getVariable("service", variable.clear()));
+        xpath.replaceString("{$request}", xpathContext->getVariable("request", variable.clear()));
+
+        IPropertyTree *child = tree->queryPropTree(xpath.str());  //get pointer to the write-able area
+        if (!child)
+            throw MakeStringException(ESDL_SCRIPT_Error, "EsdlCustomTransform error getting target xpath %s", xpath.str());
+        return child;
+    }
+    return tree;
+}
+static IPropertyTree *getOperationTargetPTree(MapStringToMyClass<IPropertyTree> &treeMap, IPropertyTree *currentTree, IEsdlTransformOperation &operation, IPropertyTree *tree, IXpathContext *xpathContext, const char *target)
+{
+    const char *mergedTarget = operation.queryMergedTarget();
+    if (isEmptyString(mergedTarget) || streq(mergedTarget, target))
+        return currentTree;
+    IPropertyTree *opTree = treeMap.getValue(mergedTarget);
+    if (opTree)
+        return opTree;
+    opTree = getTargetPTree(tree, xpathContext, mergedTarget);
+    if (opTree)
+        treeMap.setValue(mergedTarget, LINK(opTree));
+    return opTree;
+}
+
+class CEsdlCustomTransform : public CInterfaceOf<IEsdlCustomTransform>
+{
+private:
+    IArrayOf<IEsdlTransformOperation> m_variables; //keep separate and only at top level for now
+    IArrayOf<IEsdlTransformOperation> m_operations;
+    StringAttr m_name;
+    StringAttr m_target;
+    StringBuffer m_prefix;
+
+public:
+    CEsdlCustomTransform(){}
+    void verifyPrefixDeclared(IPropertyTree &tree, const char *prefix)
+    {
+        StringBuffer attpath("@xmlns");
+        if (!isEmptyString(prefix))
+            attpath.append(':').append(prefix);
+        const char *uri = tree.queryProp(attpath.str());
+        if (!uri || !streq(uri, "urn:hpcc:esdl:script"))
+            throw MakeStringException(ESDL_SCRIPT_Error, "Undeclared script xmlns prefix %s", prefix);
+    }
+    CEsdlCustomTransform(IPropertyTree &tree, const char *ns_prefix) : m_prefix(ns_prefix)
+    {
+        if (m_prefix.length())
+            m_prefix.append(':');
+        else
+        {
+            const char *tag = tree.queryName();
+            if (!tag)
+                m_prefix.set("xsdl:");
+            else
+            {
+                const char *colon = strchr(tag, ':');
+                if (!colon)
+                    verifyPrefixDeclared(tree, nullptr);
+                else
+                {
+                    if (colon == tag)
+                        throw MakeStringException(ESDL_SCRIPT_Error, "Tag shouldn't start with colon %s", tag);
+                    m_prefix.append(colon-tag, tag);
+                    if (!streq(m_prefix, "xsdl"))
+                        verifyPrefixDeclared(tree, m_prefix);
+                    //add back the colon for easy comparison
+                    m_prefix.append(':');
+                }
+            }
+        }
+
+        m_name.set(tree.queryProp("@name"));
+        m_target.set(tree.queryProp("@target"));
+
+        DBGLOG("Compiling custom ESDL Transform: '%s'", m_name.str());
+
+        StringBuffer xpath;
+        Owned<IPropertyTreeIterator> parameters = tree.getElements(makeOperationTagName(xpath, m_prefix, "param"));
+        ForEach(*parameters)
+            m_variables.append(*new CEsdlTransformOperationParameter(&parameters->query(), m_prefix));
+
+        Owned<IPropertyTreeIterator> variables = tree.getElements(makeOperationTagName(xpath.clear(), m_prefix, "variable"));
+        ForEach(*variables)
+            m_variables.append(*new CEsdlTransformOperationVariable(&variables->query(), m_prefix));
+
+        Owned<IPropertyTreeIterator> children = tree.getElements("*");
+        ForEach(*children)
+        {
+            Owned<IEsdlTransformOperation> operation = createEsdlTransformOperation(&children->query(), m_prefix);
+            if (operation)
+                m_operations.append(*operation.getClear());
+        }
+    }
+
+    virtual void appendEsdlURIPrefixes(StringArray &prefixes) override
+    {
+        if (m_prefix.length())
+        {
+            StringAttr copy(m_prefix.str(), m_prefix.length()-1); //remove the colon
+            prefixes.appendUniq(copy.str());
+        }
+        else
+            prefixes.appendUniq("");
+    }
+    virtual void toDBGLog() override
+    {
+#if defined(_DEBUG)
+        DBGLOG(">>>>>>>>>>>>>>>>transform: '%s'>>>>>>>>>>", m_name.str());
+        ForEachItemIn(i, m_operations)
+            m_operations.item(i).toDBGLog();
+        DBGLOG("<<<<<<<<<<<<<<<<transform<<<<<<<<<<<<");
+#endif
+     }
+
+    virtual ~CEsdlCustomTransform(){}
+
+    void processTransformImpl(IEspContext * context, IPropertyTree *theroot, IXpathContext *xpathContext, const char *target)
+    {
+        CXpathContextScope scope(xpathContext, "transform");
+
+        if (m_target.length())
+            target = m_target.str();
+        MapStringToMyClass<IPropertyTree> treeMap; //cache trees because when there are merged targets they are likely to repeat
+        IPropertyTree *txTree = getTargetPTree(theroot, xpathContext, target);
+        treeMap.setValue(target, LINK(txTree));
+        ForEachItemIn(v, m_variables)
+            m_variables.item(v).process(context, txTree, xpathContext);
+        ForEachItemIn(i, m_operations)
+        {
+            IPropertyTree *opTree = getOperationTargetPTree(treeMap, txTree, m_operations.item(i), theroot, xpathContext, target);
+            m_operations.item(i).process(context, opTree, xpathContext);
+        }
+    }
+
+    void processTransform(IEspContext * context, IPropertyTree *tgtcfg, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & content, IPropertyTree * bindingCfg) override
+    {
+        processServiceAndMethodTransforms({static_cast<IEsdlCustomTransform*>(this)}, context, tgtcfg, srvdef, mthdef, content, bindingCfg);
+    }
+    void processTransform(IEspContext * context, IPropertyTree *tgtcfg, const char *service, const char *method, const char* reqtype, StringBuffer & content, IPropertyTree * bindingCfg) override
+    {
+        processServiceAndMethodTransforms({static_cast<IEsdlCustomTransform*>(this)}, context, tgtcfg, service, method, reqtype, content, bindingCfg);
+    }
+};
+
+void processServiceAndMethodTransforms(std::initializer_list<IEsdlCustomTransform *> const &transforms, IEspContext * context, IPropertyTree *tgtcfg, const char *service, const char *method, const char* reqtype, StringBuffer & content, IPropertyTree * bindingCfg)
+{
+    LogLevel level = LogMin;
+    if (!transforms.size())
+        return;
+    if (tgtcfg)
+        level = (unsigned) tgtcfg->getPropInt("@traceLevel", level);
+
+    if (content.length()!=0)
+    {
+        if (level >= LogMax)
+        {
+            DBGLOG("ORIGINAL content: %s", content.str());
+            StringBuffer marshalled;
+            if (bindingCfg)
+                toXML(bindingCfg, marshalled.clear());
+            DBGLOG("BINDING CONFIG: %s", marshalled.str());
+            if (tgtcfg)
+                toXML(tgtcfg, marshalled.clear());
+            DBGLOG("TARGET CONFIG: %s", marshalled.str());
+        }
+
+        bool strictParams = bindingCfg ? bindingCfg->getPropBool("@strictParams", false) : false;
+        Owned<IXpathContext> xpathContext = getXpathContext(content.str(), strictParams);
+
+        StringArray prefixes;
+        for ( IEsdlCustomTransform * const & item : transforms)
+        {
+            if (item)
+                item->appendEsdlURIPrefixes(prefixes);
+        }
+
+        registerEsdlXPathExtensions(xpathContext, context, prefixes);
+
+        VStringBuffer ver("%g", context->getClientVersion());
+        if(!xpathContext->addVariable("clientversion", ver.str()))
+            OERRLOG("Could not set ESDL Script variable: clientversion:'%s'", ver.str());
+
+        //in case transform wants to make use of these values:
+        //make them few well known values variables rather than inputs so they are automatically available
+        xpathContext->addVariable("query", tgtcfg->queryProp("@queryname"));
+        xpathContext->addVariable("method", method);
+        xpathContext->addVariable("service", service);
+        xpathContext->addVariable("request", reqtype);
+
+        ISecUser *user = context->queryUser();
+        if (user)
+        {
+            static const std::map<SecUserStatus, const char*> statusLabels =
+            {
+#define STATUS_LABEL_NODE(s) { s, #s }
+                STATUS_LABEL_NODE(SecUserStatus_Inhouse),
+                STATUS_LABEL_NODE(SecUserStatus_Active),
+                STATUS_LABEL_NODE(SecUserStatus_Exempt),
+                STATUS_LABEL_NODE(SecUserStatus_FreeTrial),
+                STATUS_LABEL_NODE(SecUserStatus_csdemo),
+                STATUS_LABEL_NODE(SecUserStatus_Rollover),
+                STATUS_LABEL_NODE(SecUserStatus_Suspended),
+                STATUS_LABEL_NODE(SecUserStatus_Terminated),
+                STATUS_LABEL_NODE(SecUserStatus_TrialExpired),
+                STATUS_LABEL_NODE(SecUserStatus_Status_Hold),
+                STATUS_LABEL_NODE(SecUserStatus_Unknown),
+#undef STATUS_LABEL_NODE
+            };
+
+            Owned<IPropertyIterator> userPropIt = user->getPropertyIterator();
+            ForEach(*userPropIt)
+            {
+                const char *name = userPropIt->getPropKey();
+                if (name && *name)
+                    xpathContext->addInputValue(name, user->getProperty(name));
+            }
+
+            auto it = statusLabels.find(user->getStatus());
+
+            xpathContext->addInputValue("espUserName", user->getName());
+            xpathContext->addInputValue("espUserRealm", user->getRealm() ? user->getRealm() : "");
+            xpathContext->addInputValue("espUserPeer", user->getPeer() ? user->getPeer() : "");
+            xpathContext->addInputValue("espUserStatus", VStringBuffer("%d", int(user->getStatus())));
+            if (it != statusLabels.end())
+                xpathContext->addInputValue("espUserStatusString", it->second);
+            else
+                throw MakeStringException(ESDL_SCRIPT_Error, "encountered unexpected secure user status (%d) while processing transform", int(user->getStatus()));
+        }
+        else
+        {
+            // enable transforms to distinguish secure versus insecure requests
+            xpathContext->addInputValue("espUserName", "");
+            xpathContext->addInputValue("espUserRealm", "");
+            xpathContext->addInputValue("espUserPeer", "");
+            xpathContext->addInputValue("espUserStatus", "");
+            xpathContext->addInputValue("espUserStatusString", "");
+        }
+
+        //external parameters need <es:param> statements to make them accessible (in strict mode)
+        Owned<IPropertyTreeIterator> configParams;
+        if (bindingCfg)
+            configParams.setown(bindingCfg->getElements("Transform/Param"));
+        if (configParams)
+        {
+            ForEach(*configParams)
+            {
+                IPropertyTree & currentParam = configParams->query();
+                if (currentParam.hasProp("@select"))
+                    xpathContext->addInputXpath(currentParam.queryProp("@name"), currentParam.queryProp("@select"));
+                else
+                    xpathContext->addInputValue(currentParam.queryProp("@name"), currentParam.queryProp("@value"));
+            }
+        }
+        if (!strictParams)
+            xpathContext->declareRemainingInputs();
+
+        Owned<IPropertyTree> theroot = createPTreeFromXMLString(content.str());
+        StringBuffer defaultTarget;
+            //This default gives us backward compatibility with only being able to write to the actual request
+        const char *tgtQueryName = tgtcfg->queryProp("@queryname");
+        defaultTarget.setf("soap:Body/%s/%s", tgtQueryName ? tgtQueryName : method, reqtype);
+
+        for ( auto&& item : transforms)
+        {
+            if (item)
+            {
+                CEsdlCustomTransform *transform = static_cast<CEsdlCustomTransform*>(item);
+                transform->processTransformImpl(context, theroot, xpathContext, defaultTarget);
+            }
+        }
+
+        toXML(theroot, content.clear());
+
+        if (level >= LogMax)
+            DBGLOG(1,"MODIFIED content: %s", content.str());
+    }
+}
+
+void processServiceAndMethodTransforms(std::initializer_list<IEsdlCustomTransform *> const &transforms, IEspContext * context, IPropertyTree *tgtcfg, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & content, IPropertyTree * bindingCfg)
+{
+    processServiceAndMethodTransforms(transforms, context, tgtcfg, srvdef.queryName(), mthdef.queryMethodName(), mthdef.queryRequestType(), content, bindingCfg);
+}
+
+IEsdlCustomTransform *createEsdlCustomTransform(IPropertyTree &tree, const char *ns_prefix)
+{
+    return new CEsdlCustomTransform(tree, ns_prefix);
+}

esp/esdllib/esdl_script.hpp

@@ -0,0 +1,63 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef ESDL_SCRIPT_HPP_
+#define ESDL_SCRIPT_HPP_
+
+#ifdef ESDLLIB_EXPORTS
+ #define esdl_decl DECL_EXPORT
+#else
+ #define esdl_decl
+#endif
+
+#include "jlib.hpp"
+#include "jstring.hpp"
+#include "jptree.hpp"
+#include "jlog.hpp"
+#include "esp.hpp"
+
+#include "esdl_def.hpp"
+
+#include <map>
+#include <mutex>
+#include <thread>
+#include <initializer_list>
+
+#include "tokenserialization.hpp"
+#include "xpathprocessor.hpp"
+
+#define ESDL_SCRIPT_Error                         5700
+#define ESDL_SCRIPT_MissingOperationAttr          5710
+#define ESDL_SCRIPT_UnknownOperation              5720
+
+interface IEsdlCustomTransform : extends IInterface
+{
+    virtual void processTransform(IEspContext * context, IPropertyTree *tgtcfg, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & content, IPropertyTree * bindingCfg) = 0;
+    virtual void processTransform(IEspContext * context, IPropertyTree *tgtcfg, const char *service, const char *method, const char* reqtype, StringBuffer & content, IPropertyTree * bindingCfg) = 0;
+    virtual void appendEsdlURIPrefixes(StringArray &prefixes) = 0;
+    virtual void toDBGLog() = 0;
+};
+
+
+esdl_decl void processServiceAndMethodTransforms(std::initializer_list<IEsdlCustomTransform *> const &transforms, IEspContext * context, IPropertyTree *tgtcfg, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & content, IPropertyTree * bindingCfg);
+esdl_decl void processServiceAndMethodTransforms(std::initializer_list<IEsdlCustomTransform *> const &transforms, IEspContext * context, IPropertyTree *tgtcfg, const char *service, const char *method, const char* reqtype, StringBuffer & content, IPropertyTree * bindingCfg);
+
+esdl_decl IEsdlCustomTransform *createEsdlCustomTransform(IPropertyTree &customRequestTransform, const char *ns_prefix);
+
+esdl_decl void registerEsdlXPathExtensions(IXpathContext *xpathCtx, IEspContext *espCtx, const StringArray &prefixes);
+
+#endif /* ESDL_SCRIPT_HPP_ */

esp/esdllib/esdl_xpath_extensions_libxml.cpp

@@ -0,0 +1,206 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jstring.hpp"
+#include "jdebug.hpp"
+#include "jptree.hpp"
+#include "jexcept.hpp"
+#include "jlog.hpp"
+
+#include <libxml/xmlmemory.h>
+#include <libxml/parserInternals.h>
+#include <libxml/debugXML.h>
+#include <libxml/HTMLtree.h>
+#include <libxml/xmlIO.h>
+#include <libxml/xinclude.h>
+#include <libxml/catalog.h>
+#include <libxml/xpathInternals.h>
+#include <libxml/xpath.h>
+#include <libxml/xmlschemas.h>
+#include <libxml/hash.h>
+
+#include "xpathprocessor.hpp"
+#include "xmlerror.hpp"
+
+#include "espcontext.hpp"
+#include "esdl_script.hpp"
+
+//only support libxml2 for now
+
+void addFeaturesToAccessMap(MapStringTo<SecAccessFlags> &accessmap, const char *s)
+{
+    StringArray entries;
+    entries.appendList(s, ",");
+
+    ForEachItemIn(i, entries)
+    {
+        StringArray pair;
+        pair.appendList(entries.item(i), ":");
+        if (pair.length()==0)
+            continue;
+        if (pair.length()==1)
+            accessmap.setValue(pair.item(0), SecAccess_Read);
+        else
+        {
+            SecAccessFlags required = getSecAccessFlagValue(pair.item(1));
+            if (required >= SecAccess_None)
+                accessmap.setValue(pair.item(0), required);
+        }
+    }
+}
+
+/**
+ * validateFeaturesAccessFunction:
+ * @ctxt:  an XPath parser context
+ * @nargs:  the number of arguments
+ *
+ * Wraps IEspContext::validateFeaturesAccess()
+ */
+static void validateFeaturesAccessFunction (xmlXPathParserContextPtr ctxt, int nargs)
+{
+    if (!ctxt || !ctxt->context || !ctxt->context->userData)
+    {
+        xmlXPathSetError((ctxt), XPATH_INVALID_CTXT);
+        return;
+    }
+
+    IEspContext *espContext = reinterpret_cast<IEspContext *>(ctxt->context->userData);
+
+    if (nargs != 1)
+    {
+        xmlXPathSetArityError(ctxt);
+        return;
+    }
+
+    xmlChar *authstring = xmlXPathPopString(ctxt);
+    if (xmlXPathCheckError(ctxt)) //includes null check
+        return;
+
+    MapStringTo<SecAccessFlags> accessmap;
+    addFeaturesToAccessMap(accessmap, (const char *)authstring);
+
+    bool ok = true;
+    if (accessmap.ordinality()!=0)
+        ok = espContext->validateFeaturesAccess(accessmap, false);
+
+    if (authstring != nullptr)
+        xmlFree(authstring);
+
+    xmlXPathReturnBoolean(ctxt, ok ? 1 : 0);
+}
+
+/**
+ * evaluateSecAccessFlagsFunction
+ * @ctxt:  an XPath parser context
+ * @nargs:  the number of arguments
+ *
+ */
+static void secureAccessFlagsFunction (xmlXPathParserContextPtr ctxt, int nargs)
+{
+    if (!ctxt || !ctxt->context || !ctxt->context->userData)
+    {
+        xmlXPathSetError((ctxt), XPATH_INVALID_CTXT);
+        return;
+    }
+
+    IEspContext *espContext = reinterpret_cast<IEspContext *>(ctxt->context->userData);
+
+    if (nargs == 0)
+    {
+        xmlXPathSetArityError(ctxt);
+        return;
+    }
+
+    unsigned flags = 0;
+    while(nargs--)
+    {
+        xmlChar *s = xmlXPathPopString(ctxt);
+        if (xmlXPathCheckError(ctxt)) //includes null check
+            return;
+        SecAccessFlags f = getSecAccessFlagValue((const char *)s);
+        xmlFree(s);
+        if (f < SecAccess_None)
+        {
+            xmlXPathSetArityError(ctxt);
+            return;
+        }
+        flags |= (unsigned)f;
+    }
+
+    xmlXPathReturnNumber(ctxt, flags);
+}
+/**
+ * getFeatureSecAccessFlags
+ * @ctxt:  an XPath parser context
+ * @nargs:  the number of arguments
+ *
+ */
+static void getFeatureSecAccessFlagsFunction (xmlXPathParserContextPtr ctxt, int nargs)
+{
+    if (!ctxt || !ctxt->context || !ctxt->context->userData)
+    {
+        xmlXPathSetError((ctxt), XPATH_INVALID_CTXT);
+        return;
+    }
+
+    IEspContext *espContext = reinterpret_cast<IEspContext *>(ctxt->context->userData);
+
+    if (nargs != 1)
+    {
+        xmlXPathSetArityError(ctxt);
+        return;
+    }
+
+    xmlChar *authstring = xmlXPathPopString(ctxt);
+    if (xmlXPathCheckError(ctxt)) //includes null check
+        return;
+
+    SecAccessFlags access = SecAccess_None;
+    espContext->authorizeFeature((const char *)authstring, access);
+    xmlFree(authstring);
+
+    xmlXPathReturnNumber(ctxt, access);
+}
+
+void registerEsdlXPathExtensions(IXpathContext *xpathContext, IEspContext *context, const StringArray &prefixes)
+{
+    bool includeDefaultNS = false;
+    xpathContext->setUserData(context);
+    if (!prefixes.ordinality())
+        xpathContext->registerNamespace("esdl", "urn:hpcc:esdl:script");
+    else
+    {
+        ForEachItemIn(i, prefixes)
+        {
+            if (isEmptyString(prefixes.item(i)))
+                includeDefaultNS=true;
+            else
+                xpathContext->registerNamespace(prefixes.item(i), "urn:hpcc:esdl:script");
+        }
+    }
+
+    if (includeDefaultNS)
+    {
+        xpathContext->registerFunction(nullptr, "validateFeaturesAccess", (void  *)validateFeaturesAccessFunction);
+        xpathContext->registerFunction(nullptr, "secureAccessFlags", (void  *)secureAccessFlagsFunction);
+        xpathContext->registerFunction(nullptr, "getFeatureSecAccessFlags", (void  *)getFeatureSecAccessFlagsFunction);
+    }
+
+    xpathContext->registerFunction("urn:hpcc:esdl:script", "validateFeaturesAccess", (void  *)validateFeaturesAccessFunction);
+    xpathContext->registerFunction("urn:hpcc:esdl:script", "secureAccessFlags", (void  *)secureAccessFlagsFunction);
+    xpathContext->registerFunction("urn:hpcc:esdl:script", "getFeatureSecAccessFlags", (void  *)getFeatureSecAccessFlagsFunction);
+}

esp/esdllib/esdl_xpath_extensions_unsupported.cpp

@@ -0,0 +1,32 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jstring.hpp"
+#include "jdebug.hpp"
+#include "jptree.hpp"
+#include "jexcept.hpp"
+#include "jlog.hpp"
+
+#include "espcontext.hpp"
+#include "esdl_script.hpp"
+
+void registerEsdlXPathExtensions(IXpathContext *xpathContext, IEspContext *context, const StringArray &prefixes)
+{
+    //nothing to register.  xpath will error on use of function attempt
+
+    //if another xpath library is used in future add support for our functions there.
+}

esp/esdllib/wsexcept.cpp

@@ -1,208 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems®.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#include "platform.h"
-#include "wsexcept.hpp"
-
-class DECL_EXCEPTION CWsException : implements IWsException, public CInterface
-{
-public:
-    IMPLEMENT_IINTERFACE
-
-    CWsException (const char* source, WsErrorType errorType )
-    {
-        if (source)
-            source_.append(source);
-        errorType_ = errorType;
-    }
-    CWsException( IMultiException& me, WsErrorType errorType )
-    {
-        append(me);
-        errorType_ = errorType;
-
-        const char* source = me.source();
-
-        if (source)
-            source_.append(source);
-    }
-    CWsException( IException& e, const char* source, WsErrorType errorType )
-    {
-        IMultiException* me = dynamic_cast<IMultiException*>(&e);
-        if ( me ) {
-            append(*me);
-        } else
-            append(e);
-        errorType_ = errorType;
-
-        if (source)
-            source_.append(source);
-    }
-
-    //convenience methods for handling this as an array
-    virtual aindex_t ordinality() const
-    {
-        synchronized block(m_mutex);
-        return array_.ordinality();
-    }
-    virtual IException& item(aindex_t pos) const
-    {
-        synchronized block(m_mutex);
-        return array_.item(pos);
-    }
-    virtual const char* source() const
-    {
-        synchronized block(m_mutex);
-        return source_.str();
-    }
-
-    //for complete control...caller is responsible for thread safety!
-    virtual IArrayOf<IException>& getArray()     { return array_;              }
-
-    // add another exception. Pass ownership to this obj:
-    // i.e., caller needs to make sure that e has one consumable ref count
-    virtual void append(IException& e)
-    {
-        synchronized block(m_mutex);
-        array_.append(e);
-    }
-    virtual void append(IMultiException& me)
-    {
-        synchronized block(m_mutex);
-
-        IArrayOf<IException>& exceptions = me.getArray();
-        const char* source = me.source();
-        ForEachItemIn(i, exceptions)
-        {
-            IException& e = exceptions.item(i);
-            if (source && *source)
-            {
-                StringBuffer msg;
-                msg.appendf("[%s] ",source);
-                e.errorMessage(msg);
-                array_.append(*MakeStringExceptionDirect(e.errorAudience(), e.errorCode(), msg));
-            }
-            else
-                array_.append(*LINK(&e));
-        }
-    }
-
-
-    StringBuffer& serialize(StringBuffer& buffer, unsigned indent = 0, bool simplified=false, bool root=true) const
-    {
-        synchronized block(m_mutex);
-
-        if (root)
-            buffer.append("<Exceptions>");
-
-        if (!simplified)
-        {
-            if (indent) buffer.append("\n\t");
-            buffer.appendf("<Source>%s</Source>", source_.str());
-        }
-
-        ForEachItemIn(i, array_)
-        {
-            IException& exception = array_.item(i);
-
-            if (indent) buffer.append("\n\t");
-            buffer.append("<Exception>");
-
-            if (indent) buffer.append("\n\t\t");
-            buffer.appendf("<Code>%d</Code>", exception.errorCode());
-
-            if (indent) buffer.append("\n\t\t");
-            buffer.appendf("<Audience>%s</Audience>", serializeMessageAudience( exception.errorAudience() ));
-
-            if (simplified)
-            {
-                if (indent) buffer.append("\n\t\t");
-                StringBuffer msg;
-                buffer.appendf("<Source>%s</Source>", source_.str());
-            }
-
-            if (indent) buffer.append("\n\t\t");
-
-            StringBuffer msg;
-            StringBuffer encoded;
-            encodeXML(exception.errorMessage(msg).str(), encoded);
-            buffer.appendf("<Message>%s</Message>", encoded.str());
-
-            if (indent) buffer.append("\n\t");
-            buffer.append("</Exception>");
-        }
-
-        if (root)
-            buffer.append("</Exceptions>");
-        return buffer;
-    }
-
-    virtual int errorCode() const
-    {
-        synchronized block(m_mutex);
-        return ordinality() == 1 ? item(0).errorCode() : -1;
-    }
-    virtual StringBuffer& errorMessage(StringBuffer &msg) const
-    {
-        synchronized block(m_mutex);
-        ForEachItemIn(i, array_)
-        {
-            IException& e = item(i);
-
-            StringBuffer buf;
-            msg.appendf("[%3d: %s] ", e.errorCode(), e.errorMessage(buf).str());
-        }
-        return msg;
-    }
-    virtual MessageAudience errorAudience() const
-    {
-        synchronized block(m_mutex);
-        return ordinality() == 1 ? item(0).errorAudience() : MSGAUD_unknown;
-    }
-    virtual WsErrorType errorType() const
-    {
-        synchronized block(m_mutex);
-        return errorType_;
-    }
-private:
-    CWsException( const CWsException& );
-    IArrayOf<IException> array_;
-    StringBuffer         source_;
-    mutable Mutex        m_mutex;
-    WsErrorType          errorType_;
-};
-
-IWsException esdl_decl *makeWsException(IMultiException& me, WsErrorType errorType)
-{
-    return new CWsException(me, errorType);
-}
-IWsException esdl_decl *makeWsException(IException& e, WsErrorType errorType, const char* source)
-{
-    return new CWsException(  e, source, errorType );
-}
-IWsException esdl_decl *makeWsException(int errorCode, WsErrorType errorType, const char* source, const char *format, ...)
-{
-    va_list args;
-    va_start(args, format);
-    IException *e = MakeStringExceptionVA(errorCode, format, args);
-    va_end(args);
-
-    return new CWsException(  *e, source, errorType );
-}
-IWsException esdl_decl *makeWsException(const char *source, WsErrorType errorType)
-{
-    return new CWsException(source, errorType);
-}

esp/services/esdl_svc_engine/CMakeLists.txt

@@ -30,7 +30,6 @@ set (   SRCS
         esdl_svc_engine.cpp
         esdl_store.cpp
         esdl_monitor.cpp
-        esdl_svc_custom.cpp
     )
 
 include_directories (

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -417,7 +417,7 @@ void EsdlServiceImpl::addServiceLevelRequestTransform(IPropertyTree *customReque
 
     try
     {
-        m_serviceLevelRequestTransform.setown(new CEsdlCustomTransform(*customRequestTransform));
+        m_serviceLevelRequestTransform.setown(createEsdlCustomTransform(*customRequestTransform, nullptr));
     }
     catch(IException* e)
     {
@@ -441,7 +441,7 @@ void EsdlServiceImpl::addMethodLevelRequestTransform(const char *method, IProper
 
     try
     {
-        Owned<CEsdlCustomTransform> crt = new CEsdlCustomTransform(*customRequestTransform);
+        Owned<IEsdlCustomTransform> crt = createEsdlCustomTransform(*customRequestTransform, nullptr);
         m_customRequestTransformMap.setValue(method, crt.get());
     }
     catch(IException* e)
@@ -467,7 +467,7 @@ static void ensureMergeOrderedEsdlTransform(Owned<IPropertyTree> &dest, IPropert
     if (!src)
         return;
     if (!dest)
-        dest.setown(createPTree(ipt_ordered));
+        dest.setown(createPTree(src->queryName(), ipt_ordered));
     //copy so we can make changes, like adding calculated targets
     Owned<IPropertyTree> copy = createPTreeFromIPT(src, ipt_ordered);
     const char *target = copy->queryProp("@target");
@@ -1114,7 +1114,7 @@ void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
         if (serviceCrt || methodCrt)
         {
             context.addTraceSummaryTimeStamp(LogNormal, "srt-custreqtrans");
-            processServiceAndMethodTransforms({serviceCrt, methodCrt}, &context, tgtcfg.get(), tgtctx.get(), srvdef, mthdef, soapmsg, m_oEspBindingCfg.get());
+            processServiceAndMethodTransforms({serviceCrt, methodCrt}, &context, tgtcfg.get(), srvdef, mthdef, soapmsg, m_oEspBindingCfg.get());
             context.addTraceSummaryTimeStamp(LogNormal, "end-custreqtrans");
         }
 

esp/services/esdl_svc_engine/esdl_binding.hpp

@@ -17,7 +17,7 @@
 #ifndef _EsdlBinding_HPP__
 #define _EsdlBinding_HPP__
 
-#include "esdl_svc_custom.hpp"
+#include "esdl_script.hpp"
 #include "esdl_def.hpp"
 #include "esdl_transformer.hpp"
 #include "esdl_def_helper.hpp"
@@ -80,7 +80,7 @@ private:
     Owned<ILoggingManager> m_oLoggingManager;
     bool m_bGenerateLocalTrxId;
     MapStringToMyClass<IEsdlCustomTransform> m_customRequestTransformMap;
-    Owned<CEsdlCustomTransform> m_serviceLevelRequestTransform;
+    Owned<IEsdlCustomTransform> m_serviceLevelRequestTransform;
     bool m_serviceLevelCrtFail = false;
     using MethodAccessMap = MapStringTo<SecAccessFlags>;
     using MethodAccessMaps = MapStringTo<Owned<MethodAccessMap> >;

esp/services/esdl_svc_engine/esdl_svc_custom.cpp

@@ -1,443 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#include "espcontext.hpp"
-#include "esdl_svc_custom.hpp"
-
-//
-// CEsdlCustomTransformChoose methods
-//
-
-CEsdlCustomTransformChoose::CEsdlCustomTransformChoose(IPropertyTree * choosewhen)
-{
-    if (choosewhen)
-    {
-        if (choosewhen->hasProp("@_crtTarget"))
-            crtTarget.set(choosewhen->queryProp("@_crtTarget"));
-        IPropertyTree * whentree = choosewhen->queryPropTree("xsdl:when"); //should support multiple when statements
-        if (whentree)
-        {
-            StringBuffer testatt;
-            testatt.set(whentree->queryProp("@test"));
-            m_compiledConditionalXpath.setown(compileXpath(testatt.str()));
-
-            compileClauses(whentree, false);
-            compileChildChoose(whentree, false);
-
-            IPropertyTree * otherwise = choosewhen->queryPropTree("xsdl:otherwise");
-            if (otherwise)
-            {
-                compileClauses(otherwise, true);
-                compileChildChoose(otherwise, true);
-            }
-        }
-        else
-            IERRLOG("CEsdlCustomTransformChoose: Found xsdl:choose clause without required xsdl:when");
-    }
-}
-
-void CEsdlCustomTransformChoose::processClauses(IPropertyTree *request, IXpathContext * xpathContext, CIArrayOf<CEsdlCustomTransformRule> & transforms)
-{
-    if (request)
-    {
-        StringBuffer evaluatedValue;
-        ForEachItemIn(i, transforms)
-        {
-            CEsdlCustomTransformRule & cur = transforms.item(i);
-            const char * targetField = cur.queryTargetField();
-            bool optional = cur.isOptional();
-
-            if (xpathContext)
-            {
-                if (targetField && *targetField)
-                {
-                    try
-                    {
-                        xpathContext->evaluateAsString(cur.queryCompiledValuePath(), evaluatedValue.clear());
-                        if (cur.isPerformingASet())
-                        {
-                            request->setProp(targetField, evaluatedValue.str());
-                        }
-                        else
-                        {
-                            ensurePTree(request, targetField);
-                            request->appendProp(targetField, evaluatedValue.str());
-                        }
-                    }
-                    catch (IException* e)
-                    {
-                        StringBuffer eMsg;
-                        e->errorMessage(eMsg);
-                        e->Release();
-
-                        VStringBuffer msg("Could not process Custom Transform: '%s' [%s]", cur.queryName(), eMsg.str());
-                        if (!optional)
-                            throw MakeStringException(-1, "%s", msg.str());
-                        else
-                            ERRLOG("%s", msg.str());
-                    }
-                    catch (...)
-                    {
-                        VStringBuffer msg("Could not process Custom Transform: '%s' ", cur.queryName());
-                        if (!optional)
-                            throw MakeStringException(-1, "%s", msg.str());
-                        else
-                            OERRLOG("%s", msg.str());
-                    }
-                }
-                else
-                    throw MakeStringException(-1, "Encountered field transform rule without target field declaration.");
-            }
-            else
-                throw MakeStringException(-1, "Could not process custom transform (xpathcontext == null)");
-
-        }
-    }
-}
-
-void CEsdlCustomTransformChoose::processClauses(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext, bool othwerwise)
-{
-    if (request)
-    {
-        if (!othwerwise)
-        {
-            processClauses(request, xpathContext, m_chooseClauses);
-        }
-        else
-        {
-            processClauses(request, xpathContext, m_otherwiseClauses);
-        }
-    }
-}
-
-void CEsdlCustomTransformChoose::processChildClauses(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext,  bool otherwise, IPropertyTree *origTree)
-{
-    if (!otherwise)
-    {
-        ForEachItemIn(currNestedConditionalIndex, m_childChooseClauses)
-        {
-            m_childChooseClauses.item(currNestedConditionalIndex).process(context, request, xpathContext, origTree, nullptr);
-        }
-    }
-    else
-    {
-        ForEachItemIn(currNestedConditionalIndex, m_childOtherwiseClauses)
-        {
-            m_childOtherwiseClauses.item(currNestedConditionalIndex).process(context, request, xpathContext, origTree, nullptr);
-        }
-    }
-}
-
-void CEsdlCustomTransformChoose::compileChildChoose(IPropertyTree * nested, bool otherwise)
-{
-    Owned<IPropertyTreeIterator> conditionalIterator = nested->getElements("xsdl:choose");
-    ForEach(*conditionalIterator)
-    {
-        auto xslchooseelement = &conditionalIterator->query();
-        Owned<CEsdlCustomTransformChoose> esdlCustomTransformChoose = new CEsdlCustomTransformChoose(xslchooseelement);
-        addChildTransformClause(esdlCustomTransformChoose, otherwise);
-    }
-}
-
-void CEsdlCustomTransformChoose::compileClauses(IPropertyTreeIterator * rulesiter, bool otherwise, bool typeIsSet)
-{
-    ForEach(*rulesiter)
-    {
-        IPropertyTree & cur = rulesiter->query();
-
-        bool optional = cur.getPropBool("@optional", false);
-        const char * ruleName = cur.queryProp("@name");
-        const char * targetFieldPath = cur.queryProp("@target");
-        if (!targetFieldPath || !*targetFieldPath)
-        {
-            VStringBuffer msg("Encountered custom transform rule '%s' without TargetField path", ruleName ? ruleName : "No Name Provided");
-            if(!optional)
-                throw MakeStringException(-1, "%s", msg.str());
-            else
-                IERRLOG("%s", msg.str());
-                continue;
-        }
-
-        const char * pathToValue = cur.queryProp("@value");
-        if (!pathToValue || !*pathToValue)
-        {
-            VStringBuffer msg("Encountered custom transform rule '%s' without Value path", ruleName ? ruleName : "No Name Provided");
-            if(!optional)
-                throw MakeStringException(-1, "%s", msg.str());
-            else
-                IERRLOG("%s", msg.str());
-                continue;
-        }
-
-        Owned<CEsdlCustomTransformRule> esdlCustomTransformRule = new CEsdlCustomTransformRule(ruleName, targetFieldPath, pathToValue, optional, typeIsSet );
-        addTransformClause(esdlCustomTransformRule, otherwise);
-    }
-}
-
-void CEsdlCustomTransformChoose::compileClauses(IPropertyTree * rules, bool otherwise)
-{
-    Owned<IPropertyTreeIterator> setValueItr = rules->getElements("xsdl:SetValue");
-    Owned<IPropertyTreeIterator> appendValueItr = rules->getElements("xsdl:AppendValue");
-    compileClauses(setValueItr, otherwise, true);
-    compileClauses(appendValueItr, otherwise, false);
-}
-
-void CEsdlCustomTransformChoose::addTransformClause(CEsdlCustomTransformRule * fieldmapping, bool otherwise)
-{
-    if (!otherwise)
-        m_chooseClauses.append(*LINK(fieldmapping));
-    else
-        m_otherwiseClauses.append(*LINK(fieldmapping));
-}
-
-void CEsdlCustomTransformChoose::addChildTransformClause(CEsdlCustomTransformChoose * nestedConditional, bool otherwise)
-{
-    if(!otherwise)
-        m_childChooseClauses.append(*LINK(nestedConditional));
-    else
-        m_childOtherwiseClauses.append(*LINK(nestedConditional));
-}
-
-bool CEsdlCustomTransformChoose::evaluate(IXpathContext * xpathContext)
-{
-    bool evalresp = false;
-    try
-    {
-        evalresp = xpathContext->evaluateAsBoolean(m_compiledConditionalXpath);
-    }
-    catch (IException* e)
-    {
-        StringBuffer msg;
-        DBGLOG("%s", e->errorMessage(msg).str());
-        e->Release();
-    }
-    catch (...)
-    {
-        DBGLOG("CEsdlCustomTransformChoose:evaluate: Could not evaluate xpath '%s'", m_compiledConditionalXpath->getXpath());
-    }
-    return evalresp;
-}
-
-void CEsdlCustomTransformChoose::process(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext, IPropertyTree *origTree, const char *defaultTarget)
-{
-    StringBuffer xpath;
-
-    if (crtTarget.length())
-        xpath.set(crtTarget.str());
-    else if (defaultTarget && *defaultTarget)
-        xpath.set(defaultTarget);
-
-    IPropertyTree *reqTarget = request;
-
-    if (xpath.length())
-    {
-        //we can use real xpath processing in the future, for now simple substitution is fine
-        StringBuffer variable;
-        xpath.replaceString("{$query}", xpathContext->getVariable("query", variable));
-        xpath.replaceString("{$method}", xpathContext->getVariable("method", variable.clear()));
-        xpath.replaceString("{$service}", xpathContext->getVariable("service", variable.clear()));
-        xpath.replaceString("{$request}", xpathContext->getVariable("request", variable.clear()));
-
-        reqTarget = origTree->queryPropTree(xpath.str());  //get pointer to the write-able area
-        if (!reqTarget)
-            throw MakeStringException(-1, "EsdlCustomTransformChoose::process error getting request target %s", xpath.str());
-    }
-
-    bool result = false;
-    try
-    {
-        bool result = evaluate(xpathContext);
-        processClauses(context, reqTarget, xpathContext, !result);
-        processChildClauses(context, reqTarget, xpathContext, !result, origTree);
-    }
-    catch (...)
-    {
-        IERRLOG("EsdlCustomTransformClause::process internal error");
-    }
-}
-
-#if defined(_DEBUG)
-void CEsdlCustomTransformChoose::toDBGLog ()
-{
-    DBGLOG (">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>WHEN>>>>>>>>>>");
-    StringBuffer str;
-
-    str.set (m_compiledConditionalXpath->getXpath ());
-    DBGLOG ("WHEN %s ", str.str ());
-
-    ForEachItemIn(i, m_chooseClauses)
-    {
-        CEsdlCustomTransformRule & cur = m_chooseClauses.item (i);
-        cur.toDBGLog ();
-    }
-
-    ForEachItemIn(nci, m_childChooseClauses)
-    {
-        CEsdlCustomTransformChoose & cur = m_childChooseClauses.item (nci);
-        cur.toDBGLog ();
-    }
-
-    if (m_otherwiseClauses.ordinality () != 0 || m_childOtherwiseClauses.ordinality () != 0)
-    {
-        DBGLOG ("OTHERWISE ");
-        ForEachItemIn(i, m_otherwiseClauses)
-        {
-            CEsdlCustomTransformRule & cur = m_otherwiseClauses.item (i);
-            cur.toDBGLog ();
-        }
-
-        ForEachItemIn(nci, m_childOtherwiseClauses)
-        {
-            CEsdlCustomTransformChoose & cur =
-            m_childOtherwiseClauses.item (nci);
-            cur.toDBGLog ();
-         }
-    }
-    DBGLOG (">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>WHEN>>>>>>>>>>");
-}
-#endif
-
-void processServiceAndMethodTransforms(std::initializer_list<IEsdlCustomTransform *> const &transforms, IEspContext * context, IPropertyTree *tgtcfg, IPropertyTree *tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & request, IPropertyTree * bindingCfg)
-{
-    if (!transforms.size())
-        return;
-
-    if (request.length()!=0)
-    {
-        if (getEspLogLevel() >= LogMax)
-        {
-            DBGLOG("ORIGINAL REQUEST: %s", request.str());
-            StringBuffer marshalled;
-            toXML(bindingCfg, marshalled.clear());
-            DBGLOG("INCOMING CONFIG: %s", marshalled.str());
-        }
-
-        Owned<IXpathContext> xpathContext = getXpathContext(request.str());
-
-        VStringBuffer ver("%g", context->getClientVersion());
-        if(!xpathContext->addVariable("clientversion", ver.str()))
-            OERRLOG("Could not set custom transform variable: clientversion:'%s'", ver.str());
-        //in case transform wants to make use of these values:
-        xpathContext->addVariable("query", tgtcfg->queryProp("@queryname"));
-        xpathContext->addVariable("method", mthdef.queryMethodName());
-        xpathContext->addVariable("service", srvdef.queryName());
-        xpathContext->addVariable("request", mthdef.queryRequestType());
-
-        auto user = context->queryUser();
-        if (user)
-        {
-            static const std::map<SecUserStatus, const char*> statusLabels =
-            {
-#define STATUS_LABEL_NODE(s) { s, #s }
-                STATUS_LABEL_NODE(SecUserStatus_Inhouse),
-                STATUS_LABEL_NODE(SecUserStatus_Active),
-                STATUS_LABEL_NODE(SecUserStatus_Exempt),
-                STATUS_LABEL_NODE(SecUserStatus_FreeTrial),
-                STATUS_LABEL_NODE(SecUserStatus_csdemo),
-                STATUS_LABEL_NODE(SecUserStatus_Rollover),
-                STATUS_LABEL_NODE(SecUserStatus_Suspended),
-                STATUS_LABEL_NODE(SecUserStatus_Terminated),
-                STATUS_LABEL_NODE(SecUserStatus_TrialExpired),
-                STATUS_LABEL_NODE(SecUserStatus_Status_Hold),
-                STATUS_LABEL_NODE(SecUserStatus_Unknown),
-#undef STATUS_LABEL_NODE
-            };
-
-            Owned<IPropertyIterator> userPropIt = user->getPropertyIterator();
-            ForEach(*userPropIt)
-            {
-                const char *name = userPropIt->getPropKey();
-                if (name && *name)
-                    xpathContext->addVariable(name, user->getProperty(name));
-            }
-
-            auto it = statusLabels.find(user->getStatus());
-
-            xpathContext->addVariable("espUserName", user->getName());
-            xpathContext->addVariable("espUserRealm", user->getRealm() ? user->getRealm() : "");
-            xpathContext->addVariable("espUserPeer", user->getPeer() ? user->getPeer() : "");
-            xpathContext->addVariable("espUserStatus", VStringBuffer("%d", int(user->getStatus())));
-            if (it != statusLabels.end())
-                xpathContext->addVariable("espUserStatusString", it->second);
-            else
-                throw MakeStringException(-1, "encountered unexpected secure user status (%d) while processing transform", int(user->getStatus()));
-        }
-        else
-        {
-            // enable transforms to distinguish secure versus insecure requests
-            xpathContext->addVariable("espUserName", "");
-        }
-
-        Owned<IPropertyTreeIterator> configParams = bindingCfg->getElements("Transform/Param");
-        {
-            ForEach(*configParams)
-            {
-                IPropertyTree & currentParam = configParams->query();
-                xpathContext->addVariable(currentParam.queryProp("@name"), currentParam.queryProp("@value"));
-            }
-        }
-
-        Owned<IPropertyTree> theroot = createPTreeFromXMLString(request.str());
-        StringBuffer defaultTarget;
-            //This default gives us backward compatibility with only being able to write to the actual request
-        const char *tgtQueryName = tgtcfg->queryProp("@queryname");
-        defaultTarget.setf("soap:Body/%s/%s", tgtQueryName ? tgtQueryName : mthdef.queryMethodName(), mthdef.queryRequestType());
-
-        for ( auto&& item : transforms)
-        {
-            if (item)
-                item->processTransform(context, theroot, xpathContext, defaultTarget);
-        }
-
-        toXML(theroot, request.clear());
-
-        ESPLOG(1,"MODIFIED REQUEST: %s", request.str());
-    }
-}
-
-//
-// CEsdlCustomTransform methods
-//
-CEsdlCustomTransform::CEsdlCustomTransform(IPropertyTree &currentTransform)
-{
-    m_name.set(currentTransform.queryProp("@name"));
-    m_target.set(currentTransform.queryProp("@target"));
-    DBGLOG("Compiling custom ESDL Transform: '%s'", m_name.str());
-
-    Owned<IPropertyTreeIterator> conditionalIterator = currentTransform.getElements("xsdl:choose");
-    ForEach(*conditionalIterator)
-    {
-        auto xslchooseelement = &conditionalIterator->query();
-        Owned<CEsdlCustomTransformChoose> currconditional = new CEsdlCustomTransformChoose(xslchooseelement);
-        m_customTransformClauses.append(*LINK(currconditional));
-    }
-
-#if defined(_DEBUG)
-    toDBGLog();
-#endif
-}
-
-void CEsdlCustomTransform::processTransform(IEspContext * context, IPropertyTree *theroot, IXpathContext *xpathContext, const char *defaultTarget)
-{
-    ForEachItemIn(currConditionalIndex, m_customTransformClauses)
-        m_customTransformClauses.item(currConditionalIndex).process(context, theroot, xpathContext, theroot, defaultTarget);
-}
-
-void CEsdlCustomTransform::processTransform(IEspContext * context, IPropertyTree *tgtcfg, IPropertyTree *tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & request, IPropertyTree * bindingCfg)
-{
-    processServiceAndMethodTransforms({static_cast<IEsdlCustomTransform*>(this)}, context, tgtcfg, tgtctx, srvdef, mthdef, request, bindingCfg);
-}

esp/services/esdl_svc_engine/esdl_svc_custom.hpp

@@ -1,178 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#ifndef ESDL_SVC_CUSTOM_HPP_
-#define ESDL_SVC_CUSTOM_HPP_
-
-#ifdef ESDL_TRANSFORM_EXPORTS
- #define esdl_svc_cust_decl DECL_EXPORT
-#else
- #define esdl_svc_cust_decl
-#endif
-
-#include "jlib.hpp"
-#include "jstring.hpp"
-#include "jptree.hpp"
-#include "jlog.hpp"
-#include "esp.hpp"
-
-#include "esdl_def.hpp"
-
-#include <map>
-#include <mutex>
-#include <thread>
-#include <initializer_list>
-
-#include "tokenserialization.hpp"
-#include "xpathprocessor.hpp"
-
-class CEsdlCustomTransformRule : public CInterface
-{
-private:
-    StringAttr m_targetField;
-    StringAttr m_xpathToValue;
-    StringAttr m_ruleName;
-    Owned<ICompiledXpath> m_compiledValueXpath;
-    bool m_optional = false;
-    bool m_attemptToSet = false;
-
-public:
-
-    IMPLEMENT_IINTERFACE;
-    CEsdlCustomTransformRule(const char * transferName, const char * targetfield, const char * xpathToValue, bool optional, bool attemptToSet)
-    :  m_ruleName(transferName), m_targetField(targetfield), m_xpathToValue(xpathToValue), m_optional(optional), m_attemptToSet(attemptToSet)
-    {
-        m_compiledValueXpath.setown(compileXpath(xpathToValue));
-    }
-
-#if defined(_DEBUG)
-    void toDBGLog()
-    {
-        DBGLOG(">%s> %s field '%s' to '%s'", m_ruleName.str(), (m_attemptToSet) ? "SetValue" : "AppendValue", m_targetField.str() , m_xpathToValue.str());
-    }
-#endif
-
-    virtual ~CEsdlCustomTransformRule(){}
-
-    const char * queryValueXpath() const
-    {
-        return m_xpathToValue.str();
-    }
-    const char * queryTargetField() const
-    {
-        return m_targetField.str();
-    }
-
-    const char * queryName() const
-    {
-        return m_ruleName.str();
-    }
-
-    ICompiledXpath * queryCompiledValuePath()
-    {
-        return m_compiledValueXpath;
-    }
-
-    bool isOptional () const { return m_optional; }
-    bool isPerformingASet() const { return m_attemptToSet; }
-
-};
-
-class CEsdlCustomTransformChoose : public CInterface
-{
-private:
-    Owned<ICompiledXpath> m_compiledConditionalXpath;
-    CIArrayOf<CEsdlCustomTransformRule> m_chooseClauses;
-    CIArrayOf<CEsdlCustomTransformRule> m_otherwiseClauses;
-    CIArrayOf<CEsdlCustomTransformChoose> m_childChooseClauses;
-    CIArrayOf<CEsdlCustomTransformChoose> m_childOtherwiseClauses;
-    StringAttr crtTarget;
-
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CEsdlCustomTransformChoose(IPropertyTree * choosewhen);
-    ~CEsdlCustomTransformChoose()
-    {
-#if defined(_DEBUG)
-        DBGLOG("CEsdlCustomTransformClause released!");
-#endif
-    }
-    void process(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext, IPropertyTree *origTree, const char *defTarget);
-#if defined(_DEBUG)
-    void toDBGLog();
-#endif
-
-private:
-    void compileChildChoose(IPropertyTree * nested, bool otherwise);
-    void compileClauses(IPropertyTreeIterator * clauseIter, bool otherwise, bool typeIsSet);
-    void compileClauses(IPropertyTree * clauses, bool otherwise);
-    void addTransformClause(CEsdlCustomTransformRule * fieldmapping, bool otherwise);
-    void addChildTransformClause(CEsdlCustomTransformChoose * childConditional, bool otherwise);
-    bool evaluate(IXpathContext * xpathContext);
-    void processClauses(IPropertyTree *request, IXpathContext * xpathContext, CIArrayOf<CEsdlCustomTransformRule> & m_fieldTransforms);
-    void processClauses(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext, bool otherwise);
-    void processChildClauses(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext,  bool otherwise, IPropertyTree *origTree);
-};
-
-interface IEsdlCustomTransform : extends IInterface
-{
-       virtual void processTransform(IEspContext * context, IPropertyTree *tgtcfg, IPropertyTree *tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & request, IPropertyTree * bindingCfg)=0;
-       virtual void processTransform(IEspContext * context, IPropertyTree *theroot, IXpathContext *xpathContext, const char *defaultTarget)=0;
-
-};
-
-class CEsdlCustomTransform : implements IEsdlCustomTransform, public CInterface
-{
-private:
-    CIArrayOf<CEsdlCustomTransformChoose> m_customTransformClauses;
-    StringAttr m_name;
-    StringAttr m_target;
-
-public:
-    IMPLEMENT_IINTERFACE;
-    CEsdlCustomTransform(){}
-    CEsdlCustomTransform(IPropertyTree &cfg);
-
-#if defined(_DEBUG)
-    void toDBGLog()
-    {
-        DBGLOG(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>transform: '%s'>>>>>>>>>>", m_name.str());
-        ForEachItemIn(i, m_customTransformClauses)
-        {
-            CEsdlCustomTransformChoose & cur = m_customTransformClauses.item(i);
-            cur.toDBGLog();
-        }
-        DBGLOG("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<transform<<<<<<<<<<<<");
-      }
-#endif
-
-    virtual ~CEsdlCustomTransform()
-    {
-#if defined(_DEBUG)
-        DBGLOG("CEsdltransformProcessor '%s' released!", m_name.str());
-#endif
-    }
-
-    void processTransform(IEspContext * context, IPropertyTree *tgtcfg, IPropertyTree *tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & request, IPropertyTree * bindingCfg) override;
-    void processTransform(IEspContext * context, IPropertyTree *theroot, IXpathContext *xpathContext, const char *defaultTarget) override;
-
-};
-
-void processServiceAndMethodTransforms(std::initializer_list<IEsdlCustomTransform *> const &transforms, IEspContext * context, IPropertyTree *tgtcfg, IPropertyTree *tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer & request, IPropertyTree * bindingCfg);
-
-#endif /* ESDL_SVC_CUSTOM_HPP_ */

system/security/shared/seclib.hpp

@@ -78,6 +78,42 @@ inline const char * getSecAccessFlagName(SecAccessFlags flag)
     }
 }
 
+inline SecAccessFlags getSecAccessFlagValue(const char *s)
+{
+    if (isEmptyString(s))
+        return SecAccess_None;
+    switch (tolower(*s))
+    {
+    case 'u':
+        if (strieq("unavailable", s))
+            return SecAccess_Unavailable;
+        break;
+    case 'n':
+        if (strieq("none", s))
+            return SecAccess_None;
+        break;
+    case 'a':
+        if (strieq("access", s))
+            return SecAccess_Access;
+        break;
+    case 'r':
+        if (strieq("read", s))
+            return SecAccess_Read;
+        break;
+    case 'w':
+        if (strieq("write", s))
+            return SecAccess_Write;
+        break;
+    case 'f':
+        if (strieq("full", s))
+            return SecAccess_Full;
+        break;
+    default:
+        break;
+    }
+    return SecAccess_Unknown;
+}
+
 
 enum SecResourceType : int
 {

system/xmllib/libxml_xpathprocessor.cpp

@@ -32,11 +32,51 @@
 #include <libxml/xpath.h>
 #include <libxml/xmlschemas.h>
 #include <libxml/hash.h>
+#include <libexslt/exslt.h>
 
 #include "xpathprocessor.hpp"
 #include "xmlerror.hpp"
 
-class CLibCompiledXpath : public CInterface, public ICompiledXpath
+#include <map>
+#include <stack>
+#include <memory>
+
+static inline char *skipWS(char *s)
+{
+    while (isspace(*s)) s++;
+    return s;
+}
+static char *markEndGetNext(char *line)
+{
+    char *end = (char *)strchr(line, '\n');
+    if (!end)
+        return nullptr;
+    *end=0;
+    if (isEmptyString(++end))
+        return nullptr;
+    return end;
+}
+static char *extractFromLineGetNext(StringArray &functions, StringArray &variables, char *line)
+{
+    line = skipWS(line);
+    if (isEmptyString(line))
+        return nullptr;
+    char *next = markEndGetNext(line);
+    if (strncmp(line, "FUNCTION", 8)==0)
+    {
+        char *paren = (char *)strchr(line, '(');
+        if (paren)
+            *paren=0;
+        functions.append(skipWS(line+8+1));
+    }
+    else if (strncmp(line, "VARIABLE", 8)==0)
+    {
+        variables.append(skipWS(line+8+1));
+    }
+    return next;
+}
+
+class CLibCompiledXpath : public CInterfaceOf<ICompiledXpath>
 {
 private:
     xmlXPathCompExprPtr m_compiledXpathExpression = nullptr;
@@ -44,8 +84,6 @@ private:
     ReadWriteLock m_rwlock;
 
 public:
-    IMPLEMENT_IINTERFACE;
-
     CLibCompiledXpath(const char * xpath)
     {
         m_xpath.set(xpath);
@@ -63,70 +101,417 @@ public:
     {
         return m_compiledXpathExpression;
     }
+
+    virtual void extractReferences(StringArray &functions, StringArray &variables) override
+    {
+        char *buf = nullptr;
+        size_t len = 0;
+
+        FILE *stream = open_memstream(&buf, &len);
+        if (stream == nullptr)
+            return;
+
+        xmlXPathDebugDumpCompExpr(stream, m_compiledXpathExpression, 0);
+        fputc(0, stream);
+        fflush(stream);
+        fclose (stream);
+        char *line = buf;
+        while (line)
+            line = extractFromLineGetNext(functions, variables, line);
+        free (buf);
+    }
+};
+static xmlXPathObjectPtr variableLookupFunc(void *data, const xmlChar *name, const xmlChar *ns_uri);
+
+typedef std::map<std::string, xmlXPathObjectPtr> XPathObjectMap;
+
+class CLibXpathScope
+{
+public:
+    StringAttr name; //in future allow named parent access?
+    XPathObjectMap variables;
+
+public:
+    CLibXpathScope(const char *_name) : name(_name){}
+    ~CLibXpathScope()
+    {
+        for (XPathObjectMap::iterator it=variables.begin(); it!=variables.end(); ++it)
+            xmlXPathFreeObject(it->second);
+    }
+    bool setObject(const char *key, xmlXPathObjectPtr obj)
+    {
+        std::pair<XPathObjectMap::iterator,bool> ret = variables.emplace(key, obj);
+        if (ret.second==true)
+            return true;
+        //within scope, behave exactly like xmlXPathContext variables are added now, which seems to be that they are replaced
+        //if we're preventing replacing variables we need to handle elsewhere
+        //  and still replace external values when treated as xsdl:variables, but not when treated as xsdl:params
+        if (ret.first->second)
+            xmlXPathFreeObject(ret.first->second);
+        ret.first->second = obj;
+        return true;
+    }
+    xmlXPathObjectPtr getObject(const char *key)
+    {
+        XPathObjectMap::iterator it = variables.find(key);
+        if (it == variables.end())
+            return nullptr;
+        return it->second;
+    }
 };
 
-class CLibXpathContext : public CInterface, public IXpathContext
+typedef std::vector<std::unique_ptr<CLibXpathScope>> XPathScopeVector;
+typedef std::map<std::string, ICompiledXpath*> XPathInputMap;
+
+class XpathContextState
 {
 private:
+    xmlDocPtr doc = nullptr;
+    xmlNodePtr node = nullptr;
+    int contextSize = 0;
+    int proximityPosition = 0;
+public:
+    XpathContextState(xmlXPathContextPtr ctx)
+    {
+        doc = ctx->doc;
+        node = ctx->node;
+        contextSize = ctx->contextSize;
+        proximityPosition = ctx->proximityPosition;
+    }
+
+    void restore(xmlXPathContextPtr ctx)
+    {
+        ctx->doc = doc;
+        ctx->node = node;
+        ctx->contextSize = contextSize;
+        ctx->proximityPosition = proximityPosition;
+    }
+};
+
+typedef std::vector<XpathContextState> XPathContextStateVector;
+
+class CLibXpathContext : public CInterfaceOf<IXpathContext>
+{
+public:
+    XPathInputMap provided;
     xmlDocPtr m_xmlDoc = nullptr;
     xmlXPathContextPtr m_xpathContext = nullptr;
-    StringBuffer m_xpath;
     ReadWriteLock m_rwlock;
+    XPathScopeVector scopes;
+    bool strictParameterDeclaration = true;
 
-public:
-    IMPLEMENT_IINTERFACE;
+    //saved state
+    XPathContextStateVector saved;
 
-    CLibXpathContext(const char * xmldoc) //not thread safe
+public:
+    CLibXpathContext(const char * xmldoc, bool _strictParameterDeclaration) : strictParameterDeclaration(_strictParameterDeclaration)
     {
+        beginScope("/");
         setXmlDoc(xmldoc);
+        exsltDateXpathCtxtRegister(m_xpathContext, (xmlChar*)"date");
+        exsltMathXpathCtxtRegister(m_xpathContext, (xmlChar*)"math");
+        exsltSetsXpathCtxtRegister(m_xpathContext, (xmlChar*)"set");
+        exsltStrXpathCtxtRegister(m_xpathContext, (xmlChar*)"str");
     }
+
     ~CLibXpathContext()
     {
+        for (XPathInputMap::iterator it=provided.begin(); it!=provided.end(); ++it)
+            it->second->Release();
         xmlXPathFreeContext(m_xpathContext);
         xmlFreeDoc(m_xmlDoc);
     }
 
+    void pushLocation()
+    {
+        WriteLockBlock wblock(m_rwlock);
+        saved.emplace_back(XpathContextState(m_xpathContext));
+    }
+
+    void setLocation(xmlDocPtr doc, xmlNodePtr node, int contextSize, int proximityPosition)
+    {
+        WriteLockBlock wblock(m_rwlock);
+        m_xpathContext->doc = doc;
+        m_xpathContext->node = node;
+        m_xpathContext->contextSize = contextSize;
+        m_xpathContext->proximityPosition = proximityPosition;
+    }
+
+    void setLocation(xmlXPathContextPtr ctx)
+    {
+        WriteLockBlock wblock(m_rwlock);
+        m_xpathContext->doc = ctx->doc;
+        m_xpathContext->node = ctx->node;
+        m_xpathContext->contextSize = ctx->contextSize;
+        m_xpathContext->proximityPosition = ctx->proximityPosition;
+    }
+
+    void popLocation()
+    {
+        WriteLockBlock wblock(m_rwlock);
+        saved.back().restore(m_xpathContext);
+        saved.pop_back();
+    }
+
+    void beginScope(const char *name) override
+    {
+        WriteLockBlock wblock(m_rwlock);
+        scopes.emplace_back(new CLibXpathScope(name));
+    }
+
+    void endScope() override
+    {
+        WriteLockBlock wblock(m_rwlock);
+        if (scopes.size()>1) //preserve root scope
+            scopes.pop_back();
+    }
+
     static void tableScanCallback(void *payload, void *data, xmlChar *name)
     {
         DBGLOG("k/v == [%s,%s]\n", (char *) name, (char *) payload);
     }
 
-    virtual const char * getXpath() override
+    virtual void registerNamespace(const char *prefix, const char * uri) override
     {
-        return m_xpath.str();
+        if (m_xpathContext)
+        {
+            WriteLockBlock wblock(m_rwlock);
+            xmlXPathRegisterNs(m_xpathContext, (const xmlChar *) prefix, (const xmlChar *) uri);
+        }
     }
 
-    virtual bool addVariable(const char * name,  const char * val) override
+    virtual void registerFunction(const char *xmlns, const char * name, void *f) override
     {
-        WriteLockBlock wblock(m_rwlock);
-        if (m_xpathContext && val)
+        if (m_xpathContext)
+        {
+            WriteLockBlock wblock(m_rwlock);
+            xmlXPathRegisterFuncNS(m_xpathContext, (const xmlChar *) name, (const xmlChar *) xmlns, (xmlXPathFunction) f);
+        }
+    }
+
+    virtual void setUserData(void *userdata) override
+    {
+        if (m_xpathContext)
+        {
+            WriteLockBlock wblock(m_rwlock);
+            m_xpathContext->userData = userdata;
+        }
+    }
+
+    virtual void *getUserData() override
+    {
+        if (m_xpathContext)
         {
-            return xmlXPathRegisterVariable(m_xpathContext, (xmlChar *)name, xmlXPathNewCString(val)) == 0;
+            ReadLockBlock wblock(m_rwlock);
+            return m_xpathContext->userData;
+        }
+        return nullptr;
+    }
+
+    inline CLibXpathScope *getCurrentScope()
+    {
+        ReadLockBlock wblock(m_rwlock);
+        assertex(scopes.size());
+        return scopes.back().get();
+    }
+    xmlXPathObjectPtr findVariable(const char *name, const char *ns_uri, CLibXpathScope *scope)
+    {
+        const char *fullname = name;
+        StringBuffer s;
+        if (!isEmptyString(ns_uri))
+            fullname = s.append(ns_uri).append(':').append(name).str();
+
+        ReadLockBlock wblock(m_rwlock);
+        xmlXPathObjectPtr obj = nullptr;
+        if (scope)
+            return scope->getObject(fullname);
+
+        for (XPathScopeVector::const_reverse_iterator it=scopes.crbegin(); !obj && it!=scopes.crend(); ++it)
+            obj = it->get()->getObject(fullname);
+
+        //check libxml2 level variables, shouldn't happen currently but we may want to wrap existing xpathcontexts in the future
+        if (!obj)
+            obj = (xmlXPathObjectPtr)xmlHashLookup2(m_xpathContext->varHash, (const xmlChar *)name, (const xmlChar *)ns_uri);
+        return obj;
+    }
+
+    xmlXPathObjectPtr getVariableObject(const char *name, const char *ns_uri, CLibXpathScope *scope)
+    {
+        return xmlXPathObjectCopy(findVariable(name, ns_uri, scope));
+    }
+
+    bool hasVariable(const char *name, const char *ns_uri, CLibXpathScope *scope)
+    {
+        return (findVariable(name, ns_uri, scope)!=nullptr);
+    }
+
+    virtual bool addObjectVariable(const char * name, xmlXPathObjectPtr obj, CLibXpathScope *scope)
+    {
+        if (isEmptyString(name))
+            return false;
+        if (m_xpathContext)
+        {
+            if (!obj)
+                throw MakeStringException(-1, "addObjectVariable %s error", name);
+            WriteLockBlock wblock(m_rwlock);
+            if (!scope && !scopes.empty())
+                scope = scopes.back().get();
+            if (scope)
+                return scope->setObject(name, obj);
+            return xmlXPathRegisterVariable(m_xpathContext, (xmlChar *)name, obj) == 0;
         }
         return false;
     }
 
-    virtual const char * getVariable(const char * name, StringBuffer & variable) override
+    bool addStringVariable(const char * name,  const char * val, CLibXpathScope *scope)
+    {
+        if (!val)
+            return false;
+        return addObjectVariable(name, xmlXPathNewCString(val), scope);
+    }
+
+    virtual bool addXpathVariable(const char * name, const char * xpath, CLibXpathScope *scope)
+    {
+        if (isEmptyString(xpath))
+            addVariable(name, "");
+        if (m_xpathContext)
+        {
+            xmlXPathObjectPtr obj = evaluate(xpath);
+            if (!obj)
+                throw MakeStringException(-1, "addXpathVariable xpath error %s", xpath);
+            return addObjectVariable(name, obj, scope);
+        }
+        return false;
+    }
+
+    bool addCompiledVariable(const char * name, ICompiledXpath * compiled, CLibXpathScope *scope)
+    {
+        if (!compiled)
+            addVariable(name, "");
+        if (m_xpathContext)
+        {
+            CLibCompiledXpath * clibCompiledXpath = static_cast<CLibCompiledXpath *>(compiled);
+            xmlXPathObjectPtr obj = evaluate(clibCompiledXpath->getCompiledXPathExpression(), clibCompiledXpath->getXpath());
+            if (!obj)
+                throw MakeStringException(-1, "addEvaluateVariable xpath error %s", clibCompiledXpath->getXpath());
+            return addObjectVariable(name, obj, scope);
+        }
+
+        return false;
+    }
+
+    virtual bool addInputValue(const char * name, const char * value) override
+    {
+        if (isEmptyString(name)||isEmptyString(value))
+            return false;
+        VStringBuffer xpath("'%s'", value);
+        return addInputXpath(name, xpath);
+    }
+    virtual bool addInputXpath(const char * name, const char * xpath) override
+    {
+        if (isEmptyString(name)||isEmptyString(xpath))
+            return false;
+        Owned<ICompiledXpath> compiled = compileXpath(xpath);
+        if (compiled)
+        {
+            WriteLockBlock wblock(m_rwlock);
+            provided.emplace(name, compiled.getClear());
+            return true;
+        }
+        return false;
+    }
+
+    inline ICompiledXpath *findInput(const char *name)
     {
         ReadLockBlock rblock(m_rwlock);
+        XPathInputMap::iterator it = provided.find(name);
+        if (it == provided.end())
+            return nullptr;
+         return it->second;
+    }
+
+    virtual bool declareCompiledParameter(const char * name, ICompiledXpath * compiled) override
+    {
+        if (hasVariable(name, nullptr, getCurrentScope()))
+            return false;
+
+        //use input value
+        ICompiledXpath *inputxp = findInput(name);
+        if (inputxp)
+            return addCompiledVariable(name, inputxp, getCurrentScope());
+
+        //use default provided
+        return addCompiledVariable(name, compiled, getCurrentScope());
+    }
+
+    virtual void declareRemainingInputs() override
+    {
+        for (XPathInputMap::iterator it=provided.begin(); it!=provided.end(); ++it)
+            declareCompiledParameter(it->first.c_str(), it->second);
+    }
+
+    virtual bool declareParameter(const char * name, const char *value) override
+    {
+        if (hasVariable(name, nullptr, getCurrentScope()))
+            return false;
+
+        //use input value
+        ICompiledXpath *input = findInput(name);
+        if (input)
+            return addCompiledVariable(name, input, getCurrentScope());
+
+        //use default provided
+        return addStringVariable(name, value, getCurrentScope());
+    }
+
+    virtual bool addXpathVariable(const char * name, const char * xpath) override
+    {
+        return addXpathVariable(name, xpath, nullptr);
+    }
+
+
+    virtual bool addVariable(const char * name,  const char * val) override
+    {
+        return addStringVariable(name, val, nullptr);
+    }
+
+    virtual bool addCompiledVariable(const char * name, ICompiledXpath * compiled) override
+    {
+        return addCompiledVariable(name, compiled, nullptr);
+    }
+
+    virtual const char * getVariable(const char * name, StringBuffer & variable) override
+    {
         if (m_xpathContext)
         {
-            xmlXPathObjectPtr ptr = xmlXPathVariableLookup(m_xpathContext, (const xmlChar *)name);
+            ReadLockBlock rblock(m_rwlock);
+            xmlXPathObjectPtr ptr = xmlXPathVariableLookupNS(m_xpathContext, (const xmlChar *)name, nullptr);
+            if (!ptr)
+                return nullptr;
             variable.append((const char *) ptr->stringval);
             xmlXPathFreeObject(ptr);
             return variable;
         }
-        else
-            return nullptr;
+        return nullptr;
+    }
+    virtual  IXpathContextIterator *evaluateAsNodeSet(ICompiledXpath * compiled) override
+    {
+        CLibCompiledXpath * clCompiled = static_cast<CLibCompiledXpath *>(compiled);
+        if (!clCompiled)
+            throw MakeStringException(XPATHERR_MissingInput,"XpathProcessor:evaluateAsNodeSet: Error: Could not evaluate XPATH");
+        return evaluateAsNodeSet(evaluate(clCompiled->getCompiledXPathExpression(), compiled->getXpath()), compiled->getXpath());
     }
 
-    virtual bool evaluateAsBoolean(const char * xpath)
+    IXpathContextIterator *evaluateAsNodeSet(xmlXPathObjectPtr evaluated, const char* xpath);
+
+    virtual bool evaluateAsBoolean(const char * xpath) override
     {
         if (!xpath || !*xpath)
             throw MakeStringException(XPATHERR_MissingInput,"XpathProcessor:evaluateAsBoolean: Error: Could not evaluate empty XPATH");
         return evaluateAsBoolean(evaluate(xpath), xpath);
     }
-    virtual bool evaluateAsString(const char * xpath, StringBuffer & evaluated)
+
+    virtual bool evaluateAsString(const char * xpath, StringBuffer & evaluated) override
     {
         if (!xpath || !*xpath)
             throw MakeStringException(XPATHERR_MissingInput,"XpathProcessor:evaluateAsString: Error: Could not evaluate empty XPATH");
@@ -149,30 +534,45 @@ public:
         return evaluateAsString(evaluate(clibCompiledXpath->getCompiledXPathExpression(), compiledXpath->getXpath()), evaluated, compiledXpath->getXpath());
     }
 
-private:
+    virtual double evaluateAsNumber(ICompiledXpath * compiledXpath) override
+    {
+        CLibCompiledXpath * clibCompiledXpath = static_cast<CLibCompiledXpath *>(compiledXpath);
+        if (!clibCompiledXpath)
+            throw MakeStringException(XPATHERR_MissingInput,"XpathProcessor:evaluateAsNumber: Error: Missing compiled XPATH");
+        return evaluateAsNumber(evaluate(clibCompiledXpath->getCompiledXPathExpression(), compiledXpath->getXpath()), compiledXpath->getXpath());
+    }
+
     virtual bool setXmlDoc(const char * xmldoc) override
     {
-        if (xmldoc && * xmldoc)
+        if (isEmptyString(xmldoc))
+            return false;
+        xmlDocPtr doc = xmlParseDoc((const unsigned char *)xmldoc);
+        if (doc == nullptr)
         {
-            m_xmlDoc = xmlParseDoc((const unsigned char *)xmldoc);
-            if (m_xmlDoc == nullptr)
-            {
-                ERRLOG("XpathProcessor:setxmldoc Error: Unable to parse XMLLib document");
-                return false;
-            }
-
-            // Create xpath evaluation context
-            m_xpathContext = xmlXPathNewContext(m_xmlDoc);
-            if(m_xpathContext == nullptr)
-            {
-                ERRLOG("XpathProcessor:setxmldoc: Error: Unable to create new XMLLib XPath context");
-                return false;
-            }
-            return true;
+            ERRLOG("XpathProcessor:setXmlDoc Error: Unable to parse XMLLib document");
+            return false;
         }
-        return false;
+        return setContextDocument(doc, xmlDocGetRootElement(doc));
     }
 
+private:
+    bool setContextDocument(xmlDocPtr doc, xmlNodePtr node)
+    {
+        WriteLockBlock rblock(m_rwlock);
+
+        m_xmlDoc = doc;
+        m_xpathContext = xmlXPathNewContext(m_xmlDoc);
+        if(m_xpathContext == nullptr)
+        {
+            ERRLOG("XpathProcessor:setContextDocument: Error: Unable to create new XMLLib XPath context");
+            return false;
+        }
+
+        //relative paths need something to be relative to
+        xmlXPathSetContextNode(node, m_xpathContext);
+        xmlXPathRegisterVariableLookup(m_xpathContext, variableLookupFunc, this);
+        return true;
+    }
     bool evaluateAsBoolean(xmlXPathObjectPtr evaluatedXpathObj, const char* xpath)
     {
         if (!evaluatedXpathObj)
@@ -237,19 +637,28 @@ private:
         return evaluated.str();
     }
 
-    virtual xmlXPathObjectPtr evaluate(xmlXPathCompExprPtr compiledXpath, const char* xpath)
+    double evaluateAsNumber(xmlXPathObjectPtr evaluatedXpathObj, const char* xpath)
+    {
+        if (!evaluatedXpathObj)
+            throw MakeStringException(XPATHERR_InvalidInput,"XpathProcessor:evaluateAsNumber: Error: Could not evaluate XPATH '%s'", xpath);
+        double ret = xmlXPathCastToNumber(evaluatedXpathObj);
+        xmlXPathFreeObject(evaluatedXpathObj);
+        return ret;
+    }
+
+    virtual xmlXPathObjectPtr evaluate(xmlXPathCompExprPtr compiled, const char *xpath)
     {
         xmlXPathObjectPtr evaluatedXpathObj = nullptr;
-        if (compiledXpath)
+        if (compiled)
         {
             ReadLockBlock rlock(m_rwlock);
             if ( m_xpathContext)
             {
-                evaluatedXpathObj = xmlXPathCompiledEval(compiledXpath, m_xpathContext);
+                evaluatedXpathObj = xmlXPathCompiledEval(compiled, m_xpathContext);
             }
             else
             {
-                throw MakeStringException(XPATHERR_InvalidState,"XpathProcessor:evaluate: Error: Could not evaluate XPATH '%s'; ensure xmldoc has been set", xpath);
+                throw MakeStringException(XPATHERR_InvalidState,"XpathProcessor:evaluate: Error: Could not evaluate XPATH '%s'", xpath);
             }
         }
 
@@ -276,12 +685,95 @@ private:
     }
 };
 
+class XPathNodeSetIterator : public CInterfaceOf<IXpathContextIterator>
+{
+public:
+    CLibXpathContext *context;
+    xmlNodeSetPtr list;
+    unsigned pos = 0;
+
+public:
+    XPathNodeSetIterator(CLibXpathContext *xpctx, xmlNodeSetPtr nodeset) : context(xpctx), list(nodeset)
+    {
+        context->pushLocation();
+        context->m_xpathContext->contextSize = xmlXPathNodeSetGetLength(list);
+    }
+
+    virtual ~XPathNodeSetIterator()
+    {
+        context->popLocation();
+        if (list)
+            xmlXPathFreeNodeSet(list);
+    }
+
+    bool update(int newpos)
+    {
+        pos = newpos;
+        if (!isValid())
+            return false;
+        xmlNodePtr node = xmlXPathNodeSetItem(list, pos);;
+        context->m_xpathContext->node = node;
+        if ((node->type != XML_NAMESPACE_DECL && node->doc != nullptr))
+            context->m_xpathContext->doc = node->doc;
+        context->m_xpathContext->proximityPosition = pos + 1;
+        return true;
+    }
+    virtual bool first()
+    {
+        if (xmlXPathNodeSetGetLength(list)==0)
+            return false;
+        return update(0);
+    }
+
+    virtual bool next()
+    {
+        return update(pos+1);
+    }
+    virtual bool isValid()
+    {
+        return (pos < xmlXPathNodeSetGetLength(list));
+    }
+    virtual IXpathContext  & query()
+    {
+        return *context;
+    }
+};
+
+IXpathContextIterator *CLibXpathContext::evaluateAsNodeSet(xmlXPathObjectPtr evaluated, const char* xpath)
+{
+    if (!evaluated)
+    {
+        throw MakeStringException(XPATHERR_InvalidInput, "XpathProcessor:evaluateAsNodeSet: Error: Could not evaluate XPATH '%s'", xpath);
+    }
+
+    if (XPATH_NODESET != evaluated->type)
+    {
+        xmlXPathFreeObject(evaluated);
+        throw MakeStringException(XPATHERR_UnexpectedInput, "XpathProcessor:evaluateAsNodeSet: Error: Could not evaluate XPATH '%s' as NodeSet", xpath);
+    }
+
+
+    xmlNodeSetPtr ns = evaluated->nodesetval;
+    evaluated->nodesetval = nullptr;
+    xmlXPathFreeObject(evaluated);
+
+    return new XPathNodeSetIterator(this, ns);
+}
+
+static xmlXPathObjectPtr variableLookupFunc(void *data, const xmlChar *name, const xmlChar *ns_uri)
+{
+    CLibXpathContext *ctxt = (CLibXpathContext *) data;
+    if (!ctxt)
+        return nullptr;
+    return ctxt->getVariableObject((const char *)name, (const char *)ns_uri, nullptr);
+}
+
 extern ICompiledXpath* compileXpath(const char * xpath)
 {
     return new CLibCompiledXpath(xpath);
 }
 
-extern IXpathContext* getXpathContext(const char * xmldoc)
+extern IXpathContext* getXpathContext(const char * xmldoc, bool strictParameterDeclaration)
 {
-    return new CLibXpathContext(xmldoc);
+    return new CLibXpathContext(xmldoc, strictParameterDeclaration);
 }

system/xmllib/xpathprocessor.hpp

@@ -23,26 +23,62 @@
 
 interface XMLLIB_API ICompiledXpath : public IInterface
 {
-public:
-
     virtual const char * getXpath() = 0;
+    virtual void extractReferences(StringArray &functions, StringArray &variables) = 0;
 };
 
+interface IXpathContextIterator;
+
 interface XMLLIB_API IXpathContext : public IInterface
 {
-public:
-
     virtual bool addVariable(const char * name, const char * val) = 0;
+    virtual bool addXpathVariable(const char * name, const char * xpath) = 0;
+    virtual bool addCompiledVariable(const char * name, ICompiledXpath * compiled) = 0;
+
     virtual const char * getVariable(const char * name, StringBuffer & variable) = 0;
+
     virtual bool evaluateAsBoolean(const char * xpath) = 0;
     virtual bool evaluateAsString(const char * xpath, StringBuffer & evaluated) = 0;
     virtual bool evaluateAsBoolean(ICompiledXpath * compiledXpath) = 0;
     virtual const char * evaluateAsString(ICompiledXpath * compiledXpath, StringBuffer & evaluated) = 0;
-    virtual const char * getXpath() = 0;
+    virtual double evaluateAsNumber(ICompiledXpath * compiledXpath) = 0;
+    virtual  IXpathContextIterator *evaluateAsNodeSet(ICompiledXpath * compiledXpath) = 0;
+
     virtual bool setXmlDoc(const char * xmldoc) = 0;
+    virtual void setUserData(void *) = 0;
+    virtual void *getUserData() = 0;
+
+    virtual void registerFunction(const char *xmlns, const char * name, void *f) = 0;
+    virtual void registerNamespace(const char *prefix, const char *uri) = 0;
+    virtual void beginScope(const char *name) = 0;
+    virtual void endScope() = 0;
+
+    virtual bool addInputXpath(const char * name, const char * xpath) = 0; //values should be declared as parameters before use, "strict parameter mode" requires it
+    virtual bool addInputValue(const char * name, const char * value) = 0; //values should be declared as parameters before use, "strict parameter mode" requires it
+    virtual bool declareParameter(const char * name, const char *value) = 0;
+    virtual bool declareCompiledParameter(const char * name, ICompiledXpath * compiled) = 0;
+    virtual void declareRemainingInputs() = 0;
+};
+
+interface IXpathContextIterator : extends IIteratorOf<IXpathContext> { };
+
+class CXpathContextScope : CInterface
+{
+private:
+    Linked<IXpathContext> context;
+public:
+    IMPLEMENT_IINTERFACE;
+    CXpathContextScope(IXpathContext *ctx, const char *name) : context(ctx)
+    {
+        context->beginScope(name);
+    }
+    virtual ~CXpathContextScope()
+    {
+        context->endScope();
+    }
 };
 
 extern "C" XMLLIB_API ICompiledXpath* compileXpath(const char * xpath);
-extern "C" XMLLIB_API IXpathContext*  getXpathContext(const char * xmldoc);
+extern "C" XMLLIB_API IXpathContext*  getXpathContext(const char * xmldoc, bool strictParameterDeclaration);
 
 #endif /* XPATH_MANAGER_HPP_ */

testing/unittests/CMakeLists.txt

@@ -35,6 +35,7 @@ set (    SRCS
          jlibtests.cpp
          cryptotests.cpp
          hqltests.cpp
+         esdltests.cpp
          configmgr/ConfigMgrUnitTests.cpp
          configmgr/ConfigMgrTemplateTests.cpp
          configmgr/ConfigMgrHPCCTests.cpp
@@ -53,8 +54,14 @@ include_directories (
          ./../../common/deftype
          ./../../system/security/cryptohelper
          ./../../configuration/configmgr/configmgrlib
+         ${HPCC_SOURCE_DIR}/system/xmllib
          ${HPCC_SOURCE_DIR}/ecl/hql
          ${HPCC_SOURCE_DIR}/configuration/configmgr/RapidJSON/include
+         ${HPCC_SOURCE_DIR}/esp/bindings
+         ${HPCC_SOURCE_DIR}/esp/esdllib
+         ${HPCC_SOURCE_DIR}/esp/platform
+         ${HPCC_SOURCE_DIR}/esp/services/common
+         ${CMAKE_BINARY_DIR}/generated
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
     )
@@ -72,6 +79,8 @@ target_link_libraries ( unittests
          libbase58
          thorhelper
          configmgr
+         esphttp
+         esdllib
          ${CPPUNIT_LIBRARIES}
     )