HPCC-25420 ESDL Script: add mysql support and other features

Add the following features:
1. MySql support in ESDL integration scripts
2. Support the creation of stand alone purely scripted services.
3. Add ESDLResponse entry point supporting modifying the
   post ESDL response.
4. Add xpath functions getDataSection(name) and ensureDataSection(name).
   Scripts should no longer have to use absolute xpaths they should
   use these functions instead.
5. Add IXpathContext::createXmlWriter() supporting writing directly into
   an xpath context using an IXmlWriter inteface.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexisrisk.com>

esp/esdllib/CMakeLists.txt

@@ -31,6 +31,7 @@ include_directories (
     ${HPCC_SOURCE_DIR}/rtl/eclrtl
     ${HPCC_SOURCE_DIR}/rtl/include #IXMLWriter
     ${HPCC_SOURCE_DIR}/common/thorhelper #JSONWRITER
+    ${HPCC_SOURCE_DIR}/common/dllserver #loading plugins
     ${HPCC_SOURCE_DIR}/tools/hidl #TAccessMapGenerator
 )
 

esp/esdllib/esdl_script.cpp

@@ -19,6 +19,12 @@
 #include "esdl_script.hpp"
 #include "wsexcept.hpp"
 #include "httpclient.hpp"
+#include "dllserver.hpp"
+#include "thorplugin.hpp"
+#include "eclrtl.hpp"
+#include "rtlformat.hpp"
+#include "jsecrets.hpp"
+#include "esdl_script.hpp"
 
 #include <xpp/XmlPullParser.h>
 using namespace xpp;
@@ -52,10 +58,9 @@ inline void esdlOperationError(int code, const char *op, const char *msg, const
     if (!isEmptyString(op))
         s.append(" ").append(op).append(" ");
     s.append(msg);
+    IERRLOG("%s", s.str());
     if(exception)
         throw MakeStringException(code, "%s", s.str());
-
-    IERRLOG("%s", s.str());
 }
 
 inline void esdlOperationError(int code, const char *op, const char *msg, bool exception)
@@ -234,6 +239,410 @@ interface IEsdlTransformOperationHttpHeader : public IInterface
     virtual bool processHeader(IEsdlScriptContext * scriptContext, IXpathContext * targetContext, IXpathContext * sourceContext, IProperties *headers) = 0;
 };
 
+static Owned<ILoadedDllEntry> mysqlPluginDll;
+static Owned<IEmbedContext> mysqlplugin;
+
+IEmbedContext &ensureMysqlEmbed()
+{
+    if (!mysqlplugin)
+    {
+        mysqlPluginDll.setown(createDllEntry("mysqlembed", false, NULL, false));
+        if (!mysqlPluginDll)
+            throw makeStringException(0, "Failed to load mysqlembed plugin");
+        GetEmbedContextFunction pf = (GetEmbedContextFunction) mysqlPluginDll->getEntry("getEmbedContextDynamic");
+        if (!pf)
+            throw makeStringException(0, "Failed to load mysqlembed plugin");
+        mysqlplugin.setown(pf());
+    }
+    return *mysqlplugin;
+}
+
+class CEsdlTransformOperationMySqlBindParmeter : public CEsdlTransformOperationWithoutChildren
+{
+protected:
+    StringAttr m_name;
+    StringAttr m_mysql_type;
+    Owned<ICompiledXpath> m_value;
+    bool m_bitfield = false;
+
+public:
+    IMPLEMENT_IINTERFACE_USING(CEsdlTransformOperationWithoutChildren)
+
+    CEsdlTransformOperationMySqlBindParmeter(XmlPullParser &xpp, StartTag &stag, const StringBuffer &prefix) : CEsdlTransformOperationWithoutChildren(xpp, stag, prefix)
+    {
+        m_name.set(stag.getValue("name"));
+        if (m_name.isEmpty())
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, "without name or xpath_name", m_traceName, !m_ignoreCodingErrors);
+
+        const char *value = stag.getValue("value");
+        if (isEmptyString(value))
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, "without value", m_traceName, !m_ignoreCodingErrors);
+        m_value.setown(compileXpath(value));
+
+        //optional, conversions normally work well, ONLY WHEN NEEDED we may need to have special handling for mysql types
+        m_mysql_type.set(stag.getValue("type"));
+        if (m_mysql_type.length() && 0==strnicmp(m_mysql_type.str(), "BIT", 3))
+            m_bitfield = true;
+    }
+
+    virtual ~CEsdlTransformOperationMySqlBindParmeter(){}
+
+    bool process(IEsdlScriptContext * scriptContext, IXpathContext * targetContext, IXpathContext * sourceContext) override
+    {
+        return false;
+    }
+
+    void bindParameter(IEsdlScriptContext * scriptContext, IXpathContext * targetContext, IXpathContext * sourceContext, IEmbedFunctionContext *functionContext)
+    {
+        if (!functionContext)
+            return;
+        StringBuffer value;
+        if (m_value)
+            sourceContext->evaluateAsString(m_value, value);
+        if (value.isEmpty())
+            functionContext->bindUTF8Param(m_name, 0, "");
+        else
+        {
+            if (m_bitfield)
+                functionContext->bindSignedParam(m_name, atoi64(value.str()));
+            else
+                functionContext->bindUTF8Param(m_name, rtlUtf8Length(value.length(), value), value);
+        }
+    }
+
+    virtual void toDBGLog () override
+    {
+    #if defined(_DEBUG)
+        DBGLOG ("> %s (%s, value(%s)) >>>>>>>>>>", m_tagname.str(), m_name.str(), m_value ? m_value->getXpath() : "");
+    #endif
+    }
+};
+
+static inline void buildMissingMySqlParameterMessage(StringBuffer &msg, const char *name)
+{
+    msg .append(msg.isEmpty() ? "without " : ", ").append(name);
+}
+
+static inline void addExceptionsToXpathContext(IXpathContext *targetContext, IMultiException *me)
+{
+    if (!targetContext || !me)
+        return;
+    StringBuffer xml;
+    me->serialize(xml);
+    CXpathContextLocation content_location(targetContext);
+    targetContext->ensureSetValue("@status", "error", true);
+    targetContext->addXmlContent(xml.str());
+}
+
+static inline void addExceptionsToXpathContext(IXpathContext *targetContext, IException *E)
+{
+    if (!targetContext || !E)
+        return;
+    Owned<IMultiException> me = makeMultiException("ESDLScript");
+    me->append(*LINK(E));
+    addExceptionsToXpathContext(targetContext, me);
+}
+
+class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase
+{
+protected:
+    StringAttr m_name;
+
+    Owned<ICompiledXpath> m_vaultName;
+    Owned<ICompiledXpath> m_secretName;
+    Owned<ICompiledXpath> m_section;
+    Owned<ICompiledXpath> m_resultsetTag;
+    Owned<ICompiledXpath> m_server;
+    Owned<ICompiledXpath> m_user;
+    Owned<ICompiledXpath> m_password;
+    Owned<ICompiledXpath> m_database;
+
+    StringArray m_mysqlOptionNames;
+    IArrayOf<ICompiledXpath> m_mysqlOptionXpaths;
+
+    StringBuffer m_sql;
+
+    Owned<ICompiledXpath> m_select;
+
+    IArrayOf<CEsdlTransformOperationMySqlBindParmeter> m_parameters;
+
+public:
+    CEsdlTransformOperationMySqlCall(XmlPullParser &xpp, StartTag &stag, const StringBuffer &prefix) : CEsdlTransformOperationBase(xpp, stag, prefix)
+    {
+        ensureMysqlEmbed();
+
+        m_name.set(stag.getValue("name"));
+        if (m_traceName.isEmpty())
+            m_traceName.set(m_name.str());
+
+        //select is optional, with select, a mysql call behaves like a for-each, binding and executing each iteration of the selected content
+        //without select, it executes once in the current context
+        m_select.setown(compileOptionalXpath(stag.getValue("select")));
+
+        m_vaultName.setown(compileOptionalXpath(stag.getValue("vault")));
+        m_secretName.setown(compileOptionalXpath(stag.getValue("secret")));
+        m_section.setown(compileOptionalXpath(stag.getValue("section")));
+        m_resultsetTag.setown(compileOptionalXpath(stag.getValue("resultset-tag")));
+
+        m_server.setown(compileOptionalXpath(stag.getValue("server")));
+        m_user.setown(compileOptionalXpath(stag.getValue("user")));
+        m_password.setown(compileOptionalXpath(stag.getValue("password")));
+        m_database.setown(compileOptionalXpath(stag.getValue("database")));
+
+        //script can set any MYSQL options using an attribute with the same name as the option enum, for example
+        //    MYSQL_SET_CHARSET_NAME="'latin1'" or MYSQL_SET_CHARSET_NAME="$charset"
+        //
+        int attCount = stag.getLength();
+        for (int i=0; i<attCount; i++)
+        {
+            const char *attName = stag.getLocalName(i);
+            if (attName && hasPrefix(attName, "MYSQL_", false))
+            {
+                Owned<ICompiledXpath> attXpath = compileOptionalXpath(stag.getValue(i));
+                if (attXpath)
+                {
+                    m_mysqlOptionNames.append(attName);
+                    m_mysqlOptionXpaths.append(*attXpath.getClear());
+                }
+            }
+        }
+
+        int type = 0;
+        while((type = xpp.next()) != XmlPullParser::END_DOCUMENT)
+        {
+            switch(type)
+            {
+                case XmlPullParser::START_TAG:
+                {
+                    StartTag stag;
+                    xpp.readStartTag(stag);
+                    const char *op = stag.getLocalName();
+                    if (isEmptyString(op))
+                        esdlOperationError(ESDL_SCRIPT_Error, m_tagname, "unknown error", m_traceName, !m_ignoreCodingErrors);
+                    if (streq(op, "bind"))
+                        m_parameters.append(*new CEsdlTransformOperationMySqlBindParmeter(xpp, stag, prefix));
+                    else if (streq(op, "sql"))
+                        readFullContent(xpp, m_sql);
+                    else
+                        xpp.skipSubTreeEx();
+                    break;
+                }
+                case XmlPullParser::END_TAG:
+                case XmlPullParser::END_DOCUMENT:
+                    return;
+            }
+        }
+
+        if (!m_section)
+            m_section.setown(compileXpath("'temporaries'"));
+        StringBuffer errmsg;
+        if (m_name.isEmpty())
+            buildMissingMySqlParameterMessage(errmsg, "name");
+        if (!m_server)
+            buildMissingMySqlParameterMessage(errmsg, "server");
+        if (!m_user)
+            buildMissingMySqlParameterMessage(errmsg, "user");
+        if (!m_database)
+            buildMissingMySqlParameterMessage(errmsg, "database");
+        if (m_sql.isEmpty())
+            buildMissingMySqlParameterMessage(errmsg, "sql");
+        if (errmsg.length())
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, errmsg, m_traceName, !m_ignoreCodingErrors);
+    }
+
+    virtual ~CEsdlTransformOperationMySqlCall()
+    {
+    }
+
+    void bindParameters(IEsdlScriptContext * scriptContext, IXpathContext * targetContext, IXpathContext * sourceContext, IEmbedFunctionContext *functionContext)
+    {
+        if (!m_parameters.length())
+            return;
+        CXpathContextLocation location(targetContext);
+        ForEachItemIn(i, m_parameters)
+            m_parameters.item(i).bindParameter(scriptContext, targetContext, sourceContext, functionContext);
+    }
+
+    void missingMySqlOptionError(const char *name, bool required)
+    {
+        if (required)
+        {
+            StringBuffer msg("empty or missing ");
+            esdlOperationError(ESDL_SCRIPT_MissingOperationAttr, m_tagname, msg.append(name), m_traceName, true);
+        }
+    }
+    IPropertyTree *getSecretInfo(IXpathContext * sourceContext)
+    {
+        //leaving flexibility for the secret to be configured multiple ways
+        //  the most secure option in my opinion is to at least have the server, name, and password all in the secret
+        //  with the server included the credentials can't be hijacked and sent somewhere else for capture.
+        //
+        if (!m_secretName)
+            return nullptr;
+        StringBuffer name;
+        sourceContext->evaluateAsString(m_secretName, name);
+        if (name.isEmpty())
+        {
+            missingMySqlOptionError(name, true);
+            return nullptr;
+        }
+        StringBuffer vault;
+        if (m_vaultName)
+            sourceContext->evaluateAsString(m_vaultName, vault);
+        if (vault.isEmpty())
+            return getSecret("esp", name);
+        return getVaultSecret("esp", vault, name);
+    }
+    void appendOption(StringBuffer &options, const char *name, const char *value, bool required)
+    {
+        if (isEmptyString(value))
+        {
+            missingMySqlOptionError(name, required);
+            return;
+        }
+        if (options.length())
+            options.append(',');
+        options.append(name).append('=').append(value);
+
+    }
+    void appendOption(StringBuffer &options, const char *name, IXpathContext * sourceContext, ICompiledXpath *cx, IPropertyTree *secret, bool required)
+    {
+        if (secret && secret->hasProp(name))
+        {
+            StringBuffer value;
+            getSecretKeyValue(value, secret, name);
+            appendOption(options, name, value, required);
+            return;
+        }
+
+        if (!cx)
+        {
+            missingMySqlOptionError(name, required);
+            return;
+        }
+        StringBuffer value;
+        sourceContext->evaluateAsString(cx, value);
+        if (!value.length())
+        {
+            missingMySqlOptionError(name, required);
+            return;
+        }
+        if (options.length())
+            options.append(',');
+        options.append(name).append('=').append(value);
+    }
+    IEmbedFunctionContext *createFunctionContext(IXpathContext * sourceContext)
+    {
+        Owned<IPropertyTree> secret = getSecretInfo(sourceContext);
+        StringBuffer options;
+        appendOption(options, "server", sourceContext, m_server, secret, true);
+        appendOption(options, "user", sourceContext, m_user, secret, true);
+        appendOption(options, "database", sourceContext, m_database, secret, true);
+        appendOption(options, "password", sourceContext, m_password, secret, true);
+
+        aindex_t count = m_mysqlOptionNames.length();
+        for (aindex_t i=0; i<count; i++)
+            appendOption(options, m_mysqlOptionNames.item(i), sourceContext, &m_mysqlOptionXpaths.item(i), nullptr, true);
+
+        Owned<IEmbedFunctionContext> fc = ensureMysqlEmbed().createFunctionContext(EFembed, options.str());
+        fc->compileEmbeddedScript(m_sql.length(), m_sql);
+        return fc.getClear();
+    }
+
+    void processCurrent(IEmbedFunctionContext *fc, IXmlWriter *writer, const char *tag, IEsdlScriptContext * scriptContext, IXpathContext * targetContext, IXpathContext * sourceContext)
+    {
+        bindParameters(scriptContext, targetContext, sourceContext, fc);
+        fc->callFunction();
+        if (!isEmptyString(tag))
+            writer->outputBeginNested(tag, true);
+        fc->writeResult(nullptr, nullptr, nullptr, writer);
+        if (!isEmptyString(tag))
+            writer->outputEndNested(tag);
+    }
+
+    IXpathContextIterator *select(IXpathContext * xpathContext)
+    {
+        IXpathContextIterator *xpathset = nullptr;
+        try
+        {
+            xpathset = xpathContext->evaluateAsNodeSet(m_select);
+        }
+        catch (IException* e)
+        {
+            int code = e->errorCode();
+            StringBuffer msg;
+            e->errorMessage(msg);
+            e->Release();
+            esdlOperationError(code, m_tagname, msg, !m_ignoreCodingErrors);
+        }
+        catch (...)
+        {
+            VStringBuffer msg("unknown exception evaluating select '%s'", m_select.get() ? m_select->getXpath() : "undefined!");
+            esdlOperationError(ESDL_SCRIPT_Error, m_tagname, msg, !m_ignoreCodingErrors);
+        }
+        return xpathset;
+    }
+
+    void getXpathStringValue(StringBuffer &s, IXpathContext * sourceContext, ICompiledXpath *cx, const char *defaultValue)
+    {
+        if (cx)
+            sourceContext->evaluateAsString(cx, s);
+        if (defaultValue && s.isEmpty())
+            s.set(defaultValue);
+    }
+    virtual bool process(IEsdlScriptContext * scriptContext, IXpathContext * targetContext, IXpathContext * sourceContext) override
+    {
+        StringBuffer section;
+        getXpathStringValue(section, sourceContext, m_section, "temporaries");
+
+        VStringBuffer xpath("/esdl_script_context/%s/%s", section.str(), m_name.str());
+        CXpathContextLocation location(targetContext);
+        targetContext->ensureLocation(xpath, true);
+
+        Owned<IXpathContextIterator> selected;
+        if (m_select)
+        {
+            selected.setown(select(sourceContext));
+            if (!selected || !selected->first())
+                return false;
+        }
+
+        try
+        {
+            Owned<IEmbedFunctionContext> fc = createFunctionContext(sourceContext);
+            Owned<IXmlWriter> writer = targetContext->createXmlWriter();
+            StringBuffer rstag;
+            getXpathStringValue(rstag, sourceContext, m_resultsetTag, nullptr);
+            if (!selected)
+                processCurrent(fc, writer, rstag, scriptContext, targetContext, sourceContext);
+            else
+            {
+                ForEach(*selected)
+                    processCurrent(fc, writer, rstag, scriptContext, targetContext, &selected->query());
+            }
+        }
+        catch(IMultiException *me)
+        {
+            addExceptionsToXpathContext(targetContext, me);
+            me->Release();
+        }
+        catch(IException *E)
+        {
+            addExceptionsToXpathContext(targetContext, E);
+            E->Release();
+        }
+
+        sourceContext->addXpathVariable(m_name, xpath);
+        return true;
+    }
+
+    virtual void toDBGLog() override
+    {
+#if defined(_DEBUG)
+        DBGLOG(">%s> %s with name(%s) server(%s) database(%s)", m_name.str(), m_tagname.str(), m_name.str(), m_server->getXpath(), m_database->getXpath());
+#endif
+    }
+};
 
 class CEsdlTransformOperationHttpHeader : public CEsdlTransformOperationWithoutChildren, implements IEsdlTransformOperationHttpHeader
 {
@@ -298,8 +707,6 @@ public:
     }
 };
 
-
-
 class CEsdlTransformOperationHttpPostXml : public CEsdlTransformOperationBase
 {
 protected:
@@ -1488,6 +1895,8 @@ IEsdlTransformOperation *createEsdlTransformOperation(XmlPullParser &xpp, const
         return new CEsdlTransformOperationNamespace(xpp, stag, prefix);
     if (streq(op, "http-post-xml"))
         return new CEsdlTransformOperationHttpPostXml(xpp, stag, prefix);
+    if (streq(op, "mysql"))
+        return new CEsdlTransformOperationMySqlCall(xpp, stag, prefix);
     return nullptr;
 }
 

esp/esdllib/esdl_script.hpp

@@ -69,6 +69,8 @@ inline bool isEmptyTransformSet(IEsdlTransformSet *set)
 #define ESDLScriptEntryPoint_Legacy "CustomRequestTransform"
 #define ESDLScriptEntryPoint_BackendRequest "BackendRequest"
 #define ESDLScriptEntryPoint_BackendResponse "BackendResponse"
+#define ESDLScriptEntryPoint_ScriptedService "Service"
+#define ESDLScriptEntryPoint_InitialEsdlResponse "EsdlResponse"
 #define ESDLScriptEntryPoint_PreLogging "PreLogging"
 
 interface IEsdlTransformEntryPointMap : extends IInterface

esp/esdllib/esdl_transformer2.cpp

@@ -1628,12 +1628,15 @@ int Esdl2Transformer::process(IEspContext &ctx, EsdlProcessMode mode, const char
             tctx.root_type.set(root->queryName());
             root->process(tctx, (root_name) ? root_name : root_type);
             rc = tctx.counter;
-            if (mode==EsdlRequestMode)
+            if (mode!=EsdlRequestMode)
+                out.set(respWriter->str());
+            else
             {
                 Esdl2Request *rootreq = dynamic_cast<Esdl2Request *>(root);
-                if (rootreq)
+                if (!rootreq || !rootreq->hasDefaults())
+                    out.set(respWriter->str());
+                else
                 {
-                    DBGLOG("XML: %s", respWriter->str());
                     OwnedPTree req = createPTreeFromXMLString(respWriter->str(), false);
                     if (!req.get())
                         req.setown(createPTree(root_type,false));

esp/esdllib/esdl_transformer2.ipp

@@ -387,6 +387,13 @@ public:
     {
         Esdl2Base::serialize(out, "EsdlRequest");
     }
+    bool hasDefaults()
+    {
+        if (!defvals)
+            return false;
+        Owned<IPropertyIterator> it = defvals->getIterator();
+        return (it && it->first());
+    }
 
     virtual void buildDefaults(Esdl2Transformer *xformer);
 

esp/esdllib/esdl_xpath_extensions_libxml.cpp

@@ -223,6 +223,70 @@ static void getStoredStringValueFunction (xmlXPathParserContextPtr ctxt, int nar
 }
 
 /**
+ * scriptGetDataSectionFunctionImpl
+ * @ctxt:  an XPath parser context
+ * @nargs:  the number of arguments
+ *
+ */
+static void scriptGetDataSectionFunctionImpl (xmlXPathParserContextPtr ctxt, int nargs, bool ensure)
+{
+    IEsdlScriptContext *scriptContext = getEsdlScriptContext(ctxt);
+    if (!scriptContext)
+    {
+        xmlXPathSetError((ctxt), XPATH_INVALID_CTXT);
+        return;
+    }
+
+    if (nargs != 1)
+    {
+        xmlXPathSetArityError(ctxt);
+        return;
+    }
+
+    xmlChar *namestring = xmlXPathPopString(ctxt);
+    if (xmlXPathCheckError(ctxt)) //includes null check
+        return;
+    const char *sectionName = isEmptyString((const char *) namestring) ? "temporaries" : (const char *) namestring;
+
+    if (ensure)
+        scriptContext->appendContent(sectionName, nullptr, nullptr);
+
+    StringBuffer xpath("/esdl_script_context/");
+    xpath.append((const char *) sectionName);
+
+    xmlFree(namestring);
+
+    xmlXPathObjectPtr ret = xmlXPathEval((const xmlChar *) xpath.str(), ctxt->context);
+    if (ret)
+        valuePush(ctxt, ret);
+    else
+        xmlXPathReturnEmptyNodeSet(ctxt);
+}
+
+
+/**
+ * scriptEnsureDataSectionFunction
+ * @ctxt:  an XPath parser context
+ * @nargs:  the number of arguments
+ *
+ */
+static void scriptEnsureDataSectionFunction (xmlXPathParserContextPtr ctxt, int nargs)
+{
+    scriptGetDataSectionFunctionImpl (ctxt, nargs, true);
+}
+
+/**
+ * scriptGetDataSectionFunction
+ * @ctxt:  an XPath parser context
+ * @nargs:  the number of arguments
+ *
+ */
+static void scriptGetDataSectionFunction (xmlXPathParserContextPtr ctxt, int nargs)
+{
+    scriptGetDataSectionFunctionImpl (ctxt, nargs, false);
+}
+
+/**
  * getLogOptionFunction
  * @ctxt:  an XPath parser context
  * @nargs:  the number of arguments
@@ -414,6 +478,8 @@ void registerEsdlXPathExtensionsForURI(IXpathContext *xpathContext, const char *
     xpathContext->registerFunction(uri, "secureAccessFlags", (void *)secureAccessFlagsFunction);
     xpathContext->registerFunction(uri, "getFeatureSecAccessFlags", (void *)getFeatureSecAccessFlagsFunction);
     xpathContext->registerFunction(uri, "getStoredStringValue", (void *)getStoredStringValueFunction);
+    xpathContext->registerFunction(uri, "getDataSection", (void *)scriptGetDataSectionFunction);
+    xpathContext->registerFunction(uri, "ensureDataSection", (void *)scriptEnsureDataSectionFunction);
     xpathContext->registerFunction(uri, "storedValueExists", (void *)storedValueExistsFunction);
     xpathContext->registerFunction(uri, "getLogProfile", (void *)getLogProfileFunction);
     xpathContext->registerFunction(uri, "getLogOption", (void *)getLogOptionFunction);

esp/logging/logginglib/CMakeLists.txt

@@ -20,6 +20,7 @@ project( logginglib )
 include(${HPCC_SOURCE_DIR}/esp/scm/espscm.cmake)
 
 include_directories (
+    ${HPCC_SOURCE_DIR}/rtl/include                  #eclhelper.hpp for IXmlWriter
     ${HPCC_SOURCE_DIR}/system/include
     ${HPCC_SOURCE_DIR}/system/jlib
     ${HPCC_SOURCE_DIR}/system/xmllib

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -614,6 +614,8 @@ void EsdlServiceImpl::configureTargets(IPropertyTree *cfg, const char *service)
                         localCppPluginMap.setValue(pluginName, plugin);
                 }
             }
+            else if (type && strieq(type, "script"))
+                DBGLOG("Purely scripted service method %s", method);
             else
                 configureUrlMethod(method, methodCfg);
             DBGLOG("Method %s configured", method);
@@ -639,7 +641,8 @@ enum EsdlMethodImplType
     EsdlMethodImplWsEcl,
     EsdlMethodImplProxy,
     EsdlMethodImplJava,
-    EsdlMethodImplCpp
+    EsdlMethodImplCpp,
+    EsdlMethodImplScript
 };
 
 inline EsdlMethodImplType getEsdlMethodImplType(const char *querytype)
@@ -656,6 +659,8 @@ inline EsdlMethodImplType getEsdlMethodImplType(const char *querytype)
             return EsdlMethodImplJava;
         if (strieq(querytype, "cpp"))
             return EsdlMethodImplCpp;
+        if (strieq(querytype, "script"))
+            return EsdlMethodImplScript;
     }
     return EsdlMethodImplRoxie;
 }
@@ -665,7 +670,7 @@ static inline bool isPublishedQuery(EsdlMethodImplType implType)
     return (implType==EsdlMethodImplRoxie || implType==EsdlMethodImplWsEcl);
 }
 
-IEsdlScriptContext* EsdlServiceImpl::checkCreateEsdlServiceScriptContext(IEspContext &context, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, IPropertyTree *tgtcfg)
+IEsdlScriptContext* EsdlServiceImpl::checkCreateEsdlServiceScriptContext(IEspContext &context, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, IPropertyTree *tgtcfg, IPropertyTree *originalRequest)
 {
     IEsdlTransformEntryPointMap *serviceEPm = m_transforms->queryMethod("");
     IEsdlTransformEntryPointMap *methodEPm = m_transforms->queryMethod(mthdef.queryMethodName());
@@ -679,6 +684,8 @@ IEsdlScriptContext* EsdlServiceImpl::checkCreateEsdlServiceScriptContext(IEspCon
     scriptContext->setAttribute(ESDLScriptCtxSection_ESDLInfo, "request_type", mthdef.queryRequestType());
     scriptContext->setAttribute(ESDLScriptCtxSection_ESDLInfo, "request", mthdef.queryRequestType());  //this could diverge from request_type in the future
 
+    if (originalRequest)
+        scriptContext->setContent(ESDLScriptCtxSection_OriginalRequest, originalRequest);
     if (tgtcfg)
         scriptContext->setContent(ESDLScriptCtxSection_TargetConfig, tgtcfg);
     if (m_oEspBindingCfg)
@@ -686,6 +693,68 @@ IEsdlScriptContext* EsdlServiceImpl::checkCreateEsdlServiceScriptContext(IEspCon
     return scriptContext.getClear();
 }
 
+void EsdlServiceImpl::runPostEsdlScript(IEspContext &context,
+                                         IEsdlScriptContext *scriptContext,
+                                         IEsdlDefService &srvdef,
+                                         IEsdlDefMethod &mthdef,
+                                         StringBuffer &content,
+                                         unsigned txResultFlags,
+                                         const char *ns,
+                                         const char *schema_location)
+{
+    if (scriptContext)
+    {
+        IEsdlTransformSet *serviceIRTs = m_transforms->queryMethodEntryPoint("", ESDLScriptEntryPoint_InitialEsdlResponse);
+        IEsdlTransformSet *methodIRTs = m_transforms->queryMethodEntryPoint(mthdef.queryName(), ESDLScriptEntryPoint_InitialEsdlResponse);
+
+        if (serviceIRTs || methodIRTs)
+        {
+            scriptContext->setContent(ESDLScriptCtxSection_InitialESDLResponse, content.str());
+
+            context.addTraceSummaryTimeStamp(LogNormal, "srt-modifytrans");
+            processServiceAndMethodTransforms(scriptContext, {serviceIRTs, methodIRTs}, ESDLScriptCtxSection_InitialESDLResponse, ESDLScriptCtxSection_ModifiedESDLResponse);
+            scriptContext->toXML(content.clear(), ESDLScriptCtxSection_ModifiedESDLResponse);
+            context.addTraceSummaryTimeStamp(LogNormal, "end-modifytrans");
+
+            //have to make a second ESDL pass (this stage is always a basic response format, never an HPCC/Roxie format)
+            StringBuffer out;
+            m_pEsdlTransformer->process(context, EsdlResponseMode, srvdef.queryName(), mthdef.queryName(), out, content.str(), txResultFlags, ns, schema_location);
+            content.swapWith(out);
+        }
+    }
+
+}
+
+void EsdlServiceImpl::runServiceScript(IEspContext &context,
+                                         IEsdlScriptContext *scriptContext,
+                                         IEsdlDefService &srvdef,
+                                         IEsdlDefMethod &mthdef,
+                                         const char *reqcontent,
+                                         StringBuffer &respcontent,
+                                         unsigned txResultFlags,
+                                         const char *ns,
+                                         const char *schema_location)
+{
+    if (scriptContext)
+    {
+        IEsdlTransformSet *serviceSTs = m_transforms->queryMethodEntryPoint("", ESDLScriptEntryPoint_ScriptedService);
+        IEsdlTransformSet *methodSTs = m_transforms->queryMethodEntryPoint(mthdef.queryName(), ESDLScriptEntryPoint_ScriptedService);
+
+        if (serviceSTs || methodSTs)
+        {
+            scriptContext->setContent(ESDLScriptCtxSection_ScriptRequest, reqcontent);
+
+            context.addTraceSummaryTimeStamp(LogNormal, "srt-script-service");
+            VStringBuffer emptyResponse("<%s/>", mthdef.queryResponseType());
+            scriptContext->setContent(ESDLScriptCtxSection_ScriptResponse, emptyResponse.str());
+            processServiceAndMethodTransforms(scriptContext, {serviceSTs, methodSTs}, ESDLScriptCtxSection_ScriptRequest, ESDLScriptCtxSection_ScriptResponse);
+            scriptContext->toXML(respcontent.clear(), ESDLScriptCtxSection_ScriptResponse);
+            context.addTraceSummaryTimeStamp(LogNormal, "end-script-service");
+        }
+    }
+
+}
+
 void EsdlServiceImpl::handleServiceRequest(IEspContext &context,
                                            Owned<IEsdlScriptContext> &scriptContext,
                                            IEsdlDefService &srvdef,
@@ -905,7 +974,7 @@ void EsdlServiceImpl::handleServiceRequest(IEspContext &context,
         else
         {
             //Future: support transforms for all transaction types by moving scripts and script processing up
-            scriptContext.setown(checkCreateEsdlServiceScriptContext(context, srvdef, mthdef, tgtcfg));
+            scriptContext.setown(checkCreateEsdlServiceScriptContext(context, srvdef, mthdef, tgtcfg, req));
 
             Owned<IXmlWriterExt> reqWriter = createIXmlWriterExt(0, 0, NULL, WTStandard);
 
@@ -922,24 +991,28 @@ void EsdlServiceImpl::handleServiceRequest(IEspContext &context,
             reqcontent.set(reqWriter->str());
             context.addTraceSummaryTimeStamp(LogNormal, "serialized-xmlreq");
 
-            handleFinalRequest(context, scriptContext, tgtcfg, tgtctx, srvdef, mthdef, ns, reqcontent, origResp, isPublishedQuery(implType), implType==EsdlMethodImplProxy, soapmsg);
+            if (implType==EsdlMethodImplScript)
+                runServiceScript(context, scriptContext, srvdef, mthdef, reqcontent, origResp, txResultFlags, ns, schema_location);
+            else
+                handleFinalRequest(context, scriptContext, tgtcfg, tgtctx, srvdef, mthdef, ns, reqcontent, origResp, isPublishedQuery(implType), implType==EsdlMethodImplProxy, soapmsg);
             context.addTraceSummaryTimeStamp(LogNormal, "end-HFReq");
 
-
-
             if (isPublishedQuery(implType))
             {
                 context.addTraceSummaryTimeStamp(LogNormal, "srt-procres");
                 Owned<IXmlWriterExt> respWriter = createIXmlWriterExt(0, 0, NULL, (flags & ESDL_BINDING_RESPONSE_JSON) ? WTJSONRootless : WTStandard);
                 m_pEsdlTransformer->processHPCCResult(context, mthdef, origResp.str(), respWriter.get(), logdata, txResultFlags, ns, schema_location);
                 context.addTraceSummaryTimeStamp(LogNormal, "end-procres");
-
                 out.append(respWriter->str());
+                runPostEsdlScript(context, scriptContext, srvdef, mthdef, out, txResultFlags, ns, schema_location);
             }
             else if(implType==EsdlMethodImplProxy)
                 getSoapBody(out, origResp);
             else
+            {
                 m_pEsdlTransformer->process(context, EsdlResponseMode, srvdef.queryName(), mthdef.queryName(), out, origResp.str(), txResultFlags, ns, schema_location);
+                runPostEsdlScript(context, scriptContext, srvdef, mthdef, out, txResultFlags, ns, schema_location);
+            }
         }
     }
 
@@ -3514,7 +3587,7 @@ int EsdlBindingImpl::onGetRoxieBuilder(CHttpRequest* request, CHttpResponse* res
 
                     tgtctx.setown( m_pESDLService->createTargetContext(*context, tgtcfg, *defsrv, *defmth, req_pt));
 
-                    Owned<IEsdlScriptContext> scriptContext = m_pESDLService->checkCreateEsdlServiceScriptContext(*context, *defsrv, *defmth, tgtcfg.get());
+                    Owned<IEsdlScriptContext> scriptContext = m_pESDLService->checkCreateEsdlServiceScriptContext(*context, *defsrv, *defmth, tgtcfg.get(), req_pt);
                     m_pESDLService->prepareFinalRequest(*context, scriptContext, tgtcfg, tgtctx, *defsrv, *defmth, true, ns.str(), reqcontent, roxiemsg);
                 }
                 else

esp/services/esdl_svc_engine/esdl_binding.hpp

@@ -177,7 +177,9 @@ public:
     void handleTransformError(StringAttr &serviceError, TransformErrorMap &methodErrors, IException *e, const char *service, const char *method);
     void addTransforms(IPropertyTree *cfgParent, const char *service, const char *method, bool removeCfgIEntries);
 
-    IEsdlScriptContext* checkCreateEsdlServiceScriptContext(IEspContext &context, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, IPropertyTree *tgtcfg);
+    IEsdlScriptContext* checkCreateEsdlServiceScriptContext(IEspContext &context, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, IPropertyTree *tgtcfg, IPropertyTree *origReq);
+    void runPostEsdlScript(IEspContext &context, IEsdlScriptContext *scriptContext, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer &out, unsigned txResultFlags, const char *ns, const char *schema_location);
+    void runServiceScript(IEspContext &context, IEsdlScriptContext *scriptContext, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, const char *reqcontent, StringBuffer &out, unsigned txResultFlags, const char *ns, const char *schema_location);
 
     virtual void handleServiceRequest(IEspContext &context, Owned<IEsdlScriptContext> &scriptContext, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, Owned<IPropertyTree> &tgtcfg, Owned<IPropertyTree> &tgtctx, const char *ns, const char *schema_location, IPropertyTree *req, StringBuffer &out, StringBuffer &logdata, StringBuffer &origResp, StringBuffer &soapmsg, unsigned int flags);
     virtual void generateTransactionId(IEspContext & context, StringBuffer & trxid)=0;

esp/services/ws_esdlconfig/CMakeLists.txt

@@ -46,6 +46,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/mp
          ${HPCC_SOURCE_DIR}/system/xmllib
          ${HPCC_SOURCE_DIR}/rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/rtl/include
          ${HPCC_SOURCE_DIR}/esp/platform
          ${HPCC_SOURCE_DIR}/esp/clients
          ${HPCC_SOURCE_DIR}/esp/bindings

initfiles/examples/EsdlExample/esdl_binding_entrypoints.xml

@@ -21,6 +21,11 @@
               </es:target>
             </es:target>
          </es:BackendResponse>
+         <es:EsdlResponse xmlns:ns1="urn:hpccsystems:ws:esdlexample:roxieechopersoninfo@ver=0.01">
+             <es:target xpath="ns1:Name">
+               <es:append-to-value target="ns1:Last" value="'-post-esdl-fix'" />
+             </es:target>
+         </es:EsdlResponse>
          <es:BackendResponse>
             <es:set-value target="BRESPSRV2" value="'s22'" />
          </es:BackendResponse>

initfiles/examples/EsdlExample/esdl_binding_mysql_service.xml

@@ -0,0 +1,51 @@
+<Methods>
+   <Method name="MySqlEchoAddressReset" querytype="script">
+      <Scripts><![CDATA[
+         <Transforms xmlns:es='urn:hpcc:esdl:script'>
+           <es:Service>
+            <es:variable name="secret" select="'mydb'"/>
+            <es:variable name="database" select="'classicmodels'"/>
+            <es:variable name="section" select="'sql'"/>
+            <es:mysql secret="$secret" database="$database" section="$section" name="drop">
+              <es:sql>DROP TABLE IF EXISTS esdlecho;</es:sql>
+            </es:mysql>
+           </es:Service>
+         </Transforms>
+      ]]>
+      </Scripts>
+   </Method>
+   <Method name="MySqlEchoAddressInfo" querytype="script">
+      <Scripts><![CDATA[
+         <Transforms xmlns:es='urn:hpcc:esdl:script'>
+           <es:Service>
+            <es:variable name="secret" select="'mydb'"/>
+            <es:variable name="database" select="'classicmodels'"/>
+            <es:variable name="section" select="'sql'"/>
+            <es:mysql secret="$secret" database="$database" section="$section" name="create">
+              <es:sql>CREATE TABLE IF NOT EXISTS esdlecho ( type VARCHAR(5), Line1 VARCHAR(30), Line2 VARCHAR(30), City VARCHAR(30),  State VARCHAR(30), Zip INT );</es:sql>
+            </es:mysql>
+            <es:mysql select="Addresses/Address" secret="$secret" database="$database" section="$section" name="insert_request">
+              <es:bind name="type" value="type"/>
+              <es:bind name="Line1" value="Line1"/>
+              <es:bind name="Line2" value="Line2"/>
+              <es:bind name="City" value="City"/>
+              <es:bind name="State" value="State"/>
+              <es:bind name="Zip" value="Zip"/>
+              <es:sql>INSERT INTO esdlecho (type, Line1, Line2, City, State, Zip) values (?, ?, ?, ?, ?, ?);</es:sql>
+            </es:mysql>
+            <es:mysql secret="$secret" database="$database" section="$section" name="select_all">
+              <es:sql>SELECT * FROM esdlecho;</es:sql>
+            </es:mysql>
+            <es:ensure-target xpath="Addresses">
+             <es:for-each select="$select_all/Row">
+              <es:element name="Address">
+                <es:copy-of select="*"/>
+              </es:element>
+             </es:for-each>
+            </es:ensure-target>
+           </es:Service>
+         </Transforms>
+      ]]>
+      </Scripts>
+   </Method>
+</Methods>

initfiles/examples/EsdlExample/esdl_example.esdl

@@ -80,10 +80,31 @@ ESPresponse RoxieEchoPersonInfoResponse
      ESParray<ESPstruct AddressInfo, Address> Addresses;
 };
 
+ESPrequest MySqlEchoAddressInfoRequest
+{
+     ESParray<ESPstruct AddressInfo, Address> Addresses;
+};
+
+ESPresponse MySqlEchoAddressInfoResponse
+{
+     ESParray<ESPstruct AddressInfo, Address> Addresses;
+};
+
+ESPrequest MySqlEchoAddressResetRequest
+{
+};
+
+ESPresponse MySqlEchoAddressResetResponse
+{
+};
+
+
 ESPservice [version("0.01")] EsdlExample
 {
     ESPmethod CppEchoPersonInfo(CppEchoPersonInfoRequest, CppEchoPersonInfoResponse);
     ESPmethod JavaEchoPersonInfo(JavaEchoPersonInfoRequest, JavaEchoPersonInfoResponse);
     ESPmethod RoxieEchoPersonInfo(RoxieEchoPersonInfoRequest, RoxieEchoPersonInfoResponse);
+    ESPmethod MySqlEchoAddressInfo(MySqlEchoAddressInfoRequest, MySqlEchoAddressInfoResponse);
+    ESPmethod MySqlEchoAddressReset(MySqlEchoAddressResetRequest, MySqlEchoAddressResetResponse);
 };
 

plugins/mysql/mysqlembed.cpp

@@ -534,6 +534,10 @@ public:
     {
         return stmt;
     }
+    uint64_t getAffectedRows()
+    {
+        return (stmt) ? mysql_stmt_affected_rows(stmt) : 0;
+    }
 private:
     MySQLStatement(const MySQLStatement &);
     MYSQL_STMT *stmt;
@@ -730,6 +734,14 @@ public:
     {
         return *res != NULL;
     }
+    inline const MySQLResult *queryResultInfo() const
+    {
+        return res;
+    }
+    uint64_t getAffectedRows()
+    {
+        return (stmt) ? stmt->getAffectedRows() : 0;
+    }
 protected:
     Linked<MySQLConnection> conn;
     Linked<MySQLStatement> stmt;
@@ -1140,6 +1152,107 @@ protected:
     int colIdx;
 };
 
+class MySQLXmlRowBuilder : public CInterface
+{
+public:
+    MySQLXmlRowBuilder(const MySQLResult *_result, const MySQLBindingArray &_boundInfo)
+    : result(_result), boundInfo(_boundInfo), colIdx(-1)
+    {
+    }
+    virtual void writeXmlColumn(IXmlWriter *writer, const MYSQL_FIELD &col, const MYSQL_BIND &bound)
+    {
+        if (*bound.is_null)
+            return;
+        switch (bound.buffer_type)
+        {
+            case MYSQL_TYPE_TINY:
+            case MYSQL_TYPE_SHORT:
+            case MYSQL_TYPE_LONG:
+            case MYSQL_TYPE_LONGLONG:
+            case MYSQL_TYPE_INT24:
+            {
+                if (bound.is_unsigned)
+                    writer->outputUInt(rtlReadUInt(bound.buffer, *bound.length), *bound.length, col.name);
+                else
+                    writer->outputInt(rtlReadInt(bound.buffer, *bound.length), *bound.length, col.name);
+                break;
+            }
+            case MYSQL_TYPE_BIT:
+                writer->outputUInt(rtlReadSwapUInt(bound.buffer, *bound.length), *bound.length, col.name);
+                break;
+            case MYSQL_TYPE_FLOAT:
+                writer->outputReal(* (float *) bound.buffer, col.name);
+                break;
+            case MYSQL_TYPE_DOUBLE:
+                writer->outputReal(* (double *) bound.buffer, col.name);
+                break;
+            case MYSQL_TYPE_VARCHAR:
+            case MYSQL_TYPE_ENUM:
+            case MYSQL_TYPE_SET:
+            case MYSQL_TYPE_TINY_BLOB:
+            case MYSQL_TYPE_MEDIUM_BLOB:
+            case MYSQL_TYPE_LONG_BLOB:
+            case MYSQL_TYPE_BLOB:
+            case MYSQL_TYPE_VAR_STRING:
+            case MYSQL_TYPE_STRING:
+            case MYSQL_TYPE_GEOMETRY:
+            {
+                const char *text = (const char *) bound.buffer;
+                //For XML treat all strings as UTF-8, caller does have to get the characterset of the datase right if it's not utf8
+                //  (for example by setting MYSQL_SET_CHARSET_NAME="latin1"), may need improving as we get experience writing ESDL scripts that use it,
+                //  but I did test with both utf8 and latin1 databases
+                writer->outputUtf8(rtlUtf8Length(*bound.length, text), text, col.name);
+                break;
+            }
+            case MYSQL_TYPE_TIMESTAMP:
+            case MYSQL_TYPE_DATETIME:
+            case MYSQL_TYPE_DATE:
+            case MYSQL_TYPE_TIME:
+            {
+                const MYSQL_TIME * time = (const MYSQL_TIME *) bound.buffer;
+                char temp[20];
+                switch (bound.buffer_type)
+                {
+                    case MYSQL_TYPE_TIMESTAMP:
+                    case MYSQL_TYPE_DATETIME:
+                        _snprintf(temp, sizeof(temp), "%4u-%02u-%02u %02u:%02u:%02u", time->year, time->month, time->day, time->hour, time->minute, time->second);
+                        break;
+                    case MYSQL_TYPE_DATE:
+                        _snprintf(temp, sizeof(temp), "%4u-%02u-%02u", time->year, time->month, time->day);
+                        break;
+                    case MYSQL_TYPE_TIME:
+                        _snprintf(temp, sizeof(temp), "%02u:%02u:%02u", time->hour, time->minute, time->second);
+                        break;
+                }
+                writer->outputCString((const char *)temp, col.name);
+                break;
+            }
+            default:
+                break;
+        }
+    }
+
+   virtual void writeXmlRow(IXmlWriter *writer)
+    {
+        if (!result)
+            return;
+        writer->outputBeginNested("Row", true);
+        for (int colIdx=0; colIdx < boundInfo.numColumns(); colIdx++)
+        {
+            const MYSQL_FIELD *col = mysql_fetch_field_direct(*result, colIdx);
+            if (!col)
+                continue;
+            const MYSQL_BIND &bound = boundInfo.queryColumn(colIdx, nullptr);
+            writeXmlColumn(writer, *col, bound);
+        }
+        writer->outputEndNested("Row");
+    }
+protected:
+    const MySQLBindingArray &boundInfo;
+    const MySQLResult *result;
+    int colIdx;
+};
+
 // Bind MySQL variables from an ECL record
 
 class MySQLRecordBinder : public CInterfaceOf<IFieldProcessor>
@@ -1394,6 +1507,71 @@ protected:
     bool eof;
 };
 
+class MySQLXmlWriter : public CInterface
+{
+public:
+    MySQLXmlWriter(MySQLDatasetBinder *_inputStream, MySQLPreparedStatement *_stmtInfo, IXmlWriter *_writer)
+    : inputStream(_inputStream), stmtInfo(_stmtInfo), writer(_writer)
+    {
+        executePending = true;
+        eof = false;
+    }
+    bool nextRow()
+    {
+        // leave the streaming input support as is for possible future use, but not being used now
+        if (eof)
+            return false;
+        for (;;)
+        {
+            if (executePending)
+            {
+                executePending = false;
+                if (inputStream && !inputStream->bindNext())
+                {
+                    noteEOF();
+                    return false;
+                }
+                stmtInfo->execute();
+            }
+            if (!stmtInfo->hasResult())
+            {
+                writer->outputUInt(stmtInfo->getAffectedRows(), 64, "@affectedRows");
+                noteEOF();
+                return false;
+            }
+            if (stmtInfo->next())
+                break;
+            if (inputStream)
+                executePending = true;
+            else
+            {
+                noteEOF();
+                return false;
+            }
+        }
+
+        MySQLXmlRowBuilder mysqlXmlRowBuilder(stmtInfo->queryResultInfo(), stmtInfo->queryResultBindings());
+        mysqlXmlRowBuilder.writeXmlRow(writer);
+        return true;
+    }
+    void write()
+    {
+        while (nextRow());
+    }
+
+protected:
+    void noteEOF()
+    {
+        if (!eof)
+            eof = true;
+    }
+    Linked<MySQLDatasetBinder> inputStream;
+    Linked<MySQLPreparedStatement> stmtInfo;
+    Linked<IXmlWriter> writer;
+    bool executePending;
+    bool eof;
+};
+
 // Each call to a MySQL function will use a new MySQLEmbedFunctionContext object
 
 static bool mysqlInitialized = false;
@@ -1627,8 +1805,15 @@ public:
     virtual void paramWriterCommit(IInterface *writer)
     {
     }
-    virtual void writeResult(IInterface *esdl, const char *esdlservice, const char *esdltype, IInterface *writer)
+    virtual void writeResult(IInterface *esdl, const char *esdlservice, const char *esdltype, IInterface *iifWriter)
     {
+        if (esdl)
+            UNSUPPORTED("ESDL defined result formatting");
+
+        IXmlWriter *writer = dynamic_cast<IXmlWriter*>(iifWriter);
+        MySQLXmlWriter xmlWriter(inputStream, stmtInfo, writer);
+        xmlWriter.write();
+        nextParam = 0; //if you start binding parameters again after writing the result you are working on the next call
     }
 
     virtual void importFunction(size32_t lenChars, const char *text)
@@ -1761,4 +1946,11 @@ extern DECL_EXPORT bool syntaxCheck(const char *script)
     return true; // MORE
 }
 
+// Used for dynamically loading in ESDL
+
+extern "C" DECL_EXPORT IEmbedContext *getEmbedContextDynamic()
+{
+    return mysqlembed::getEmbedContext();
+}
+
 } // namespace

system/jlib/jsecrets.cpp

@@ -566,7 +566,7 @@ static IPropertyTree *loadLocalSecret(const char *category, const char * name)
         if (!validateXMLTag(name))
             continue;
         MemoryBuffer content;
-        Owned<IFileIO> io = entries->get().open(IFOread);
+        Owned<IFileIO> io = entries->query().open(IFOread);
         read(io, 0, (size32_t)-1, content);
         if (!content.length())
             continue;

system/xmllib/CMakeLists.txt

@@ -56,6 +56,8 @@ include_directories (
          ./../../system/include
          ./../../system/jlib
          ./../../ecl/hql
+         ./../../rtl/include
+         ./../../rtl/eclrtl
          ${LIB_INCLUDE_DIR}
     )
 
@@ -65,5 +67,6 @@ HPCC_ADD_LIBRARY( xmllib SHARED ${SRCS} )
 install ( TARGETS xmllib RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
 target_link_libraries ( xmllib
          ${LIB_LIBRARIES}
+         eclrtl
          jlib
     )

system/xmllib/libxml_xpathprocessor.cpp

@@ -20,6 +20,8 @@
 #include "jptree.hpp"
 #include "jexcept.hpp"
 #include "jlog.hpp"
+#include "eclhelper.hpp" //IXmlWriter
+#include "eclrtl.hpp" //IXmlWriter
 
 #include <libxml/xmlmemory.h>
 #include <libxml/parserInternals.h>
@@ -193,6 +195,133 @@ public:
     }
 };
 
+class XpathContextXmlWriter : public CInterfaceOf<IXmlWriter>
+{
+public:
+    XpathContextXmlWriter(xmlNodePtr _location) : location(_location)
+    {
+    }
+    void addNameValue(const char *name, const char *value)
+    {
+        if (isEmptyString(name))
+            return;
+        if (*name=='@')
+            xmlSetProp(location, (const xmlChar *)name+1, (const xmlChar *)value);
+        else
+            xmlNewTextChild(location, nullptr, (const xmlChar *) name, (const xmlChar *) value);
+    }
+    virtual void outputQuoted(const char *text) override
+    {
+
+    }
+    virtual void outputString(unsigned len, const char *field, const char *fieldname) override
+    {
+        StringAttr out(field, len);
+        addNameValue(fieldname, out.str());
+    }
+    virtual void outputBool(bool field, const char *fieldname) override
+    {
+        addNameValue(fieldname, field ? "true" : "false");
+    }
+
+    virtual void outputData(unsigned len, const void *field, const char *fieldname) override
+    {
+        StringBuffer out;
+        outputXmlData(len, field, nullptr, out);
+        addNameValue(fieldname, out.str());
+    }
+    virtual void outputInt(__int64 field, unsigned size, const char *fieldname) override
+    {
+        StringBuffer out;
+        outputXmlInt(field, nullptr, out);
+        addNameValue(fieldname, out.str());
+    }
+    virtual void outputUInt(unsigned __int64 field, unsigned size, const char *fieldname) override
+    {
+        StringBuffer out;
+        outputXmlUInt(field, nullptr, out);
+        addNameValue(fieldname, out.str());
+    }
+    virtual void outputReal(double field, const char *fieldname) override
+    {
+        StringBuffer out;
+        outputXmlReal(field, nullptr, out);
+        addNameValue(fieldname, out.str());
+    }
+    virtual void outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname) override
+    {
+        StringBuffer out;
+        outputXmlDecimal(field, size, precision, fieldname, out);
+        addNameValue(fieldname, out.str());
+    }
+    virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname) override
+    {
+        StringBuffer out;
+        outputXmlUDecimal(field, size, precision, fieldname, out);
+        addNameValue(fieldname, out.str());
+    }
+    virtual void outputUnicode(unsigned len, const UChar *field, const char *fieldname) override
+    {
+        StringBuffer out;
+        outputXmlUnicode(len, field, nullptr, out);
+        addNameValue(fieldname, out.str());
+    }
+    virtual void outputQString(unsigned len, const char *field, const char *fieldname) override
+    {
+        MemoryAttr tempBuffer;
+        char * temp;
+        if (len <= 100)
+            temp = (char *)alloca(len);
+        else
+            temp = (char *)tempBuffer.allocate(len);
+        rtlQStrToStr(len, temp, len, field);
+        outputString(len, temp, fieldname);
+    }
+    virtual void outputBeginDataset(const char *dsname, bool nestChildren) override
+    {
+        outputBeginNested("Dataset", nestChildren); //indent row, not dataset for backward compatibility
+        if (!isEmptyString(dsname))
+            outputUtf8(rtlUtf8Length(strlen(dsname), dsname), dsname, "@name");
+    }
+    virtual void outputEndDataset(const char *dsname) override
+    {
+        outputEndNested("Dataset");
+    }
+    virtual void outputBeginNested(const char *fieldname, bool nestChildren) override
+    {
+        location = xmlNewChild(location, nullptr, (const xmlChar *)fieldname, nullptr);
+    }
+    virtual void outputEndNested(const char *fieldname) override
+    {
+        if (location->parent)
+            location = location->parent;
+    }
+    virtual void outputSetAll() override
+    {
+        StringBuffer out;
+        outputXmlSetAll(out);
+        xmlNodeAddContent(location, (const xmlChar *)out.str());
+    }
+    virtual void outputUtf8(unsigned len, const char *field, const char *fieldname) override
+    {
+        StringBuffer out;
+        outputXmlUtf8(len, field, nullptr, out);
+        addNameValue(fieldname, out.str());
+    }
+    virtual void outputBeginArray(const char *fieldname){} //no op for libxml structure
+    virtual void outputEndArray(const char *fieldname){}
+
+    virtual void outputInlineXml(const char *text){} //um no
+    virtual void outputXmlns(const char *name, const char *uri)
+    {
+        xmlNewNs(location, (const xmlChar *) uri, (const xmlChar *) name);
+    }
+    virtual void flushContent(bool close){}
+private:
+    xmlNodePtr location;
+};
+
+
 typedef std::vector<XpathContextState> XPathContextStateVector;
 
 class CLibXpathContext : public CInterfaceOf<IXpathContext>, implements IVariableSubstitutionHelper
@@ -478,8 +607,12 @@ public:
         xmlFreeParserCtxt(parserCtx);
         if (!wellFormed)
             throw MakeStringException(-1, "XpathContext:addXmlContent: Unable to parse %s XML content", xml);
-
     }
+    virtual IXmlWriter *createXmlWriter() override
+    {
+        return new XpathContextXmlWriter(m_xpathContext->node);
+    }
+
     virtual bool ensureLocation(const char *xpath, bool required) override
     {
         if (isEmptyString(xpath))
@@ -1286,6 +1419,13 @@ extern ICompiledXpath* compileXpath(const char * xpath)
     return new CLibCompiledXpath(xpath);
 }
 
+extern ICompiledXpath* compileOptionalXpath(const char * xpath)
+{
+    if (isEmptyString(xpath))
+        return nullptr;
+    return compileXpath(xpath);
+}
+
 void addChildFromPtree(xmlNodePtr parent, IPropertyTree &tree, const char *name)
 {
     if (isEmptyString(name))
@@ -1584,10 +1724,10 @@ private:
     }
     virtual void appendContent(const char *section, const char *name, const char *xml) override
     {
+        xmlNodePtr sect = ensureSection(section);
+
         if (xml==nullptr)
             return;
-
-        xmlNodePtr sect = ensureSection(section);
         xmlNodePtr sectNode = getSectionNode(section, name);
 
         xmlDocPtr doc = xmlParseDoc((const xmlChar *)xml);
@@ -1605,7 +1745,7 @@ private:
             if (!isEmptyString(name))
                 xmlNodeSetName(content, (const xmlChar *) name);
             xmlAddChild(sect, content);
-            xmlFree(doc);
+            xmlFreeDoc(doc);
             return;
         }
         xmlAttrPtr prop = content->properties;

system/xmllib/xalan_processor.cpp

@@ -694,6 +694,11 @@ void CXslTransform::message(StringBuffer& out, const char* in, IXslTransform* pT
     pTrans->m_sMessages.append(in).append('\n');
 }
 
+extern ICompiledXpath* compileOptionalXpath(const char * xpath)
+{
+    UNIMPLEMENTED;
+}
+
 extern ICompiledXpath* compileXpath(const char * xpath)
 {
     UNIMPLEMENTED;

system/xmllib/xmllib_unsupported.cpp

@@ -24,6 +24,11 @@ extern IXpathContext* getXpathContext(const char * xmldoc)
     throw MakeStringException(XMLERR_MissingDependency, "XSLT library unavailable");
 }
 
+extern ICompiledXpath* compileOptionalXpath(const char * xpath)
+{
+    UNIMPLEMENTED;
+}
+
 extern ICompiledXpath* compileXpath(const char * xpath)
 {
     UNIMPLEMENTED;

system/xmllib/xpathprocessor.hpp

@@ -20,6 +20,7 @@
 
 #include "xmllib.hpp"
 #include "jliball.hpp"
+#include "eclhelper.hpp"
 
 interface XMLLIB_API ICompiledXpath : public IInterface
 {
@@ -74,6 +75,7 @@ interface XMLLIB_API IXpathContext : public IInterface
     virtual  IXpathContextIterator *evaluateAsNodeSet(ICompiledXpath * compiledXpath) = 0;
     virtual StringBuffer &toXml(const char *xpath, StringBuffer & xml) = 0;
     virtual void addXmlContent(const char *xml) = 0;
+    virtual IXmlWriter *createXmlWriter() = 0;
 };
 
 interface IXpathContextIterator : extends IIteratorOf<IXpathContext> { };
@@ -124,6 +126,7 @@ public:
 };
 
 extern "C" XMLLIB_API ICompiledXpath* compileXpath(const char * xpath);
+extern "C" XMLLIB_API ICompiledXpath* compileOptionalXpath(const char * xpath);
 extern "C" XMLLIB_API IXpathContext*  getXpathContext(const char * xmldoc, bool strictParameterDeclaration, bool removeDocNamespaces);
 
 #define ESDLScriptCtxSection_Store "store"
@@ -132,10 +135,15 @@ extern "C" XMLLIB_API IXpathContext*  getXpathContext(const char * xmldoc, bool
 #define ESDLScriptCtxSection_TargetConfig "target"
 #define ESDLScriptCtxSection_BindingConfig "config"
 #define ESDLScriptCtxSection_ESDLInfo "esdl"
+#define ESDLScriptCtxSection_OriginalRequest "original_request"
 #define ESDLScriptCtxSection_ESDLRequest "esdl_request"
 #define ESDLScriptCtxSection_FinalRequest "final_request"
 #define ESDLScriptCtxSection_InitialResponse "initial_response"
 #define ESDLScriptCtxSection_PreESDLResponse "pre_esdl_response"
+#define ESDLScriptCtxSection_InitialESDLResponse "initial_esdl_response"
+#define ESDLScriptCtxSection_ModifiedESDLResponse "modified_esdl_response"
+#define ESDLScriptCtxSection_ScriptRequest "script_request"
+#define ESDLScriptCtxSection_ScriptResponse "script_response"
 
 interface IEsdlScriptContext : extends IInterface
 {

testing/unittests/CMakeLists.txt

@@ -63,6 +63,9 @@ include_directories (
          ./../../system/security/cryptohelper
          ./../../configuration/configmgr/configmgrlib
          ${HPCC_SOURCE_DIR}/system/xmllib
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/rtl/include
+         ${HPCC_SOURCE_DIR}/common/dllserver
          ${HPCC_SOURCE_DIR}/ecl/hql
          ${HPCC_SOURCE_DIR}/configuration/configmgr/RapidJSON/include
          ${HPCC_SOURCE_DIR}/esp/bindings

testing/unittests/esdltests.cpp

@@ -23,6 +23,10 @@
 #include "wsexcept.hpp"
 
 #include <stdio.h>
+#include "dllserver.hpp"
+#include "thorplugin.hpp"
+#include "eclrtl.hpp"
+#include "rtlformat.hpp"
 
 // =============================================================== URI parser
 
@@ -292,6 +296,8 @@ class ESDLTests : public CppUnit::TestFixture
         CPPUNIT_TEST(testEsdlTransformRequestNamespaces);
         CPPUNIT_TEST(testScriptContext);
         CPPUNIT_TEST(testTargetElement);
+      //The following require setup, uncomment for development testing for now:
+      //CPPUNIT_TEST(testMysql);
       //CPPUNIT_TEST(testScriptMap); //requires a particular roxie query
       //CPPUNIT_TEST(testHTTPPostXml); //requires a particular roxie query
     CPPUNIT_TEST_SUITE_END();
@@ -1844,6 +1850,172 @@ constexpr const char * result = R"!!(<soap:Envelope xmlns:soap="http://schemas.x
             CPPUNIT_ASSERT(false);
         }
     }
+    void testMysql()
+    {
+        constexpr const char *config1 = R"!!(<config>
+          <Transform>
+            <Param name='testcase' value="new features"/>
+          </Transform>
+        </config>)!!";
+
+        static constexpr const char * data = R"!!(<?xml version="1.0" encoding="UTF-8"?>
+        <root>
+          <insert>
+            <common_value>178</common_value>
+            <common_r8>1.2</common_r8>
+            <Row>
+              <name>selected1</name>
+              <bval>65</bval>
+              <boolval>1</boolval>
+              <r4>3.4</r4>
+              <d>aa55aa55</d>
+              <ddd>1234567.89</ddd>
+              <u1>Straße1</u1>
+              <u2>ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ</u2>
+              <dt>2019-02-01 12:59:59</dt>
+            </Row>
+            <Row>
+              <name>selected2</name>
+              <bval>65</bval>
+              <boolval>1</boolval>
+              <r4>4.5</r4>
+              <d>bb66bb66</d>
+              <ddd>1234567.89</ddd>
+              <u1>Straße3</u1>
+              <u2>Straße4</u2>
+              <dt>2019-02-01 13:59:59</dt>
+            </Row>
+            <Row>
+              <name>selected3</name>
+              <bval>65</bval>
+              <boolval>1</boolval>
+              <r4>5.6</r4>
+              <d>cc77cc77</d>
+              <ddd>1234567.89</ddd>
+              <u1>Straße5</u1>
+              <u2>色は匂へど 散りぬるを</u2>
+              <dt>2019-02-01 14:59:59</dt>
+            </Row>
+          </insert>
+          <cities>
+            <name>aeiou</name>
+            <name>aeou</name>
+            <name>aoui</name>
+            <name>ei</name>
+            <name>aaa</name>
+            <name>bbb</name>
+          </cities>
+          <read>
+            <name>selected1</name>
+            <name>selected3</name>
+          </read>
+        </root>
+        )!!";
+
+        static constexpr const char * input = R"!!(<?xml version="1.0" encoding="UTF-8"?>
+        <root>
+          <Person>
+            <FullName>
+              <First>Joe</First>
+            </FullName>
+          </Person>
+        </root>
+        )!!";
+
+        static constexpr const char * script = R"!!(<es:CustomRequestTransform xmlns:es="urn:hpcc:esdl:script" target="Person">
+          <es:variable name="secret" select="'mydb'"/>
+          <es:variable name="database" select="'classicmodels'"/>
+          <es:variable name="section" select="'sql'"/>
+          <es:mysql secret="$secret" database="$database" section="$section" name="drop">
+            <es:sql>DROP TABLE IF EXISTS tbl1;</es:sql>
+          </es:mysql>
+          <es:mysql secret="$secret" database="$database" section="$section" name="create">
+            <es:sql>CREATE TABLE tbl1 ( name VARCHAR(20), bval BIT(15), value INT, boolval TINYINT, r8 DOUBLE, r4 FLOAT, d BLOB, ddd DECIMAL(10,2), u1 VARCHAR(10), u2 VARCHAR(10), dt DATETIME );</es:sql>
+          </es:mysql>
+          <es:mysql select="getDataSection('whatever')/this/insert/Row" secret="$secret" database="$database" section="$section" name="insert_each_row" resultset-tag="'inserted'">
+            <es:bind name="name" value="name"/>
+            <es:bind name="bval" value="bval" type="BIT(15)"/>
+            <es:bind name="value" value="../common_value"/>
+            <es:bind name="boolval" value="boolval"/>
+            <es:bind name="r8" value="../common_r8"/>
+            <es:bind name="r4" value="r4"/>
+            <es:bind name="d" value="d"/>
+            <es:bind name="ddd" value="ddd"/>
+            <es:bind name="u1" value="u1"/>
+            <es:bind name="u2" value="u2"/>
+            <es:bind name="dt" value="dt"/>
+            <es:sql>INSERT INTO tbl1 values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);</es:sql>
+          </es:mysql>
+          <es:mysql secret="$secret" database="$database" section="$section" name="drop">
+            <es:sql>DROP TABLE IF EXISTS tblcities;</es:sql>
+          </es:mysql>
+          <es:mysql secret="$secret" database="$database" section="$section" name="createcites">
+            <es:sql>CREATE TABLE tblcities ( city VARCHAR(20) );</es:sql>
+          </es:mysql>
+          <es:mysql select="getDataSection('whatever')/this/cities/name" secret="$secret" database="$database" section="$section" name="insert_each_row" resultset-tag="'inserted'">
+            <es:bind name="city" value="."/>
+            <es:sql>INSERT INTO tblcities values (?);</es:sql>
+          </es:mysql>
+          <es:mysql select="getDataSection('whatever')/this/read/name" secret="$secret" database="$database" section="$section" name="select_each_name" resultset-tag="'selected'">
+            <es:bind name="name" value="."/>
+            <es:sql>SELECT * FROM tbl1 where name = ?;</es:sql>
+          </es:mysql>
+          <es:mysql secret="$secret" database="$database" section="$section" name="mysql_session_info" MYSQL_SET_CHARSET_NAME="'latin1'">
+            <es:sql>
+            SELECT * FROM performance_schema.session_variables
+            WHERE VARIABLE_NAME IN (
+            'character_set_client', 'character_set_connection',
+            'character_set_results', 'collation_connection'
+            ) ORDER BY VARIABLE_NAME;
+            </es:sql>
+          </es:mysql>
+          <es:mysql secret="$secret" database="$database" section="$section" name="select_all" resultset-tag="'onecall'">
+            <es:sql>SELECT * FROM tbl1;</es:sql>
+          </es:mysql>
+          <es:for-each select="$select_all/onecall/Row">
+            <es:element name="r">
+              <es:copy-of select="*"/>
+            </es:element>
+          </es:for-each>
+            <es:mysql secret="$secret" database="$database" section="$section" name="mysqlresult">
+              <es:sql>SELECT * FROM tblcities;</es:sql>
+            </es:mysql>
+            <es:variable name="i" select="$mysqlresult/Row/city[contains(.,'i')]" />
+            <es:variable name="e" select="$mysqlresult/Row/city[contains(.,'e')]" />
+            <es:ensure-target xpath="iii">
+                <es:copy-of select="$i" />
+            </es:ensure-target>
+            <es:ensure-target xpath="eee">
+                <es:copy-of select="$e" />
+            </es:ensure-target>
+            <es:ensure-target xpath="cities">
+              <es:for-each select="set:intersection($i, $e)">
+                <es:copy-of select="." />
+              </es:for-each>
+            </es:ensure-target>
+        </es:CustomRequestTransform>
+        )!!";
+
+        Owned<IEspContext> ctx = createEspContext(nullptr);
+        Owned<IEsdlScriptContext> scriptContext = createTestScriptContext(ctx, input, config1);
+        scriptContext->appendContent("whatever", "this", data);
+
+        try
+        {
+            runTransform(scriptContext, script, ESDLScriptCtxSection_ESDLRequest, "MyResult", "http post xml", 0);
+        }
+        catch (IException *E)
+        {
+            StringBuffer m;
+            fprintf(stdout, "\nTest(%s) Exception %d - %s\n", "mysql", E->errorCode(), E->errorMessage(m).str());
+            E->Release();
+        }
+
+        StringBuffer output;
+        scriptContext->toXML(output);
+        fputs(output.str(), stdout);
+        fflush(stdout);
+    }
 
     void testScriptMap()
     {