HPCC-9805: ECL support for HTTP POST JSON and XML

Enable HTTPCALL support for HTTP POST of JSON and XML content.
HTTPCALL/POST behaves like SOAPCALL.
Also adds support for extra hints inside the XPATH attribute.
Besides providing the route xpath the user can now specify the
exact path to the row and to the exceptions.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexisrisk.com>

common/thorhelper/thorsoapcall.cpp

@@ -657,32 +657,63 @@ interface IWSCAsyncFor: public IInterface
 };
 
 class CWSCHelper;
-IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, CommonXmlWriter &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options);
+IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, IXmlWriterExt &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options);
 
 //=================================================================================================
 
+#define CBExceptionPathExc     0x0001
+#define CBExceptionPathExcsExc 0x0002
+#define CBExceptionPathExcs    0x0004
+
 class CMatchCB : implements IXMLSelect, public CInterface
 {
     IWSCAsyncFor &parent;
     const Url &url;
     StringAttr tail;
     ColumnProvider * meta;
+    StringAttr excPath;
+    unsigned excFlags = 0;
 public:
     IMPLEMENT_IINTERFACE;
 
-    CMatchCB(IWSCAsyncFor &_parent, const Url &_url, const char *_tail, ColumnProvider * _meta) : parent(_parent), url(_url), tail(_tail), meta(_meta)
+    CMatchCB(IWSCAsyncFor &_parent, const Url &_url, const char *_tail, ColumnProvider * _meta, const char *_excPath, unsigned _excFlags) : parent(_parent), url(_url), tail(_tail), meta(_meta), excFlags(_excFlags), excPath(_excPath)
     {
     }
 
+    bool checkGetExceptionEntry(bool check, Owned<IColumnProviderIterator> &excIter, IColumnProvider &parent, IColumnProvider *&excEntry, const char *path)
+    {
+        if (!check)
+            return false;
+        excIter.setown(parent.getChildIterator(path));
+        excEntry = excIter->first();
+        return excEntry != nullptr;
+    }
+
+    IColumnProvider *getExceptionEntry(Owned<IColumnProviderIterator> &excIter, IColumnProvider &parent)
+    {
+        IColumnProvider *excEntry = nullptr;
+        if (excPath.length())
+        {
+            checkGetExceptionEntry(true, excIter, parent, excEntry, excPath.str()); //set by user so don't try others
+            return excEntry;
+        }
+        if (checkGetExceptionEntry((excFlags & CBExceptionPathExc)!=0, excIter, parent, excEntry, "Exception"))
+            return excEntry;
+        if (checkGetExceptionEntry((excFlags & CBExceptionPathExcsExc)!=0, excIter, parent, excEntry, "Exceptions/Exception")) //ESP xml array
+            return excEntry;
+        checkGetExceptionEntry((excFlags & CBExceptionPathExcs)!=0, excIter, parent, excEntry, "Exceptions"); //json array
+        return excEntry;
+    }
+
     virtual void match(IColumnProvider &entry, offset_t startOffset, offset_t endOffset)
     {
         Owned<IException> e;
-        if (tail.length())
+        if (tail.length()||excPath.length())
         {
             StringBuffer path(parent.getResponsePath());
             unsigned idx = (unsigned)entry.getInt(path.append("/@sequence").str());
-            Owned<IColumnProviderIterator> excIter = entry.getChildIterator("Exception");
-            IColumnProvider *excptEntry = excIter->first();
+            Owned<IColumnProviderIterator> excIter;
+            IColumnProvider *excptEntry = getExceptionEntry(excIter, entry);
             if (excptEntry)
             {
                 int code = (int)excptEntry->getInt("Code");
@@ -739,10 +770,11 @@ class CWSCHelperThread : public Thread
 {
 private:
     CWSCHelper * master;
-    virtual void outputXmlRows(CommonXmlWriter &xmlWriter, ConstPointerArray &inputRows, const char *itemtag=NULL, bool encode_off=false, char const * itemns = NULL);
-    virtual void createESPQuery(CommonXmlWriter &xmlWriter, ConstPointerArray &inputRows);
-    virtual void createSOAPliteralOrEncodedQuery(CommonXmlWriter &xmlWriter, ConstPointerArray &inputRows);
-    virtual void createXmlSoapQuery(CommonXmlWriter &xmlWriter, ConstPointerArray &inputRows);
+    virtual void outputRows(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows, const char *itemtag=NULL, bool encode_off=false, char const * itemns = NULL);
+    virtual void createESPQuery(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows);
+    virtual void createSOAPliteralOrEncodedQuery(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows);
+    virtual void createXmlSoapQuery(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows);
+    virtual void createHttpPostQuery(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows, bool appendRequestToName, bool appendEncodeFlag);
     virtual void processQuery(ConstPointerArray &inputRows);
     
     //Thread
@@ -767,6 +799,7 @@ private:
     SpinLock outputQLock;
     CriticalSection toXmlCrit, transformCrit, onfailCrit, timeoutCrit;
     unsigned done;
+    Owned<IPropertyTree> xpathHints;
     Linked<ClientCertificate> clientCert;
 
     static CriticalSection secureContextCrit;
@@ -795,6 +828,7 @@ public:
         helper = rowProvider->queryActionHelper();
         callHelper = rowProvider->queryCallHelper();
         flags = helper->getFlags();
+
         OwnedRoxieString s;
 
         authToken.append(_authToken);
@@ -825,6 +859,11 @@ public:
 
         if (flags & SOAPFhttpheaders)
             httpHeaders.set(s.setown(helper->getHttpHeaders()));
+        if (flags & SOAPFxpathhints)
+        {
+            s.setown(helper->getXpathHintsXml());
+            xpathHints.setown(createPTreeFromXMLString(s.get()));
+        }
 
         StringAttr proxyAddress;
         proxyAddress.set(s.setown(helper->getProxyAddress()));
@@ -852,8 +891,13 @@ public:
             if ((flags & SOAPFliteral) && (flags & SOAPFencoding))
                 throw MakeStringException(0, "SOAPCALL 'LITERAL' and 'ENCODING' options are mutually exclusive");
 
-            header.set(s.setown(helper->getHeader()));
-            footer.set(s.setown(helper->getFooter()));
+            rowHeader.set(s.setown(helper->getHeader()));
+            rowFooter.set(s.setown(helper->getFooter()));
+            if (flags & SOAPFmarkupinfo)
+            {
+                rootHeader.set(s.setown(helper->getRequestHeader()));
+                rootFooter.set(s.setown(helper->getRequestFooter()));
+            }
             if(flags & SOAPFnamespace)
             {
                 OwnedRoxieString ns = helper->getNamespaceName();
@@ -1080,7 +1124,7 @@ protected:
         else
             error.setown(e);
     }
-    void toXML(const byte * self, IXmlWriter & out) { CriticalBlock block(toXmlCrit); helper->toXML(self, out); }
+    void toXML(const byte * self, IXmlWriterExt & out) { CriticalBlock block(toXmlCrit); helper->toXML(self, out); }
     size32_t transformRow(ARowBuilder & rowBuilder, IColumnProvider * row) 
     { 
         CriticalBlock block(transformCrit); 
@@ -1118,8 +1162,10 @@ protected:
     StringAttr inputpath;
     StringBuffer service;
     StringBuffer acceptType;//for httpcall, text/plain, text/html, text/xml, etc
-    StringAttr header;
-    StringAttr footer;
+    StringAttr rowHeader;
+    StringAttr rowFooter;
+    StringAttr rootHeader;
+    StringAttr rootFooter;
     StringAttr xmlnamespace;
     IXmlToRowTransformer * rowTransformer;
 };
@@ -1129,64 +1175,65 @@ Owned<ISecureSocketContext> CWSCHelper::secureContext; // created on first use
 
 //=================================================================================================
 
-void CWSCHelperThread::outputXmlRows(CommonXmlWriter &xmlWriter, ConstPointerArray &inputRows, const char *itemtag, bool encode_off, char const * itemns)
+void CWSCHelperThread::outputRows(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows, const char *itemtag, bool encode_off, char const * itemns)
 {
     ForEachItemIn(idx, inputRows)
     {
-        if (itemtag)                //TAG
+        if (idx!=0)
+            xmlWriter.checkDelimiter();
+
+        if (itemtag && *itemtag)                //TAG
         {
-            xmlWriter.outputQuoted("<");
-            xmlWriter.outputQuoted(itemtag);
+            xmlWriter.outputBeginNested(itemtag, true);
             if(itemns)
-            {
-                xmlWriter.outputQuoted(" xmlns=\"");
-                xmlWriter.outputQuoted(itemns);
-                xmlWriter.outputQuoted("\"");
-            }
-            xmlWriter.outputQuoted(">");
+                xmlWriter.outputXmlns("xmlns", itemns);
         }
 
-        if (master->header.get())   //OPTIONAL HEADER (specified by "HEADING" option)
-            xmlWriter.outputQuoted(master->header.get());
+        if (master->rowHeader.get())   //OPTIONAL HEADER (specified by "HEADING" option)
+            xmlWriter.outputInline(master->rowHeader.get());
 
                                     //XML ROW CONTENT
         master->toXML((const byte *)inputRows.item(idx), xmlWriter);
 
-        if (master->footer.get())   //OPTION FOOTER
-            xmlWriter.outputQuoted(master->footer.get());
+        if (master->rowFooter.get())   //OPTION FOOTER
+            xmlWriter.outputInline(master->rowFooter.get());
 
         if (encode_off)             //ENCODING
-            xmlWriter.outputQuoted("<encode_>0</encode_>");
+            xmlWriter.outputInt(0, 1, "encode_");
 
-        if (itemtag)                //CLOSE TAG
-        {
-            xmlWriter.outputQuoted("</");
-            xmlWriter.outputQuoted(itemtag);
-            xmlWriter.outputQuoted(">");
-        }
+        if (itemtag && *itemtag)                //TAG
+            xmlWriter.outputEndNested(itemtag);
 
         master->addUserLogMsg((const byte *)inputRows.item(idx));
     }
 }
 
-void CWSCHelperThread::createESPQuery(CommonXmlWriter &xmlWriter, ConstPointerArray &inputRows)
+void CWSCHelperThread::createHttpPostQuery(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows, bool appendRequestToName, bool appendEncodeFlag)
 {
     StringBuffer method_tag;
-    method_tag.append(master->service).append("Request");
+    method_tag.append(master->service);
+    if (method_tag.length() && appendRequestToName)
+        method_tag.append("Request");
+
+    StringBuffer array_tag;
     StringAttr method_ns;
 
+    if (master->rootHeader.get())   //OPTIONAL ROOT REQUEST HEADER
+        xmlWriter.outputInline(master->rootHeader.get());
+
     if (inputRows.ordinality() > 1)
     {
-        xmlWriter.outputQuoted("<");
-        xmlWriter.outputQuoted(method_tag.str());
-        xmlWriter.outputQuoted("Array");
-        if (master->xmlnamespace.get())
+        if (!(master->flags & SOAPFnoroot))
         {
-            xmlWriter.outputQuoted(" xmlns=\"");
-            xmlWriter.outputQuoted(master->xmlnamespace.get());
-            xmlWriter.outputQuoted("\"");
+            if (method_tag.length())
+            {
+                array_tag.append(method_tag).append("Array");
+                xmlWriter.outputBeginNested(array_tag, true);
+                if (master->xmlnamespace.get())
+                    xmlWriter.outputXmlns("xmlns", master->xmlnamespace);
+            }
         }
-        xmlWriter.outputQuoted(">");
+        xmlWriter.outputBeginArray(method_tag);
     }
     else
     {
@@ -1194,65 +1241,68 @@ void CWSCHelperThread::createESPQuery(CommonXmlWriter &xmlWriter, ConstPointerAr
             method_ns.set(master->xmlnamespace.get());
     }
 
-    outputXmlRows(xmlWriter, inputRows, method_tag.str(), (inputRows.ordinality() == 1), method_ns.get());
+    outputRows(xmlWriter, inputRows, method_tag.str(), appendEncodeFlag ? (inputRows.ordinality() == 1) : false, method_ns.get());
 
     if (inputRows.ordinality() > 1)
     {
-        xmlWriter.outputQuoted("<encode_>0</encode_>");
-        xmlWriter.outputQuoted("</");
-        xmlWriter.outputQuoted(method_tag.str());
-        xmlWriter.outputQuoted("Array>");
+        xmlWriter.outputEndArray(method_tag);
+        if (appendEncodeFlag)
+            xmlWriter.outputInt(0, 1, "encode_");
+        if (!(master->flags & SOAPFnoroot))
+        {
+            if (method_tag.length())
+                xmlWriter.outputEndNested(array_tag);
+        }
     }
+
+    if (master->rootFooter.get())   //OPTIONAL ROOT REQUEST FOOTER
+        xmlWriter.outputInline(master->rootFooter.get());
+}
+
+void CWSCHelperThread::createESPQuery(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows)
+{
+    createHttpPostQuery(xmlWriter, inputRows, true, true);
 }
 
 //Create servce xml request body, with binding usage of either Literal or Encoded
 //Note that Encoded usage requires type encoding for data fields
-void CWSCHelperThread::createSOAPliteralOrEncodedQuery(CommonXmlWriter &xmlWriter, ConstPointerArray &inputRows)
+void CWSCHelperThread::createSOAPliteralOrEncodedQuery(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows)
 {
-    xmlWriter.outputQuoted("<");
-    xmlWriter.outputQuoted(master->service);
+    xmlWriter.outputBeginNested(master->service, true);
 
     if (master->flags & SOAPFencoding)
-        xmlWriter.outputQuoted(" soapenv:encodingStyle=\"http://schemas.xmlsoap.org/soap/encoding/\"");
+        xmlWriter.outputCString("http://schemas.xmlsoap.org/soap/encoding/", "@soapenv:encodingStyle");
 
     if (master->xmlnamespace.get())
-    {
-        xmlWriter.outputQuoted(" xmlns=\"");
-        xmlWriter.outputQuoted(master->xmlnamespace.get());
-        xmlWriter.outputQuoted("\"");
-    }
-
-    xmlWriter.outputQuoted(">");
+        xmlWriter.outputXmlns("xmlns", master->xmlnamespace.get());
 
-    outputXmlRows(xmlWriter, inputRows);
+    outputRows(xmlWriter, inputRows);
 
-    xmlWriter.outputQuoted("</");
-    xmlWriter.outputQuoted(master->service);
-    xmlWriter.outputQuoted(">");
+    xmlWriter.outputEndNested(master->service);
 }
 
 //Create SOAP body of http request
-void CWSCHelperThread::createXmlSoapQuery(CommonXmlWriter &xmlWriter, ConstPointerArray &inputRows)
+void CWSCHelperThread::createXmlSoapQuery(IXmlWriterExt &xmlWriter, ConstPointerArray &inputRows)
 {
     xmlWriter.outputQuoted("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
-    xmlWriter.outputQuoted("<soap:Envelope");
+    xmlWriter.outputBeginNested("soap:Envelope", true);
 
-    xmlWriter.outputQuoted(" xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\"");
+    xmlWriter.outputXmlns("soap", "http://schemas.xmlsoap.org/soap/envelope/");
     if (master->flags & SOAPFencoding)
     {   //SOAP RPC/encoded.  'Encoded' usage includes type encoding 
-        xmlWriter.outputQuoted(" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\"");
-        xmlWriter.outputQuoted(" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"");
+        xmlWriter.outputXmlns("xsd", "http://www.w3.org/2001/XMLSchema");
+        xmlWriter.outputXmlns("xsi", "http://www.w3.org/2001/XMLSchema-instance");
     }
-    xmlWriter.outputQuoted(">");
 
-    xmlWriter.outputQuoted("<soap:Body>");
+    xmlWriter.outputBeginNested("soap:Body", true);
 
     if (master->flags & SOAPFliteral  ||  master->flags & SOAPFencoding)
         createSOAPliteralOrEncodedQuery(xmlWriter, inputRows);
     else
         createESPQuery(xmlWriter, inputRows);
 
-    xmlWriter.outputQuoted("</soap:Body></soap:Envelope>");
+    xmlWriter.outputEndNested("soap:Body");
+    xmlWriter.outputEndNested("soap:Envelope");
 }
 
 void CWSCHelperThread::processQuery(ConstPointerArray &inputRows)
@@ -1263,14 +1313,24 @@ void CWSCHelperThread::processQuery(ConstPointerArray &inputRows)
         xmlWriteFlags |= XWFtrim;
     if ((master->flags & SOAPFpreserveSpace) == 0)
         xmlReadFlags |= ptr_ignoreWhiteSpace;
-        XMLWriterType xmlType = !(master->flags & SOAPFencoding) ? WTStandard : WTEncodingData64; 
-    CommonXmlWriter *xmlWriter = CreateCommonXmlWriter(xmlWriteFlags, 0, NULL, xmlType);
-    if (master->wscType == STsoap)
+
+    bool useMarkup = (master->flags & SOAPFmarkupinfo);
+
+    XMLWriterType xmlType = WTStandard;
+    if (useMarkup && (master->flags & SOAPFjson))
+        xmlType = (master->flags & SOAPFnoroot) ? WTJSONRootless : WTJSONObject;
+    else if (master->flags & SOAPFencoding)
+        xmlType = WTEncodingData64;
+
+    Owned<IXmlWriterExt> xmlWriter = createIXmlWriterExt(xmlWriteFlags, 0, nullptr, xmlType);
+    if (useMarkup)
+        createHttpPostQuery(*xmlWriter, inputRows, false, false);
+    else if (master->wscType == STsoap )
         createXmlSoapQuery(*xmlWriter, inputRows);
+    xmlWriter->finalize();
 
     Owned<IWSCAsyncFor> casyncfor = createWSCAsyncFor(master, *xmlWriter, inputRows, (PTreeReaderOptions) xmlReadFlags);
     casyncfor->For(master->numUrls, master->numUrlThreads,false,true); // shuffle URLS for poormans load balance
-    delete xmlWriter;
 }
 
 int CWSCHelperThread::run()
@@ -1435,7 +1495,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
 private:
     CWSCHelper * master;
     ConstPointerArray &inputRows;
-    CommonXmlWriter &xmlWriter;
+    IXmlWriterExt &xmlWriter;
     IEngineRowAllocator * outputAllocator;
     CriticalSection processExceptionCrit;
     StringBuffer responsePath;
@@ -1602,7 +1662,13 @@ private:
                 request.append(hdr.append("\r\n"));
             }
             if (!httpHeaderBlockContainsHeader(httpheaders, "Content-Type"))
-                request.append("Content-Type: text/xml\r\n");
+            {
+                bool isJson = ((master->flags & SOAPFmarkupinfo) && (master->flags & SOAPFjson));
+                if (isJson)
+                    request.append("Content-Type: application/json\r\n");
+                else
+                    request.append("Content-Type: text/xml\r\n");
+            }
         }
         else if(master->wscType == SThttp)
             request.append("Accept: ").append(master->acceptType).append("\r\n");
@@ -1806,60 +1872,76 @@ private:
         return rval;
     }
 
+    inline const char *queryXpathHint(const char *name)
+    {
+        if (!master->xpathHints)
+            return nullptr;
+        return master->xpathHints->queryProp(name);
+    }
+
     void processEspResponse(Url &url, StringBuffer &response, ColumnProvider * meta)
     {
         StringBuffer path(responsePath);
         path.append("/Results/Result/");
-        const char *tail;
+        const char *tail = nullptr;
+        const char *excPath = nullptr;
         if (master->rowTransformer && master->inputpath.get())
         {
             StringBuffer ipath;
             ipath.append("/Envelope/Body/").append(master->inputpath.get());
-            if((ipath.length() >= path.length()) && (0 == memcmp(ipath.str(), path.str(), path.length())))
-            {
+            tail = queryXpathHint("rowpath");
+            if(!tail && (ipath.length() >= path.length()) && (0 == memcmp(ipath.str(), path.str(), path.length())))
                 tail = ipath.str() + path.length();
-            }
             else
-            {
                 path.clear().append(ipath);
-                tail = NULL;
-            }
+            excPath = queryXpathHint("excpath");
         }
         else
             tail = "Dataset/Row";
 
-        CMatchCB matchCB(*this, url, tail, meta);
+        CMatchCB matchCB(*this, url, tail, meta, excPath, CBExceptionPathExc);
         Owned<IXMLParse> xmlParser = createXMLParse((const void *)response.str(), (unsigned)response.length(), path.str(), matchCB, options, (master->flags&SOAPFusescontents)!=0);
         while (xmlParser->next());
     }
-
     void processLiteralResponse(Url &url, StringBuffer &response, ColumnProvider * meta)
     {
         StringBuffer path("/Envelope/Body/");
+        const char *tail = nullptr;
+        const char *excPath = nullptr;
         if(master->rowTransformer && master->inputpath.get())
+        {
             path.append(master->inputpath.get());
-        CMatchCB matchCB(*this, url, NULL, meta);
+            tail = queryXpathHint("rowpath");
+            excPath = queryXpathHint("excpath");
+        }
+        CMatchCB matchCB(*this, url, tail, meta, excPath, CBExceptionPathExc);
         Owned<IXMLParse> xmlParser = createXMLParse((const void *)response.str(), (unsigned)response.length(), path.str(), matchCB, options, (master->flags&SOAPFusescontents)!=0);
         while (xmlParser->next());
     }
 
     void processHttpResponse(Url &url, StringBuffer &response, ColumnProvider * meta)
     {
-        StringBuffer path;
+        const char *path = nullptr;
+        const char *tail = nullptr;
+        const char *excPath = nullptr;
         if(master->rowTransformer && master->inputpath.get())
-            path.append(master->inputpath.get());
-        CMatchCB matchCB(*this, url, NULL, meta);
+        {
+            path = master->inputpath.get();
+            tail = queryXpathHint("rowpath");
+            excPath = queryXpathHint("excpath");
+        }
+        CMatchCB matchCB(*this, url, tail, meta, excPath, CBExceptionPathExc | CBExceptionPathExcs | CBExceptionPathExcsExc);
         Owned<IXMLParse> xmlParser;
-        if (strieq(master->acceptType.str(), "application/json"))
-            xmlParser.setown(createJSONParse((const void *)response.str(), (unsigned)response.length(), path.str(), matchCB, options, (master->flags&SOAPFusescontents)!=0, true));
+        if (strieq(master->acceptType.str(), "application/json") || (master->flags & SOAPFjson))
+            xmlParser.setown(createJSONParse((const void *)response.str(), (unsigned)response.length(), path, matchCB, options, (master->flags&SOAPFusescontents)!=0, true));
         else
-            xmlParser.setown(createXMLParse((const void *)response.str(), (unsigned)response.length(), path.str(), matchCB, options, (master->flags&SOAPFusescontents)!=0));
+            xmlParser.setown(createXMLParse((const void *)response.str(), (unsigned)response.length(), path, matchCB, options, (master->flags&SOAPFusescontents)!=0));
         while (xmlParser->next());
     }
 
     void processResponse(Url &url, StringBuffer &response, ColumnProvider * meta)
     {
-        if (master->wscType == SThttp)
+        if (master->wscType == SThttp || master->flags & SOAPFmarkupinfo)
             processHttpResponse(url, response, meta);
         else if (master->flags & SOAPFliteral)
             processLiteralResponse(url, response, meta);
@@ -1928,7 +2010,7 @@ private:
     }
 
 public:
-    CWSCAsyncFor(CWSCHelper * _master, CommonXmlWriter &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options): xmlWriter(_xmlWriter), inputRows(_inputRows), options(_options)
+    CWSCAsyncFor(CWSCHelper * _master, IXmlWriterExt &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options): xmlWriter(_xmlWriter), inputRows(_inputRows), options(_options)
     {
         master = _master;
         outputAllocator = master->queryOutputAllocator();
@@ -2152,7 +2234,7 @@ public:
     inline virtual IEngineRowAllocator * getOutputAllocator() { return outputAllocator; }
 };
 
-IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, CommonXmlWriter &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options)
+IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, IXmlWriterExt &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options)
 {
     return new CWSCAsyncFor(_master, _xmlWriter, _inputRows, _options);
 }

ecl/hql/hqlgram.hpp

@@ -803,6 +803,9 @@ protected:
     IHqlExpression * createBuildIndexFromIndex(attribute & indexAttr, attribute & flagsAttr, attribute & errpos);
     void checkOutputRecord(attribute & errpos, bool outerLevel);
     void checkSoapRecord(attribute & errpos);
+    IHqlExpression * processHttpMarkupFlag(__int64 op);
+    IHqlExpression * processHttpMarkupFlag(__int64 op, IHqlExpression *flags);
+    IHqlExpression * processHttpMarkupFlag(__int64 op, IHqlExpression *flags, IHqlExpression *p1);
     IHqlExpression * checkOutputRecord(IHqlExpression *record, const attribute & errpos, bool & allConstant, bool outerLevel);
     void checkDefaultValueVirtualAttr(const attribute &errpos, IHqlExpression * attrs);
 

ecl/hql/hqlgram.y

@@ -2689,58 +2689,66 @@ actionStmt
                             parser->endList(actions);
                             $$.setExpr(createValue(no_orderedactionlist, makeVoidType(), actions), $1);
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ')'
                         {
                             parser->normalizeExpression($3, type_stringorunicode, false);
                             parser->normalizeExpression($5, type_stringorunicode, false);
                             parser->checkSoapRecord($7);
-                            $$.setExpr(createValue(no_soapcall, makeVoidType(), $3.getExpr(), $5.getExpr(), $7.getExpr()), $1);
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            $$.setExpr(createValue(no_soapcall, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), flags.getClear() }, true), $1);
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' soapFlags ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' soapFlags ')'
                         {
                             parser->normalizeExpression($3, type_stringorunicode, false);
                             parser->normalizeExpression($5, type_stringorunicode, false);
                             parser->checkSoapRecord($7);
-                            $$.setExpr(createValue(no_soapcall, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr() }, true), $1);
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $9.getExpr());
+                            $$.setExpr(createValue(no_soapcall, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), flags.getClear() }, true), $1);
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' transform ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' transform ')'
                         {
                             parser->normalizeExpression($3, type_stringorunicode, false);
                             parser->normalizeExpression($5, type_stringorunicode, false);
-                            $$.setExpr(createValue(no_newsoapcall, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr() }, true), $1);
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            $$.setExpr(createValue(no_newsoapcall, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), flags.getClear() }, true), $1);
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' transform ',' soapFlags ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' transform ',' soapFlags ')'
                         {
                             parser->normalizeExpression($3, type_stringorunicode, false);
                             parser->normalizeExpression($5, type_stringorunicode, false);
-                            $$.setExpr(createValue(no_newsoapcall, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr() }, true), $1);
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $11.getExpr());
+                            $$.setExpr(createValue(no_newsoapcall, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), flags.getClear() }, true), $1);
                         }
-    | SOAPCALL '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ')' endTopLeftFilter endSelectorSequence
+    | HTTPorSOAPcall '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ')' endTopLeftFilter endSelectorSequence
                         {
                             parser->normalizeExpression($5, type_stringorunicode, false);
                             parser->normalizeExpression($7, type_stringorunicode, false);
                             parser->checkSoapRecord($9);
-                            $$.setExpr(createValue(no_soapaction_ds, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $12.getExpr() }, true), $1);
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            $$.setExpr(createValue(no_soapaction_ds, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), flags.getClear(), $12.getExpr() }, true), $1);
                         }
-    | SOAPCALL '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' soapFlags ')' endTopLeftFilter endSelectorSequence
+    | HTTPorSOAPcall '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' soapFlags ')' endTopLeftFilter endSelectorSequence
                         {
                             parser->normalizeExpression($5, type_stringorunicode, false);
                             parser->normalizeExpression($7, type_stringorunicode, false);
                             parser->checkSoapRecord($9);
-                            $$.setExpr(createValue(no_soapaction_ds, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr(), $14.getExpr() }, true), $1);
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $11.getExpr());
+                            $$.setExpr(createValue(no_soapaction_ds, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), flags.getClear(), $14.getExpr() }, true), $1);
                         }
-    | SOAPCALL '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' transform ')' endTopLeftFilter endSelectorSequence
+    | HTTPorSOAPcall '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' transform ')' endTopLeftFilter endSelectorSequence
                         {
                             parser->normalizeExpression($5, type_stringorunicode, false);
                             parser->normalizeExpression($7, type_stringorunicode, false);
-                            $$.setExpr(createValue(no_newsoapaction_ds, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr(), $14.getExpr() }, true));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            $$.setExpr(createValue(no_newsoapaction_ds, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr(), flags.getClear(), $14.getExpr() }, true));
                             $$.setPosition($1);
                         }
-    | SOAPCALL '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' transform ',' soapFlags ')' endTopLeftFilter endSelectorSequence
+    | HTTPorSOAPcall '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' transform ',' soapFlags ')' endTopLeftFilter endSelectorSequence
                         {
                             parser->normalizeExpression($5, type_stringorunicode, false);
                             parser->normalizeExpression($7, type_stringorunicode, false);
-                            $$.setExpr(createValue(no_newsoapaction_ds, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr(), $13.getExpr(), $16.getExpr() }, true));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $13.getExpr());
+                            $$.setExpr(createValue(no_newsoapaction_ds, makeVoidType(), { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr(), flags.getClear(), $16.getExpr() }, true));
                             $$.setPosition($1);
                         }
     | KEYDIFF '(' dataSet ',' dataSet ',' expression keyDiffFlags ')'
@@ -3586,6 +3594,30 @@ outputFlag
                         }
     ;
 
+HTTPorSOAPcall
+    : HTTPCALL          {   $$.setInt(no_httpcall); $$.setPosition($1); }
+    | SOAPCALL          {   $$.setInt(no_soapcall); $$.setPosition($1); }
+    ;
+
+httpMarkupOptions
+    : httpMarkupOption
+    | httpMarkupOptions ',' httpMarkupOption
+                        {   $$.setExpr(createComma($1.getExpr(), $3.getExpr())); }
+    ;
+
+httpMarkupOption
+    : NOROOT            {   $$.setExpr(createAttribute(noRootAtom)); }
+    | HEADING '(' expression optCommaExpression ')'
+                        {
+                            //Markup heading is the root level heading, SOAPCALL/HTTPCALL heading is once per row for batched requests
+                            parser->normalizeExpression($3, type_string, false);
+                            if ($4.queryExpr())
+                                parser->normalizeExpression($4, type_string, false);
+                            $$.setExpr(createExprAttribute(headingAtom, $3.getExpr(), $4.getExpr()));
+                            $$.setPosition($1);
+                        }
+    ;
+
 soapFlags
     : soapFlag
     | soapFlags ',' soapFlag
@@ -3615,6 +3647,17 @@ soapFlag
                             $$.setExpr(createExprAttribute(xpathAtom, $3.getExpr()));
                             $$.setPosition($1);
                         }
+    | XPATH '(' expression ',' hintList ')'
+                        {
+                            //MORE: Really type_utf8 - and in lots of other places!
+                            parser->normalizeExpression($3, type_string, false);
+                            parser->validateXPath($3);
+                            HqlExprArray args;
+                            args.append(*$3.getExpr());
+                            $5.unwindCommaList(args);
+                            $$.setExpr(createExprAttribute(xpathAtom, args));
+                            $$.setPosition($1);
+                        }
     | GROUP             {
                             $$.setExpr(createAttribute(groupAtom));
                             $$.setPosition($1);
@@ -3716,6 +3759,26 @@ soapFlag
                             parser->normalizeExpression($3, type_string, false);
                             $$.setExpr(createExprAttribute(logAtom, $3.getExpr()), $1);
                         }
+    | XML_TOKEN         {
+                            $$.setExpr(createAttribute(xmlAtom));
+                            $$.setPosition($1);
+                        }
+    | XML_TOKEN '(' httpMarkupOptions ')'
+                        {
+                            HqlExprArray args;
+                            $3.unwindCommaList(args);
+                            $$.setExpr(createExprAttribute(xmlAtom, args), $1);
+                        }
+    | JSON_TOKEN        {
+                            $$.setExpr(createAttribute(jsonAtom));
+                            $$.setPosition($1);
+                        }
+    | JSON_TOKEN '(' httpMarkupOptions ')'
+                        {
+                            HqlExprArray args;
+                            $3.unwindCommaList(args);
+                            $$.setExpr(createExprAttribute(jsonAtom, args), $1);
+                        }
     ;
 
 onFailAction
@@ -7599,50 +7662,58 @@ simpleDataRow
                             OwnedHqlExpr ds = parser->processIfProduction($3, $5, NULL);
                             $$.setExpr(createRow(no_selectnth, ds.getClear(), getSizetConstant(1)), $1);
                         }
-    | HTTPCALL '(' expression ',' expression ',' expression ',' recordDef ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' expression ',' recordDef ')'
                         {
+                            if ($1.getInt() != (__int64) no_httpcall)
+                                parser->reportError(ERR_EXPECTED, $1, "Expected HTTPCALL rather than SOAPCALL");
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($5);
                             parser->normalizeExpression($7);
-                            IHqlExpression * ds = createDataset(no_httpcall, $3.getExpr(), createComma($5.getExpr(), $7.getExpr(), $9.getExpr()));
+                            IHqlExpression * ds = createDataset(no_httpcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr() });
                             $$.setExpr(createRow(no_selectnth, ds, createConstantOne()));
                         }
-    | HTTPCALL '(' expression ',' expression ',' expression ',' recordDef ',' soapFlags ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' expression ',' recordDef ',' soapFlags ')'
                         {
+                            if ($1.getInt() != (__int64) no_httpcall)
+                                parser->reportError(ERR_EXPECTED, $1, "Expected HTTPCALL rather than SOAPCALL");
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($5);
                             parser->normalizeExpression($7);
                             IHqlExpression * ds = createDataset(no_httpcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr() });
                             $$.setExpr(createRow(no_selectnth, ds, createConstantOne()));
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' recordDef ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' recordDef ')'
                         {
                             parser->normalizeExpression($3);
                             parser->checkSoapRecord($7);
-                            IHqlExpression * ds = createDataset(no_soapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr() });
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            IHqlExpression * ds = createDataset(no_soapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), flags.getClear() });
                             $$.setExpr(createRow(no_selectnth, ds, createConstantOne()));
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' recordDef ',' soapFlags ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' recordDef ',' soapFlags ')'
                         {
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($5);
                             parser->checkSoapRecord($7);
-                            IHqlExpression * ds = createDataset(no_soapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr() });
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $11.getExpr());
+                            IHqlExpression * ds = createDataset(no_soapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), flags.getClear() });
                             $$.setExpr(createRow(no_selectnth, ds, createConstantOne()));
                             parser->checkOnFailRecord($$.queryExpr(), $1);
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' transform ',' recordDef ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' transform ',' recordDef ')'
                         {
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($5);
-                            IHqlExpression * ds = createDataset(no_newsoapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr() });
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            IHqlExpression * ds = createDataset(no_newsoapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), flags.getClear(), $11.getExpr() });
                             $$.setExpr(createRow(no_selectnth, ds, createConstantOne()));
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' transform ',' recordDef ',' soapFlags ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' transform ',' recordDef ',' soapFlags ')'
                         {
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($5);
-                            IHqlExpression * ds = createDataset(no_newsoapcall, $3.getExpr(), createComma($5.getExpr(), $7.getExpr(), createComma($9.getExpr(), $11.getExpr(), $13.getExpr())));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $13.getExpr());
+                            IHqlExpression * ds = createDataset(no_newsoapcall, $3.getExpr(), createComma($5.getExpr(), $7.getExpr(), createComma($9.getExpr(), $11.getExpr(), flags.getClear())));
                             $$.setExpr(createRow(no_selectnth, ds, createConstantOne()));
                             parser->checkOnFailRecord($$.queryExpr(), $1);
                         }
@@ -9585,67 +9656,75 @@ simpleDataSet
                                 parser->reportError(ERR_PARSER_CANNOTRECOVER,$1,"SKIP is only valid inside a TRANSFORM");
                             $$.setExpr(createDataset(no_skip, $3.getExpr()));
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' DATASET '(' recordDef ')' ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' DATASET '(' recordDef ')' ')'
                         {
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($5);
                             parser->checkSoapRecord($7);
-                            $$.setExpr(createDataset(no_soapcall, $3.getExpr(), createComma($5.getExpr(), $7.getExpr(), $11.getExpr())));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            $$.setExpr(createDataset(no_soapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $11.getExpr(), flags.getClear() }));
                             $$.setPosition($1);
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' DATASET '(' recordDef ')' ',' soapFlags ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' DATASET '(' recordDef ')' ',' soapFlags ')'
                         {
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($5);
                             parser->checkSoapRecord($7);
-                            $$.setExpr(createDataset(no_soapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $11.getExpr(), $14.getExpr() }));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $14.getExpr());
+                            $$.setExpr(createDataset(no_soapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $11.getExpr(), flags.getClear() }));
                             parser->checkOnFailRecord($$.queryExpr(), $1);
                             $$.setPosition($1);
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' transform ',' DATASET '(' recordDef ')' ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' transform ',' DATASET '(' recordDef ')' ')'
                         {
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($5);
-                            $$.setExpr(createDataset(no_newsoapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $13.getExpr() }));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            $$.setExpr(createDataset(no_newsoapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), flags.getClear(), $13.getExpr() }));
                             $$.setPosition($1);
                         }
-    | SOAPCALL '(' expression ',' expression ',' recordDef ',' transform ',' DATASET '(' recordDef ')' ',' soapFlags ')'
+    | HTTPorSOAPcall '(' expression ',' expression ',' recordDef ',' transform ',' DATASET '(' recordDef ')' ',' soapFlags ')'
                         {
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($5);
-                            $$.setExpr(createDataset(no_newsoapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $13.getExpr(), $16.getExpr() }));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $16.getExpr());
+                            $$.setExpr(createDataset(no_newsoapcall, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $13.getExpr(), flags.getClear() }));
                             parser->checkOnFailRecord($$.queryExpr(), $1);
                             $$.setPosition($1);
                         }
-    | SOAPCALL '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' DATASET '(' recordDef ')' ')' endTopLeftFilter endSelectorSequence
+    | HTTPorSOAPcall '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' DATASET '(' recordDef ')' ')' endTopLeftFilter endSelectorSequence
                         {
                             parser->normalizeExpression($5);
                             parser->normalizeExpression($7);
                             parser->checkSoapRecord($9);
-                            $$.setExpr(createDataset(no_soapcall_ds, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $13.getExpr(), $17.getExpr() }));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            $$.setExpr(createDataset(no_soapcall_ds, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $13.getExpr(), flags.getClear(), $17.getExpr() }));
                             $$.setPosition($1);
                         }
-    | SOAPCALL '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' DATASET '(' recordDef ')' ',' soapFlags ')' endTopLeftFilter endSelectorSequence
+    | HTTPorSOAPcall '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' DATASET '(' recordDef ')' ',' soapFlags ')' endTopLeftFilter endSelectorSequence
                         {
                             parser->normalizeExpression($5);
                             parser->normalizeExpression($7);
                             parser->checkSoapRecord($9);
-                            $$.setExpr(createDataset(no_soapcall_ds, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $13.getExpr(), $16.getExpr(), $19.getExpr() }));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $16.getExpr());
+                            $$.setExpr(createDataset(no_soapcall_ds, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $13.getExpr(), flags.getClear(), $19.getExpr() }));
                             parser->checkOnFailRecord($$.queryExpr(), $1);
                             $$.setPosition($1);
                         }
-    | SOAPCALL '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' transform ',' DATASET '(' recordDef ')' ')' endTopLeftFilter endSelectorSequence
+    | HTTPorSOAPcall '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' transform ',' DATASET '(' recordDef ')' ')' endTopLeftFilter endSelectorSequence
                         {
                             parser->normalizeExpression($5);
                             parser->normalizeExpression($7);
-                            $$.setExpr(createDataset(no_newsoapcall_ds, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr(), $15.getExpr(), $19.getExpr() }));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt());
+                            $$.setExpr(createDataset(no_newsoapcall_ds, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr(), $15.getExpr(), flags.getClear(), $19.getExpr() }));
                             $$.setPosition($1);
                         }
-    | SOAPCALL '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' transform ',' DATASET '(' recordDef ')' ',' soapFlags ')' endTopLeftFilter endSelectorSequence
+    | HTTPorSOAPcall '(' startTopLeftSeqFilter ',' expression ',' expression ',' recordDef ',' transform ',' DATASET '(' recordDef ')' ',' soapFlags ')' endTopLeftFilter endSelectorSequence
                         {
                             parser->normalizeExpression($5);
                             parser->normalizeExpression($7);
-                            $$.setExpr(createDataset(no_newsoapcall_ds, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr(), $15.getExpr(), $18.getExpr(), $21.getExpr() }));
+                            OwnedHqlExpr flags = parser->processHttpMarkupFlag($1.getInt(), $18.getExpr());
+                            $$.setExpr(createDataset(no_newsoapcall_ds, { $3.getExpr(), $5.getExpr(), $7.getExpr(), $9.getExpr(), $11.getExpr(), $15.getExpr(), flags.getClear(), $21.getExpr() }));
                             parser->checkOnFailRecord($$.queryExpr(), $1);
                             $$.setPosition($1);
                         }

ecl/hql/hqlgram2.cpp

@@ -7598,6 +7598,31 @@ void HqlGram::checkSoapRecord(attribute & errpos)
     OwnedHqlExpr mapped = checkOutputRecord(record, errpos, allConstant, true);
 }
 
+inline bool hasHttpMarkupFlag(IHqlExpression *flags)
+{
+    if (!flags)
+        return false;
+    return (queryAttributeInList(jsonAtom, flags)!=nullptr || queryAttributeInList(xmlAtom, flags)!=nullptr);
+}
+
+IHqlExpression * HqlGram::processHttpMarkupFlag(__int64 op)
+{
+    if (op == (__int64) no_httpcall)
+        return createAttribute(jsonAtom);
+    return nullptr;
+}
+
+IHqlExpression * HqlGram::processHttpMarkupFlag(__int64 op, IHqlExpression *flags)
+{
+    if (op != (__int64) no_httpcall || hasHttpMarkupFlag(flags))
+        return LINK(flags);
+    return createComma(createAttribute(jsonAtom), flags);
+}
+
+IHqlExpression * HqlGram::processHttpMarkupFlag(__int64 op, IHqlExpression *flags, IHqlExpression *p1)
+{
+    return createComma(processHttpMarkupFlag(op, flags), p1);
+}
 
 IHqlExpression * HqlGram::checkIndexRecord(IHqlExpression * record, const attribute & errpos, OwnedHqlExpr & indexAttrs)
 {

ecl/hqlcpp/hqlhtcpp.cpp

@@ -17834,39 +17834,26 @@ void HqlCppTranslator::doBuildHttpHeaderStringFunction(BuildCtx &ctx, IHqlExpres
 ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpression * expr, bool isSink, bool isRoot)
 {
     ThorActivityKind tak;
-    const char * helper;
     unsigned firstArg = 0;
     IHqlExpression * dataset = NULL;
     Owned<ABoundActivity> boundDataset;
     IHqlExpression * selSeq = querySelSeq(expr);
-    if (expr->getOperator() == no_newsoapcall)
+
+    const char * helper= (isSink) ? "SoapAction" : "SoapCall";
+
+    switch (expr->getOperator())
     {
-        if (isSink)
-        {
-            tak = TAKsoap_rowaction;
-            helper = "SoapAction";
-        }
-        else
-        {
-            tak = TAKsoap_rowdataset;
-            helper = "SoapCall";
-        }
-    }
-    else
+    case no_newsoapcall:
+        tak = (isSink) ? TAKsoap_rowaction : TAKsoap_rowdataset;
+        break;
+    default:
     {
-        if (isSink)
-        {
-            tak = TAKsoap_datasetaction;
-            helper = "SoapAction";
-        }
-        else
-        {
-            tak = TAKsoap_datasetdataset;
-            helper = "SoapCall";
-        }
+        tak = (isSink) ? TAKsoap_datasetaction : TAKsoap_datasetdataset;
         dataset = expr->queryChild(0);
         boundDataset.setown(buildCachedActivity(ctx, dataset));
         firstArg = 1;
+        break;
+    }
     }
 
     StringBuffer s;
@@ -17899,6 +17886,26 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
     if (separator)
         doBuildVarStringFunction(instance->startctx, "queryOutputIteratorPath", separator->queryChild(0));
 
+    bool isJSON = false;
+    IHqlExpression * markupAttr = expr->queryAttribute(xmlAtom);
+    if (!markupAttr)
+    {
+        markupAttr = expr->queryAttribute(jsonAtom);
+        if (markupAttr)
+            isJSON = true;
+    }
+
+    if (markupAttr)
+    {
+        //request header and footer are for the entire request, row header and footer is for each row
+        IHqlExpression * reqHeader = markupAttr->queryAttribute(headingAtom);
+        if (reqHeader)
+        {
+            doBuildVarStringFunction(instance->startctx, "getRequestHeader", reqHeader->queryChild(0));
+            doBuildVarStringFunction(instance->startctx, "getRequestFooter", reqHeader->queryChild(1));
+        }
+    }
+
     //virtual const char * getHeader()
     //virtual const char * getFooter()
     IHqlExpression * header = expr->queryAttribute(headingAtom);
@@ -17972,16 +17979,36 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
         doBuildFunctionReturn(func.ctx, unknownStringType, logText);
     }
     bool usesContents = false;
+    bool hasXpathHints = false;
     if (!isSink)
     {
         //virtual IXmlToRowTransformer * queryTransformer()
         doBuildXmlReadMember(*instance, expr, "queryInputTransformer", usesContents);
 
         //virtual const char * getInputIteratorPath()
+        StringBuffer xpathHints;
         IHqlExpression * xpath = expr->queryAttribute(xpathAtom);
         if (xpath)
+        {
             doBuildVarStringFunction(instance->startctx, "getInputIteratorPath", xpath->queryChild(0));
 
+            if (xpath->numChildren()>1)
+            {
+                hasXpathHints = true;
+                appendXMLOpenTag(xpathHints, "Hints");
+                ForEachChildFrom(i, xpath, 1)
+                {
+                    StringBuffer name, value;
+                    OwnedHqlExpr folded = foldHqlExpression(xpath->queryChild(i));
+                    getHintNameValue(folded, name, value);
+                    appendXMLTag(xpathHints, name.str(), value.str());
+                }
+                appendXMLCloseTag(xpathHints, "Hints");
+                OwnedHqlExpr xpathHintsExpr = createConstant(createStringValue(xpathHints.str(), xpathHints.length()));
+                doBuildVarStringFunction(instance->startctx, "getXpathHintsXml", xpathHintsExpr);
+            }
+        }
+
         IHqlExpression * onFail = expr->queryAttribute(onFailAtom);
         if (onFail)
         {
@@ -18023,7 +18050,19 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
             flags.append("|SOAPFhttpheaders");
         if (usesContents)
             flags.append("|SOAPFusescontents");
-
+        if (markupAttr)
+            flags.append("|SOAPFmarkupinfo");
+        if (hasXpathHints)
+            flags.append("|SOAPFxpathhints");
+        if (markupAttr)
+        {
+            if (markupAttr->hasAttribute(noRootAtom))
+                flags.append("|SOAPFnoroot");
+            if (isJSON)
+                flags.append("|SOAPFjson");
+            else
+                flags.append("|SOAPFxml");
+        }
         if (flags.length())
             doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
     }

plugins/javaembed/javaembed.cpp

@@ -2483,6 +2483,7 @@ public:
     virtual void finalize() override
     {
     }
+    virtual void checkDelimiter() override                        {}
 
     virtual IInterface *saveLocation() const {return nullptr;}
     virtual void rewindTo(IInterface *loc)

rtl/eclrtl/eclhelper_base.cpp

@@ -614,6 +614,9 @@ IXmlToRowTransformer * CThorSoapActionArg::queryInputTransformer() { return NULL
 const char * CThorSoapActionArg::getInputIteratorPath() { return NULL; }
 size32_t CThorSoapActionArg::onFailTransform(ARowBuilder & rowBuilder, const void * left, IException * e) { return 0; }
 void CThorSoapActionArg::getLogText(size32_t & lenText, char * & text, const void * left) { lenText =0; text = NULL; }
+const char * CThorSoapActionArg::getXpathHintsXml() { return nullptr;}
+const char * CThorSoapActionArg::getRequestHeader() { return nullptr; }
+const char * CThorSoapActionArg::getRequestFooter() { return nullptr; }
 
 //CThorSoapCallArg
 
@@ -636,6 +639,10 @@ const char * CThorSoapCallArg::getProxyAddress() { return NULL; }
 const char * CThorSoapCallArg::getAcceptType() { return NULL; }
 IXmlToRowTransformer * CThorSoapCallArg::queryInputTransformer() { return NULL; }
 const char * CThorSoapCallArg::getInputIteratorPath() { return NULL; }
+const char * CThorSoapCallArg::getXpathHintsXml() { return nullptr; }
+const char * CThorSoapCallArg::getRequestHeader() { return nullptr; }
+const char * CThorSoapCallArg::getRequestFooter() { return nullptr; }
+
 size32_t CThorSoapCallArg::onFailTransform(ARowBuilder & rowBuilder, const void * left, IException * e) { return 0; }
 void CThorSoapCallArg::getLogText(size32_t & lenText, char * & text, const void * left) { lenText =0; text = NULL; }
 

rtl/eclrtl/rtlformat.hpp

@@ -19,6 +19,7 @@ interface IXmlWriterExt : extends IXmlWriter
     virtual void outputNumericString(const char *field, const char *fieldname) = 0;
     virtual void outputInline(const char* text) = 0;
     virtual void finalize() = 0;
+    virtual void checkDelimiter() = 0;
 };
 
 class ECLRTL_API SimpleOutputWriter : implements IXmlWriterExt, public CInterface
@@ -33,6 +34,7 @@ public:
     virtual size32_t length() const override                { return out.length(); }
     virtual const char * str() const override               { return out.str(); }
     virtual void finalize() override                        {}
+    virtual void checkDelimiter() override                  {}
 
 
     virtual void outputQuoted(const char *text) override;
@@ -131,6 +133,7 @@ public:
     virtual unsigned length() const                                 { return out.length(); }
     virtual const char * str() const                                { return out.str(); }
     virtual void finalize() override                                {}
+    virtual void checkDelimiter() override                          {}
 
     virtual IInterface *saveLocation() const
     {
@@ -224,6 +227,7 @@ public:
     virtual unsigned length() const                                 { return out.length(); }
     virtual const char * str() const                                { return out.str(); }
     virtual void finalize() override                                {}
+    virtual void checkDelimiter() override                          { checkDelimit(); }
     virtual void rewindTo(unsigned int prevlen)                     { if (prevlen < out.length()) out.setLength(prevlen); }
     virtual IInterface *saveLocation() const
     {
@@ -549,6 +553,8 @@ public:
     virtual unsigned length() const { return out.length(); }
     virtual const char* str() const { return out.str(); }
     virtual void finalize() override {}
+    virtual void checkDelimiter() override {}
+
     virtual void rewindTo(IInterface* location) { };
     virtual void cutFrom(IInterface *location, StringBuffer& databuf) { };
     virtual IInterface* saveLocation() const

rtl/include/eclhelper.hpp

@@ -44,7 +44,7 @@ typedef unsigned short UChar;
 
 //Should be incremented whenever the virtuals in the context or a helper are changed, so
 //that a work unit can't be rerun.  Try as hard as possible to retain compatibility.
-#define ACTIVITY_INTERFACE_VERSION      651
+#define ACTIVITY_INTERFACE_VERSION      652
 #define MIN_ACTIVITY_INTERFACE_VERSION  650             //minimum value that is compatible with current interface
 
 typedef unsigned char byte;
@@ -2198,18 +2198,27 @@ struct IHThorPipeThroughArg : public IHThorArg
 
 enum
 {
-    SOAPFgroup          = 0x0001,
-    SOAPFonfail         = 0x0002,
-    SOAPFlog            = 0x0004,
-    SOAPFtrim           = 0x0008,
-    SOAPFliteral        = 0x0010,
-    SOAPFnamespace      = 0x0020,
-    SOAPFencoding       = 0x0040,
-    SOAPFpreserveSpace  = 0x0080,
-    SOAPFlogmin         = 0x0100,
-    SOAPFlogusermsg     = 0x0200,
-    SOAPFhttpheaders    = 0x0400,
-    SOAPFusescontents   = 0x0800
+    SOAPFgroup          = 0x000001,
+    SOAPFonfail         = 0x000002,
+    SOAPFlog            = 0x000004,
+    SOAPFtrim           = 0x000008,
+    SOAPFliteral        = 0x000010,
+    SOAPFnamespace      = 0x000020,
+    SOAPFencoding       = 0x000040,
+    SOAPFpreserveSpace  = 0x000080,
+    SOAPFlogmin         = 0x000100,
+    SOAPFlogusermsg     = 0x000200,
+    SOAPFhttpheaders    = 0x000400,
+    SOAPFusescontents   = 0x000800,
+    SOAPFmarkupinfo     = 0x001000,
+    SOAPFxpathhints     = 0x002000,
+    SOAPFnoroot         = 0x004000,
+    SOAPFjson           = 0x008000,
+    SOAPFxml            = 0x010000
+};
+
+enum
+{
 };
 
 struct IHThorWebServiceCallActionArg : public IHThorArg
@@ -2240,6 +2249,9 @@ struct IHThorWebServiceCallActionArg : public IHThorArg
     virtual const char * getInputIteratorPath() = 0;
     virtual size32_t onFailTransform(ARowBuilder & rowBuilder, const void * left, IException * e) = 0;
     virtual void getLogText(size32_t & lenText, char * & text, const void * left) = 0;  // iff SOAPFlogusermsg set
+    virtual const char * getXpathHintsXml() = 0; //iff SOAPFxpathhints
+    virtual const char * getRequestHeader() = 0;
+    virtual const char * getRequestFooter() = 0;
 };
 typedef IHThorWebServiceCallActionArg IHThorSoapActionArg ;
 typedef IHThorWebServiceCallActionArg IHThorHttpActionArg ;

rtl/include/eclhelper_base.hpp

@@ -817,6 +817,9 @@ class ECLRTL_API CThorSoapActionArg : public CThorSinkArgOf<IHThorSoapActionArg>
     virtual const char * getInputIteratorPath() override;
     virtual size32_t onFailTransform(ARowBuilder & rowBuilder, const void * left, IException * e) override;
     virtual void getLogText(size32_t & lenText, char * & text, const void * left) override;
+    virtual const char * getXpathHintsXml() override;
+    virtual const char * getRequestHeader() override;
+    virtual const char * getRequestFooter() override;
 };
 
 class ECLRTL_API CThorSoapCallArg : public CThorArgOf<IHThorSoapCallArg>
@@ -843,6 +846,9 @@ class ECLRTL_API CThorSoapCallArg : public CThorArgOf<IHThorSoapCallArg>
     virtual const char * getInputIteratorPath() override;
     virtual size32_t onFailTransform(ARowBuilder & rowBuilder, const void * left, IException * e) override;
     virtual void getLogText(size32_t & lenText, char * & text, const void * left) override;
+    virtual const char * getXpathHintsXml() override;
+    virtual const char * getRequestHeader() override;
+    virtual const char * getRequestFooter() override;
 };
 
 typedef CThorSoapCallArg CThorHttpCallArg;

testing/regress/ecl/httpcall_jsonpost.ecl

@@ -0,0 +1,99 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//version targetIP='127.0.0.1',goodPort='9876',blacListedPort='9875'
+
+#option('generateGlobalId', true);
+
+string targetIP := '.' : stored('targetIP');
+string goodPort := '9876' : stored('goodPort');
+string blacListedPort := '9875' : stored('blacListedPort');
+
+string targetURL := 'http://' + targetIP + ':' + goodPort;
+string doubleTargetURL := 'http://'+ targetIP +':' + goodPort + '|http://'+ targetIP +':' + goodPort;
+string blacklistedTargetURL := 'http://'+ targetIP +':' + goodPort + '|http://' + targetIP + ':' + blacListedPort;
+
+d := dataset([{'FRED'},{'WILMA'}], {string unkname});
+
+ServiceOutRecord :=
+    RECORD
+        string name;
+        data pic;
+        unsigned4 id{xpath('r')};
+        unsigned4 novalue;
+    END;
+
+// simple query->dataset form
+output(SORT(HTTPCALL(targetURL,'soapbase', { string unkname := 'FRED' }, dataset(ServiceOutRecord), JSON, XPATH('*/Results', ROWPATH('result_1/Row'), ExcPath('Exception')), LOG('simple'), HTTPHEADER('HPCC-Global-Id','12345678900'), HTTPHEADER('HPCC-Caller-Id','1111')),record));
+
+// noroot /no service name -- recreate root
+output(SORT(HTTPCALL(targetURL,'', { string unkname := 'FRED' }, dataset(ServiceOutRecord), JSON(NOROOT, HEADING('{"soapbase":{', '}}')), XPATH('*/Results', ROWPATH('result_1/Row'), ExcPath('Exception')), LOG('simple'), HTTPHEADER('HPCC-Global-Id','12345678900'), HTTPHEADER('HPCC-Caller-Id','1111')),record));
+
+// double query->dataset form
+output(SORT(HTTPCALL(doubleTargetURL,'soapbase', { string unkname := 'FRED' }, dataset(ServiceOutRecord), JSON, XPATH('soapbaseResponse/Results', ROWPATH('*/Row'), ExcPath('Exception')), LOG, HTTPHEADER('HPCC-Global-Id','12345678900'), HTTPHEADER('HPCC-Caller-Id','2222')),record));
+
+// simple dataset->dataset form
+output(sort(HTTPCALL(d, targetURL,'soapbase', { unkname }, DATASET(ServiceOutRecord), JSON, XPATH('*', ROWPATH('Results/result_1/Row'), ExcPath('Results/Exception')), HTTPHEADER('HPCC-Global-Id','12345678900'), HTTPHEADER('HPCC-Caller-Id','3333')),record));
+
+// double query->dataset form
+ServiceOutRecord doError(d l) := TRANSFORM
+  //SELF.name := 'ERROR: \'' + l.unkname + '\'-\'' + failmessage[1..18] + '\''; //if (l.unkname='FRED' AND failmessage[1..18]='blacklisted socket','blacklisted socket', failmessage);
+  SELF.name := 'ERROR: ' + if (l.unkname='FRED' AND (failmessage[1..18]='blacklisted socket' OR failmessage[1..18]='connection failed '),'blacklisted socket', failmessage[1..18]);
+  SELF.pic := x'01020304';
+  SELF.id := if (l.unkname='FRED' AND failcode=-3,-1,failcode);
+  SELF.novalue := 0;
+END;
+
+ServiceOutRecord doError2 := TRANSFORM
+  SELF.name := 'ERROR: ' + failmessage;
+  SELF.pic := x'01020304';
+  SELF.id := failcode;
+  SELF.novalue := 0;
+END;
+
+ServiceOutRecord doError3(d l) := TRANSFORM
+  SELF.name := 'ERROR: ' + failmessage;
+  SELF.pic := x'01020304';
+  SELF.id := if (l.unkname='FRED' AND failcode=-3,-1,failcode);
+  SELF.novalue := 0;
+END;
+
+// Test some failure cases
+
+output(SORT(HTTPCALL(d, blacklistedTargetURL,'soapbase', { unkname }, DATASET(ServiceOutRecord), JSON, XPATH('*', ROWPATH('Results/result_1/Row'), ExcPath('Results/Exception')), onFail(doError(LEFT)),RETRY(0), log('SOAP: ' + unkname),TIMEOUT(1)), record));
+
+output(SORT(HTTPCALL(targetURL,'soapbase', { string unkname := 'FAIL' }, dataset(ServiceOutRecord), JSON, XPATH('soapbaseResponse/Results', ROWPATH('result_1/Row'), ExcPath('Exception')),onFail(doError2),RETRY(0), LOG(MIN), HTTPHEADER('Global-Id','12345678900'), HTTPHEADER('Caller-Id','4444')),record));
+
+output(SORT(HTTPCALL(d, targetURL,'soapbaseNOSUCHQUERY', { unkname }, DATASET(ServiceOutRecord), JSON, XPATH('*/Results', ROWPATH('result_1/Row'), ExcPath('Exception')), onFail(doError3(LEFT)),PARALLEL(1),RETRY(0), LOG(MIN)), record));
+
+childRecord := record
+unsigned            id;
+    end;
+
+FullServiceOutRecord :=
+    RECORD
+        string name;
+        data pic;
+        unsigned4 id{xpath('r')};
+        dataset(childRecord) ids{maxcount(5)};
+        unsigned4 novalue;
+    END;
+
+//leak children when linked counted rows are enabled, because not all records are read
+//Use a count so the results are consistent, and nofold to prevent the code generator removing the child dataset...
+output(count(nofold(choosen(HTTPCALL(d, targetURL,'soapbase', { string unkname := d.unkname+'1' }, dataset(FullServiceOutRecord), JSON, XPATH('soapbaseResponse/Results', ROWPATH('result_1/Row'), ExcPath('Exception'))),1))));
+output(count(nofold(choosen(HTTPCALL(d, targetURL,'soapbase', { string unkname := d.unkname+'1' }, dataset(FullServiceOutRecord), JSON, XPATH('soapbaseResponse/Results', ROWPATH('result_1/Row'), ExcPath('Exception'))),2))));

testing/regress/ecl/httpcall_xmlpost.ecl

@@ -0,0 +1,97 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//version targetIP='127.0.0.1',goodPort='9876',blacListedPort='9875'
+
+#option('generateGlobalId', true);
+
+string targetIP := '.' : stored('targetIP');
+string goodPort := '9876' : stored('goodPort');
+string blacListedPort := '9875' : stored('blacListedPort');
+
+string targetURL := 'http://' + targetIP + ':' + goodPort;
+string doubleTargetURL := 'http://'+ targetIP +':' + goodPort + '|http://'+ targetIP +':' + goodPort;
+string blacklistedTargetURL := 'http://'+ targetIP +':' + goodPort + '|http://' + targetIP + ':' + blacListedPort;
+
+d := dataset([{'FRED'},{'WILMA'}], {string unkname});
+
+ServiceOutRecord :=
+    RECORD
+        string name;
+        data pic;
+        unsigned4 id{xpath('r')};
+        unsigned4 novalue;
+    END;
+
+// simple query->dataset form
+output(SORT(HTTPCALL(targetURL,'soapbase', { string unkname := 'FRED' }, dataset(ServiceOutRecord), XML, XPATH('*/Results/Result', rowpath('Dataset/Row')), LOG('simple'), HTTPHEADER('HPCC-Global-Id','12345678900'), HTTPHEADER('HPCC-Caller-Id','1111')),record));
+
+// noroot / service name -- recreate root
+output(SORT(HTTPCALL(targetURL,'', { string unkname := 'FRED' }, dataset(ServiceOutRecord), XML(NOROOT, HEADING('<soapbase>', '</soapbase>')), XPATH('*/Results/Result', rowpath('Dataset/Row')), LOG('simple'), HTTPHEADER('HPCC-Global-Id','12345678900'), HTTPHEADER('HPCC-Caller-Id','1111')),record));
+
+// double query->dataset form
+output(SORT(HTTPCALL(doubleTargetURL,'soapbase', { string unkname := 'FRED' }, dataset(ServiceOutRecord), XML, XPATH('soapbaseResponse/Results/Result', rowpath('Dataset[@name="Result 1"]/Row')), LOG, HTTPHEADER('HPCC-Global-Id','12345678900'), HTTPHEADER('HPCC-Caller-Id','2222')),record));
+
+// simple dataset->dataset form
+output(sort(HTTPCALL(d, targetURL,'soapbase', { unkname }, DATASET(ServiceOutRecord), XML, XPATH('soapbaseResponse/Results/Result', rowpath('Dataset/Row')), HTTPHEADER('HPCC-Global-Id','12345678900'), HTTPHEADER('HPCC-Caller-Id','3333')),record));
+
+// double query->dataset form
+ServiceOutRecord doError(d l) := TRANSFORM
+  //SELF.name := 'ERROR: \'' + l.unkname + '\'-\'' + failmessage[1..18] + '\''; //if (l.unkname='FRED' AND failmessage[1..18]='blacklisted socket','blacklisted socket', failmessage);
+  SELF.name := 'ERROR: ' + if (l.unkname='FRED' AND (failmessage[1..18]='blacklisted socket' OR failmessage[1..18]='connection failed '),'blacklisted socket', failmessage[1..18]);
+  SELF.pic := x'01020304';
+  SELF.id := if (l.unkname='FRED' AND failcode=-3,-1,failcode);
+  SELF.novalue := 0;
+END;
+
+ServiceOutRecord doError2 := TRANSFORM
+  SELF.name := 'ERROR: ' + failmessage;
+  SELF.pic := x'01020304';
+  SELF.id := failcode;
+  SELF.novalue := 0;
+END;
+
+ServiceOutRecord doError3(d l) := TRANSFORM
+  SELF.name := 'ERROR: ' + failmessage;
+  SELF.pic := x'01020304';
+  SELF.id := if (l.unkname='FRED' AND failcode=-3,-1,failcode);
+  SELF.novalue := 0;
+END;
+
+// Test some failure cases
+
+output(SORT(HTTPCALL(d, blacklistedTargetURL,'soapbase', { unkname }, DATASET(ServiceOutRecord), XML, XPATH('soapbaseResponse/Results/Result', rowpath('Dataset/Row')), onFail(doError(LEFT)),RETRY(0), log('SOAP: ' + unkname),TIMEOUT(1)), record));
+output(SORT(HTTPCALL(targetURL,'soapbase', { string unkname := 'FAIL' }, dataset(ServiceOutRecord), XML, XPATH('soapbaseResponse/Results/Result', rowpath('Dataset/Row')),onFail(doError2),RETRY(0), LOG(MIN), HTTPHEADER('Global-Id','12345678900'), HTTPHEADER('Caller-Id','4444')),record));
+output(SORT(HTTPCALL(d, targetURL,'soapbaseNOSUCHQUERY', { unkname }, DATASET(ServiceOutRecord), XML, XPATH('soapbaseNOSUCHQUERYResponse/Results/Result', rowpath('Dataset/Row')), onFail(doError3(LEFT)),PARALLEL(1),RETRY(0), LOG(MIN)), record));
+
+childRecord := record
+unsigned            id;
+    end;
+
+FullServiceOutRecord :=
+    RECORD
+        string name;
+        data pic;
+        unsigned4 id{xpath('r')};
+        dataset(childRecord) ids{maxcount(5)};
+        unsigned4 novalue;
+    END;
+
+//leak children when linked counted rows are enabled, because not all records are read
+//Use a count so the results are consistent, and nofold to prevent the code generator removing the child dataset...
+output(count(nofold(choosen(HTTPCALL(d, targetURL,'soapbase', { string unkname := d.unkname+'1' }, dataset(FullServiceOutRecord), XML, XPATH('soapbaseResponse/Results/Result', rowpath('Dataset/Row'))),1))));
+output(count(nofold(choosen(HTTPCALL(d, targetURL,'soapbase', { string unkname := d.unkname+'1' }, dataset(FullServiceOutRecord), XML, XPATH('soapbaseResponse/Results/Result', rowpath('Dataset/Row'))),2))));

testing/regress/ecl/key/httpcall_jsonpost.xml

@@ -0,0 +1,49 @@
+<Dataset name='Result 1'>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>WILMA</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><name>ERROR: blacklisted socket</name><pic>01020304</pic><r>4294967295</r><novalue>0</novalue></Row>
+ <Row><name>ERROR: blacklisted socket</name><pic>01020304</pic><r>4294967295</r><novalue>0</novalue></Row>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>WILMA</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><name>ERROR: ReceivedRoxieException: (You asked me to fail)</name><pic>01020304</pic><r>1</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><name>ERROR: ReceivedRoxieException: (Unknown query soapbaseNOSUCHQUERY)</name><pic>01020304</pic><r>1403</r><novalue>0</novalue></Row>
+ <Row><name>ERROR: ReceivedRoxieException: (Unknown query soapbaseNOSUCHQUERY)</name><pic>01020304</pic><r>1403</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>1</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>2</Result_9></Row>
+</Dataset>

testing/regress/ecl/key/httpcall_xmlpost.xml

@@ -0,0 +1,49 @@
+<Dataset name='Result 1'>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>WILMA</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><name>ERROR: blacklisted socket</name><pic>01020304</pic><r>4294967295</r><novalue>0</novalue></Row>
+ <Row><name>ERROR: blacklisted socket</name><pic>01020304</pic><r>4294967295</r><novalue>0</novalue></Row>
+ <Row><name>FRED</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>LORRAINE</name><pic>0102030405</pic><r>2</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>RICHARD</name><pic>0102</pic><r>1</r><novalue>0</novalue></Row>
+ <Row><name>WILMA</name><pic>5432</pic><r>3</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><name>ERROR: ReceivedRoxieException: (You asked me to fail)</name><pic>01020304</pic><r>1</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><name>ERROR: ReceivedRoxieException: (Unknown query soapbaseNOSUCHQUERY)</name><pic>01020304</pic><r>1403</r><novalue>0</novalue></Row>
+ <Row><name>ERROR: ReceivedRoxieException: (Unknown query soapbaseNOSUCHQUERY)</name><pic>01020304</pic><r>1403</r><novalue>0</novalue></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>1</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>2</Result_9></Row>
+</Dataset>

testing/regress/ecl/soapcall.ecl

@@ -15,9 +15,6 @@
     limitations under the License.
 ############################################################################## */
 
-//nohthor
-//nothor
-
 //version targetIP='127.0.0.1',goodPort='9876',blacListedPort='9875'
 
 #option('generateGlobalId', true);