HPCC-13421 Add roxie support for POST FormUrlEncoded

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/thorhelper/roxiehelper.cpp

@@ -16,7 +16,6 @@
 ############################################################################## */
 
 #include "jexcept.hpp"
-#include "thorherror.h"
 #include "thorsort.hpp"
 #include "roxiehelper.hpp"
 #include "roxielmj.hpp"
@@ -1529,7 +1528,6 @@ extern ISortAlgorithm *createSortAlgorithm(RoxieSortAlgorithm _algorithm, ICompa
 CSafeSocket::CSafeSocket(ISocket *_sock)
 {
     httpMode = false;
-    mlFmt = MarkupFmt_Unknown;
     sent = 0; 
     heartbeat = false; 
     sock.setown(_sock);
@@ -1648,6 +1646,31 @@ int readHttpHeaderLine(IBufferedSocket *linereader, char *headerline, unsigned m
     return bytesread;
 }
 
+void parseHttpParameterString(IProperties *p, const char *str)
+{
+    while (*str)
+    {
+        StringBuffer s, prop, val;
+        while (*str && *str != '&' && *str != '=')
+            s.append(*str++);
+        appendDecodedURL(prop, s.trim());
+        if (!*str || *str == '&')
+            val.set("1");
+        else
+        {
+            s.clear();
+            str++;
+            while (*str && *str != '&')
+                s.append(*str++);
+            appendDecodedURL(val, s.trim());
+        }
+        if (prop.length())
+            p->setProp(prop, val);
+        if (*str)
+            str++;
+    }
+}
+
 bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHttpHelper, bool &continuationNeeded, bool &isStatus, unsigned maxBlockSize)
 {
     continuationNeeded = false;
@@ -1677,7 +1700,7 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
         if (pHttpHelper != NULL && strncmp((char *)&len, "POST", 4) == 0)
         {
 #define MAX_HTTP_HEADERSIZE 8000
-            pHttpHelper->setIsHttp(true);
+            pHttpHelper->setHttpMethod(HttpMethod::POST);
             char header[MAX_HTTP_HEADERSIZE + 1]; // allow room for \0
             sock->read(header, 1, MAX_HTTP_HEADERSIZE, bytesRead, timeout);
             header[bytesRead] = 0;
@@ -1706,7 +1729,12 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
                     buf = ret.reserveTruncate(len);
                     left = len - (bytesRead - (payload - header));
                     if (len > left)
-                        memcpy(buf, payload, len - left); 
+                        memcpy(buf, payload, len - left);
+                    if (pHttpHelper->isFormPost())
+                    {
+                        pHttpHelper->checkTarget();
+                        pHttpHelper->setFormContent(ret);
+                    }
                 }
                 else
                     left = len = 0;
@@ -1720,7 +1748,7 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
         else if (pHttpHelper != NULL && strncmp((char *)&len, "GET", 3) == 0)
         {
 #define MAX_HTTP_GET_LINE 16000 //arbitrary per line limit, most web servers are lower, but urls for queries can be complex..
-                pHttpHelper->setIsHttp(true);
+                pHttpHelper->setHttpMethod(HttpMethod::GET);
                 char headerline[MAX_HTTP_GET_LINE + 1];
                 Owned<IBufferedSocket> linereader = createBufferedSocket(sock);
 
@@ -1736,22 +1764,10 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
                     bytesread = readHttpHeaderLine(linereader, headerline, MAX_HTTP_GET_LINE);
                 }
 
-                StringBuffer queryName;
-                const char *target = pHttpHelper->queryTarget();
-                if (!target || !*target)
-                    throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Target not specified");
-                else if (!pHttpHelper->validateTarget(target))
-                    throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Target not found");
+                pHttpHelper->checkTarget();
                 const char *query = pHttpHelper->queryQueryName();
                 if (!query || !*query)
                     throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Query not specified");
-
-                queryName.append(query);
-                Owned<IPropertyTree> req = createPTreeFromHttpParameters(queryName, pHttpHelper->queryUrlParameters(), true, pHttpHelper->queryContentFormat()==MarkupFmt_JSON);
-                if (pHttpHelper->queryContentFormat()==MarkupFmt_JSON)
-                    toJSON(req, ret);
-                else
-                    toXML(req, ret);
                 return true;
         }
         else if (strnicmp((char *)&len, "STAT", 4) == 0)
@@ -1797,10 +1813,10 @@ void CSafeSocket::setHttpMode(const char *queryName, bool arrayMode, HttpHelper
 {
     CriticalBlock c(crit); // Should not be needed
     httpMode = true;
-    mlFmt = httphelper.queryContentFormat();
+    mlResponseFmt = httphelper.queryResponseMlFormat();
     heartbeat = false;
     assertex(contentHead.length()==0 && contentTail.length()==0);
-    if (mlFmt==MarkupFmt_JSON)
+    if (mlResponseFmt==MarkupFmt_JSON)
     {
         contentHead.set("{");
         contentTail.set("}");
@@ -1827,7 +1843,7 @@ void CSafeSocket::checkSendHttpException(HttpHelper &httphelper, IException *E,
 {
     if (!httphelper.isHttp())
         return;
-    if (httphelper.queryContentFormat()==MarkupFmt_JSON)
+    if (httphelper.queryResponseMlFormat()==MarkupFmt_JSON)
         sendJsonException(E, queryName);
     else
         sendSoapException(E, queryName);
@@ -1948,7 +1964,7 @@ void CSafeSocket::flush()
 
         StringBuffer header;
         header.append("HTTP/1.0 200 OK\r\n");
-        header.append("Content-Type: ").append(mlFmt == MarkupFmt_JSON ? "application/json" : "text/xml").append("\r\n");
+        header.append("Content-Type: ").append(mlResponseFmt == MarkupFmt_JSON ? "application/json" : "text/xml").append("\r\n");
         header.append("Content-Length: ").append(length).append("\r\n\r\n");
 
 
@@ -2819,26 +2835,5 @@ void HttpHelper::parseURL()
         pathNodes.appendList(path, "/");
     if (!finger)
         return;
-    finger++;
-    while (*finger)
-    {
-        StringBuffer s, prop, val;
-        while (*finger && *finger != '&' && *finger != '=')
-            s.append(*finger++);
-        appendDecodedURL(prop, s.trim());
-        if (!*finger || *finger == '&')
-            val.set("1");
-        else
-        {
-            s.clear();
-            finger++;
-            while (*finger && *finger != '&')
-                s.append(*finger++);
-            appendDecodedURL(val, s.trim());
-        }
-        if (prop.length())
-            parameters->setProp(prop, val);
-        if (*finger)
-            finger++;
-    }
+    parseHttpParameterString(parameters, ++finger);
 }

common/thorhelper/roxiehelper.hpp

@@ -18,6 +18,7 @@
 #ifndef ROXIEHELPER_HPP
 #define ROXIEHELPER_HPP
 
+#include "thorherror.h"
 #include "thorxmlwrite.hpp"
 #include "roxiehelper.ipp"
 #include "roxiemem.hpp"
@@ -27,10 +28,14 @@
 
 //========================================================================================= 
 
+void parseHttpParameterString(IProperties *p, const char *str);
+
+enum class HttpMethod {NONE, GET, POST};
+
 class THORHELPER_API HttpHelper : public CInterface
 {
 private:
-    bool _isHttp;
+    HttpMethod method;
     bool useEnvelope;
     StringAttr url;
     StringAttr authToken;
@@ -38,6 +43,7 @@ private:
     StringArray pathNodes;
     StringArray *validTargets;
     Owned<IProperties> parameters;
+    Owned<IProperties> form;
 private:
     inline void setHttpHeaderValue(StringAttr &s, const char *v, bool ignoreExt)
     {
@@ -53,15 +59,27 @@ private:
 
 public:
     IMPLEMENT_IINTERFACE;
-    HttpHelper(StringArray *_validTargets) : validTargets(_validTargets) { _isHttp = false; useEnvelope=true; parameters.setown(createProperties(true));}
-    bool isHttp() { return _isHttp; }
+    HttpHelper(StringArray *_validTargets) : validTargets(_validTargets), method(HttpMethod::NONE) {useEnvelope=true; parameters.setown(createProperties(true));}
+    inline bool isHttp() { return method!=HttpMethod::NONE; }
+    inline bool isHttpGet(){ return method==HttpMethod::GET; }
+    inline bool isControlUrl(){return (pathNodes.isItem(1) && strieq(pathNodes.item(1), "control"));}
+
     bool getUseEnvelope(){return useEnvelope;}
     void setUseEnvelope(bool _useEnvelope){useEnvelope=_useEnvelope;}
     bool getTrim() {return parameters->getPropBool(".trim", true); /*http currently defaults to true, maintain compatibility */}
-    void setIsHttp(bool __isHttp) { _isHttp = __isHttp; }
+    void setHttpMethod(HttpMethod _method) { method = _method; }
     const char *queryAuthToken() { return authToken.str(); }
     const char *queryTarget() { return (pathNodes.length()) ? pathNodes.item(0) : NULL; }
-    const char *queryQueryName() { return (pathNodes.length()>1) ? pathNodes.item(1) : NULL; }
+    const char *queryQueryName()
+    {
+        unsigned namePos = 1;
+        if (!pathNodes.isItem(namePos))
+            return nullptr;
+        if (isControlUrl())
+            if (!pathNodes.isItem(++namePos))
+                return nullptr;
+        return pathNodes.item(namePos);
+    }
 
     inline void setAuthToken(const char *v)
     {
@@ -81,20 +99,65 @@ public:
             parseURL();
         }
     }
-    TextMarkupFormat queryContentFormat()
+    inline bool isFormPost()
+    {
+        return (strnicmp(queryContentType(), "application/x-www-form-urlencoded", strlen("application/x-www-form-urlencoded"))==0);
+    }
+    TextMarkupFormat getUrlResponseFormat()
+    {
+        if (pathNodes.length()>2 && strieq(pathNodes.item(2), "json"))
+            return MarkupFmt_JSON;
+        return MarkupFmt_XML;
+    }
+    TextMarkupFormat getContentTypeMlFormat()
     {
         if (!contentType.length())
         {
-            if (pathNodes.length()>2 && strieq(pathNodes.item(2), "json"))
+            TextMarkupFormat fmt = getUrlResponseFormat();
+            if (fmt == MarkupFmt_JSON)
                 contentType.set("application/json");
             else
                 contentType.set("text/xml");
+            return fmt;
         }
 
         return (strieq(queryContentType(), "application/json")) ? MarkupFmt_JSON : MarkupFmt_XML;
     }
+    TextMarkupFormat queryResponseMlFormat()
+    {
+        if (isFormPost())
+            return getUrlResponseFormat();
+        return getContentTypeMlFormat();
+    }
+    TextMarkupFormat queryRequestMlFormat()
+    {
+        if (isHttpGet() || isFormPost())
+            return MarkupFmt_URL;
+        return getContentTypeMlFormat();
+    }
     IProperties *queryUrlParameters(){return parameters;}
     bool validateTarget(const char *target){return (validTargets) ? validTargets->contains(target) : false;}
+    inline void checkTarget()
+    {
+        const char *target = queryTarget();
+        if (!target || !*target)
+            throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Target not specified");
+        else if (!validateTarget(target))
+            throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Target not found");
+    }
+    inline void setFormContent(const char *content)
+    {
+        if (!form)
+            form.setown(createProperties(false));
+        parseHttpParameterString(form, content);
+    }
+    IPropertyTree *createPTreeFromParameters(byte flags)
+    {
+        const char *query = queryQueryName();
+        if (!query || !*query)
+            throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Query not specified");
+        return createPTreeFromHttpParameters(query, form ? form : parameters, true, false, (ipt_flags) flags);
+    }
 };
 
 //==============================================================================================================
@@ -183,7 +246,7 @@ protected:
     Linked<ISocket> sock;
     bool httpMode;
     bool heartbeat;
-    TextMarkupFormat mlFmt;
+    TextMarkupFormat mlResponseFmt = MarkupFmt_Unknown;
     StringAttr contentHead;
     StringAttr contentTail;
     PointerArray queued;

roxie/ccd/ccdprotocol.cpp

@@ -1104,7 +1104,7 @@ IHpccProtocolResponse *createProtocolResponse(const char *queryname, SafeSocket
 {
     if (protocolFlags & HPCC_PROTOCOL_NATIVE_RAW || protocolFlags & HPCC_PROTOCOL_NATIVE_ASCII)
         return new CHpccNativeProtocolResponse(queryname, client, MarkupFmt_Unknown, protocolFlags, false, logctx, xmlReadFlags);
-    else if (httpHelper.queryContentFormat()==MarkupFmt_JSON)
+    else if (httpHelper.queryResponseMlFormat()==MarkupFmt_JSON)
         return new CHpccJsonResponse(queryname, client, protocolFlags, httpHelper.isHttp(), logctx, xmlReadFlags);
     return new CHpccXmlResponse(queryname, client, protocolFlags, httpHelper.isHttp(), logctx, xmlReadFlags);
 
@@ -1350,7 +1350,7 @@ private:
             isRequestArray = false;
             if (httpHelper.isHttp())
             {
-                if (httpHelper.queryContentFormat()==MarkupFmt_JSON)
+                if (httpHelper.queryRequestMlFormat()==MarkupFmt_JSON)
                 {
                     if (strieq(queryName, "__object__"))
                     {
@@ -1418,12 +1418,14 @@ private:
         else
             throw MakeStringException(ROXIE_DATA_ERROR, "Malformed request");
     }
-    void parseQueryPTFromString(Owned<IPropertyTree> &queryPT, HttpHelper &httpHelper, const char *text, PTreeReaderOptions options)
+    void createQueryPTree(Owned<IPropertyTree> &queryPT, HttpHelper &httpHelper, const char *text, byte flags, PTreeReaderOptions options)
     {
-        if (strieq(httpHelper.queryContentType(), "application/json"))
-            queryPT.setown(createPTreeFromJSONString(text, ipt_caseInsensitive, options));
+        if (httpHelper.queryRequestMlFormat()==MarkupFmt_URL)
+            queryPT.setown(httpHelper.createPTreeFromParameters(flags));
+        else if (httpHelper.queryRequestMlFormat()==MarkupFmt_JSON)
+            queryPT.setown(createPTreeFromJSONString(text, flags, options));
         else
-            queryPT.setown(createPTreeFromXMLString(text, ipt_caseInsensitive, options));
+            queryPT.setown(createPTreeFromXMLString(text, flags, options));
     }
 
     void doMain(const char *runQuery)
@@ -1477,7 +1479,8 @@ readAnother:
         }
 
         bool isHTTP = httpHelper.isHttp();
-        TextMarkupFormat mlFmt = isHTTP ? httpHelper.queryContentFormat() : MarkupFmt_XML;
+        TextMarkupFormat mlResponseFmt = isHTTP ? httpHelper.queryResponseMlFormat() : MarkupFmt_XML;
+        TextMarkupFormat mlRequestFmt = isHTTP ? httpHelper.queryRequestMlFormat() : MarkupFmt_XML;
 
         bool failed = false;
         bool isRequest = false;
@@ -1497,15 +1500,20 @@ readAnother:
         bool stripWhitespace = msgctx->getStripWhitespace();
         try
         {
-            if (mlFmt==MarkupFmt_XML || mlFmt==MarkupFmt_JSON)
+            if (httpHelper.isHttpGet() || httpHelper.isFormPost())
             {
-                QueryNameExtractor extractor(mlFmt, stripWhitespace);
+                queryName.set(httpHelper.queryQueryName());
+                if (httpHelper.isControlUrl())
+                    queryPrefix.set("control");
+            }
+            else if (mlRequestFmt==MarkupFmt_XML || mlRequestFmt==MarkupFmt_JSON)
+            {
+                QueryNameExtractor extractor(mlRequestFmt, stripWhitespace);
                 extractor.extractName(rawText.str(), logctx, peerStr, ep.port);
                 queryName.set(extractor.name);
                 queryPrefix.set(extractor.prefix);
                 stripWhitespace = extractor.stripWhitespace;
             }
-
             if (streq(queryPrefix.str(), "control"))
             {
                 if (httpHelper.isHttp())
@@ -1514,10 +1522,7 @@ readAnother:
                 bool aclupdate = strieq(queryName, "aclupdate"); //ugly
                 byte iptFlags = aclupdate ? ipt_caseInsensitive : 0;
 
-                if (mlFmt==MarkupFmt_JSON)
-                    queryPT.setown(createPTreeFromJSONString(rawText.str(), iptFlags, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces)));
-                else
-                    queryPT.setown(createPTreeFromXMLString(rawText.str(), iptFlags, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces)));
+                createQueryPTree(queryPT, httpHelper, rawText, iptFlags, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces));
 
                 IPropertyTree *root = queryPT;
                 skipProtocolRoot(queryPT, httpHelper, queryName, isRequest, isRequestArray);
@@ -1543,7 +1548,7 @@ readAnother:
                 readFlags |= (stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
                 try
                 {
-                    parseQueryPTFromString(queryPT, httpHelper, rawText.str(), (PTreeReaderOptions)readFlags);
+                    createQueryPTree(queryPT, httpHelper, rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)readFlags);
                 }
                 catch (IException *E)
                 {
@@ -1619,7 +1624,6 @@ readAnother:
                         IArrayOf<IPropertyTree> requestArray;
                         if (isHTTP)
                         {
-                            mlFmt = httpHelper.queryContentFormat();
                             if (isRequestArray)
                             {
                                 StringBuffer reqIterString;
@@ -1658,7 +1662,7 @@ readAnother:
                                 if (stricmp(format, "raw") == 0)
                                 {
                                     protocolFlags |= HPCC_PROTOCOL_NATIVE_RAW;
-                                    mlFmt = MarkupFmt_Unknown;
+                                    mlResponseFmt = MarkupFmt_Unknown;
                                 }
                                 else if (stricmp(format, "bxml") == 0)
                                 {
@@ -1667,7 +1671,7 @@ readAnother:
                                 else if (stricmp(format, "ascii") == 0)
                                 {
                                     protocolFlags |= HPCC_PROTOCOL_NATIVE_ASCII;
-                                    mlFmt = MarkupFmt_Unknown;
+                                    mlResponseFmt = MarkupFmt_Unknown;
                                 }
                                 else if (stricmp(format, "xml") != 0) // xml is the default
                                     throw MakeStringException(ROXIE_INVALID_INPUT, "Unsupported format specified: %s", format);
@@ -1770,7 +1774,7 @@ readAnother:
                 {
                     if (msgctx->getIntercept())
                     {
-                        FlushingStringBuffer response(client, (protocolFlags & HPCC_PROTOCOL_BLOCKED), mlFmt, (protocolFlags & HPCC_PROTOCOL_NATIVE_RAW), false, logctx);
+                        FlushingStringBuffer response(client, (protocolFlags & HPCC_PROTOCOL_BLOCKED), mlResponseFmt, (protocolFlags & HPCC_PROTOCOL_NATIVE_RAW), false, logctx);
                         response.startDataset("Tracing", NULL, (unsigned) -1);
                         msgctx->outputLogXML(response);
                     }

system/jlib/jptree.hpp

@@ -28,7 +28,8 @@ enum TextMarkupFormat
 {
     MarkupFmt_Unknown=0,
     MarkupFmt_XML,
-    MarkupFmt_JSON
+    MarkupFmt_JSON,
+    MarkupFmt_URL
 };
 enum PTreeExceptionCodes
 {