浏览代码

HPCC-15284 Add roxie support for "Adaptive" rest

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
Anthony Fishbeck 9 年之前
父节点
当前提交
cd44ad43aa

+ 86 - 62
common/thorhelper/roxiehelper.cpp

@@ -1857,6 +1857,7 @@ void CSafeSocket::sendSoapException(IException *E, const char *queryName)
 {
     try
     {
+        adaptiveRoot = false;
         if (!queryName)
             queryName = "Unknown"; // Exceptions when parsing query XML can leave queryName unset/unknowable....
 
@@ -1888,6 +1889,7 @@ void CSafeSocket::sendJsonException(IException *E, const char *queryName)
 {
     try
     {
+        adaptiveRoot = false;
         if (!queryName)
             queryName = "Unknown"; // Exceptions when parsing query XML can leave queryName unset/unknowable....
 
@@ -1962,7 +1964,9 @@ void CSafeSocket::flush()
 {
     if (httpMode)
     {
-        unsigned length = contentHead.length() + contentTail.length();
+        unsigned length = 0;
+        if (!adaptiveRoot)
+            length = contentHead.length() + contentTail.length();
         ForEachItemIn(idx, lengths)
             length += lengths.item(idx);
 
@@ -1977,10 +1981,13 @@ void CSafeSocket::flush()
             DBGLOG("Writing HTTP header length %d to HTTP socket", header.length());
         sock->write(header.str(), header.length());
         sent += header.length();
-        if (traceLevel > 5)
-            DBGLOG("Writing content head length %d to HTTP socket", contentHead.length());
-        sock->write(contentHead.str(), contentHead.length());
-        sent += contentHead.length();
+        if (!adaptiveRoot || mlResponseFmt != MarkupFmt_JSON)
+        {
+            if (traceLevel > 5)
+                DBGLOG("Writing content head length %d to HTTP socket", contentHead.length());
+            sock->write(contentHead.str(), contentHead.length());
+            sent += contentHead.length();
+        }
         ForEachItemIn(idx2, queued)
         {
             unsigned length = lengths.item(idx2);
@@ -1989,10 +1996,13 @@ void CSafeSocket::flush()
             sock->write(queued.item(idx2), length);
             sent += length;
         }
-        if (traceLevel > 5)
-            DBGLOG("Writing content tail length %d to HTTP socket", contentTail.length());
-        sock->write(contentTail.str(), contentTail.length());
-        sent += contentTail.length();
+        if (!adaptiveRoot || mlResponseFmt != MarkupFmt_JSON)
+        {
+            if (traceLevel > 5)
+                DBGLOG("Writing content tail length %d to HTTP socket", contentTail.length());
+            sock->write(contentTail.str(), contentTail.length());
+            sent += contentTail.length();
+        }
         if (traceLevel > 5)
             DBGLOG("Total written %d", sent);
     }
@@ -2292,7 +2302,7 @@ void *FlushingStringBuffer::getPayload(size32_t &length)
     return length ? s.detach() : NULL;
 }
 
-void FlushingStringBuffer::startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend, const IProperties *xmlns)
+void FlushingStringBuffer::startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend, const IProperties *xmlns, bool adaptive)
 {
     CriticalBlock b(crit);
     extend = _extend;
@@ -2303,35 +2313,38 @@ void FlushingStringBuffer::startDataset(const char *elementName, const char *res
         startBlock();
         if (!isBlocked)
         {
-            if (mlFmt==MarkupFmt_XML)
+            if (mlFmt==MarkupFmt_XML && elementName)
             {
                 s.append('<').append(elementName);
-                if (isSoap && (resultName || (sequence != (unsigned) -1)))
+                if (!adaptive)
                 {
-                    s.append(" xmlns=\'urn:hpccsystems:ecl:").appendLower(queryName.length(), queryName.str()).append(":result:");
-                    if (resultName && *resultName)
-                        s.appendLower(strlen(resultName), resultName).append('\'');
-                    else
-                        s.append("result_").append(sequence+1).append('\'');
-                    if (xmlns)
+                    if (isSoap && (resultName || (sequence != (unsigned) -1)))
                     {
-                        Owned<IPropertyIterator> it = const_cast<IProperties*>(xmlns)->getIterator(); //should fix IProperties to be const friendly
-                        ForEach(*it)
+                        s.append(" xmlns=\'urn:hpccsystems:ecl:").appendLower(queryName.length(), queryName.str()).append(":result:");
+                        if (resultName && *resultName)
+                            s.appendLower(strlen(resultName), resultName).append('\'');
+                        else
+                            s.append("result_").append(sequence+1).append('\'');
+                        if (xmlns)
                         {
-                            const char *name = it->getPropKey();
-                            s.append(' ');
-                            if (!streq(name, "xmlns"))
-                                s.append("xmlns:");
-                            s.append(name).append("='");
-                            encodeUtf8XML(const_cast<IProperties*>(xmlns)->queryProp(name), s);
-                            s.append("'");
+                            Owned<IPropertyIterator> it = const_cast<IProperties*>(xmlns)->getIterator(); //should fix IProperties to be const friendly
+                            ForEach(*it)
+                            {
+                                const char *name = it->getPropKey();
+                                s.append(' ');
+                                if (!streq(name, "xmlns"))
+                                    s.append("xmlns:");
+                                s.append(name).append("='");
+                                encodeUtf8XML(const_cast<IProperties*>(xmlns)->queryProp(name), s);
+                                s.append("'");
+                            }
                         }
                     }
+                    if (resultName && *resultName)
+                        s.appendf(" name='%s'",resultName);
+                    else if (sequence != (unsigned) -1)
+                        s.appendf(" name='Result %d'",sequence+1);
                 }
-                if (resultName && *resultName)
-                    s.appendf(" name='%s'",resultName);
-                else if (sequence != (unsigned) -1)
-                    s.appendf(" name='Result %d'",sequence+1);
                 s.append(">\n");
                 tail.clear().appendf("</%s>\n", elementName);
             }
@@ -2339,8 +2352,7 @@ void FlushingStringBuffer::startDataset(const char *elementName, const char *res
         isEmpty = false;
     }
 }
-
-void FlushingStringBuffer::startScalar(const char *resultName, unsigned sequence)
+void FlushingStringBuffer::startScalar(const char *resultName, unsigned sequence, bool simpleTag, const char *simpleName)
 {
     if (s.length())
         throw MakeStringException(0, "Attempt to output scalar ('%s',%d) multiple times", resultName ? resultName : "", (int)sequence);
@@ -2354,32 +2366,38 @@ void FlushingStringBuffer::startScalar(const char *resultName, unsigned sequence
     {
         if (mlFmt==MarkupFmt_XML)
         {
-            tail.clear();
-            s.append("<Dataset");
-            if (isSoap && (resultName || (sequence != (unsigned) -1)))
+            if (!simpleTag)
             {
-                s.append(" xmlns=\'urn:hpccsystems:ecl:").appendLower(queryName.length(), queryName.str()).append(":result:");
+                tail.clear();
+                s.append("<Dataset");
+                if (isSoap && (resultName || (sequence != (unsigned) -1)))
+                {
+                    s.append(" xmlns=\'urn:hpccsystems:ecl:").appendLower(queryName.length(), queryName.str()).append(":result:");
+                    if (resultName && *resultName)
+                        s.appendLower(strlen(resultName), resultName).append('\'');
+                    else
+                        s.append("result_").append(sequence+1).append('\'');
+                }
                 if (resultName && *resultName)
-                    s.appendLower(strlen(resultName), resultName).append('\'');
+                    s.appendf(" name='%s'>\n",resultName);
                 else
-                    s.append("result_").append(sequence+1).append('\'');
+                    s.appendf(" name='Result %d'>\n",sequence+1);
+                s.append(" <Row>");
             }
-            if (resultName && *resultName)
-                s.appendf(" name='%s'>\n",resultName);
-            else
-                s.appendf(" name='Result %d'>\n",sequence+1);
-            s.append(" <Row>");
-            if (resultName && *resultName)
+            if (!simpleName)
+                simpleName = resultName;
+            if (simpleName && *simpleName)
             {
-                s.appendf("<%s>", resultName);
-                tail.appendf("</%s>", resultName);
+                s.appendf("<%s>", simpleName);
+                tail.appendf("</%s>", simpleName);
             }
             else
             {
                 s.appendf("<Result_%d>", sequence+1);
                 tail.appendf("</Result_%d>", sequence+1);
             }
-            tail.appendf("</Row>\n</Dataset>\n");
+            if (!simpleTag)
+                tail.appendf("</Row>\n</Dataset>\n");
         }
         else if (!isRaw)
         {
@@ -2423,7 +2441,7 @@ void FlushingJsonBuffer::encodeData(const void *data, unsigned len)
     appendJSONDataValue(s, NULL, len, data);
 }
 
-void FlushingJsonBuffer::startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend, const IProperties *xmlns)
+void FlushingJsonBuffer::startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend, const IProperties *xmlns, bool adaptive)
 {
     CriticalBlock b(crit);
     extend = _extend;
@@ -2432,7 +2450,7 @@ void FlushingJsonBuffer::startDataset(const char *elementName, const char *resul
         name.clear().append(resultName ? resultName : elementName);
         sequenceNumber = 0;
         startBlock();
-        if (!isBlocked)
+        if (elementName && !isBlocked)
         {
             StringBuffer seqName;
             if (!resultName || !*resultName)
@@ -2444,7 +2462,7 @@ void FlushingJsonBuffer::startDataset(const char *elementName, const char *resul
     }
 }
 
-void FlushingJsonBuffer::startScalar(const char *resultName, unsigned sequence)
+void FlushingJsonBuffer::startScalar(const char *resultName, unsigned sequence, bool simpleTag, const char *simpleName)
 {
     if (s.length())
         throw MakeStringException(0, "Attempt to output scalar ('%s',%d) multiple times", resultName ? resultName : "", (int)sequence);
@@ -2456,28 +2474,34 @@ void FlushingJsonBuffer::startScalar(const char *resultName, unsigned sequence)
     startBlock();
     if (!isBlocked)
     {
-        StringBuffer seqName;
-        if (!resultName || !*resultName)
-            resultName = seqName.appendf("Result_%d", sequence+1).str();
-        appendJSONName(s, resultName).append('{');
-        appendJSONName(s, "Row").append("[{");
-        appendJSONName(s, resultName);
-        tail.set("}]}");
+        if (!simpleTag)
+        {
+            StringBuffer seqName;
+            if (!resultName || !*resultName)
+                resultName = seqName.appendf("Result_%d", sequence+1).str();
+            appendJSONName(s, resultName).append('{');
+            appendJSONName(s, "Row").append("[");
+        }
+        s.append('{');
+        appendJSONName(s, (simpleName && *simpleName) ? simpleName : resultName);
+        tail.set("}");
+        if (!simpleTag)
+            tail.append("]}");
     }
 }
 
-void FlushingJsonBuffer::setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size)
+void FlushingJsonBuffer::setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size, bool simpleTag, const char *simpleName)
 {
-    startScalar(resultName, sequence);
+    startScalar(resultName, sequence, simpleTag, simpleName);
     if (size < 7) //JavaScript only supports 53 significant bits
         s.append(value);
     else
         s.append('"').append(value).append('"');
 }
 
-void FlushingJsonBuffer::setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size)
+void FlushingJsonBuffer::setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size, bool simpleTag, const char *simpleName)
 {
-    startScalar(resultName, sequence);
+    startScalar(resultName, sequence, simpleTag, simpleName);
     if (size < 7) //JavaScript doesn't support unsigned, and only supports 53 significant bits
         s.append(value);
     else

+ 61 - 8
common/thorhelper/roxiehelper.hpp

@@ -171,12 +171,58 @@ public:
     }
     IPropertyTree *createPTreeFromParameters(byte flags)
     {
+        if (!pathNodes.isItem(1))
+            throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Query not specified");
         StringBuffer query;
         appendDecodedURL(query, pathNodes.item(1));
-        if (!query.length())
-            throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Query not specified");
+        aindex_t count = pathNodes.ordinality();
+        if (count>2)
+            for (aindex_t x = 2; x<count; ++x)
+                appendDecodedURL(query.append('/'), pathNodes.item(x));
         return createPTreeFromHttpParameters(query, form ? form : parameters, true, false, (ipt_flags) flags);
     }
+    bool isMappedToInputParameter()
+    {
+        if (isHttp())
+        {
+            aindex_t count = pathNodes.ordinality();
+            if (count>2)
+                for (aindex_t x = 2; x<count; ++x)
+                    if (strncmp(pathNodes.item(x), "input(", 6)==0)
+                        return true;
+        }
+        return false;
+    }
+    IPropertyTree *checkAddWrapperForAdaptiveInput(IPropertyTree *content, byte flags)
+    {
+        if (!isMappedToInputParameter())
+            return content;
+        if (!pathNodes.isItem(1))
+            throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Query not specified");
+        StringBuffer query;
+        appendDecodedURL(query, pathNodes.item(1));
+        aindex_t count = pathNodes.ordinality();
+        if (count>2)
+            for (aindex_t x = 2; x<count; ++x)
+                appendDecodedURL(query.append('/'), pathNodes.item(x));
+        return createPTreeFromHttpPath(query, content, false, (ipt_flags) flags);
+    }
+    void getResultFilterAndTag(StringAttr &filter, StringAttr &tag)
+    {
+        if (!isHttp())
+            return;
+        aindex_t count = pathNodes.ordinality();
+        if (count<=2)
+            return;
+        StringBuffer temp;
+        for (aindex_t x = 2; x<count; ++x)
+        {
+            if (strncmp(pathNodes.item(x), "result(", 6)==0)
+                checkParseUrlPathNodeValue(pathNodes.item(x), temp, filter);
+            else if (strncmp(pathNodes.item(x), "tag(", 4)==0)
+                checkParseUrlPathNodeValue(pathNodes.item(x), temp, tag);
+        }
+    }
 };
 
 //==============================================================================================================
@@ -257,6 +303,9 @@ interface SafeSocket : extends IInterface
     // TO be removed and replaced with better mechanism when SafeSocket merged with tht new output sequencer...
     // until then you may need to lock using this if you are making multiple calls and they need to stay together in the output
     virtual CriticalSection &queryCrit() = 0;
+
+    virtual void setAdaptiveRoot(bool adaptive)=0;
+    virtual bool getAdaptiveRoot()=0;
 };
 
 class THORHELPER_API CSafeSocket : public CInterface, implements SafeSocket
@@ -265,6 +314,7 @@ protected:
     Linked<ISocket> sock;
     bool httpMode;
     bool heartbeat;
+    bool adaptiveRoot = false;
     TextMarkupFormat mlResponseFmt = MarkupFmt_Unknown;
     StringAttr contentHead;
     StringAttr contentTail;
@@ -284,6 +334,8 @@ public:
     bool readBlock(MemoryBuffer &ret, unsigned maxBlockSize, unsigned timeout = (unsigned) WAIT_FOREVER);
     bool readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHttpHelper, bool &, bool &, unsigned maxBlockSize);
     void setHttpMode(const char *queryName, bool arrayMode, HttpHelper &httphelper);
+    void setAdaptiveRoot(bool adaptive){adaptiveRoot=adaptive;}
+    bool getAdaptiveRoot(){return adaptiveRoot;}
     void checkSendHttpException(HttpHelper &httphelper, IException *E, const char *queryName);
     void sendSoapException(IException *E, const char *queryName);
     void sendJsonException(IException *E, const char *queryName);
@@ -342,12 +394,13 @@ public:
     virtual void addPayload(StringBuffer &s, unsigned int reserve=0);
     virtual void *getPayload(size32_t &length);
     virtual void startBlock();
-    virtual void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false, const IProperties *xmlns=NULL);
-    virtual void startScalar(const char *resultName, unsigned sequence);
+    virtual void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false, const IProperties *xmlns=NULL, bool adaptive=false);
+    virtual void startScalar(const char *resultName, unsigned sequence, bool simpleTag=false, const char *simplename=nullptr);
     virtual void setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size);
     virtual void setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size);
     virtual void incrementRowCount();
     void setTail(const char *value){tail.set(value);}
+    const char *queryResultName(){return name;}
 };
 
 class THORHELPER_API FlushingJsonBuffer : public FlushingStringBuffer
@@ -361,10 +414,10 @@ public:
     void append(double data);
     void encodeString(const char *x, unsigned len, bool utf8=false);
     void encodeData(const void *data, unsigned len);
-    void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false, const IProperties *xmlns=NULL);
-    void startScalar(const char *resultName, unsigned sequence);
-    virtual void setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size);
-    virtual void setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size);
+    void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false, const IProperties *xmlns=NULL, bool adaptive=false);
+    void startScalar(const char *resultName, unsigned sequence, bool simpleTag, const char *simplename=nullptr);
+    virtual void setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size, bool simpleTag = false, const char *simplename=nullptr);
+    virtual void setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size, bool simpleTag = false, const char *simplename=nullptr);
 };
 
 inline const char *getFormatName(TextMarkupFormat fmt)

+ 7 - 2
common/thorhelper/thorxmlwrite.cpp

@@ -551,10 +551,10 @@ void CommonJsonWriter::outputUtf8(unsigned len, const char *field, const char *f
     appendJSONStringValue(out, checkItemName(fieldname), rtlUtf8Size(len, field), field, true);
 }
 
-void CommonJsonWriter::outputBeginArray(const char *fieldname)
+void CommonJsonWriter::prepareBeginArray(const char *fieldname)
 {
     arrays.append(*new CJsonWriterItem(fieldname));
-    const char * sep = strchr(fieldname, '/');
+    const char * sep = (fieldname) ? strchr(fieldname, '/') : NULL;
     while (sep)
     {
         StringAttr leading(fieldname, sep-fieldname);
@@ -563,6 +563,11 @@ void CommonJsonWriter::outputBeginArray(const char *fieldname)
         sep = strchr(fieldname, '/');
     }
     checkFormat(false, false, 1);
+}
+
+void CommonJsonWriter::outputBeginArray(const char *fieldname)
+{
+    prepareBeginArray(fieldname);
     appendJSONName(out, fieldname).append('[');
 }
 

+ 1 - 0
common/thorhelper/thorxmlwrite.hpp

@@ -122,6 +122,7 @@ public:
 
     void checkDelimit(int inc=0);
     void checkFormat(bool doDelimit, bool needDelimiter=true, int inc=0);
+    void prepareBeginArray(const char *fieldname);
 
     virtual void outputInlineXml(const char *text) //for appending raw xml content
     {

+ 202 - 37
roxie/ccd/ccdprotocol.cpp

@@ -329,6 +329,104 @@ protected:
 
 };
 
+enum class AdaptiveRoot {NamedArray, RootArray, FirstRow};
+
+class AdaptiveRESTJsonWriter : public CommonJsonWriter
+{
+    AdaptiveRoot model;
+    unsigned depth = 0;
+public:
+    AdaptiveRESTJsonWriter(AdaptiveRoot _model, unsigned _flags, unsigned _initialIndent, IXmlStreamFlusher *_flusher) :
+        CommonJsonWriter(_flags, _initialIndent, _flusher), model(_model)
+    {
+    }
+
+    virtual void outputBeginArray(const char *fieldname)
+    {
+        prepareBeginArray(fieldname);
+        if (model == AdaptiveRoot::NamedArray || arrays.length()>1)
+            appendJSONName(out, fieldname).append('[');
+        else if (model == AdaptiveRoot::RootArray)
+            out.append('[');
+    }
+    void outputEndArray(const char *fieldname)
+    {
+        arrays.pop();
+        checkFormat(false, true, -1);
+        if (arrays.length() || model != AdaptiveRoot::FirstRow)
+            out.append(']');
+        const char * sep = (fieldname) ? strchr(fieldname, '/') : NULL;
+        while (sep)
+        {
+            out.append('}');
+            sep = strchr(sep+1, '/');
+        }
+    }
+    void outputBeginNested(const char *fieldname, bool nestChildren)
+    {
+        CommonJsonWriter::outputBeginNested(fieldname, nestChildren);
+        if (model == AdaptiveRoot::FirstRow)
+            depth++;
+    }
+    void outputEndNested(const char *fieldname)
+    {
+        CommonJsonWriter::outputEndNested(fieldname);
+        if (model == AdaptiveRoot::FirstRow)
+        {
+            depth--;
+            if (fieldname && streq(fieldname, "Row") && depth==0)
+            {
+                flush(true);
+                flusher = nullptr;
+            }
+        }
+    }
+};
+
+class AdaptiveRESTXmlWriter : public CommonXmlWriter
+{
+    StringAttr tag;
+    AdaptiveRoot model = AdaptiveRoot::NamedArray;
+    unsigned depth = 0;
+public:
+    AdaptiveRESTXmlWriter(AdaptiveRoot _model, const char *tagname, unsigned _flags, unsigned _initialIndent, IXmlStreamFlusher *_flusher) :
+        CommonXmlWriter(_flags, _initialIndent, _flusher), tag(tagname), model(_model)
+    {
+    }
+    void outputBeginNested(const char *fieldname, bool nestChildren)
+    {
+        if (model == AdaptiveRoot::FirstRow)
+        {
+            if (!depth && tag.length())
+                fieldname = tag.str();
+            depth++;
+        }
+        CommonXmlWriter::outputBeginNested(fieldname, nestChildren);
+    }
+    void outputEndNested(const char *fieldname)
+    {
+        if (model == AdaptiveRoot::FirstRow)
+        {
+            depth--;
+            if (!depth)
+            {
+                CommonXmlWriter::outputEndNested(tag.length() ? tag.str() : fieldname);
+                flush(true);
+                flusher = nullptr;
+                return;
+            }
+        }
+        CommonXmlWriter::outputEndNested(fieldname);
+    }
+};
+
+IXmlWriterExt * createAdaptiveRESTWriterExt(AdaptiveRoot model, const char *tagname, unsigned _flags, unsigned _initialIndent, IXmlStreamFlusher *_flusher, XMLWriterType xmlType)
+{
+    if (xmlType==WTJSON)
+        return new AdaptiveRESTJsonWriter(model, _flags, _initialIndent, _flusher);
+    return new AdaptiveRESTXmlWriter(model, tagname, _flags, _initialIndent, _flusher);
+}
+
 //================================================================================================================
 
 class CHpccNativeResultsWriter : public CInterface, implements IHpccNativeProtocolResultsWriter
@@ -339,6 +437,9 @@ protected:
     IPointerArrayOf<FlushingStringBuffer> resultMap;
 
     StringAttr queryName;
+    StringAttr tagName;
+    StringAttr resultFilter;
+
     const IContextLogger &logctx;
     Owned<FlushingStringBuffer> probe;
     TextMarkupFormat mlFmt;
@@ -348,6 +449,8 @@ protected:
     bool isHTTP;
     bool trim;
     bool failed;
+    bool adaptiveRoot = false;
+    bool onlyUseFirstRow = false;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -358,6 +461,10 @@ public:
     ~CHpccNativeResultsWriter()
     {
     }
+    inline void setAdaptiveRoot(){adaptiveRoot = true; client->setAdaptiveRoot(true);}
+    inline void setTagName(const char *tag){tagName.set(tag);}
+    inline void setOnlyUseFirstRow(){onlyUseFirstRow = true;}
+    inline void setResultFilter(const char *_resultFilter){resultFilter.set(_resultFilter);}
     virtual FlushingStringBuffer *queryResult(unsigned sequence)
     {
         CriticalBlock procedure(resultsCrit);
@@ -381,18 +488,42 @@ public:
     {
         return new FlushingStringBuffer(client, isBlocked, mlFmt, isRaw, isHTTP, logctx);
     }
+    bool checkAdaptiveResult(const char *name)
+    {
+        if (!adaptiveRoot)
+            return false;
+        if (!resultFilter || !*resultFilter)
+            return true;
+        return (streq(resultFilter, name));
+    }
     virtual IXmlWriter *addDataset(const char *name, unsigned sequence, const char *elementName, bool &appendRawData, unsigned writeFlags, bool _extend, const IProperties *xmlns)
     {
         FlushingStringBuffer *response = queryResult(sequence);
         if (response)
         {
             appendRawData = response->isRaw;
-            response->startDataset(elementName, name, sequence, _extend, xmlns);
+            bool adaptive = checkAdaptiveResult(name);
+            if (adaptive)
+            {
+                elementName = nullptr;
+                if (response->mlFmt!=MarkupFmt_JSON && !onlyUseFirstRow && tagName.length())
+                    elementName = tagName.str();
+            }
+            response->startDataset(elementName, name, sequence, _extend, xmlns, adaptive);
             if (response->mlFmt==MarkupFmt_XML || response->mlFmt==MarkupFmt_JSON)
             {
                 if (response->mlFmt==MarkupFmt_JSON)
                     writeFlags |= XWFnoindent;
-                Owned<IXmlWriter> xmlwriter = createIXmlWriterExt(writeFlags, 1, response, (response->mlFmt==MarkupFmt_JSON) ? WTJSON : WTStandard);
+                AdaptiveRoot rootType = AdaptiveRoot::NamedArray;
+                if (adaptive)
+                {
+                    if (onlyUseFirstRow)
+                        rootType = AdaptiveRoot::FirstRow;
+                    else
+                        rootType = AdaptiveRoot::RootArray;
+                }
+
+                Owned<IXmlWriter> xmlwriter = createAdaptiveRESTWriterExt(rootType, tagName, writeFlags, 1, response, (response->mlFmt==MarkupFmt_JSON) ? WTJSON : WTStandard);
                 xmlwriter->outputBeginArray("Row");
                 return xmlwriter.getClear();
             }
@@ -411,6 +542,15 @@ public:
             }
         }
     }
+    inline void startScalar(FlushingStringBuffer *r, const char *name, unsigned sequence)
+    {
+        if (checkAdaptiveResult(name))
+        {
+            r->startScalar(name, sequence, true, tagName.length() ? tagName.str() : name);
+            return;
+        }
+        r->startScalar(name, sequence);
+    }
     virtual void appendRaw(unsigned sequence, unsigned len, const char *data)
     {
         FlushingStringBuffer *r = queryResult(sequence);
@@ -439,7 +579,7 @@ public:
         FlushingStringBuffer *r = queryResult(sequence);
         if (r)
         {
-            r->startScalar(name, sequence);
+            startScalar(r, name, sequence);
             if (isRaw)
                 r->append(sizeof(value), (char *)&value);
             else
@@ -451,7 +591,7 @@ public:
         FlushingStringBuffer *r = queryResult(sequence);
         if (r)
         {
-            r->startScalar(name, sequence);
+            startScalar(r, name, sequence);
             r->encodeData(data, len);
         }
     }
@@ -460,7 +600,7 @@ public:
         FlushingStringBuffer *r = queryResult(sequence);
         if (r)
         {
-            r->startScalar(name, sequence);
+            startScalar(r, name, sequence);
             if (isRaw)
                 r->append(len, (const char *) data);
             else
@@ -472,7 +612,7 @@ public:
         FlushingStringBuffer *r = queryResult(sequence);
         if (r)
         {
-            r->startScalar(name, sequence);
+            startScalar(r, name, sequence);
             if (isRaw)
                 r->append(len, (char *)data);
             else if (mlFmt==MarkupFmt_XML)
@@ -510,7 +650,7 @@ public:
         FlushingStringBuffer *r = queryResult(sequence);
         if (r)
         {
-            r->startScalar(name, sequence);
+            startScalar(r, name, sequence);
             if (isRaw)
                 r->append(len, (char *)val);
             else
@@ -531,7 +671,7 @@ public:
         {
             if (isRaw)
             {
-                r->startScalar(name, sequence);
+                startScalar(r, name, sequence);
                 r->append(sizeof(value), (char *)&value);
             }
             else
@@ -546,7 +686,7 @@ public:
         {
             if (isRaw)
             {
-                r->startScalar(name, sequence);
+                startScalar(r, name, sequence);
                 r->append(sizeof(value), (char *)&value);
             }
             else
@@ -559,7 +699,7 @@ public:
         FlushingStringBuffer *r = queryResult(sequence);
         if (r)
         {
-            r->startScalar(name, sequence);
+            startScalar(r, name, sequence);
             r->append(value);
         }
     }
@@ -568,7 +708,7 @@ public:
         FlushingStringBuffer *r = queryResult(sequence);
         if (r)
         {
-            r->startScalar(name, sequence);
+            startScalar(r, name, sequence);
             if (r->isRaw)
             {
                 r->append(len, str);
@@ -584,7 +724,7 @@ public:
         FlushingStringBuffer *r = queryResult(sequence);
         if (r)
         {
-            r->startScalar(name, sequence);
+            startScalar(r, name, sequence);
             if (r->isRaw)
             {
                 r->append(len*2, (const char *) str);
@@ -615,13 +755,13 @@ public:
                 result->flush(true);
         }
     }
-    virtual void finalize(unsigned seqNo, const char *delim)
+    virtual void finalize(unsigned seqNo, const char *delim, const char *filter)
     {
         bool needDelimiter = false;
         ForEachItemIn(seq, resultMap)
         {
             FlushingStringBuffer *result = resultMap.item(seq);
-            if (result)
+            if (result && (!filter || !*filter || streq(filter, result->queryResultName())))
             {
                 result->flush(true);
                 for(;;)
@@ -787,6 +927,8 @@ class CHpccNativeProtocolResponse : public CInterface, implements IHpccNativePro
 protected:
     SafeSocket *client;
     StringAttr queryName;
+    StringArray resultFilter;
+    StringBuffer rootTag;
     const IContextLogger &logctx;
     TextMarkupFormat mlFmt;
     PTreeReaderOptions xmlReadFlags;
@@ -799,9 +941,12 @@ protected:
 
 public:
     IMPLEMENT_IINTERFACE;
-    CHpccNativeProtocolResponse(const char *queryname, SafeSocket *_client, TextMarkupFormat _mlFmt, unsigned flags, bool _isHTTP, const IContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags) :
-        client(_client), queryName(queryname), logctx(_logctx), mlFmt(_mlFmt), xmlReadFlags(_xmlReadFlags), protocolFlags(flags), isHTTP(_isHTTP)
+    CHpccNativeProtocolResponse(const char *queryname, SafeSocket *_client, TextMarkupFormat _mlFmt, unsigned flags, bool _isHTTP, const IContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags, const char *_resultFilterString, const char *_rootTag) :
+        client(_client), queryName(queryname), logctx(_logctx), mlFmt(_mlFmt), xmlReadFlags(_xmlReadFlags), protocolFlags(flags), isHTTP(_isHTTP), rootTag(_rootTag)
     {
+        resultFilter.appendList(_resultFilterString, ".");
+        if (!rootTag.length() && resultFilter.length())
+            rootTag.set(resultFilter.item(0)).replace(' ', '_');
     }
     ~CHpccNativeProtocolResponse()
     {
@@ -842,7 +987,18 @@ public:
     virtual IHpccProtocolResultsWriter *queryHpccResultsSection()
     {
         if (!results)
+        {
             results.setown(new CHpccNativeResultsWriter(queryName, client, getIsBlocked(), mlFmt, getIsRaw(), isHTTP, logctx, xmlReadFlags));
+            if (rootTag.length())
+                results->setTagName(rootTag);
+            if (resultFilter.length())
+            {
+                results->setAdaptiveRoot();
+                results->setResultFilter(resultFilter.item(0));
+            }
+            if (resultFilter.isItem(1) && strieq("row", resultFilter.item(1)))
+                results->setOnlyUseFirstRow();
+        }
         return results;
     }
 
@@ -892,8 +1048,8 @@ public:
 class CHpccJsonResponse : public CHpccNativeProtocolResponse
 {
 public:
-    CHpccJsonResponse(const char *queryname, SafeSocket *_client, unsigned flags, bool _isHttp, const IContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags) :
-        CHpccNativeProtocolResponse(queryname, _client, MarkupFmt_JSON, flags, _isHttp, _logctx, _xmlReadFlags)
+    CHpccJsonResponse(const char *queryname, SafeSocket *_client, unsigned flags, bool _isHttp, const IContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags, const char *_resultFilter, const char *_rootTag) :
+        CHpccNativeProtocolResponse(queryname, _client, MarkupFmt_JSON, flags, _isHttp, _logctx, _xmlReadFlags, _resultFilter, _rootTag)
     {
     }
 
@@ -976,7 +1132,7 @@ public:
         CriticalBlock b(contentsCrit);
 
         StringBuffer responseHead, responseTail;
-        if (!(protocolFlags & HPCC_PROTOCOL_CONTROL))
+        if (!resultFilter.ordinality() && !(protocolFlags & HPCC_PROTOCOL_CONTROL))
         {
             StringBuffer name(queryName.get());
             if (isHTTP)
@@ -988,10 +1144,11 @@ public:
             unsigned len = responseHead.length();
             client->write(responseHead.detach(), len, true);
         }
-        outputContent();
+        if (!resultFilter.ordinality())
+            outputContent();
         if (results)
-            results->finalize(seqNo, ",");
-        if (!(protocolFlags & HPCC_PROTOCOL_CONTROL))
+            results->finalize(seqNo, ",", resultFilter.ordinality() ? resultFilter.item(0) : NULL);
+        if (!resultFilter.ordinality() && !(protocolFlags & HPCC_PROTOCOL_CONTROL))
         {
             responseTail.append("}");
             unsigned len = responseTail.length();
@@ -1003,8 +1160,8 @@ public:
 class CHpccXmlResponse : public CHpccNativeProtocolResponse
 {
 public:
-    CHpccXmlResponse(const char *queryname, SafeSocket *_client, unsigned flags, bool _isHTTP, const IContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags) :
-        CHpccNativeProtocolResponse(queryname, _client, MarkupFmt_XML, flags, _isHTTP, _logctx, _xmlReadFlags)
+    CHpccXmlResponse(const char *queryname, SafeSocket *_client, unsigned flags, bool _isHTTP, const IContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags, const char *_resultFilter, const char *_rootTag) :
+        CHpccNativeProtocolResponse(queryname, _client, MarkupFmt_XML, flags, _isHTTP, _logctx, _xmlReadFlags, _resultFilter, _rootTag)
     {
     }
 
@@ -1085,7 +1242,7 @@ public:
         CriticalBlock b(contentsCrit);
 
         StringBuffer responseHead, responseTail;
-        if (!(protocolFlags & HPCC_PROTOCOL_CONTROL))
+        if (!resultFilter.ordinality() && !(protocolFlags & HPCC_PROTOCOL_CONTROL))
         {
             responseHead.append("<").append(queryName);
             responseHead.append("Response").append(" xmlns=\"urn:hpccsystems:ecl:").appendLower(queryName.length(), queryName.str()).append('\"');
@@ -1094,11 +1251,12 @@ public:
             client->write(responseHead.detach(), len, true);
         }
 
-        outputContent();
+        if (!resultFilter.ordinality())
+            outputContent();
         if (results)
-            results->finalize(seqNo, NULL);
+            results->finalize(seqNo, NULL, resultFilter.ordinality() ? resultFilter.item(0) : NULL);
 
-        if (!(protocolFlags & HPCC_PROTOCOL_CONTROL))
+        if (!resultFilter.ordinality() && !(protocolFlags & HPCC_PROTOCOL_CONTROL))
         {
             responseTail.append("</").append(queryName);
             if (isHTTP)
@@ -1112,11 +1270,13 @@ public:
 
 IHpccProtocolResponse *createProtocolResponse(const char *queryname, SafeSocket *client, HttpHelper &httpHelper, const IContextLogger &logctx, unsigned protocolFlags, PTreeReaderOptions xmlReadFlags)
 {
+    StringAttr filter, tag;
+    httpHelper.getResultFilterAndTag(filter, tag);
     if (protocolFlags & HPCC_PROTOCOL_NATIVE_RAW || protocolFlags & HPCC_PROTOCOL_NATIVE_ASCII)
-        return new CHpccNativeProtocolResponse(queryname, client, MarkupFmt_Unknown, protocolFlags, false, logctx, xmlReadFlags);
+        return new CHpccNativeProtocolResponse(queryname, client, MarkupFmt_Unknown, protocolFlags, false, logctx, xmlReadFlags, filter, tag);
     else if (httpHelper.queryResponseMlFormat()==MarkupFmt_JSON)
-        return new CHpccJsonResponse(queryname, client, protocolFlags, httpHelper.isHttp(), logctx, xmlReadFlags);
-    return new CHpccXmlResponse(queryname, client, protocolFlags, httpHelper.isHttp(), logctx, xmlReadFlags);
+        return new CHpccJsonResponse(queryname, client, protocolFlags, httpHelper.isHttp(), logctx, xmlReadFlags, filter, tag);
+    return new CHpccXmlResponse(queryname, client, protocolFlags, httpHelper.isHttp(), logctx, xmlReadFlags, filter, tag);
 
 }
 
@@ -1362,18 +1522,16 @@ private:
             {
                 if (httpHelper.queryRequestMlFormat()==MarkupFmt_JSON)
                 {
+                    if (strieq(queryName, "__array__"))
+                        throw MakeStringException(ROXIE_DATA_ERROR, "JSON request array not implemented");
+                    isRequest=true;
                     if (strieq(queryName, "__object__"))
                     {
                         queryPT.setown(queryPT->getPropTree("*[1]"));
                         queryName.set(queryPT->queryName());
-                        isRequest = true;
                         if (!queryPT)
                             throw MakeStringException(ROXIE_DATA_ERROR, "Malformed JSON request (missing Body)");
                     }
-                    else if (strieq(queryName, "__array__"))
-                        throw MakeStringException(ROXIE_DATA_ERROR, "JSON request array not implemented");
-                    else
-                        throw MakeStringException(ROXIE_DATA_ERROR, "Malformed JSON request");
                 }
                 else
                 {
@@ -1428,12 +1586,19 @@ private:
     }
     void createQueryPTree(Owned<IPropertyTree> &queryPT, HttpHelper &httpHelper, const char *text, byte flags, PTreeReaderOptions options)
     {
+        StringBuffer logxml;
         if (httpHelper.queryRequestMlFormat()==MarkupFmt_URL)
+        {
             queryPT.setown(httpHelper.createPTreeFromParameters(flags));
-        else if (httpHelper.queryRequestMlFormat()==MarkupFmt_JSON)
+            toXML(queryPT, logxml);
+            DBGLOG("%s", logxml.str());
+            return;
+        }
+        if (httpHelper.queryRequestMlFormat()==MarkupFmt_JSON)
             queryPT.setown(createPTreeFromJSONString(text, flags, options));
         else
             queryPT.setown(createPTreeFromXMLString(text, flags, options));
+        queryPT.setown(httpHelper.checkAddWrapperForAdaptiveInput(queryPT.getClear(), flags));
     }
 
     void doMain(const char *runQuery)

+ 79 - 18
system/jlib/jptree.cpp

@@ -7240,27 +7240,94 @@ static void ensureHttpParameter(IPropertyTree *pt, const char *path, const char
     ensureHttpParameter(pt, tag, path, value, fullpath);
 }
 
-//URL node nameWithAttrs is of the form: "TagName;attr1=abc;attr2;attr3=;"
-IPropertyTree *createPTreeFromHttpParameters(const char *nameWithAttrs, IProperties *parameters, bool skipLeadingDotParameters, bool nestedRoot, ipt_flags flags)
+bool checkParseUrlPathNodeValue(const char *s, StringBuffer &name, StringAttr &value)
+{
+    s = skipWhitespace(s);
+    const char *pn = strchr(s, '(');
+    if (pn) //strict format param('value') so we can extend later
+    {
+        const char *vp = pn + 1;
+        if (*vp!='\'')
+            return false;
+        const char *end =strchr(++vp, '\'');
+        if (!end || *(end+1)!=')')
+            return false;
+        if (!validateXMLTag(name.append(pn-s, s).trim()))
+            return false;
+        value.set(vp, end-vp);
+    }
+    else
+    {
+        if (!validateXMLTag(name.append(s).trim()))
+            return false;
+    }
+    return true;
+}
+IPropertyTree *createPTreeFromHttpPath(const char *nameWithAttrs, IPropertyTree *content, bool nestedRoot, ipt_flags flags)
 {
     StringArray nameAttrList;
-    nameAttrList.appendList(nameWithAttrs, ";");
+    nameAttrList.appendList(nameWithAttrs, "/");
     if (!nameAttrList.ordinality())
         return NULL;
     Owned<IPropertyTree> pt = createPTree(nameAttrList.item(0), flags);
     for (aindex_t pos=1; nameAttrList.isItem(pos); pos++)
     {
-        const char *attr = skipWhitespace(nameAttrList.item(pos));
-        if (*attr=='=')
-            continue;
+        StringBuffer name;
+        StringAttr value;
+        if (!checkParseUrlPathNodeValue(nameAttrList.item(pos), name, value))
+            throw MakeStringException(-1, "Invalid URL parameter format %s", nameAttrList.item(pos));
         StringBuffer xpath("@");
-        const char *eq = strchr(attr, '=');
-        if (eq)
-            pt->setProp(xpath.append(eq-attr, attr).trim(), eq+1);
+        xpath.append(name.str());
+        if (!value.get())
+            pt->setPropBool(xpath, true);
         else
-            pt->setPropBool(xpath.append(attr).trim(), true);
+            pt->setProp(xpath, value);
+    }
+    IPropertyTree *parent = pt;
+    const char *input = pt->queryProp("@input");
+    if (input)
+    {
+        StringArray inputNodes;
+        inputNodes.appendList(input, ".");
+        ForEachItemIn(in, inputNodes)
+        {
+            const char *tag = inputNodes.item(in);
+            if (!validateXMLTag(tag))
+                throw MakeStringException(-1, "Invalid REST query input specifier %s", input);
+            parent = parent->addPropTree(tag, createPTree(tag, flags));
+        }
     }
 
+    if (streq("__array__", content->queryName()))
+    {
+        Owned<IAttributeIterator> aiter = content->getAttributes();
+        ForEach (*aiter)
+            parent->addProp(aiter->queryName(), aiter->queryValue());
+        Owned<IPropertyTreeIterator> iter = content->getElements("__item__");
+        ForEach (*iter)
+        {
+            IPropertyTree &e = iter->query();
+            e.renameProp("/", "Row");
+            parent->addPropTree("Row", LINK(&e));
+        }
+    }
+    else
+        mergePTree(parent, content);
+
+    if (nestedRoot)
+    {
+        Owned<IPropertyTree> root = createPTree(flags);
+        root->setPropTree(nameAttrList.item(0), pt.getClear());
+        return root.getClear();
+    }
+
+    return pt.getClear();
+}
+
+//URL node nameWithAttrs is of the form: "TagName/attr1('abc')/attr2/attr3('xyz')"
+IPropertyTree *createPTreeFromHttpParameters(const char *nameWithAttrs, IProperties *parameters, bool skipLeadingDotParameters, bool nestedRoot, ipt_flags flags)
+{
+    Owned<IPropertyTree> content = createPTree("content", flags);
     Owned<IPropertyIterator> iter = parameters->getIterator();
     ForEach(*iter)
     {
@@ -7272,14 +7339,8 @@ IPropertyTree *createPTreeFromHttpParameters(const char *nameWithAttrs, IPropert
         const char *value = parameters->queryProp(key);
         if (!value || !*value)
             continue;
-        ensureHttpParameter(pt, key, value);
-    }
-    if (nestedRoot)
-    {
-        Owned<IPropertyTree> root = createPTree(flags);
-        root->setPropTree(nameAttrList.item(0), pt.getClear());
-        return root.getClear();
+        ensureHttpParameter(content, key, value);
     }
 
-    return pt.getClear();
+    return createPTreeFromHttpPath(nameWithAttrs, content.getClear(), nestedRoot, flags);
 }

+ 3 - 1
system/jlib/jptree.hpp

@@ -209,8 +209,10 @@ jlib_decl IPropertyTree *createPTreeFromIPT(const IPropertyTree *srcTree, ipt_fl
 jlib_decl IPropertyTree *createPTreeFromJSONString(const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 
-//URL node nameWithAttrs is of the form: "TagName;attr1=abc;attr2;attr3=;"
+//URL node nameWithAttrs is of the form: "TagName/attr1('abc')/attr2/attr3('')"
+jlib_decl IPropertyTree *createPTreeFromHttpPath(const char *nameWithAttrs, IPropertyTree *content, bool nestedRoot, ipt_flags flags);
 jlib_decl IPropertyTree *createPTreeFromHttpParameters(const char *nameWithAttrs, IProperties *parameters, bool skipLeadingDotParameters, bool nestedRoot, ipt_flags flags=ipt_none);
+jlib_decl bool checkParseUrlPathNodeValue(const char *s, StringBuffer &name, StringAttr &value);
 
 
 #define XML_SortTags 0x01