HPCC-17999 Fix Roxie OUTPUT(extend) support for JSON

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/thorhelper/roxiehelper.cpp

@@ -2051,9 +2051,16 @@ void FlushingStringBuffer::addPayload(StringBuffer &s, unsigned int reserve)
         s.ensureCapacity(reserve);
 }
 
-void FlushingStringBuffer::flushXML(StringBuffer &current, bool isClosing)
+void FlushingStringBuffer::flushXML(StringBuffer &current, bool isClosing, const char *delim)
 {
     CriticalBlock b(crit);
+    if (isClosing && delim && current.length())
+    {
+        if (!first)
+            s.append(delim);
+        else
+            first = false;
+    }
     if (isHttp) // we don't do any chunking for non-HTTP yet
     {
         if (isClosing || current.length() > HTTP_SPLIT_THRESHOLD)
@@ -2311,7 +2318,9 @@ void FlushingJsonBuffer::startDataset(const char *elementName, const char *resul
             if (!resultName || !*resultName)
                 resultName = seqName.appendf("result_%d", sequence+1).str();
             appendJSONName(s, resultName).append('{');
-            tail.set("}");
+            if (!adaptive)
+                appendJSONName(s, "Row").append('[');
+            tail.set(adaptive ? "}" : "]}");
         }
         isEmpty = false;
     }

common/thorhelper/roxiehelper.hpp

@@ -414,6 +414,7 @@ protected:
     CriticalSection crit;
     PointerArray queued;
     UnsignedArray lengths;
+    bool first = true;
 
     bool needsFlush(bool closing);
 public:
@@ -440,7 +441,11 @@ public:
     virtual void appendf(const char *format, ...) __attribute__((format(printf, 2, 3)));
     virtual void encodeString(const char *x, unsigned len, bool utf8=false);
     virtual void encodeData(const void *data, unsigned len);
-    virtual void flushXML(StringBuffer &current, bool isClosing);
+    virtual void flushXML(StringBuffer &current, bool isClosing)
+    {
+        flushXML(current, isClosing, nullptr);
+    }
+    void flushXML(StringBuffer &current, bool isClosing, const char *delim);
     virtual void flush(bool closing) ;
     virtual void addPayload(StringBuffer &s, unsigned int reserve=0);
     virtual void *getPayload(size32_t &length);
@@ -456,9 +461,12 @@ public:
 
 class THORHELPER_API FlushingJsonBuffer : public FlushingStringBuffer
 {
+protected:
+    bool extend;
+
 public:
-    FlushingJsonBuffer(SafeSocket *_sock, bool _isBlocked, bool _isHttp, const IContextLogger &_logctx) :
-        FlushingStringBuffer(_sock, _isBlocked, MarkupFmt_JSON, false, _isHttp, _logctx)
+    FlushingJsonBuffer(SafeSocket *_sock, bool _isBlocked, bool _isHttp, const IContextLogger &_logctx, bool _extend = false) :
+        FlushingStringBuffer(_sock, _isBlocked, MarkupFmt_JSON, false, _isHttp, _logctx), extend(_extend)
     {
     }
 
@@ -469,6 +477,10 @@ public:
     void startScalar(const char *resultName, unsigned sequence, bool simpleTag, const char *simplename=nullptr);
     virtual void setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size, bool simpleTag = false, const char *simplename=nullptr);
     virtual void setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size, bool simpleTag = false, const char *simplename=nullptr);
+    virtual void flushXML(StringBuffer &current, bool isClosing)
+    {
+        FlushingStringBuffer::flushXML(current, isClosing, (extend) ? "," : nullptr);
+    }
 };
 
 inline const char *getFormatName(TextMarkupFormat fmt)

common/thorhelper/thorxmlwrite.cpp

@@ -645,7 +645,6 @@ void CommonJsonWriter::outputBeginNested(const char *fieldname, bool nestChildre
     if (!*fieldname && !checkUnamedArrayItem(true))
         return;
 
-    flush(false);
     checkFormat(true, false, 1);
     fieldname = checkItemNameBeginNested(fieldname);
     if (fieldname && *fieldname)
@@ -672,7 +671,6 @@ void CommonJsonWriter::outputEndNested(const char *fieldname)
     if (!*fieldname && !checkUnamedArrayItem(false))
         return;
 
-    flush(false);
     checkFormat(false, true, -1);
     fieldname = checkItemNameEndNested(fieldname);
     if (fieldname && *fieldname)
@@ -691,7 +689,6 @@ void CommonJsonWriter::outputEndNested(const char *fieldname)
 
 void CommonJsonWriter::outputSetAll()
 {
-    flush(false);
     checkDelimit();
     appendJSONValue(out, "All", true);
 }

roxie/ccd/ccdprotocol.cpp

@@ -408,7 +408,7 @@ protected:
 
 };
 
-enum class AdaptiveRoot {NamedArray, RootArray, FirstRow};
+enum class AdaptiveRoot {NamedArray, RootArray, ExtendArray, FirstRow};
 
 class AdaptiveRESTJsonWriter : public CommonJsonWriter
 {
@@ -432,7 +432,7 @@ public:
     {
         arrays.pop();
         checkFormat(false, true, -1);
-        if (arrays.length() || model != AdaptiveRoot::FirstRow)
+        if (arrays.length() || (model != AdaptiveRoot::FirstRow && model != AdaptiveRoot::ExtendArray))
             out.append(']');
         const char * sep = (fieldname) ? strchr(fieldname, '/') : NULL;
         while (sep)
@@ -543,7 +543,7 @@ public:
     inline void setTagName(const char *tag){tagName.set(tag);}
     inline void setOnlyUseFirstRow(){onlyUseFirstRow = true;}
     inline void setResultFilter(const char *_resultFilter){resultFilter.set(_resultFilter);}
-    virtual FlushingStringBuffer *queryResult(unsigned sequence)
+    virtual FlushingStringBuffer *queryResult(unsigned sequence, bool extend=false)
     {
         CriticalBlock procedure(resultsCrit);
         while (!resultMap.isItem(sequence))
@@ -552,7 +552,7 @@ public:
         if (!result)
         {
             if (mlFmt==MarkupFmt_JSON)
-                result = new FlushingJsonBuffer(client, isBlocked, isHTTP, logctx);
+                result = new FlushingJsonBuffer(client, isBlocked, isHTTP, logctx, extend);
             else
                 result = new FlushingStringBuffer(client, isBlocked, mlFmt, isRaw, isHTTP, logctx);
             result->isSoap = isHTTP;
@@ -576,7 +576,7 @@ public:
     }
     virtual IXmlWriter *addDataset(const char *name, unsigned sequence, const char *elementName, bool &appendRawData, unsigned writeFlags, bool _extend, const IProperties *xmlns)
     {
-        FlushingStringBuffer *response = queryResult(sequence);
+        FlushingStringBuffer *response = queryResult(sequence, _extend);
         if (response)
         {
             appendRawData = response->isRaw;
@@ -592,7 +592,7 @@ public:
             {
                 if (response->mlFmt==MarkupFmt_JSON)
                     writeFlags |= XWFnoindent;
-                AdaptiveRoot rootType = AdaptiveRoot::NamedArray;
+                AdaptiveRoot rootType = AdaptiveRoot::ExtendArray;
                 if (adaptive)
                 {
                     if (onlyUseFirstRow)