HPCC-8953 Add roxie support for JSON/HTTP queries

New feature allowing native support for JSON queries in roxie.

Signed-off-by: Anthony Fishbeck <Anthony.Fishbeck@lexisnexis.com>

common/thorhelper/roxiedebug.cpp

@@ -506,6 +506,14 @@ public:
     {
         // nothing for now
     }
+    virtual void outputBeginArray(const char *fieldname)
+    {
+        // nothing for now
+    }
+    virtual void outputEndArray(const char *fieldname)
+    {
+        // nothing for now
+    }
     virtual void outputSetAll()
     {
         // nothing for now

common/thorhelper/roxiehelper.cpp

@@ -415,7 +415,8 @@ bool CRHLimitedCompareHelper::getGroup(OwnedRowArray &group, const void *left)
 
 CSafeSocket::CSafeSocket(ISocket *_sock)
 {
-    httpMode = false; 
+    httpMode = false;
+    mlFmt = MarkupFmt_Unknown;
     sent = 0; 
     heartbeat = false; 
     sock.setown(_sock);
@@ -564,16 +565,11 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
 
                 // capture authentication token
                 if ((str = strstr(header, "Authorization: Basic ")) != NULL)
-                {
-                    char *authToken = str + strlen("Authorization: Basic ");
-                    str = strchr(authToken, '\r');
-                    if (str)
-                    {
-                        *str = 0;
-                        pHttpHelper->setAuthToken(authToken);
-                        *str = '\r';  // need to remove the 0 so other str comparisons will work
-                    }
-                }
+                    pHttpHelper->setAuthToken(str+21);
+
+                // capture content type
+                if ((str = strstr(header, "Content-Type: ")) != NULL)
+                    pHttpHelper->setContentType(str+14);
 
                 // determine payload length
                 str = strstr(header, "Content-Length: ");
@@ -627,22 +623,31 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
     }
 }
 
-void CSafeSocket::setHttpMode(const char *queryName, bool arrayMode)
+void CSafeSocket::setHttpMode(const char *queryName, bool arrayMode, TextMarkupFormat _mlfmt)
 {
     CriticalBlock c(crit); // Should not be needed
     httpMode = true;
+    mlFmt = _mlfmt;
     heartbeat = false;
-    assertex(xmlhead.length()==0 && xmltail.length()==0);
-    xmlhead.append(
-        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
-        "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\">"
-        "<soap:Body>");
-    if (arrayMode)
+    assertex(contentHead.length()==0 && contentTail.length()==0);
+    if (mlFmt==MarkupFmt_JSON)
+    {
+        contentHead.append("{");
+        contentTail.append("}");
+    }
+    else
     {
-        xmlhead.append("<").append(queryName).append("ResponseArray>");
-        xmltail.append("</").append(queryName).append("ResponseArray>");
+        contentHead.append(
+            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+            "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\">"
+            "<soap:Body>");
+        if (arrayMode)
+        {
+            contentHead.append("<").append(queryName).append("ResponseArray>");
+            contentTail.append("</").append(queryName).append("ResponseArray>");
+        }
+        contentTail.append("</soap:Body></soap:Envelope>");
     }
-    xmltail.append("</soap:Body></soap:Envelope>");
 }
 
 void CSafeSocket::setHeartBeat()
@@ -694,13 +699,13 @@ void CSafeSocket::flush()
 {
     if (httpMode)
     {
-        unsigned length = xmlhead.length() + xmltail.length();
+        unsigned length = contentHead.length() + contentTail.length();
         ForEachItemIn(idx, lengths)
             length += lengths.item(idx);
 
         StringBuffer header;
         header.append("HTTP/1.0 200 OK\r\n");
-        header.append("Content-Type: text/xml\r\n");
+        header.append("Content-Type: ").append(mlFmt == MarkupFmt_JSON ? "application/json" : "text/xml").append("\r\n");
         header.append("Content-Length: ").append(length).append("\r\n\r\n");
 
 
@@ -710,9 +715,9 @@ void CSafeSocket::flush()
         sock->write(header.str(), header.length());
         sent += header.length();
         if (traceLevel > 5)
-            DBGLOG("Writing xml head length %d to HTTP socket", xmlhead.length());
-        sock->write(xmlhead.str(), xmlhead.length());
-        sent += xmlhead.length();
+            DBGLOG("Writing content head length %d to HTTP socket", contentHead.length());
+        sock->write(contentHead.str(), contentHead.length());
+        sent += contentHead.length();
         ForEachItemIn(idx2, queued)
         {
             unsigned length = lengths.item(idx2);
@@ -722,9 +727,9 @@ void CSafeSocket::flush()
             sent += length;
         }
         if (traceLevel > 5)
-            DBGLOG("Writing xml tail length %d to HTTP socket", xmltail.length());
-        sock->write(xmltail.str(), xmltail.length());
-        sent += xmltail.length();
+            DBGLOG("Writing content tail length %d to HTTP socket", contentTail.length());
+        sock->write(contentTail.str(), contentTail.length());
+        sent += contentTail.length();
         if (traceLevel > 5)
             DBGLOG("Total written %d", sent);
     }
@@ -734,7 +739,7 @@ void CSafeSocket::sendException(const char *source, unsigned code, const char *m
 {
     try
     {
-        FlushingStringBuffer response(this, isBlocked, true, false, httpMode, logctx);
+        FlushingStringBuffer response(this, isBlocked, MarkupFmt_XML, false, httpMode, logctx);
         response.startDataset("Exception", NULL, (unsigned) -1);
         response.appendf("<Source>%s</Source><Code>%d</Code>", source, code);
         response.append("<Message>");
@@ -758,11 +763,11 @@ void CSafeSocket::sendException(const char *source, unsigned code, const char *m
 #define RESULT_FLUSH_THRESHOLD 10000u
 
 #ifdef _DEBUG
-#define SOAP_SPLIT_THRESHOLD 100u
-#define SOAP_SPLIT_RESERVE 200u
+#define HTTP_SPLIT_THRESHOLD 100u
+#define HTTP_SPLIT_RESERVE 200u
 #else
-#define SOAP_SPLIT_THRESHOLD 64000u
-#define SOAP_SPLIT_RESERVE 65535u
+#define HTTP_SPLIT_THRESHOLD 64000u
+#define HTTP_SPLIT_RESERVE 65535u
 #endif
 interface IXmlStreamFlusher;
 
@@ -783,7 +788,7 @@ void FlushingStringBuffer::startBlock()
 {
     size32_t len = 0;
     s.clear();
-    if (!isSoap)
+    if (!isHttp)
         append(sizeof(size32_t), (char *) &len);
     rowCount = 0;
     if (isBlocked)
@@ -801,8 +806,8 @@ void FlushingStringBuffer::startBlock()
     // MORE - should probably pre-reserve string at RESULT_FLUSH_THRESHOLD plus a bit
 }
 
-FlushingStringBuffer::FlushingStringBuffer(SafeSocket *_sock, bool _isBlocked, bool _isXml, bool _isRaw, bool _isHttp, const IContextLogger &_logctx) 
-  : sock(_sock), isBlocked(_isBlocked), isXml(_isXml), isRaw(_isRaw), isHttp(_isHttp), logctx(_logctx)
+FlushingStringBuffer::FlushingStringBuffer(SafeSocket *_sock, bool _isBlocked, TextMarkupFormat _mlFmt, bool _isRaw, bool _isHttp, const IContextLogger &_logctx)
+  : sock(_sock), isBlocked(_isBlocked), mlFmt(_mlFmt), isRaw(_isRaw), isHttp(_isHttp), logctx(_logctx)
 {
     sequenceNumber = 0;
     rowCount = 0;
@@ -878,44 +883,44 @@ void FlushingStringBuffer::encodeXML(const char *x, unsigned flags, unsigned len
 void FlushingStringBuffer::flushXML(StringBuffer &current, bool isClosing)
 {
     CriticalBlock b(crit);
-    if (isSoap) // we don't do any chunking for non-SOAP yet
+    if (isHttp) // we don't do any chunking for non-HTTP yet
     {
-        if (isClosing || current.length() > SOAP_SPLIT_THRESHOLD)
+        if (isClosing || current.length() > HTTP_SPLIT_THRESHOLD)
         {
             if (s.length())
             {
                 lengths.append(s.length());
                 queued.append(s.detach());
-                s.ensureCapacity(SOAP_SPLIT_RESERVE);
+                s.ensureCapacity(HTTP_SPLIT_RESERVE);
             }
             lengths.append(current.length());
             queued.append(current.detach());
             if (!isClosing)
-                current.ensureCapacity(SOAP_SPLIT_RESERVE);
+                current.ensureCapacity(HTTP_SPLIT_RESERVE);
         }
     }
     else if (isClosing)
         append(current.length(), current.str());
 }
 
-    void FlushingStringBuffer::flush(bool closing) 
-    {
+void FlushingStringBuffer::flush(bool closing)
+{
     CriticalBlock b(crit);
     if (closing && tail.length())
     {
         s.append(tail);
         tail.clear();
     }
-    if (isSoap)
+    if (isHttp)
     {
         if (!closing)
         {
             unsigned length = s.length();
-            if (length > SOAP_SPLIT_THRESHOLD)
+            if (length > HTTP_SPLIT_THRESHOLD)
             {
                 queued.append(s.detach());
                 lengths.append(length);
-                s.ensureCapacity(SOAP_SPLIT_RESERVE);
+                s.ensureCapacity(HTTP_SPLIT_RESERVE);
             }
         }
     }
@@ -929,7 +934,7 @@ void FlushingStringBuffer::flushXML(StringBuffer &current, bool isClosing)
         if (logctx.queryTraceLevel() > 1)
         {
             if (isBlocked)
-                logctx.CTXLOG("Sending reply: Sending blocked %s data", (isXml)?"xml":"raw");
+                logctx.CTXLOG("Sending reply: Sending blocked %s data", getFormatName(mlFmt));
             else
 #ifdef _DEBUG
                 logctx.CTXLOG("Sending reply length %d: %.1024s", (unsigned) (s.length() - sizeof(size32_t)), s.str()+sizeof(size32_t));
@@ -983,7 +988,7 @@ void FlushingStringBuffer::flushXML(StringBuffer &current, bool isClosing)
 
 void *FlushingStringBuffer::getPayload(size32_t &length)
 {
-    assertex(isSoap);
+    assertex(isHttp);
     CriticalBlock b(crit);
     if (queued.ordinality())
     {
@@ -1008,7 +1013,7 @@ void FlushingStringBuffer::startDataset(const char *elementName, const char *res
         startBlock();
         if (!isBlocked)
         {
-            if (isXml)
+            if (mlFmt==MarkupFmt_XML)
             {
                 s.append('<').append(elementName);
                 if (isSoap && (resultName || (sequence != (unsigned) -1)))
@@ -1041,7 +1046,7 @@ void FlushingStringBuffer::startScalar(const char *resultName, unsigned sequence
     startBlock();
     if (!isBlocked)
     {
-        if (isXml)
+        if (mlFmt==MarkupFmt_XML)
         {
             tail.clear();
             s.append("<Dataset");
@@ -1083,6 +1088,53 @@ void FlushingStringBuffer::incrementRowCount()
     rowCount++;
 }
 
+void FlushingJsonBuffer::encodeXML(const char *x, unsigned flags, unsigned len, bool utf8)
+{
+    StringBuffer t;
+    ::encodeJSON(t, x);
+    append(t.length(), t.str());
+}
+
+void FlushingJsonBuffer::startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend)
+{
+    CriticalBlock b(crit);
+    extend = _extend;
+    if (isEmpty || !extend)
+    {
+        name.clear().append(resultName ? resultName : elementName);
+        sequenceNumber = 0;
+        startBlock();
+        if (!isBlocked)
+        {
+            StringBuffer seqName;
+            if (!resultName || !*resultName)
+                resultName = seqName.appendf("result_%d", sequence+1).str();
+            appendJSONName(s, resultName).append('{');
+            tail.set("}");
+        }
+        isEmpty = false;
+    }
+}
+
+void FlushingJsonBuffer::startScalar(const char *resultName, unsigned sequence)
+{
+    CriticalBlock b(crit);
+    assertex(!s.length());
+    name.set(resultName ? resultName : "Dataset");
+
+    sequenceNumber = 0;
+    startBlock();
+    if (!isBlocked)
+    {
+        StringBuffer seqName;
+        if (!resultName || !*resultName)
+            resultName = seqName.appendf("Result_%d", sequence+1).str();
+        appendJSONName(s, resultName).append('{');
+        appendJSONName(s, "Row").append("[{");
+        appendJSONName(s, resultName);
+        tail.set("}]}");
+    }
+}
 //=====================================================================================================
 
 ClusterWriteHandler::ClusterWriteHandler(char const * _logicalName, char const * _activityType)

common/thorhelper/roxiehelper.hpp

@@ -31,14 +31,35 @@ class THORHELPER_API HttpHelper : public CInterface
 {
 private:
     bool _isHttp;
-    StringBuffer authToken;
+    StringAttr authToken;
+    StringAttr contentType;
+private:
+    inline void setHttpHeaderValue(StringAttr &s, const char *v, bool ignoreExt)
+    {
+        if (!v || !*v)
+            return;
+        unsigned len=0;
+        while (v[len] && v[len]!='\r' && (!ignoreExt || v[len]!=';'))
+            len++;
+        if (len)
+            s.set(v, len);
+    }
 public:
     IMPLEMENT_IINTERFACE;
     HttpHelper() { _isHttp = false; };
     bool isHttp() { return _isHttp; };
     void setIsHttp(bool __isHttp) { _isHttp = __isHttp; };
-    const char *queryAuthToken() { return authToken.str(); };
-    void setAuthToken(const char *_authToken) { authToken.clear().append(_authToken); };
+    const char *queryAuthToken() { return authToken.sget(); };
+    inline void setAuthToken(const char *v)
+    {
+        setHttpHeaderValue(authToken, v, false);
+    };
+    const char *queryContentType() { return contentType.sget(); };
+    inline void setContentType(const char *v)
+    {
+        setHttpHeaderValue(contentType, v, true);
+    };
+    TextMarkupFormat queryContentFormat(){return (strieq(queryContentType(), "application/json")) ? MarkupFmt_JSON : MarkupFmt_XML;}
 };
 
 //========================================================================================= 
@@ -49,7 +70,7 @@ interface SafeSocket : extends IInterface
     virtual size32_t write(const void *buf, size32_t size, bool takeOwnership=false) = 0;
     virtual bool readBlock(MemoryBuffer &ret, unsigned maxBlockSize, unsigned timeout = (unsigned) WAIT_FOREVER) = 0;
     virtual bool readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHttpHelper, bool &, bool &, unsigned maxBlockSize) = 0;
-    virtual void setHttpMode(const char *queryName, bool arrayMode) = 0;
+    virtual void setHttpMode(const char *queryName, bool arrayMode, TextMarkupFormat txtfmt) = 0;
     virtual void setHeartBeat() = 0;
     virtual bool sendHeartBeat(const IContextLogger &logctx) = 0;
     virtual void flush() = 0;
@@ -68,8 +89,9 @@ protected:
     Linked<ISocket> sock;
     bool httpMode;
     bool heartbeat;
-    StringBuffer xmlhead;
-    StringBuffer xmltail;
+    TextMarkupFormat mlFmt;
+    StringBuffer contentHead;
+    StringBuffer contentTail;
     PointerArray queued;
     UnsignedArray lengths;
     unsigned sent;
@@ -85,7 +107,7 @@ public:
     size32_t write(const void *buf, size32_t size, bool takeOwnership=false);
     bool readBlock(MemoryBuffer &ret, unsigned maxBlockSize, unsigned timeout = (unsigned) WAIT_FOREVER);
     bool readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHttpHelper, bool &, bool &, unsigned maxBlockSize);
-    void setHttpMode(const char *queryName, bool arrayMode);
+    void setHttpMode(const char *queryName, bool arrayMode, TextMarkupFormat txtfmt);
     void setHeartBeat();
     bool sendHeartBeat(const IContextLogger &logctx);
     void flush();
@@ -98,7 +120,7 @@ public:
 class THORHELPER_API FlushingStringBuffer : extends CInterface, implements IXmlStreamFlusher, implements IInterface
 {
     // MORE this code is yukky. Overdue for cleanup!
-
+protected:
     SafeSocket *sock;
     StringBuffer name;
     StringBuffer tail;
@@ -113,7 +135,7 @@ class THORHELPER_API FlushingStringBuffer : extends CInterface, implements IXmlS
     bool needsFlush(bool closing);
     void startBlock();
 public:
-    bool isXml;      // controls whether xml elements are output
+    TextMarkupFormat mlFmt;      // controls whether xml/json elements are output
     bool isRaw;      // controls whether output as binary or ascii
     bool isBlocked;
     bool isHttp;
@@ -127,21 +149,42 @@ public:
 
     IMPLEMENT_IINTERFACE;
 
-    FlushingStringBuffer(SafeSocket *_sock, bool _isBlocked, bool _isXml, bool _isRaw, bool _isHttp, const IContextLogger &_logctx);
+    FlushingStringBuffer(SafeSocket *_sock, bool _isBlocked, TextMarkupFormat _mlFmt, bool _isRaw, bool _isHttp, const IContextLogger &_logctx);
     ~FlushingStringBuffer();
-    void append(char data) {append(1, &data);}
-    void append(const char *data);
-    void append(unsigned len, const char *data);
-    void appendf(const char *format, ...) __attribute__((format(printf, 2, 3)));
-    void encodeXML(const char *x, unsigned flags=0, unsigned len=(unsigned)-1, bool utf8=false);
+    virtual void append(char data) {append(1, &data);}
+    virtual void append(const char *data);
+    virtual void append(unsigned len, const char *data);
+    virtual void appendf(const char *format, ...) __attribute__((format(printf, 2, 3)));
+    virtual void encodeXML(const char *x, unsigned flags=0, unsigned len=(unsigned)-1, bool utf8=false);
     virtual void flushXML(StringBuffer &current, bool isClosing);
-    void flush(bool closing) ;
-    void *getPayload(size32_t &length);
+    virtual void flush(bool closing) ;
+    virtual void *getPayload(size32_t &length);
+    virtual void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false);
+    virtual void startScalar(const char *resultName, unsigned sequence);
+    virtual void incrementRowCount();
+};
+
+class THORHELPER_API FlushingJsonBuffer : public FlushingStringBuffer
+{
+public:
+    FlushingJsonBuffer(SafeSocket *_sock, bool _isBlocked, bool _isHttp, const IContextLogger &_logctx) :
+        FlushingStringBuffer(_sock, _isBlocked, MarkupFmt_JSON, false, _isHttp, _logctx)
+    {
+    }
+
+    void encodeXML(const char *x, unsigned flags=0, unsigned len=(unsigned)-1, bool utf8=false);
     void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false);
     void startScalar(const char *resultName, unsigned sequence);
-    void incrementRowCount();
 };
 
+inline const char *getFormatName(TextMarkupFormat fmt)
+{
+    if (fmt==MarkupFmt_XML)
+        return "xml";
+    if (fmt==MarkupFmt_JSON)
+        return "json";
+    return "raw";
+}
 //==============================================================================================================
 
 class THORHELPER_API OwnedRowArray

common/thorhelper/thorxmlwrite.cpp

@@ -304,6 +304,221 @@ void CommonXmlWriter::outputSetAll()
         out.newline();
 }
 
+//=====================================================================================
+
+CommonJsonWriter::CommonJsonWriter(unsigned _flags, unsigned initialIndent, IXmlStreamFlusher *_flusher)
+{
+    flusher = _flusher;
+    flags = _flags;
+    indent = initialIndent;
+    nestLimit = flags & XWFnoindent ? (unsigned) -1 : 0;
+    needDelimiter = false;
+}
+
+CommonJsonWriter::~CommonJsonWriter()
+{
+    flush(true);
+}
+
+CommonJsonWriter & CommonJsonWriter::clear()
+{
+    out.clear();
+    indent = 0;
+    nestLimit = flags & XWFnoindent ? (unsigned) -1 : 0;
+    return *this;
+}
+
+void CommonJsonWriter::checkFormat(bool doDelimit, bool delimitNext, int inc)
+{
+    if (doDelimit && needDelimiter)
+    {
+        if (!out.length()) //new block
+           out.append(',');
+        else
+            delimitJSON(out);
+    }
+    if (!nestLimit)
+    {
+        out.append('\n').pad(indent);
+        if (inc!=0)
+            indent+=inc;
+    }
+    needDelimiter = delimitNext;
+}
+
+void CommonJsonWriter::checkDelimit(int inc)
+{
+    checkFormat(true, true, inc);
+}
+
+void CommonJsonWriter::outputQuoted(const char *text)
+{
+    checkDelimit();
+    appendJSONValue(out, NULL, text);
+}
+
+void CommonJsonWriter::outputString(unsigned len, const char *field, const char *fieldname)
+{
+    if (flags & XWFtrim)
+        len = rtlTrimStrLen(len, field);
+    if ((flags & XWFopt) && (rtlTrimStrLen(len, field) == 0))
+        return;
+    checkDelimit();
+    appendJSONValue(out, fieldname, len, field);
+}
+
+void CommonJsonWriter::outputQString(unsigned len, const char *field, const char *fieldname)
+{
+    MemoryAttr tempBuffer;
+    char * temp;
+    if (len <= 100)
+        temp = (char *)alloca(len);
+    else
+        temp = (char *)tempBuffer.allocate(len);
+    rtlQStrToStr(len, temp, len, field);
+    outputString(len, temp, fieldname);
+}
+
+void CommonJsonWriter::outputBool(bool field, const char *fieldname)
+{
+    checkDelimit();
+    appendJSONValue(out, fieldname, field);
+}
+
+void CommonJsonWriter::outputData(unsigned len, const void *field, const char *fieldname)
+{
+    checkDelimit();
+    appendJSONValue(out, fieldname, len, field);
+}
+
+void CommonJsonWriter::outputInt(__int64 field, const char *fieldname)
+{
+    checkDelimit();
+    appendJSONValue(out, fieldname, field);
+}
+
+void CommonJsonWriter::outputUInt(unsigned __int64 field, const char *fieldname)
+{
+    checkDelimit();
+    appendJSONValue(out, fieldname, field);
+}
+
+void CommonJsonWriter::outputReal(double field, const char *fieldname)
+{
+    checkDelimit();
+    appendJSONValue(out, fieldname, field);
+}
+
+void CommonJsonWriter::outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname)
+{
+    checkDelimit();
+    outputJsonDecimal(field, size, precision, fieldname, out);
+}
+
+void CommonJsonWriter::outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname)
+{
+    checkDelimit();
+    outputJsonUDecimal(field, size, precision, fieldname, out);
+}
+
+void CommonJsonWriter::outputUnicode(unsigned len, const UChar *field, const char *fieldname)
+{
+    if (flags & XWFtrim)
+        len = rtlTrimUnicodeStrLen(len, field);
+    if ((flags & XWFopt) && (rtlTrimUnicodeStrLen(len, field) == 0))
+        return;
+    checkDelimit();
+    outputJsonUnicode(len, field, fieldname, out);
+}
+
+void CommonJsonWriter::outputUtf8(unsigned len, const char *field, const char *fieldname)
+{
+    if (flags & XWFtrim)
+        len = rtlTrimUtf8StrLen(len, field);
+    if ((flags & XWFopt) && (rtlTrimUtf8StrLen(len, field) == 0))
+        return;
+    checkDelimit();
+    appendJSONValue(out, fieldname, len, field);
+}
+
+void CommonJsonWriter::outputBeginArray(const char *fieldname)
+{
+    arrays.append(fieldname);
+    const char * sep = strchr(fieldname, '/');
+    while (sep)
+    {
+        StringAttr leading(fieldname, sep-fieldname);
+        appendJSONName(out, leading).append(" {");
+        fieldname = sep+1;
+        sep = strchr(fieldname, '/');
+    }
+    checkFormat(true, false, 1);
+    appendJSONName(out, fieldname).append('[');
+}
+
+void CommonJsonWriter::outputEndArray(const char *fieldname)
+{
+    arrays.pop();
+    checkFormat(false, true, -1);
+    out.append(']');
+    const char * sep = (fieldname) ? strchr(fieldname, '/') : NULL;
+    while (sep)
+    {
+        out.append('}');
+        sep = strchr(sep+1, '/');
+    }
+}
+
+void CommonJsonWriter::outputBeginNested(const char *fieldname, bool nestChildren)
+{
+    const char *parentArray = (arrays.length()) ? arrays.tos() : NULL;
+    if (parentArray && !streq(parentArray, fieldname))
+        parentArray = NULL;
+    flush(false);
+    checkFormat(true, false, 1);
+    const char * sep = (fieldname) ? strchr(fieldname, '/') : NULL;
+    while (sep)
+    {
+        if (!parentArray)
+        {
+            StringAttr leading(fieldname, sep-fieldname);
+            appendJSONName(out, leading).append("{");
+        }
+        fieldname = sep+1;
+        sep = strchr(fieldname, '/');
+    }
+    if (!parentArray)
+        appendJSONName(out, fieldname);
+    out.append("{");
+    if (!nestChildren && !nestLimit)
+        nestLimit = indent;
+}
+
+void CommonJsonWriter::outputEndNested(const char *fieldname)
+{
+    const char *parentArray = (arrays.length()) ? arrays.tos() : NULL;
+    if (parentArray && !streq(parentArray, fieldname))
+        parentArray = NULL;
+    flush(false);
+    checkFormat(false, true, -1);
+    const char * sep = (fieldname) ? strchr(fieldname, '/') : NULL;
+    while (sep)
+    {
+        if (!parentArray)
+            out.append('}');
+        sep = strchr(sep+1, '/');
+    }
+    out.append("}");
+    if (indent==nestLimit)
+        nestLimit = 0;
+}
+
+void CommonJsonWriter::outputSetAll()
+{
+    flush(false);
+    checkDelimit();
+    appendJSONValue(out, NULL, "All");
+}
 
 //=====================================================================================
 
@@ -635,6 +850,15 @@ CommonXmlWriter * CreateCommonXmlWriter(unsigned _flags, unsigned _initialIndent
 
 //=====================================================================================
 
+IXmlWriter * createIXmlWriter(unsigned _flags, unsigned _initialIndent, IXmlStreamFlusher *_flusher, XMLWriterType xmlType)
+{
+    if (xmlType==WTJSON)
+        return new CommonJsonWriter(_flags, _initialIndent, _flusher);
+    return CreateCommonXmlWriter(_flags, _initialIndent, _flusher, xmlType);
+}
+
+//=====================================================================================
+
 SimpleOutputWriter::SimpleOutputWriter()
 {
     separatorNeeded = false;
@@ -731,15 +955,19 @@ void SimpleOutputWriter::outputUtf8(unsigned len, const char *field, const char
     outputXmlUtf8(len, field, NULL, out);
 }
 
-void SimpleOutputWriter::outputBeginNested(const char *, bool)
+void SimpleOutputWriter::outputBeginNested(const char *s, bool)
 {
+    if (!s || !*s)
+        return;
     outputFieldSeparator();
     out.append('[');
     separatorNeeded = false;
 }
 
-void SimpleOutputWriter::outputEndNested(const char *)
+void SimpleOutputWriter::outputEndNested(const char *s)
 {
+    if (!s || !*s)
+        return;
     out.append(']');
     separatorNeeded = true;
 }

common/thorhelper/thorxmlwrite.hpp

@@ -62,6 +62,8 @@ public:
     virtual void outputUtf8(unsigned len, const char *field, const char *fieldname);
     virtual void outputBeginNested(const char *fieldname, bool nestChildren);
     virtual void outputEndNested(const char *fieldname);
+    virtual void outputBeginArray(const char *fieldname){}; //repeated elements are inline for xml
+    virtual void outputEndArray(const char *fieldname){};
     virtual void outputSetAll();
 
 protected:
@@ -82,6 +84,53 @@ protected:
     bool tagClosed;
 };
 
+class thorhelper_decl CommonJsonWriter : public CInterface, implements IXmlWriter
+{
+public:
+    CommonJsonWriter(unsigned _flags, unsigned initialIndent=0,  IXmlStreamFlusher *_flusher=NULL);
+    ~CommonJsonWriter();
+    IMPLEMENT_IINTERFACE;
+
+    CommonJsonWriter & clear();
+    unsigned length() const                                 { return out.length(); }
+    const char * str() const                                { return out.str(); }
+    void checkDelimit(int inc=0);
+    void checkFormat(bool doDelimit, bool needDelimiter=true, int inc=0);
+
+    virtual void outputQuoted(const char *text);
+    virtual void outputQString(unsigned len, const char *field, const char *fieldname);
+    virtual void outputString(unsigned len, const char *field, const char *fieldname);
+    virtual void outputBool(bool field, const char *fieldname);
+    virtual void outputData(unsigned len, const void *field, const char *fieldname);
+    virtual void outputInt(__int64 field, const char *fieldname);
+    virtual void outputUInt(unsigned __int64 field, const char *fieldname);
+    virtual void outputReal(double field, const char *fieldname);
+    virtual void outputDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
+    virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
+    virtual void outputUnicode(unsigned len, const UChar *field, const char *fieldname);
+    virtual void outputUtf8(unsigned len, const char *field, const char *fieldname);
+    virtual void outputBeginNested(const char *fieldname, bool nestChildren);
+    virtual void outputEndNested(const char *fieldname);
+    virtual void outputBeginArray(const char *fieldname);
+    virtual void outputEndArray(const char *fieldname);
+    virtual void outputSetAll();
+
+protected:
+    inline void flush(bool isClose)
+    {
+        if (flusher)
+            flusher->flushXML(out, isClose);
+    }
+
+protected:
+    IXmlStreamFlusher *flusher;
+    StringArray arrays;
+    StringBuffer out;
+    unsigned flags;
+    unsigned indent;
+    unsigned nestLimit;
+    bool needDelimiter;
+};
 
 //Writes type encoded XML strings  (xsi:type="xsd:string", xsi:type="xsd:boolean" etc)
 class thorhelper_decl CommonEncodedXmlWriter : public CommonXmlWriter
@@ -110,8 +159,9 @@ public:
     virtual void outputData(unsigned len, const void *field, const char *fieldname);
 };
 
-enum XMLWriterType{WTStandard, WTEncoding, WTEncodingData64} ;
+enum XMLWriterType{WTStandard, WTEncoding, WTEncodingData64, WTJSON} ;
 CommonXmlWriter * CreateCommonXmlWriter(unsigned _flags, unsigned initialIndent=0, IXmlStreamFlusher *_flusher=NULL, XMLWriterType xmlType=WTStandard);
+thorhelper_decl IXmlWriter * createIXmlWriter(unsigned _flags, unsigned initialIndent=0, IXmlStreamFlusher *_flusher=NULL, XMLWriterType xmlType=WTStandard);
 
 class thorhelper_decl SimpleOutputWriter : public CInterface, implements IXmlWriter
 {
@@ -139,6 +189,8 @@ public:
     virtual void outputUtf8(unsigned len, const char *field, const char *fieldname);
     virtual void outputBeginNested(const char *fieldname, bool nestChildren);
     virtual void outputEndNested(const char *fieldname);
+    virtual void outputBeginArray(const char *fieldname){}
+    virtual void outputEndArray(const char *fieldname){}
     virtual void outputSetAll();
 
     void newline();

ecl/eclagent/eclagent.cpp

@@ -399,7 +399,7 @@ public:
             sanitizeQuery(queryXml, queryName, sanitizedText, isHTTP, uid, isRequest, isRequestArray);
             DBGLOG("Received debug query %s", sanitizedText.str());
 
-            FlushingStringBuffer response(client, false, true, false, false, queryDummyContextLogger());
+            FlushingStringBuffer response(client, false, MarkupFmt_XML, false, false, queryDummyContextLogger());
             response.startDataset("Debug", NULL, (unsigned) -1);
 
             if (!debugCmdHandler.get())

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -2328,7 +2328,7 @@ bool xppGotoTag(XmlPullParser &xppx, const char *tagname, StartTag &stag)
     return false;
 }
 
-void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query)
+void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query, const char *contentType)
 {
     ISmartSocketFactory *conn = NULL;
     SocketEndpoint ep;
@@ -2351,7 +2351,7 @@ void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, Stri
         ep.getIpText(url).append(':').append(ep.port);
 
         Owned<IHttpClient> httpclient = httpctx->createHttpClient(NULL, url);
-        if (0 > httpclient->sendRequest("POST", "text/xml", req, resp, status))
+        if (0 > httpclient->sendRequest("POST", contentType, req, resp, status))
             throw MakeStringException(-1, "Process cluster communication error: %s", process.str());
     }
     catch (IException *e)
@@ -2360,15 +2360,24 @@ void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, Stri
             conn->setStatus(ep, false);
 
         StringBuffer s;
-        VStringBuffer uri("urn:hpccsystems:ecl:%s", query);
-        resp.set("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
-        resp.append("<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\"><soap:Body>");
-        resp.append('<').append(query).append("Response xmlns='").append(uri).append("'>");
-        resp.append("<Results><Result><Exception><Source>WsEcl</Source>");
-        resp.append("<Code>").append(e->errorCode()).append("</Code>");
-        resp.append("<Message>").append(e->errorMessage(s)).append("</Message>");
-        resp.append("</Exception></Result></Results>");
-        resp.append("</").append(query).append("Response></soap:Body></soap:Envelope>");
+        if (strieq(contentType, "application/json"))
+        {
+            resp.set("{").append("\"").append(query).append("Response\": {\"Results\": {");
+            appendJSONException(resp, e);
+            resp.append("}}}");
+        }
+        else
+        {
+            VStringBuffer uri("urn:hpccsystems:ecl:%s", query);
+            resp.set("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+            resp.append("<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\"><soap:Body>");
+            resp.append('<').append(query).append("Response xmlns='").append(uri).append("'>");
+            resp.append("<Results><Result><Exception><Source>WsEcl</Source>");
+            resp.append("<Code>").append(e->errorCode()).append("</Code>");
+            resp.append("<Message>").append(e->errorMessage(s)).append("</Message>");
+            resp.append("</Exception></Result></Results>");
+            resp.append("</").append(query).append("Response></soap:Body></soap:Envelope>");
+        }
         e->Release();
     }
 }
@@ -2849,48 +2858,45 @@ void CWsEclBinding::handleJSONPost(CHttpRequest *request, CHttpResponse *respons
         }
 
         WsEclWuInfo wsinfo(wuid.str(), queryset.str(), queryname.str(), ctx->queryUserId(), ctx->queryPassword());
-
-        StringBuffer content(request->queryContent());
-        StringBuffer status;
-        StringBuffer soapfromjson;
-        soapfromjson.append(
-            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
-            "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
-              " xmlns:SOAP-ENC=\"http://schemas.xmlsoap.org/soap/encoding/\">"
-                " <soap:Body>"
-            );
-        createPTreeFromJsonString(content.str(), false, soapfromjson, "Request");
-        soapfromjson.append("</soap:Body></soap:Envelope>");
-        DBGLOG("soap from json req: %s", soapfromjson.str());
-
-        StringBuffer soapresp;
-
         SCMStringBuffer clustertype;
         wsinfo.wu->getDebugValue("targetclustertype", clustertype);
 
-        unsigned xmlflags = WWV_ADD_SOAP | WWV_ADD_RESULTS_TAG | WWV_ADD_RESPONSE_TAG | WWV_INCL_NAMESPACES | WWV_INCL_GENERATED_NAMESPACES;
-        if (ctx->queryRequestParameters()->hasProp("display"))
-            xmlflags |= WWV_USE_DISPLAY_XSLT;
-        if (streq(action.str(), "expanded"))
-            xmlflags |= WWV_CDATA_SCHEMAS;
-        else
-            xmlflags |= WWV_OMIT_SCHEMAS;
-
+        StringBuffer content(request->queryContent());
+        StringBuffer status;
         if (strieq(clustertype.str(), "roxie"))
         {
             StringBuffer output;
-            sendRoxieRequest(wsinfo.qsetname.get(), soapfromjson, output, status, wsinfo.queryname);
-            Owned<IWuWebView> web = createWuWebView(*wsinfo.wu, NULL, getCFD(), true);
-            if (web.get())
-                web->expandResults(output.str(), soapresp, xmlflags);
+            DBGLOG("json req: %s", content.str());
+            sendRoxieRequest(wsinfo.qsetname.get(), content, jsonresp, status, wsinfo.queryname, "application/json");
+            DBGLOG("json resp: %s", jsonresp.str());
         }
         else
         {
+            StringBuffer soapfromjson;
+            soapfromjson.append(
+                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+                "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
+                  " xmlns:SOAP-ENC=\"http://schemas.xmlsoap.org/soap/encoding/\">"
+                    " <soap:Body>"
+                );
+            createPTreeFromJsonString(content.str(), false, soapfromjson, "Request");
+            soapfromjson.append("</soap:Body></soap:Envelope>");
+            DBGLOG("soap from json req: %s", soapfromjson.str());
+
+            StringBuffer soapresp;
+            unsigned xmlflags = WWV_ADD_SOAP | WWV_ADD_RESULTS_TAG | WWV_ADD_RESPONSE_TAG | WWV_INCL_NAMESPACES | WWV_INCL_GENERATED_NAMESPACES;
+            if (ctx->queryRequestParameters()->hasProp("display"))
+                xmlflags |= WWV_USE_DISPLAY_XSLT;
+            if (streq(action.str(), "expanded"))
+                xmlflags |= WWV_CDATA_SCHEMAS;
+            else
+                xmlflags |= WWV_OMIT_SCHEMAS;
+
             submitWsEclWorkunit(*ctx, wsinfo, soapfromjson.str(), soapresp, xmlflags);
+            DBGLOG("HandleSoapRequest response: %s", soapresp.str());
+            getWsEclJsonResponse(jsonresp, *ctx, request, soapresp.str(), wsinfo);
         }
 
-        DBGLOG("HandleSoapRequest response: %s", soapresp.str());
-        getWsEclJsonResponse(jsonresp, *ctx, request, soapresp.str(), wsinfo);
     }
     catch (IException *e)
     {

esp/services/ws_ecl/ws_ecl_service.hpp

@@ -194,7 +194,7 @@ public:
     void getWsEclJsonRequest(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags);
     void getWsEclJsonResponse(StringBuffer& jsonmsg, IEspContext &context, CHttpRequest *request, const char *xml, WsEclWuInfo &wsinfo);
     
-    void sendRoxieRequest(const char *process, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query);
+    void sendRoxieRequest(const char *process, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query, const char *contentType="text/xml");
 };
 
 #endif //_WS_ECL_SERVICE_HPP__

roxie/ccd/ccdcontext.cpp

@@ -1809,12 +1809,9 @@ class CRoxieServerContext : public CSlaveContext, implements IRoxieServerContext
     CriticalSection daliUpdateCrit;
     Owned<IRoxiePackage> dynamicPackage;
 
-    bool isXml;
+    TextMarkupFormat mlFmt;
     bool isRaw;
-    bool isBlocked;
-    bool isHttp;
     bool sendHeartBeats;
-    bool trim;
     unsigned warnTimeLimit;
     unsigned lastSocketCheckTime;
     unsigned lastHeartBeat;
@@ -1822,6 +1819,9 @@ class CRoxieServerContext : public CSlaveContext, implements IRoxieServerContext
 protected:
     Owned<WorkflowMachine> workflow;
     SafeSocket *client;
+    bool isBlocked;
+    bool isHttp;
+    bool trim;
 
     void doPostProcess()
     {
@@ -1838,7 +1838,7 @@ protected:
 
         if (probeQuery)
         {
-            FlushingStringBuffer response(client, isBlocked, true, false, isHttp, *this);
+            FlushingStringBuffer response(client, isBlocked, MarkupFmt_XML, false, isHttp, *this);
 
             // create output stream
             response.startDataset("_Probe", NULL, (unsigned) -1);  // initialize it
@@ -1866,7 +1866,7 @@ protected:
     {
         client = NULL;
         totSlavesReplyLen = 0;
-        isXml = true;
+        mlFmt = MarkupFmt_XML;
         isRaw = false;
         isBlocked = false;
         isHttp = false;
@@ -1962,13 +1962,13 @@ public:
         startWorkUnit();
     }
 
-    CRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, bool _isXml, bool _isRaw, bool _isBlocked, HttpHelper &httpHelper, bool _trim, unsigned _priority, const IRoxieContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags)
+    CRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, TextMarkupFormat _mlFmt, bool _isRaw, bool _isBlocked, HttpHelper &httpHelper, bool _trim, unsigned _priority, const IRoxieContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags)
         : CSlaveContext(_factory, _logctx, 0, 0, NULL, false, false), serverQueryFactory(_factory)
     {
         init();
         context.set(_context);
         client = &_client;
-        isXml = _isXml;
+        mlFmt = _mlFmt;
         isRaw = _isRaw;
         isBlocked = _isBlocked;
         isHttp = httpHelper.isHttp();
@@ -2168,7 +2168,7 @@ public:
         FlushingStringBuffer *result = resultMap.item(sequence);
         if (!result)
         {
-            result = new FlushingStringBuffer(client, isBlocked, isXml, isRaw, isHttp, *this);
+            result = new FlushingStringBuffer(client, isBlocked, mlFmt, isRaw, isHttp, *this);
             result->isSoap = isHttp;
             result->trim = trim;
             result->queryName.set(context->queryName());
@@ -2393,12 +2393,19 @@ public:
                 r->startScalar(name, sequence);
                 if (isRaw)
                     r->append(len, (char *)data);
-                else if (isXml)
+                else if (mlFmt==MarkupFmt_XML)
                 {
                     assertex(transformer);
-                    CommonXmlWriter xmlwrite(getXmlFlags()|XWFnoindent, 0);
-                    transformer->toXML(isAll, len, (byte *)data, xmlwrite);
-                    r->append(xmlwrite.str());
+                    CommonXmlWriter writer(getXmlFlags()|XWFnoindent, 0);
+                    transformer->toXML(isAll, len, (byte *)data, writer);
+                    r->append(writer.str());
+                }
+                else if (mlFmt==MarkupFmt_JSON)
+                {
+                    assertex(transformer);
+                    CommonJsonWriter writer(getXmlFlags()|XWFnoindent, 0);
+                    transformer->toXML(isAll, len, (byte *)data, writer);
+                    r->append(writer.str());
                 }
                 else
                 {
@@ -2916,7 +2923,7 @@ private:
 
 public:
     CSoapRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, HttpHelper &httpHelper, unsigned _priority, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags)
-        : CRoxieServerContext(_context, _factory, _client, true, false, false, httpHelper, true, _priority, _logctx, xmlReadFlags)
+        : CRoxieServerContext(_context, _factory, _client, MarkupFmt_XML, false, false, httpHelper, true, _priority, _logctx, xmlReadFlags)
     {
         queryName.set(_context->queryName());
     }
@@ -2968,12 +2975,92 @@ public:
     }
 };
 
-IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, const IQueryFactory *factory, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, unsigned priority, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags)
+class CJsonRoxieServerContext : public CRoxieServerContext
+{
+private:
+    StringAttr queryName;
+
+public:
+    CJsonRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, HttpHelper &httpHelper, unsigned _priority, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags)
+        : CRoxieServerContext(_context, _factory, _client, MarkupFmt_JSON, false, false, httpHelper, true, _priority, _logctx, xmlReadFlags)
+    {
+        queryName.set(_context->queryName());
+    }
+
+    virtual void process()
+    {
+        EclProcessFactory pf = (EclProcessFactory) factory->queryDll()->getEntry("createProcess");
+        Owned<IEclProcess> p = pf();
+        if (workflow)
+            workflow->perform(this, p);
+        else
+            p->perform(this, 0);
+    }
+
+    virtual void flush(unsigned seqNo)
+    {
+        CriticalBlock b(resultsCrit);
+        CriticalBlock b1(client->queryCrit());
+
+        StringBuffer responseHead, responseTail;
+        appendfJSONName(responseHead, "%sResponse", queryName.get()).append(" {");
+        appendJSONValue(responseHead, "sequence", seqNo);
+        appendJSONName(responseHead, "Results").append(" {");
+
+        unsigned len = responseHead.length();
+        client->write(responseHead.detach(), len, true);
+
+        ForEachItemIn(seq, resultMap)
+        {
+            FlushingStringBuffer *result = resultMap.item(seq);
+            if (result)
+            {
+                result->flush(true);
+                for(;;)
+                {
+                    size32_t length;
+                    void *payload = result->getPayload(length);
+                    if (!length)
+                        break;
+                    client->write(payload, length, true);
+                }
+            }
+        }
+
+        responseTail.append("}}");
+        len = responseTail.length();
+        client->write(responseTail.detach(), len, true);
+    }
+
+    virtual FlushingStringBuffer *queryResult(unsigned sequence)
+    {
+        if (!client && workUnit)
+            return NULL;    // when outputting to workunit only, don't output anything to stdout
+        CriticalBlock procedure(resultsCrit);
+        while (!resultMap.isItem(sequence))
+            resultMap.append(NULL);
+        FlushingStringBuffer *result = resultMap.item(sequence);
+        if (!result)
+        {
+            result = new FlushingJsonBuffer(client, isBlocked, isHttp, *this);
+            result->trim = trim;
+            result->queryName.set(context->queryName());
+            resultMap.replace(result, sequence);
+        }
+        return result;
+    }
+};
+
+IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, const IQueryFactory *factory, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, unsigned priority, const IRoxieContextLogger &_logctx, PTreeReaderOptions readFlags)
 {
     if (httpHelper.isHttp())
-        return new CSoapRoxieServerContext(context, factory, client, httpHelper, priority, _logctx, xmlReadFlags);
+    {
+        if (httpHelper.queryContentFormat()==MarkupFmt_JSON)
+            return new CJsonRoxieServerContext(context, factory, client, httpHelper, priority, _logctx, readFlags);
+        return new CSoapRoxieServerContext(context, factory, client, httpHelper, priority, _logctx, readFlags);
+    }
     else
-        return new CRoxieServerContext(context, factory, client, isXml, isRaw, isBlocked, httpHelper, trim, priority, _logctx, xmlReadFlags);
+        return new CRoxieServerContext(context, factory, client, isXml ? MarkupFmt_XML : MarkupFmt_Unknown, isRaw, isBlocked, httpHelper, trim, priority, _logctx, readFlags);
 }
 
 IRoxieServerContext *createOnceServerContext(const IQueryFactory *factory, const IRoxieContextLogger &_logctx)

roxie/ccd/ccdlistener.cpp

@@ -89,9 +89,46 @@ static void sendSoapException(SafeSocket &client, IException *E, const char *que
 #endif
 }
 
+static void sendJsonException(SafeSocket &client, IException *E, const char *queryName)
+{
+    try
+    {
+        if (!queryName)
+            queryName = "Unknown"; // Exceptions when parsing query XML can leave queryName unset/unknowable....
+
+        StringBuffer response("{");
+        appendfJSONName(response, "%sResponse", queryName).append(" {");
+        appendJSONName(response, "Results").append(" {");
+        appendJSONName(response, "Exception").append(" [{");
+        appendJSONValue(response, "Source", "Roxie");
+        appendJSONValue(response, "Code", E->errorCode());
+        StringBuffer s;
+        appendJSONValue(response, "Message", E->errorMessage(s).str());
+        response.append("}]}}}");
+        client.write(response.str(), response.length());
+    }
+    catch(IException *EE)
+    {
+        StringBuffer error("While reporting exception: ");
+        DBGLOG("%s", EE->errorMessage(error).str());
+        EE->Release();
+    }
+#ifndef _DEBUG
+    catch(...) {}
+#endif
+}
+
+static void sendHttpException(SafeSocket &client, TextMarkupFormat fmt, IException *E, const char *queryName)
+{
+    if (fmt==MarkupFmt_JSON)
+        sendJsonException(client, E, queryName);
+    else
+        sendSoapException(client, E, queryName);
+}
+
 //================================================================================================================
 
-class CSoapRequestAsyncFor : public CInterface, public CAsyncFor
+class CHttpRequestAsyncFor : public CInterface, public CAsyncFor
 {
 private:
     const char *queryName, *queryText;
@@ -106,7 +143,7 @@ private:
     CriticalSection crit;
 
 public:
-    CSoapRequestAsyncFor(const char *_queryName, IQueryFactory *_f, IArrayOf<IPropertyTree> &_requestArray, SafeSocket &_client, HttpHelper &_httpHelper, unsigned &_memused, unsigned &_slaveReplyLen, const char *_queryText, const IRoxieContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags) :
+    CHttpRequestAsyncFor(const char *_queryName, IQueryFactory *_f, IArrayOf<IPropertyTree> &_requestArray, SafeSocket &_client, HttpHelper &_httpHelper, unsigned &_memused, unsigned &_slaveReplyLen, const char *_queryText, const IRoxieContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags) :
       f(_f), requestArray(_requestArray), client(_client), httpHelper(_httpHelper), memused(_memused), slaveReplyLen(_slaveReplyLen), logctx(_logctx), xmlReadFlags(_xmlReadFlags)
     {
         queryName = _queryName;
@@ -122,7 +159,7 @@ public:
         StringBuffer error("EXCEPTION: ");
         E->errorMessage(error);
         DBGLOG("%s", error.str());
-        sendSoapException(client, E, queryName);
+        sendHttpException(client, httpHelper.queryContentFormat(), E, queryName);
         E->Release();
     }
 
@@ -131,7 +168,7 @@ public:
         try
         {
             IPropertyTree &request = requestArray.item(idx);
-            Owned<IRoxieServerContext> ctx = f->createContext(&request, client, true, false, false, httpHelper, true, logctx, xmlReadFlags);
+            Owned<IRoxieServerContext> ctx = f->createContext(&request, client, httpHelper.queryContentFormat(), false, false, httpHelper, true, logctx, xmlReadFlags);
             ctx->process();
             ctx->flush(idx);
             CriticalBlock b(crit);
@@ -1246,16 +1283,31 @@ private:
         }
     }
 
-    void sanitizeQuery(Owned<IPropertyTree> &queryXML, StringAttr &queryName, StringBuffer &saniText, bool isHTTP, const char *&uid, bool &isRequest, bool &isRequestArray, bool &isBlind, bool &isDebug)
+    void sanitizeQuery(Owned<IPropertyTree> &queryXML, StringAttr &queryName, StringBuffer &saniText, HttpHelper &httpHelper, const char *&uid, bool &isRequest, bool &isRequestArray, bool &isBlind, bool &isDebug)
     {
         if (queryXML)
         {
             queryName.set(queryXML->queryName());
             isRequest = false;
             isRequestArray = false;
-            if (isHTTP)
+            if (httpHelper.isHttp())
             {
-                if (stricmp(queryName, "envelope") == 0)
+                if (httpHelper.queryContentFormat()==MarkupFmt_JSON)
+                {
+                    if (strieq(queryName, "__object__"))
+                    {
+                        queryXML.setown(queryXML->getPropTree("*[1]"));
+                        queryName.set(queryXML->queryName());
+                        isRequest = true;
+                        if (!queryXML)
+                            throw MakeStringException(ROXIE_DATA_ERROR, "Malformed JSON request (missing Body)");
+                    }
+                    else if (strieq(queryName, "__array__"))
+                        throw MakeStringException(ROXIE_DATA_ERROR, "JSON request array not implemented");
+                    else
+                        throw MakeStringException(ROXIE_DATA_ERROR, "Malformed JSON request");
+                }
+                else if (strieq(queryName, "envelope"))
                 {
                     queryXML.setown(queryXML->getPropTree("Body/*"));
                     if (!queryXML)
@@ -1297,7 +1349,13 @@ private:
         else
             throw MakeStringException(ROXIE_DATA_ERROR, "Malformed request");
     }
-
+    void parseQueryPTFromString(Owned<IPropertyTree> &queryPT, HttpHelper &httpHelper, const char *text, PTreeReaderOptions options)
+    {
+        if (strieq(httpHelper.queryContentType(), "application/json"))
+            queryPT.setown(createPTreeFromJSONString(text, ipt_caseInsensitive, options));
+        else
+            queryPT.setown(createPTreeFromXMLString(text, ipt_caseInsensitive, options));
+    }
     void doMain(const char *runQuery)
     {
         StringBuffer rawText(runQuery);
@@ -1347,7 +1405,7 @@ readAnother:
             return;
         }
 
-        bool isXml = true;
+        TextMarkupFormat mlFmt = MarkupFmt_XML;
         bool isRaw = false;
         bool isHTTP = httpHelper.isHttp();
         bool isBlocked = false;
@@ -1371,7 +1429,7 @@ readAnother:
             {
                 if (logctx.queryTraceLevel() > 8)
                     logctx.CTXLOG("Got lock request %s", rawText.str());
-                FlushingStringBuffer response(client, false, true, false, false, logctx);
+                FlushingStringBuffer response(client, false, MarkupFmt_XML, false, false, logctx);
                 response.startDataset("Control", NULL, (unsigned) -1);
                 if (!cascade)
                     cascade.setown(new CascadeManager(logctx));
@@ -1391,7 +1449,7 @@ readAnother:
             {
                 if (logctx.queryTraceLevel() > 8)
                     logctx.CTXLOG("Got childlock request %s", rawText.str());
-                FlushingStringBuffer response(client, false, true, false, false, logctx);
+                FlushingStringBuffer response(client, false, MarkupFmt_XML, false, false, logctx);
                 response.startDataset("Control", NULL, (unsigned) -1);
                 if (!cascade)
                     cascade.setown(new CascadeManager(logctx));
@@ -1413,7 +1471,7 @@ readAnother:
                 queryXML.clear();
                 bool doControlQuery = true;
 
-                FlushingStringBuffer response(client, false, true, false, isHTTP, logctx);
+                FlushingStringBuffer response(client, false, MarkupFmt_XML, false, isHTTP, logctx);
                 response.startDataset("Control", NULL, (unsigned) -1);
 
                 if (strnicmp(rawText.str(), "<control:aclupdate", 18)==0 && !isalpha(rawText.charAt(18)))
@@ -1472,7 +1530,7 @@ readAnother:
             {
                 try
                 {
-                    queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)(defaultXmlReadFlags | ptr_ignoreNameSpaces)));
+                    parseQueryPTFromString(queryXml, httpHelper, rawText.str(), (PTreeReaderOptions)(defaultXmlReadFlags | ptr_ignoreNameSpaces));
                 }
                 catch (IException *E)
                 {
@@ -1485,7 +1543,7 @@ readAnother:
                 bool isBlind = false;
                 bool isDebug = false;
 
-                sanitizeQuery(queryXml, queryName, sanitizedText, isHTTP, uid, isRequest, isRequestArray, isBlind, isDebug);
+                sanitizeQuery(queryXml, queryName, sanitizedText, httpHelper, uid, isRequest, isRequestArray, isBlind, isDebug);
                 pool->checkAccess(peer, queryName, sanitizedText, isBlind);
                 if (isDebug)
                 {
@@ -1518,7 +1576,7 @@ readAnother:
                         if (!debugCmdHandler.get())
                             debugCmdHandler.setown(new CDebugCommandHandler);
                     }
-                    FlushingStringBuffer response(client, false, true, false, isHTTP, logctx);
+                    FlushingStringBuffer response(client, false, MarkupFmt_XML, false, isHTTP, logctx);
                     response.startDataset("Debug", NULL, (unsigned) -1);
                     debugCmdHandler->doDebugCommand(queryXml, debuggerContext, response);
                 }
@@ -1545,7 +1603,7 @@ readAnother:
                     {
                         queryFactory.setown(globalPackageSetManager->getQuery(queryName, logctx));
                         if (isHTTP)
-                            client->setHttpMode(queryName, isRequestArray);
+                            client->setHttpMode(queryName, isRequestArray, httpHelper.queryContentFormat());
                         if (queryFactory)
                         {
                             bool stripWhitespace = queryFactory->getDebugValueBool("stripWhitespaceFromStoredDataset", 0 != (ptr_ignoreWhiteSpace & defaultXmlReadFlags));
@@ -1555,8 +1613,8 @@ readAnother:
                             if (xmlReadFlags != defaultXmlReadFlags)
                             {
                                 // we need to reparse input xml, as global whitespace setting has been overridden
-                                queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)(xmlReadFlags|ptr_ignoreNameSpaces)));
-                                sanitizeQuery(queryXml, queryName, sanitizedText, isHTTP, uid, isRequest, isRequestArray, isBlind, isDebug);
+                                parseQueryPTFromString(queryXml, httpHelper, rawText.str(), (PTreeReaderOptions)(xmlReadFlags | ptr_ignoreNameSpaces));
+                                sanitizeQuery(queryXml, queryName, sanitizedText, httpHelper, uid, isRequest, isRequestArray, isBlind, isDebug);
                             }
                             IArrayOf<IPropertyTree> requestArray;
                             if (isHTTP)
@@ -1598,7 +1656,7 @@ readAnother:
                                     if (stricmp(format, "raw") == 0)
                                     {
                                         isRaw = true;
-                                        isXml = false;
+                                        mlFmt = MarkupFmt_Unknown;
                                         isBlocked = (client != NULL);
                                     }
                                     else if (stricmp(format, "bxml") == 0)
@@ -1608,7 +1666,7 @@ readAnother:
                                     else if (stricmp(format, "ascii") == 0)
                                     {
                                         isRaw = false;
-                                        isXml = false;
+                                        mlFmt = MarkupFmt_Unknown;
                                     }
                                     else if (stricmp(format, "xml") != 0) // xml is the default
                                         throw MakeStringException(ROXIE_INVALID_INPUT, "Unsupported format specified: %s", format);
@@ -1629,12 +1687,12 @@ readAnother:
                             combinedQueryStats.noteActive();
                             if (isHTTP)
                             {
-                                CSoapRequestAsyncFor af(queryName, queryFactory, requestArray, *client, httpHelper, memused, slavesReplyLen, sanitizedText, logctx, xmlReadFlags);
+                                CHttpRequestAsyncFor af(queryName, queryFactory, requestArray, *client, httpHelper, memused, slavesReplyLen, sanitizedText, logctx, xmlReadFlags);
                                 af.For(requestArray.length(), numRequestArrayThreads);
                             }
                             else
                             {
-                                Owned<IRoxieServerContext> ctx = queryFactory->createContext(queryXml, *client, isXml, isRaw, isBlocked, httpHelper, trim, logctx, xmlReadFlags);
+                                Owned<IRoxieServerContext> ctx = queryFactory->createContext(queryXml, *client, mlFmt, isRaw, isBlocked, httpHelper, trim, logctx, xmlReadFlags);
                                 if (client && !ctx->outputResultsToSocket())
                                 {
                                     unsigned replyLen = 0;
@@ -1706,7 +1764,7 @@ readAnother:
             if (client)
             {
                 if (isHTTP)
-                    sendSoapException(*client, E, queryName);
+                    sendHttpException(*client, mlFmt, E, queryName);
                 else
                     client->sendException("Roxie", code, error.str(), isBlocked, logctx);
             }
@@ -1782,7 +1840,7 @@ readAnother:
                 {
                     if (logctx.intercept)
                     {
-                        FlushingStringBuffer response(client, isBlocked, isXml, isRaw, false, logctx);
+                        FlushingStringBuffer response(client, isBlocked, mlFmt, isRaw, false, logctx);
                         response.startDataset("Tracing", NULL, (unsigned) -1);
                         logctx.outputXML(response);
                     }

roxie/ccd/ccdquery.cpp

@@ -1193,7 +1193,7 @@ public:
         throwUnexpected();   // only implemented in derived slave class
     }
 
-    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags) const
+    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, TextMarkupFormat mlFmt, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags) const
     {
         throwUnexpected();   // only implemented in derived server class
     }
@@ -1312,10 +1312,10 @@ public:
         return activities;
     }
 
-    virtual IRoxieServerContext *createContext(IPropertyTree *context, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags) const
+    virtual IRoxieServerContext *createContext(IPropertyTree *context, SafeSocket &client, TextMarkupFormat mlFmt, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags) const
     {
         checkSuspended();
-        return createRoxieServerContext(context, this, client, isXml, isRaw, isBlocked, httpHelper, trim, priority, _logctx, _xmlReadFlags);
+        return createRoxieServerContext(context, this, client, mlFmt, isRaw, isBlocked, httpHelper, trim, priority, _logctx, _xmlReadFlags);
     }
 
     virtual IRoxieServerContext *createContext(IConstWorkUnit *wu, const IRoxieContextLogger &_logctx) const

roxie/ccd/ccdquery.hpp

@@ -115,7 +115,7 @@ interface IQueryFactory : extends IInterface
     virtual int getDebugValueInt(const char * propname, int defVal) const = 0;
     virtual bool getDebugValueBool(const char * propname, bool defVal) const = 0;
 
-    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags) const = 0;
+    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, TextMarkupFormat mlFmt, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags) const = 0;
     virtual IRoxieServerContext *createContext(IConstWorkUnit *wu, const IRoxieContextLogger &_logctx) const = 0;
     virtual void noteQuery(time_t startTime, bool failed, unsigned elapsed, unsigned memused, unsigned slavesReplyLen, unsigned bytesOut) = 0;
     virtual IPropertyTree *getQueryStats(time_t from, time_t to) = 0;

roxie/ccd/ccdserver.cpp

@@ -19114,11 +19114,18 @@ public:
         if (!meta.queryOriginal()) // this is a bit of a hack - don't know why no meta on an output....
             meta.set(input->queryOutputMeta());
         Owned<IOutputRowSerializer> rowSerializer;
+        Owned<IXmlWriter> writer;
         if ((int) sequence >= 0)
         {
             response = serverContext->queryResult(sequence);
             if (response)
                 response->startDataset("Dataset", helper.queryName(), sequence, (helper.getFlags() & POFextend) != 0);
+            if (response->mlFmt==MarkupFmt_XML || response->mlFmt==MarkupFmt_JSON)
+            {
+                writer.setown(createIXmlWriter(serverContext->getXmlFlags(), 1, response, (response->mlFmt==MarkupFmt_JSON) ? WTJSON : WTStandard));
+                writer->outputBeginArray("Row");
+            }
+
         }
         if (serverContext->outputResultsToWorkUnit()||(response && response->isRaw))
         {
@@ -19173,12 +19180,11 @@ public:
                     rowSerializer->serialize(serializerTarget, (const byte *) row);
                     response->append(rowbuff.length(), rowbuff.toByteArray());
                 }
-                else if (response->isXml)
+                else if (writer)
                 {
-                    CommonXmlWriter xmlwrite(serverContext->getXmlFlags(), 1, response);
-                    xmlwrite.outputBeginNested("Row", false);
-                    helper.serializeXml((byte *) row, xmlwrite);
-                    xmlwrite.outputEndNested("Row");
+                    writer->outputBeginNested("Row", false);
+                    helper.serializeXml((byte *) row, *writer);
+                    writer->outputEndNested("Row");
                 }
                 else
                 {
@@ -19192,6 +19198,8 @@ public:
             }
             ReleaseRoxieRow(row);
         }
+        if (writer)
+            writer->outputEndArray("Row");
         if (saveInContext)
             serverContext->appendResultDeserialized(storedName, sequence, builder.getcount(), builder.linkrows(), (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
         if (serverContext->outputResultsToWorkUnit())

rtl/eclrtl/eclrtl.hpp

@@ -520,6 +520,10 @@ ECLRTL_API void outputXmlAttrUDecimal(const void *field, unsigned size, unsigned
 ECLRTL_API void outputXmlAttrUnicode(unsigned len, const UChar *field, const char *fieldname, StringBuffer &out);
 ECLRTL_API void outputXmlAttrUtf8(unsigned len, const char *field, const char *fieldname, StringBuffer &out);
 
+ECLRTL_API void outputJsonDecimal(const void *field, unsigned digits, unsigned precision, const char *fieldname, StringBuffer &out);
+ECLRTL_API void outputJsonUDecimal(const void *field, unsigned digits, unsigned precision, const char *fieldname, StringBuffer &out);
+ECLRTL_API void outputJsonUnicode(unsigned len, const UChar *field, const char *fieldname, StringBuffer &out);
+
 ECLRTL_API void deserializeRaw(unsigned size, void *record, MemoryBuffer & in);
 ECLRTL_API void deserializeDataX(size32_t & len, void * & data, MemoryBuffer &in);
 ECLRTL_API void deserializeStringX(size32_t & len, char * & data, MemoryBuffer &in);

rtl/eclrtl/rtlfield.cpp

@@ -715,6 +715,9 @@ size32_t RtlSetTypeInfo::toXML(const byte * self, const byte * selfrow, const Rt
         target.outputBeginNested(outerTag, false);
     }
 
+    const char *innerPath = queryXPath(field);
+    target.outputBeginArray(innerPath);
+
     if (*(bool *)self)
         target.outputSetAll();
     else
@@ -726,6 +729,7 @@ size32_t RtlSetTypeInfo::toXML(const byte * self, const byte * selfrow, const Rt
         }
     }
 
+    target.outputEndArray(innerPath);
     if (outerTag)
         target.outputEndNested(outerTag);
     return max;
@@ -815,6 +819,9 @@ size32_t RtlDatasetTypeInfo::toXML(const byte * self, const byte * selfrow, cons
         target.outputBeginNested(outerTag, false);
     }
 
+    const char *innerPath = queryXPath(field);
+    target.outputBeginArray(innerPath);
+
     unsigned thisSize;
     if (isLinkCounted())
     {
@@ -840,6 +847,7 @@ size32_t RtlDatasetTypeInfo::toXML(const byte * self, const byte * selfrow, cons
         thisSize = max;
     }
 
+    target.outputEndArray(innerPath);
     if (outerTag)
         target.outputEndNested(outerTag);
 
@@ -889,6 +897,9 @@ size32_t RtlDictionaryTypeInfo::toXML(const byte * self, const byte * selfrow, c
         target.outputBeginNested(outerTag, false);
     }
 
+    const char *innerPath = queryXPath(field);
+    target.outputBeginArray(innerPath);
+
     unsigned thisSize;
     if (isLinkCounted())
     {
@@ -908,6 +919,7 @@ size32_t RtlDictionaryTypeInfo::toXML(const byte * self, const byte * selfrow, c
         UNIMPLEMENTED;
     }
 
+    target.outputEndArray(innerPath);
     if (outerTag)
         target.outputEndNested(outerTag);
 

rtl/eclrtl/rtlxml.cpp

@@ -239,3 +239,44 @@ void outputXmlAttrUtf8(unsigned len, const char *field, const char *fieldname, S
 }
 
 //---------------------------------------------------------------------------
+
+void outputJsonUnicode(unsigned len, const UChar *field, const char *fieldname, StringBuffer &out)
+{
+    char * buff = 0;
+    unsigned bufflen = 0;
+    rtlUnicodeToCodepageX(bufflen, buff, len, field, "utf-8");
+    appendJSONValue(out, fieldname, bufflen, buff); // output as UTF-8
+    rtlFree(buff);
+}
+
+void outputJsonDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname, StringBuffer &out)
+{
+    char dec[50];
+    appendJSONNameOrDelimit(out, fieldname);
+    DecLock();
+    if (DecValid(true, size*2-1, field))
+    {
+        DecPushDecimal(field, size, precision);
+        DecPopCString(sizeof(dec), dec);
+        const char *finger = dec;
+        while(isspace(*finger)) finger++;
+        out.append(finger);
+    }
+    DecUnlock();
+}
+
+void outputJsonUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname, StringBuffer &out)
+{
+    char dec[50];
+    appendJSONNameOrDelimit(out, fieldname);
+    DecLock();
+    if (DecValid(false, size*2, field))
+    {
+        DecPushUDecimal(field, size, precision);
+        DecPopCString(sizeof(dec), dec);
+        const char *finger = dec;
+        while(isspace(*finger)) finger++;
+        out.append(finger);
+    }
+    DecUnlock();
+}

rtl/include/eclhelper.hpp

@@ -167,6 +167,8 @@ public:
     virtual void outputEndNested(const char *fieldname) = 0;
     virtual void outputSetAll() = 0;
     virtual void outputUtf8(unsigned len, const char *field, const char *fieldname) = 0;
+    virtual void outputBeginArray(const char *fieldname) = 0;
+    virtual void outputEndArray(const char *fieldname) = 0;
     inline void outputCString(const char *field, const char *fieldname) { outputString((size32_t)strlen(field), field, fieldname); }
 };
 

system/jlib/jptree.cpp

@@ -6368,12 +6368,12 @@ public:
         else
         {
             if ('{' == nextChar)
-                readObject("__root__");
+                readObject("__object__");
             else if ('[' == nextChar)
             {
-                iEvent->beginNode("__root__", curOffset);
+                iEvent->beginNode("__array__", curOffset);
                 readArray("__item__");
-                iEvent->endNode("__root__", 0, "", false, curOffset);
+                iEvent->endNode("__array__", 0, "", false, curOffset);
             }
             else
                 error("expected '{' or '['");

system/jlib/jptree.hpp

@@ -23,6 +23,12 @@
 #include "jexcept.hpp"
 #include "jiter.hpp"
 
+enum TextMarkupFormat
+{
+    MarkupFmt_Unknown=0,
+    MarkupFmt_XML,
+    MarkupFmt_JSON
+};
 enum PTreeExceptionCodes
 {
     PTreeExcpt_XPath_Ambiguity,

system/jlib/jstring.cpp

@@ -1872,37 +1872,72 @@ jlib_decl StringBuffer &appendJSONName(StringBuffer &s, const char *name)
     return encodeJSON(s.append('"'), name).append("\": ");
 }
 
+jlib_decl StringBuffer &appendfJSONName(StringBuffer &s, const char *format, ...)
+{
+    va_list args;
+    va_start(args, format);
+    StringBuffer vs;
+    vs.valist_appendf(format, args);
+    va_end(args);
+    return appendJSONName(s, vs);
+}
+
+static char hexchar[] = "0123456789ABCDEF";
+jlib_decl StringBuffer &appendJSONValue(StringBuffer& s, const char *name, unsigned len, const void *_value)
+{
+    appendJSONNameOrDelimit(s, name);
+    s.append('"');
+    const unsigned char *value = (const unsigned char *) _value;
+    for (unsigned int i = 0; i < len; i++)
+        s.append(hexchar[value[i] >> 4]).append(hexchar[value[i] & 0x0f]);
+    return s.append('"');
+}
+
+inline StringBuffer &encodeJSON(StringBuffer &s, const char ch)
+{
+    switch (ch)
+    {
+        case '\b':
+            s.append("\\b");
+            break;
+        case '\f':
+            s.append("\\f");
+            break;
+        case '\n':
+            s.append("\\n");
+            break;
+        case '\r':
+            s.append("\\r");
+            break;
+        case '\t':
+            s.append("\\t");
+            break;
+        case '\"':
+        case '\\':
+        case '/':
+            s.append('\\'); //fall through
+        default:
+            s.append(ch);
+    }
+    return s;
+}
+
+StringBuffer &encodeJSON(StringBuffer &s, unsigned len, const char *value)
+{
+    if (!value)
+        return s;
+    unsigned pos=0;
+    while(pos<len && value[pos]!=0)
+        encodeJSON(s, value[pos++]);
+    return s;
+}
+
 StringBuffer &encodeJSON(StringBuffer &s, const char *value)
 {
     if (!value)
         return s;
-    for (; *value; value++)
-    {
-        switch (*value)
-        {
-            case '\b':
-                s.append("\\b");
-                break;
-            case '\f':
-                s.append("\\f");
-                break;
-            case '\n':
-                s.append("\\n");
-                break;
-            case '\r':
-                s.append("\\r");
-                break;
-            case '\t':
-                s.append("\\t");
-                break;
-            case '\"':
-            case '\\':
-            case '/':
-                s.append('\\'); //fall through
-            default:
-                s.append(*value);
-        }
-    }
+    while (*value)
+        encodeJSON(s, *value++);
     return s;
 }
 

system/jlib/jstring.hpp

@@ -431,7 +431,11 @@ inline StringBuffer &delimitJSON(StringBuffer &s, bool addNewline=false, bool es
 }
 
 jlib_decl StringBuffer &encodeJSON(StringBuffer &s, const char *value);
+jlib_decl StringBuffer &encodeJSON(StringBuffer &s, unsigned len, const char *value);
+
 jlib_decl StringBuffer &appendJSONName(StringBuffer &s, const char *name);
+jlib_decl StringBuffer &appendfJSONName(StringBuffer &s, const char *format, ...);
+jlib_decl StringBuffer &appendJSONValue(StringBuffer& s, const char *name, unsigned len, const void *_value);
 
 inline StringBuffer &appendJSONNameOrDelimit(StringBuffer &s, const char *name)
 {
@@ -478,6 +482,14 @@ inline StringBuffer &appendJSONValue(StringBuffer& s, const char *name, unsigned
     return s.appendulong(value);
 }
 
+inline StringBuffer &appendJSONValue(StringBuffer& s, const char *name, unsigned len, const char *value)
+{
+    appendJSONNameOrDelimit(s, name);
+    if (!value)
+        return s.append("null");
+    return encodeJSON(s.append('"'), len, value).append('"');
+}
+
 extern jlib_decl void decodeCppEscapeSequence(StringBuffer & out, const char * in, bool errorIfInvalid);
 extern jlib_decl bool strToBool(const char * text);
 extern jlib_decl bool strToBool(size_t len, const char * text);