HPCC-3060 Add fileview2 support for JSON format workunit results

Also integrate fileview2 JSON support into WsEcl to common up JSON
result formatting and improve performance.

Also add JSON result format to /WsWorkunits/WUResult.json

Also fix JSON set item formatting, and improve formatting for "ALL"
in a Set.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/fileview2/CMakeLists.txt

@@ -77,6 +77,7 @@ include_directories (
          ./../remote 
          ./../../common/workunit 
          ./../../common/environment 
+         ./../../common/thorhelper
     )
 
 HPCC_ADD_LIBRARY( fileview2 SHARED ${SRCS} )

common/fileview2/fileview.hpp

@@ -60,7 +60,7 @@ enum ResultSetType
 };
 
 
-
+interface IXmlWriter;
 
 interface IResultSetMetaData : extends IInterface
 {
@@ -87,6 +87,7 @@ interface IResultSetMetaData : extends IInterface
 
 typedef double xdouble;
 interface INewResultSet;
+interface IXmlWriter;
 interface IResultSetCursor : extends IInterface
 {
     virtual bool absolute(__int64 row) = 0;
@@ -120,6 +121,10 @@ interface IResultSetCursor : extends IInterface
     virtual IStringVal & getXml(IStringVal & ret, int columnIndex) = 0;
     virtual IStringVal & getXmlRow(IStringVal & ret) = 0;
     virtual IStringVal & getXmlItem(IStringVal & ret) = 0;
+    virtual void beginWriteXmlRows(IXmlWriter & writer) = 0;
+    virtual void writeXmlRow(IXmlWriter & writer) = 0;
+    virtual void endWriteXmlRows(IXmlWriter & writer) = 0;
+    virtual void writeXmlItem(IXmlWriter & writer) = 0;
     virtual __int64 getNumRows() const = 0;
 };
 
@@ -186,6 +191,9 @@ extern FILEVIEW_API int findResultSetColumn(const INewResultSet * results, const
 
 extern FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
 extern FILEVIEW_API unsigned getResultXml(IStringVal & ret, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
+extern FILEVIEW_API unsigned getResultJSON(IStringVal & ret, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0);
+extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
+extern FILEVIEW_API unsigned writeResultXml(IXmlWriter & writer, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
 
 extern FILEVIEW_API unsigned getResultCursorBin(MemoryBuffer & ret, IResultSetCursor * cursor, unsigned start=0, unsigned count=0);
 extern FILEVIEW_API unsigned getResultBin(MemoryBuffer & ret, INewResultSet * cursor, unsigned start=0, unsigned count=0);
@@ -194,7 +202,9 @@ extern FILEVIEW_API unsigned getResultBin(MemoryBuffer & ret, INewResultSet * cu
 #define WorkUnitXML_NoRoot          0x0002
 #define WorkUnitXML_SeverityTags    0x0004
 
+extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const char *password, const IConstWorkUnit *cw, IXmlWriter &writer, unsigned flags, WUExceptionSeverity minSeverity, const char *rootTag);
 extern FILEVIEW_API IStringVal& getFullWorkUnitResultsXML(const char *user, const char *pw, const IConstWorkUnit *wu, IStringVal &str, unsigned flags=0, WUExceptionSeverity minSeverity=ExceptionSeverityInformation);
+extern FILEVIEW_API IStringVal& getFullWorkUnitResultsJSON(const char *user, const char *pw, const IConstWorkUnit *wu, IStringVal &str, unsigned flags=0, WUExceptionSeverity minSeverity=ExceptionSeverityInformation);
 
 extern FILEVIEW_API void startRemoteDataSourceServer(const char * queue, const char * cluster);
 extern FILEVIEW_API void stopRemoteDataSourceServer();

common/fileview2/fvresultset.cpp

@@ -34,6 +34,9 @@
 #include "fvrelate.ipp"
 #include "dasess.hpp"
 
+#include "thorxmlwrite.hpp"
+#include "eclhelper.hpp"
+
 #define DEFAULT_FETCH_SIZE 100
 #define FILEVIEW_VERSION 1
 #define MAX_SORT_ELEMENTS 1000
@@ -1673,6 +1676,7 @@ void CResultSetCursor::getXmlText(StringBuffer & out, int columnIndex, const cha
     rtlFree(resultStr);
 }
 
+
 void CResultSetCursor::getXmlAttrText(StringBuffer & out, int columnIndex, const char *tag)
 {
     if (!isValid())
@@ -1791,6 +1795,274 @@ void CResultSetCursor::getXmlAttrText(StringBuffer & out, int columnIndex, const
     rtlFree(resultStr);
 }
 
+void CResultSetCursor::writeXmlText(IXmlWriter &writer, int columnIndex, const char *tag)
+{
+    if (!isValid())
+        return;
+
+    const char * name = (tag) ? tag : meta.meta->queryXmlTag(columnIndex);
+    CResultSetColumnInfo & column = meta.columns.item(columnIndex);
+    unsigned flags = column.flag;
+    switch (flags)
+    {
+    case FVFFbeginif:
+    case FVFFendif:
+        return;
+    case FVFFbeginrecord:
+        {
+            if (name && *name)
+            {
+                writer.outputBeginNested(name, false);
+                const IntArray &attributes = meta.meta->queryAttrList(columnIndex);
+                ForEachItemIn(ac, attributes)
+                    writeXmlAttrText(writer, attributes.item(ac));
+            }
+        }
+        return;
+    case FVFFendrecord:
+        writer.outputEndNested(name);
+        return;
+    }
+
+    const byte * cur = getColumn(columnIndex);
+    unsigned resultLen;
+    char * resultStr = NULL;
+
+    ITypeInfo & type = *column.type;
+    unsigned size = type.getSize();
+    unsigned len = UNKNOWN_LENGTH;
+    switch (type.getTypeCode())
+    {
+    case type_boolean:
+        writer.outputBool(*((byte *)cur) != 0, name);
+        break;
+    case type_int:
+        {
+            __int64 value = getIntFromInt(type, cur, isMappedIndexField(columnIndex));
+            if (type.isSigned())
+                writer.outputInt((__int64) value, name);
+            else
+                writer.outputUInt((unsigned __int64) value, name);
+            break;
+        }
+    case type_swapint:
+        {
+            __int64 value = getIntFromSwapInt(type, cur, isMappedIndexField(columnIndex));
+            if (type.isSigned())
+                writer.outputInt((__int64) value, name);
+            else
+                writer.outputUInt((unsigned __int64) value, name);
+            break;
+        }
+    case type_packedint:
+        {
+            if (type.isSigned())
+                writer.outputInt(rtlGetPackedSigned(cur), name);
+            else
+                writer.outputUInt(rtlGetPackedUnsigned(cur), name);
+            break;
+        }
+    case type_decimal:
+        if (type.isSigned())
+            writer.outputDecimal(cur, size, type.getPrecision(), name);
+        else
+            writer.outputUDecimal(cur, size, type.getPrecision(), name);
+        break;
+    case type_real:
+        if (size == 4)
+            writer.outputReal(*(float *)cur, name);
+        else
+            writer.outputReal(*(double *)cur, name);
+        break;
+    case type_qstring:
+        len = getLength(type, cur);
+        rtlQStrToStrX(resultLen, resultStr, len, (const char *)cur);
+        writer.outputString(resultLen, resultStr, name);
+        break;
+    case type_data:
+        len = getLength(type, cur);
+        writer.outputData(len, cur, name);
+        break;
+    case type_string:
+        len = getLength(type, cur);
+        if (meta.isEBCDIC(columnIndex))
+        {
+            rtlStrToEStrX(resultLen, resultStr, len, (const char *)cur);
+            writer.outputString(resultLen, resultStr, name);
+        }
+        else
+            writer.outputString(len, (const char *)cur, name);
+        break;
+    case type_unicode:
+        len = getLength(type, cur);
+        writer.outputUnicode(len, (UChar const *)cur, name);
+        break;
+    case type_varstring:
+        if (meta.isEBCDIC(columnIndex))
+        {
+            rtlStrToEStrX(resultLen, resultStr, strlen((const char *)cur), (const char *)cur);
+            writer.outputString(resultLen, resultStr, name);
+        }
+        else
+            writer.outputString(strlen((const char *)cur), (const char *)cur, name);
+        break;
+    case type_varunicode:
+        writer.outputUnicode(rtlUnicodeStrlen((UChar const *)cur), (UChar const *)cur, name);
+        break;
+    case type_utf8:
+        len = getLength(type, cur);
+        writer.outputUtf8(len, (const char *)cur, name);
+        break;
+    case type_table:
+    case type_groupedtable:
+        {
+            writer.outputBeginNested(name, false);
+            Owned<IResultSetCursor> childCursor = getChildren(columnIndex);
+            childCursor->beginWriteXmlRows(writer);
+            ForEach(*childCursor)
+                childCursor->writeXmlRow(writer);
+            childCursor->endWriteXmlRows(writer);
+            writer.outputEndNested(name);
+            break;
+        }
+    case type_set:
+        {
+            writer.outputBeginNested(name, false);
+            if (getIsAll(columnIndex))
+                writer.outputSetAll();
+            else
+            {
+                Owned<IResultSetCursor> childCursor = getChildren(columnIndex);
+                childCursor->beginWriteXmlRows(writer);
+                ForEach(*childCursor)
+                    childCursor->writeXmlItem(writer);
+                childCursor->endWriteXmlRows(writer);
+            }
+            writer.outputEndNested(name);
+            break;
+        }
+    default:
+        UNIMPLEMENTED;
+    }
+    rtlFree(resultStr);
+}
+
+void CResultSetCursor::writeXmlAttrText(IXmlWriter &writer, int columnIndex, const char *tag)
+{
+    if (!isValid())
+        return;
+
+    const char * name = (tag) ? tag : meta.meta->queryXmlTag(columnIndex);
+    CResultSetColumnInfo & column = meta.columns.item(columnIndex);
+    unsigned flags = column.flag;
+    switch (flags)
+    {
+    case FVFFbeginif:
+    case FVFFendif:
+    case FVFFbeginrecord:
+    case FVFFendrecord:
+        return;
+    }
+
+    const byte * cur = getColumn(columnIndex);
+    unsigned resultLen;
+    char * resultStr = NULL;
+
+    ITypeInfo & type = *column.type;
+    unsigned size = type.getSize();
+    unsigned len = UNKNOWN_LENGTH;
+    switch (type.getTypeCode())
+    {
+    case type_boolean:
+        writer.outputBool((*((byte *)cur)) != 0, name);
+        break;
+    case type_int:
+        {
+            __int64 value = getIntFromInt(type, cur, isMappedIndexField(columnIndex));
+            if (type.isSigned())
+                writer.outputInt((__int64) value, name);
+            else
+                writer.outputUInt((unsigned __int64) value, name);
+            break;
+        }
+    case type_swapint:
+        {
+            __int64 value = getIntFromSwapInt(type, cur, isMappedIndexField(columnIndex));
+            if (type.isSigned())
+                writer.outputInt((__int64) value, name);
+            else
+                writer.outputUInt((unsigned __int64) value, name);
+            break;
+        }
+    case type_packedint:
+        {
+            if (type.isSigned())
+                writer.outputInt(rtlGetPackedSigned(cur), name);
+            else
+                writer.outputUInt(rtlGetPackedUnsigned(cur), name);
+            break;
+        }
+    case type_decimal:
+        if (type.isSigned())
+            writer.outputDecimal(cur, size, type.getPrecision(), name);
+        else
+            writer.outputUDecimal(cur, size, type.getPrecision(), name);
+        break;
+    case type_real:
+        if (size == 4)
+            writer.outputReal(*(float *)cur, name);
+        else
+            writer.outputReal(*(double *)cur, name);
+        break;
+    case type_qstring:
+        len = getLength(type, cur);
+        rtlQStrToStrX(resultLen, resultStr, len, (const char *)cur);
+        writer.outputString(resultLen, resultStr, name);
+        break;
+    case type_data:
+        len = getLength(type, cur);
+        writer.outputData(len, cur, name);
+        break;
+    case type_string:
+        len = getLength(type, cur);
+        if (meta.isEBCDIC(columnIndex))
+        {
+            rtlStrToEStrX(resultLen, resultStr, len, (const char *)cur);
+            writer.outputString(resultLen, resultStr, name);
+        }
+        else
+            writer.outputString(len, (const char *)cur, name);
+        break;
+    case type_unicode:
+        len = getLength(type, cur);
+        writer.outputUnicode(len, (UChar const *)cur, name);
+        break;
+    case type_varstring:
+        if (meta.isEBCDIC(columnIndex))
+        {
+            rtlStrToEStrX(resultLen, resultStr, strlen((const char *)cur), (const char *)cur);
+            writer.outputString(resultLen, resultStr, name);
+        }
+        else
+            writer.outputString(strlen((const char *)cur), (const char *)cur, name);
+        break;
+    case type_varunicode:
+        writer.outputUnicode(rtlUnicodeStrlen((UChar const *)cur), (UChar const *)cur, name);
+        break;
+    case type_utf8:
+        len = getLength(type, cur);
+        writer.outputUtf8(len, (const char *)cur, name);
+        break;
+    case type_table:
+    case type_groupedtable:
+    case type_set:
+        break;
+    default:
+        UNIMPLEMENTED;
+    }
+    rtlFree(resultStr);
+}
+
 IStringVal & CResultSetCursor::getXml(IStringVal &ret, int columnIndex)
 {
     StringBuffer temp;
@@ -1807,6 +2079,11 @@ IStringVal & CResultSetCursor::getXmlItem(IStringVal & ret)
     return ret;
 }
 
+void CResultSetCursor::writeXmlItem(IXmlWriter &writer)
+{
+    writeXmlText(writer, 0, meta.meta->queryXmlTag());
+}
+
 //More efficient than above...
 IStringVal & CResultSetCursor::getXmlRow(IStringVal &ret)
 {
@@ -1865,6 +2142,74 @@ IStringVal & CResultSetCursor::getXmlRow(IStringVal &ret)
     return ret;
 }
 
+void CResultSetCursor::beginWriteXmlRows(IXmlWriter & writer)
+{
+    const char *rowtag = meta.meta->queryXmlTag();
+    if (rowtag && *rowtag)
+        writer.outputBeginArray(rowtag);
+}
+
+void CResultSetCursor::endWriteXmlRows(IXmlWriter & writer)
+{
+    const char *rowtag = meta.meta->queryXmlTag();
+    if (rowtag && *rowtag)
+        writer.outputEndArray(rowtag);
+}
+
+void CResultSetCursor::writeXmlRow(IXmlWriter &writer)
+{
+    StringBuffer temp;
+    const char *rowtag = meta.meta->queryXmlTag();
+    if (rowtag && *rowtag)
+    {
+        writer.outputBeginNested(rowtag, false);
+        const IntArray &attributes = meta.meta->queryAttrList();
+        ForEachItemIn(ac, attributes)
+            writeXmlAttrText(writer, attributes.item(ac));
+    }
+    unsigned numColumns = meta.getColumnCount();
+    unsigned ignoreNesting = 0;
+    for (unsigned col = 0; col < numColumns; col++)
+    {
+        unsigned flags = meta.columns.item(col).flag;
+        const char *tag = meta.meta->queryXmlTag(col);
+        if (tag && *tag=='@')
+            continue;
+        switch (flags)
+        {
+        case FVFFbeginif:
+            if (ignoreNesting || !getBoolean(col))
+                ignoreNesting++;
+            break;
+        case FVFFendif:
+            if (ignoreNesting)
+                ignoreNesting--;
+            break;
+        case FVFFbeginrecord:
+            if (ignoreNesting)
+                ignoreNesting++;
+            else
+                writeXmlText(writer, col);
+            break;
+        case FVFFendrecord:
+            if (ignoreNesting)
+                ignoreNesting--;
+            else
+                writeXmlText(writer, col);
+            break;
+        case FVFFnone:
+        case FVFFvirtual:
+        case FVFFdataset:
+        case FVFFset:
+            if (ignoreNesting == 0)
+                writeXmlText(writer, col);
+            break;
+        }
+    }
+    assertex(ignoreNesting == 0);
+    writer.outputEndNested(rowtag);
+}
+
 bool CResultSetCursor::isAfterLast() const
 {
     return (curRowData.length() == 0) && (getCurRow() != BEFORE_FIRST_ROW);
@@ -2101,6 +2446,22 @@ IStringVal & IndirectResultSetCursor::getXmlItem(IStringVal &ret)
 {
     return queryBase()->getXmlItem(ret);
 }
+void IndirectResultSetCursor::beginWriteXmlRows(IXmlWriter & writer)
+{
+    return queryBase()->beginWriteXmlRows(writer);
+}
+void IndirectResultSetCursor::writeXmlRow(IXmlWriter &writer)
+{
+    return queryBase()->writeXmlRow(writer);
+}
+void IndirectResultSetCursor::endWriteXmlRows(IXmlWriter & writer)
+{
+    return queryBase()->endWriteXmlRows(writer);
+}
+void IndirectResultSetCursor::writeXmlItem(IXmlWriter &writer)
+{
+    return queryBase()->writeXmlItem(writer);
+}
 void IndirectResultSetCursor::noteRelatedFileChanged()
 {
     queryBase()->noteRelatedFileChanged();
@@ -3292,6 +3653,59 @@ extern FILEVIEW_API unsigned getResultXml(IStringVal & ret, INewResultSet * resu
     return getResultCursorXml(ret, cursor, name, start, count, schemaName);
 }
 
+extern FILEVIEW_API unsigned getResultJSON(IStringVal & ret, INewResultSet * result, const char* name,unsigned start, unsigned count)
+{
+    Owned<IResultSetCursor> cursor = result->createCursor();
+    Owned<CommonJsonWriter> writer = new CommonJsonWriter(0);
+    writer->outputBeginRoot();
+    unsigned rc = writeResultCursorXml(*writer, cursor, name, start, count, NULL);
+    writer->outputEndRoot();
+    ret.set(writer->str());
+    return rc;
+}
+
+extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSetCursor * cursor, const char * name, unsigned start, unsigned count, const char * schemaName)
+{
+    if (schemaName)
+    {
+        CommonXmlWriter *xmlwriter = dynamic_cast<CommonXmlWriter *>(&writer);
+        if (xmlwriter)
+        {
+            StringBuffer xsd;
+            xsd.append("<XmlSchema name=\"").append(schemaName).append("\">");
+            const IResultSetMetaData & meta = cursor->queryResultSet()->getMetaData();
+            StringBufferAdaptor adaptor(xsd);
+            meta.getXmlXPathSchema(adaptor, false);
+            xsd.append("</XmlSchema>").newline();
+            xmlwriter->outputInlineXml(xsd.str());
+        }
+    }
+
+    writer.outputBeginDataset(name, true);
+    if (schemaName)
+        writer.outputCString(schemaName, "@xmlSchema");
+
+    cursor->beginWriteXmlRows(writer);
+    unsigned c=0;
+    for(bool ok=cursor->absolute(start);ok;ok=cursor->next())
+    {
+        cursor->writeXmlRow(writer);
+
+        c++;
+        if(count && c>=count)
+            break;
+    }
+    cursor->endWriteXmlRows(writer);
+    writer.outputEndDataset(name);
+    return c;
+}
+
+extern FILEVIEW_API unsigned writeResultXml(IXmlWriter & writer, INewResultSet * result, const char* name,unsigned start, unsigned count, const char * schemaName)
+{
+    Owned<IResultSetCursor> cursor = result->createCursor();
+    return writeResultCursorXml(writer, cursor, name, start, count, schemaName);
+}
+
 extern FILEVIEW_API unsigned getResultCursorBin(MemoryBuffer & ret, IResultSetCursor * cursor, unsigned start, unsigned count)
 {
     const IResultSetMetaData & meta = cursor->queryResultSet()->getMetaData();
@@ -3334,13 +3748,13 @@ inline const char *getSeverityTagname(WUExceptionSeverity severity, unsigned fla
     return "Exception";
 }
 
-extern FILEVIEW_API IStringVal& getFullWorkUnitResultsXML(const char *username, const char *password, const IConstWorkUnit *cw, IStringVal &str, unsigned flags, WUExceptionSeverity minSeverity)
+extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const char *password, const IConstWorkUnit *cw, IXmlWriter &writer, unsigned flags, WUExceptionSeverity minSeverity, const char *rootTag)
 {
     SCMStringBuffer wuid;
     cw->getWuid(wuid);
-    StringBuffer result;
-    if (!(flags & WorkUnitXML_NoRoot))
-        result.append("<Result>").newline();
+
+    if (rootTag && *rootTag && !(flags & WorkUnitXML_NoRoot))
+        writer.outputBeginNested(rootTag, true);
 
     Owned<IConstWUExceptionIterator> exceptions = &cw->getExceptions();
     ForEach(*exceptions)
@@ -3348,15 +3762,13 @@ extern FILEVIEW_API IStringVal& getFullWorkUnitResultsXML(const char *username,
         WUExceptionSeverity severity = exceptions->query().getSeverity();
         if (severity>=minSeverity)
         {
-            SCMStringBuffer x, y;
-            exceptions->query().getExceptionSource(x);
-            exceptions->query().getExceptionMessage(y);
-            
-            result.appendf("<%s><Source>", getSeverityTagname(severity, flags));
-            encodeUtf8XML(x.str(), result);
-            result.append("</Source><Message>");
-            encodeUtf8XML(y.str(), result);
-            result.appendf("</Message></%s>", getSeverityTagname(severity, flags)).newline();
+            SCMStringBuffer src, msg;
+            exceptions->query().getExceptionSource(src);
+            exceptions->query().getExceptionMessage(msg);
+            writer.outputBeginNested(getSeverityTagname(severity, flags), false);
+            writer.outputCString(src.str(), "Source");
+            writer.outputCString(msg.str(), "Message");
+            writer.outputEndNested(getSeverityTagname(severity, flags));
         }
     }
 
@@ -3372,22 +3784,47 @@ extern FILEVIEW_API IStringVal& getFullWorkUnitResultsXML(const char *username,
                 IConstWUResult &ds = results->query();
                 if (ds.getResultSequence()>=0)
                 {
-                    SCMStringBuffer resultXML, name;
+                    SCMStringBuffer name;
                     ds.getResultName(name);
                     Owned<INewResultSet> nr = factory->createNewResultSet(&ds, wuid.str());
-                    getResultXml(resultXML, nr.get(), name.str(), 0, 0, (flags & WorkUnitXML_InclSchema) ? name.str() : NULL);
-                    result.append(resultXML);
+                    writeResultXml(writer, nr.get(), name.str(), 0, 0, (flags & WorkUnitXML_InclSchema) ? name.str() : NULL);
                 }
             }
         }
         break;
 
         case WUStateAborted:
-            result.append("<Exception><Source>System</Source><Message>Query aborted by operator</Message></Exception>");
+            writer.outputBeginNested("Exception", false);
+            writer.outputCString("System", "Source");
+            writer.outputCString("Query aborted by operator", "Message");
+            writer.outputEndNested("Exception");
             break;
     }
-    if (!(flags & WorkUnitXML_NoRoot))
-        result.append("</Result>");
-    str.set(result.str());
+    if (rootTag && *rootTag && !(flags & WorkUnitXML_NoRoot))
+        writer.outputEndNested(rootTag);
+}
+
+extern FILEVIEW_API IStringVal& getFullWorkUnitResultsXML(const char *username, const char *password, const IConstWorkUnit *cw, IStringVal &str, unsigned flags, WUExceptionSeverity minSeverity)
+{
+    SCMStringBuffer wuid;
+    cw->getWuid(wuid);
+
+    Owned<CommonXmlWriter> writer = CreateCommonXmlWriter(0);
+    writeFullWorkUnitResults(username, password, cw, *writer, flags, minSeverity, "Result");
+
+    str.set(writer->str());
+    return str;
+}
+
+extern FILEVIEW_API IStringVal& getFullWorkUnitResultsJSON(const char *username, const char *password, const IConstWorkUnit *cw, IStringVal &str, unsigned flags, WUExceptionSeverity minSeverity)
+{
+    SCMStringBuffer wuid;
+    cw->getWuid(wuid);
+
+    Owned<CommonJsonWriter> writer = new CommonJsonWriter(0, 0, 0);
+    writer->outputBeginRoot();
+    writeFullWorkUnitResults(username, password, cw, *writer, flags, minSeverity, "Results");
+    writer->outputEndRoot();
+    str.set(writer->str());
     return str;
 }

common/fileview2/fvresultset.ipp

@@ -364,6 +364,11 @@ public:
     virtual IStringVal & getXmlRow(IStringVal &ret);
     virtual IStringVal & getXmlItem(IStringVal & ret);
 
+    virtual void beginWriteXmlRows(IXmlWriter & writer);
+    virtual void writeXmlRow(IXmlWriter &writer);
+    virtual void endWriteXmlRows(IXmlWriter & writer);
+    virtual void writeXmlItem(IXmlWriter &writer);
+
 //IExtendedResultSetCursor
     virtual void noteRelatedFileChanged() {}
 
@@ -374,6 +379,8 @@ protected:
     const byte * getColumn(unsigned idx) const      { return (const byte *)curRowData.toByteArray() + offsets[idx]; }
     void getXmlText(StringBuffer & out, int columnIndex, const char *tag=NULL);
     void getXmlAttrText(StringBuffer & out, int columnIndex, const char *tag=NULL);
+    void writeXmlText(IXmlWriter &writer, int columnIndex, const char *tag=NULL);
+    void writeXmlAttrText(IXmlWriter &writer, int columnIndex, const char *tag=NULL);
 
     virtual __int64 getCurRow() const;
     virtual __int64 translateRow(__int64 row) const;
@@ -428,6 +435,10 @@ public:
     virtual IStringVal & getXml(IStringVal & ret, int columnIndex);
     virtual IStringVal & getXmlRow(IStringVal &ret);
     virtual IStringVal & getXmlItem(IStringVal &ret);
+    virtual void beginWriteXmlRows(IXmlWriter & writer);
+    virtual void writeXmlRow(IXmlWriter &writer);
+    virtual void endWriteXmlRows(IXmlWriter & writer);
+    virtual void writeXmlItem(IXmlWriter &writer);
     virtual void noteRelatedFileChanged();
 
 protected:

common/thorhelper/roxiedebug.cpp

@@ -498,6 +498,14 @@ public:
         // Searching/breaking on unicode not supported at the moment
     }
 
+    virtual void outputBeginDataset(const char *dsname, bool nestChildren)
+    {
+        // nothing for now
+    }
+    virtual void outputEndDataset(const char *dsname)
+    {
+        // nothing for now
+    }
     virtual void outputBeginNested(const char *fieldname, bool nestChildren) 
     {
         // nothing for now

common/thorhelper/roxiehelper.cpp

@@ -1085,7 +1085,7 @@ void FlushingStringBuffer::incrementRowCount()
 void FlushingJsonBuffer::encodeXML(const char *x, unsigned flags, unsigned len, bool utf8)
 {
     CriticalBlock b(crit);
-    appendJSONValue(s, NULL, len, x);
+    appendJSONStringValue(s, NULL, len, x, true);
 }
 
 void FlushingJsonBuffer::startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend)

common/thorhelper/thorxmlwrite.cpp

@@ -241,8 +241,22 @@ void CommonXmlWriter::outputUtf8(unsigned len, const char *field, const char *fi
     }
 }
 
+void CommonXmlWriter::outputBeginDataset(const char *dsname, bool nestChildren)
+{
+    outputBeginNested("Dataset", nestChildren, false); //no indent for backward compatibility
+    if (!dsname || !*dsname)
+        return;
+    out.append(" name='"); //single quote for backward compatibility
+    outputXmlUtf8(strlen(dsname), dsname, NULL, out);
+    out.append("'");
+}
 
-void CommonXmlWriter::outputBeginNested(const char *fieldname, bool nestChildren)
+void CommonXmlWriter::outputEndDataset(const char *dsname)
+{
+    outputEndNested("Dataset", false);
+}
+
+void CommonXmlWriter::outputBeginNested(const char *fieldname, bool nestChildren, bool doIndent)
 {
     const char * sep = strchr(fieldname, '/');
     if (sep)
@@ -254,16 +268,22 @@ void CommonXmlWriter::outputBeginNested(const char *fieldname, bool nestChildren
     }
 
     closeTag();
-    if (!nestLimit)
+    if (!nestLimit && doIndent)
         out.pad(indent);
     out.append('<').append(fieldname);
-    indent += 1;
+    if (doIndent)
+        indent += 1;
     if (!nestChildren && !nestLimit)
         nestLimit = indent;
     tagClosed = false;
 }
 
-void CommonXmlWriter::outputEndNested(const char *fieldname)
+void CommonXmlWriter::outputBeginNested(const char *fieldname, bool nestChildren)
+{
+    outputBeginNested(fieldname, nestChildren, true);
+}
+
+void CommonXmlWriter::outputEndNested(const char *fieldname, bool doIndent)
 {
     const char * sep = strchr(fieldname, '/');
     if (sep)
@@ -289,11 +309,16 @@ void CommonXmlWriter::outputEndNested(const char *fieldname)
     }
     if (indent==nestLimit)
         nestLimit = 0;
-    indent -= 1;
+    if (doIndent)
+        indent -= 1;
     if (!nestLimit)
         out.newline();
 }
 
+void CommonXmlWriter::outputEndNested(const char *fieldname)
+{
+    outputEndNested(fieldname, true);
+}
 void CommonXmlWriter::outputSetAll()
 {
     closeTag();
@@ -351,23 +376,26 @@ void CommonJsonWriter::checkDelimit(int inc)
     checkFormat(true, true, inc);
 }
 
-const char *CommonJsonWriter::checkItemName(CJsonWriterItem *item, const char *name)
+
+const char *CommonJsonWriter::checkItemName(CJsonWriterItem *item, const char *name, bool simpleType)
 {
+    if (simpleType && (!name || !*name))
+        name = "#value"; //xml mixed content
     if (item && item->depth==0 && strieq(item->name, name))
         return NULL;
     return name;
 }
 
-const char *CommonJsonWriter::checkItemName(const char *name)
+const char *CommonJsonWriter::checkItemName(const char *name, bool simpleType)
 {
     CJsonWriterItem *item = (arrays.length()) ? &arrays.tos() : NULL;
-    return checkItemName(item, name);
+    return checkItemName(item, name, simpleType);
 }
 
 const char *CommonJsonWriter::checkItemNameBeginNested(const char *name)
 {
     CJsonWriterItem *item = (arrays.length()) ? &arrays.tos() : NULL;
-    name = checkItemName(item, name);
+    name = checkItemName(item, name, false);
     if (item)
         item->depth++;
     return name;
@@ -378,7 +406,7 @@ const char *CommonJsonWriter::checkItemNameEndNested(const char *name)
     CJsonWriterItem *item = (arrays.length()) ? &arrays.tos() : NULL;
     if (item)
         item->depth--;
-    return checkItemName(item, name);
+    return checkItemName(item, name, false);
 }
 
 void CommonJsonWriter::outputQuoted(const char *text)
@@ -394,7 +422,7 @@ void CommonJsonWriter::outputString(unsigned len, const char *field, const char
     if ((flags & XWFopt) && (rtlTrimStrLen(len, field) == 0))
         return;
     checkDelimit();
-    appendJSONValue(out, checkItemName(fieldname), len, field);
+    appendJSONStringValue(out, checkItemName(fieldname), len, field, true);
 }
 
 void CommonJsonWriter::outputQString(unsigned len, const char *field, const char *fieldname)
@@ -418,7 +446,7 @@ void CommonJsonWriter::outputBool(bool field, const char *fieldname)
 void CommonJsonWriter::outputData(unsigned len, const void *field, const char *fieldname)
 {
     checkDelimit();
-    appendJSONValue(out, checkItemName(fieldname), len, field);
+    appendJSONDataValue(out, checkItemName(fieldname), len, field);
 }
 
 void CommonJsonWriter::outputInt(__int64 field, const char *fieldname)
@@ -468,7 +496,7 @@ void CommonJsonWriter::outputUtf8(unsigned len, const char *field, const char *f
     if ((flags & XWFopt) && (rtlTrimUtf8StrLen(len, field) == 0))
         return;
     checkDelimit();
-    appendJSONValue(out, checkItemName(fieldname), len, field);
+    appendJSONStringValue(out, checkItemName(fieldname), len, field, true);
 }
 
 void CommonJsonWriter::outputBeginArray(const char *fieldname)
@@ -499,8 +527,23 @@ void CommonJsonWriter::outputEndArray(const char *fieldname)
     }
 }
 
+void CommonJsonWriter::outputBeginDataset(const char *dsname, bool nestChildren)
+{
+    if (dsname && *dsname)
+        outputBeginNested(dsname, nestChildren);
+}
+
+void CommonJsonWriter::outputEndDataset(const char *dsname)
+{
+    if (dsname && *dsname)
+        outputEndNested(dsname);
+}
+
 void CommonJsonWriter::outputBeginNested(const char *fieldname, bool nestChildren)
 {
+    if (!fieldname || !*fieldname)
+        return;
+
     flush(false);
     checkFormat(true, false, 1);
     fieldname = checkItemNameBeginNested(fieldname);
@@ -523,6 +566,9 @@ void CommonJsonWriter::outputBeginNested(const char *fieldname, bool nestChildre
 
 void CommonJsonWriter::outputEndNested(const char *fieldname)
 {
+    if (!fieldname || !*fieldname)
+        return;
+
     flush(false);
     checkFormat(false, true, -1);
     fieldname = checkItemNameEndNested(fieldname);
@@ -544,7 +590,7 @@ void CommonJsonWriter::outputSetAll()
 {
     flush(false);
     checkDelimit();
-    appendJSONValue(out, NULL, "All");
+    appendJSONValue(out, "All", true);
 }
 
 //=====================================================================================

common/thorhelper/thorxmlwrite.hpp

@@ -48,6 +48,10 @@ public:
     unsigned length() const                                 { return out.length(); }
     const char * str() const                                { return out.str(); }
 
+    void outputInlineXml(const char *text){out.append(text); flush(false);} //for appending xml only content
+    void outputBeginNested(const char *fieldname, bool nestChildren, bool doIndent);
+    void outputEndNested(const char *fieldname, bool doIndent);
+
     virtual void outputQuoted(const char *text);
     virtual void outputQString(unsigned len, const char *field, const char *fieldname);
     virtual void outputString(unsigned len, const char *field, const char *fieldname);
@@ -60,6 +64,8 @@ public:
     virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
     virtual void outputUnicode(unsigned len, const UChar *field, const char *fieldname);
     virtual void outputUtf8(unsigned len, const char *field, const char *fieldname);
+    virtual void outputBeginDataset(const char *dsname, bool nestChildren);
+    virtual void outputEndDataset(const char *dsname);
     virtual void outputBeginNested(const char *fieldname, bool nestChildren);
     virtual void outputEndNested(const char *fieldname);
     virtual void outputBeginArray(const char *fieldname){}; //repeated elements are inline for xml
@@ -109,12 +115,17 @@ public:
     virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname);
     virtual void outputUnicode(unsigned len, const UChar *field, const char *fieldname);
     virtual void outputUtf8(unsigned len, const char *field, const char *fieldname);
+    virtual void outputBeginDataset(const char *dsname, bool nestChildren);
+    virtual void outputEndDataset(const char *dsname);
     virtual void outputBeginNested(const char *fieldname, bool nestChildren);
     virtual void outputEndNested(const char *fieldname);
     virtual void outputBeginArray(const char *fieldname);
     virtual void outputEndArray(const char *fieldname);
     virtual void outputSetAll();
 
+    void outputBeginRoot(){out.append('{');}
+    void outputEndRoot(){out.append('}');}
+
 protected:
     inline void flush(bool isClose)
     {
@@ -131,8 +142,8 @@ protected:
         unsigned depth;
     };
 
-    const char *checkItemName(CJsonWriterItem *item, const char *name);
-    const char *checkItemName(const char *name);
+    const char *checkItemName(CJsonWriterItem *item, const char *name, bool simpleType=true);
+    const char *checkItemName(const char *name, bool simpleType=true);
     const char *checkItemNameBeginNested(const char *name);
     const char *checkItemNameEndNested(const char *name);
 
@@ -173,7 +184,7 @@ public:
 };
 
 enum XMLWriterType{WTStandard, WTEncoding, WTEncodingData64, WTJSON} ;
-CommonXmlWriter * CreateCommonXmlWriter(unsigned _flags, unsigned initialIndent=0, IXmlStreamFlusher *_flusher=NULL, XMLWriterType xmlType=WTStandard);
+thorhelper_decl CommonXmlWriter * CreateCommonXmlWriter(unsigned _flags, unsigned initialIndent=0, IXmlStreamFlusher *_flusher=NULL, XMLWriterType xmlType=WTStandard);
 thorhelper_decl IXmlWriter * createIXmlWriter(unsigned _flags, unsigned initialIndent=0, IXmlStreamFlusher *_flusher=NULL, XMLWriterType xmlType=WTStandard);
 
 class thorhelper_decl SimpleOutputWriter : public CInterface, implements IXmlWriter
@@ -202,6 +213,8 @@ public:
     virtual void outputUtf8(unsigned len, const char *field, const char *fieldname);
     virtual void outputBeginNested(const char *fieldname, bool nestChildren);
     virtual void outputEndNested(const char *fieldname);
+    virtual void outputBeginDataset(const char *dsname, bool nestChildren){}
+    virtual void outputEndDataset(const char *dsname){}
     virtual void outputBeginArray(const char *fieldname){}
     virtual void outputEndArray(const char *fieldname){}
     virtual void outputSetAll();

common/wuwebview/wuwebview.cpp

@@ -287,6 +287,8 @@ public:
     virtual void renderResults(const char *viewName, const char *xml, StringBuffer &html);
     virtual void renderResults(const char *viewName, StringBuffer &html);
     virtual void renderSingleResult(const char *viewName, const char *resultname, StringBuffer &html);
+    virtual void renderResultsJSON(StringBuffer &out, const char *jsonp);
+
     virtual void expandResults(const char *xml, StringBuffer &out, unsigned flags);
     virtual void expandResults(StringBuffer &out, unsigned flags);
     virtual void createWuidResponse(StringBuffer &out, unsigned flags);
@@ -630,6 +632,22 @@ void WuWebView::renderResults(const char *viewName, StringBuffer &out)
     renderExpandedResults(viewName, buffer, out);
 }
 
+void WuWebView::renderResultsJSON(StringBuffer &out, const char *jsonp)
+{
+    if (jsonp && *jsonp)
+        out.append(jsonp).append('(');
+    out.append('{');
+    StringBuffer responseName(name.str());
+    responseName.append("Response");
+    appendJSONName(out, responseName);
+    StringBufferAdaptor json(out);
+    getFullWorkUnitResultsJSON(username, pw, cw, json, 0, ExceptionSeverityError);
+    out.append("}");
+    if (jsonp && *jsonp)
+        out.append(");");
+}
+
+
 void WuWebView::renderSingleResult(const char *viewName, const char *resultname, StringBuffer &out)
 {
     WuExpandedResultBuffer buffer(name.str(), WWV_ADD_RESPONSE_TAG | WWV_ADD_RESULTS_TAG);

common/wuwebview/wuwebview.hpp

@@ -46,6 +46,7 @@ interface IWuWebView : extends IInterface
     virtual void renderResults(const char *viewName, const char *xml, StringBuffer &html)=0;
     virtual void renderResults(const char *viewName, StringBuffer &html)=0;
     virtual void renderSingleResult(const char *viewName, const char *resultname, StringBuffer &html)=0;
+    virtual void renderResultsJSON(StringBuffer &out, const char *jsonp)=0;
     virtual void applyResultsXSLT(const char *file, const char *xml, StringBuffer &html)=0;
     virtual void applyResultsXSLT(const char *file, StringBuffer &html)=0;
     virtual StringBuffer &aggregateResources(const char *type, StringBuffer &content)=0;

esp/bindings/SOAP/Platform/soapparam.cpp

@@ -24,12 +24,12 @@
 #include "soapmessage.hpp"
 #include "soapparam.hpp"
 
-void BaseEspParam::toJSON(IEspContext *ctx, StringBuffer &s, const char *tagname)
+void BaseEspParam::toJSON(IEspContext *ctx, StringBuffer &s, const char *tagname, bool encode)
 {
     if (isNil && nilBH==nilRemove)
         return;
     appendJSONName(s, tagname);
-    toJSONValue(s);
+    toJSONValue(s, encode);
 }
 
 void BaseEspParam::toXML(IEspContext* ctx, StringBuffer &s, const char *tagname, const char *prefix, bool encode)
@@ -47,7 +47,7 @@ void BaseEspParam::toStr(IEspContext* ctx, StringBuffer &s, const char *tagname,
         return;
 
     if (ctx && ctx->getResponseFormat()==ESPSerializationJSON)
-        return toJSON(ctx, s, tagname);
+        return toJSON(ctx, s, tagname, true);
     toXML(ctx, s, tagname, prefix, encode);
 }
 
@@ -107,9 +107,20 @@ void SoapStringParam::toXMLValue(StringBuffer &s, bool encode)
         encodeUtf8XML(value.str(), s, getEncodeNewlines() ? ENCODE_NEWLINES : 0);
 }
 
-void SoapStringParam::toJSONValue(StringBuffer &s)
+void SoapStringParam::toStr(IEspContext* ctx, StringBuffer &s, const char *tagname, const char *basepath, bool encode, const char *xsdtype, const char *prefix, bool encodeJSON)
+{
+    if (isNil && nilBH!=nilIgnore)
+        return;
+
+    if (ctx && ctx->getResponseFormat()==ESPSerializationJSON)
+        toJSON(ctx, s, tagname, encodeJSON);
+    else
+        toXML(ctx, s, tagname, prefix, encode);
+}
+
+void SoapStringParam::toJSONValue(StringBuffer &s, bool encode)
 {
-    appendJSONValue(s, NULL, (isNil) ? NULL : value.str());
+    appendJSONStringValue(s, NULL, (isNil) ? NULL : value.str(), encode, encode);
 }
 
 void SoapStringParam::addRpcValue(IRpcMessage &rpc_call, const char *tagname, const char *basepath, const char *xsdtype, const char *prefix, bool *encodex)

esp/bindings/SOAP/Platform/soapparam.hpp

@@ -83,9 +83,9 @@ public:
     virtual void addRpcValue(IRpcMessage &rpc_call, const char *tagname, const char *basepath, const char *xsdtype, const char *prefix, bool *encodex)=0;
 
     virtual void toXMLValue(StringBuffer &s, bool encode)=0;
-    virtual void toJSONValue(StringBuffer &s)=0;
+    virtual void toJSONValue(StringBuffer &s, bool encode)=0;
     virtual void toXML(IEspContext* ctx, StringBuffer &s, const char *tagname, const char *prefix, bool encode);
-    virtual void toJSON(IEspContext* ctx, StringBuffer &s, const char *tagname);
+    virtual void toJSON(IEspContext* ctx, StringBuffer &s, const char *tagname, bool encode);
     void toStr(IEspContext* ctx, StringBuffer &str, const char *tagname, const char *basepath="", bool encodeXml=true, const char *xsdtype="", const char *prefix="");
 
     void marshall(IRpcMessage &rpc_call, const char *tagname, const char *basepath="", const char *xsdtype="", const char *prefix="");
@@ -129,7 +129,7 @@ public:
         appendStringBuffer(s, value);
     }
 
-    virtual void toJSONValue(StringBuffer &s)
+    virtual void toJSONValue(StringBuffer &s, bool encode)
     {
         if (!isNil)
             appendJSONValue(s, NULL, value);
@@ -194,7 +194,7 @@ public:
         s.append(value);
     }
 
-    virtual void toJSONValue(StringBuffer &s)
+    virtual void toJSONValue(StringBuffer &s, bool encode)
     {
         if (!isNil)
             appendJSONValue(s, NULL, value);
@@ -272,8 +272,10 @@ public:
         encodeNewlines = b;
     }
 
+    void toStr(IEspContext* ctx, StringBuffer &str, const char *tagname, const char *basepath="", bool encodeXml=true, const char *xsdtype="", const char *prefix="", bool encodeJSON=true);
+
     virtual void toXMLValue(StringBuffer &s, bool encode);
-    virtual void toJSONValue(StringBuffer &s);
+    virtual void toJSONValue(StringBuffer &s, bool encode);
 
     void addRpcValue(IRpcMessage &rpc_call, const char *tagname, const char *basepath, const char *xsdtype, const char *prefix, bool *encodex);
     virtual bool updateValue(IRpcMessage &rpc_call, const char *path);

esp/scm/ws_workunits.ecm

@@ -704,7 +704,7 @@ ESPresponse [exceptions_inline,http_encode(0)] WUResultResponse
     int Count;
     int64 Total;
 
-    string Result;
+    [json_inline(1)] string Result;
 };
 
 ESPrequest WUResultViewRequest

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -2138,7 +2138,7 @@ int CWsEclBinding::getWsEcl2Form(CHttpRequest* request, CHttpResponse* response,
     return 0;
 }
 
-int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, const char *xml, StringBuffer &out, unsigned flags, const char *viewname, const char *xsltname)
+int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, const char *xml, StringBuffer &out, unsigned flags, TextMarkupFormat fmt, const char *viewname, const char *xsltname)
 {
     Owned <IWorkUnitFactory> factory = getSecWorkUnitFactory(*context.querySecManager(), *context.queryUser());
     Owned <IWorkUnit> workunit = factory->createWorkUnit(NULL, "wsecl", context.queryUserId());
@@ -2193,6 +2193,8 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf
             web->renderResults(viewname, out);
         else if (xsltname)
             web->applyResultsXSLT(xsltname, out);
+        else if (fmt==MarkupFmt_JSON)
+            web->renderResultsJSON(out, context.queryRequestParameters()->queryProp("jsonp"));
         else
             web->expandResults(out, flags);
     }
@@ -2378,15 +2380,7 @@ int CWsEclBinding::onSubmitQueryOutput(IEspContext &context, CHttpRequest* reque
         }
     }
     else
-    {
-        submitWsEclWorkunit(context, wsinfo, soapmsg.str(), output, xmlflags);
-        if (outputJSON)
-        {
-            StringBuffer jsonresp;
-            getWsEclJsonResponse(jsonresp, context, request, output.str(), wsinfo);
-            output.swapWith(jsonresp);
-        }
-    }
+        submitWsEclWorkunit(context, wsinfo, soapmsg.str(), output, xmlflags, outputJSON ? MarkupFmt_JSON : MarkupFmt_XML);
 
     response->setContent(output.str());
     response->setContentType(outputJSON ? "application/json" : "application/xml");
@@ -2427,7 +2421,7 @@ int CWsEclBinding::onSubmitQueryOutputView(IEspContext &context, CHttpRequest* r
     }
     else
     {
-        submitWsEclWorkunit(context, wsinfo, soapmsg.str(), html, 0, view, xsltfile.str());
+        submitWsEclWorkunit(context, wsinfo, soapmsg.str(), html, 0, MarkupFmt_XML, view, xsltfile.str());
     }
 
     response->setContent(html.str());
@@ -2937,7 +2931,6 @@ void CWsEclBinding::handleJSONPost(CHttpRequest *request, CHttpResponse *respons
             if (getEspLogLevel()>LogNormal)
                 DBGLOG("soap from json req: %s", soapfromjson.str());
 
-            StringBuffer soapresp;
             unsigned xmlflags = WWV_ADD_SOAP | WWV_ADD_RESULTS_TAG | WWV_ADD_RESPONSE_TAG | WWV_INCL_NAMESPACES | WWV_INCL_GENERATED_NAMESPACES;
             if (ctx->queryRequestParameters()->hasProp("display"))
                 xmlflags |= WWV_USE_DISPLAY_XSLT;
@@ -2946,10 +2939,9 @@ void CWsEclBinding::handleJSONPost(CHttpRequest *request, CHttpResponse *respons
             else
                 xmlflags |= WWV_OMIT_SCHEMAS;
 
-            submitWsEclWorkunit(*ctx, wsinfo, soapfromjson.str(), soapresp, xmlflags);
+            submitWsEclWorkunit(*ctx, wsinfo, soapfromjson.str(), jsonresp, xmlflags, MarkupFmt_JSON);
             if (getEspLogLevel()>LogNormal)
-                DBGLOG("HandleSoapRequest response: %s", soapresp.str());
-            getWsEclJsonResponse(jsonresp, *ctx, request, soapresp.str(), wsinfo);
+                DBGLOG("HandleJSONRequest response: %s", jsonresp.str());
         }
 
     }

esp/services/ws_ecl/ws_ecl_service.hpp

@@ -168,7 +168,7 @@ public:
     int onSubmitQueryOutput(IEspContext &context, CHttpRequest* request, CHttpResponse* response,    WsEclWuInfo &wsinfo, const char *format);
     int onSubmitQueryOutputView(IEspContext &context, CHttpRequest* request, CHttpResponse* response, WsEclWuInfo &wsinfo);
 
-    int submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, const char *xml, StringBuffer &out, unsigned flags, const char *viewname=NULL, const char *xsltname=NULL);
+    int submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, const char *xml, StringBuffer &out, unsigned flags, TextMarkupFormat fmt=MarkupFmt_XML, const char *viewname=NULL, const char *xsltname=NULL);
 
     void handleHttpPost(CHttpRequest *request, CHttpResponse *response);
     void handleJSONPost(CHttpRequest *request, CHttpResponse *response);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2476,7 +2476,7 @@ bool CWsWorkunitsEx::onWUQuery(IEspContext &context, IEspWUQueryRequest & req, I
     return true;
 }
 
-void appendResultSet(MemoryBuffer& mb, INewResultSet* result, const char *name, __int64 start, unsigned& count, __int64& total, bool bin, bool xsd)
+void appendResultSet(MemoryBuffer& mb, INewResultSet* result, const char *name, __int64 start, unsigned& count, __int64& total, bool bin, bool xsd, ESPSerializationFormat fmt)
 {
     if (!result)
         return;
@@ -2502,7 +2502,10 @@ void appendResultSet(MemoryBuffer& mb, INewResultSet* result, const char *name,
             MemoryBuffer & buffer;
         } adaptor(mb);
 
-        count = getResultXml(adaptor, result, name, (unsigned) start, count, (xsd) ? "myschema" : NULL);
+        if (fmt==ESPSerializationJSON)
+            count = getResultJSON(adaptor, result, name, (unsigned) start, count);
+        else
+            count = getResultXml(adaptor, result, name, (unsigned) start, count, (xsd) ? "myschema" : NULL);
     }
 }
 
@@ -2549,7 +2552,7 @@ void getWsWuResult(IEspContext &context, const char* wuid, const char *name, con
     }
     else
         rs.setown(resultSetFactory->createNewResultSet(result, wuid));
-    appendResultSet(mb, rs, name, start, count, total, bin, xsd);
+    appendResultSet(mb, rs, name, start, count, total, bin, xsd, context.getResponseFormat());
 }
 
 void openSaveFile(IEspContext &context, int opt, const char* filename, const char* origMimeType, MemoryBuffer& buf, IEspWULogFileResponse &resp)
@@ -2910,7 +2913,7 @@ void getFileResults(IEspContext &context, const char* logicalName, const char* c
 {
     Owned<IResultSetFactory> resultSetFactory = getSecResultSetFactory(context.querySecManager(), context.queryUser(), context.queryUserId(), context.queryPassword());
     Owned<INewResultSet> result(resultSetFactory->createNewFileResultSet(logicalName, cluster));
-    appendResultSet(buf, result, resname.str(), start, count, total, bin, xsd);
+    appendResultSet(buf, result, resname.str(), start, count, total, bin, xsd, context.getResponseFormat());
 }
 
 void getWorkunitCluster(IEspContext &context, const char* wuid, SCMStringBuffer& cluster, bool checkArchiveWUs)
@@ -2963,6 +2966,8 @@ bool CWsWorkunitsEx::onWUResult(IEspContext &context, IEspWUResultRequest &req,
         filter.appendf(";seq=%d;", seq);
         if (inclXsd)
             filter.append("xsd;");
+        if (context.getResponseFormat()==ESPSerializationJSON)
+            filter.append("json;");
 
         const char* logicalName = req.getLogicalName();
         const char* clusterName = req.getCluster();

rtl/eclrtl/rtlfield.cpp

@@ -1026,21 +1026,20 @@ size32_t RtlSetTypeInfo::toXML(const byte * self, const byte * selfrow, const Rt
         target.outputBeginNested(outerTag, false);
     }
 
-    const char *innerPath = queryXPath(field);
-    target.outputBeginArray(innerPath);
-
     if (*(bool *)self)
         target.outputSetAll();
     else
     {
+        const char *innerPath = queryXPath(field);
+        target.outputBeginArray(innerPath);
         while (offset < max)
         {
             child->toXML(self+offset, selfrow, field, target);
             offset += child->size(self+offset, selfrow);
         }
+        target.outputEndArray(innerPath);
     }
 
-    target.outputEndArray(innerPath);
     if (outerTag)
         target.outputEndNested(outerTag);
     return max;

rtl/eclrtl/rtlxml.cpp

@@ -245,7 +245,7 @@ void outputJsonUnicode(unsigned len, const UChar *field, const char *fieldname,
     char * buff = 0;
     unsigned bufflen = 0;
     rtlUnicodeToCodepageX(bufflen, buff, len, field, "utf-8");
-    appendJSONValue(out, fieldname, bufflen, buff); // output as UTF-8
+    appendJSONStringValue(out, fieldname, bufflen, buff, true); // output as UTF-8
     rtlFree(buff);
 }
 

rtl/include/eclhelper.hpp

@@ -168,6 +168,8 @@ public:
     virtual void outputUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname) = 0;
     virtual void outputUnicode(unsigned len, const UChar *field, const char *fieldname) = 0;
     virtual void outputQString(unsigned len, const char *field, const char *fieldname) = 0;
+    virtual void outputBeginDataset(const char *dsname, bool nestChildren) = 0;
+    virtual void outputEndDataset(const char *dsname) = 0;
     virtual void outputBeginNested(const char *fieldname, bool nestChildren) = 0;
     virtual void outputEndNested(const char *fieldname) = 0;
     virtual void outputSetAll() = 0;

system/jlib/jstring.cpp

@@ -1953,7 +1953,7 @@ jlib_decl StringBuffer &appendfJSONName(StringBuffer &s, const char *format, ...
 }
 
 static char hexchar[] = "0123456789ABCDEF";
-jlib_decl StringBuffer &appendJSONValue(StringBuffer& s, const char *name, unsigned len, const void *_value)
+jlib_decl StringBuffer &appendJSONDataValue(StringBuffer& s, const char *name, unsigned len, const void *_value)
 {
     appendJSONNameOrDelimit(s, name);
     s.append('"');

system/jlib/jstring.hpp

@@ -439,7 +439,7 @@ jlib_decl StringBuffer &encodeJSON(StringBuffer &s, unsigned len, const char *va
 
 jlib_decl StringBuffer &appendJSONName(StringBuffer &s, const char *name);
 jlib_decl StringBuffer &appendfJSONName(StringBuffer &s, const char *format, ...);
-jlib_decl StringBuffer &appendJSONValue(StringBuffer& s, const char *name, unsigned len, const void *_value);
+jlib_decl StringBuffer &appendJSONDataValue(StringBuffer& s, const char *name, unsigned len, const void *_value);
 
 inline StringBuffer &appendJSONNameOrDelimit(StringBuffer &s, const char *name)
 {
@@ -448,6 +448,27 @@ inline StringBuffer &appendJSONNameOrDelimit(StringBuffer &s, const char *name)
     return delimitJSON(s);
 }
 
+inline StringBuffer &appendJSONStringValue(StringBuffer& s, const char *name, unsigned len, const char *value, bool encode, bool quoted=true)
+{
+    appendJSONNameOrDelimit(s, name);
+    if (!value)
+        return s.append("null");
+    if (quoted)
+        s.append('"');
+    if (encode)
+        encodeJSON(s, len, value);
+    else
+        s.append(len, value);
+    if (quoted)
+        s.append('"');
+    return s;
+}
+
+inline StringBuffer &appendJSONStringValue(StringBuffer& s, const char *name, const char *value, bool encode, bool quoted=true)
+{
+    return appendJSONStringValue(s, name, value ? strlen(value) : 0, value, encode, quoted);
+}
+
 template <typename type>
 inline StringBuffer &appendJSONValue(StringBuffer& s, const char *name, type value)
 {
@@ -466,10 +487,7 @@ inline StringBuffer &appendJSONValue(StringBuffer& s, const char *name, bool val
 template <>
 inline StringBuffer &appendJSONValue(StringBuffer& s, const char *name, const char *value)
 {
-    appendJSONNameOrDelimit(s, name);
-    if (!value)
-        return s.append("null");
-    return encodeJSON(s.append('"'), value).append('"');
+    return appendJSONStringValue(s, name, value, true);
 }
 
 template <>
@@ -486,14 +504,6 @@ inline StringBuffer &appendJSONValue(StringBuffer& s, const char *name, unsigned
     return s.appendulong(value);
 }
 
-inline StringBuffer &appendJSONValue(StringBuffer& s, const char *name, unsigned len, const char *value)
-{
-    appendJSONNameOrDelimit(s, name);
-    if (!value)
-        return s.append("null");
-    return encodeJSON(s.append('"'), len, value).append('"');
-}
-
 extern jlib_decl void decodeCppEscapeSequence(StringBuffer & out, const char * in, bool errorIfInvalid);
 extern jlib_decl bool strToBool(const char * text);
 extern jlib_decl bool strToBool(size_t len, const char * text);

tools/hidl/hidlcomp.cpp

@@ -1955,12 +1955,16 @@ void ParamInfo::write_esp_marshall(bool isRpc, bool encodeXml, bool checkVer, in
         {
             outf("\"%s\", \"\", ", getXmlTag());
             if (isRpc)
-                outf("\"\", ");
+                outf("\"\", %s);\n", prefix);
             else if (kind==TK_ESPSTRUCT)
-                outf("false, \"\", ");
+                outf("false, \"\", %s);\n", prefix);
             else
-                outf("%s, \"\", ", encode);
-            outf("%s);\n", prefix);
+            {
+                outf("%s, \"\", %s", encode, prefix);
+                if (getMetaInt("json_inline"))
+                    outs(", false");
+                outs(");\n");
+            }
         }
     }
 }