Merge pull request #6405 from rpastrana/HPCC-11922-NEW-JSON-Support

HPCC-11922 new json support

Reviewed-By: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/roxiedebug.cpp

@@ -410,7 +410,7 @@ public:
     virtual void outputInlineXml(const char *xml){}
     virtual void outputQuoted(const char *text) { }
 
-    virtual void outputString(unsigned len, const char *field, const char *fieldname) 
+    virtual void outputString(unsigned len, const char *field, const char *fieldname)
     {
         if (!matchSeen && checkFieldName(fieldname) && (len >= searchStringLength))
         {
@@ -480,7 +480,7 @@ public:
     {
         // Searching/breaking on unicode not supported at the moment
     }
-    virtual void outputQString(unsigned len, const char *field, const char *fieldname) 
+    virtual void outputQString(unsigned len, const char *field, const char *fieldname)
     {
         if (!matchSeen && checkFieldName(fieldname))
         {

common/thorhelper/thorxmlwrite.cpp

@@ -103,6 +103,7 @@ void CommonXmlWriter::outputQString(unsigned len, const char *field, const char
     else
         temp = (char *)tempBuffer.allocate(len);
     rtlQStrToStr(len, temp, len, field);
+//    outputString(len, temp, fieldname, isnumeric);
     outputString(len, temp, fieldname);
 }
 
@@ -439,6 +440,18 @@ void CommonJsonWriter::outputQuoted(const char *text)
     appendJSONValue(out, NULL, text);
 }
 
+void CommonJsonWriter::outputNumericString(const char *field, const char *fieldname)
+{
+    unsigned len = (size32_t)strlen(field);
+
+    if (flags & XWFtrim)
+        len = rtlTrimStrLen(len, field);
+    if ((flags & XWFopt) && (rtlTrimStrLen(len, field) == 0))
+        return;
+    checkDelimit();
+    appendJSONStringValue(out, checkItemName(fieldname), len, field, true, false);
+}
+
 void CommonJsonWriter::outputString(unsigned len, const char *field, const char *fieldname)
 {
     if (flags & XWFtrim)

common/thorhelper/thorxmlwrite.hpp

@@ -42,6 +42,8 @@ interface IXmlWriterExt : extends IXmlWriter
     virtual IXmlWriterExt & clear() = 0;
     virtual size32_t length() const = 0;
     virtual const char *str() const = 0;
+    virtual void rewindTo(unsigned int prevlen) = 0;
+    virtual void outputNumericString(const char *field, const char *fieldname) = 0;
 };
 
 class thorhelper_decl CommonXmlWriter : public CInterface, implements IXmlWriterExt
@@ -80,6 +82,18 @@ public:
     virtual IXmlWriterExt & clear();
     virtual unsigned length() const                                 { return out.length(); }
     virtual const char * str() const                                { return out.str(); }
+    virtual void rewindTo(unsigned int prevlen)
+    {
+        if (flusher)
+            throwUnexpected();
+
+        if (prevlen < out.length()) out.setLength(prevlen);
+    }
+
+    virtual void outputNumericString(const char *field, const char *fieldname)
+    {
+        outputCString(field, fieldname);
+    }
 
 protected:
     bool checkForAttribute(const char * fieldname);
@@ -134,11 +148,13 @@ public:
     virtual void outputEndArray(const char *fieldname);
     virtual void outputSetAll();
     virtual void outputXmlns(const char *name, const char *uri){}
+    virtual void outputNumericString(const char *field, const char *fieldname);
 
     //IXmlWriterExt
     virtual IXmlWriterExt & clear();
     virtual unsigned length() const                                 { return out.length(); }
     virtual const char * str() const                                { return out.str(); }
+    virtual void rewindTo(unsigned int prevlen)                     { if (prevlen < out.length()) out.setLength(prevlen); }
 
     void outputBeginRoot(){out.append('{');}
     void outputEndRoot(){out.append('}');}

esp/bindings/http/platform/httptransport.cpp

@@ -1451,6 +1451,8 @@ void CHttpRequest::parseEspPathInfo()
                     m_sstype=sub_serv_soap_builder;
                 else if (m_queryparams && (m_queryparams->hasProp("roxie_builder_")))
                     m_sstype=sub_serv_roxie_builder;
+                else if (m_queryparams && (m_queryparams->hasProp("json_builder_")))
+                    m_sstype=sub_serv_json_builder;
                 else if (m_queryparams && m_queryparams->hasProp("config_"))
                     m_sstype=sub_serv_config;
                 else if (m_espServiceName.length()==0)

esp/bindings/http/platform/httptransport.ipp

@@ -264,6 +264,7 @@ typedef enum sub_service_
     sub_serv_instant_query,
     sub_serv_soap_builder,
     sub_serv_roxie_builder,
+    sub_serv_json_builder,
     sub_serv_wsdl,
     sub_serv_xsd,
     sub_serv_config,

esp/files/gen_form.js

@@ -375,6 +375,14 @@ function onSubmit(reqType)  // reqType: 0: regular form, 1: soap, 2: form param
                 form.action += c + "roxie_builder_";
          }
     }
+    if (reqType==4)
+    {
+         if (form.action.indexOf('json_builder_')<0) // add only if does not exist already
+         {
+                var c =  (form.action.indexOf('?')>0) ? '&' : '?';
+                form.action += c + "json_builder_";
+         }
+    }
     // alert("Form action = " + form.action);
 
     // firefox now save input values (version 1.5)  

esp/services/CMakeLists.txt

@@ -13,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
+HPCC_ADD_SUBDIRECTORY (common)
 HPCC_ADD_SUBDIRECTORY (ecldirect "PLATFORM")
 IF (USE_OPENLDAP)
     HPCC_ADD_SUBDIRECTORY (ws_access "PLATFORM")

esp/services/common/CMakeLists.txt

@@ -13,3 +13,10 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
+# Component: esp_services_common
+
+project( esp_services_common )
+
+set ( SRCS
+    ${HPCC_SOURCE_DIR}/esp/services/common/wsexcept.cpp
+)

esp/services/common/jsonhelpers.hpp

@@ -0,0 +1,411 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+// jsonhelpers.hpp:
+//
+//////////////////////////////////////////////////////////////////////
+#pragma warning( disable : 4786)
+
+#ifndef _JSONHELPERS_HPP__
+#define _JSONHELPERS_HPP__
+#include "jliball.hpp"
+#include "wsexcept.hpp"
+
+#define REQSF_ROOT         0x0001
+#define REQSF_SAMPLE_DATA  0x0002
+#define REQSF_TRIM         0x0004
+#define REQSF_ESCAPEFORMATTERS 0x0008
+#define REQSF_EXCLUSIVE (REQSF_SAMPLE_DATA | REQSF_TRIM)
+
+namespace HttpParamHelpers
+{
+    static const char * nextParameterTag(StringBuffer &tag, const char *path)
+    {
+        while (*path=='.')
+            path++;
+        const char *finger = strchr(path, '.');
+        if (finger)
+        {
+            tag.clear().append(finger - path, path);
+            finger++;
+        }
+        else
+            tag.set(path);
+        return finger;
+    }
+
+    static void ensureParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
+    {
+        if (!tag.length())
+            return;
+
+        unsigned idx = 1;
+        if (path && isdigit(*path))
+        {
+            StringBuffer pos;
+            path = nextParameterTag(pos, path);
+            idx = (unsigned) atoi(pos.str())+1;
+            if (idx>25) //adf
+                throw MakeStringException(-1, "Array items above 25 not supported in HPCC WS HTTP parameters: %s", fullpath);
+        }
+
+        if (tag.charAt(tag.length()-1)=='$')
+        {
+            if (path && *path)
+                throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
+            tag.setLength(tag.length()-1);
+            StringArray values;
+            values.appendList(value, "\r");
+            ForEachItemIn(pos, values)
+            {
+                const char *itemValue = values.item(pos);
+                while (*itemValue=='\n')
+                    itemValue++;
+                pt->addProp(tag, itemValue);
+            }
+            return;
+        }
+        unsigned count = pt->getCount(tag);
+        while (count++ < idx)
+            pt->addPropTree(tag, createPTree(tag));
+        StringBuffer xpath(tag);
+        xpath.append('[').append(idx).append(']');
+        pt = pt->queryPropTree(xpath);
+
+        if (!path || !*path)
+        {
+            pt->setProp(NULL, value);
+            return;
+        }
+
+        StringBuffer nextTag;
+        path = HttpParamHelpers::nextParameterTag(nextTag, path);
+        ensureParameter(pt, nextTag, path, value, fullpath);
+    }
+
+    static void ensureParameter(IPropertyTree *pt, const char *path, const char *value)
+    {
+        const char *fullpath = path;
+        StringBuffer tag;
+        path = HttpParamHelpers::nextParameterTag(tag, path);
+        ensureParameter(pt, tag, path, value, fullpath);
+    }
+
+    static IPropertyTree *createPTreeFromHttpParameters(const char *name, IProperties *parameters)
+    {
+        Owned<IPropertyTree> pt = createPTree(name);
+        Owned<IPropertyIterator> props = parameters->getIterator();
+        ForEach(*props)
+        {
+            const char *key = props->getPropKey();
+            const char *value = parameters->queryProp(key);
+            ensureParameter(pt, key, value);
+        }
+        return pt.getClear();
+    }
+};
+
+namespace JsonHelpers
+{
+    static StringBuffer &appendJSONExceptionItem(StringBuffer &s, int code, const char *msg, const char *objname="Exceptions", const char *arrayName = "Exception")
+    {
+        if (objname && *objname)
+            appendJSONName(s, objname).append('{');
+        if (arrayName && *arrayName)
+            appendJSONName(s, arrayName).append('[');
+        delimitJSON(s);
+        s.append('{');
+        appendJSONValue(s, "Code", code);
+        appendJSONValue(s, "Message", msg);
+        s.append('}');
+        if (arrayName && *arrayName)
+            s.append(']');
+        if (objname && *objname)
+            s.append('}');
+        return s;
+    }
+
+    static StringBuffer &appendJSONException(StringBuffer &s, IException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
+    {
+        if (!e)
+            return s;
+        StringBuffer temp;
+        return appendJSONExceptionItem(s, e->errorCode(), e->errorMessage(temp).str(), objname, arrayName);
+    }
+
+    static StringBuffer &appendJSONException(StringBuffer &s, IWsException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
+    {
+        if (!e)
+            return s;
+
+        StringBuffer temp;
+        IArrayOf<IException>& exceptions = e->getArray();
+        for (int i = 0 ; i < exceptions.ordinality(); i++)
+        {
+            appendJSONExceptionItem(s, e->errorCode(), e->errorMessage(temp.clear()).str(), objname, arrayName);
+        }
+
+        return s;
+    }
+
+    static StringBuffer &appendJSONExceptions(StringBuffer &s, IMultiException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
+    {
+        if (!e)
+            return s;
+        if (objname && *objname)
+            appendJSONName(s, objname).append('{');
+        if (arrayName && *arrayName)
+            appendJSONName(s, arrayName).append('[');
+        ForEachItemIn(i, *e)
+            appendJSONException(s, &e->item(i), NULL, NULL);
+        if (arrayName && *arrayName)
+            s.append(']');
+        if (objname && *objname)
+            s.append('}');
+        return s;
+    }
+
+    static IException *MakeJSONValueException(int code, const char *start, const char *pos, const char *tail, const char *intro="Invalid json format: ")
+    {
+         StringBuffer s(intro);
+         s.append(pos-start, start).append('^').append(pos);
+         if (tail && *tail)
+             s.append(" - ").append(tail);
+         return MakeStringException(code, "%s", s.str());
+    }
+
+    static inline StringBuffer &jsonNumericNext(StringBuffer &s, const char *&c, bool &allowDecimal, bool &allowExponent, const char *start)
+    {
+        if (isdigit(*c))
+            s.append(*c++);
+        else if ('.'==*c)
+        {
+            if (!allowDecimal || !allowExponent)
+                throw MakeJSONValueException(-1, start, c, "Unexpected decimal");
+            allowDecimal=false;
+            s.append(*c++);
+        }
+        else if ('e'==*c || 'E'==*c)
+        {
+            if (!allowExponent)
+                throw MakeJSONValueException(-1, start, c, "Unexpected exponent");
+
+            allowDecimal=false;
+            allowExponent=false;
+            s.append(*c++);
+            if ('-'==*c || '+'==*c)
+                s.append(*c++);
+            if (!isdigit(*c))
+                throw MakeJSONValueException(-1, start, c, "Unexpected token");
+        }
+        else
+            throw MakeJSONValueException(-1, start, c, "Unexpected token");
+
+        return s;
+    }
+
+    static inline StringBuffer &jsonNumericStart(StringBuffer &s, const char *&c, const char *start)
+    {
+        if ('-'==*c)
+            return jsonNumericStart(s.append(*c++), c, start);
+        else if ('0'==*c)
+        {
+            s.append(*c++);
+            if (*c && '.'!=*c)
+                throw MakeJSONValueException(-1, start, c, "Unexpected token");
+        }
+        else if (isdigit(*c))
+            s.append(*c++);
+        else
+            throw MakeJSONValueException(-1, start, c, "Unexpected token");
+        return s;
+    }
+
+    static StringBuffer &appendJSONNumericString(StringBuffer &s, const char *value, bool allowDecimal)
+    {
+        if (!value || !*value)
+            return s.append("null");
+
+        bool allowExponent = allowDecimal;
+
+        const char *pos = value;
+        jsonNumericStart(s, pos, value);
+        while (*pos)
+            jsonNumericNext(s, pos, allowDecimal, allowExponent, value);
+        return s;
+    }
+
+    typedef enum _JSONFieldCategory
+    {
+        JSONField_String,
+        JSONField_Integer,
+        JSONField_Real,
+        JSONField_Boolean,
+        JSONField_Present  //true or remove
+    } JSONField_Category;
+
+    static JSONField_Category xsdTypeToJSONFieldCategory(const char *xsdtype)
+    {
+        //map XML Schema types used in ECL generated schemas to basic JSON formatting types
+        if (streq(xsdtype, "integer") || streq(xsdtype, "nonNegativeInteger"))
+            return JSONField_Integer;
+        if (streq(xsdtype, "boolean"))
+            return JSONField_Boolean;
+        if (streq(xsdtype, "double"))
+            return JSONField_Real;
+        if (!strncmp(xsdtype, "decimal", 7)) //ecl creates derived types of the form decimal#_#
+            return JSONField_Real;
+        if (streq(xsdtype, "none")) //maps to an eml schema element with no type.  set to true or don't add
+            return JSONField_Present;
+        return JSONField_String;
+    }
+
+    static void buildJsonAppendValue(IXmlType* type, StringBuffer& out, const char* tag, const char *value, unsigned flags)
+    {
+        JSONField_Category ct = xsdTypeToJSONFieldCategory(type->queryName());
+
+        if (ct==JSONField_Present && (!value || !*value))
+            return;
+
+        if (tag && *tag)
+            out.appendf("\"%s\": ", tag);
+        StringBuffer sample;
+        if ((!value || !*value) && (flags & REQSF_SAMPLE_DATA))
+        {
+            type->getSampleValue(sample, NULL);
+            value = sample.str();
+        }
+
+        if (value)
+        {
+            switch (ct)
+            {
+            case JSONField_String:
+                appendJSONValue(out, NULL, value);
+                break;
+            case JSONField_Integer:
+                appendJSONNumericString(out, value, false);
+                break;
+            case JSONField_Real:
+                appendJSONNumericString(out, value, true);
+                break;
+            case JSONField_Boolean:
+                if (strieq(value, "default"))
+                    out.append("null");
+                else
+                    appendJSONValue(out, NULL, strToBool(value));
+                break;
+            case JSONField_Present:
+                appendJSONValue(out, NULL, true);
+                break;
+            }
+        }
+        else
+            out.append("null");
+    }
+    static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer& out, const char* tag, IPropertyTree *reqTree, unsigned flags)
+    {
+        assertex(type!=NULL);
+
+        if (flags & REQSF_ROOT)
+            out.append("{");
+
+        const char* typeName = type->queryName();
+        if (type->isComplexType())
+        {
+            if (typeName && !parentTypes.appendUniq(typeName))
+                return; // recursive
+
+            int startlen = out.length();
+            if (tag)
+                appendJSONName(out, tag);
+            out.append('{');
+            int taglen=out.length()+1;
+            if (type->getSubType()==SubType_Complex_SimpleContent)
+            {
+                if (reqTree)
+                {
+                    const char *attrval = reqTree->queryProp(NULL);
+                    out.appendf("\"%s\" ", (attrval) ? attrval : "");
+                }
+                else if (flags & REQSF_SAMPLE_DATA)
+                {
+                    out.append("\"");
+                    type->queryFieldType(0)->getSampleValue(out,tag);
+                    out.append("\" ");
+                }
+            }
+            else
+            {
+                int flds = type->getFieldCount();
+                for (int idx=0; idx<flds; idx++)
+                {
+                    delimitJSON(out);
+                    IPropertyTree *childtree = NULL;
+                    const char *childname = type->queryFieldName(idx);
+                    if (reqTree)
+                        childtree = reqTree->queryPropTree(childname);
+                    buildJsonMsg(parentTypes, type->queryFieldType(idx), out, childname, childtree, flags & ~REQSF_ROOT);
+                }
+            }
+
+            if (typeName)
+                parentTypes.pop();
+            out.append("}");
+        }
+        else if (type->isArray())
+        {
+            if (typeName && !parentTypes.appendUniq(typeName))
+                return; // recursive
+
+            const char* itemName = type->queryFieldName(0);
+            IXmlType*   itemType = type->queryFieldType(0);
+            if (!itemName || !itemType)
+                throw MakeStringException(-1,"*** Invalid array definition: tag=%s, itemName=%s", tag, itemName?itemName:"NULL");
+
+            int startlen = out.length();
+            if (tag)
+                out.appendf("\"%s\": ", tag);
+            out.append('{');
+            out.appendf("\"%s\": [", itemName);
+            int taglen=out.length();
+            if (reqTree)
+            {
+                Owned<IPropertyTreeIterator> items = reqTree->getElements(itemName);
+                ForEach(*items)
+                    buildJsonMsg(parentTypes, itemType, delimitJSON(out), NULL, &items->query(), flags & ~REQSF_ROOT);
+            }
+            else
+                buildJsonMsg(parentTypes, itemType, out, NULL, NULL, flags & ~REQSF_ROOT);
+
+            out.append(']');
+
+            if (typeName)
+                parentTypes.pop();
+            out.append("}");
+        }
+        else // simple type
+        {
+            const char *parmval = (reqTree) ? reqTree->queryProp(NULL) : NULL;
+            buildJsonAppendValue(type, out, tag, parmval, flags);
+        }
+
+        if (flags & REQSF_ROOT)
+            out.append('}');
+    }
+};
+#endif // _JSONHELPERS_HPP__

esp/services/common/wsexcept.cpp

@@ -0,0 +1,192 @@
+#include "platform.h"
+#include "wsexcept.hpp"
+
+class CWsException : public CInterface, implements IWsException
+{
+public:
+    IMPLEMENT_IINTERFACE
+
+    CWsException (const char* source, WsErrorType errorType )
+    {
+        if (source)
+            exSource.append(source);
+        exErrorType = errorType;
+    }
+
+    CWsException( IMultiException& me, WsErrorType errorType )
+    {
+        append(me);
+        exErrorType = errorType;
+
+        const char* source = me.source();
+
+        if (source)
+            exSource.append(source);
+    }
+    CWsException( IException& e, const char* source, WsErrorType errorType )
+    {
+        IMultiException* me = dynamic_cast<IMultiException*>(&e);
+        if ( me ) {
+            append(*me);
+        } else
+            append(e);
+        exErrorType = errorType;
+
+        if (source)
+            exSource.append(source);
+    }
+
+    //convenience methods for handling this as an array
+    virtual aindex_t ordinality() const
+    {
+        synchronized block(mutex);
+        return exceptions.ordinality();
+    }
+    virtual IException& item(aindex_t pos) const
+    {
+        synchronized block(mutex);
+        return exceptions.item(pos);
+    }
+    virtual const char* source() const
+    {
+        synchronized block(mutex);
+        return exSource.str();
+    }
+
+    //for complete control...caller is responsible for thread safety!
+    virtual IArrayOf<IException>& getArray()     { return exceptions;              }
+
+    // add another exception. Pass ownership to this obj:
+    // i.e., caller needs to make sure that e has one consumable ref count
+    virtual void append(IException& e)
+    {
+        synchronized block(mutex);
+        exceptions.append(e);
+    }
+    virtual void append(IMultiException& me)
+    {
+        synchronized block(mutex);
+
+        IArrayOf<IException>& exceptions = me.getArray();
+        const char* source = me.source();
+        ForEachItemIn(i, exceptions)
+        {
+            IException& e = exceptions.item(i);
+            if (source && *source)
+            {
+                StringBuffer msg;
+                msg.appendf("[%s] ",source);
+                e.errorMessage(msg);
+                exceptions.append(*MakeStringExceptionDirect(e.errorAudience(), e.errorCode(), msg));
+            }
+            else
+                exceptions.append(*LINK(&e));
+        }
+    }
+
+
+    StringBuffer& serialize(StringBuffer& buffer, unsigned indent = 0, bool simplified=false, bool root=true) const
+    {
+        synchronized block(mutex);
+
+        if (root)
+            buffer.append("<Exceptions>");
+
+        if (!simplified)
+        {
+            if (indent) buffer.append("\n\t");
+            buffer.appendf("<Source>%s</Source>", exSource.str());
+        }
+
+        ForEachItemIn(i, exceptions)
+        {
+            IException& exception = exceptions.item(i);
+
+            if (indent) buffer.append("\n\t");
+            buffer.append("<Exception>");
+
+            if (indent) buffer.append("\n\t\t");
+            buffer.appendf("<Code>%d</Code>", exception.errorCode());
+
+            if (indent) buffer.append("\n\t\t");
+            buffer.appendf("<Audience>%s</Audience>", serializeMessageAudience( exception.errorAudience() ));
+
+            if (simplified)
+            {
+                if (indent) buffer.append("\n\t\t");
+                StringBuffer msg;
+                buffer.appendf("<Source>%s</Source>", exSource.str());
+            }
+
+            if (indent) buffer.append("\n\t\t");
+
+            StringBuffer msg;
+            StringBuffer encoded;
+            encodeXML(exception.errorMessage(msg).str(), encoded);
+            buffer.appendf("<Message>%s</Message>", encoded.str());
+
+            if (indent) buffer.append("\n\t");
+            buffer.append("</Exception>");
+        }
+
+        if (root)
+            buffer.append("</Exceptions>");
+        return buffer;
+    }
+
+    virtual int errorCode() const
+    {
+        synchronized block(mutex);
+        return ordinality() == 1 ? item(0).errorCode() : -1;
+    }
+    virtual StringBuffer& errorMessage(StringBuffer &msg) const
+    {
+        synchronized block(mutex);
+        ForEachItemIn(i, exceptions)
+        {
+            IException& e = item(i);
+
+            StringBuffer buf;
+            msg.appendf("[%3d: %s] ", e.errorCode(), e.errorMessage(buf).str());
+        }
+        return msg;
+    }
+    virtual MessageAudience errorAudience() const
+    {
+        synchronized block(mutex);
+        return ordinality() == 1 ? item(0).errorAudience() : MSGAUD_unknown;
+    }
+    virtual WsErrorType errorType() const
+    {
+        synchronized block(mutex);
+        return exErrorType;
+    }
+private:
+    CWsException( const CWsException& );
+    IArrayOf<IException> exceptions;
+    StringBuffer         exSource;
+    mutable Mutex        mutex;
+    WsErrorType          exErrorType;
+};
+
+IWsException esdl_decl *makeWsException(IMultiException& me, WsErrorType errorType)
+{
+    return new CWsException(me, errorType);
+}
+IWsException esdl_decl *makeWsException(IException& e, WsErrorType errorType, const char* source)
+{
+    return new CWsException(  e, source, errorType );
+}
+IWsException esdl_decl *makeWsException(int errorCode, WsErrorType errorType, const char* source, const char *format, ...)
+{
+    va_list args;
+    va_start(args, format);
+    IException *e = MakeStringExceptionVA(errorCode, format, args);
+    va_end(args);
+
+    return new CWsException(  *e, source, errorType );
+}
+IWsException esdl_decl *makeWsException(const char *source, WsErrorType errorType)
+{
+    return new CWsException(source, errorType);
+}

esp/services/common/wsexcept.hpp

@@ -0,0 +1,67 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef __WSEXCEPT__
+#define __WSEXCEPT__
+
+#include "jexcept.hpp"
+
+#ifdef _WIN32
+ #ifdef ESDLLIB_EXPORTS
+  #define esdl_decl __declspec(dllexport)
+ #else
+  #define esdl_decl
+ #endif
+#else
+ #define esdl_decl
+#endif
+
+typedef enum
+{
+    WSERR_NOERR=-1,WSERR_CLIENT, WSERR_SERVER, WSERR_VERSION, WSERR_MUSTUNDERSTAND
+} WsErrorType;
+
+interface IWsException : extends IException
+{
+   //convenience methods for handling this as an array
+   virtual aindex_t ordinality() const = 0;
+   virtual IException& item(aindex_t pos) const = 0;
+   virtual const char* source() const = 0;
+
+   //for complete control...
+   virtual IArrayOf<IException>& getArray()= 0;
+
+   //add another exception
+   virtual void append(IException& e) = 0;
+   virtual void append(IMultiException& e) = 0;
+
+   virtual StringBuffer& serialize(StringBuffer& ret, unsigned indent = 0, bool simplified=false, bool root=true) const = 0;
+
+   //the following methods override those in IIException
+   //
+    virtual int errorCode() const = 0;
+    virtual StringBuffer&     errorMessage(StringBuffer &msg) const = 0;
+    virtual MessageAudience errorAudience() const = 0;
+    virtual WsErrorType     errorType() const = 0;
+};
+
+IWsException esdl_decl *makeWsException(IMultiException& me, WsErrorType errorType);
+IWsException esdl_decl *makeWsException(const char *source, WsErrorType errorType);
+IWsException esdl_decl *makeWsException(IException& e, WsErrorType errorType, const char* source = NULL );
+IWsException esdl_decl *makeWsException(int errorCode, WsErrorType errorType, const char* source, const char *format, ...);
+
+#endif

esp/services/common/wshelpers.hpp

@@ -85,5 +85,4 @@ inline StringBuffer& operator<<(StringBuffer& buf, const JScript& j)
     return buf;
 }
 
-
 #endif // _ESPWIZ_WSHELPERS_HPP__

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -9,6 +9,7 @@
 #include "ws_ecl_wuinfo.hpp"
 #include "xsdparser.hpp"
 #include "httpclient.hpp"
+#include "jsonhelpers.hpp"
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
 
@@ -194,92 +195,6 @@ static void appendServerAddress(StringBuffer &s, IPropertyTree &env, IPropertyTr
     s.append(netAddress).append(':').append(port ? port : "9876");
 }
 
-
-const char *nextParameterTag(StringBuffer &tag, const char *path)
-{
-    while (*path=='.')
-        path++;
-    const char *finger = strchr(path, '.');
-    if (finger)
-    {
-        tag.clear().append(finger - path, path);
-        finger++;
-    }
-    else
-        tag.set(path);
-    return finger;
-}
-
-void ensureParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
-{
-    if (!tag.length())
-        return;
-
-    unsigned idx = 1;
-    if (path && isdigit(*path))
-    {
-        StringBuffer pos;
-        path = nextParameterTag(pos, path);
-        idx = (unsigned) atoi(pos.str())+1;
-        if (idx>25) //adf
-            throw MakeStringException(-1, "Array items above 25 not supported in WsECL HTTP parameters: %s", fullpath);
-    }
-
-    if (tag.charAt(tag.length()-1)=='$')
-    {
-        if (path && *path)
-            throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
-        tag.setLength(tag.length()-1);
-        StringArray values;
-        values.appendList(value, "\r");
-        ForEachItemIn(pos, values)
-        {
-            const char *itemValue = values.item(pos);
-            while (*itemValue=='\n')
-                itemValue++;
-            pt->addProp(tag, itemValue);
-        }
-        return;
-    }
-    unsigned count = pt->getCount(tag);
-    while (count++ < idx)
-        pt->addPropTree(tag, createPTree(tag));
-    StringBuffer xpath(tag);
-    xpath.append('[').append(idx).append(']');
-    pt = pt->queryPropTree(xpath);
-
-    if (!path || !*path)
-    {
-        pt->setProp(NULL, value);
-        return;
-    }
-
-    StringBuffer nextTag;
-    path = nextParameterTag(nextTag, path);
-    ensureParameter(pt, nextTag, path, value, fullpath);
-}
-
-void ensureParameter(IPropertyTree *pt, const char *path, const char *value)
-{
-    const char *fullpath = path;
-    StringBuffer tag;
-    path = nextParameterTag(tag, path);
-    ensureParameter(pt, tag, path, value, fullpath);
-}
-
-IPropertyTree *createPTreeFromHttpParameters(const char *name, IProperties *parameters)
-{
-    Owned<IPropertyTree> pt = createPTree(name);
-    Owned<IPropertyIterator> props = parameters->getIterator();
-    ForEach(*props)
-    {
-        const char *key = props->getPropKey();
-        const char *value = parameters->queryProp(key);
-        ensureParameter(pt, key, value);
-    }
-    return pt.getClear();
-}
-
 bool CWsEclService::init(const char * name, const char * type, IPropertyTree * cfg, const char * process)
 {
     StringBuffer xpath;
@@ -827,238 +742,6 @@ static void buildRestURL(StringArray& parentTypes, StringArray &path, IXmlType*
     }
 }
 
-
-IException *MakeJSONValueException(int code, const char *start, const char *pos, const char *tail, const char *intro="Invalid json format: ")
-{
-     StringBuffer s(intro);
-     s.append(pos-start, start).append('^').append(pos);
-     if (tail && *tail)
-         s.append(" - ").append(tail);
-     return MakeStringException(code, "%s", s.str());
-}
-
-inline StringBuffer &jsonNumericNext(StringBuffer &s, const char *&c, bool &allowDecimal, bool &allowExponent, const char *start)
-{
-    if (isdigit(*c))
-        s.append(*c++);
-    else if ('.'==*c)
-    {
-        if (!allowDecimal || !allowExponent)
-            throw MakeJSONValueException(-1, start, c, "Unexpected decimal");
-        allowDecimal=false;
-        s.append(*c++);
-    }
-    else if ('e'==*c || 'E'==*c)
-    {
-        if (!allowExponent)
-            throw MakeJSONValueException(-1, start, c, "Unexpected exponent");
-
-        allowDecimal=false;
-        allowExponent=false;
-        s.append(*c++);
-        if ('-'==*c || '+'==*c)
-            s.append(*c++);
-        if (!isdigit(*c))
-            throw MakeJSONValueException(-1, start, c, "Unexpected token");
-    }
-    else
-        throw MakeJSONValueException(-1, start, c, "Unexpected token");
-
-    return s;
-}
-
-inline StringBuffer &jsonNumericStart(StringBuffer &s, const char *&c, const char *start)
-{
-    if ('-'==*c)
-        return jsonNumericStart(s.append(*c++), c, start);
-    else if ('0'==*c)
-    {
-        s.append(*c++);
-        if (*c && '.'!=*c)
-            throw MakeJSONValueException(-1, start, c, "Unexpected token");
-    }
-    else if (isdigit(*c))
-        s.append(*c++);
-    else
-        throw MakeJSONValueException(-1, start, c, "Unexpected token");
-    return s;
-}
-
-StringBuffer &appendJSONNumericString(StringBuffer &s, const char *value, bool allowDecimal)
-{
-    if (!value || !*value)
-        return s.append("null");
-
-    bool allowExponent = allowDecimal;
-
-    const char *pos = value;
-    jsonNumericStart(s, pos, value);
-    while (*pos)
-        jsonNumericNext(s, pos, allowDecimal, allowExponent, value);
-    return s;
-}
-
-typedef enum _JSONFieldCategory
-{
-    JSONField_String,
-    JSONField_Integer,
-    JSONField_Real,
-    JSONField_Boolean,
-    JSONField_Present  //true or remove
-} JSONField_Category;
-
-JSONField_Category xsdTypeToJSONFieldCategory(const char *xsdtype)
-{
-    //map XML Schema types used in ECL generated schemas to basic JSON formatting types
-    if (streq(xsdtype, "integer") || streq(xsdtype, "nonNegativeInteger"))
-        return JSONField_Integer;
-    if (streq(xsdtype, "boolean"))
-        return JSONField_Boolean;
-    if (streq(xsdtype, "double"))
-        return JSONField_Real;
-    if (!strncmp(xsdtype, "decimal", 7)) //ecl creates derived types of the form decimal#_#
-        return JSONField_Real;
-    if (streq(xsdtype, "none")) //maps to an eml schema element with no type.  set to true or don't add
-        return JSONField_Present;
-    return JSONField_String;
-}
-
-static void buildJsonAppendValue(IXmlType* type, StringBuffer& out, const char* tag, const char *value, unsigned flags)
-{
-    JSONField_Category ct = xsdTypeToJSONFieldCategory(type->queryName());
-
-    if (ct==JSONField_Present && (!value || !*value))
-        return;
-
-    if (tag && *tag)
-        out.appendf("\"%s\": ", tag);
-    StringBuffer sample;
-    if ((!value || !*value) && (flags & REQSF_SAMPLE_DATA))
-    {
-        type->getSampleValue(sample, NULL);
-        value = sample.str();
-    }
-
-    if (value)
-    {
-        switch (ct)
-        {
-        case JSONField_String:
-            appendJSONValue(out, NULL, value);
-            break;
-        case JSONField_Integer:
-            appendJSONNumericString(out, value, false);
-            break;
-        case JSONField_Real:
-            appendJSONNumericString(out, value, true);
-            break;
-        case JSONField_Boolean:
-            if (strieq(value, "default"))
-                out.append("null");
-            else
-                appendJSONValue(out, NULL, strToBool(value));
-            break;
-        case JSONField_Present:
-            appendJSONValue(out, NULL, true);
-            break;
-        }
-    }
-    else
-        out.append("null");
-}
-
-static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer& out, const char* tag, IPropertyTree *reqTree, unsigned flags)
-{
-    assertex(type!=NULL);
-
-    if (flags & REQSF_ROOT)
-        out.append("{");
-
-    const char* typeName = type->queryName();
-    if (type->isComplexType())
-    {
-        if (typeName && !parentTypes.appendUniq(typeName))
-            return; // recursive
-
-        int startlen = out.length();
-        if (tag)
-            appendJSONName(out, tag);
-        out.append('{');
-        int taglen=out.length()+1;
-        if (type->getSubType()==SubType_Complex_SimpleContent)
-        {
-            if (reqTree)
-            {
-                const char *attrval = reqTree->queryProp(NULL);
-                out.appendf("\"%s\" ", (attrval) ? attrval : "");
-            }
-            else if (flags & REQSF_SAMPLE_DATA)
-            {
-                out.append("\"");
-                type->queryFieldType(0)->getSampleValue(out,tag);
-                out.append("\" ");
-            }
-        }
-        else
-        {
-            int flds = type->getFieldCount();
-            for (int idx=0; idx<flds; idx++)
-            {
-                delimitJSON(out);
-                IPropertyTree *childtree = NULL;
-                const char *childname = type->queryFieldName(idx);
-                if (reqTree)
-                    childtree = reqTree->queryPropTree(childname);
-                buildJsonMsg(parentTypes, type->queryFieldType(idx), out, childname, childtree, flags & ~REQSF_ROOT);
-            }
-        }
-
-        if (typeName)
-            parentTypes.pop();
-        out.append("}");
-    }
-    else if (type->isArray())
-    {
-        if (typeName && !parentTypes.appendUniq(typeName))
-            return; // recursive
-
-        const char* itemName = type->queryFieldName(0);
-        IXmlType*   itemType = type->queryFieldType(0);
-        if (!itemName || !itemType)
-            throw MakeStringException(-1,"*** Invalid array definition: tag=%s, itemName=%s", tag, itemName?itemName:"NULL");
-
-        int startlen = out.length();
-        if (tag)
-            out.appendf("\"%s\": ", tag);
-        out.append('{');
-        out.appendf("\"%s\": [", itemName);
-        int taglen=out.length();
-        if (reqTree)
-        {
-            Owned<IPropertyTreeIterator> items = reqTree->getElements(itemName);
-            ForEach(*items)
-                buildJsonMsg(parentTypes, itemType, delimitJSON(out), NULL, &items->query(), flags & ~REQSF_ROOT);
-        }
-        else
-            buildJsonMsg(parentTypes, itemType, out, NULL, NULL, flags & ~REQSF_ROOT);
-
-        out.append(']');
-
-        if (typeName)
-            parentTypes.pop();
-        out.append("}");
-    }
-    else // simple type
-    {
-        const char *parmval = (reqTree) ? reqTree->queryProp(NULL) : NULL;
-        buildJsonAppendValue(type, out, tag, parmval, flags);
-    }
-
-    if (flags & REQSF_ROOT)
-        out.append('}');
-
-}
-
 static inline StringBuffer &appendNamespaceSpecificString(StringBuffer &dest, const char *src)
 {
     if (src)
@@ -1766,7 +1449,7 @@ void CWsEclBinding::getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &cont
         return;
     }
 
-    Owned<IPropertyTree> reqTree = createPTreeFromHttpParameters(wsinfo.queryname, parameters);
+    Owned<IPropertyTree> reqTree = HttpParamHelpers::createPTreeFromHttpParameters(wsinfo.queryname, parameters);
 
     if (!validate)
         toXML(reqTree, soapmsg, 0, 0);
@@ -1792,56 +1475,13 @@ void CWsEclBinding::getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &cont
     }
 }
 
-StringBuffer &appendJSONExceptionItem(StringBuffer &s, int code, const char *msg, const char *objname="Exceptions", const char *arrayName = "Exception")
-{
-    if (objname && *objname)
-        appendJSONName(s, objname).append('{');
-    if (arrayName && *arrayName)
-        appendJSONName(s, arrayName).append('[');
-    delimitJSON(s);
-    s.append('{');
-    appendJSONValue(s, "Code", code);
-    appendJSONValue(s, "Message", msg);
-    s.append('}');
-    if (arrayName && *arrayName)
-        s.append(']');
-    if (objname && *objname)
-        s.append('}');
-    return s;
-}
-
-StringBuffer &appendJSONException(StringBuffer &s, IException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
-{
-    if (!e)
-        return s;
-    StringBuffer temp;
-    return appendJSONExceptionItem(s, e->errorCode(), e->errorMessage(temp).str(), objname, arrayName);
-}
-
-StringBuffer &appendJSONExceptions(StringBuffer &s, IMultiException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
-{
-    if (!e)
-        return s;
-    if (objname && *objname)
-        appendJSONName(s, objname).append('{');
-    if (arrayName && *arrayName)
-        appendJSONName(s, arrayName).append('[');
-    ForEachItemIn(i, *e)
-        appendJSONException(s, &e->item(i), NULL, NULL);
-    if (arrayName && *arrayName)
-        s.append(']');
-    if (objname && *objname)
-        s.append('}');
-    return s;
-}
-
 void CWsEclBinding::getWsEclJsonRequest(StringBuffer& jsonmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags, bool validate)
 {
     size32_t start = jsonmsg.length();
     try
     {
         IProperties *parameters = context.queryRequestParameters();
-        Owned<IPropertyTree> reqTree = createPTreeFromHttpParameters(wsinfo.queryname, parameters);
+        Owned<IPropertyTree> reqTree = HttpParamHelpers::createPTreeFromHttpParameters(wsinfo.queryname, parameters);
 
         if (!validate)
         {
@@ -1866,14 +1506,14 @@ void CWsEclBinding::getWsEclJsonRequest(StringBuffer& jsonmsg, IEspContext &cont
             if (type)
             {
                 StringArray parentTypes;
-                buildJsonMsg(parentTypes, type, jsonmsg, wsinfo.queryname.sget(), reqTree, flags|REQSF_ROOT);
+                JsonHelpers::buildJsonMsg(parentTypes, type, jsonmsg, wsinfo.queryname.sget(), reqTree, flags|REQSF_ROOT);
             }
         }
     }
     catch (IException *e)
     {
         jsonmsg.setLength(start);
-        appendJSONException(jsonmsg.append('{'), e);
+        JsonHelpers::appendJSONException(jsonmsg.append('{'), e);
         jsonmsg.append('}');
     }
 }
@@ -2201,7 +1841,7 @@ void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, Stri
         if (strieq(contentType, "application/json"))
         {
             resp.set("{").append("\"").append(query).append("Response\": {\"Results\": {");
-            appendJSONException(resp, e);
+            JsonHelpers::appendJSONException(resp, e);
             resp.append("}}}");
         }
         else
@@ -2687,7 +2327,7 @@ int CWsEclBinding::onGet(CHttpRequest* request, CHttpResponse* response)
 
             if (!wsecl->connMap.getValue(target.str()))
                 throw MakeStringException(-1, "Target cluster not mapped to roxie process!");
-            Owned<IPropertyTree> pt = createPTreeFromHttpParameters(qid.str(), parms);
+            Owned<IPropertyTree> pt = HttpParamHelpers::createPTreeFromHttpParameters(qid.str(), parms);
             StringBuffer soapreq(
                 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                 "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
@@ -2910,7 +2550,7 @@ void CWsEclBinding::handleJSONPost(CHttpRequest *request, CHttpResponse *respons
     }
     catch (IException *e)
     {
-        appendJSONException(jsonresp.set("{"), e);
+        JsonHelpers::appendJSONException(jsonresp.set("{"), e);
         jsonresp.append('}');
     }
 

esp/xslt/gen_form.xsl

@@ -39,6 +39,7 @@
     <xsl:param name="noDefaultValue" select="0"/> 
     <xsl:param name="includeSoapTest" select="1"/>
     <xsl:param name="includeRoxieTest" select="0"/>
+    <xsl:param name="includeJsonTest" select="0"/>
     <!--xsl:param name="includeGatewayTest" select="0"/-->
     <xsl:param name="schemaRoot" select="xsd:schema"/>
     <xsl:param name="esdl_links" select="0"/>
@@ -256,6 +257,9 @@
                    <xsl:if test="$includeRoxieTest">
                     &nbsp;<input type='submit' value='Roxie Test' onclick='return onSubmit(3)'/>
                    </xsl:if>
+                   <xsl:if test="$includeJsonTest">
+                    &nbsp;<input type='submit' value='Json Test' onclick='return onSubmit(4)'/>
+                   </xsl:if>
                    &nbsp;<input type='reset' value='Reset'  title='Reset the form'/>
                    &nbsp;<input type='button' value='Clear All' onclick='onClearAll()'  title='Reset the form, and remove all arrays you added'/>
                    &nbsp;<input type='button' value='Link to This Form' title='Generate a link to this page with form filled' onclick='onSubmit(2)'/>

rtl/include/eclhelper.hpp

@@ -181,7 +181,6 @@ public:
     inline void outputCString(const char *field, const char *fieldname) { outputString((size32_t)strlen(field), field, fieldname); }
 };
 
-
 interface IFieldProcessor : public IInterface
 {
 public: