HPCC-11922 Changes from code review

- IXmlWriterExt.rewindTo() not valid for flusher type
- Move wsexcept from LN/esp/esdllib to esp/services/common
- Move non json specific logic from JsonHelpers to HttpParamHelpers
- Cleared tmp strbuffer upon every appendjsonexceptions iteration
- Moved buildJsonMsg from ws_ecl to JsonHelpers
- ws_ecl now utilizes JsonHelpers

Signed-off-by: rpastrana <rodrigo.pastrana@lexisnexis.com>

common/thorhelper/thorxmlwrite.hpp

@@ -80,7 +80,13 @@ public:
     virtual IXmlWriterExt & clear();
     virtual unsigned length() const                                 { return out.length(); }
     virtual const char * str() const                                { return out.str(); }
-    virtual void rewindTo(unsigned int prevlen)                     { if (prevlen < out.length()) out.setLength(prevlen); }
+    virtual void rewindTo(unsigned int prevlen)
+    {
+        if (flusher)
+            UNIMPLEMENTED;
+
+        if (prevlen < out.length()) out.setLength(prevlen);
+    }
 
 protected:
     bool checkForAttribute(const char * fieldname);

esp/services/CMakeLists.txt

@@ -13,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
+HPCC_ADD_SUBDIRECTORY (common)
 HPCC_ADD_SUBDIRECTORY (ecldirect "PLATFORM")
 IF (USE_OPENLDAP)
     HPCC_ADD_SUBDIRECTORY (ws_access "PLATFORM")

esp/services/common/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,3 +13,10 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
+# Component: esp_services_common
+
+project( esp_services_common )
+
+set ( SRCS
+    ${HPCC_SOURCE_DIR}/esp/services/common/wsexcept.cpp
+)

esp/services/common/jsonhelpers.hpp

@@ -23,6 +23,7 @@
 #ifndef _JSONHELPERS_HPP__
 #define _JSONHELPERS_HPP__
 #include "jliball.hpp"
+#include "wsexcept.hpp"
 
 #define REQSF_ROOT         0x0001
 #define REQSF_SAMPLE_DATA  0x0002
@@ -30,6 +31,95 @@
 #define REQSF_ESCAPEFORMATTERS 0x0008
 #define REQSF_EXCLUSIVE (REQSF_SAMPLE_DATA | REQSF_TRIM)
 
+class HttpParamHelpers
+{
+public:
+    static void ensureParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
+    {
+        if (!tag.length())
+            return;
+
+        unsigned idx = 1;
+        if (path && isdigit(*path))
+        {
+            StringBuffer pos;
+            path = nextParameterTag(pos, path);
+            idx = (unsigned) atoi(pos.str())+1;
+            if (idx>25) //adf
+                throw MakeStringException(-1, "Array items above 25 not supported in HPCC WS HTTP parameters: %s", fullpath);
+        }
+
+        if (tag.charAt(tag.length()-1)=='$')
+        {
+            if (path && *path)
+                throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
+            tag.setLength(tag.length()-1);
+            StringArray values;
+            values.appendList(value, "\r");
+            ForEachItemIn(pos, values)
+            {
+                const char *itemValue = values.item(pos);
+                while (*itemValue=='\n')
+                    itemValue++;
+                pt->addProp(tag, itemValue);
+            }
+            return;
+        }
+        unsigned count = pt->getCount(tag);
+        while (count++ < idx)
+            pt->addPropTree(tag, createPTree(tag));
+        StringBuffer xpath(tag);
+        xpath.append('[').append(idx).append(']');
+        pt = pt->queryPropTree(xpath);
+
+        if (!path || !*path)
+        {
+            pt->setProp(NULL, value);
+            return;
+        }
+
+        StringBuffer nextTag;
+        path = nextParameterTag(nextTag, path);
+        ensureParameter(pt, nextTag, path, value, fullpath);
+    }
+
+    static void ensureParameter(IPropertyTree *pt, const char *path, const char *value)
+    {
+        const char *fullpath = path;
+        StringBuffer tag;
+        path = nextParameterTag(tag, path);
+        ensureParameter(pt, tag, path, value, fullpath);
+    }
+
+    static IPropertyTree *createPTreeFromHttpParameters(const char *name, IProperties *parameters)
+    {
+        Owned<IPropertyTree> pt = createPTree(name);
+        Owned<IPropertyIterator> props = parameters->getIterator();
+        ForEach(*props)
+        {
+            const char *key = props->getPropKey();
+            const char *value = parameters->queryProp(key);
+            ensureParameter(pt, key, value);
+        }
+        return pt.getClear();
+    }
+
+    static const char *nextParameterTag(StringBuffer &tag, const char *path)
+        {
+            while (*path=='.')
+                path++;
+            const char *finger = strchr(path, '.');
+            if (finger)
+            {
+                tag.clear().append(finger - path, path);
+                finger++;
+            }
+            else
+                tag.set(path);
+            return finger;
+        }
+};
+
 class JsonHelpers
 {
 public:
@@ -68,7 +158,7 @@ public:
         IArrayOf<IException>& exceptions = e->getArray();
         for (int i = 0 ; i < exceptions.ordinality(); i++)
         {
-            appendJSONExceptionItem(s, e->errorCode(), e->errorMessage(temp).str(), objname, arrayName);
+            appendJSONExceptionItem(s, e->errorCode(), e->errorMessage(temp.clear()).str(), objname, arrayName);
         }
 
         return s;
@@ -229,90 +319,95 @@ public:
         else
             out.append("null");
     }
-
-    static const char *nextParameterTag(StringBuffer &tag, const char *path)
+    static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer& out, const char* tag, IPropertyTree *reqTree, unsigned flags)
     {
-        while (*path=='.')
-            path++;
-        const char *finger = strchr(path, '.');
-        if (finger)
-        {
-            tag.clear().append(finger - path, path);
-            finger++;
-        }
-        else
-            tag.set(path);
-        return finger;
-    }
+        assertex(type!=NULL);
 
-    static void ensureParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
-    {
-        if (!tag.length())
-            return;
-
-        unsigned idx = 1;
-        if (path && isdigit(*path))
-        {
-            StringBuffer pos;
-            path = nextParameterTag(pos, path);
-            idx = (unsigned) atoi(pos.str())+1;
-            if (idx>25) //adf
-                throw MakeStringException(-1, "Array items above 25 not supported in HPCC WS HTTP parameters: %s", fullpath);
-        }
+        if (flags & REQSF_ROOT)
+            out.append("{");
 
-        if (tag.charAt(tag.length()-1)=='$')
+        const char* typeName = type->queryName();
+        if (type->isComplexType())
         {
-            if (path && *path)
-                throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
-            tag.setLength(tag.length()-1);
-            StringArray values;
-            values.appendList(value, "\r");
-            ForEachItemIn(pos, values)
+            if (typeName && !parentTypes.appendUniq(typeName))
+                return; // recursive
+
+            int startlen = out.length();
+            if (tag)
+                appendJSONName(out, tag);
+            out.append('{');
+            int taglen=out.length()+1;
+            if (type->getSubType()==SubType_Complex_SimpleContent)
             {
-                const char *itemValue = values.item(pos);
-                while (*itemValue=='\n')
-                    itemValue++;
-                pt->addProp(tag, itemValue);
+                if (reqTree)
+                {
+                    const char *attrval = reqTree->queryProp(NULL);
+                    out.appendf("\"%s\" ", (attrval) ? attrval : "");
+                }
+                else if (flags & REQSF_SAMPLE_DATA)
+                {
+                    out.append("\"");
+                    type->queryFieldType(0)->getSampleValue(out,tag);
+                    out.append("\" ");
+                }
+            }
+            else
+            {
+                int flds = type->getFieldCount();
+                for (int idx=0; idx<flds; idx++)
+                {
+                    delimitJSON(out);
+                    IPropertyTree *childtree = NULL;
+                    const char *childname = type->queryFieldName(idx);
+                    if (reqTree)
+                        childtree = reqTree->queryPropTree(childname);
+                    buildJsonMsg(parentTypes, type->queryFieldType(idx), out, childname, childtree, flags & ~REQSF_ROOT);
+                }
             }
-            return;
-        }
-        unsigned count = pt->getCount(tag);
-        while (count++ < idx)
-            pt->addPropTree(tag, createPTree(tag));
-        StringBuffer xpath(tag);
-        xpath.append('[').append(idx).append(']');
-        pt = pt->queryPropTree(xpath);
 
-        if (!path || !*path)
-        {
-            pt->setProp(NULL, value);
-            return;
+            if (typeName)
+                parentTypes.pop();
+            out.append("}");
         }
+        else if (type->isArray())
+        {
+            if (typeName && !parentTypes.appendUniq(typeName))
+                return; // recursive
+
+            const char* itemName = type->queryFieldName(0);
+            IXmlType*   itemType = type->queryFieldType(0);
+            if (!itemName || !itemType)
+                throw MakeStringException(-1,"*** Invalid array definition: tag=%s, itemName=%s", tag, itemName?itemName:"NULL");
+
+            int startlen = out.length();
+            if (tag)
+                out.appendf("\"%s\": ", tag);
+            out.append('{');
+            out.appendf("\"%s\": [", itemName);
+            int taglen=out.length();
+            if (reqTree)
+            {
+                Owned<IPropertyTreeIterator> items = reqTree->getElements(itemName);
+                ForEach(*items)
+                    buildJsonMsg(parentTypes, itemType, delimitJSON(out), NULL, &items->query(), flags & ~REQSF_ROOT);
+            }
+            else
+                buildJsonMsg(parentTypes, itemType, out, NULL, NULL, flags & ~REQSF_ROOT);
 
-        StringBuffer nextTag;
-        path = nextParameterTag(nextTag, path);
-        ensureParameter(pt, nextTag, path, value, fullpath);
-    }
+            out.append(']');
 
-    static void ensureParameter(IPropertyTree *pt, const char *path, const char *value)
-    {
-        const char *fullpath = path;
-        StringBuffer tag;
-        path = nextParameterTag(tag, path);
-        ensureParameter(pt, tag, path, value, fullpath);
-    }
-
-    static IPropertyTree *createPTreeFromHttpParameters(const char *name, IProperties *parameters)
-    {
-        Owned<IPropertyTree> pt = createPTree(name);
-        Owned<IPropertyIterator> props = parameters->getIterator();
-        ForEach(*props)
+            if (typeName)
+                parentTypes.pop();
+            out.append("}");
+        }
+        else // simple type
         {
-            const char *key = props->getPropKey();
-            const char *value = parameters->queryProp(key);
-            ensureParameter(pt, key, value);
+            const char *parmval = (reqTree) ? reqTree->queryProp(NULL) : NULL;
+            buildJsonAppendValue(type, out, tag, parmval, flags);
         }
-        return pt.getClear();
+
+        if (flags & REQSF_ROOT)
+            out.append('}');
     }
 };
 #endif // _JSONHELPERS_HPP__

esp/services/common/wsexcept.cpp

@@ -0,0 +1,192 @@
+#include "platform.h"
+#include "wsexcept.hpp"
+
+class CWsException : public CInterface,
+implements IWsException
+{
+public:
+    IMPLEMENT_IINTERFACE
+
+    CWsException (const char* source, WsErrorType errorType )
+    {
+        if (source)
+            source_.append(source);
+        errorType_ = errorType;
+    }
+    CWsException( IMultiException& me, WsErrorType errorType )
+    {
+        append(me);
+        errorType_ = errorType;
+
+        const char* source = me.source();
+
+        if (source)
+            source_.append(source);
+    }
+    CWsException( IException& e, const char* source, WsErrorType errorType )
+    {
+        IMultiException* me = dynamic_cast<IMultiException*>(&e);
+        if ( me ) {
+            append(*me);
+        } else
+            append(e);
+        errorType_ = errorType;
+
+        if (source)
+            source_.append(source);
+    }
+
+    //convenience methods for handling this as an array
+    virtual aindex_t ordinality() const
+    {
+        synchronized block(m_mutex);
+        return array_.ordinality();
+    }
+    virtual IException& item(aindex_t pos) const
+    {
+        synchronized block(m_mutex);
+        return array_.item(pos);
+    }
+    virtual const char* source() const
+    {
+        synchronized block(m_mutex);
+        return source_.str();
+    }
+
+    //for complete control...caller is responsible for thread safety!
+    virtual IArrayOf<IException>& getArray()     { return array_;              }
+
+    // add another exception. Pass ownership to this obj:
+    // i.e., caller needs to make sure that e has one consumable ref count
+    virtual void append(IException& e)
+    {
+        synchronized block(m_mutex);
+        array_.append(e);
+    }
+    virtual void append(IMultiException& me)
+    {
+        synchronized block(m_mutex);
+
+        IArrayOf<IException>& exceptions = me.getArray();
+        const char* source = me.source();
+        ForEachItemIn(i, exceptions)
+        {
+            IException& e = exceptions.item(i);
+            if (source && *source)
+            {
+                StringBuffer msg;
+                msg.appendf("[%s] ",source);
+                e.errorMessage(msg);
+                array_.append(*MakeStringExceptionDirect(e.errorAudience(), e.errorCode(), msg));
+            }
+            else
+                array_.append(*LINK(&e));
+        }
+    }
+
+
+    StringBuffer& serialize(StringBuffer& buffer, unsigned indent = 0, bool simplified=false, bool root=true) const
+    {
+        synchronized block(m_mutex);
+
+        if (root)
+            buffer.append("<Exceptions>");
+
+        if (!simplified)
+        {
+            if (indent) buffer.append("\n\t");
+            buffer.appendf("<Source>%s</Source>", source_.str());
+        }
+
+        ForEachItemIn(i, array_)
+        {
+            IException& exception = array_.item(i);
+
+            if (indent) buffer.append("\n\t");
+            buffer.append("<Exception>");
+
+            if (indent) buffer.append("\n\t\t");
+            buffer.appendf("<Code>%d</Code>", exception.errorCode());
+
+            if (indent) buffer.append("\n\t\t");
+            buffer.appendf("<Audience>%s</Audience>", serializeMessageAudience( exception.errorAudience() ));
+
+            if (simplified)
+            {
+                if (indent) buffer.append("\n\t\t");
+                StringBuffer msg;
+                buffer.appendf("<Source>%s</Source>", source_.str());
+            }
+
+            if (indent) buffer.append("\n\t\t");
+
+            StringBuffer msg;
+            StringBuffer encoded;
+            encodeXML(exception.errorMessage(msg).str(), encoded);
+            buffer.appendf("<Message>%s</Message>", encoded.str());
+
+            if (indent) buffer.append("\n\t");
+            buffer.append("</Exception>");
+        }
+
+        if (root)
+            buffer.append("</Exceptions>");
+        return buffer;
+    }
+
+    virtual int errorCode() const
+    {
+        synchronized block(m_mutex);
+        return ordinality() == 1 ? item(0).errorCode() : -1;
+    }
+    virtual StringBuffer& errorMessage(StringBuffer &msg) const
+    {
+        synchronized block(m_mutex);
+        ForEachItemIn(i, array_)
+        {
+            IException& e = item(i);
+
+            StringBuffer buf;
+            msg.appendf("[%3d: %s] ", e.errorCode(), e.errorMessage(buf).str());
+        }
+        return msg;
+    }
+    virtual MessageAudience errorAudience() const
+    {
+        synchronized block(m_mutex);
+        return ordinality() == 1 ? item(0).errorAudience() : MSGAUD_unknown;
+    }
+    virtual WsErrorType errorType() const
+    {
+        synchronized block(m_mutex);
+        return errorType_;
+    }
+private:
+    CWsException( const CWsException& );
+    IArrayOf<IException> array_;
+    StringBuffer         source_;
+    mutable Mutex        m_mutex;
+    WsErrorType          errorType_;
+};
+
+IWsException esdl_decl *makeWsException(IMultiException& me, WsErrorType errorType)
+{
+    return new CWsException(me, errorType);
+}
+IWsException esdl_decl *makeWsException(IException& e, WsErrorType errorType, const char* source)
+{
+    return new CWsException(  e, source, errorType );
+}
+IWsException esdl_decl *makeWsException(int errorCode, WsErrorType errorType, const char* source, const char *format, ...)
+{
+    va_list args;
+    va_start(args, format);
+    IException *e = MakeStringExceptionVA(errorCode, format, args);
+    va_end(args);
+
+    return new CWsException(  *e, source, errorType );
+}
+IWsException esdl_decl *makeWsException(const char *source, WsErrorType errorType)
+{
+    return new CWsException(source, errorType);
+}

esp/services/common/wsexcept.hpp

@@ -0,0 +1,67 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef __WSEXCEPT__
+#define __WSEXCEPT__
+
+#include "jexcept.hpp"
+
+#ifdef _WIN32
+ #ifdef ESDLLIB_EXPORTS
+  #define esdl_decl __declspec(dllexport)
+ #else
+  #define esdl_decl
+ #endif
+#else
+ #define esdl_decl
+#endif
+
+typedef enum
+{
+    WSERR_NOERR=-1,WSERR_CLIENT, WSERR_SERVER, WSERR_VERSION, WSERR_MUSTUNDERSTAND
+} WsErrorType;
+
+interface IWsException : extends IException
+{
+   //convenience methods for handling this as an array
+   virtual aindex_t ordinality() const = 0;
+   virtual IException& item(aindex_t pos) const = 0;
+   virtual const char* source() const = 0;
+
+   //for complete control...
+   virtual IArrayOf<IException>& getArray()= 0;
+
+   //add another exception
+   virtual void append(IException& e) = 0;
+   virtual void append(IMultiException& e) = 0;
+
+   virtual StringBuffer& serialize(StringBuffer& ret, unsigned indent = 0, bool simplified=false, bool root=true) const = 0;
+
+   //the following methods override those in IIException
+   //
+    virtual int errorCode() const = 0;
+    virtual StringBuffer&     errorMessage(StringBuffer &msg) const = 0;
+    virtual MessageAudience errorAudience() const = 0;
+    virtual WsErrorType     errorType() const = 0;
+};
+
+IWsException esdl_decl *makeWsException(IMultiException& me, WsErrorType errorType);
+IWsException esdl_decl *makeWsException(const char *source, WsErrorType errorType);
+IWsException esdl_decl *makeWsException(IException& e, WsErrorType errorType, const char* source = NULL );
+IWsException esdl_decl *makeWsException(int errorCode, WsErrorType errorType, const char* source, const char *format, ...);
+
+#endif

esp/services/common/wshelpers.hpp

@@ -85,5 +85,4 @@ inline StringBuffer& operator<<(StringBuffer& buf, const JScript& j)
     return buf;
 }
 
-
 #endif // _ESPWIZ_WSHELPERS_HPP__

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -9,6 +9,7 @@
 #include "ws_ecl_wuinfo.hpp"
 #include "xsdparser.hpp"
 #include "httpclient.hpp"
+#include "jsonhelpers.hpp"
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
 
@@ -194,92 +195,6 @@ static void appendServerAddress(StringBuffer &s, IPropertyTree &env, IPropertyTr
     s.append(netAddress).append(':').append(port ? port : "9876");
 }
 
-
-const char *nextParameterTag(StringBuffer &tag, const char *path)
-{
-    while (*path=='.')
-        path++;
-    const char *finger = strchr(path, '.');
-    if (finger)
-    {
-        tag.clear().append(finger - path, path);
-        finger++;
-    }
-    else
-        tag.set(path);
-    return finger;
-}
-
-void ensureParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
-{
-    if (!tag.length())
-        return;
-
-    unsigned idx = 1;
-    if (path && isdigit(*path))
-    {
-        StringBuffer pos;
-        path = nextParameterTag(pos, path);
-        idx = (unsigned) atoi(pos.str())+1;
-        if (idx>25) //adf
-            throw MakeStringException(-1, "Array items above 25 not supported in WsECL HTTP parameters: %s", fullpath);
-    }
-
-    if (tag.charAt(tag.length()-1)=='$')
-    {
-        if (path && *path)
-            throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
-        tag.setLength(tag.length()-1);
-        StringArray values;
-        values.appendList(value, "\r");
-        ForEachItemIn(pos, values)
-        {
-            const char *itemValue = values.item(pos);
-            while (*itemValue=='\n')
-                itemValue++;
-            pt->addProp(tag, itemValue);
-        }
-        return;
-    }
-    unsigned count = pt->getCount(tag);
-    while (count++ < idx)
-        pt->addPropTree(tag, createPTree(tag));
-    StringBuffer xpath(tag);
-    xpath.append('[').append(idx).append(']');
-    pt = pt->queryPropTree(xpath);
-
-    if (!path || !*path)
-    {
-        pt->setProp(NULL, value);
-        return;
-    }
-
-    StringBuffer nextTag;
-    path = nextParameterTag(nextTag, path);
-    ensureParameter(pt, nextTag, path, value, fullpath);
-}
-
-void ensureParameter(IPropertyTree *pt, const char *path, const char *value)
-{
-    const char *fullpath = path;
-    StringBuffer tag;
-    path = nextParameterTag(tag, path);
-    ensureParameter(pt, tag, path, value, fullpath);
-}
-
-IPropertyTree *createPTreeFromHttpParameters(const char *name, IProperties *parameters)
-{
-    Owned<IPropertyTree> pt = createPTree(name);
-    Owned<IPropertyIterator> props = parameters->getIterator();
-    ForEach(*props)
-    {
-        const char *key = props->getPropKey();
-        const char *value = parameters->queryProp(key);
-        ensureParameter(pt, key, value);
-    }
-    return pt.getClear();
-}
-
 bool CWsEclService::init(const char * name, const char * type, IPropertyTree * cfg, const char * process)
 {
     StringBuffer xpath;
@@ -827,238 +742,6 @@ static void buildRestURL(StringArray& parentTypes, StringArray &path, IXmlType*
     }
 }
 
-
-IException *MakeJSONValueException(int code, const char *start, const char *pos, const char *tail, const char *intro="Invalid json format: ")
-{
-     StringBuffer s(intro);
-     s.append(pos-start, start).append('^').append(pos);
-     if (tail && *tail)
-         s.append(" - ").append(tail);
-     return MakeStringException(code, "%s", s.str());
-}
-
-inline StringBuffer &jsonNumericNext(StringBuffer &s, const char *&c, bool &allowDecimal, bool &allowExponent, const char *start)
-{
-    if (isdigit(*c))
-        s.append(*c++);
-    else if ('.'==*c)
-    {
-        if (!allowDecimal || !allowExponent)
-            throw MakeJSONValueException(-1, start, c, "Unexpected decimal");
-        allowDecimal=false;
-        s.append(*c++);
-    }
-    else if ('e'==*c || 'E'==*c)
-    {
-        if (!allowExponent)
-            throw MakeJSONValueException(-1, start, c, "Unexpected exponent");
-
-        allowDecimal=false;
-        allowExponent=false;
-        s.append(*c++);
-        if ('-'==*c || '+'==*c)
-            s.append(*c++);
-        if (!isdigit(*c))
-            throw MakeJSONValueException(-1, start, c, "Unexpected token");
-    }
-    else
-        throw MakeJSONValueException(-1, start, c, "Unexpected token");
-
-    return s;
-}
-
-inline StringBuffer &jsonNumericStart(StringBuffer &s, const char *&c, const char *start)
-{
-    if ('-'==*c)
-        return jsonNumericStart(s.append(*c++), c, start);
-    else if ('0'==*c)
-    {
-        s.append(*c++);
-        if (*c && '.'!=*c)
-            throw MakeJSONValueException(-1, start, c, "Unexpected token");
-    }
-    else if (isdigit(*c))
-        s.append(*c++);
-    else
-        throw MakeJSONValueException(-1, start, c, "Unexpected token");
-    return s;
-}
-
-StringBuffer &appendJSONNumericString(StringBuffer &s, const char *value, bool allowDecimal)
-{
-    if (!value || !*value)
-        return s.append("null");
-
-    bool allowExponent = allowDecimal;
-
-    const char *pos = value;
-    jsonNumericStart(s, pos, value);
-    while (*pos)
-        jsonNumericNext(s, pos, allowDecimal, allowExponent, value);
-    return s;
-}
-
-typedef enum _JSONFieldCategory
-{
-    JSONField_String,
-    JSONField_Integer,
-    JSONField_Real,
-    JSONField_Boolean,
-    JSONField_Present  //true or remove
-} JSONField_Category;
-
-JSONField_Category xsdTypeToJSONFieldCategory(const char *xsdtype)
-{
-    //map XML Schema types used in ECL generated schemas to basic JSON formatting types
-    if (streq(xsdtype, "integer") || streq(xsdtype, "nonNegativeInteger"))
-        return JSONField_Integer;
-    if (streq(xsdtype, "boolean"))
-        return JSONField_Boolean;
-    if (streq(xsdtype, "double"))
-        return JSONField_Real;
-    if (!strncmp(xsdtype, "decimal", 7)) //ecl creates derived types of the form decimal#_#
-        return JSONField_Real;
-    if (streq(xsdtype, "none")) //maps to an eml schema element with no type.  set to true or don't add
-        return JSONField_Present;
-    return JSONField_String;
-}
-
-static void buildJsonAppendValue(IXmlType* type, StringBuffer& out, const char* tag, const char *value, unsigned flags)
-{
-    JSONField_Category ct = xsdTypeToJSONFieldCategory(type->queryName());
-
-    if (ct==JSONField_Present && (!value || !*value))
-        return;
-
-    if (tag && *tag)
-        out.appendf("\"%s\": ", tag);
-    StringBuffer sample;
-    if ((!value || !*value) && (flags & REQSF_SAMPLE_DATA))
-    {
-        type->getSampleValue(sample, NULL);
-        value = sample.str();
-    }
-
-    if (value)
-    {
-        switch (ct)
-        {
-        case JSONField_String:
-            appendJSONValue(out, NULL, value);
-            break;
-        case JSONField_Integer:
-            appendJSONNumericString(out, value, false);
-            break;
-        case JSONField_Real:
-            appendJSONNumericString(out, value, true);
-            break;
-        case JSONField_Boolean:
-            if (strieq(value, "default"))
-                out.append("null");
-            else
-                appendJSONValue(out, NULL, strToBool(value));
-            break;
-        case JSONField_Present:
-            appendJSONValue(out, NULL, true);
-            break;
-        }
-    }
-    else
-        out.append("null");
-}
-
-static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer& out, const char* tag, IPropertyTree *reqTree, unsigned flags)
-{
-    assertex(type!=NULL);
-
-    if (flags & REQSF_ROOT)
-        out.append("{");
-
-    const char* typeName = type->queryName();
-    if (type->isComplexType())
-    {
-        if (typeName && !parentTypes.appendUniq(typeName))
-            return; // recursive
-
-        int startlen = out.length();
-        if (tag)
-            appendJSONName(out, tag);
-        out.append('{');
-        int taglen=out.length()+1;
-        if (type->getSubType()==SubType_Complex_SimpleContent)
-        {
-            if (reqTree)
-            {
-                const char *attrval = reqTree->queryProp(NULL);
-                out.appendf("\"%s\" ", (attrval) ? attrval : "");
-            }
-            else if (flags & REQSF_SAMPLE_DATA)
-            {
-                out.append("\"");
-                type->queryFieldType(0)->getSampleValue(out,tag);
-                out.append("\" ");
-            }
-        }
-        else
-        {
-            int flds = type->getFieldCount();
-            for (int idx=0; idx<flds; idx++)
-            {
-                delimitJSON(out);
-                IPropertyTree *childtree = NULL;
-                const char *childname = type->queryFieldName(idx);
-                if (reqTree)
-                    childtree = reqTree->queryPropTree(childname);
-                buildJsonMsg(parentTypes, type->queryFieldType(idx), out, childname, childtree, flags & ~REQSF_ROOT);
-            }
-        }
-
-        if (typeName)
-            parentTypes.pop();
-        out.append("}");
-    }
-    else if (type->isArray())
-    {
-        if (typeName && !parentTypes.appendUniq(typeName))
-            return; // recursive
-
-        const char* itemName = type->queryFieldName(0);
-        IXmlType*   itemType = type->queryFieldType(0);
-        if (!itemName || !itemType)
-            throw MakeStringException(-1,"*** Invalid array definition: tag=%s, itemName=%s", tag, itemName?itemName:"NULL");
-
-        int startlen = out.length();
-        if (tag)
-            out.appendf("\"%s\": ", tag);
-        out.append('{');
-        out.appendf("\"%s\": [", itemName);
-        int taglen=out.length();
-        if (reqTree)
-        {
-            Owned<IPropertyTreeIterator> items = reqTree->getElements(itemName);
-            ForEach(*items)
-                buildJsonMsg(parentTypes, itemType, delimitJSON(out), NULL, &items->query(), flags & ~REQSF_ROOT);
-        }
-        else
-            buildJsonMsg(parentTypes, itemType, out, NULL, NULL, flags & ~REQSF_ROOT);
-
-        out.append(']');
-
-        if (typeName)
-            parentTypes.pop();
-        out.append("}");
-    }
-    else // simple type
-    {
-        const char *parmval = (reqTree) ? reqTree->queryProp(NULL) : NULL;
-        buildJsonAppendValue(type, out, tag, parmval, flags);
-    }
-
-    if (flags & REQSF_ROOT)
-        out.append('}');
-
-}
-
 static inline StringBuffer &appendNamespaceSpecificString(StringBuffer &dest, const char *src)
 {
     if (src)
@@ -1766,7 +1449,7 @@ void CWsEclBinding::getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &cont
         return;
     }
 
-    Owned<IPropertyTree> reqTree = createPTreeFromHttpParameters(wsinfo.queryname, parameters);
+    Owned<IPropertyTree> reqTree = HttpParamHelpers::createPTreeFromHttpParameters(wsinfo.queryname, parameters);
 
     if (!validate)
         toXML(reqTree, soapmsg, 0, 0);
@@ -1792,56 +1475,13 @@ void CWsEclBinding::getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &cont
     }
 }
 
-StringBuffer &appendJSONExceptionItem(StringBuffer &s, int code, const char *msg, const char *objname="Exceptions", const char *arrayName = "Exception")
-{
-    if (objname && *objname)
-        appendJSONName(s, objname).append('{');
-    if (arrayName && *arrayName)
-        appendJSONName(s, arrayName).append('[');
-    delimitJSON(s);
-    s.append('{');
-    appendJSONValue(s, "Code", code);
-    appendJSONValue(s, "Message", msg);
-    s.append('}');
-    if (arrayName && *arrayName)
-        s.append(']');
-    if (objname && *objname)
-        s.append('}');
-    return s;
-}
-
-StringBuffer &appendJSONException(StringBuffer &s, IException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
-{
-    if (!e)
-        return s;
-    StringBuffer temp;
-    return appendJSONExceptionItem(s, e->errorCode(), e->errorMessage(temp).str(), objname, arrayName);
-}
-
-StringBuffer &appendJSONExceptions(StringBuffer &s, IMultiException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
-{
-    if (!e)
-        return s;
-    if (objname && *objname)
-        appendJSONName(s, objname).append('{');
-    if (arrayName && *arrayName)
-        appendJSONName(s, arrayName).append('[');
-    ForEachItemIn(i, *e)
-        appendJSONException(s, &e->item(i), NULL, NULL);
-    if (arrayName && *arrayName)
-        s.append(']');
-    if (objname && *objname)
-        s.append('}');
-    return s;
-}
-
 void CWsEclBinding::getWsEclJsonRequest(StringBuffer& jsonmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags, bool validate)
 {
     size32_t start = jsonmsg.length();
     try
     {
         IProperties *parameters = context.queryRequestParameters();
-        Owned<IPropertyTree> reqTree = createPTreeFromHttpParameters(wsinfo.queryname, parameters);
+        Owned<IPropertyTree> reqTree = HttpParamHelpers::createPTreeFromHttpParameters(wsinfo.queryname, parameters);
 
         if (!validate)
         {
@@ -1866,14 +1506,14 @@ void CWsEclBinding::getWsEclJsonRequest(StringBuffer& jsonmsg, IEspContext &cont
             if (type)
             {
                 StringArray parentTypes;
-                buildJsonMsg(parentTypes, type, jsonmsg, wsinfo.queryname.sget(), reqTree, flags|REQSF_ROOT);
+                JsonHelpers::buildJsonMsg(parentTypes, type, jsonmsg, wsinfo.queryname.sget(), reqTree, flags|REQSF_ROOT);
             }
         }
     }
     catch (IException *e)
     {
         jsonmsg.setLength(start);
-        appendJSONException(jsonmsg.append('{'), e);
+        JsonHelpers::appendJSONException(jsonmsg.append('{'), e);
         jsonmsg.append('}');
     }
 }
@@ -2201,7 +1841,7 @@ void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, Stri
         if (strieq(contentType, "application/json"))
         {
             resp.set("{").append("\"").append(query).append("Response\": {\"Results\": {");
-            appendJSONException(resp, e);
+            JsonHelpers::appendJSONException(resp, e);
             resp.append("}}}");
         }
         else
@@ -2687,7 +2327,7 @@ int CWsEclBinding::onGet(CHttpRequest* request, CHttpResponse* response)
 
             if (!wsecl->connMap.getValue(target.str()))
                 throw MakeStringException(-1, "Target cluster not mapped to roxie process!");
-            Owned<IPropertyTree> pt = createPTreeFromHttpParameters(qid.str(), parms);
+            Owned<IPropertyTree> pt = HttpParamHelpers::createPTreeFromHttpParameters(qid.str(), parms);
             StringBuffer soapreq(
                 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                 "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
@@ -2910,7 +2550,7 @@ void CWsEclBinding::handleJSONPost(CHttpRequest *request, CHttpResponse *respons
     }
     catch (IException *e)
     {
-        appendJSONException(jsonresp.set("{"), e);
+        JsonHelpers::appendJSONException(jsonresp.set("{"), e);
         jsonresp.append('}');
     }