HPCC-9401 Optimize query time WsEcl

The dependency on the xml schema from the query workunit to process
incoming queries greatly slows down WsECL.

Past optimizations have removed this dependency for some query paths,
but not others.  This removes the dependency for the rest of the query
paths.

Also add a toJSON function to jptree.

Also fix WsECL form, xsd, and other processing of child sets.

Also add a new type of REST input parameter.  Adding a $ to the end of
an input parameter will cause it to be treated as a \r\n delimited
input array (so that WsEcl no longer needs to use schema to process arrays on
input).

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

esp/services/ws_ecl/CMakeLists.txt

@@ -27,8 +27,6 @@ project( ws_ecl )
 set (    SRCS 
          ws_ecl_plugin.cpp 
          ws_ecl_service.cpp 
-         ws_ecl_json.cpp
-         JSON_parser.c
          ws_ecl_wuinfo.cpp 
     )
 

esp/services/ws_ecl/JSON_parser.h

@@ -1,226 +0,0 @@
-/* See JSON_parser.c for copyright information and licensing. */
-
-#ifndef JSON_PARSER_H
-#define JSON_PARSER_H
-
-/* JSON_parser.h */
-
-
-#include <stddef.h>
-
-/* Windows DLL stuff */
-#ifdef JSON_PARSER_DLL
-#   ifdef _MSC_VER
-#       ifdef JSON_PARSER_DLL_EXPORTS
-#           define JSON_PARSER_DLL_API __declspec(dllexport)
-#       else
-#           define JSON_PARSER_DLL_API __declspec(dllimport)
-#       endif
-#   else
-#       define JSON_PARSER_DLL_API 
-#   endif
-#else
-#   define JSON_PARSER_DLL_API 
-#endif
-
-/* Determine the integer type use to parse non-floating point numbers */
-#if __STDC_VERSION__ >= 199901L || HAVE_LONG_LONG == 1
-typedef long long JSON_int_t;
-#define JSON_PARSER_INTEGER_SSCANF_TOKEN "%lld"
-#define JSON_PARSER_INTEGER_SPRINTF_TOKEN "%lld"
-#else 
-typedef long JSON_int_t;
-#define JSON_PARSER_INTEGER_SSCANF_TOKEN "%ld"
-#define JSON_PARSER_INTEGER_SPRINTF_TOKEN "%ld"
-#endif
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif 
-
-typedef enum 
-{
-    JSON_E_NONE = 0,
-    JSON_E_INVALID_CHAR,
-    JSON_E_INVALID_KEYWORD,
-    JSON_E_INVALID_ESCAPE_SEQUENCE,
-    JSON_E_INVALID_UNICODE_SEQUENCE,
-    JSON_E_INVALID_NUMBER,
-    JSON_E_NESTING_DEPTH_REACHED,
-    JSON_E_UNBALANCED_COLLECTION,
-    JSON_E_EXPECTED_KEY,
-    JSON_E_EXPECTED_COLON,
-    JSON_E_OUT_OF_MEMORY
-} JSON_error;
-
-typedef enum 
-{
-    JSON_T_NONE = 0,
-    JSON_T_ARRAY_BEGIN,
-    JSON_T_ARRAY_END,
-    JSON_T_OBJECT_BEGIN,
-    JSON_T_OBJECT_END,
-    JSON_T_INTEGER,
-    JSON_T_FLOAT,
-    JSON_T_NULL,
-    JSON_T_TRUE,
-    JSON_T_FALSE,
-    JSON_T_STRING,
-    JSON_T_KEY,
-    JSON_T_MAX
-} JSON_type;
-
-typedef struct JSON_value_struct {
-    union {
-        JSON_int_t integer_value;
-        
-        double float_value;
-        
-        struct {
-            const char* value;
-            size_t length;
-        } str;
-    } vu;
-} JSON_value;
-
-typedef struct JSON_parser_struct* JSON_parser;
-
-/*! \brief JSON parser callback 
-
-    \param ctx The pointer passed to new_JSON_parser.
-    \param type An element of JSON_type but not JSON_T_NONE.    
-    \param value A representation of the parsed value. This parameter is NULL for
-        JSON_T_ARRAY_BEGIN, JSON_T_ARRAY_END, JSON_T_OBJECT_BEGIN, JSON_T_OBJECT_END,
-        JSON_T_NULL, JSON_T_TRUE, and JSON_T_FALSE. String values are always returned
-        as zero-terminated C strings.
-
-    \return Non-zero if parsing should continue, else zero.
-*/    
-typedef int (*JSON_parser_callback)(void* ctx, int type, const struct JSON_value_struct* value);
-
-
-/**
-   A typedef for allocator functions semantically compatible with malloc().
-*/
-typedef void* (*JSON_malloc_t)(size_t n);
-/**
-   A typedef for deallocator functions semantically compatible with free().
-*/
-typedef void (*JSON_free_t)(void* mem);
-
-/*! \brief The structure used to configure a JSON parser object 
-*/
-typedef struct {
-    /** Pointer to a callback, called when the parser has something to tell
-        the user. This parameter may be NULL. In this case the input is
-        merely checked for validity.
-    */
-    JSON_parser_callback    callback;
-    /**
-       Callback context - client-specified data to pass to the
-       callback function. This parameter may be NULL.
-    */
-    void*                   callback_ctx;
-    /** Specifies the levels of nested JSON to allow. Negative numbers yield unlimited nesting.
-        If negative, the parser can parse arbitrary levels of JSON, otherwise
-        the depth is the limit.
-    */
-    int                     depth;
-    /**
-       To allow C style comments in JSON, set to non-zero.
-    */
-    int                     allow_comments;
-    /**
-       To decode floating point numbers manually set this parameter to
-       non-zero.
-    */
-    int                     handle_floats_manually;
-    /**
-       The memory allocation routine, which must be semantically
-       compatible with malloc(3). If set to NULL, malloc(3) is used.
-
-       If this is set to a non-NULL value then the 'free' member MUST be
-       set to the proper deallocation counterpart for this function.
-       Failure to do so results in undefined behaviour at deallocation
-       time.
-    */
-    JSON_malloc_t       malloc;
-    /**
-       The memory deallocation routine, which must be semantically
-       compatible with free(3). If set to NULL, free(3) is used.
-
-       If this is set to a non-NULL value then the 'alloc' member MUST be
-       set to the proper allocation counterpart for this function.
-       Failure to do so results in undefined behaviour at deallocation
-       time.
-    */
-    JSON_free_t         free;
-} JSON_config;
-
-/*! \brief Initializes the JSON parser configuration structure to default values.
-
-    The default configuration is
-    - 127 levels of nested JSON (depends on JSON_PARSER_STACK_SIZE, see json_parser.c)
-    - no parsing, just checking for JSON syntax
-    - no comments
-    - Uses realloc() for memory de/allocation.
-
-    \param config. Used to configure the parser.
-*/
-JSON_PARSER_DLL_API void init_JSON_config(JSON_config * config);
-
-/*! \brief Create a JSON parser object 
-
-    \param config. Used to configure the parser. Set to NULL to use
-        the default configuration. See init_JSON_config.  Its contents are
-        copied by this function, so it need not outlive the returned
-        object.
-    
-    \return The parser object, which is owned by the caller and must eventually
-    be freed by calling delete_JSON_parser().
-*/
-JSON_PARSER_DLL_API JSON_parser new_JSON_parser(JSON_config const* config);
-
-/*! \brief Destroy a previously created JSON parser object. */
-JSON_PARSER_DLL_API void delete_JSON_parser(JSON_parser jc);
-
-/*! \brief Parse a character.
-
-    \return Non-zero, if all characters passed to this function are part of are valid JSON.
-*/
-JSON_PARSER_DLL_API int JSON_parser_char(JSON_parser jc, int next_char);
-
-/*! \brief Finalize parsing.
-
-    Call this method once after all input characters have been consumed.
-    
-    \return Non-zero, if all parsed characters are valid JSON, zero otherwise.
-*/
-JSON_PARSER_DLL_API int JSON_parser_done(JSON_parser jc);
-
-/*! \brief Determine if a given string is valid JSON white space 
-
-    \return Non-zero if the string is valid, zero otherwise.
-*/
-JSON_PARSER_DLL_API int JSON_parser_is_legal_white_space_string(const char* s);
-
-/*! \brief Gets the last error that occurred during the use of JSON_parser.
-
-    \return A value from the JSON_error enum.
-*/
-JSON_PARSER_DLL_API int JSON_parser_get_last_error(JSON_parser jc);
-
-/*! \brief Re-sets the parser to prepare it for another parse run.
-
-    \return True (non-zero) on success, 0 on error (e.g. !jc).
-*/
-JSON_PARSER_DLL_API int JSON_parser_reset(JSON_parser jc);
-
-
-#ifdef __cplusplus
-}
-#endif 
-    
-
-#endif /* JSON_PARSER_H */

esp/services/ws_ecl/ws_ecl_json.cpp

@@ -1,190 +0,0 @@
-#include "ws_ecl_service.hpp"
-
-//#include <stdlib.h>
-//#include <stdio.h>
-//#include <string.h>
-//#include <assert.h>
-//#include <locale.h>
-
-#include <vector>
-#include <clocale>
-
-#include "JSON_parser.h"
-
-static int ptree_builder(void* ctx, int type, const JSON_value* value);
-
-class jsonkey
-{
-public:
-    StringBuffer name;
-    bool isArray;
-
-    jsonkey(const char *name_) : name(name_), isArray(false){}
-};
-
-class JSonToXmlStringContext
-{
-public:
-    StringBuffer &xml;
-    std::vector<jsonkey> stack;
-    StringBuffer tail;
-
-    const char *tagname(){if (stack.size()>0) return stack.back().name.str(); return NULL;}
-    bool isArray(){return stack.back().isArray;}
-    void setIsArray(bool isArray=true){stack.back().isArray = isArray;}
-    void push(const char *name){stack.push_back(name);}
-    void pop(){stack.pop_back();}
-    bool isEmpty(){return !stack.size();}
-
-    JSonToXmlStringContext(StringBuffer &xml_, const char *tail_) : xml(xml_), tail(tail_) {}
-    void startKey(const char *name)
-    {
-        const char *curname = tagname();
-        if (tail.length() && isEmpty())
-        {
-            StringBuffer fullname(name);
-            push(fullname.append(tail).str());
-        }
-        else
-            push(name);
-    }
-    void endKey()
-    {
-        const char *curname = tagname();
-        pop();
-    }
-    void startObject()
-    {
-        const char *curname = tagname();
-        if (!isEmpty()) 
-            xml.appendf("<%s>", tagname());
-    }
-    void endObject()
-    {
-        const char *curname = tagname();
-        if (!isEmpty()) 
-        { 
-            xml.appendf("</%s>", tagname()); 
-            if (!isArray())
-                pop();
-        }
-    }
-    void startArray()
-    {
-        const char *curname = tagname();
-        setIsArray();
-    }
-    void endArray()
-    {
-        const char *curname = tagname();
-        pop();
-    }
-
-    void setValueStr(const char *val){xml.appendf("<%s>%s</%s>", tagname(), val, tagname()); if (!isArray()) pop();}
-    void setValueBool(bool val){xml.appendf("<%s>%s</%s>", tagname(), val ? "true" : "false", tagname()); if (!isArray()) pop();}
-    void setValueInt(int val){xml.appendf("<%s>%d</%s>", tagname(), val, tagname()); if (!isArray()) pop();}
-    void setValueDouble(double val){xml.appendf("<%s>%f</%s>", tagname(), val, tagname()); if (!isArray()) pop();}
-    void setNULL(){xml.appendf("<%s></%s>", tagname(), tagname()); if (!isArray()) pop();}
-};
-
-
-void createPTreeFromJsonString(const char *json, bool caseInsensitive, StringBuffer &xml, const char *tail)
-{
-    int count = 0, result = 0;
-        
-    JSonToXmlStringContext jcontext(xml, tail);
-    
-    struct JSON_parser_struct* jc = NULL;
-
-    JSON_config config;
-    init_JSON_config(&config);
-    
-    config.depth                  = 19;
-    config.callback               = &ptree_builder;
-    config.allow_comments         = 1;
-    config.handle_floats_manually = 0;
-    config.callback_ctx = &jcontext;
-    
-    /* Important! Set locale before parser is created.*/
-    setlocale(LC_ALL, "english");
-    
-    jc = new_JSON_parser(&config);
-    
-    const char *finger = json;
-    for (; *finger ; finger++) {
-        if (!JSON_parser_char(jc, *finger)) {
-            throw MakeStringException(-1, "JSON_parser_char: syntax error, byte %d\n", (int) (finger - json));
-        }
-    }
-    if (!JSON_parser_done(jc)) {
-        throw MakeStringException(-1, "JSON_parser_end: syntax error\n");
-    }
-
-    //TBD: worry about exception cleanup later
-    delete_JSON_parser(jc);
-}
-
-
-
-static size_t s_Level = 0;
-static const char* s_pIndention = "  ";
-static int s_IsKey = 0;
-
-static void print_indention(StringBuffer &xml)
-{
-    xml.appendN(s_Level * 2, ' ');
-}
- 
-
-static int ptree_builder(void* ctx, int type, const JSON_value* value)
-{
-    JSonToXmlStringContext* jcx = (JSonToXmlStringContext*) ctx;
-
-    switch(type) {
-    case JSON_T_ARRAY_BEGIN:    
-        jcx->startArray();
-        ++s_Level;
-        break;
-    case JSON_T_ARRAY_END:
-        jcx->endArray();
-        if (s_Level > 0) --s_Level;
-        break;
-   case JSON_T_OBJECT_BEGIN:
-       jcx->startObject();
-        ++s_Level;
-        break;
-    case JSON_T_OBJECT_END:
-        if (s_Level > 0) --s_Level;
-        jcx->endObject();
-        break;
-    case JSON_T_INTEGER:
-        jcx->setValueInt(value->vu.integer_value);
-        break;
-    case JSON_T_FLOAT:
-        jcx->setValueDouble(value->vu.float_value);
-        break;
-    case JSON_T_NULL:
-        jcx->setNULL();
-        break;
-    case JSON_T_TRUE:
-        s_IsKey = 0;
-        jcx->setValueBool(true);
-        break;
-    case JSON_T_FALSE:
-        jcx->setValueBool(false);
-        break;
-    case JSON_T_KEY:
-        jcx->startKey(value->vu.str.value);
-        break;   
-    case JSON_T_STRING:
-        jcx->setValueStr(value->vu.str.value);
-        break;
-    default:
-        assert(0);
-        break;
-    }
-    
-    return 1;
-}
-
-

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -9,7 +9,6 @@
 #include "ws_ecl_wuinfo.hpp"
 #include "xsdparser.hpp"
 #include "httpclient.hpp"
-#include "xpp/XmlPullParser.h"
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
 
@@ -198,14 +197,14 @@ static void appendServerAddress(StringBuffer &s, IPropertyTree &env, IPropertyTr
 }
 
 
-const char *nextParameterTag(StringAttr &tag, const char *path)
+const char *nextParameterTag(StringBuffer &tag, const char *path)
 {
     while (*path=='.')
         path++;
     const char *finger = strchr(path, '.');
     if (finger)
     {
-        tag.set(path, finger - path);
+        tag.clear().append(finger - path, path);
         finger++;
     }
     else
@@ -213,17 +212,37 @@ const char *nextParameterTag(StringAttr &tag, const char *path)
     return finger;
 }
 
-void ensureParameter(IPropertyTree *pt, const char *tag, const char *path, const char *value, const char *fullpath)
+void ensureParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
 {
+    if (!tag || !*tag)
+        return;
+
     unsigned idx = 1;
     if (path && isdigit(*path))
     {
-        StringAttr pos;
+        StringBuffer pos;
         path = nextParameterTag(pos, path);
-        idx = (unsigned) atoi(pos.sget())+1;
+        idx = (unsigned) atoi(pos.str())+1;
         if (idx>25) //adf
             throw MakeStringException(-1, "Array items above 25 not supported in WsECL HTTP parameters: %s", fullpath);
     }
+
+    if (tag.charAt(tag.length()-1)=='$')
+    {
+        if (path && *path)
+            throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
+        tag.setLength(tag.length()-1);
+        StringArray values;
+        values.appendList(value, "\r");
+        ForEachItemIn(pos, values)
+        {
+            const char *itemValue = values.item(pos);
+            while (*itemValue=='\n')
+                itemValue++;
+            pt->addProp(tag, itemValue);
+        }
+        return;
+    }
     unsigned count = pt->getCount(tag);
     while (count++ < idx)
         pt->addPropTree(tag, createPTree(tag));
@@ -237,7 +256,7 @@ void ensureParameter(IPropertyTree *pt, const char *tag, const char *path, const
         return;
     }
 
-    StringAttr nextTag;
+    StringBuffer nextTag;
     path = nextParameterTag(nextTag, path);
     ensureParameter(pt, nextTag, path, value, fullpath);
 }
@@ -245,7 +264,7 @@ void ensureParameter(IPropertyTree *pt, const char *tag, const char *path, const
 void ensureParameter(IPropertyTree *pt, const char *path, const char *value)
 {
     const char *fullpath = path;
-    StringAttr tag;
+    StringBuffer tag;
     path = nextParameterTag(tag, path);
     ensureParameter(pt, tag, path, value, fullpath);
 }
@@ -471,9 +490,6 @@ void CWsEclBinding::getDynNavData(IEspContext &context, IProperties *params, IPr
     }
 }
 
-
-
-
 static void splitPathTailAndExt(const char *s, StringBuffer &path, StringBuffer &tail, StringBuffer *ext)
 {
     if (s)
@@ -564,15 +580,14 @@ StringBuffer &CWsEclBinding::generateNamespace(IEspContext &context, CHttpReques
 #define REQSF_SAMPLE_DATA  0x0002
 #define REQSF_TRIM         0x0004
 #define REQSF_ESCAPEFORMATTERS 0x0008
-
 #define REQSF_EXCLUSIVE (REQSF_SAMPLE_DATA | REQSF_TRIM)
 
-static void buildReqXml(StringArray& parentTypes, IXmlType* type, StringBuffer& out, const char* tag, IPropertyTree *parmtree, unsigned flags, const char* ns=NULL)
+static void buildReqXml(StringArray& parentTypes, IXmlType* type, StringBuffer& out, const char* tag, IPropertyTree *reqTree, unsigned flags, const char* ns=NULL)
 {
     assertex(type!=NULL);
     assertex((flags & REQSF_EXCLUSIVE)!= REQSF_EXCLUSIVE);
 
-    if (!parmtree && (flags & REQSF_TRIM) && !(flags & REQSF_ROOT))
+    if (!reqTree && (flags & REQSF_TRIM) && !(flags & REQSF_ROOT))
         return;
 
     const char* typeName = type->queryName();
@@ -591,8 +606,8 @@ static void buildReqXml(StringArray& parentTypes, IXmlType* type, StringBuffer&
             IXmlAttribute* attr = type->queryAttr(i);
             StringBuffer s;
             const char *attrval;
-            if (parmtree)
-                attrval = parmtree->queryProp(s.append('@').append(attr->queryName()));
+            if (reqTree)
+                attrval = reqTree->queryProp(s.append('@').append(attr->queryName()));
             else
                 attrval = attr->getSampleValue(s);
             if (attrval)
@@ -605,9 +620,9 @@ static void buildReqXml(StringArray& parentTypes, IXmlType* type, StringBuffer&
         {
         case SubType_Complex_SimpleContent:
             assertex(flds==0);
-            if (parmtree)
+            if (reqTree)
             {
-                const char *val = parmtree->queryProp(NULL);
+                const char *val = reqTree->queryProp(NULL);
                 if (val)
                     encodeXML(val, out);
             }
@@ -620,8 +635,8 @@ static void buildReqXml(StringArray& parentTypes, IXmlType* type, StringBuffer&
             {
                 IPropertyTree *childtree = NULL;
                 const char *childname = type->queryFieldName(idx);
-                if (parmtree)
-                    childtree = parmtree->queryPropTree(childname);
+                if (reqTree)
+                    childtree = reqTree->queryPropTree(childname);
                 buildReqXml(parentTypes,type->queryFieldType(idx), out, childname, childtree, flags & ~REQSF_ROOT);
             }
             break;
@@ -650,40 +665,11 @@ static void buildReqXml(StringArray& parentTypes, IXmlType* type, StringBuffer&
             out.append(' ').append(ns);
         out.append(">");
         int taglen=out.length();
-        if (parmtree)
+        if (reqTree)
         {
-            VStringBuffer countpath("%s/itemcount", itemName);
-            const char *countstr=parmtree->queryProp(countpath.str());
-            if (countstr && *countstr)
-            {
-                int count = atoi(countstr);
-                for (int idx=0; idx<count; idx++)
-                {
-                    StringBuffer itempath;
-                    itempath.append(itemName).append(idx);
-                    IPropertyTree *itemtree = parmtree->queryPropTree(itempath.str());
-                    if (itemtree)
-                        buildReqXml(parentTypes,itemType,out,itemName, itemtree, flags & ~REQSF_ROOT);
-                }
-            }
-            else if (parmtree->hasProp(itemName))
-            {
-                Owned<IPropertyTreeIterator> items = parmtree->getElements(itemName);
-                ForEach(*items)
-                    buildReqXml(parentTypes,itemType,out,itemName, &items->query(), flags & ~REQSF_ROOT);
-            }
-            else
-            {
-                const char *s = parmtree->queryProp(NULL);
-                if (s && *s)
-                {
-                    StringArray items;
-                    items.appendList(s, "\n");
-                    ForEachItemIn(i, items)
-                        appendXMLTag(out, itemName, items.item(i));
-                }
-
-            }
+            Owned<IPropertyTreeIterator> items = reqTree->getElements(itemName);
+            ForEach(*items)
+                buildReqXml(parentTypes,itemType,out,itemName, &items->query(), flags & ~REQSF_ROOT);
         }
         else
             buildReqXml(parentTypes,itemType,out,itemName, NULL, flags & ~REQSF_ROOT);
@@ -698,8 +684,8 @@ static void buildReqXml(StringArray& parentTypes, IXmlType* type, StringBuffer&
     else // simple type
     {
         StringBuffer parmval;
-        if (parmtree)
-            parmval.append(parmtree->queryProp(NULL));
+        if (reqTree)
+            parmval.append(reqTree->queryProp(NULL));
         if (!parmval.length() && (flags & REQSF_SAMPLE_DATA))
             type->getSampleValue(parmval, NULL);
         
@@ -900,7 +886,8 @@ typedef enum _JSONFieldCategory
     JSONField_String,
     JSONField_Integer,
     JSONField_Real,
-    JSONField_Boolean
+    JSONField_Boolean,
+    JSONField_Present  //true or remove
 } JSONField_Category;
 
 JSONField_Category xsdTypeToJSONFieldCategory(const char *xsdtype)
@@ -914,11 +901,18 @@ JSONField_Category xsdTypeToJSONFieldCategory(const char *xsdtype)
         return JSONField_Real;
     if (!strncmp(xsdtype, "decimal", 7)) //ecl creates derived types of the form decimal#_#
         return JSONField_Real;
+    if (!strncmp(xsdtype, "none", 4)) //maps to an eml schema element with no type.  set to true or don't add
+        return JSONField_Present;
     return JSONField_String;
 }
 
 static void buildJsonAppendValue(IXmlType* type, StringBuffer& out, const char* tag, const char *value, unsigned flags)
 {
+    JSONField_Category ct = xsdTypeToJSONFieldCategory(type->queryName());
+
+    if (ct==JSONField_Present && (!value || !*value))
+        return;
+
     if (tag && *tag)
         out.appendf("\"%s\": ", tag);
     StringBuffer sample;
@@ -930,7 +924,7 @@ static void buildJsonAppendValue(IXmlType* type, StringBuffer& out, const char*
 
     if (value)
     {
-        switch (xsdTypeToJSONFieldCategory(type->queryName()))
+        switch (ct)
         {
         case JSONField_String:
             appendJSONValue(out, NULL, value);
@@ -942,7 +936,13 @@ static void buildJsonAppendValue(IXmlType* type, StringBuffer& out, const char*
             appendJSONNumericString(out, value, true);
             break;
         case JSONField_Boolean:
-            appendJSONValue(out, NULL, strToBool(value));
+            if (strieq(value, "default"))
+                out.append("null");
+            else
+                appendJSONValue(out, NULL, strToBool(value));
+            break;
+        case JSONField_Present:
+            appendJSONValue(out, NULL, true);
             break;
         }
     }
@@ -950,7 +950,7 @@ static void buildJsonAppendValue(IXmlType* type, StringBuffer& out, const char*
         out.append("null");
 }
 
-static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer& out, const char* tag, IPropertyTree *parmtree, unsigned flags)
+static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer& out, const char* tag, IPropertyTree *reqTree, unsigned flags)
 {
     assertex(type!=NULL);
 
@@ -970,9 +970,9 @@ static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer&
         int taglen=out.length()+1;
         if (type->getSubType()==SubType_Complex_SimpleContent)
         {
-            if (parmtree)
+            if (reqTree)
             {
-                const char *attrval = parmtree->queryProp(NULL);
+                const char *attrval = reqTree->queryProp(NULL);
                 out.appendf("\"%s\" ", (attrval) ? attrval : "");
             }
             else if (flags & REQSF_SAMPLE_DATA)
@@ -984,18 +984,14 @@ static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer&
         }
         else
         {
-            bool first=true;
             int flds = type->getFieldCount();
             for (int idx=0; idx<flds; idx++)
             {
-                if (first)
-                    first=false;
-                else
-                    out.append(',');
+                delimitJSON(out);
                 IPropertyTree *childtree = NULL;
                 const char *childname = type->queryFieldName(idx);
-                if (parmtree)
-                    childtree = parmtree->queryPropTree(childname);
+                if (reqTree)
+                    childtree = reqTree->queryPropTree(childname);
                 buildJsonMsg(parentTypes, type->queryFieldType(idx), out, childname, childtree, flags & ~REQSF_ROOT);
             }
         }
@@ -1020,55 +1016,11 @@ static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer&
         out.append('{');
         out.appendf("\"%s\": [", itemName);
         int taglen=out.length();
-        if (parmtree)
+        if (reqTree)
         {
-            VStringBuffer countpath("%s/itemcount", itemName);
-            const char *countstr=parmtree->queryProp(countpath.str());
-            if (countstr && *countstr)
-            {
-                bool first=true;
-                int count = atoi(countstr);
-                for (int idx=0; idx<count; idx++)
-                {
-                    if (first)
-                        first=false;
-                    else
-                        out.append(",");
-                    StringBuffer itempath;
-                    itempath.append(itemName).append(idx);
-                    IPropertyTree *itemtree = parmtree->queryPropTree(itempath.str());
-                    if (itemtree)
-                        buildJsonMsg(parentTypes,itemType,out, NULL, itemtree, flags & ~REQSF_ROOT);
-                }
-            }
-            else if (parmtree->hasProp(itemName))
-            {
-                Owned<IPropertyTreeIterator> items = parmtree->getElements(itemName);
-                bool first=true;
-                ForEach(*items)
-                {
-                    if (first)
-                        first=false;
-                    else
-                        out.append(",");
-                    buildJsonMsg(parentTypes,itemType,out, NULL, &items->query(), flags & ~REQSF_ROOT);
-                }
-            }
-            else
-            {
-                const char *s = parmtree->queryProp(NULL);
-                if (s && *s)
-                {
-                    StringArray items;
-                    items.appendList(s, "\n");
-                    ForEachItemIn(i, items)
-                    {
-                        delimitJSON(out, false);
-                        buildJsonAppendValue(type, out, NULL, items.item(i), flags & ~REQSF_ROOT);
-                    }
-                }
-
-            }
+            Owned<IPropertyTreeIterator> items = reqTree->getElements(itemName);
+            ForEach(*items)
+                buildJsonMsg(parentTypes, itemType, delimitJSON(out), NULL, &items->query(), flags & ~REQSF_ROOT);
         }
         else
             buildJsonMsg(parentTypes, itemType, out, NULL, NULL, flags & ~REQSF_ROOT);
@@ -1081,7 +1033,7 @@ static void buildJsonMsg(StringArray& parentTypes, IXmlType* type, StringBuffer&
     }
     else // simple type
     {
-        const char *parmval = (parmtree) ? parmtree->queryProp(NULL) : NULL;
+        const char *parmval = (reqTree) ? reqTree->queryProp(NULL) : NULL;
         buildJsonAppendValue(type, out, tag, parmval, flags);
     }
 
@@ -1696,7 +1648,7 @@ int CWsEclBinding::getGenForm(IEspContext &context, CHttpRequest* request, CHttp
     if (box)
     {
         StringBuffer xmlreq;
-        getWsEcl2XmlRequest(xmlreq, context, request, wsinfo, "xml", NULL, 0);
+        getWsEcl2XmlRequest(xmlreq, context, request, wsinfo, "xml", NULL, 0, true);
         if (xmlreq.length())
         {
             Owned<IPropertyTree> pretty = createPTreeFromXMLString(xmlreq.str(), ipt_ordered);
@@ -1778,42 +1730,6 @@ inline void appendParameterNode(StringBuffer &xpath, StringBuffer &node)
     }
 }
 
-void buildParametersXml(IPropertyTree *parmtree, IProperties *parms)
-{
-    Owned<IPropertyIterator> it = parms->getIterator();
-    ForEach(*it)
-    {
-        const char *key = it->getPropKey();
-        const char *val = parms->queryProp(key);
-        StringBuffer xpath;
-        if (key && *key && val && *val)
-        {
-            bool isidx=false;
-            StringBuffer node;
-            for (int pos=0; key[pos]!=0; pos++)
-            {
-                if (key[pos]!='.')
-                    node.append(key[pos]);
-                else
-                {
-                    appendParameterNode(xpath, node);
-                    xpath.append('/');
-                }
-            }
-            appendParameterNode(xpath, node);
-
-            ensurePTree(parmtree, xpath.str());
-            parmtree->setProp(xpath.str(), val);
-        }
-    }
-    if (getEspLogLevel()>LogNormal)
-    {
-        StringBuffer xml;
-        toXML(parmtree, xml);
-        DBGLOG("parmtree: %s", xml.str());
-    }
-}
-
 void appendValidInputBoxContent(StringBuffer &xml, const char *in)
 {
     //more later
@@ -1821,36 +1737,38 @@ void appendValidInputBoxContent(StringBuffer &xml, const char *in)
     toXML(validAndFlat, xml, 0, 0);
 }
 
-void CWsEclBinding::getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags)
+void CWsEclBinding::getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags, bool validate)
 {
-    Owned<IPropertyTree> parmtree = createPTree();
-    IProperties *parms = context.queryRequestParameters();
-
-    const char *boxInput = parms->queryProp("_boxFormInput");
+    IProperties *parameters = context.queryRequestParameters();
+    const char *boxInput = parameters->queryProp("_boxFormInput");
     if (boxInput)
     {
         appendValidInputBoxContent(soapmsg, boxInput);
         return;
     }
 
-    buildParametersXml(parmtree, parms);
+    Owned<IPropertyTree> reqTree = createPTreeFromHttpParameters(wsinfo.queryname, parameters);
 
-    StringBuffer element;
-    element.append(wsinfo.queryname.sget());
-        element.append("Request");
-
-    StringBuffer schemaXml;
-    getSchema(schemaXml, context, request, wsinfo);
-    if (getEspLogLevel()>LogNormal)
-        DBGLOG("request schema: %s", schemaXml.str());
-    Owned<IXmlSchema> schema = createXmlSchemaFromString(schemaXml);
-    if (schema.get())
+    if (!validate)
+        toXML(reqTree, soapmsg, 0, 0);
+    else
     {
-        IXmlType* type = schema->queryElementType(element);
-        if (type)
+        StringBuffer element;
+        element.append(wsinfo.queryname.sget()).append("Request");
+
+        StringBuffer schemaXml;
+        getSchema(schemaXml, context, request, wsinfo);
+        if (getEspLogLevel()>LogNormal)
+            DBGLOG("request schema: %s", schemaXml.str());
+        Owned<IXmlSchema> schema = createXmlSchemaFromString(schemaXml);
+        if (schema.get())
         {
-            StringArray parentTypes;
-            buildReqXml(parentTypes, type, soapmsg, (!stricmp(xmltype, "roxiexml")) ? wsinfo.queryname.sget() : element.str(), parmtree, flags|REQSF_ROOT, ns);
+            IXmlType* type = schema->queryElementType(element);
+            if (type)
+            {
+                StringArray parentTypes;
+                buildReqXml(parentTypes, type, soapmsg, (!stricmp(xmltype, "roxiexml")) ? wsinfo.queryname.sget() : element.str(), reqTree, flags|REQSF_ROOT, ns);
+            }
         }
     }
 }
@@ -1903,10 +1821,8 @@ void CWsEclBinding::getWsEclJsonRequest(StringBuffer& jsonmsg, IEspContext &cont
     size32_t start = jsonmsg.length();
     try
     {
-        Owned<IPropertyTree> parmtree = createPTree();
-        IProperties *parms = context.queryRequestParameters();
-
-        buildParametersXml(parmtree, parms);
+        IProperties *parameters = context.queryRequestParameters();
+        Owned<IPropertyTree> reqTree = createPTreeFromHttpParameters(wsinfo.queryname, parameters);
 
         StringBuffer element;
         element.append(wsinfo.queryname.sget());
@@ -1923,7 +1839,7 @@ void CWsEclBinding::getWsEclJsonRequest(StringBuffer& jsonmsg, IEspContext &cont
             if (type)
             {
                 StringArray parentTypes;
-                buildJsonMsg(parentTypes, type, jsonmsg, wsinfo.queryname.sget(), parmtree, flags|REQSF_ROOT);
+                buildJsonMsg(parentTypes, type, jsonmsg, wsinfo.queryname.sget(), reqTree, flags|REQSF_ROOT);
             }
         }
     }
@@ -2023,8 +1939,7 @@ void CWsEclBinding::getWsEclJsonResponse(StringBuffer& jsonmsg, IEspContext &con
     }
 }
 
-
-void CWsEclBinding::getSoapMessage(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, unsigned flags)
+void CWsEclBinding::getSoapMessage(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, unsigned flags, bool validate)
 {
     soapmsg.append(
         "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
@@ -2035,7 +1950,7 @@ void CWsEclBinding::getSoapMessage(StringBuffer& soapmsg, IEspContext &context,
 
     StringBuffer ns;
     ns.append("xmlns=\"urn:hpccsystems:ecl:").appendLower(wsinfo.queryname.length(), wsinfo.queryname.sget()).append('\"');
-    getWsEcl2XmlRequest(soapmsg, context, request, wsinfo, "soap", ns.str(), flags);
+    getWsEcl2XmlRequest(soapmsg, context, request, wsinfo, "soap", ns.str(), flags, validate);
 
     soapmsg.append("</soap:Body></soap:Envelope>");
 }
@@ -2063,7 +1978,7 @@ int CWsEclBinding::getXmlTestForm(IEspContext &context, CHttpRequest* request, C
     IProperties *parms = context.queryRequestParameters();
 
     StringBuffer soapmsg, pageName;
-    getSoapMessage(soapmsg, context, request, wsinfo, 0);
+    getSoapMessage(soapmsg, context, request, wsinfo, 0, true);
 
     StringBuffer params;
     const char* excludes[] = {"soap_builder_",NULL};
@@ -2237,7 +2152,7 @@ int CWsEclBinding::getWsEcl2Form(CHttpRequest* request, CHttpResponse* response,
     return 0;
 }
 
-int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, const char *xml, StringBuffer &out, unsigned flags, TextMarkupFormat fmt, const char *viewname, const char *xsltname)
+int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, IPropertyTree *reqTree, StringBuffer &out, unsigned flags, TextMarkupFormat fmt, const char *viewname, const char *xsltname)
 {
     Owned <IWorkUnitFactory> factory = getSecWorkUnitFactory(*context.querySecManager(), *context.queryUser());
     Owned <IWorkUnit> workunit = factory->createWorkUnit(NULL, "wsecl", context.queryUserId());
@@ -2263,13 +2178,14 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf
     workunit->setState(WUStateSubmitted);
     workunit->commit();
 
-    Owned<IPropertyTree> req = createPTreeFromXMLString(xml, ipt_none, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces));
-    IPropertyTree *start = req.get();
-    if (start->hasProp("Envelope"))
-        start=start->queryPropTree("Envelope");
-    if (start->hasProp("Body"))
-        start=start->queryPropTree("Body/*[1]");
-    workunit->setXmlParams(LINK(start));
+    if (reqTree)
+    {
+        if (reqTree->hasProp("Envelope"))
+            reqTree=reqTree->queryPropTree("Envelope");
+        if (reqTree->hasProp("Body"))
+            reqTree=reqTree->queryPropTree("Body/*[1]");
+        workunit->setXmlParams(LINK(reqTree));
+    }
 
     workunit->schedule();
     workunit.clear();
@@ -2309,89 +2225,10 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf
     return true;
 }
 
-void xppToXmlString(XmlPullParser &xpp, StartTag &stag, StringBuffer &buffer)
-{
-    int level = 1; //assumed due to the way gotonextdataset works.
-    int type = XmlPullParser::END_TAG;
-    const char * content = "";
-    const char *tag = NULL;
-    EndTag etag;
-
-    tag = stag.getLocalName();
-    if (tag && *tag)
-    {
-        buffer.appendf("<%s", tag);
-        for (int idx=0; idx<stag.getLength(); idx++)
-            buffer.appendf(" %s=\"%s\"", stag.getRawName(idx), stag.getValue(idx));
-        buffer.append(">");
-    }
-
-    do  
-    {
-        type = xpp.next();
-        switch(type) 
-        {
-            case XmlPullParser::START_TAG:
-            {
-                xpp.readStartTag(stag);
-                ++level;
-                tag = stag.getLocalName();
-                if (tag && *tag)
-                {
-                    buffer.appendf("<%s", tag);
-                    for (int idx=0; idx<stag.getLength(); idx++)
-                        buffer.appendf(" %s=\"%s\"", stag.getRawName(idx), stag.getValue(idx));
-                    buffer.append(">");
-                }
-                break;
-            }
-            case XmlPullParser::END_TAG:
-                xpp.readEndTag(etag);
-                tag = etag.getLocalName();
-                if (tag && *tag)
-                    buffer.appendf("</%s>", tag);
-                --level;
-            break;
-            case XmlPullParser::CONTENT:
-                content = xpp.readContent();
-                encodeUtf8XML(content, buffer);
-                break;
-            case XmlPullParser::END_DOCUMENT:
-                level=0;
-            break;
-        }
-    }
-    while (level > 0);
-}
-
-bool xppGotoTag(XmlPullParser &xppx, const char *tagname, StartTag &stag)
+int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, const char *xml, StringBuffer &out, unsigned flags, TextMarkupFormat fmt, const char *viewname, const char *xsltname)
 {
-    int level = 0;
-    int type = XmlPullParser::END_TAG;
-    do  
-    {
-        type = xppx.next();
-        switch(type) 
-        {
-            case XmlPullParser::START_TAG:
-            {
-                xppx.readStartTag(stag);
-                ++level;
-                const char *tag = stag.getLocalName();
-                if (tag && strieq(tag, tagname))
-                    return true;
-                break;
-            }
-            case XmlPullParser::END_TAG:
-                --level;
-            break;
-            case XmlPullParser::END_DOCUMENT:
-                level=0;
-            break;
-        }
-    }
-    while (level > 0);
-    return false;
+    Owned<IPropertyTree> reqTree = createPTreeFromXMLString(xml, ipt_none, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces));
+    return submitWsEclWorkunit(context, wsinfo, reqTree, out, flags, fmt, viewname, xsltname);
 }
 
 void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query, const char *contentType)
@@ -2443,43 +2280,42 @@ void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, Stri
 
 int CWsEclBinding::onSubmitQueryOutput(IEspContext &context, CHttpRequest* request, CHttpResponse* response, WsEclWuInfo &wsinfo, const char *format)
 {
-    bool outputJSON = !format ? false : strieq(format, "json");
-
-    StringBuffer soapmsg;
-
-    getSoapMessage(soapmsg, context, request, wsinfo, REQSF_TRIM|REQSF_ROOT);
-    if (getEspLogLevel()>LogNormal)
-        DBGLOG("submitQuery soap: %s", soapmsg.str());
-
-    const char *thepath = request->queryPath();
-
     StringBuffer status;
     StringBuffer output;
 
     SCMStringBuffer clustertype;
-    wsinfo.wu->getDebugValue("targetclustertype", clustertype);
 
-    unsigned xmlflags = WWV_ADD_RESPONSE_TAG | WWV_INCL_NAMESPACES | WWV_INCL_GENERATED_NAMESPACES;
-    if (context.queryRequestParameters()->hasProp("display"))
-        xmlflags |= WWV_USE_DISPLAY_XSLT;
-    if (!format || !streq(format, "expanded"))
-        xmlflags |= WWV_OMIT_SCHEMAS;
-    if (strieq(clustertype.str(), "roxie"))
+    bool isRoxieReq = wsecl->connMap.getValue(wsinfo.qsetname.get())!=NULL;
+    bool outputJSON = (format && strieq(format, "json"));
+    if (isRoxieReq && outputJSON)
     {
-        StringBuffer roxieresp;
-        sendRoxieRequest(wsinfo.qsetname.get(), soapmsg, roxieresp, status, wsinfo.queryname);
-
-        if (outputJSON)
-            getWsEclJsonResponse(output, context, request, roxieresp.str(), wsinfo);
+        StringBuffer jsonmsg;
+        getWsEclJsonRequest(jsonmsg, context, request, wsinfo, "json", NULL, 0);
+        sendRoxieRequest(wsinfo.qsetname.get(), jsonmsg, output, status, wsinfo.queryname, "application/json");
+    }
+    else
+    {
+        StringBuffer soapmsg;
+        getSoapMessage(soapmsg, context, request, wsinfo, REQSF_TRIM|REQSF_ROOT, false);
+        if (getEspLogLevel()>LogNormal)
+            DBGLOG("submitQuery soap: %s", soapmsg.str());
+
+        unsigned xmlflags = WWV_ADD_RESPONSE_TAG | WWV_INCL_NAMESPACES | WWV_INCL_GENERATED_NAMESPACES;
+        if (context.queryRequestParameters()->hasProp("display"))
+            xmlflags |= WWV_USE_DISPLAY_XSLT;
+        if (!format || !streq(format, "expanded"))
+            xmlflags |= WWV_OMIT_SCHEMAS;
+        if (!isRoxieReq)
+            submitWsEclWorkunit(context, wsinfo, soapmsg.str(), output, xmlflags, outputJSON ? MarkupFmt_JSON : MarkupFmt_XML);
         else
         {
+            StringBuffer roxieresp;
+            sendRoxieRequest(wsinfo.qsetname.get(), soapmsg, roxieresp, status, wsinfo.queryname);
             Owned<IWuWebView> web = createWuWebView(*wsinfo.wu, wsinfo.queryname.get(), getCFD(), true);
             if (web.get())
                 web->expandResults(roxieresp.str(), output, xmlflags);
         }
     }
-    else
-        submitWsEclWorkunit(context, wsinfo, soapmsg.str(), output, xmlflags, outputJSON ? MarkupFmt_JSON : MarkupFmt_XML);
 
     response->setContent(output.str());
     response->setContentType(outputJSON ? "application/json" : "application/xml");
@@ -2493,7 +2329,7 @@ int CWsEclBinding::onSubmitQueryOutputView(IEspContext &context, CHttpRequest* r
 {
     StringBuffer soapmsg;
 
-    getSoapMessage(soapmsg, context, request, wsinfo, REQSF_TRIM|REQSF_ROOT);
+    getSoapMessage(soapmsg, context, request, wsinfo, REQSF_TRIM|REQSF_ROOT, false);
     if (getEspLogLevel()>LogNormal)
         DBGLOG("submitQuery soap: %s", soapmsg.str());
 
@@ -2997,9 +2833,6 @@ void checkForXmlResponseName(StartTag &starttag, StringBuffer &respname, int &so
         respname.setLength(len-8);
 }
 
-
-void createPTreeFromJsonString(const char *json, bool caseInsensitive, StringBuffer &xml, const char *tail);
-
 void CWsEclBinding::handleJSONPost(CHttpRequest *request, CHttpResponse *response)
 {
     IEspContext *ctx = request->queryContext();
@@ -3044,51 +2877,41 @@ void CWsEclBinding::handleJSONPost(CHttpRequest *request, CHttpResponse *respons
             nextPathNode(thepath, queryname);
         }
 
+        const char *jsonp = ctx->queryRequestParameters()->queryProp("jsonp");
+        if (jsonp && *jsonp)
+            jsonresp.append(jsonp).append('(');
 
         StringBuffer content(request->queryContent());
+        if (getEspLogLevel()>LogNormal)
+            DBGLOG("json request: %s", content.str());
+
         StringBuffer status;
         if (wsecl->connMap.getValue(queryset.str()))
-        {
-            StringBuffer output;
-            if (getEspLogLevel()>LogNormal)
-                DBGLOG("roxie json req: %s", content.str());
-            const char *jsonp = ctx->queryRequestParameters()->queryProp("jsonp");
-            if (jsonp && *jsonp)
-                jsonresp.append(jsonp).append('(');
             sendRoxieRequest(queryset.str(), content, jsonresp, status, queryname.str(), "application/json");
-            if (jsonp && *jsonp)
-                jsonresp.append(");");
-            if (getEspLogLevel()>LogNormal)
-                DBGLOG("roxie json resp: %s", jsonresp.str());
-        }
         else
         {
             WsEclWuInfo wsinfo(wuid.str(), queryset.str(), queryname.str(), ctx->queryUserId(), ctx->queryPassword());
+            Owned<IPropertyTree> contentTree = createPTreeFromJSONString(content.str());
+            IPropertyTree *reqTree = contentTree.get();
 
-            StringBuffer soapfromjson;
-            soapfromjson.append(
-                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
-                "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
-                  " xmlns:SOAP-ENC=\"http://schemas.xmlsoap.org/soap/encoding/\">"
-                    " <soap:Body>"
-                );
-            createPTreeFromJsonString(content.str(), false, soapfromjson, "Request");
-            soapfromjson.append("</soap:Body></soap:Envelope>");
-            if (getEspLogLevel()>LogNormal)
-                DBGLOG("soap from json req: %s", soapfromjson.str());
-
-            unsigned xmlflags = WWV_ADD_SOAP | WWV_ADD_RESULTS_TAG | WWV_ADD_RESPONSE_TAG | WWV_INCL_NAMESPACES | WWV_INCL_GENERATED_NAMESPACES;
-            if (ctx->queryRequestParameters()->hasProp("display"))
-                xmlflags |= WWV_USE_DISPLAY_XSLT;
-            if (streq(action.str(), "expanded"))
-                xmlflags |= WWV_CDATA_SCHEMAS;
-            else
-                xmlflags |= WWV_OMIT_SCHEMAS;
-
-            submitWsEclWorkunit(*ctx, wsinfo, soapfromjson.str(), jsonresp, xmlflags, MarkupFmt_JSON);
-            if (getEspLogLevel()>LogNormal)
-                DBGLOG("HandleJSONRequest response: %s", jsonresp.str());
+            StringBuffer fullname(queryname);
+            fullname.append("Request");
+            Owned<IPropertyTreeIterator> it = reqTree->getElements("*");
+            ForEach(*it)
+            {
+                const char *name = it->query().queryName();
+                if (strieq(name, queryname) || strieq(name, fullname))
+                {
+                    reqTree = &it->query();
+                    break;
+                }
+            }
+            submitWsEclWorkunit(*ctx, wsinfo, reqTree, jsonresp, 0, MarkupFmt_JSON);
         }
+        if (jsonp && *jsonp)
+            jsonresp.append(");");
+        if (getEspLogLevel()>LogNormal)
+            DBGLOG("json response: %s", jsonresp.str());
 
     }
     catch (IException *e)
@@ -3207,4 +3030,3 @@ int CWsEclBinding::HandleSoapRequest(CHttpRequest* request, CHttpResponse* respo
 
     return 0;
 }
-

esp/services/ws_ecl/ws_ecl_service.hpp

@@ -162,13 +162,14 @@ public:
     int getXmlTestForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *formtype, WsEclWuInfo &wsinfo);
     int getXmlTestForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, WsEclWuInfo &wsinfo, const char *formtype);
 
-    void getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags);
+    void getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags, bool validate);
     void buildSampleResponseXml(StringBuffer& msg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo);
-    void getSoapMessage(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, unsigned flags);
+    void getSoapMessage(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, unsigned flags, bool validate);
     int onGetSoapBuilder(IEspContext &context, CHttpRequest* request, CHttpResponse* response,  WsEclWuInfo &wsinfo);
     int onSubmitQueryOutput(IEspContext &context, CHttpRequest* request, CHttpResponse* response,    WsEclWuInfo &wsinfo, const char *format);
     int onSubmitQueryOutputView(IEspContext &context, CHttpRequest* request, CHttpResponse* response, WsEclWuInfo &wsinfo);
 
+    int submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, IPropertyTree *reqTree, StringBuffer &out, unsigned flags, TextMarkupFormat fmt=MarkupFmt_XML, const char *viewname=NULL, const char *xsltname=NULL);
     int submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, const char *xml, StringBuffer &out, unsigned flags, TextMarkupFormat fmt=MarkupFmt_XML, const char *viewname=NULL, const char *xsltname=NULL);
 
     void handleHttpPost(CHttpRequest *request, CHttpResponse *response);

esp/xslt/wsecl3_form.xsl

@@ -597,12 +597,16 @@ function switchInputForm()
             <xsl:value-of select="concat('GetInputCtrlHtml(node=', $translated_name, ',fieldId=', $fieldId, ',collapsed=',  $collapsed, ')&lt;br/&gt;') "/>
         </xsl:if>
         <xsl:choose>
+            <xsl:when test="not($type) and not($cpxType)">
+                <xsl:text disable-output-escaping="yes"><![CDATA[<input type='checkbox' name=']]></xsl:text><xsl:value-of select="$fieldId"/><xsl:text disable-output-escaping="yes"><![CDATA['/>]]></xsl:text>
+            </xsl:when>
             <xsl:when test="starts-with($type, 'xsd:') or starts-with($type, 'xs:')">
                 <xsl:call-template name="GenXsdTypeHtml">
                     <xsl:with-param name="typeName" select="substring-after($type,':')"/>
                     <xsl:with-param name="fieldId" select="$fieldId"/>
                     <xsl:with-param name="value" select="$node/@default"/>
                     <xsl:with-param name="annot" select="$node/xsd:annotation/xsd:appinfo/form"/>
+                    <xsl:with-param name="maxoccurs" select="$node/@maxOccurs"/>
                 </xsl:call-template>
             </xsl:when>
             <xsl:when test="starts-with($type, 'tns:ArrayOf')">
@@ -801,12 +805,16 @@ function switchInputForm()
                     <xsl:when test="$schemaRoot/xsd:complexType[@name=$bareType]/xsd:all">
                     </xsl:when>
                     <xsl:otherwise>
-                        <xsl:value-of select="concat('WARNING[1]: unknown type: ', $type, ',fieldId=',$fieldId)"/>
+                        <xsl:if test="$type">
+                            <xsl:value-of select="concat('WARNING[1]: unknown type: ', $type, ',fieldId=',$fieldId)"/>
+                        </xsl:if>
                     </xsl:otherwise>
                 </xsl:choose>
             </xsl:when>
             <xsl:otherwise>
-                <xsl:value-of select="concat('WARNING[1]: unknown type: ', $type, ',fieldId=',$fieldId)"/>
+                <xsl:if test="$type">
+                    <xsl:value-of select="concat('WARNING[1]: unknown type: ', $type, ',fieldId=',$fieldId)"/>
+                </xsl:if>
             </xsl:otherwise>
         </xsl:choose>
     </xsl:template>
@@ -1067,6 +1075,7 @@ function switchInputForm()
         <xsl:param name="value"/>
         <xsl:param name="annot"/>
         <xsl:param name="field_len"/>
+        <xsl:param name="maxoccurs"/>
         <xsl:choose>
             <!-- string -->
             <xsl:when test="$typeName='string'">
@@ -1075,6 +1084,9 @@ function switchInputForm()
                         <xsl:when test="$annot/@formRows">
                             <xsl:value-of select="$annot/@formRows"/>
                         </xsl:when>
+                        <xsl:when test="$maxoccurs='unbounded'">
+                            <xsl:number value="4"/>
+                        </xsl:when>
                         <xsl:otherwise>0</xsl:otherwise>
                     </xsl:choose>
                 </xsl:variable>
@@ -1092,12 +1104,16 @@ function switchInputForm()
                 <xsl:choose>
                     <!-- use text area for string type -->
                     <xsl:when test="number($inputRows)">
+                        <xsl:if test="$maxoccurs='unbounded'">
+                            <xsl:text disable-output-escaping="yes"><![CDATA[[Enter one item per line]<br/>]]></xsl:text>
+                        </xsl:if>
                         <xsl:text disable-output-escaping="yes"><![CDATA[<textarea rows=']]></xsl:text>
                         <xsl:value-of select="$inputRows"/>
                         <xsl:text disable-output-escaping="yes"><![CDATA[' cols=']]></xsl:text>
                         <xsl:value-of select="$inputCols"/>
                         <xsl:text disable-output-escaping="yes"><![CDATA[' name=']]></xsl:text>
                         <xsl:value-of select="$fieldId"/>
+                        <xsl:if test="$maxoccurs='unbounded'">$</xsl:if>
                         <xsl:text disable-output-escaping="yes"><![CDATA[' id=']]></xsl:text>
                         <xsl:value-of select="$fieldId"/>
                         <xsl:text disable-output-escaping="yes"><![CDATA['>]]></xsl:text>
@@ -1153,22 +1169,61 @@ function switchInputForm()
                         <xsl:otherwise>20</xsl:otherwise>
                     </xsl:choose>
                 </xsl:variable>
-                <xsl:text disable-output-escaping="yes"><![CDATA[<input type='text' name=']]></xsl:text>
-                <xsl:value-of select="$fieldId"/>
-                <xsl:text disable-output-escaping="yes"><![CDATA[' id=']]></xsl:text>
-                <xsl:value-of select="$fieldId"/>
+                <xsl:variable name="inputRows">
+                    <xsl:choose>
+                        <xsl:when test="$maxoccurs='unbounded'">
+                            <xsl:number value="4"/>
+                        </xsl:when>
+                        <xsl:otherwise>0</xsl:otherwise>
+                    </xsl:choose>
+                </xsl:variable>
                 <xsl:choose>
-                    <xsl:when test="not($noDefaultValue) and $value">
-                        <xsl:text disable-output-escaping="yes"><![CDATA[' value=']]></xsl:text>
-                        <xsl:value-of select="$value"/>
-                    </xsl:when>
-                    <xsl:when test="$set_ctrl_value">
-                        <xsl:text disable-output-escaping="yes"><![CDATA[' value='12]]></xsl:text>
+                    <!-- use text area for string type -->
+                    <xsl:when test="number($inputRows)">
+                        <xsl:if test="$maxoccurs='unbounded'">
+                            <xsl:text disable-output-escaping="yes"><![CDATA[[Enter one item per line]<br/>]]></xsl:text>
+                        </xsl:if>
+                        <xsl:text disable-output-escaping="yes"><![CDATA[<textarea rows=']]></xsl:text>
+                        <xsl:value-of select="$inputRows"/>
+                        <xsl:text disable-output-escaping="yes"><![CDATA[' cols=']]></xsl:text>
+                        <xsl:value-of select="$inputCols"/>
+                        <xsl:text disable-output-escaping="yes"><![CDATA[' name=']]></xsl:text>
+                        <xsl:value-of select="$fieldId"/>
+                        <xsl:if test="$maxoccurs='unbounded'">$</xsl:if>
+                        <xsl:text disable-output-escaping="yes"><![CDATA[' id=']]></xsl:text>
+                        <xsl:value-of select="$fieldId"/>
+                        <xsl:text disable-output-escaping="yes"><![CDATA['>]]></xsl:text>
+                        <xsl:choose>
+                            <xsl:when test="not($noDefaultValue) and $value">
+                                <xsl:value-of select="$value"/>
+                            </xsl:when>
+                            <xsl:when test="$set_ctrl_value">
+                                <xsl:value-of select="$fieldId"/>
+                            </xsl:when>
+                        </xsl:choose>
+                        <xsl:text disable-output-escaping="yes"><![CDATA[</textarea>]]></xsl:text>
                     </xsl:when>
+                    <!-- use input for string type -->
+                    <!-- -->
+                    <xsl:otherwise>
+                        <xsl:text disable-output-escaping="yes"><![CDATA[<input type='text' name=']]></xsl:text>
+                        <xsl:value-of select="$fieldId"/>
+                        <xsl:text disable-output-escaping="yes"><![CDATA[' id=']]></xsl:text>
+                        <xsl:value-of select="$fieldId"/>
+                        <xsl:choose>
+                            <xsl:when test="not($noDefaultValue) and $value">
+                                <xsl:text disable-output-escaping="yes"><![CDATA[' value=']]></xsl:text>
+                                <xsl:value-of select="$value"/>
+                            </xsl:when>
+                            <xsl:when test="$set_ctrl_value">
+                                <xsl:text disable-output-escaping="yes"><![CDATA[' value='12]]></xsl:text>
+                            </xsl:when>
+                        </xsl:choose>
+                        <xsl:text disable-output-escaping="yes"><![CDATA[' size=']]></xsl:text>
+                        <xsl:value-of select="$inputCols"/>
+                        <xsl:text disable-output-escaping="yes"><![CDATA[' ></input>]]></xsl:text>
+                    </xsl:otherwise>
                 </xsl:choose>
-                <xsl:text disable-output-escaping="yes"><![CDATA[' size=']]></xsl:text>
-                <xsl:value-of select="$inputCols"/>
-                <xsl:text disable-output-escaping="yes"><![CDATA[' ></input>]]></xsl:text>
             </xsl:when>
             <!-- boolean -->
             <xsl:when test="$typeName='boolean'">

system/jlib/jptree.cpp

@@ -5448,6 +5448,237 @@ void saveXML(IIOStream &stream, const IPropertyTree *tree, unsigned indent, byte
     toXML(tree, stream, indent, flags);
 }
 
+/////////////////////////
+
+void checkWriteJSONDelimiter(IIOStream &out, bool &delimit)
+{
+    if (delimit)
+        writeCharToStream(out, ',');
+    delimit = false;
+}
+
+static void writeJSONNameToStream(IIOStream &out, const char *name, unsigned indent, bool &delimit)
+{
+    if (!name || !*name)
+        return;
+    checkWriteJSONDelimiter(out, delimit);
+    if (indent)
+    {
+        writeCharToStream(out, '\n');
+        writeCharsNToStream(out, ' ', indent);
+    }
+    else
+        writeCharToStream(out, ' ');
+
+    writeCharToStream(out, '"');
+    writeStringToStream(out, name);
+    writeStringToStream(out, "\": ");
+    delimit = false;
+}
+
+static void writeJSONValueToStream(IIOStream &out, const char *val, bool &delimit, bool hidden=false)
+{
+    checkWriteJSONDelimiter(out, delimit);
+    delimit = true;
+    if (!val)
+    {
+        writeStringToStream(out, "null");
+        return;
+    }
+    writeCharToStream(out, '"');
+    if (hidden)
+        writeCharsNToStream(out, '*', strlen(val));
+    else
+        writeStringToStream(out, val);
+    writeCharToStream(out, '"');
+}
+
+static void writeJSONBase64ValueToStream(IIOStream &out, const char *val, size32_t len, bool &delimit)
+{
+    checkWriteJSONDelimiter(out, delimit);
+    delimit = true;
+    if (!val)
+    {
+        writeStringToStream(out, "null");
+        return;
+    }
+    writeCharToStream(out, '"');
+    JBASE64_Encode(val, len, out);
+    writeCharToStream(out, '"');
+}
+
+static void checkWriteJSONCloseArrayToStream(IIOStream &out, StringAttr &prevItem, const char *name, bool format, unsigned &indent, bool &delimit)
+{
+    if (prevItem.isEmpty())
+        return;
+    if (!name || !streq(prevItem, name))
+    {
+        if (format)
+        {
+            writeCharToStream(out, '\n');
+            writeCharsNToStream(out, ' ', indent);
+            indent--;
+        }
+        writeCharToStream(out, ']');
+        prevItem.clear();
+        delimit = true;
+    }
+}
+
+static void _toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags, bool &delimit, bool root=false)
+{
+    Owned<IAttributeIterator> it = tree->getAttributes(true);
+    bool hasAttributes = it->first();
+    bool complex = (hasAttributes || tree->hasChildren());
+    bool isBinary = tree->isBinary(NULL);
+    bool isItem = tree->isArrayItem();
+
+    const char *name = tree->queryName();
+    if (!root && !isItem)
+    {
+        if (!name || !*name)
+            name = "__unnamed__";
+        writeJSONNameToStream(out, name, (flags & JSON_Format) ? indent : 0, delimit);
+    }
+
+    checkWriteJSONDelimiter(out, delimit);
+
+    if (isItem && (flags & JSON_Format))
+    {
+        writeCharToStream(out, '\n');
+        writeCharsNToStream(out, ' ', indent);
+    }
+
+    if (root || complex)
+    {
+        writeCharToStream(out, '{');
+        delimit = false;
+    }
+
+    if (hasAttributes)
+    {
+        ForEach(*it)
+        {
+            const char *key = it->queryName();
+            if (!isBinary || stricmp(key, "@xsi:type")!=0)
+            {
+                const char *val = it->queryValue();
+                if (val)
+                {
+                    writeJSONNameToStream(out, key, (flags & JSON_Format) ? indent+1 : 0, delimit);
+                    if (flags & JSON_SanitizeAttributeValues)
+                    {
+                        bool hide = !(streq(val, "0") || streq(val, "1") || strieq(val, "true") || strieq(val, "false") || strieq(val, "yes") || strieq(val, "no"));
+                        writeJSONValueToStream(out, val, delimit, hide);
+                    }
+                    else
+                    {
+                        StringBuffer encoded;
+                        encodeJSON(encoded, val);
+                        writeJSONValueToStream(out, encoded.str(), delimit);
+                    }
+                }
+            }
+        }
+    }
+    MemoryBuffer thislevelbin;
+    StringBuffer _thislevel;
+    const char *thislevel = NULL; // to avoid uninitialized warning
+    bool empty;
+    if (isBinary)
+    {
+        empty = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
+    }
+    else
+    {
+        if (tree->isCompressed(NULL))
+        {
+            empty = false; // can't be empty if compressed;
+            verifyex(tree->getProp(NULL, _thislevel));
+            thislevel = _thislevel.toCharArray();
+        }
+        else
+            empty = (NULL == (thislevel = tree->queryProp(NULL)));
+    }
+
+    if (empty && !complex)
+    {
+        writeJSONValueToStream(out, NULL, delimit);
+        return;
+    }
+
+    Owned<IPropertyTreeIterator> sub = tree->getElements("*", 0 != (flags & JSON_SortTags) ? iptiter_sort : iptiter_null);
+    StringAttr lastArrayItem;
+    ForEach(*sub)
+    {
+        IPropertyTree &element = sub->query();
+        const char *name = element.queryName();
+        checkWriteJSONCloseArrayToStream(out, lastArrayItem, name, (flags & JSON_Format), indent, delimit);
+
+        if (element.isArrayItem() && lastArrayItem.isEmpty())
+        {
+            if (flags & JSON_Format)
+                indent++;
+            writeJSONNameToStream(out, name, (flags & JSON_Format) ? indent : 0, delimit);
+            writeCharToStream(out, '[');
+            lastArrayItem.set(name);
+        }
+
+        _toJSON(&sub->query(), out, indent+1, flags, delimit);
+    }
+
+    checkWriteJSONCloseArrayToStream(out, lastArrayItem, NULL, (flags & JSON_Format), indent, delimit);
+
+    if (!empty)
+    {
+        if (complex)
+            writeJSONNameToStream(out, "#value", (flags & JSON_Format) ? indent+1 : 0, delimit);
+        if (isBinary)
+            writeJSONBase64ValueToStream(out, thislevelbin.toByteArray(), thislevelbin.length(), delimit);
+        else
+        {
+            // NOTE - JSON_Sanitize won't output anything for binary.... is that ok?
+            bool hide = (flags & JSON_Sanitize) && thislevel && !(streq(thislevel, "0") || streq(thislevel, "1") || strieq(thislevel, "true") || strieq(thislevel, "false") || strieq(thislevel, "yes") || strieq(thislevel, "no"));
+            writeJSONValueToStream(out, thislevel, delimit, hide);
+        }
+    }
+
+    if (root || complex)
+    {
+        if (flags & JSON_Format)
+        {
+            writeCharToStream(out, '\n');
+            writeCharsNToStream(out, ' ', indent);
+        }
+        writeCharToStream(out, '}');
+        delimit = true;
+    }
+}
+
+jlib_decl StringBuffer &toJSON(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, byte flags)
+{
+    class CAdapter : public CInterface, implements IIOStream
+    {
+        StringBuffer &out;
+    public:
+        IMPLEMENT_IINTERFACE;
+        CAdapter(StringBuffer &_out) : out(_out) { }
+        virtual void flush() { }
+        virtual size32_t read(size32_t len, void * data) { UNIMPLEMENTED; return 0; }
+        virtual size32_t write(size32_t len, const void * data) { out.append(len, (const char *)data); return len; }
+    } adapter(ret);
+    bool delimit = false;
+    _toJSON(tree->queryBranch(NULL), adapter, indent, flags, delimit, true);
+    return ret;
+}
+
+void toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags)
+{
+    bool delimit = false;
+    _toJSON(tree, out, indent, flags, delimit, true);
+}
+
+
 static inline void skipWS(const char *&xpath)
 {
     while (isspace(*xpath)) xpath++;

system/jlib/jptree.hpp

@@ -115,6 +115,7 @@ interface jlib_decl IPropertyTree : extends serializable
     virtual bool IsShared() const = 0;
     virtual void localizeElements(const char *xpath, bool allTail=false) = 0;
     virtual unsigned getCount(const char *xpath) = 0;
+    virtual bool isArrayItem() const = 0;
     
 private:
     void setProp(const char *, int); // dummy to catch accidental use of setProp when setPropInt() intended
@@ -221,6 +222,14 @@ jlib_decl void saveXML(IFile &ifile, const IPropertyTree *tree, unsigned indent
 jlib_decl void saveXML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent = 0, byte flags=XML_Format);
 jlib_decl void saveXML(IIOStream &stream, const IPropertyTree *tree, unsigned indent = 0, byte flags=XML_Format);
 
+#define JSON_SortTags 0x01
+#define JSON_Format   0x02
+#define JSON_Sanitize 0x08
+#define JSON_SanitizeAttributeValues 0x10
+
+jlib_decl StringBuffer &toJSON(const IPropertyTree *tree, StringBuffer &ret, unsigned indent = 0, byte flags=JSON_Format);
+jlib_decl void toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent = 0, byte flags=JSON_Format);
+
 jlib_decl const char *splitXPath(const char *xpath, StringBuffer &head); // returns tail, fills 'head' with leading xpath
 jlib_decl bool validateXPathSyntax(const char *xpath, StringBuffer *error=NULL);
 jlib_decl bool validateXMLParseXPath(const char *xpath, StringBuffer *error=NULL);

system/jlib/jptree.ipp

@@ -385,6 +385,7 @@ public:
     virtual IPropertyTreeIterator *getElements(const char *xpath, IPTIteratorCodes flags = iptiter_null) const;
     virtual void localizeElements(const char *xpath, bool allTail=false);
     virtual bool hasChildren() const { return children && children->count()?true:false; }
+    virtual bool isArrayItem() const { return parent!=NULL; }
     virtual unsigned numUniq() { return checkChildren()?children->count():0; }  
     virtual unsigned numChildren();
     virtual bool isCaseInsensitive() { return isnocase(); }

system/xmllib/xsdparser.cpp

@@ -795,15 +795,22 @@ IXmlType* CXmlSchema::parseComplexType(IPTree* complexDef)
         Owned<IPTreeIterator> els = sub->getElements(VStringBuffer("%selement",xsdNs()));
         
         size_t fldCount = 0;
+        size_t typelessCount = 0;
         for (els->first(); els->isValid(); els->next())
+        {
             fldCount++;
+            if (!els->query().hasProp("@type") && !els->query().hasChildren())
+                typelessCount++;
+        }
 
         // TODO: verify with struct with one field
-        if (fldCount==1) // hack: assume to be array
+        if ((fldCount-typelessCount)==1) // hack: assume 1 to be array
         {
-            els->first();
-            IPTree& el = els->query();
+            for (els->first(); els->isValid(); els->next())
+                if (els->query().hasProp("@type") || els->query().hasChildren())
+                    break;
 
+            IPTree& el = els->query();
             const char* maxOccurs = sub->queryProp("@maxOccurs");
             if (!maxOccurs)
                 maxOccurs = els->query().queryProp("@maxOccurs");
@@ -879,11 +886,8 @@ IXmlType* CXmlSchema::parseComplexType(IPTree* complexDef)
                 const char* itemName = el.queryProp("@name");
                 const char* typeName = el.queryProp("@type");
                 IXmlType* type = typeName ? queryTypeByName(typeName,el.queryProp("@default")) : parseTypeDef(&el);
-                if (!type) 
-                {
-                    DBGLOG(-1,"Parse schema failed: itemName=%s, typeName=%s", itemName?itemName:"<no-name>", typeName?typeName:"<no-type-name>");
-                    throw MakeStringException(-1, "Internal error: parse schema failed");
-                }
+                if (!type)
+                    type = getNativeSchemaType("none", el.queryProp("@default")); //really should be tag only, no content?
                 
                 types[fldIdx] = type;
                 names[fldIdx] = strdup(itemName);