HPCC-10675 Support spraying of JSON files

This initial implementation focuses on maximizing function over
performance.  Major optimizations will probably be necessary.

Also some highly complex JSON structures may not be partitioned or
recombined correctly.  More testing of complex files is needed.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

dali/ft/daftformat.cpp

@@ -113,7 +113,7 @@ void CPartitioner::commonCalcPartitions()
         //Don't add an empty block on the start of the this chunk to transfer.
         if ((split != firstSplit) || (inputOffset != startInputOffset))
         {
-            results.append(*new PartitionPoint(whichInput, split-1, startInputOffset-thisOffset+thisHeaderSize, inputOffset - startInputOffset, cursor.outputOffset-startOutputOffset));
+            results.append(*new PartitionPoint(whichInput, split-1, startInputOffset-thisOffset+thisHeaderSize, inputOffset - startInputOffset - cursor.trimLength, cursor.outputOffset-startOutputOffset));
             startInputOffset = inputOffset;
             startOutputOffset = cursor.outputOffset;
         }
@@ -1577,6 +1577,64 @@ offset_t XmlSplitter::getFooterLength(BufferedDirectReader & reader, offset_t si
 }
 
 
+CJsonInputPartitioner::CJsonInputPartitioner(const FileFormat & _format)
+{
+    format.set(_format);
+    CriticalBlock block(openfilecachesect);
+    if (!openfilecache)
+        openfilecache = createFileIOCache(16);
+    else
+        openfilecache->Link();
+}
+
+IFileIOCache *CJsonInputPartitioner::openfilecache = NULL;
+CriticalSection CJsonInputPartitioner::openfilecachesect;
+
+CJsonInputPartitioner::~CJsonInputPartitioner()
+{
+    json.clear();
+    inStream.clear();
+    if (openfilecache) {
+        CriticalBlock block(openfilecachesect);
+        if (openfilecache->Release())
+            openfilecache = NULL;
+    }
+}
+
+void CJsonInputPartitioner::setSource(unsigned _whichInput, const RemoteFilename & _fullPath, bool _compressedInput, const char *_decryptKey)
+{
+    CPartitioner::setSource(_whichInput, _fullPath, _compressedInput,_decryptKey);
+    Owned<IFileIO> inIO;
+    Owned<IFile> inFile = createIFile(inputName);
+    if (!inFile->exists()) {
+        StringBuffer tmp;
+        inputName.getRemotePath(tmp);
+        throwError1(DFTERR_CouldNotOpenFilePart, tmp.str());
+    }
+    inIO.setown(openfilecache->addFile(inputName,IFOread));
+
+    if (_compressedInput) {
+        Owned<IExpander> expander;
+        if (_decryptKey&&*_decryptKey) {
+            StringBuffer key;
+            decrypt(key,_decryptKey);
+            expander.setown(createAESExpander256(key.length(),key.str()));
+        }
+        inIO.setown(createCompressedFileReader(inIO,expander));
+    }
+
+    inStream.setown(createIOStream(inIO));
+    json.setown(new JsonSplitter(format, *inStream));
+    json->getHeaderLength();
+}
+
+CJsonPartitioner::CJsonPartitioner(const FileFormat & _format) : CJsonInputPartitioner(_format)
+{
+    unitSize = format.getUnitSize();
+    utfFormat = getUtfFormatType(format.type);
+}
+
+
 CXmlPartitioner::CXmlPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize), splitter(_format)
 {
     LOG(MCdebugProgressDetail, unknownJob, "CXmlPartitioner::CXmlPartitioner(_format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), format.getUnitSize());
@@ -2128,13 +2186,15 @@ IFormatProcessor * createFormatProcessor(const FileFormat & srcFormat, const Fil
             partitioner = new CCsvQuickPartitioner(srcFormat, sameFormats);
         break;
     case FFTutf8: case FFTutf8n: case FFTutf16: case FFTutf16be: case FFTutf16le: case FFTutf32: case FFTutf32be: case FFTutf32le:
-        if (srcFormat.rowTag)
+        if (srcFormat.hasXmlMarkup())
         {
             if (calcOutput && !sameFormats)
                 partitioner = new CXmlPartitioner(srcFormat);
             else
                 partitioner = new CXmlQuickPartitioner(srcFormat, sameFormats);
         }
+        else if (srcFormat.hasJsonMarkup())
+            partitioner = new CJsonPartitioner(srcFormat);
         else
         {
             if (calcOutput && !sameFormats)
@@ -2190,15 +2250,13 @@ IFormatPartitioner * createFormatPartitioner(const SocketEndpoint & ep, const Fi
                 return new CCsvQuickPartitioner(srcFormat, sameFormats);
             break;
         case FFTutf: case FFTutf8: case FFTutf8n: case FFTutf16: case FFTutf16be: case FFTutf16le: case FFTutf32: case FFTutf32be: case FFTutf32le:
-            if (srcFormat.rowTag)
+            if (srcFormat.hasXmlMarkup())
                 return new CXmlQuickPartitioner(srcFormat, sameFormats);
-            else
-            {
-                if (srcFormat.hasQuote() && srcFormat.hasQuotedTerminator())
-                    return new CUtfPartitioner(srcFormat);
-                else
-                    return new CUtfQuickPartitioner(srcFormat, sameFormats);
-            }
+            if (srcFormat.hasJsonMarkup())
+                return new CJsonPartitioner(srcFormat);
+            if (srcFormat.hasQuote() && srcFormat.hasQuotedTerminator())
+                return new CUtfPartitioner(srcFormat);
+            return new CUtfQuickPartitioner(srcFormat, sameFormats);
         }
     }
     if (!calcOutput)
@@ -2220,10 +2278,11 @@ IFormatPartitioner * createFormatPartitioner(const SocketEndpoint & ep, const Fi
         case FFTcsv:
             return new CCsvQuickPartitioner(srcFormat, sameFormats);
         case FFTutf: case FFTutf8: case FFTutf8n: case FFTutf16: case FFTutf16be: case FFTutf16le: case FFTutf32: case FFTutf32be: case FFTutf32le:
-            if (srcFormat.rowTag)
+            if (srcFormat.hasXmlMarkup())
                 return new CXmlQuickPartitioner(srcFormat, sameFormats);
-            else
-                return new CUtfQuickPartitioner(srcFormat, sameFormats);
+            if (srcFormat.hasJsonMarkup())
+                return new CJsonPartitioner(srcFormat);
+            return new CUtfQuickPartitioner(srcFormat, sameFormats);
         default:
             throwError(DFTERR_UnknownFileFormatType);
             break;

dali/ft/daftformat.hpp

@@ -33,11 +33,12 @@
 struct PartitionCursor
 {
 public:
-    PartitionCursor(offset_t _inputOffset)  { inputOffset = nextInputOffset = _inputOffset; outputOffset = 0; }
+    PartitionCursor(offset_t _inputOffset)  { inputOffset = nextInputOffset = _inputOffset; outputOffset = 0; trimLength = 0; }
     
     offset_t        inputOffset;
     offset_t        nextInputOffset;
     offset_t        outputOffset;
+    offset_t        trimLength;
 };
 
 struct TransformCursor

dali/ft/daftformat.ipp

@@ -23,6 +23,7 @@
 #include "daft.hpp"
 #include "daftformat.hpp"
 #include "rmtpass.hpp"
+#include "jptree.hpp"
 
 //---------------------------------------------------------------------------
 
@@ -355,6 +356,224 @@ protected:
     Linked<IFileIOStream> stream;
 };
 
+class DALIFT_API JsonSplitter : public CInterface, implements IPTreeNotifyEvent
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    JsonSplitter(const FileFormat & format, IFileIOStream &stream) : headerLength(0), pathPos(0), tangent(0), rowDepth(0), rowStart(0), rowEnd(0), footerLength(0), newRowSet(true)
+    {
+        size = stream.size();
+        const char *rowPath = format.rowTag;
+        while (*rowPath=='/')
+            rowPath++;
+        pathNodes.appendList(rowPath, "/");
+        noPath = !pathNodes.length();
+        reader.setown(createPullJSONStreamReader(stream, *this, ptr_noRoot));
+    }
+    virtual void beginNode(const char *tag, offset_t startOffset)
+    {
+        if (tangent)
+        {
+            tangent++;
+            return;
+        }
+        if (rowDepth)
+        {
+            rowDepth++;
+            return;
+        }
+        if (!pathPos)
+        {
+            if (streq(tag, "__array__")) //ignore root array
+                return;
+            if (!noPath && streq(tag, "__object__")) //paths start inside this object, with no path this starts a row
+                return;
+        }
+        if (noPath || streq(tag, pathNodes.item(pathPos))) //closer to a row
+        {
+            if (!noPath)
+                pathPos++;
+            if (noPath || pathPos==pathNodes.ordinality()) //start a row
+            {
+                rowDepth=1;
+                rowStart=startOffset;
+            }
+            return;
+        }
+        //off on a tangent
+        tangent=1;
+    }
+    virtual void newAttribute(const char *name, const char *value){}
+    virtual void beginNodeContent(const char *tag){}
+    virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset)
+    {
+        if (rowDepth)
+        {
+            rowDepth--;
+            if (!rowDepth)
+            {
+                rowEnd=endOffset;
+                if (newRowSet)
+                    newRowSet = false;  //individual row ended, but track whether the rowset itself ends before next row start
+                pathPos--;
+            }
+        }
+        else if (tangent)
+            tangent--;
+        else if (pathPos)
+        {
+            if (!newRowSet)
+                newRowSet=true;
+            pathPos--;
+        }
+    }
+
+    bool findNextRow()
+    {
+        if (rowDepth && !exitRow())
+            return false;
+        while (reader->next())
+            if (rowDepth==1)
+                return true;
+        return false;
+    }
+
+    bool exitRow()
+    {
+        if (!rowDepth)
+            return false;
+        while (reader->next())
+            if (rowDepth==0)
+                return true;
+        return false;
+    }
+    bool findNextRowEnd()
+    {
+        if (rowDepth)
+            return exitRow();
+        while (reader->next())
+            if (rowDepth==1)
+                return exitRow();
+        return false;
+    }
+    bool findRowEnd(offset_t splitOffset, offset_t &prevRowEnd)
+    {
+        while (rowEnd < splitOffset + headerLength)
+        {
+            prevRowEnd = rowEnd;
+            if (!findNextRowEnd())
+                return false;
+        }
+        return true;
+    }
+    offset_t getHeaderLength()
+    {
+        if (!headerLength)
+        {
+            while (!rowStart && reader->next());
+            if (!rowStart)
+                throw MakeStringException(DFTERR_CannotFindFirstJsonRecord, "Could not find first json record (check path)");
+            else
+                headerLength = rowStart-1;
+        }
+        return headerLength;
+    }
+    offset_t getFooterLength()
+    {
+        if (!footerLength)
+        {
+            while (reader->next());
+            if (rowEnd)
+                footerLength = size + 1 - rowEnd;
+        }
+        return footerLength;
+    }
+    offset_t getRowOffset()
+    {
+        if (rowStart <= headerLength)
+            return 0;
+        return rowStart - headerLength - 1;
+    }
+
+public:
+    Owned<IFileIOStream> inStream;
+    Owned<IPullPTreeReader> reader;
+    StringArray pathNodes;
+    offset_t rowStart;
+    offset_t rowEnd;
+    offset_t headerLength;
+    offset_t footerLength;
+    offset_t size;
+    unsigned pathPos;
+    unsigned tangent;
+    unsigned rowDepth;
+    bool noPath;
+    bool newRowSet;
+};
+
+class DALIFT_API CJsonInputPartitioner : public CPartitioner
+{
+public:
+    CJsonInputPartitioner(const FileFormat & _format);
+    ~CJsonInputPartitioner();
+
+    virtual void setSource(unsigned _whichInput, const RemoteFilename & _fullPath, bool compressedInput, const char *decryptKey);
+
+protected:
+    virtual void findSplitPoint(offset_t splitOffset, PartitionCursor & cursor)
+    {
+        if (!splitOffset) //header + 0 is first offset
+            return;
+
+        offset_t prevRowEnd;
+        json->findRowEnd(splitOffset, prevRowEnd);
+        if (!json->rowStart)
+            return;
+        if (!json->newRowSet) //get rid of extra delimiter if we haven't closed and reopened in the meantime
+            cursor.trimLength = json->rowStart - prevRowEnd;
+        cursor.inputOffset = json->getRowOffset();
+        if (json->findNextRow())
+            cursor.nextInputOffset = json->getRowOffset();
+        else
+            cursor.nextInputOffset = cursor.inputOffset;  //eof
+    }
+
+protected:
+    FileFormat      format;
+    Owned<IFileIOStream>   inStream;
+    Owned<JsonSplitter> json;
+    static IFileIOCache    *openfilecache;
+    static CriticalSection openfilecachesect;
+};
+
+
+class DALIFT_API CJsonPartitioner : public CJsonInputPartitioner
+{
+public:
+    CJsonPartitioner(const FileFormat & _format);
+
+    virtual void setTarget(IOutputProcessor * _target){UNIMPLEMENTED;}
+
+    //Processing.
+    virtual void beginTransform(offset_t thisOffset, offset_t thisLength, TransformCursor & cursor){UNIMPLEMENTED;}
+    virtual void endTransform(TransformCursor & cursor){UNIMPLEMENTED;}
+    virtual crc32_t getInputCRC(){return 0;}
+    virtual void setInputCRC(crc32_t value){UNIMPLEMENTED;}
+    virtual unsigned transformBlock(offset_t endOffset, TransformCursor & cursor){UNIMPLEMENTED;}
+    virtual void killBuffer(){}
+
+protected:
+    virtual size32_t getSplitRecordSize(const byte * record, unsigned maxToRead, bool processFullBuffer){UNIMPLEMENTED;}
+    virtual size32_t getTransformRecordSize(const byte * record, unsigned maxToRead){UNIMPLEMENTED;}
+
+protected:
+    size32_t recordSize;
+    unsigned unitSize;
+    UtfReader::UtfFormat utfFormat;
+};
+
+
 class DALIFT_API XmlSplitter
 {
 public:

dali/ft/filecopy.cpp

@@ -1041,17 +1041,31 @@ void FileSprayer::calculateSplitPrefixPartition(const char * splitPrefix)
     }
 }
 
-
 void FileSprayer::calculateMany2OnePartition()
 {
     LOG(MCdebugProgressDetail, job, "Setting up many2one partition");
-
+    bool isJSON = srcFormat.hasJsonMarkup();
+    offset_t lastContentLength = 0;
     ForEachItemIn(idx, sources)
     {
         FilePartInfo & cur = sources.item(idx);
         RemoteFilename curFilename;
         curFilename.set(cur.filename);
         setCanAccessDirectly(curFilename);
+        if (isJSON)
+        {
+            offset_t contentLength = cur.size - cur.xmlHeaderLength - cur.xmlFooterLength;
+            if (contentLength)
+            {
+                if (lastContentLength)
+                {
+                    PartitionPoint &part = createLiteral(1, ",", (unsigned) -1);
+                    part.whichOutput = 0;
+                    partition.append(part);
+                }
+                lastContentLength = contentLength;
+            }
+        }
         partition.append(*new PartitionPoint(idx, 0, cur.headerSize, cur.size, cur.size));
     }
 }
@@ -1531,6 +1545,15 @@ void FileSprayer::analyseFileHeaders(bool setcurheadersize)
 void FileSprayer::locateXmlHeader(IFileIO * io, unsigned headerSize, offset_t & xmlHeaderLength, offset_t & xmlFooterLength)
 {
     Owned<IFileIOStream> in = createIOStream(io);
+
+    if (srcFormat.hasJsonMarkup())
+    {
+        JsonSplitter jsplitter(srcFormat, *in);
+        xmlHeaderLength = jsplitter.getHeaderLength();
+        xmlFooterLength = jsplitter.getFooterLength();
+        return;
+    }
+
     XmlSplitter splitter(srcFormat);
     BufferedDirectReader reader;
 
@@ -2763,7 +2786,7 @@ void FileSprayer::spray()
         insertHeaders();
     }
     addEmptyFilesToPartition();
-    
+
     derivePartitionExtra();
     if (partition.ordinality() < 1000)
         displayPartition();

dali/ft/filecopy.hpp

@@ -72,6 +72,8 @@ public:
     bool hasQuote() const                           { return (quote == NULL) || (*quote != '\0'); }
     bool hasQuotedTerminator() const                { return quotedTerminator; }
     const char * getFileFormatTypeString() const        { return FileFormatTypeStr[type]; }
+    bool hasJsonMarkup() const {return (rowTag.length() && *rowTag.get()=='/');} //need to add a more explicit markup indicator
+    bool hasXmlMarkup() const {return (rowTag.length() && *rowTag.get()!='/');}
 
 public:
     FileFormatType      type;

dali/ft/fterror.hpp

@@ -79,6 +79,7 @@
 #define DFTERR_WrongRECFMvbBlockSize            8106
 #define DFTERR_WrongRECFMvRecordSize            8107
 #define DFTERR_WrongSplitRecordSize             8108
+#define DFTERR_CannotFindFirstJsonRecord        8109
 
 //Internal errors
 #define DFTERR_UnknownFormatType                8190

esp/scm/ws_fs.ecm

@@ -367,6 +367,8 @@ ESPrequest [nil_remove] SprayVariable
     [min_ver("1.09")] bool recordStructurePresent(false);
 
     [min_ver("1.10")] bool quotedTerminator(true);
+    [min_ver("1.11")] string sourceRowPath;
+    [min_ver("1.11")] bool isJSON(false);
 };
 
 ESPresponse [exceptions_inline] 
@@ -627,7 +629,7 @@ ESPresponse [exceptions_inline, nil_remove] GetSprayTargetsResponse
 };
 
 ESPservice [
-    version("1.10"), default_client_version("1.10"),
+    version("1.11"), default_client_version("1.10"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPuses ESPstruct DFUWorkunit;

esp/services/ws_fs/ws_fsService.cpp

@@ -2072,9 +2072,19 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
         source->setMaxRecordSize(req.getSourceMaxRecordSize());
         source->setFormat((DFUfileformat)req.getSourceFormat());
 
-        // if rowTag specified, it means it's xml format, otherwise it's csv
-        const char* rowtag = req.getSourceRowTag();
-        if(rowtag != NULL && *rowtag != '\0')
+        StringBuffer rowtag;
+        if (req.getIsJSON())
+        {
+            const char *srcRowPath = req.getSourceRowPath();
+            if (!srcRowPath || *srcRowPath != '/')
+                rowtag.append("/");
+            rowtag.append(srcRowPath);
+        }
+        else
+            rowtag.append(req.getSourceRowTag());
+
+        // if rowTag specified, it means it's xml or json format, otherwise it's csv
+        if(rowtag.length())
         {
             source->setRowTag(rowtag);
             options->setKeepHeader(true);

esp/src/eclwatch/LZBrowseWidget.js

@@ -89,6 +89,9 @@ define([
             this.sprayXmlForm = registry.byId(this.id + "SprayXmlForm");
             this.sprayXmlDestinationSelect = registry.byId(this.id + "SprayXmlDestinationSelect");
             this.sprayXmlGrid = registry.byId(this.id + "SprayXmlGrid");
+            this.sprayJsonForm = registry.byId(this.id + "SprayJsonForm");
+            this.sprayJsonDestinationSelect = registry.byId(this.id + "SprayJsonDestinationSelect");
+            this.sprayJsonGrid = registry.byId(this.id + "SprayJsonGrid");
             this.sprayVariableForm = registry.byId(this.id + "SprayVariableForm");
             this.sprayVariableDestinationSelect = registry.byId(this.id + "SprayVariableDestination");
             this.sprayVariableGrid = registry.byId(this.id + "SprayVariableGrid");
@@ -387,6 +390,21 @@ define([
             });
         },
 
+        _onSprayJson: function(event) {
+            var context = this;
+            this._spraySelectedOneAtATime("SprayJsonDropDown", "SprayJsonForm", function (request, item) {
+                lang.mixin(request, {
+                    sourceRowPath: item.targetRowPath,
+                    isJSON: true
+                });
+                FileSpray.SprayVariable({
+                    request: request
+                }).then(function (response) {
+                    context._handleResponse("SprayResponse.wuid", response);
+                });
+            });
+        },
+
         _onSprayVariable: function (event) {
             var context = this;
             this._spraySelectedOneAtATime("SprayVariableDropDown", "SprayVariableForm", function (request, item) {
@@ -427,6 +445,9 @@ define([
             this.sprayXmlDestinationSelect.init({
                 SprayTargets: true
             });
+            this.sprayJsonDestinationSelect.init({
+                Groups: true
+            });
             this.sprayVariableDestinationSelect.init({
                 SprayTargets: true
             });
@@ -577,6 +598,24 @@ define([
                 }
             });
 
+            this.sprayJsonGrid.createGrid({
+                idProperty: "calculatedID",
+                columns: {
+                    targetName: editor({
+                        label: this.i18n.TargetName,
+                        width: 144,
+                        autoSave: true,
+                        editor: "text"
+                    }),
+                    targetRowPath: editor({
+                        label: this.i18n.RowPath,
+                        width: 72,
+                        autoSave: true,
+                        editor: "text"
+                    })
+                }
+            });
+
             this.sprayVariableGrid.createGrid({
                 idProperty: "calculatedID",
                 columns: {
@@ -622,6 +661,7 @@ define([
             registry.byId(this.id + "SprayFixedDropDown").set("disabled", !hasSelection);
             registry.byId(this.id + "SprayDelimitedDropDown").set("disabled", !hasSelection);
             registry.byId(this.id + "SprayXmlDropDown").set("disabled", !hasSelection);
+            registry.byId(this.id + "SprayJsonDropDown").set("disabled", !hasSelection);
             registry.byId(this.id + "SprayVariableDropDown").set("disabled", !hasSelection);
             registry.byId(this.id + "SprayBlobDropDown").set("disabled", !hasSelection);
 
@@ -632,13 +672,15 @@ define([
                     lang.mixin(item, lang.mixin({
                         targetName: item.displayName,
                         targetRecordLength: "",
-                        targetRowTag: context.i18n.tag
+                        targetRowTag: context.i18n.tag,
+                        targetRowPath: "/"
                     }, item));
                     data.push(item);
                 });
                 this.sprayFixedGrid.setData(data);
                 this.sprayDelimitedGrid.setData(data);
                 this.sprayXmlGrid.setData(data);
+                this.sprayJsonGrid.setData(data);
                 this.sprayVariableGrid.setData(data);
                 this.sprayBlobGrid.setData(data);
             }

esp/src/eclwatch/nls/hpcc.js

@@ -402,6 +402,7 @@ define({root:
     RetainSuperfileStructure: "Retain Superfile Structure",
     RetypePassword: "Retype Password",
     Rows: "Rows",
+    RowPath: "Row Path",
     RowTag: "Row Tag",
     RoxieCluster: "Roxie Cluster",
     Sample: "Sample",

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -145,6 +145,46 @@
                             </div>
                         </div>
                     </div>
+                    <div id="${id}SprayJsonDropDown" data-dojo-type="dijit.form.DropDownButton">
+                        <span>${i18n.JSON}</span>
+                        <div data-dojo-type="dijit.TooltipDialog">
+                            <div id="${id}SprayJsonForm" style="width: 530px;" data-dojo-type="dijit.form.Form">
+                                <div data-dojo-type="dijit.Fieldset">
+                                    <legend>${i18n.Target}</legend>
+                                    <div data-dojo-type="hpcc.TableContainer">
+                                        <input id="${id}SprayJsonDestinationSelect" title="${i18n.Group}:" style="width: 95%;" name="destGroup" data-dojo-type="TargetSelectWidget" />
+                                        <input title="${i18n.NamePrefix}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
+                                    </div>
+                                    <div id="${id}SprayJsonGrid" data-dojo-type="SelectionGridWidget">
+                                    </div>
+                                </div>
+                                <div data-dojo-type="dijit.Fieldset">
+                                    <legend>${i18n.Options}</legend>
+                                    <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
+                                        <select id="${id}jsonsourceFormat" title="${i18n.Format}:" name="sourceFormat" colspan="2" data-dojo-type="dijit.form.Select">
+                                            <option value="2">UTF-8</option>
+                                            <option value="3">UTF-8N</option>
+                                            <option value="4">UTF-16</option>
+                                            <option value="5">UTF-16LE</option>
+                                            <option value="6">UTF-16BE</option>
+                                            <option value="7">UTF-32</option>
+                                            <option value="8">UTF-32LE</option>
+                                            <option value="9">UTF-32BE</option>
+                                        </select>
+                                        <input id="${id}SprayJsonMaxRecordLength" title="${i18n.MaxRecordLength}:" value="8192" style="width: 95%;" name="sourceMaxRecordSize" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
+                                        <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
+                                    </div>
+                                </div>
+                                <div class="dijitDialogPaneActionBar">
+                                    <button data-dojo-attach-event="onClick:_onSprayJson" data-dojo-type="dijit.form.Button">${i18n.Spray}</button>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
                     <div id="${id}SprayVariableDropDown" data-dojo-type="dijit.form.DropDownButton">
                         <span>${i18n.Variable}</span>
                         <div data-dojo-type="dijit.TooltipDialog">

system/jlib/jptree.cpp

@@ -6587,12 +6587,12 @@ public:
                     readChild(tagName.str(), true);
                     break;
                 case '{':  //treat unnamed objects like we're in a noroot array
-                    readObject("__item__");
+                    readObject("__object__");
                     break;
                 case '[':  //treat unnamed arrays like we're in a noroot array
-                    iEvent->beginNode("__item__", curOffset);
+                    iEvent->beginNode("__array__", curOffset);
                     readArray("__item__");
-                    iEvent->endNode("__item__", 0, "", false, curOffset);
+                    iEvent->endNode("__array__", 0, "", false, curOffset);
                     break;
                 default:
                     expecting("{[ or \"");
@@ -6775,11 +6775,11 @@ public:
         }
     }
 
-    inline const char *arrayItemName()
+    inline const char *arrayItemName(const char *defaultName)
     {
         if (stack.ordinality()>1)
             return stateInfo->wnsTag;
-        return "__item__";
+        return defaultName;
     }
 
     bool arrayItem(offset_t offset)
@@ -6797,18 +6797,18 @@ public:
         case '{':
             state=objAttributes;
             readNext();
-            beginNode(arrayItemName(), offset, elementTypeObject);
+            beginNode(arrayItemName("__object__"), offset, elementTypeObject);
             break;
         case '[':
             state=valueStart;
             readNext();
-            beginNode(arrayItemName(), offset, elementTypeArray, true);
+            beginNode(arrayItemName("__array__"), offset, elementTypeArray, true);
             break;
         default:
             state=valueStart;
             ptElementType type = readValue(value.clear());
             readNext();
-            beginNode(arrayItemName(), offset, type, true);
+            beginNode(arrayItemName("__item__"), offset, type, true);
             stateInfo->tagText.swapWith(value);
             break;
         }