Merge pull request #6428 from afishbeck/jsonFileRead2

HPCC-10673 ECL support for reading JSON Files

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/thorhelper/commonext.cpp

@@ -133,6 +133,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     kindArray[TAKchildthroughnormalize] = "childthroughnormalize";
     kindArray[TAKcsvread] = "csvread";
     kindArray[TAKxmlread] = "xmlread";
+    kindArray[TAKjsonread] = "jsonread";
     kindArray[TAKlocalresultread] = "localresultread";
     kindArray[TAKlocalresultwrite] = "localresultwrite";
     kindArray[TAKcombine] = "combine";

common/thorhelper/thorcommon.cpp

@@ -704,6 +704,7 @@ extern const char * getActivityText(ThorActivityKind kind)
     case TAKchildthroughnormalize:  return "Normalize";
     case TAKcsvread:                return "Csv Read";
     case TAKxmlread:                return "Xml Read";
+    case TAKjsonread:               return "Json Read";
     case TAKlocalresultread:        return "Read Local Result";
     case TAKlocalresultwrite:       return "Local Result";
     case TAKcombine:                return "Combine";
@@ -812,6 +813,7 @@ extern bool isActivitySource(ThorActivityKind kind)
     case TAKchildgroupaggregate:
     case TAKcsvread:
     case TAKxmlread:
+    case TAKjsonread:
     case TAKlocalresultread:
     case TAKsimpleaction:
     case TAKlocalstreamread:

common/thorhelper/thorxmlread.cpp

@@ -1614,7 +1614,7 @@ class CXMLParse : public CInterface, implements IXMLParse
     StringAttr xpath;
     IXMLSelect *iXMLSelect;  // NOTE - not linked - creates circular links
     PTreeReaderOptions xmlOptions;
-    bool step, contentRequired;
+    bool step, contentRequired, isJson;
 
     class CXMLMaker : public CInterface, implements IPTreeMaker
     {
@@ -1910,12 +1910,12 @@ class CXMLParse : public CInterface, implements IXMLParse
 public:
     IMPLEMENT_IINTERFACE;
 
-    CXMLParse(const char *fName, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(fName); }
-    CXMLParse(IFile &ifile, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(ifile); }
-    CXMLParse(IFileIO &fileio, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(fileio); }
-    CXMLParse(ISimpleReadStream &stream, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(stream); }
-    CXMLParse(const void *buffer, unsigned bufLen, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(buffer, bufLen); }
-    CXMLParse(const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); }
+    CXMLParse(const char *fName, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true, bool _isJson=false) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step), isJson(_isJson) { init(); go(fName); }
+    CXMLParse(IFile &ifile, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true, bool _isJson=false) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step), isJson(_isJson) { init(); go(ifile); }
+    CXMLParse(IFileIO &fileio, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true, bool _isJson=false) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step), isJson(_isJson) { init(); go(fileio); }
+    CXMLParse(ISimpleReadStream &stream, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true, bool _isJson=false) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step), isJson(_isJson) { init(); go(stream); }
+    CXMLParse(const void *buffer, unsigned bufLen, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true, bool _isJson=false) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step), isJson(_isJson) { init(); go(buffer, bufLen); }
+    CXMLParse(const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true, bool _isJson=false) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step), isJson(_isJson) { init(); }
     ~CXMLParse()
     {
         ::Release(iXMLMaker);
@@ -1946,6 +1946,7 @@ public:
         Owned<IIOStream> stream = createIOStream(&fileio);
         go(*stream);
     }
+
     void go(ISimpleReadStream &stream)
     {
         if (contentRequired)
@@ -1953,8 +1954,13 @@ public:
             // only need marking stream if fetching xml text content.
             Owned<CMarkReadStream> markingStream = new CMarkReadStream(*LINK(&stream));
             iXMLMaker->setMarkingStream(*markingStream);
-            xmlReader = createPullXMLStreamReader(*markingStream, *iXMLMaker, xmlOptions);
+            if (isJson)
+                xmlReader = createPullJSONStreamReader(*markingStream, *iXMLMaker, xmlOptions);
+            else
+                xmlReader = createPullXMLStreamReader(*markingStream, *iXMLMaker, xmlOptions);
         }
+        else if (isJson)
+            xmlReader = createPullJSONStreamReader(stream, *iXMLMaker, xmlOptions);
         else
             xmlReader = createPullXMLStreamReader(stream, *iXMLMaker, xmlOptions);
         if (!step)
@@ -1971,7 +1977,10 @@ public:
             Owned<CMarkReadBase> markingStream = new CMarkRead(buffer, bufLen);
             iXMLMaker->setMarkingStream(*markingStream);
         }
-        xmlReader = createPullXMLBufferReader(buffer, bufLen, *iXMLMaker, xmlOptions);
+        if (isJson)
+            xmlReader = createPullJSONBufferReader(buffer, bufLen, *iXMLMaker, xmlOptions);
+        else
+            xmlReader = createPullXMLBufferReader(buffer, bufLen, *iXMLMaker, xmlOptions);
         if (!step)
         {
             xmlReader->load();
@@ -1986,7 +1995,10 @@ public:
             Owned<CMarkReadBase> markingStream = new CMarkRead(str, strlen(str));
             iXMLMaker->setMarkingStream(*markingStream);
         }
-        xmlReader = createPullXMLStringReader(str, *iXMLMaker, xmlOptions);
+        if (isJson)
+            xmlReader = createPullJSONStringReader(str, *iXMLMaker, xmlOptions);
+        else
+            xmlReader = createPullXMLStringReader(str, *iXMLMaker, xmlOptions);
         if (!step)
         {
             xmlReader->load();
@@ -2029,3 +2041,24 @@ IXMLParse *createXMLParseString(const char *string, const char *xpath, IXMLSelec
     return parser;
 }
 
+IXMLParse *createJSONParse(const char *filename, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions, bool contentRequired)
+{
+    return new CXMLParse(filename, xpath, iselect, xmlOptions, contentRequired, true, true);
+}
+IXMLParse *createJSONParse(ISimpleReadStream &stream, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions, bool contentRequired)
+{
+    return new CXMLParse(stream, xpath, iselect, xmlOptions, contentRequired, true, true);
+}
+
+IXMLParse *createJSONParse(const void *buffer, unsigned bufLen, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions, bool contentRequired)
+{
+    return new CXMLParse(buffer, bufLen, xpath, iselect, xmlOptions, contentRequired, true, true);
+}
+
+IXMLParse *createJSONParseString(const char *string, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions, bool contentRequired)
+{
+    CXMLParse *parser = new CXMLParse(xpath, iselect, xmlOptions, contentRequired, true, true);
+    parser->provideXML(string);
+    return parser;
+}
+

common/thorhelper/thorxmlread.hpp

@@ -217,6 +217,11 @@ thorhelper_decl IXMLParse *createXMLParseString(const char *str, const char *xpa
 thorhelper_decl size32_t createRowFromXml(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace);
 thorhelper_decl const void * createRowFromXml(IEngineRowAllocator * rowAllocator, size32_t len, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace);
 
+thorhelper_decl IXMLParse *createJSONParse(const char *filename, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions=ptr_none, bool contentRequired=true);
+thorhelper_decl IXMLParse *createJSONParse(ISimpleReadStream &stream, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions=ptr_none, bool contentRequired=true);
+thorhelper_decl IXMLParse *createJSONParse(const void *buffer, unsigned bufLen, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions=ptr_none, bool contentRequired=true);
+thorhelper_decl IXMLParse *createJSONParseString(const char *str, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions=ptr_none, bool contentRequired=true);
+
 thorhelper_decl size32_t createRowFromJson(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace);
 thorhelper_decl const void * createRowFromJson(IEngineRowAllocator * rowAllocator, size32_t len, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace);
 

ecl/eclagent/eclgraph.cpp

@@ -257,6 +257,7 @@ static IHThorActivity * createActivity(IAgentContext & agent, unsigned activityI
     case TAKcsvread:
         return createCsvReadActivity(agent, activityId, subgraphId, (IHThorCsvReadArg &)arg, kind);
     case TAKxmlread:
+    case TAKjsonread:
         return createXmlReadActivity(agent, activityId, subgraphId, (IHThorXmlReadArg &)arg, kind);
     case TAKlocalresultread:
         return createLocalResultReadActivity(agent, activityId, subgraphId, (IHThorLocalResultReadArg &)arg, kind, node->getPropInt("att[@name='_graphId']/@value"));

ecl/hql/hqlattr.cpp

@@ -515,6 +515,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_joined:
     case no_any:
     case no_xml:
+    case no_json:
     case no_distributer:
     case no_keyed:
     case no_sortpartition:

ecl/hql/hqlexpr.cpp

@@ -1098,6 +1098,7 @@ const char *getOpString(node_operator op)
     case no_flat: return "FLAT";
     case no_csv: return "CSV";
     case no_xml: return "XML";
+    case no_json: return "JSON";
 
     case no_when: return "WHEN";
     case no_priority: return "PRIORITY";

ecl/hql/hqlexpr.hpp

@@ -528,6 +528,7 @@ enum _node_operator {
         no_pat_x_before_y,              // look ahead assertion
         no_pat_x_after_y,               // look behind assertion
         no_xml,
+        no_json,
         no_compound_fetch,
         no_pat_index,
         no_pat_beginpattern,            // marks the start of a global/child pattern.

ecl/hql/hqlgram.y

@@ -9711,6 +9711,23 @@ mode
                                 args.add(*createConstant("xml"), 0);
                             $$.setExpr(createValue(no_xml, makeNullType(), args));
                         }
+    | JSON_TOKEN         {   $$.setExpr(createValue(no_json));    }
+    | JSON_TOKEN '(' xmlOptions ')'
+                        {
+                            HqlExprArray args;
+                            $3.unwindCommaList(args);
+
+                            //Create expression in a form that is backward compatible
+                            IHqlExpression * name = queryAttribute(rowAtom, args);
+                            if (name)
+                            {
+                                args.add(*LINK(name->queryChild(0)), 0);
+                                args.zap(*name);
+                            }
+                            else
+                                args.add(*createConstant("json"), 0);
+                            $$.setExpr(createValue(no_json, makeNullType(), args));
+                        }
     | pipe
     ;
 

ecl/hql/hqlopt.cpp

@@ -1674,7 +1674,7 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateCompound(IHqlExpression * tran
         return NULL;
     IHqlExpression * tableExpr = queryRoot(transformed);
     node_operator modeOp = queryTableMode(tableExpr);
-    if (modeOp == no_csv || modeOp == no_xml)
+    if (modeOp == no_csv || modeOp == no_xml || modeOp == no_json)
         return NULL;
 
     if (isLimitedDataset(child) && !isSimpleCountExistsAggregate(transformed, true, false))

ecl/hql/hqltrans.cpp

@@ -2559,6 +2559,7 @@ bool onlyTransformOnce(IHqlExpression * expr)
     case no_thor:
     case no_csv:
     case no_xml:
+    case no_json:
     case no_list:
         return (expr->numChildren() == 0);
     case no_select:

ecl/hqlcpp/hqlgraph.cpp

@@ -631,6 +631,7 @@ const char * LogicalGraphCreator::getActivityText(IHqlExpression * expr, StringB
             {
             case no_csv:
             case no_xml:
+            case no_json:
                 temp.append(getOpString(expr->getOperator())).append(" ");
                 break;
             }

ecl/hqlcpp/hqlhtcpp.cpp

@@ -16425,12 +16425,15 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
 
 void HqlCppTranslator::doBuildXmlReadMember(ActivityInstance & instance, IHqlExpression * expr, const char * functionName, bool & usesContents)
 {
+    IHqlExpression * mode = expr->queryChild(2);
+    node_operator modeType = mode->getOperator();
+
     StringBuffer s, xmlInstanceName;
     usesContents = false;
     if (isValidXmlRecord(expr->queryRecord()))
     {
         StringBuffer xmlFactoryName;
-        getUniqueId(xmlInstanceName.append("xml"));
+        getUniqueId(xmlInstanceName.append((modeType==no_json) ? "json" : "xml"));
 
         buildXmlReadTransform(expr, xmlFactoryName, usesContents);
         instance.classctx.addQuoted(s.clear().append("Owned<IXmlToRowTransformer> ").append(xmlInstanceName).append(";"));

ecl/hqlcpp/hqlsource.cpp

@@ -1551,6 +1551,7 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
                 leftCursor = translator.bindCsvTableCursor(subctx, dataset, "Left", no_left, querySelSeq(expr), true, queryCsvTableEncoding(tableExpr));
                 break;
             case no_xml:
+            case no_json:
                 leftCursor = translator.bindXmlTableCursor(subctx, dataset, "xmlLeft", no_left, querySelSeq(expr), true);
                 break;
             default:
@@ -1807,7 +1808,7 @@ ABoundActivity * SourceBuilder::buildActivity(BuildCtx & ctx, IHqlExpression * e
     StringBuffer graphLabel;
     graphLabel.append(getActivityText(activityKind));
 
-    if ((activityKind == TAKdiskread) || (activityKind == TAKcsvread) || (activityKind == TAKxmlread))
+    if ((activityKind == TAKdiskread) || (activityKind == TAKcsvread) || (activityKind == TAKxmlread) || (activityKind == TAKjsonread))
     {
         graphLabel.clear();
         if (expr != tableExpr)
@@ -6988,7 +6989,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityXmlRead(BuildCtx & ctx, IHqlEx
     node_operator modeType = mode->getOperator();
     StringBuffer s;
 
-    Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKxmlread, expr, "XmlRead");
+    ThorActivityKind kind = (modeType == no_json) ? TAKjsonread : TAKxmlread;
+    Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, kind, expr, "XmlRead");
 
     buildActivityFramework(instance);
 
@@ -7048,6 +7050,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityTable(BuildCtx & ctx, IHqlExpr
     case no_csv:
         return doBuildActivityDiskRead(ctx, expr);
     case no_xml:
+    case no_json:
         return doBuildActivityXmlRead(ctx, expr);
     default:
         UNIMPLEMENTED;
@@ -7126,6 +7129,7 @@ void FetchBuilder::buildMembers(IHqlExpression * expr)
             break;
         }
     case no_xml:
+    case no_json:
         {
             // virtual const char * getXmlIteratorPath()
             translator.doBuildVarStringFunction(instance->classctx, "getXmlIteratorPath", queryRealChild(tableExpr->queryChild(2), 0));
@@ -7166,6 +7170,7 @@ void FetchBuilder::buildTransform(IHqlExpression * expr)
         transformCtx.addQuotedLiteral("unsigned char * right = (unsigned char *)_right;");
         break;
     case no_xml:
+    case no_json:
         transformCtx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, IColumnProvider * xmlLeft, const void * _right, unsigned __int64 _fpos)");
         transformCtx.addQuotedLiteral("unsigned char * right = (unsigned char *)_right;");
         break;

ecl/hqlcpp/hqlttcpp.ipp

@@ -242,7 +242,7 @@ public:
         return (splitCount == 1);
     }
     inline void noteUsage() { if (splitCount < 10) splitCount++; }
-    inline bool isBinary() const { return mode != no_csv && mode != no_xml; }
+    inline bool isBinary() const { return mode != no_csv && mode != no_xml && mode != no_json; }
     inline bool hasAnyLimit() const { return isLimited || hasChoosen; }
     void reset();
 

ecl/hthor/hthor.cpp

@@ -8981,7 +8981,10 @@ bool CHThorXmlReadActivity::openNext()
             inputfileiostream.setown(createIOStream(inputfileio));
 
         OwnedRoxieString xmlIterator(helper.getXmlIteratorPath());
-        xmlParser.setown(createXMLParse(*inputfileiostream, xmlIterator, *this, (0 != (TDRxmlnoroot & helper.getFlags()))?ptr_noRoot:ptr_none, (helper.getFlags() & TDRusexmlcontents) != 0));
+        if (kind==TAKjsonread)
+            xmlParser.setown(createJSONParse(*inputfileiostream, xmlIterator, *this, (0 != (TDRxmlnoroot & helper.getFlags()))?ptr_noRoot:ptr_none, (helper.getFlags() & TDRusexmlcontents) != 0));
+        else
+            xmlParser.setown(createXMLParse(*inputfileiostream, xmlIterator, *this, (0 != (TDRxmlnoroot & helper.getFlags()))?ptr_noRoot:ptr_none, (helper.getFlags() & TDRusexmlcontents) != 0));
         return true;
     }
     return false;

roxie/ccd/ccdquery.cpp

@@ -526,6 +526,7 @@ protected:
             return createRoxieServerDegroupActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKcsvread:
         case TAKxmlread:
+        case TAKjsonread:
         case TAKdiskread:
         {       
             if (node.getPropBool("att[@name='_isSpill']/@value", false) || node.getPropBool("att[@name='_isSpillGlobal']/@value", false))
@@ -1681,6 +1682,7 @@ class CSlaveQueryFactory : public CQueryFactory
         case TAKcsvread:
         case TAKxmlread:
         case TAKdiskread:
+        case TAKjsonread:
             if (node.getPropBool("att[@name='_isSpill']/@value", false) || node.getPropBool("att[@name='_isSpillGlobal']/@value", false))
                 return;
             break;
@@ -1731,6 +1733,7 @@ class CSlaveQueryFactory : public CQueryFactory
                     newAct = createRoxieCsvReadActivityFactory(node, subgraphId, *this, helperFactory);
                     break;
                 case TAKxmlread:
+                case TAKjsonread:
                     newAct = createRoxieXmlReadActivityFactory(node, subgraphId, *this, helperFactory);
                     break;
                 case TAKdisknormalize:

roxie/ccd/ccdserver.cpp

@@ -20698,7 +20698,10 @@ public:
             rowTransformer.set(readHelper->queryTransformer());
             assertex(reader != NULL);
             OwnedRoxieString xmlIterator(readHelper->getXmlIteratorPath());
-            xmlParser.setown(createXMLParse(*reader->querySimpleStream(), xmlIterator, *this, (0 != (TDRxmlnoroot & readHelper->getFlags()))?ptr_noRoot:ptr_none, (readHelper->getFlags() & TDRusexmlcontents) != 0));
+            if (factory->getKind()==TAKjsonread)
+                xmlParser.setown(createJSONParse(*reader->querySimpleStream(), xmlIterator, *this, (0 != (TDRxmlnoroot & readHelper->getFlags()))?ptr_noRoot:ptr_none, (readHelper->getFlags() & TDRusexmlcontents) != 0));
+            else
+                xmlParser.setown(createXMLParse(*reader->querySimpleStream(), xmlIterator, *this, (0 != (TDRxmlnoroot & readHelper->getFlags()))?ptr_noRoot:ptr_none, (readHelper->getFlags() & TDRusexmlcontents) != 0));
         }
     }
 
@@ -21395,6 +21398,7 @@ public:
             return new CRoxieServerCsvReadActivity(this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager,
                                                    quotes, separators, terminators, escapes);
         case TAKxmlread:
+        case TAKjsonread:
             return new CRoxieServerXmlReadActivity(this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager);
         case TAKdiskread:
             return new CRoxieServerDiskReadActivity(this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager);

rtl/include/eclhelper.hpp

@@ -920,6 +920,7 @@ enum ThorActivityKind
     TAKunknowndenormalizegroup3,
     TAKlastdenormalizegroup,
     TAKjsonwrite,
+    TAKjsonread,
 
     TAKlast
 };

system/jlib/jptree.cpp

@@ -6849,8 +6849,10 @@ public:
     }
     bool rootNext()
     {
-        if (!noRoot || !checkReadNext() || !checkSkipWS())
+        if (!noRoot)
             return false;
+        if (!checkReadNext() || !checkSkipWS())
+            return true;
         if (','!=nextChar)
             expecting(",");
         return true;
@@ -6901,7 +6903,7 @@ public:
 
     virtual bool next()
     {
-        skipWS();
+        checkSkipWS();
         switch (state)
         {
             case headerStart:

testing/regress/ecl/jsonout.ecl

@@ -17,14 +17,14 @@
 
 phoneRecord :=
             RECORD
-string5         areaCode{xpath('areaCode')};
-udecimal12      number{xpath('number')};
+string5         areaCode{xpath('@areaCode')};
+udecimal12      number{xpath('@number')};
             END;
 
 contactrecord :=
             RECORD
 phoneRecord     phone;
-boolean         hasemail{xpath('hasEmail')};
+boolean         hasemail{xpath('@hasEmail')};
                 ifblock(self.hasemail)
 string              email;
                 end;
@@ -56,7 +56,15 @@ namesTable := dataset([
         {'Halliday','Abigail','09876',654321,false,'','',false,[{'The cat in the hat','Suess'},{'Wolly the sheep',''}], ['Red','Yellow']}
         ], personRecord);
 
-output(namesTable,,'REGRESS::TEMP::output.json',overwrite, json('jrow'));
+output(namesTable,,'REGRESS::TEMP::output_object_namedArray.json',overwrite, json);
+readObjectNamedArray := dataset(DYNAMIC('REGRESS::TEMP::output_object_namedArray.json'), personRecord, json('*/Row'));
+output(readObjectNamedArray, named('ObjectNamedArray'));
 
-//will read back data and add workunit output when json read is implemented
+output(namesTable,,'REGRESS::TEMP::output_array.json',overwrite, json('', heading('[', ']')));
+readArrayOfRows := dataset(DYNAMIC('REGRESS::TEMP::output_array.json'), personRecord, json('*/*'));
+output(readArrayOfRows, named('ArrayOfRows'));
+
+output(namesTable,,'REGRESS::TEMP::output_noroot.json',overwrite, json('', heading('','')));
+readNoRootRows := dataset(DYNAMIC('REGRESS::TEMP::output_noroot.json'), personRecord, json('*', NOROOT));
+output(readNoRootRows, named('noRootRows'));
 

thorlcr/activities/xmlread/thxmlreadslave.cpp

@@ -82,7 +82,10 @@ class CXmlReadSlaveActivity : public CDiskReadSlaveActivityBase, public CThorDat
             }
             inputIOstream.setown(createBufferedIOStream(stream));
             OwnedRoxieString xmlIterator(activity.helper->getXmlIteratorPath());
-            xmlParser.setown(createXMLParse(*inputIOstream.get(), xmlIterator, *this, (0 != (TDRxmlnoroot & activity.helper->getFlags()))?ptr_noRoot:ptr_none, 0 != (TDRusexmlcontents & activity.helper->getFlags())));
+            if (activity.queryContainer().getKind()==TAKjsonread)
+                xmlParser.setown(createJSONParse(*inputIOstream.get(), xmlIterator, *this, (0 != (TDRxmlnoroot & activity.helper->getFlags()))?ptr_noRoot:ptr_none, 0 != (TDRusexmlcontents & activity.helper->getFlags())));
+            else
+                xmlParser.setown(createXMLParse(*inputIOstream.get(), xmlIterator, *this, (0 != (TDRxmlnoroot & activity.helper->getFlags()))?ptr_noRoot:ptr_none, 0 != (TDRusexmlcontents & activity.helper->getFlags())));
         }
         virtual void close(CRC32 &fileCRC)
         {

thorlcr/graph/thgraph.cpp

@@ -305,6 +305,7 @@ bool isDiskInput(ThorActivityKind kind)
     {
         case TAKcsvread:
         case TAKxmlread:
+        case TAKjsonread:
         case TAKdiskread:
         case TAKdisknormalize:
         case TAKdiskaggregate:
@@ -1013,6 +1014,7 @@ bool isGlobalActivity(CGraphElementBase &container)
         case TAKindexread:
         case TAKindexnormalize:
         case TAKxmlread:
+        case TAKjsonread:
         case TAKdiskexists:
         case TAKindexexists:
         case TAKchildexists:

thorlcr/master/thactivitymaster.cpp

@@ -340,6 +340,7 @@ public:
                 ret = createTopNActivityMaster(this);
                 break;
             case TAKxmlread:
+            case TAKjsonread:
                 ret = createXmlReadActivityMaster(this);
                 break;
             case TAKxmlwrite:

thorlcr/slave/slave.cpp

@@ -659,6 +659,7 @@ public:
                 ret = createXmlParseSlave(this);
                 break;
             case TAKxmlread:
+            case TAKjsonread:
                 ret = createXmlReadSlave(this);
                 break;
             case TAKxmlwrite: