Merge pull request #3820 from afishbeck/json_ptree

HPCC-8435 Implement JSON Pull Parser and PTree creation

Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorpipe.cpp

@@ -224,7 +224,7 @@ public:
         bool noRoot = (pipeFlags & TPFreadnoroot) != 0;
         bool useContents = (pipeFlags & TPFreadusexmlcontents) != 0;
         if (in)
-            xmlParser.setown(createXMLParse(*in, iteratorPath, *this, noRoot?xr_noRoot:xr_none, useContents));
+            xmlParser.setown(createXMLParse(*in, iteratorPath, *this, noRoot?ptr_noRoot:ptr_none, useContents));
         else
             xmlParser.clear();
     }

common/thorhelper/thorsoapcall.cpp

@@ -580,7 +580,7 @@ interface IWSCAsyncFor: public IInterface
 };
 
 class CWSCHelper;
-IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, CommonXmlWriter &_xmlWriter, ConstPointerArray &_inputRows, XmlReaderOptions _options);
+IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, CommonXmlWriter &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options);
 
 //=================================================================================================
 
@@ -1209,17 +1209,17 @@ void CWSCHelperThread::createXmlSoapQuery(CommonXmlWriter &xmlWriter, ConstPoint
 void CWSCHelperThread::processQuery(ConstPointerArray &inputRows)
 {
     unsigned xmlWriteFlags = 0;
-    unsigned xmlReadFlags = xr_ignoreNameSpaces; 
+    unsigned xmlReadFlags = ptr_ignoreNameSpaces;
     if (master->flags & SOAPFtrim)
         xmlWriteFlags |= XWFtrim;
     if ((master->flags & SOAPFpreserveSpace) == 0)
-        xmlReadFlags |= xr_ignoreWhiteSpace;
+        xmlReadFlags |= ptr_ignoreWhiteSpace;
         XMLWriterType xmlType = !(master->flags & SOAPFencoding) ? WTStandard : WTEncodingData64; 
     CommonXmlWriter *xmlWriter = CreateCommonXmlWriter(xmlWriteFlags, 0, NULL, xmlType);
     if (master->wscType == STsoap)
         createXmlSoapQuery(*xmlWriter, inputRows);
 
-    Owned<IWSCAsyncFor> casyncfor = createWSCAsyncFor(master, *xmlWriter, inputRows, (XmlReaderOptions) xmlReadFlags);
+    Owned<IWSCAsyncFor> casyncfor = createWSCAsyncFor(master, *xmlWriter, inputRows, (PTreeReaderOptions) xmlReadFlags);
     casyncfor->For(master->numUrls, master->numUrlThreads,false,true); // shuffle URLS for poormans load balance
     delete xmlWriter;
 }
@@ -1359,7 +1359,7 @@ private:
     CriticalSection processExceptionCrit;
     StringBuffer responsePath;
     Owned<CSocketDataProvider> dataProvider;
-    XmlReaderOptions options;
+    PTreeReaderOptions options;
     unsigned remainingMS;
 
     inline void checkRoxieAbortMonitor(IRoxieAbortMonitor * roxieAbortMonitor)
@@ -1709,7 +1709,7 @@ private:
     }
 
 public:
-    CWSCAsyncFor(CWSCHelper * _master, CommonXmlWriter &_xmlWriter, ConstPointerArray &_inputRows, XmlReaderOptions _options): xmlWriter(_xmlWriter), inputRows(_inputRows), options(_options)
+    CWSCAsyncFor(CWSCHelper * _master, CommonXmlWriter &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options): xmlWriter(_xmlWriter), inputRows(_inputRows), options(_options)
     {
         master = _master;
         outputAllocator = master->queryOutputAllocator();
@@ -1923,7 +1923,7 @@ public:
     inline virtual IEngineRowAllocator * getOutputAllocator() { return outputAllocator; }
 };
 
-IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, CommonXmlWriter &_xmlWriter, ConstPointerArray &_inputRows, XmlReaderOptions _options)
+IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, CommonXmlWriter &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options)
 {
     return new CWSCAsyncFor(_master, _xmlWriter, _inputRows, _options);
 }

common/thorhelper/thorxmlread.cpp

@@ -647,7 +647,7 @@ IDataVal & CXmlToRawTransformer::transformTree(IDataVal & result, IPropertyTree
 
 size32_t createRowFromXml(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace)
 {
-    Owned<IPropertyTree> root = createPTreeFromXMLString(size, utf8, ipt_none, stripWhitespace ? xr_ignoreWhiteSpace : xr_none);
+    Owned<IPropertyTree> root = createPTreeFromXMLString(size, utf8, ipt_none, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
     if (!root)
     {
         throwError(THORCERR_InvalidXmlFromXml);
@@ -695,7 +695,7 @@ IDataVal & CCsvToRawTransformer::transform(IDataVal & result, size32_t len, cons
 
 //=====================================================================================================
 
-extern thorhelper_decl IXmlToRawTransformer * createXmlRawTransformer(IXmlToRowTransformer * xmlTransformer, XmlReaderOptions xmlReadFlags)
+extern thorhelper_decl IXmlToRawTransformer * createXmlRawTransformer(IXmlToRowTransformer * xmlTransformer, PTreeReaderOptions xmlReadFlags)
 {
     if (xmlTransformer)
         return new CXmlToRawTransformer(*xmlTransformer, xmlReadFlags);
@@ -1605,10 +1605,10 @@ void CColumnIterator::setCurrent()
 
 class CXMLParse : public CInterface, implements IXMLParse
 {
-    IPullXMLReader *xmlReader;
+    IPullPTreeReader *xmlReader;
     StringAttr xpath;
     IXMLSelect *iXMLSelect;  // NOTE - not linked - creates circular links
-    XmlReaderOptions xmlOptions;
+    PTreeReaderOptions xmlOptions;
     bool step, contentRequired;
 
     class CXMLMaker : public CInterface, implements IPTreeMaker
@@ -1905,12 +1905,12 @@ class CXMLParse : public CInterface, implements IXMLParse
 public:
     IMPLEMENT_IINTERFACE;
 
-    CXMLParse(const char *fName, const char *_xpath, IXMLSelect &_iXMLSelect, XmlReaderOptions _xmlOptions=xr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(fName); }
-    CXMLParse(IFile &ifile, const char *_xpath, IXMLSelect &_iXMLSelect, XmlReaderOptions _xmlOptions=xr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(ifile); }
-    CXMLParse(IFileIO &fileio, const char *_xpath, IXMLSelect &_iXMLSelect, XmlReaderOptions _xmlOptions=xr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(fileio); }
-    CXMLParse(ISimpleReadStream &stream, const char *_xpath, IXMLSelect &_iXMLSelect, XmlReaderOptions _xmlOptions=xr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(stream); }
-    CXMLParse(const void *buffer, unsigned bufLen, const char *_xpath, IXMLSelect &_iXMLSelect, XmlReaderOptions _xmlOptions=xr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(buffer, bufLen); }
-    CXMLParse(const char *_xpath, IXMLSelect &_iXMLSelect, XmlReaderOptions _xmlOptions=xr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); }
+    CXMLParse(const char *fName, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(fName); }
+    CXMLParse(IFile &ifile, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(ifile); }
+    CXMLParse(IFileIO &fileio, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(fileio); }
+    CXMLParse(ISimpleReadStream &stream, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(stream); }
+    CXMLParse(const void *buffer, unsigned bufLen, const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); go(buffer, bufLen); }
+    CXMLParse(const char *_xpath, IXMLSelect &_iXMLSelect, PTreeReaderOptions _xmlOptions=ptr_none, bool _contentRequired=true, bool _step=true) : xpath(_xpath), iXMLSelect(&_iXMLSelect), xmlOptions(_xmlOptions), contentRequired(_contentRequired), step(_step) { init(); }
     ~CXMLParse()
     {
         ::Release(iXMLMaker);
@@ -1919,7 +1919,7 @@ public:
     void init()
     {
         xmlReader = NULL;
-        bool ignoreNameSpaces = 0 != ((unsigned)xmlOptions & (unsigned)xr_ignoreNameSpaces);
+        bool ignoreNameSpaces = 0 != ((unsigned)xmlOptions & (unsigned)ptr_ignoreNameSpaces);
         iXMLMaker = new CXMLMaker(xpath, *iXMLSelect, contentRequired, ignoreNameSpaces);
         iXMLMaker->init();
     }
@@ -2003,21 +2003,21 @@ public:
     }
 };
 
-IXMLParse *createXMLParse(const char *filename, const char *xpath, IXMLSelect &iselect, XmlReaderOptions xmlOptions, bool contentRequired)
+IXMLParse *createXMLParse(const char *filename, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions, bool contentRequired)
 {
     return new CXMLParse(filename, xpath, iselect, xmlOptions, contentRequired);
 }
-IXMLParse *createXMLParse(ISimpleReadStream &stream, const char *xpath, IXMLSelect &iselect, XmlReaderOptions xmlOptions, bool contentRequired)
+IXMLParse *createXMLParse(ISimpleReadStream &stream, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions, bool contentRequired)
 {
     return new CXMLParse(stream, xpath, iselect, xmlOptions, contentRequired);
 }
 
-IXMLParse *createXMLParse(const void *buffer, unsigned bufLen, const char *xpath, IXMLSelect &iselect, XmlReaderOptions xmlOptions, bool contentRequired)
+IXMLParse *createXMLParse(const void *buffer, unsigned bufLen, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions, bool contentRequired)
 {
     return new CXMLParse(buffer, bufLen, xpath, iselect, xmlOptions, contentRequired);
 }
 
-IXMLParse *createXMLParseString(const char *string, const char *xpath, IXMLSelect &iselect, XmlReaderOptions xmlOptions, bool contentRequired)
+IXMLParse *createXMLParseString(const char *string, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions, bool contentRequired)
 {
     CXMLParse *parser = new CXMLParse(xpath, iselect, xmlOptions, contentRequired);
     parser->provideXML(string);

common/thorhelper/thorxmlread.hpp

@@ -136,7 +136,7 @@ public:
 class thorhelper_decl CXmlToRawTransformer : public CInterface, implements IXmlToRawTransformer
 {
 public:
-    CXmlToRawTransformer(IXmlToRowTransformer & _rowTransformer, XmlReaderOptions _xmlReadFlags)
+    CXmlToRawTransformer(IXmlToRowTransformer & _rowTransformer, PTreeReaderOptions _xmlReadFlags)
     : rowTransformer(&_rowTransformer), xmlReadFlags(_xmlReadFlags)
     {
     }
@@ -147,7 +147,7 @@ public:
 
 protected:
     Linked<IXmlToRowTransformer> rowTransformer;
-    XmlReaderOptions xmlReadFlags;
+    PTreeReaderOptions xmlReadFlags;
 };
 
 class thorhelper_decl CCsvToRawTransformer : public CInterface, implements ICsvToRawTransformer
@@ -191,7 +191,7 @@ protected:
     CSVColumnProvider csvSplitter;
 };
 #endif
-extern thorhelper_decl IXmlToRawTransformer * createXmlRawTransformer(IXmlToRowTransformer * xmlTransformer, XmlReaderOptions xmlReadFlags=xr_ignoreWhiteSpace);
+extern thorhelper_decl IXmlToRawTransformer * createXmlRawTransformer(IXmlToRowTransformer * xmlTransformer, PTreeReaderOptions xmlReadFlags=ptr_ignoreWhiteSpace);
 extern thorhelper_decl ICsvToRawTransformer * createCsvRawTransformer(ICsvToRowTransformer * csvTransformer);
 
 
@@ -209,10 +209,10 @@ interface IXMLParse : extends IInterface
     virtual bool next() = 0;
     virtual void reset() = 0;
 };
-thorhelper_decl IXMLParse *createXMLParse(const char *filename, const char *xpath, IXMLSelect &iselect, XmlReaderOptions xmlOptions=xr_none, bool contentRequired=true);
-thorhelper_decl IXMLParse *createXMLParse(ISimpleReadStream &stream, const char *xpath, IXMLSelect &iselect, XmlReaderOptions xmlOptions=xr_none, bool contentRequired=true);
-thorhelper_decl IXMLParse *createXMLParse(const void *buffer, unsigned bufLen, const char *xpath, IXMLSelect &iselect, XmlReaderOptions xmlOptions=xr_none, bool contentRequired=true);
-thorhelper_decl IXMLParse *createXMLParseString(const char *str, const char *xpath, IXMLSelect &iselect, XmlReaderOptions xmlOptions=xr_none, bool contentRequired=true);
+thorhelper_decl IXMLParse *createXMLParse(const char *filename, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions=ptr_none, bool contentRequired=true);
+thorhelper_decl IXMLParse *createXMLParse(ISimpleReadStream &stream, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions=ptr_none, bool contentRequired=true);
+thorhelper_decl IXMLParse *createXMLParse(const void *buffer, unsigned bufLen, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions=ptr_none, bool contentRequired=true);
+thorhelper_decl IXMLParse *createXMLParseString(const char *str, const char *xpath, IXMLSelect &iselect, PTreeReaderOptions xmlOptions=ptr_none, bool contentRequired=true);
 
 thorhelper_decl size32_t createRowFromXml(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace);
 thorhelper_decl const void * createRowFromXml(IEngineRowAllocator * rowAllocator, size32_t len, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace);

common/wuwebview/wuwebview.cpp

@@ -86,7 +86,7 @@ public:
     void appendDatasetsFromXML(const char *xml)
     {
         assertex(!finalized);
-        Owned<IPullXMLReader> reader = createPullXMLStringReader(xml, *this);
+        Owned<IPullPTreeReader> reader = createPullXMLStringReader(xml, *this);
         reader->load();
     }
 
@@ -735,7 +735,7 @@ void WuWebView::addInputsFromPTree(IPropertyTree *pt)
 
 void WuWebView::addInputsFromXml(const char *xml)
 {
-    Owned<IPropertyTree> pt = createPTreeFromXMLString(xml, ipt_none, (XmlReaderOptions)(xr_ignoreWhiteSpace|xr_ignoreNameSpaces));
+    Owned<IPropertyTree> pt = createPTreeFromXMLString(xml, ipt_none, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces));
     addInputsFromPTree(pt.get());
 }
 

dali/base/dasds.cpp

@@ -4621,7 +4621,7 @@ IPropertyTree *loadStore(const char *storeFilename, IPTreeMaker *iMaker, unsigne
         Owned<IFileIOStream> fstream = createIOStream(iFileIOStore);
         Owned<ICrcIOStream> crcPipeStream = createCrcPipeStream(fstream);
         Owned<IIOStream> ios = createBufferedIOStream(crcPipeStream);
-        root.setown((CServerRemoteTree *) createPTree(*ios, ipt_none, xr_ignoreWhiteSpace, iMaker));
+        root.setown((CServerRemoteTree *) createPTree(*ios, ipt_none, ptr_ignoreWhiteSpace, iMaker));
         ios.clear();
         unsigned crc = crcPipeStream->queryCrc();
 
@@ -8760,7 +8760,7 @@ bool applyXmlDeltas(IPropertyTree &root, IIOStream &stream, bool stopOnError)
         }
     } deltaProcessor(root, stopOnError);
 
-    Owned<IPullXMLReader> xmlReader = createPullXMLStreamReader(stream, deltaProcessor, (XmlReaderOptions)((unsigned)xr_ignoreWhiteSpace+(unsigned)xr_noRoot), false);
+    Owned<IPullPTreeReader> xmlReader = createPullXMLStreamReader(stream, deltaProcessor, (PTreeReaderOptions)((unsigned)ptr_ignoreWhiteSpace+(unsigned)ptr_noRoot), false);
     try
     {
         xmlReader->load();

dali/daliadmin/daliadmin.cpp

@@ -1949,8 +1949,8 @@ struct CTreeItem : public CInterface
 
 class CXMLSizesParser : public CInterface
 {
-    Owned<IPullXMLReader> xmlReader;
-    XmlReaderOptions xmlOptions;
+    Owned<IPullPTreeReader> xmlReader;
+    PTreeReaderOptions xmlOptions;
     double pc;
 
     class CParse : public CInterface, implements IPTreeNotifyEvent
@@ -2059,7 +2059,7 @@ class CXMLSizesParser : public CInterface
 public:
     IMPLEMENT_IINTERFACE;
 
-    CXMLSizesParser(const char *fName, XmlReaderOptions _xmlOptions=xr_none, double _pc=1.0) : xmlOptions(_xmlOptions), pc(_pc) { go(fName); }
+    CXMLSizesParser(const char *fName, PTreeReaderOptions _xmlOptions=ptr_none, double _pc=1.0) : xmlOptions(_xmlOptions), pc(_pc) { go(fName); }
     ~CXMLSizesParser() { ::Release(parser); }
 
     void go(const char *fName)
@@ -2100,7 +2100,7 @@ static void xmlSize(const char *filename, double pc)
             OUTLOG("File '%s' not found", filename);
         else
         {
-            Owned<CXMLSizesParser> parser = new CXMLSizesParser((filename&&*filename)?filename:"dalisds.xml", xr_none, pc);
+            Owned<CXMLSizesParser> parser = new CXMLSizesParser((filename&&*filename)?filename:"dalisds.xml", ptr_none, pc);
             while (parser->next())
                 ;
             parser->printResultTree();

ecl/eclagent/eclagent.cpp

@@ -383,7 +383,7 @@ public:
 
             try
             {
-                queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (XmlReaderOptions)(xr_ignoreWhiteSpace|xr_ignoreNameSpaces)));
+                queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces)));
             }
             catch (IException *E)
             {

ecl/hthor/hthor.cpp

@@ -7020,7 +7020,7 @@ const void * CHThorXmlParseActivity::nextInGroup()
         size32_t srchLen;
         helper.getSearchText(srchLen, srchStr, in);
         OwnedRoxieString xmlIteratorPath(helper.getXmlIteratorPath());
-        xmlParser.setown(createXMLParse(srchStr, srchLen, xmlIteratorPath, *this, xr_noRoot, helper.requiresContents()));
+        xmlParser.setown(createXMLParse(srchStr, srchLen, xmlIteratorPath, *this, ptr_noRoot, helper.requiresContents()));
     }
 }
 
@@ -8787,7 +8787,7 @@ bool CHThorXmlReadActivity::openNext()
             inputfileiostream.setown(createIOStream(inputfileio));
 
         OwnedRoxieString xmlIterator(helper.getXmlIteratorPath());
-        xmlParser.setown(createXMLParse(*inputfileiostream, xmlIterator, *this, (0 != (TDRxmlnoroot & helper.getFlags()))?xr_noRoot:xr_none, (helper.getFlags() & TDRusexmlcontents) != 0));
+        xmlParser.setown(createXMLParse(*inputfileiostream, xmlIterator, *this, (0 != (TDRxmlnoroot & helper.getFlags()))?ptr_noRoot:ptr_none, (helper.getFlags() & TDRusexmlcontents) != 0));
         return true;
     }
     return false;

esp/bindings/http/platform/httptransport.cpp

@@ -185,7 +185,7 @@ public:
         m_message = message;
         m_pStart = (char*) message;
 
-        Owned<IPullXMLReader> reader = createPullXMLStringReader(m_message, *this, xr_ignoreNameSpaces);
+        Owned<IPullPTreeReader> reader = createPullXMLStringReader(m_message, *this, ptr_ignoreNameSpaces);
         while(m_readNext && reader->next())
         {
             if (m_foundPassword)

esp/clients/LoggingClient/LogThread.cpp

@@ -326,7 +326,7 @@ bool CLogThread::queueLog(IEspContext & context,const char* serviceName,int Reco
 {
     StringBuffer dataStr;
     serializeRequest(context,logInfo,dataStr);
-    Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(dataStr.str(), ipt_none, xr_none);
+    Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(dataStr.str(), ipt_none, ptr_none);
     return queueLog(context,serviceName,RecordsReturned,LogArray, *pLogTreeInfo);
 }
 
@@ -338,7 +338,7 @@ bool CLogThread::queueLog(IEspContext & context,const char* serviceName,int Reco
 
     serializeRequest(context,logInfo,_LogStruct.RequestStr);
 
-    Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(_LogStruct.RequestStr.str(), ipt_none, xr_none);
+    Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(_LogStruct.RequestStr.str(), ipt_none, ptr_none);
 
 
     addLogInfo(LogArray,*pLogTreeInfo.get());
@@ -529,7 +529,7 @@ bool CLogThread::queueLog(IEspContext & context,const char* serviceName,int Reco
 
 bool CLogThread::queueLog(IEspContext & context,const char* serviceName,int RecordsReturned,IArrayOf<IEspLogInfo>& LogArray, StringBuffer& logInfo)
 {
-    Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(logInfo, ipt_none, xr_none);
+    Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(logInfo, ipt_none, ptr_none);
     return queueLog(context,serviceName,RecordsReturned,LogArray, *pLogTreeInfo.get());
 }
 
@@ -548,7 +548,7 @@ bool CLogThread::queueLog(IEspContext & context,const char* serviceName, const c
     else
         context.getRealm(UserRealm);
 
-    Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(request, ipt_none, xr_none);
+    Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(request, ipt_none, ptr_none);
     IArrayOf<IEspLogInfo> LogArray;
     addLogInfo(LogArray, *pLogTreeInfo.get());
 
@@ -568,7 +568,7 @@ bool CLogThread::queueLog(IEspContext & context,const char* serviceName, const c
 bool CLogThread::queueLog(IEspContext & context,const char* serviceName,int RecordsReturned, const char* logInfo)
 {
     try {
-        Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(logInfo, ipt_none, xr_none);
+        Owned<IPropertyTree> pLogTreeInfo = createPTreeFromXMLString(logInfo, ipt_none, ptr_none);
         return queueLog(context,serviceName,RecordsReturned,*pLogTreeInfo);
     } catch (IException* e) {
         StringBuffer msg;
@@ -921,7 +921,7 @@ IClientLOGServiceUpdateRequest* CLogThread::DeserializeRequest(const char* reque
     m_LogFailSafe->SplitLogRecord(requestStr,GUID,Cache);
     _Info.GUID = GUID.str();
 
-    Owned<IPropertyTree> pLogTree = createPTreeFromXMLString(Cache.str(), ipt_none, xr_none);
+    Owned<IPropertyTree> pLogTree = createPTreeFromXMLString(Cache.str(), ipt_none, ptr_none);
     pRequest->setUserName(pLogTree->queryProp("UserName"));
     pRequest->setDomainName(pLogTree->queryProp("DomainName"));
     pRequest->setRecordCount(pLogTree->getPropInt("RecordCount"));

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -1362,7 +1362,7 @@ void appendEclInputXsds(StringBuffer &content, IPropertyTree *xsd, BoolHash &add
 
 void CWsEclBinding::SOAPSectionToXsd(WsEclWuInfo &wsinfo, const char *parmXml, StringBuffer &schema, bool isRequest, IPropertyTree *xsdtree)
 {
-    Owned<IPropertyTree> tree = createPTreeFromXMLString(parmXml, ipt_none, (XmlReaderOptions)(xr_ignoreWhiteSpace|xr_noRoot));
+    Owned<IPropertyTree> tree = createPTreeFromXMLString(parmXml, ipt_none, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_noRoot));
 
     schema.appendf("<xsd:element name=\"%s%s\">", wsinfo.queryname.sget(), isRequest ? "Request" : "Response");
     schema.append("<xsd:complexType>");
@@ -1824,7 +1824,7 @@ void CWsEclBinding::getWsEclJsonResponse(StringBuffer& jsonmsg, IEspContext &con
     size32_t start = jsonmsg.length();
     try
     {
-        Owned<IPropertyTree> parmtree = createPTreeFromXMLString(xml, ipt_none, (XmlReaderOptions)(xr_ignoreWhiteSpace|xr_ignoreNameSpaces));
+        Owned<IPropertyTree> parmtree = createPTreeFromXMLString(xml, ipt_none, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces));
 
         StringBuffer element;
         element.append(wsinfo.queryname.sget());
@@ -2189,7 +2189,7 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf
     workunit->setState(WUStateSubmitted);
     workunit->commit();
 
-    Owned<IPropertyTree> req = createPTreeFromXMLString(xml, ipt_none, (XmlReaderOptions)(xr_ignoreWhiteSpace|xr_ignoreNameSpaces));
+    Owned<IPropertyTree> req = createPTreeFromXMLString(xml, ipt_none, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces));
     IPropertyTree *start = req.get();
     if (start->hasProp("Envelope"))
         start=start->queryPropTree("Envelope");

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -103,7 +103,7 @@ void setWsWuXmlParameters(IWorkUnit *wu, const char *xml, bool setJobname=false)
 {
     if (!xml || !*xml)
         return;
-    Owned<IPropertyTree> tree = createPTreeFromXMLString(xml, ipt_none, (XmlReaderOptions)(xr_ignoreWhiteSpace | xr_ignoreNameSpaces));
+    Owned<IPropertyTree> tree = createPTreeFromXMLString(xml, ipt_none, (PTreeReaderOptions)(ptr_ignoreWhiteSpace | ptr_ignoreNameSpaces));
     IPropertyTree *root = tree.get();
     if (strieq(root->queryName(), "Envelope"))
         root = root->queryPropTree("Body/*[1]");

roxie/ccd/ccd.hpp

@@ -396,7 +396,7 @@ extern bool simpleLocalKeyedJoins;
 extern bool enableKeyDiff;
 extern bool enableForceKeyDiffCopy;
 extern bool useTreeCopy;
-extern XmlReaderOptions defaultXmlReadFlags;
+extern PTreeReaderOptions defaultXmlReadFlags;
 extern bool mergeSlaveStatistics;
 extern bool roxieMulticastEnabled;   // enable use of multicast for sending requests to slaves
 extern bool preloadOnceData;

roxie/ccd/ccdactivities.cpp

@@ -1619,7 +1619,7 @@ public:
 #endif
         Linked<IXmlToRowTransformer> rowTransformer = helper->queryTransformer();
         OwnedRoxieString xmlIterator(helper->getXmlIteratorPath());
-        Owned<IXMLParse> xmlParser = createXMLParse(*reader->querySimpleStream(), xmlIterator, *this, (0 != (TDRxmlnoroot & helper->getFlags()))?xr_noRoot:xr_none, (helper->getFlags() & TDRusexmlcontents) != 0);
+        Owned<IXMLParse> xmlParser = createXMLParse(*reader->querySimpleStream(), xmlIterator, *this, (0 != (TDRxmlnoroot & helper->getFlags()))?ptr_noRoot:ptr_none, (helper->getFlags() & TDRusexmlcontents) != 0);
         while (!aborted)
         {
             //call to next() will callback on the IXmlSelect interface

roxie/ccd/ccdcontext.cpp

@@ -688,7 +688,7 @@ public:
         temporaries = NULL;
         deserializedResultStore = NULL;
         rereadResults = NULL;
-        xmlStoredDatasetReadFlags = xr_none;
+        xmlStoredDatasetReadFlags = ptr_none;
         if (_debuggerActive)
         {
             CSlaveDebugContext *slaveDebugContext = new CSlaveDebugContext(this, logctx, *header);
@@ -1503,7 +1503,7 @@ protected:
     Owned<IPropertyTree> context;
     IPropertyTree *temporaries;
     IPropertyTree *rereadResults;
-    XmlReaderOptions xmlStoredDatasetReadFlags;
+    PTreeReaderOptions xmlStoredDatasetReadFlags;
     CDeserializedResultStore *deserializedResultStore;
 
     IPropertyTree &useContext(unsigned sequence)
@@ -1961,7 +1961,7 @@ public:
         startWorkUnit();
     }
 
-    CRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, bool _isXml, bool _isRaw, bool _isBlocked, HttpHelper &httpHelper, bool _trim, unsigned _priority, const IRoxieContextLogger &_logctx, XmlReaderOptions _xmlReadFlags)
+    CRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, bool _isXml, bool _isRaw, bool _isBlocked, HttpHelper &httpHelper, bool _trim, unsigned _priority, const IRoxieContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags)
         : CSlaveContext(_factory, _logctx, 0, 0, NULL, false, false), serverQueryFactory(_factory)
     {
         init();
@@ -2914,7 +2914,7 @@ private:
     StringAttr queryName;
 
 public:
-    CSoapRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, HttpHelper &httpHelper, unsigned _priority, const IRoxieContextLogger &_logctx, XmlReaderOptions xmlReadFlags)
+    CSoapRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, HttpHelper &httpHelper, unsigned _priority, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags)
         : CRoxieServerContext(_context, _factory, _client, true, false, false, httpHelper, true, _priority, _logctx, xmlReadFlags)
     {
         queryName.set(_context->queryName());
@@ -2967,7 +2967,7 @@ public:
     }
 };
 
-IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, const IQueryFactory *factory, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, unsigned priority, const IRoxieContextLogger &_logctx, XmlReaderOptions xmlReadFlags)
+IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, const IQueryFactory *factory, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, unsigned priority, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags)
 {
     if (httpHelper.isHttp())
         return new CSoapRoxieServerContext(context, factory, client, httpHelper, priority, _logctx, xmlReadFlags);

roxie/ccd/ccdcontext.hpp

@@ -109,7 +109,7 @@ typedef IEclProcess* (* EclProcessFactory)();
 
 extern IDeserializedResultStore *createDeserializedResultStore();
 extern IRoxieSlaveContext *createSlaveContext(const IQueryFactory *factory, const SlaveContextLogger &logctx, unsigned timeLimit, memsize_t memoryLimit, IRoxieQueryPacket *packet);
-extern IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, const IQueryFactory *factory, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, unsigned priority, const IRoxieContextLogger &logctx, XmlReaderOptions xmlReadFlags);
+extern IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, const IQueryFactory *factory, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, unsigned priority, const IRoxieContextLogger &logctx, PTreeReaderOptions xmlReadFlags);
 extern IRoxieServerContext *createOnceServerContext(const IQueryFactory *factory, const IRoxieContextLogger &_logctx);
 extern IRoxieServerContext *createWorkUnitServerContext(IConstWorkUnit *wu, const IQueryFactory *factory, const IRoxieContextLogger &logctx);
 extern WorkflowMachine *createRoxieWorkflowMachine(IPropertyTree *_workflowInfo, bool doOnce, const IRoxieContextLogger &_logctx);

roxie/ccd/ccdlistener.cpp

@@ -100,13 +100,13 @@ private:
     Linked<IQueryFactory> f;
     SafeSocket &client;
     HttpHelper &httpHelper;
-    XmlReaderOptions xmlReadFlags;
+    PTreeReaderOptions xmlReadFlags;
     unsigned &memused;
     unsigned &slaveReplyLen;
     CriticalSection crit;
 
 public:
-    CSoapRequestAsyncFor(const char *_queryName, IQueryFactory *_f, IArrayOf<IPropertyTree> &_requestArray, SafeSocket &_client, HttpHelper &_httpHelper, unsigned &_memused, unsigned &_slaveReplyLen, const char *_queryText, const IRoxieContextLogger &_logctx, XmlReaderOptions _xmlReadFlags) :
+    CSoapRequestAsyncFor(const char *_queryName, IQueryFactory *_f, IArrayOf<IPropertyTree> &_requestArray, SafeSocket &_client, HttpHelper &_httpHelper, unsigned &_memused, unsigned &_slaveReplyLen, const char *_queryText, const IRoxieContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags) :
       f(_f), requestArray(_requestArray), client(_client), httpHelper(_httpHelper), memused(_memused), slaveReplyLen(_slaveReplyLen), logctx(_logctx), xmlReadFlags(_xmlReadFlags)
     {
         queryName = _queryName;
@@ -1418,7 +1418,7 @@ readAnother:
 
                 if (strnicmp(rawText.str(), "<control:aclupdate", 18)==0 && !isalpha(rawText.charAt(18)))
                 {
-                    queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (XmlReaderOptions)(xr_ignoreWhiteSpace|xr_ignoreNameSpaces)));
+                    queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces)));
                     IPropertyTree *aclTree = queryXml->queryPropTree("ACL");
                     if (aclTree)
                     {
@@ -1472,7 +1472,7 @@ readAnother:
             {
                 try
                 {
-                    queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (XmlReaderOptions)(defaultXmlReadFlags | xr_ignoreNameSpaces)));
+                    queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)(defaultXmlReadFlags | ptr_ignoreNameSpaces)));
                 }
                 catch (IException *E)
                 {
@@ -1548,14 +1548,14 @@ readAnother:
                             client->setHttpMode(queryName, isRequestArray);
                         if (queryFactory)
                         {
-                            bool stripWhitespace = queryFactory->getDebugValueBool("stripWhitespaceFromStoredDataset", 0 != (xr_ignoreWhiteSpace & defaultXmlReadFlags));
+                            bool stripWhitespace = queryFactory->getDebugValueBool("stripWhitespaceFromStoredDataset", 0 != (ptr_ignoreWhiteSpace & defaultXmlReadFlags));
                             stripWhitespace = queryXml->getPropBool("_stripWhitespaceFromStoredDataset", stripWhitespace);
-                            XmlReaderOptions xmlReadFlags = (XmlReaderOptions)((defaultXmlReadFlags & ~xr_ignoreWhiteSpace) |
-                                                                               (stripWhitespace ? xr_ignoreWhiteSpace : xr_none));
+                            PTreeReaderOptions xmlReadFlags = (PTreeReaderOptions)((defaultXmlReadFlags & ~ptr_ignoreWhiteSpace) |
+                                                                               (stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none));
                             if (xmlReadFlags != defaultXmlReadFlags)
                             {
                                 // we need to reparse input xml, as global whitespace setting has been overridden
-                                queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (XmlReaderOptions)(xmlReadFlags|xr_ignoreNameSpaces)));
+                                queryXml.setown(createPTreeFromXMLString(rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)(xmlReadFlags|ptr_ignoreNameSpaces)));
                                 sanitizeQuery(queryXml, queryName, sanitizedText, isHTTP, uid, isRequest, isRequestArray, isBlind, isDebug);
                             }
                             IArrayOf<IPropertyTree> requestArray;

roxie/ccd/ccdmain.cpp

@@ -85,7 +85,7 @@ bool insertionSort = false;
 bool fieldTranslationEnabled = false;
 bool useTreeCopy = true;
 bool mergeSlaveStatistics = true;
-XmlReaderOptions defaultXmlReadFlags = xr_ignoreWhiteSpace;
+PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
 bool runOnce = false;
 
 unsigned udpMulticastBufferSize = 262142;
@@ -701,7 +701,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         defaultWarnTimeLimit[1] = (unsigned) topology->getPropInt64("@defaultHighPriorityTimeWarning", 0);
         defaultWarnTimeLimit[2] = (unsigned) topology->getPropInt64("@defaultSLAPriorityTimeWarning", 0);
 
-        defaultXmlReadFlags = topology->getPropBool("@defaultStripLeadingWhitespace", true) ? xr_ignoreWhiteSpace : xr_none;
+        defaultXmlReadFlags = topology->getPropBool("@defaultStripLeadingWhitespace", true) ? ptr_ignoreWhiteSpace : ptr_none;
         defaultParallelJoinPreload = topology->getPropInt("@defaultParallelJoinPreload", 0);
         defaultConcatPreload = topology->getPropInt("@defaultConcatPreload", 0);
         defaultFetchPreload = topology->getPropInt("@defaultFetchPreload", 0);

roxie/ccd/ccdquery.cpp

@@ -1193,7 +1193,7 @@ public:
         throwUnexpected();   // only implemented in derived slave class
     }
 
-    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, XmlReaderOptions xmlReadFlags) const
+    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags) const
     {
         throwUnexpected();   // only implemented in derived server class
     }
@@ -1312,7 +1312,7 @@ public:
         return activities;
     }
 
-    virtual IRoxieServerContext *createContext(IPropertyTree *context, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, XmlReaderOptions _xmlReadFlags) const
+    virtual IRoxieServerContext *createContext(IPropertyTree *context, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags) const
     {
         checkSuspended();
         return createRoxieServerContext(context, this, client, isXml, isRaw, isBlocked, httpHelper, trim, priority, _logctx, _xmlReadFlags);

roxie/ccd/ccdquery.hpp

@@ -115,7 +115,7 @@ interface IQueryFactory : extends IInterface
     virtual int getDebugValueInt(const char * propname, int defVal) const = 0;
     virtual bool getDebugValueBool(const char * propname, bool defVal) const = 0;
 
-    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, XmlReaderOptions xmlReadFlags) const = 0;
+    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const IRoxieContextLogger &_logctx, PTreeReaderOptions xmlReadFlags) const = 0;
     virtual IRoxieServerContext *createContext(IConstWorkUnit *wu, const IRoxieContextLogger &_logctx) const = 0;
     virtual void noteQuery(time_t startTime, bool failed, unsigned elapsed, unsigned memused, unsigned slavesReplyLen, unsigned bytesOut) = 0;
     virtual IPropertyTree *getQueryStats(time_t from, time_t to) = 0;

roxie/ccd/ccdserver.cpp

@@ -19892,7 +19892,7 @@ public:
             rowTransformer.set(readHelper->queryTransformer());
             assertex(reader != NULL);
             OwnedRoxieString xmlIterator(readHelper->getXmlIteratorPath());
-            xmlParser.setown(createXMLParse(*reader->querySimpleStream(), xmlIterator, *this, (0 != (TDRxmlnoroot & readHelper->getFlags()))?xr_noRoot:xr_none, (readHelper->getFlags() & TDRusexmlcontents) != 0));
+            xmlParser.setown(createXMLParse(*reader->querySimpleStream(), xmlIterator, *this, (0 != (TDRxmlnoroot & readHelper->getFlags()))?ptr_noRoot:ptr_none, (readHelper->getFlags() & TDRusexmlcontents) != 0));
         }
     }
 

system/jhtree/jhtree.cpp

@@ -1600,7 +1600,7 @@ IPropertyTree * CKeyIndex::getMetadata()
     {
         ret = createPTreeFromXMLString(xml.str());
     }
-    catch(IXMLReadException * e)
+    catch(IPTreeReadException * e)
     {
         StringBuffer emsg;
         IException * wrapped = MakeStringException(e->errorAudience(), e->errorCode(), "Error retrieving XML metadata: %s", e->errorMessage(emsg).str());

system/jlib/jptree.hpp

@@ -126,25 +126,25 @@ interface IPTreeNotifyEvent : extends IInterface
     virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset) = 0;
 };
 
-enum XmlReadExcptCode { XmlRead_undefined, XmlRead_EOS, XmlRead_syntax };
-interface jlib_thrown_decl IXMLReadException : extends IException
+enum PTreeReadExcptCode { PTreeRead_undefined, PTreeRead_EOS, PTreeRead_syntax };
+interface jlib_thrown_decl IPTreeReadException : extends IException
 {
     virtual const char *queryDescription() = 0;
     virtual unsigned queryLine() = 0;
     virtual offset_t queryOffset() = 0;
     virtual const char *queryContext() = 0;
 };
-extern jlib_decl IXMLReadException *createXmlReadException(int code, const char *msg, const char *context, unsigned line, offset_t offset);
+extern jlib_decl IPTreeReadException *createPTreeReadException(int code, const char *msg, const char *context, unsigned line, offset_t offset);
 
-enum XmlReaderOptions { xr_none=0x00, xr_ignoreWhiteSpace=0x01, xr_noRoot=0x02, xr_ignoreNameSpaces=0x04 };
+enum PTreeReaderOptions { ptr_none=0x00, ptr_ignoreWhiteSpace=0x01, ptr_noRoot=0x02, ptr_ignoreNameSpaces=0x04 };
 
-interface IXMLReader : extends IInterface
+interface IPTreeReader : extends IInterface
 {
     virtual void load() = 0;
     virtual offset_t queryOffset() = 0;
 };
 
-interface IPullXMLReader : extends IXMLReader
+interface IPullPTreeReader : extends IPTreeReader
 {
     virtual bool next() = 0;
     virtual void reset() = 0;
@@ -167,12 +167,19 @@ interface IPTreeNodeCreator : extends IInterface
 enum ipt_flags { ipt_none=0x00, ipt_caseInsensitive=0x01, ipt_binary=0x02, ipt_ordered=0x04, ipt_ext1=0x08, ipt_ext2=16, ipt_ext3=32, ipt_ext4=64, ipt_ext5=128 };
 jlib_decl IPTreeMaker *createPTreeMaker(byte flags=ipt_none, IPropertyTree *root=NULL, IPTreeNodeCreator *nodeCreator=NULL);
 jlib_decl IPTreeMaker *createRootLessPTreeMaker(byte flags=ipt_none, IPropertyTree *root=NULL, IPTreeNodeCreator *nodeCreator=NULL);
-jlib_decl IXMLReader *createXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, XmlReaderOptions xmlReaderOptions=xr_ignoreWhiteSpace, size32_t bufSize=0);
-jlib_decl IXMLReader *createXMLStringReader(const char *xml, IPTreeNotifyEvent &iEvent, XmlReaderOptions xmlReaderOptions=xr_ignoreWhiteSpace);
-jlib_decl IXMLReader *createXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, XmlReaderOptions xmlReaderOptions=xr_ignoreWhiteSpace);
-jlib_decl IPullXMLReader *createPullXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, XmlReaderOptions xmlReaderOptions=xr_ignoreWhiteSpace, size32_t bufSize=0);
-jlib_decl IPullXMLReader *createPullXMLStringReader(const char *xml, IPTreeNotifyEvent &iEvent, XmlReaderOptions xmlReaderOptions=xr_ignoreWhiteSpace);
-jlib_decl IPullXMLReader *createPullXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, XmlReaderOptions xmlReaderOptions=xr_ignoreWhiteSpace);
+jlib_decl IPTreeReader *createXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions=ptr_ignoreWhiteSpace, size32_t bufSize=0);
+jlib_decl IPTreeReader *createXMLStringReader(const char *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions=ptr_ignoreWhiteSpace);
+jlib_decl IPTreeReader *createXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions=ptr_ignoreWhiteSpace);
+jlib_decl IPullPTreeReader *createPullXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions=ptr_ignoreWhiteSpace, size32_t bufSize=0);
+jlib_decl IPullPTreeReader *createPullXMLStringReader(const char *xml, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions=ptr_ignoreWhiteSpace);
+jlib_decl IPullPTreeReader *createPullXMLBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions=ptr_ignoreWhiteSpace);
+
+jlib_decl IPTreeReader *createJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions=ptr_ignoreWhiteSpace, size32_t bufSize=0);
+jlib_decl IPTreeReader *createJSONStringReader(const char *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions=ptr_ignoreWhiteSpace);
+jlib_decl IPTreeReader *createJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions jsonReaderOptions=ptr_ignoreWhiteSpace);
+jlib_decl IPullPTreeReader *createPullJSONStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions=ptr_ignoreWhiteSpace, size32_t bufSize=0);
+jlib_decl IPullPTreeReader *createPullJSONStringReader(const char *json, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions=ptr_ignoreWhiteSpace);
+jlib_decl IPullPTreeReader *createPullJSONBufferReader(const void *buf, size32_t bufLength, IPTreeNotifyEvent &iEvent, PTreeReaderOptions readerOptions=ptr_ignoreWhiteSpace);
 
 jlib_decl void mergePTree(IPropertyTree *target, IPropertyTree *toMerge);
 jlib_decl void synchronizePTree(IPropertyTree *target, IPropertyTree *source);
@@ -183,14 +190,17 @@ jlib_decl IPropertyTree *createPTree(MemoryBuffer &src);
 
 jlib_decl IPropertyTree *createPTree(byte flags=ipt_none);
 jlib_decl IPropertyTree *createPTree(const char *name, byte flags=ipt_none);
-jlib_decl IPropertyTree *createPTree(IFile &ifile, byte flags=ipt_none, XmlReaderOptions readFlags=xr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
-jlib_decl IPropertyTree *createPTree(IFileIO &ifileio, byte flags=ipt_none, XmlReaderOptions readFlags=xr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
-jlib_decl IPropertyTree *createPTree(ISimpleReadStream &stream, byte flags=ipt_none, XmlReaderOptions readFlags=xr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
-jlib_decl IPropertyTree *createPTreeFromXMLString(const char *xml, byte flags=ipt_none, XmlReaderOptions readFlags=xr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
-jlib_decl IPropertyTree *createPTreeFromXMLString(unsigned len, const char *xml, byte flags=ipt_none, XmlReaderOptions readFlags=xr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
-jlib_decl IPropertyTree *createPTreeFromXMLFile(const char *filename, byte flags=ipt_none, XmlReaderOptions readFlags=xr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
+jlib_decl IPropertyTree *createPTree(IFile &ifile, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
+jlib_decl IPropertyTree *createPTree(IFileIO &ifileio, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
+jlib_decl IPropertyTree *createPTree(ISimpleReadStream &stream, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
+jlib_decl IPropertyTree *createPTreeFromXMLString(const char *xml, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
+jlib_decl IPropertyTree *createPTreeFromXMLString(unsigned len, const char *xml, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
+jlib_decl IPropertyTree *createPTreeFromXMLFile(const char *filename, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromIPT(const IPropertyTree *srcTree, ipt_flags flags=ipt_none);
 
+jlib_decl IPropertyTree *createPTreeFromJSONString(const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
+jlib_decl IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
+
 #define XML_SortTags 0x01
 #define XML_Format   0x02
 #define XML_NoEncode 0x04

system/jlib/jptree.ipp

@@ -140,6 +140,17 @@ inline static bool isValidXPathChr(char c)
     return ('\0' != c && (isalnum(c) || strchr(validChrs, c)));
 }
 
+inline static int validJSONUtf8ChrLen(unsigned char c)
+{
+    if (c <= 31)
+        return 0;
+    if ('\"' == c)
+        return 0;
+    if ('\\' == c)
+        return 2;
+    return utf8CharLen(&c);
+}
+
 inline static bool isAttribute(const char *xpath) { return (xpath && *xpath == '@'); }
 
 jlib_decl const char *splitXPathUQ(const char *xpath, StringBuffer &path);

system/jlib/jstring.cpp

@@ -1617,6 +1617,85 @@ static void writeUtf8(unsigned c, StringBuffer &out)
         assertex(false);
 }
 
+#define JSONSTRICT
+const char *decodeJSON(const char *j, StringBuffer &ret, unsigned len, const char **errMark)
+{
+    if (!j)
+        return j;
+    if ((unsigned)-1 == len)
+        len = (unsigned)strlen(j);
+    try
+    {
+        for (const char *end = j+len; j<end && *j; j++)
+        {
+            if (*j!='\\')
+                ret.append(*j);
+            else
+            {
+                switch (*++j)
+                {
+                case 'u':
+                {
+                    j++;
+                    if (end-j>=4)
+                    {
+                        char *endptr;
+                        StringAttr s(j, 4);
+                        unsigned val = strtoul(s.get(), &endptr, 16);
+                        if (endptr && !*endptr)
+                        {
+                            writeUtf8(val, ret);
+                            j+=3;
+                            break;
+                        }
+                    }
+#ifdef JSONSTRICT
+                    throw MakeStringException(-1, "invalid json \\u escaped sequence");
+#endif
+                    ret.append(*j);
+                    break;
+                }
+                case '\"':
+                case '\\':
+                case '/':
+                    ret.append(*j);
+                    break;
+                case 'b':
+                    ret.append('\b');
+                    break;
+                case 'f':
+                    ret.append('\f');
+                    break;
+                case 'n':
+                    ret.append('\n');
+                    continue;
+                case 'r':
+                    ret.append('\r');
+                    break;
+                case 't':
+                    ret.append('\t');
+                    break;
+                default:
+                {
+#ifdef JSONSTRICT
+                    throw MakeStringException(-1, "invalid json escaped sequence");
+#endif
+                    ret.append('\\');
+                    ret.append(*j);
+                    break;
+                }
+                }
+            }
+        }
+    }
+    catch (IException *)
+    {
+        if (errMark) *errMark = j;
+        throw;
+    }
+    return j;
+}
+
 void decodeXML(ISimpleReadStream &in, StringBuffer &out, unsigned len)
 {
     // TODO

system/jlib/jstring.hpp

@@ -361,12 +361,15 @@ extern jlib_decl StringBuffer & appendStringAsSQL(StringBuffer & out, unsigned l
 extern jlib_decl StringBuffer & appendStringAsECL(StringBuffer & out, unsigned len, const char * src);
 extern jlib_decl StringBuffer & appendStringAsQuotedECL(StringBuffer &out, unsigned len, const char * src);
 
-jlib_decl void extractItem(StringBuffer & res, const char * src, const char * sep, int whichItem, bool caps);
+extern jlib_decl const char *decodeJSON(const char *x, StringBuffer &ret, unsigned len=(unsigned)-1, const char **errMark=NULL);
+extern jlib_decl void extractItem(StringBuffer & res, const char * src, const char * sep, int whichItem, bool caps);
 extern jlib_decl const char *encodeXML(const char *x, StringBuffer &ret, unsigned flags=0, unsigned len=(unsigned)-1, bool utf8=false);
 extern jlib_decl const char *decodeXML(const char *x, StringBuffer &ret, unsigned len=(unsigned)-1, const char **errMark=NULL, IEntityHelper *entityHelper=NULL);
 extern jlib_decl const char *encodeXML(const char *x, IIOStream &out, unsigned flags=0, unsigned len=(unsigned)-1, bool utf8=false);
 extern jlib_decl void decodeXML(ISimpleReadStream &in, StringBuffer &out, unsigned len=(unsigned)-1);
 
+extern jlib_decl int utf8CharLen(const unsigned char *ch);
+
 inline const char *encodeUtf8XML(const char *x, StringBuffer &ret, unsigned flags=false, unsigned len=(unsigned)-1)
 {
     return encodeXML(x, ret, flags, len, true);

thorlcr/activities/fetch/thfetchslave.cpp

@@ -624,7 +624,7 @@ public:
             streams[f].setown(createBufferedIOStream(fetchStream->queryPartIO(f)));
             // NB: the index is based on path iteration matches, so on lookup the elements start at positioned stream
             // i.e. getXmlIteratorPath not used (or supplied) here.
-            parsers[f].setown(createXMLParse(*streams[f], "/", *xmlSelect, xr_none, ((IHThorXmlFetchArg *)fetchBaseHelper)->requiresContents()));
+            parsers[f].setown(createXMLParse(*streams[f], "/", *xmlSelect, ptr_none, ((IHThorXmlFetchArg *)fetchBaseHelper)->requiresContents()));
         }
     }
     virtual size32_t fetch(ARowBuilder & rowBuilder, const void *keyRow, unsigned filePartIndex, unsigned __int64 localFpos, unsigned __int64 fpos)

thorlcr/activities/xmlparse/thxmlparseslave.cpp

@@ -135,7 +135,7 @@ public:
                 unsigned len;
                 helper->getSearchText(len, searchStr, nxt);
                 OwnedRoxieString xmlIteratorPath(helper->getXmlIteratorPath());
-                xmlParser.setown(createXMLParse(searchStr, len, xmlIteratorPath, *this, xr_noRoot, helper->requiresContents()));
+                xmlParser.setown(createXMLParse(searchStr, len, xmlIteratorPath, *this, ptr_noRoot, helper->requiresContents()));
             }
         }
         catch (IOutOfMemException *e)

thorlcr/activities/xmlread/thxmlreadslave.cpp

@@ -82,7 +82,7 @@ class CXmlReadSlaveActivity : public CDiskReadSlaveActivityBase, public CThorDat
             }
             inputIOstream.setown(createBufferedIOStream(stream));
             OwnedRoxieString xmlIterator(activity.helper->getXmlIteratorPath());
-            xmlParser.setown(createXMLParse(*inputIOstream.get(), xmlIterator, *this, (0 != (TDRxmlnoroot & activity.helper->getFlags()))?xr_noRoot:xr_none, 0 != (TDRusexmlcontents & activity.helper->getFlags())));
+            xmlParser.setown(createXMLParse(*inputIOstream.get(), xmlIterator, *this, (0 != (TDRxmlnoroot & activity.helper->getFlags()))?ptr_noRoot:ptr_none, 0 != (TDRusexmlcontents & activity.helper->getFlags())));
         }
         virtual void close(CRC32 &fileCRC)
         {
@@ -124,9 +124,9 @@ class CXmlReadSlaveActivity : public CDiskReadSlaveActivityBase, public CThorDat
                 context.append("offset = ").append(localOffset);
                 throw MakeStringException(e->errorCode(), "%s", context.str());
             }
-            catch (IXMLReadException *e)
+            catch (IPTreeReadException *e)
             {
-                if (XmlRead_syntax != e->errorCode())
+                if (PTreeRead_syntax != e->errorCode())
                     throw;
                 Owned<IException> _e = e;
                 offset_t localFPos = makeLocalFposOffset(activity.queryContainer().queryJob().queryMyRank()-1, e->queryOffset());
@@ -138,7 +138,7 @@ class CXmlReadSlaveActivity : public CDiskReadSlaveActivityBase, public CThorDat
                 appendDataAsHex(context, sizeof(localFPos), &localFPos);
                 context.newline();
                 context.append(e->queryContext());
-                throw createXmlReadException(e->errorCode(), e->queryDescription(), context.str(), e->queryLine(), e->queryOffset());
+                throw createPTreeReadException(e->errorCode(), e->queryDescription(), context.str(), e->queryLine(), e->queryOffset());
             }
             catch (IOutOfMemException *e)
             {

tools/testsocket/testsocket.cpp

@@ -425,7 +425,7 @@ int doSendQuery(const char * ip, unsigned port, const char * base)
     if (useHTTP)
     {
         StringBuffer newQuery;
-        Owned<IPTree> p = createPTreeFromXMLString(base, ipt_none, xr_none);
+        Owned<IPTree> p = createPTreeFromXMLString(base, ipt_none, ptr_none);
         const char *queryName = p->queryName();
         if ((stricmp(queryName, "envelope") != 0) && (stricmp(queryName, "envelope") != 0))
         {
@@ -467,7 +467,7 @@ int doSendQuery(const char * ip, unsigned port, const char * base)
         {
             try
             {
-                Owned<IPTree> p = createPTreeFromXMLString(base, ipt_none, xr_none);
+                Owned<IPTree> p = createPTreeFromXMLString(base, ipt_none, ptr_none);
                 p->renameProp("/", queryNameOverride);
                 toXML(p, fullQuery.clear());
             }