HPCC-17625 JPTree memory usage and contention issues

Specify what form of ptree to use on different creates, to ensure we use the
low-memory one for persistant trees and the fast one for transient trees.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/environment/environment.cpp

@@ -463,14 +463,14 @@ public:
 
     virtual IEnvironment * loadLocalEnvironmentFile(const char * filename)
     {
-        Owned<IPropertyTree> ptree = createPTreeFromXMLFile(filename);
+        Owned<IPropertyTree> ptree = createPTreeFromXMLFile(filename, ipt_lowmem);
         Owned<CLocalEnvironment> pLocalEnv = new CLocalEnvironment(nullptr, ptree);
         return new CLockedEnvironment(pLocalEnv);
     }
 
     virtual IEnvironment * loadLocalEnvironment(const char * xml)
     {
-        Owned<IPropertyTree> ptree = createPTreeFromXMLString(xml);
+        Owned<IPropertyTree> ptree = createPTreeFromXMLString(xml, ipt_lowmem);
         Owned<CLocalEnvironment> pLocalEnv = new CLocalEnvironment(nullptr, ptree);
         return new CLockedEnvironment(pLocalEnv);
     }
@@ -1400,7 +1400,7 @@ void CLocalEnvironment::preload()
 
 void CLocalEnvironment::setXML(const char *xml)
 {
-    Owned<IPropertyTree> newRoot = createPTreeFromXMLString(xml);
+    Owned<IPropertyTree> newRoot = createPTreeFromXMLString(xml, ipt_lowmem);
     synchronized procedure(safeCache);
     Owned<IPropertyTreeIterator> it = p->getElements("*");
     ForEach(*it)
@@ -1709,7 +1709,7 @@ CConstDropZoneServerInfoIterator::CConstDropZoneServerInfoIterator(const IConstD
     if (0 != dropZoneMachineName.length())
     {
         // Create a ServerList for legacy element.
-        Owned<IPropertyTree> legacyServerList = createPTree();
+        Owned<IPropertyTree> legacyServerList = createPTree(ipt_lowmem);
 
         Owned<IConstMachineInfo> machineInfo = constEnv->getMachine(dropZoneMachineName.str());
         if (machineInfo)
@@ -1719,10 +1719,9 @@ CConstDropZoneServerInfoIterator::CConstDropZoneServerInfoIterator(const IConstD
 
             // Create a single ServerList record related to @computer
             //<ServerList name="ServerList" server="<IP_of_@computer>"/>
-            Owned<IPropertyTree> newRecord = createPTree();
+            IPropertyTree *newRecord = legacyServerList->addPropTree("ServerList");
             newRecord->setProp("@name", "ServerList");
             newRecord->setProp("@server", dropZoneMachineNetAddress.str());
-            legacyServerList->addPropTree("ServerList",newRecord.getClear());
 
             maxIndex = 1;
         }

common/thorhelper/roxiedebug.cpp

@@ -1763,7 +1763,7 @@ void CBaseServerDebugContext::debugPrint(IXmlWriter *output, const char *edgeId,
 #if 1
     CommonXmlWriter dummyOutput(0, 0);
     activityCtx->printEdge(&dummyOutput, startRow, numRows); // MORE - need to suppress the <Edge> if we want backward compatibility!
-    Owned<IPropertyTree> result = createPTreeFromXMLString(dummyOutput.str());
+    Owned<IPropertyTree> result = createPTreeFromXMLString(dummyOutput.str(),ipt_fast);
     if (result)
     {
         Owned<IAttributeIterator> attributes = result->getAttributes();

common/thorhelper/thorxmlread.cpp

@@ -564,7 +564,7 @@ void XmlSetColumnProvider::readUtf8X(size32_t & len, char * & target, const char
 IDataVal & CXmlToRawTransformer::transform(IDataVal & result, size32_t len, const void * text, bool isDataSet)
 {
     // MORE - should redo using a pull parser sometime
-    Owned<IPropertyTree> root = createPTreeFromXMLString(len, (const char *)text, ipt_none, xmlReadFlags);
+    Owned<IPropertyTree> root = createPTreeFromXMLString(len, (const char *)text, ipt_fast, xmlReadFlags);
     return transformTree(result, *root, isDataSet);
 }
 
@@ -595,7 +595,7 @@ IDataVal & CXmlToRawTransformer::transformTree(IDataVal & result, IPropertyTree
                     try
                     {
                         decodedXML.append("<root>").append(body).append("</root>");
-                        decodedTree.setown(createPTreeFromXMLString(decodedXML.str(), ipt_caseInsensitive));
+                        decodedTree.setown(createPTreeFromXMLString(decodedXML.str(), ipt_caseInsensitive|ipt_fast));
                         rows.setown(decodedTree->getElements("Row"));
                     }
                     catch (IException *E)
@@ -645,7 +645,7 @@ IDataVal & CXmlToRawTransformer::transformTree(IDataVal & result, IPropertyTree
 
 size32_t createRowFromXml(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace)
 {
-    Owned<IPropertyTree> root = createPTreeFromXMLString(size, utf8, ipt_none, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
+    Owned<IPropertyTree> root = createPTreeFromXMLString(size, utf8, ipt_fast, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
     if (!root)
     {
         throwError(THORCERR_InvalidXmlFromXml);
@@ -666,7 +666,7 @@ const void * createRowFromXml(IEngineRowAllocator * rowAllocator, size32_t len,
 
 size32_t createRowFromJson(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace)
 {
-    Owned<IPropertyTree> root = createPTreeFromJSONString(size, utf8, ipt_none, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
+    Owned<IPropertyTree> root = createPTreeFromJSONString(size, utf8, ipt_fast, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
     if (!root)
     {
         throwError(THORCERR_InvalidJsonFromJson);

common/workunit/package.cpp

@@ -39,7 +39,7 @@ CPackageNode::CPackageNode(IPropertyTree *p)
     if (p)
         node.set(p);
     else
-        node.setown(createPTree("HpccPackages"));
+        node.setown(createPTree("HpccPackages", ipt_lowmem));
     StringBuffer xml;
     toXML(node, xml, 0, XML_SortTags);
     hash = rtlHash64Data(xml.length(), xml.str(), 9994410);
@@ -158,7 +158,7 @@ IHpccPackageMap *createPackageMapFromPtree(IPropertyTree *t, const char *queryse
 
 IHpccPackageMap *createPackageMapFromXml(const char *xml, const char *queryset, const char *id)
 {
-    Owned<IPropertyTree> t = createPTreeFromXMLString(xml);
+    Owned<IPropertyTree> t = createPTreeFromXMLString(xml, ipt_lowmem);
     return createPackageMapFromPtree(t, queryset, id);
 }
 

common/workunit/pkgimpl.hpp

@@ -450,7 +450,7 @@ public:
                 Owned<ISimpleSuperFileEnquiry> ssfe = pkg->resolveSuperFile(rf.getLogicalName());
                 if (ssfe && ssfe->numSubFiles()>0)
                 {
-                    IPropertyTree *superInfo = fileInfo->addPropTree("SuperFile", createPTree());
+                    IPropertyTree *superInfo = fileInfo->addPropTree("SuperFile");
                     superInfo->setProp("@name", rf.getLogicalName());
                     unsigned count = ssfe->numSubFiles();
                     while (count--)
@@ -495,7 +495,7 @@ public:
                     referencedPackages.setValue(baseId, true);
             }
         }
-        Owned<IPropertyTree> tempQuerySet=createPTree();
+        Owned<IPropertyTree> tempQuerySet=createPTree(ipt_fast);
         Owned<IPropertyTreeIterator> queries;
         if (queriesToCheck.length())
         {

common/workunit/workflow.cpp

@@ -126,7 +126,7 @@ public:
     CWorkflowItem(IPropertyTree & _tree) { tree.setown(&_tree); }
     CWorkflowItem(IPropertyTree * ptree, unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
     {
-        tree.setown(LINK(ptree->addPropTree("Item", createPTree())));
+        tree.setown(LINK(ptree->addPropTree("Item")));
         tree->setPropInt("@wfid", wfid);
         setEnum(tree, "@type", type, wftypes);
         setEnum(tree, "@mode", mode, wfmodes);
@@ -136,7 +136,7 @@ public:
         {
             tree->setPropInt("@recovery", recovery);
             tree->setPropInt("@retriesAllowed", retriesAllowed);
-            tree->addPropTree("Dependency", createPTree())->setPropInt("@wfid", recovery);
+            tree->addPropTree("Dependency")->setPropInt("@wfid", recovery);
         }
         if(contingencyFor) tree->setPropInt("@contingencyFor", contingencyFor);
         reset();
@@ -165,11 +165,11 @@ public:
     virtual bool         queryPersistRefresh() const { return tree->getPropBool("@persistRefresh", true); }
     virtual IStringVal & getCriticalName(IStringVal & val) const { val.set(tree->queryProp("@criticalName")); return val; }
     virtual IStringVal & queryCluster(IStringVal & val) const { val.set(tree->queryProp("@cluster")); return val; }
-    virtual void         setScheduledNow() { tree->setPropTree("Schedule", createPTree()); setEnum(tree, "@state", WFStateReqd, wfstates); }
-    virtual void         setScheduledOn(char const * name, char const * text) { IPropertyTree * stree = createPTree(); stree->setProp("@name", name); stree->setProp("@text", text); tree->setPropTree("Schedule", createPTree())->setPropTree("Event", stree); setEnum(tree, "@state", WFStateWait, wfstates); }
+    virtual void         setScheduledNow() { tree->setPropTree("Schedule"); setEnum(tree, "@state", WFStateReqd, wfstates); }
+    virtual void         setScheduledOn(char const * name, char const * text) { IPropertyTree * stree =  tree->setPropTree("Schedule")->setPropTree("Event"); stree->setProp("@name", name); stree->setProp("@text", text);; setEnum(tree, "@state", WFStateWait, wfstates); }
     virtual void         setSchedulePriority(unsigned priority) { assertex(tree->hasProp("Schedule")); tree->setPropInt("Schedule/@priority", priority); }
     virtual void         setScheduleCount(unsigned count) { assertex(tree->hasProp("Schedule")); tree->setPropInt("Schedule/@count", count); tree->setPropInt("Schedule/@countRemaining", count); }
-    virtual void         addDependency(unsigned wfid) { tree->addPropTree("Dependency", createPTree())->setPropInt("@wfid", wfid); }
+    virtual void         addDependency(unsigned wfid) { tree->addPropTree("Dependency")->setPropInt("@wfid", wfid); }
     virtual void         setPersistInfo(char const * name, unsigned wfid, int numPersistInstances, bool refresh)
     {
         tree->setProp("@persistName", name);

common/workunit/workunit.cpp

@@ -196,17 +196,16 @@ void CWuGraphStats::beforeDispose()
     StringBuffer tag;
     tag.append("sg").append(id);
 
-    IPropertyTree * subgraph = createPTree(tag);
-    subgraph->setProp("@c", queryCreatorTypeName(creatorType));
-    subgraph->setProp("@creator", creator);
-    subgraph->setPropInt("@minActivity", minActivity);
-    subgraph->setPropInt("@maxActivity", maxActivity);
 
     //Replace the particular subgraph statistics added by this creator
     StringBuffer qualified(tag);
     qualified.append("[@creator='").append(creator).append("']");
     progress->removeProp(qualified);
-    subgraph = progress->addPropTree(tag, subgraph);
+    IPropertyTree * subgraph = progress->addPropTree(tag);
+    subgraph->setProp("@c", queryCreatorTypeName(creatorType));
+    subgraph->setProp("@creator", creator);
+    subgraph->setPropInt("@minActivity", minActivity);
+    subgraph->setPropInt("@maxActivity", maxActivity);
     subgraph->setPropBin("Stats", compressed.length(), compressed.toByteArray());
     if (!progress->getPropBool("@stats", false))
         progress->setPropBool("@stats", true);
@@ -292,7 +291,7 @@ protected:
                 throwUnexpected();
             }
 
-            IPropertyTree * next = curTarget->addPropTree(tag, createPTree());
+            IPropertyTree * next = curTarget->addPropTree(tag);
             next->setProp("@id", id);
             expandProcessTreeFromStats(rootTarget, next, &cur);
         }
@@ -1145,7 +1144,7 @@ public:
         IPropertyTree *node = progress->queryPropTree(path.str());
         if (!node)
         {
-            node = progress->addPropTree("node", createPTree());
+            node = progress->addPropTree("node");
             node->setPropInt64("@id", nodeId);
         }
         node->setPropInt("@_state", (unsigned)state);
@@ -1153,7 +1152,7 @@ public:
         {
             case WUGraphRunning:
             {
-                IPropertyTree *running = conn->queryRoot()->setPropTree("Running", createPTree());
+                IPropertyTree *running = conn->queryRoot()->setPropTree("Running");
                 running->setProp("@graph", graphName);
                 running->setPropInt64("@subId", nodeId);
                 break;
@@ -3913,7 +3912,7 @@ void CLocalWorkUnit::loadXML(const char *xml)
     CriticalBlock block(crit);
     clearCached(true);
     assertex(xml);
-    p.setown(createPTreeFromXMLString(xml));
+    p.setown(createPTreeFromXMLString(xml,ipt_lowmem));
 }
 
 void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
@@ -5269,7 +5268,7 @@ void CLocalWorkUnit::setWarningSeverity(unsigned code, ErrorSeverity severity)
     if (!mapping)
     {
         IPropertyTree * onWarnings = ensurePTree(p, "OnWarnings");
-        mapping = onWarnings->addPropTree("OnWarning", createPTree());
+        mapping = onWarnings->addPropTree("OnWarning");
         mapping->setPropInt("@code", code);
     }
 
@@ -5604,7 +5603,7 @@ void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned
     if (!p->hasProp(xpath))
     {
         IPropertyTree *node = ensurePTree(p, processType.str());
-        node = node->addPropTree(instance, createPTree());
+        node = node->addPropTree(instance);
         node->setProp("@log", log);
         node->setPropInt("@pid", pid);
     }
@@ -5888,7 +5887,7 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
     CriticalBlock block(crit);
     IPropertyTree * stats = p->queryPropTree("Statistics");
     if (!stats)
-        stats = p->addPropTree("Statistics", createPTree("Statistics"));
+        stats = p->addPropTree("Statistics");
 
     IPropertyTree * statTree = NULL;
     if (mergeAction != StatsMergeAppend)
@@ -5900,7 +5899,7 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
 
     if (!statTree)
     {
-        statTree = stats->addPropTree("Statistic", createPTree("Statistic"));
+        statTree = stats->addPropTree("Statistic");
         statTree->setProp("@creator", creator);
         statTree->setProp("@scope", scope);
         statTree->setProp("@kind", kindName);
@@ -5992,9 +5991,9 @@ IWUPlugin* CLocalWorkUnit::updatePluginByName(const char *qname)
     if (existing)
         return (IWUPlugin *) existing;
     if (!plugins.length())
-        p->addPropTree("Plugins", createPTree("Plugins"));
+        p->addPropTree("Plugins");
     IPropertyTree *pl = p->queryPropTree("Plugins");
-    IPropertyTree *s = pl->addPropTree("Plugin", createPTree("Plugin"));
+    IPropertyTree *s = pl->addPropTree("Plugin");
     s->Link();
     IWUPlugin* q = new CLocalWUPlugin(s); 
     q->Link();
@@ -6028,9 +6027,9 @@ IWULibrary* CLocalWorkUnit::updateLibraryByName(const char *qname)
     if (existing)
         return (IWULibrary *) existing;
     if (!libraries.length())
-        p->addPropTree("Libraries", createPTree("Libraries"));
+        p->addPropTree("Libraries");
     IPropertyTree *pl = p->queryPropTree("Libraries");
-    IPropertyTree *s = pl->addPropTree("Library", createPTree("Library"));
+    IPropertyTree *s = pl->addPropTree("Library");
     s->Link();
     IWULibrary* q = new CLocalWULibrary(s); 
     q->Link();
@@ -6092,9 +6091,9 @@ IWUException* CLocalWorkUnit::createException()
     loadExceptions();
 
     if (!exceptions.length())
-        p->addPropTree("Exceptions", createPTree("Exceptions"));
+        p->addPropTree("Exceptions");
     IPropertyTree *r = p->queryPropTree("Exceptions");
-    IPropertyTree *s = r->addPropTree("Exception", createPTree("Exception"));
+    IPropertyTree *s = r->addPropTree("Exception");
     s->setPropInt("@sequence", exceptions.ordinality());
     IWUException* q = new CLocalWUException(LINK(s)); 
     exceptions.append(*LINK(q));
@@ -6132,7 +6131,7 @@ IWUWebServicesInfo* CLocalWorkUnit::updateWebServicesInfo(bool create)
         if (!s)
         {
             if (create)
-                s = p->addPropTree("WebServicesInfo", createPTreeFromXMLString("<WebServicesInfo />"));
+                s = p->addPropTree("WebServicesInfo");
             else
                 return NULL;
         }
@@ -6231,9 +6230,9 @@ IWUResult* CLocalWorkUnit::createResult()
     // For this to be legally called, we must have the write-able interface. So we are already locked for write.
     loadResults();
     if (!results.length())
-        p->addPropTree("Results", createPTree("Results"));
+        p->addPropTree("Results");
     IPropertyTree *r = p->queryPropTree("Results");
-    IPropertyTree *s = r->addPropTree("Result", createPTree());
+    IPropertyTree *s = r->addPropTree("Result");
 
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
@@ -6381,10 +6380,8 @@ IWUResult* CLocalWorkUnit::updateTemporaryByName(const char *qname)
     IConstWUResult *existing = getTemporaryByName(qname);
     if (existing)
         return (IWUResult *) existing;
-    if (!temporaries.length())
-        p->addPropTree("Temporaries", createPTree("Temporaries"));
-    IPropertyTree *vars = p->queryPropTree("Temporaries");
-    IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
+    IPropertyTree *vars = (temporaries.length()) ? p->queryPropTree("Temporaries") : p->addPropTree("Temporaries");
+    IPropertyTree *s = vars->addPropTree("Variable");
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
     q->Link();
@@ -6401,9 +6398,9 @@ IWUResult* CLocalWorkUnit::updateVariableByName(const char *qname)
     if (existing)
         return (IWUResult *) existing;
     if (!variables.length())
-        p->addPropTree("Variables", createPTree("Variables"));
+        p->addPropTree("Variables");
     IPropertyTree *vars = p->queryPropTree("Variables");
-    IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
+    IPropertyTree *s = vars->addPropTree("Variable");
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
     q->Link();
@@ -6488,9 +6485,7 @@ static void _noteFileRead(IDistributedFile *file, IPropertyTree *filesRead)
                 IDistributedFile &file = iter->query();
                 StringBuffer fname;
                 file.getLogicalName(fname);
-                Owned<IPropertyTree> subfile = createPTree();
-                subfile->setProp("@name", fname.str());
-                fileTree->addPropTree("Subfile", subfile.getClear());
+                fileTree->addPropTree("Subfile")->setProp("@name", fname.str());
                 _noteFileRead(&file, filesRead);
             }
         }
@@ -6509,7 +6504,7 @@ void CLocalWorkUnit::noteFileRead(IDistributedFile *file)
         CriticalBlock block(crit);
         IPropertyTree *files = p->queryPropTree("FilesRead");
         if (!files)
-            files = p->addPropTree("FilesRead", createPTree());
+            files = p->addPropTree("FilesRead");
         _noteFileRead(file, files);
     }
 }
@@ -6553,7 +6548,7 @@ void CLocalWorkUnit::addFile(const char *fileName, StringArray *clusters, unsign
     CriticalBlock block(crit);
     IPropertyTree *files = p->queryPropTree("Files");
     if (!files)
-        files = p->addPropTree("Files", createPTree());
+        files = p->addPropTree("Files");
     if (!clusters)
         ::addFile(files, fileName, NULL, usageCount, fileKind, graphOwner);
     else
@@ -6726,7 +6721,7 @@ void CLocalWorkUnit::addDiskUsageStats(__int64 _avgNodeUsage, unsigned _minNode,
         maxNodeUsage = stats->getPropInt64("@maxNodeUsage");
     else
     {
-        stats = p->addPropTree("DiskUsageStats", createPTree());
+        stats = p->addPropTree("DiskUsageStats");
         maxNodeUsage = 0;
     }
 
@@ -7115,9 +7110,9 @@ void CLocalWorkUnit::createGraph(const char * name, const char *label, WUGraphTy
 {
     CriticalBlock block(crit);
     if (!graphs.length())
-        p->addPropTree("Graphs", createPTree("Graphs"));
+        p->addPropTree("Graphs");
     IPropertyTree *r = p->queryPropTree("Graphs");
-    IPropertyTree *s = r->addPropTree("Graph", createPTree());
+    IPropertyTree *s = r->addPropTree("Graph");
     CLocalWUGraph *q = new CLocalWUGraph(*this, LINK(s));
     q->setName(name);
     q->setLabel(label);
@@ -7171,13 +7166,13 @@ void CLocalWUGraph::setLabel(const char *str)
 
 void CLocalWUGraph::setXGMML(const char *str)
 {
-    setXGMMLTree(createPTreeFromXMLString(str));
+    setXGMMLTree(createPTreeFromXMLString(str,ipt_lowmem));
 }
 
 void CLocalWUGraph::setXGMMLTree(IPropertyTree *_graph)
 {
     assertex(strcmp(_graph->queryName(), "graph")==0);
-    IPropertyTree *xgmml = p->setPropTree("xgmml", createPTree());
+    IPropertyTree *xgmml = p->setPropTree("xgmml");
     MemoryBuffer mb;
     _graph->serialize(mb);
     // Note - we could compress further but that would introduce compatibility concerns, so don't bother
@@ -7194,7 +7189,7 @@ static void expandAttributes(IPropertyTree & targetNode, IPropertyTree & progres
         const char *aName = aIter->queryName()+1;
         if (0 != stricmp("id", aName)) // "id" reserved.
         {
-            IPropertyTree *att = targetNode.addPropTree("att", createPTree());
+            IPropertyTree *att = targetNode.addPropTree("att");
             att->setProp("@name", aName);
             att->setProp("@value", aIter->queryValue());
         }
@@ -7233,7 +7228,7 @@ void CLocalWUGraph::mergeProgress(IPropertyTree &rootNode, IPropertyTree &progre
                     ForEach (*iter)
                     {
                         IPropertyTree &t = iter->query();
-                        IPropertyTree *att = graphEdge->addPropTree("att", createPTree());
+                        IPropertyTree *att = graphEdge->addPropTree("att");
                         att->setProp("@name", t.queryName());
                         att->setProp("@value", t.queryProp(NULL));
                     }
@@ -7276,7 +7271,7 @@ IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
         // daliadmin can retrospectively compress existing graphs, so need to check for all versions
         MemoryBuffer mb;
         if (p->getPropBin("xgmml/graphBin", mb))
-            graph.setown(createPTree(mb));
+            graph.setown(createPTree(mb, ipt_lowmem));
         else
             graph.setown(p->getBranch("xgmml/graph"));
         if (!graph)
@@ -7286,7 +7281,7 @@ IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
         return graph.getLink();
     else
     {
-        Owned<IPropertyTree> copy = createPTreeFromIPT(graph);
+        Owned<IPropertyTree> copy = createPTreeFromIPT(graph, ipt_lowmem);
         Owned<IConstWUGraphProgress> progress = owner.getGraphProgress(p->queryProp("@name"));
         if (progress)
         {
@@ -7421,7 +7416,7 @@ IStringVal& CLocalWUQuery::getQueryShortText(IStringVal &str) const
         text = p->queryProp("Text");
         if (isArchiveQuery(text))
         {
-            Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive);
+            Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
             const char * path = xml->queryProp("Query/@attributePath");
             if (path)
             {
@@ -7496,7 +7491,7 @@ void CLocalWUQuery::setQueryText(const char *text)
     bool isArchive = isArchiveQuery(text);
     if (isArchive)
     {
-        Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive);
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
         const char * path = xml->queryProp("Query/@attributePath");
         if (path)
         {
@@ -7527,9 +7522,9 @@ void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const
     CriticalBlock block(crit);
     loadAssociated();
     if (!associated.length())
-        p->addPropTree("Associated", createPTree("Associated"));
+        p->addPropTree("Associated");
     IPropertyTree *pl = p->queryPropTree("Associated");
-    IPropertyTree *s = pl->addPropTree("File", createPTree("File"));
+    IPropertyTree *s = pl->addPropTree("File");
     setEnum(s, "@type", type, queryFileTypes);
     s->setProp("@filename", name);
     s->setProp("@ip", ip);
@@ -9042,10 +9037,10 @@ extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(const char *xml)
 {
     Owned<CLocalWorkUnit> cw = new CLocalWorkUnit((ISecManager *) NULL, NULL);
     if (xml)
-        cw->loadPTree(createPTreeFromXMLString(xml));
+        cw->loadPTree(createPTreeFromXMLString(xml, ipt_lowmem));
     else
     {
-        Owned<IPropertyTree> p = createPTree("W_LOCAL");
+        Owned<IPropertyTree> p = createPTree("W_LOCAL", ipt_lowmem);
         p->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
         cw->loadPTree(p.getClear());
     }
@@ -9317,7 +9312,7 @@ IWorkflowItem * CLocalWorkUnit::addWorkflowItem(unsigned wfid, WFType type, WFMo
     workflowIteratorCached = false;
     IPropertyTree * s = p->queryPropTree("Workflow");
     if(!s)
-        s = p->addPropTree("Workflow", createPTree("Workflow"));
+        s = p->addPropTree("Workflow");
     return createWorkflowItem(s, wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor);
 }
 
@@ -9329,7 +9324,7 @@ IWorkflowItemIterator * CLocalWorkUnit::updateWorkflowItems()
     {
         IPropertyTree * s = p->queryPropTree("Workflow");
         if(!s)
-            s = p->addPropTree("Workflow", createPTree("Workflow"));
+            s = p->addPropTree("Workflow");
         workflowIterator.setown(createWorkflowItemIterator(s)); 
         workflowIteratorCached = true;
     }
@@ -9520,7 +9515,7 @@ unsigned CLocalWorkUnit::addLocalFileUpload(LocalFileUploadType type, char const
     CriticalBlock block(crit);
     IPropertyTree * s = p->queryPropTree("LocalFileUploads");
     if(!s)
-        s = p->addPropTree("LocalFileUploads", createPTree());
+        s = p->addPropTree("LocalFileUploads");
     unsigned id = s->numChildren();
     Owned<CLocalFileUpload> upload = new CLocalFileUpload(id, type, source, destination, eventTag);
     s->addPropTree("LocalFileUpload", upload->getTree());
@@ -10031,12 +10026,10 @@ public:
     {
         assertex(baseconn);
         Owned<IPropertyTree> root = baseconn->getRoot();
-        ensurePTree(root, "EventQueue");
-        Owned<IPropertyTree> eventQueue = root->getPropTree("EventQueue");
-        Owned<IPropertyTree> eventItem = createPTree();
+        IPropertyTree *eventQueue = ensurePTree(root, "EventQueue");
+        IPropertyTree *eventItem = eventQueue->addPropTree("Item");
         eventItem->setProp("@name", name);
         eventItem->setProp("@text", text);
-        eventQueue->addPropTree("Item", eventItem.getLink());
     }
 
     virtual void remove()
@@ -10246,7 +10239,7 @@ IPropertyTree * addNamedQuery(IPropertyTree * queryRegistry, const char * name,
 
     StringBuffer id;
     id.append(lcName).append(".").append(seq);
-    IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive);
+    IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive|ipt_lowmem);
     newEntry->setProp("@name", lcName);
     newEntry->setProp("@wuid", wuid);
     newEntry->setProp("@dll", dll);
@@ -10306,9 +10299,8 @@ void setQueryAlias(IPropertyTree * queryRegistry, const char * name, const char
     IPropertyTree * match = queryRegistry->queryPropTree(xpath);
     if (!match)
     {
-        IPropertyTree * newEntry = createPTree("Alias");
-        newEntry->setProp("@name", lcName);
-        match = queryRegistry->addPropTree("Alias", newEntry);
+        match = queryRegistry->addPropTree("Alias");
+        match->setProp("@name", lcName);
     }
     match->setProp("@id", value);
 }

common/workunit/wujobq.cpp

@@ -113,13 +113,6 @@ public:
         item->setProp("@enqueuedt",dts.str());
     }
 
-    IPropertyTree *createBranch(CJobQueueItem)
-    {
-        IPropertyTree *item = createPTree("Item");
-        assignBranch(item,this);
-        return item;
-    }
-
     const char *queryWUID()
     {
         return wu.get();
@@ -893,7 +886,7 @@ public:
                             StringBuffer cpath;
                             cpath.appendf("Queue[@name=\"%s\"]",qd->qname.get());
                             if (!proot->hasProp(cpath.str())) {
-                                IPropertyTree *pt = proot->addPropTree("Queue",createPTree("Queue"));
+                                IPropertyTree *pt = proot->addPropTree("Queue");
                                 pt->setProp("@name",qd->qname.get());
                                 pt->setProp("@state","active");
                                 pt->setPropInt("@count", 0);
@@ -1087,8 +1080,7 @@ public:
         IPropertyTree *ret = qd.root->queryPropTree(path.str());
         if (!ret)
         {
-            ret = createPTree("Client");
-            ret = qd.root->addPropTree("Client",ret);
+            ret = qd.root->addPropTree("Client");
             ret->setPropInt64("@session",sessionid);
             StringBuffer eps;
             ret->setProp("@node",queryMyNode()->endpoint().getUrlStr(eps).str());

dali/base/dacsds.cpp

@@ -1105,13 +1105,13 @@ IPropertyTree *CClientRemoteTree::collateData()
     {
         ChangeTree(IPropertyTree *donor=NULL) { ptree = LINK(donor); }
         ~ChangeTree() { ::Release(ptree); }
-        inline void createTree() { assertex(!ptree); ptree = createPTree(RESERVED_CHANGE_NODE); }
+        inline void createTree() { assertex(!ptree); ptree = createPTree(RESERVED_CHANGE_NODE, ipt_fast); }
         inline IPropertyTree *queryTree() { return ptree; }
         inline IPropertyTree *getTree() { return LINK(ptree); }
         inline IPropertyTree *queryCreateTree()
         {
             if (!ptree)
-                ptree = createPTree(RESERVED_CHANGE_NODE);
+                ptree = createPTree(RESERVED_CHANGE_NODE, ipt_fast);
             return ptree;
         }
     private:
@@ -1133,10 +1133,10 @@ IPropertyTree *CClientRemoteTree::collateData()
         Owned<IAttributeIterator> iter = getAttributes();
         if (iter->count())
         {
-            IPropertyTree *t = createPTree();
+            IPropertyTree *t = ct.queryTree()->addPropTree(ATTRCHANGE_TAG);
             ForEach(*iter)
                 t->setProp(iter->queryName(), queryProp(iter->queryName()));
-            ct.queryTree()->addPropTree(ATTRCHANGE_TAG, t);
+
         }
         ct.queryTree()->setPropBool("@new", true);
     }
@@ -1149,10 +1149,9 @@ IPropertyTree *CClientRemoteTree::collateData()
             {
                 ct.queryTree()->removeTree(ac);
                 Owned<IAttributeIterator> iter = ac->getAttributes();
-                IPropertyTree *t = createPTree();
+                IPropertyTree *t = ct.queryTree()->addPropTree(ATTRCHANGE_TAG);
                 ForEach(*iter)
                     t->setProp(iter->queryName(), queryProp(iter->queryName()));
-                ct.queryTree()->addPropTree(ATTRCHANGE_TAG, t);
             }
         }
     }
@@ -1952,9 +1951,9 @@ IPropertyTree &CClientSDSManager::queryProperties() const
         default:
             assertex(false);
     }
-    properties = createPTree(mb);
+    properties = createPTree(mb, ipt_lowmem);
     if (!properties->hasProp("Client"))
-        properties->setPropTree("Client", createPTree());
+        properties->setPropTree("Client");
     return *properties;
 }
 
@@ -1975,7 +1974,7 @@ IPropertyTree *CClientSDSManager::getXPaths(__int64 serverId, const char *xpath,
     switch (replyMsg)
     {
         case DAMP_SDSREPLY_OK:
-            return createPTree(mb);
+            return createPTree(mb, ipt_lowmem);
         case DAMP_SDSREPLY_EMPTY:
             return NULL;
         case DAMP_SDSREPLY_ERROR:
@@ -2014,7 +2013,7 @@ IPropertyTreeIterator *CClientSDSManager::getXPathsSortLimit(const char *baseXPa
         default:
             throwUnexpected();
     }
-    Owned<IPropertyTree> matchTree = createPTree(mb);
+    Owned<IPropertyTree> matchTree = createPTree(mb, ipt_lowmem);
     Owned<CRemoteConnection> conn = (CRemoteConnection *)connect(baseXPath, myProcessSession(), RTM_LOCK_READ, INFINITE);
     if (!conn)
         return createNullPTreeIterator();
@@ -2083,7 +2082,7 @@ IPropertyTreeIterator *CClientSDSManager::getElementsRaw(const char *xpath, INod
             mb.read(count);
             for (c=0; c<count; c++)
             {
-                Owned<IPropertyTree> item = createPTree(mb);
+                Owned<IPropertyTree> item = createPTree(mb, ipt_lowmem);
                 resultIterator->array.append(*LINK(item));
             }
             return LINK(resultIterator);

dali/base/dasds.ipp

@@ -109,7 +109,7 @@ class ChangeInfo : public CInterfaceOf<IInterface>
 {
     DECL_NAMEDCOUNT;
 public:
-    ChangeInfo(IPropertyTree &_owner) : owner(&_owner) { INIT_NAMEDCOUNT; tree.setown(createPTree(RESERVED_CHANGE_NODE)); }
+    ChangeInfo(IPropertyTree &_owner) : owner(&_owner) { INIT_NAMEDCOUNT; tree.setown(createPTree(RESERVED_CHANGE_NODE, ipt_fast)); }
     const IPropertyTree *queryOwner() const { return owner; }
     const void *queryFindParam() const { return &owner; }
 public: // data
@@ -276,26 +276,25 @@ public:
     void registerRenamed(IPropertyTree &owner, const char *newName, const char *oldName, unsigned pos, __int64 id)
     {
         ChangeInfo *changes = queryCreateChangeInfo(owner);
-        IPropertyTree *t = createPTree();
+        IPropertyTree *t = changes->tree->addPropTree(RENAME_TAG);
         t->setProp("@from", oldName);
         t->setProp("@to", newName);
         t->setPropInt64("@id", id);
 #ifdef SIBLING_MOVEMENT_CHECK
         t->setProp("@pos", pos);
 #endif
-        changes->tree->addPropTree(RENAME_TAG, t);
+
     }
 
     void registerDeleted(IPropertyTree &owner, const char *name, unsigned pos, __int64 id)
     {
         ChangeInfo *changes = queryCreateChangeInfo(owner);
-        IPropertyTree *t = createPTree();
+        IPropertyTree *t = changes->tree->addPropTree(DELETE_TAG);
         t->setProp("@name", name);
         t->setPropInt64("@id", id);
 #ifdef SIBLING_MOVEMENT_CHECK
         t->setPropInt("@pos", pos+1);
 #endif
-        changes->tree->addPropTree(DELETE_TAG, t);
     }
 
     virtual void registerAttrChange(IPropertyTree &owner, const char *attr)
@@ -305,7 +304,7 @@ public:
         if (t) t->removeProp(attr);
         t = changes->tree->queryPropTree("AC");
         if (!t)
-            t = changes->tree->addPropTree("AC", createPTree());
+            t = changes->tree->addPropTree("AC");
         t->setProp(attr, "");
     }
 
@@ -316,7 +315,7 @@ public:
         if (t) t->removeProp(attr);
         t = changes->tree->queryPropTree("AD");
         if (!t)
-            t = changes->tree->addPropTree("AD", createPTree());
+            t = changes->tree->addPropTree("AD");
         t->addProp(attr, "");
     }
 
@@ -325,7 +324,7 @@ public:
         ChangeInfo *changes = queryCreateChangeInfo(owner);
         IPropertyTree *t = changes->tree->queryPropTree(APPEND_TAG);
         if (!t)
-            t = changes->tree->setPropTree(APPEND_TAG, createPTree());
+            t = changes->tree->setPropTree(APPEND_TAG);
         t->setPropInt(NULL, l);
     }
 

roxie/ccd/ccdactivities.cpp

@@ -81,7 +81,7 @@ extern void putStatsValue(IPropertyTree *node, const char *statName, const char
         IPropertyTree *att = node->queryPropTree(xpath.str());
         if (!att)
         {
-            att = node->addPropTree("att", createPTree());
+            att = node->addPropTree("att");
             att->setProp("@name", statName);
         }
         att->setProp("@type", statType);
@@ -245,15 +245,13 @@ protected:
             const char *id = strchr(levelx, '[');
             if (!id)
             {
-                child = createPTree(levelx);
-                parent->addPropTree(levelx, child);
+                child = parent->addPropTree(levelx);
             }
             else
             {
                 StringBuffer elem;
                 elem.append(id-levelx, levelx);
-                child = createPTree(elem);
-                parent->addPropTree(elem, child);
+                child = parent->addPropTree(elem);
                 for (;;)
                 {
                     StringBuffer attr, val;

roxie/ccd/ccdcontext.cpp

@@ -2018,7 +2018,7 @@ protected:
             {
                 CriticalBlock b(contextCrit);
                 if (!persists)
-                    persists = createPTree();
+                    persists = createPTree(ipt_fast);
                 return *persists;
             }
         case ResultSequenceOnce:
@@ -2031,14 +2031,14 @@ protected:
             {
                 CriticalBlock b(contextCrit);
                 if (!temporaries)
-                    temporaries = createPTree();
+                    temporaries = createPTree(ipt_fast);
                 return *temporaries;
             }
         default:
             {
                 CriticalBlock b(contextCrit);
                 if (!rereadResults)
-                    rereadResults = createPTree();
+                    rereadResults = createPTree(ipt_fast);
                 return *rereadResults;
             }
         }
@@ -2687,8 +2687,8 @@ protected:
         if (workUnit->getXmlParams(wuParams, false).length())
         {
             // Merge in params from WU. Ones on command line take precedence though...
-            Owned<IPropertyTree> wuParamTree = createPTreeFromXMLString(wuParams.str(), ipt_caseInsensitive);
-            Owned<IPropertyTreeIterator> params = wuParamTree ->getElements("*");
+            Owned<IPropertyTree> wuParamTree = createPTreeFromXMLString(wuParams.str(), ipt_caseInsensitive|ipt_fast);
+            Owned<IPropertyTreeIterator> params = wuParamTree->getElements("*");
             ForEach(*params)
             {
                 IPropertyTree &param = params->query();
@@ -2735,7 +2735,7 @@ public:
         init();
         rowManager->setMemoryLimit(options.memoryLimit);
         workflow.setown(_factory->createWorkflowMachine(workUnit, true, logctx));
-        context.setown(createPTree(ipt_caseInsensitive));
+        context.setown(createPTree(ipt_caseInsensitive|ipt_fast));
     }
 
     CRoxieServerContext(IConstWorkUnit *_workUnit, const IQueryFactory *_factory, const ContextLogger &_logctx)
@@ -2745,7 +2745,7 @@ public:
         workUnit.set(_workUnit);
         rowManager->setMemoryLimit(options.memoryLimit);
         workflow.setown(_factory->createWorkflowMachine(workUnit, false, logctx));
-        context.setown(createPTree(ipt_caseInsensitive));
+        context.setown(createPTree(ipt_caseInsensitive|ipt_fast));
 
         //MORE: Use various debug settings to override settings:
         rowManager->setActivityTracking(workUnit->getDebugValueBool("traceRoxiePeakMemory", false));
@@ -3206,7 +3206,7 @@ public:
         else
         {
             if (!val)
-                val = ctx.addPropTree(name, createPTree());
+                val = ctx.addPropTree(name, createPTree(ipt_fast));
             val->setProp("@format", "deserialized");
             val->setPropInt("@id", resultStore.addResult(count, data, meta));
         }
@@ -3317,7 +3317,7 @@ public:
     virtual void setResultXml(const char *name, unsigned sequence, const char *xml)
     {
         CriticalBlock b(contextCrit);
-        useContext(sequence).setPropTree(name, createPTreeFromXMLString(xml, ipt_caseInsensitive));
+        useContext(sequence).setPropTree(name, createPTreeFromXMLString(xml, ipt_caseInsensitive|ipt_fast));
     }
 
     virtual void setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val)

roxie/ccd/ccddali.cpp

@@ -241,11 +241,11 @@ private:
 
     static void initCache()
     {
-        IPropertyTree *tree = createPTree("Roxie");
-        tree->addPropTree("QuerySets", createPTree("QuerySets"));
-        tree->addPropTree("PackageSets", createPTree("PackageSets"));
-        tree->addPropTree("PackageMaps", createPTree("PackageMaps"));
-        tree->addPropTree("Files", createPTree("Files"));
+        IPropertyTree *tree = createPTree("Roxie", ipt_lowmem);
+        tree->addPropTree("QuerySets");
+        tree->addPropTree("PackageSets");
+        tree->addPropTree("PackageMaps");
+        tree->addPropTree("Files");
         cache.setown(tree);
     }
 
@@ -256,7 +256,7 @@ private:
             StringBuffer cacheFileName(queryDirectory);
             cacheFileName.append(roxieStateName);
             if (checkFileExists(cacheFileName))
-                cache.setown(createPTreeFromXMLFile(cacheFileName));
+                cache.setown(createPTreeFromXMLFile(cacheFileName, ipt_lowmem));
             else
                 initCache();
         }
@@ -293,7 +293,7 @@ private:
                 if (conn)
                 {
                     Owned <IPropertyTree> daliTree = conn->getRoot();
-                    localTree.setown(createPTreeFromIPT(daliTree));
+                    localTree.setown(createPTreeFromIPT(daliTree, ipt_lowmem));
                 }
                 writeCache(xpath, path, localTree);
                 return localTree.getClear();
@@ -442,7 +442,7 @@ public:
         Owned<IPropertyTree> ret = loadDaliTree("QuerySets/QuerySet", id);
         if (!ret)
         {
-            ret.setown(createPTree("QuerySet"));
+            ret.setown(createPTree("QuerySet", ipt_lowmem));
             ret->setProp("@id", id);
         }
         return ret.getClear();
@@ -453,7 +453,7 @@ public:
         Owned<IPropertyTree> ret = loadDaliTree("PackageSets", NULL);
         if (!ret)
         {
-            ret.setown(createPTree("PackageSets"));
+            ret.setown(createPTree("PackageSets", ipt_lowmem));
         }
         return ret.getClear();
     }
@@ -476,7 +476,7 @@ public:
         Owned<IPropertyTree> ret = loadDaliTree("PackageMaps/PackageMap", id);
         if (!ret)
         {
-            ret.setown(createPTree("PackageMap"));
+            ret.setown(createPTree("PackageMap", ipt_lowmem));
             ret->setProp("@id", id);
         }
         return ret.getClear();

roxie/ccd/ccddebug.cpp

@@ -410,7 +410,7 @@ public:
                                 IPropertyTree *att = edge.queryPropTree("att[@name=\"_roxieStarted\"]");
                                 if (!att)
                                 {
-                                    att = edge.addPropTree("att", createPTree());
+                                    att = edge.addPropTree("att");
                                     att->setProp("@name", "_roxieStarted");
                                 }
                                 else
@@ -440,7 +440,7 @@ public:
                                     IPropertyTree *att = edge.queryPropTree("att[@name=\"_roxieStarted\"]");
                                     if (!att)
                                     {
-                                        att = edge.addPropTree("att", createPTree());
+                                        att = edge.addPropTree("att");
                                         att->setProp("@name", "_roxieStarted");
                                     }
                                     else
@@ -1586,7 +1586,7 @@ public:
                 DebugRequestLookupActivityByEdgeId request(proxyId, edgeId);
                 CommonXmlWriter reply(0);
                 sendProxyRequest(&reply, request);
-                Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str());
+                Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str(), ipt_fast);
                 if (response)
                 {
                     memsize_t proxyId = (memsize_t) response->getPropInt64("@proxyId", 0);
@@ -1679,7 +1679,7 @@ public:
         reply.outputBeginNested("Counts", true);
         sendProxyRequest(&reply, request);
         reply.outputEndNested("Counts"); // strange way to do it...
-        Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str());
+        Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str(), ipt_fast);
         if (response)
         {
             Owned<IPropertyTreeIterator> edges = response->getElements("edge");

roxie/ccd/ccdfile.cpp

@@ -1795,7 +1795,7 @@ public:
                     addFile(sub.queryLogicalName(), fDesc.getClear(), remoteFDesc.getClear());
                 }
                 // We have to clone the properties since we don't want to keep the superfile locked
-                properties.setown(createPTreeFromIPT(&dFile->queryAttributes()));
+                properties.setown(createPTreeFromIPT(&dFile->queryAttributes(), ipt_lowmem));
                 if (!isDynamic && !lockSuperFiles)
                 {
                     notifier.setown(daliHelper->getSuperFileSubscription(lfn, this));
@@ -2205,7 +2205,7 @@ public:
         assertex(file->exists());
         offset_t size = file->size();
         Owned<IFileDescriptor> fdesc = createFileDescriptor();
-        Owned<IPropertyTree> pp = createPTree("Part");
+        Owned<IPropertyTree> pp = createPTree("Part", ipt_lowmem);
         pp->setPropInt64("@size",size);
         pp->setPropBool("@local", true);
         fdesc->setPart(0, queryMyNode(), localFileName, pp);
@@ -2372,7 +2372,7 @@ public:
                 bool propertiesPresent;
                 serverData.read(propertiesPresent);
                 if (propertiesPresent)
-                    properties.setown(createPTree(serverData));
+                    properties.setown(createPTree(serverData, ipt_lowmem));
             }
             else
                 throw MakeStringException(ROXIE_CALLBACK_ERROR, "Failed to get response from server for dynamic file callback");

roxie/ccd/ccdlistener.cpp

@@ -173,7 +173,7 @@ public:
         StringBuffer lockQuery;
         lockQuery.appendf("<control:childlock thisEndpoint='%d' parent='%d'/>", activeIdxes.item(idx), myEndpoint);
         doChildQuery(idx, lockQuery.str(), lockReply);
-        Owned<IPropertyTree> lockResult = createPTreeFromXMLString(lockReply.str(), ipt_caseInsensitive);
+        Owned<IPropertyTree> lockResult = createPTreeFromXMLString(lockReply.str(), ipt_caseInsensitive|ipt_fast);
         int lockCount = lockResult->getPropInt("Lock", 0);
         if (lockCount)
         {
@@ -340,7 +340,7 @@ public:
 
     void doLockChild(const char *queryText, StringBuffer &reply)
     {
-        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText);
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText,ipt_fast);
         doLockChild(xml, queryText, reply);
     }
 
@@ -395,7 +395,7 @@ public:
 
     void doControlQuery(SocketEndpoint &ep, const char *queryText, StringBuffer &reply)
     {
-        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText); // control queries are case sensitive
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText,ipt_fast); // control queries are case sensitive
         doControlQuery(ep, xml, queryText, reply);
     }
 
@@ -413,7 +413,7 @@ public:
             mergeType=CascadeMergeQueries;
         Owned<IPropertyTree> mergedReply;
         if (mergeType!=CascadeMergeNone)
-            mergedReply.setown(createPTree("Endpoint"));
+            mergedReply.setown(createPTree("Endpoint",ipt_fast));
 
         class casyncfor: public CAsyncFor
         {
@@ -444,7 +444,7 @@ public:
                 {
                     StringBuffer childReply;
                     parent->doChildQuery(i, queryText, childReply);
-                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(childReply);
+                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(childReply,ipt_fast);
                     if (!replyXML)
                     {
                         StringBuffer err;
@@ -490,7 +490,7 @@ public:
                 CriticalBlock cb(crit);
                 if (mergedReply)
                 {
-                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(myReply);
+                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(myReply,ipt_fast);
                     if (mergeType == CascadeMergeStats)
                         mergeStats(mergedReply, replyXML);
                     else if (mergeType == CascadeMergeQueries)

roxie/ccd/ccdmain.cpp

@@ -282,9 +282,7 @@ void getAccessList(const char *aclName, const IPropertyTree *topology, IProperty
     xpath.append("ACL[@name='").append(aclName).append("']");
     if (aclInfo->queryPropTree(xpath))
         throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - recursive ACL definition of %s", aclName);
-    Owned<IPropertyTree> X = createPTree("ACL");
-    X->setProp("@name", aclName);
-    aclInfo->addPropTree("ACL", X.getClear());
+    aclInfo->addPropTree("ACL")->setProp("@name", aclName);
 
     Owned<IPropertyTree> acl = topology->getPropTree(xpath.str());
     if (!acl)
@@ -314,10 +312,9 @@ void addSlaveChannel(unsigned channel, unsigned level)
     xpath.appendf("RoxieSlaveProcess[@channel=\"%d\"]", channel);
     if (ccdChannels->hasProp(xpath.str()))
         throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - channel %d repeated", channel);
-    IPropertyTree *ci = createPTree("RoxieSlaveProcess");
+    IPropertyTree *ci = ccdChannels->addPropTree("RoxieSlaveProcess");
     ci->setPropInt("@channel", channel);
     ci->setPropInt("@subChannel", numSlaves[channel]);
-    ccdChannels->addPropTree("RoxieSlaveProcess", ci);
 }
 
 void addChannel(unsigned nodeNumber, unsigned channel, unsigned level)
@@ -499,7 +496,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
     Thread::setDefaultStackSize(0x10000);   // NB under windows requires linker setting (/stack:)
 #endif
     srand( (unsigned)time( NULL ) );
-    ccdChannels = createPTree("Channels");
+    ccdChannels = createPTree("Channels", ipt_lowmem);
 
     char currentDirectory[_MAX_DIR];
     if (!getcwd(currentDirectory, sizeof(currentDirectory)))
@@ -524,7 +521,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         if (checkFileExists(topologyFile.str()))
         {
             DBGLOG("Loading topology file %s", topologyFile.str());
-            topology = createPTreeFromXMLFile(topologyFile.str());
+            topology = createPTreeFromXMLFile(topologyFile.str(), ipt_lowmem);
             saveTopology();
         }
         else
@@ -539,6 +536,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 " <RoxieFarmProcess/>"
                 " <RoxieServerProcess netAddress='.'/>"
                 "</RoxieTopology>"
+                , ipt_lowmem
                 );
             int port = globals->getPropInt("--port", 9876);
             topology->setPropInt("RoxieFarmProcess/@port", port);
@@ -1141,7 +1139,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 const char *aclName = roxieFarm.queryProp("@aclName");
                 if (aclName && *aclName)
                 {
-                    Owned<IPropertyTree> aclInfo = createPTree("AccessInfo");
+                    Owned<IPropertyTree> aclInfo = createPTree("AccessInfo", ipt_lowmem);
                     getAccessList(aclName, topology, aclInfo);
                     Owned<IPropertyTreeIterator> accesses = aclInfo->getElements("Access");
                     ForEach(*accesses)
@@ -1166,7 +1164,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 roxieServer->start();
             }
             writeSentinelFile(sentinelFile);
-            DBGLOG("Waiting for queries");
+            DBGLOG("Waiting for queries LPT=%u APT=%u", queryNumLocalTrees(), queryNumAtomTrees());
             if (pingInterval)
                 startPingTimer();
             LocalIAbortHandler abortHandler(waiter);

roxie/ccd/ccdprotocol.cpp

@@ -816,7 +816,7 @@ public:
             return;
         if (fmt==MarkupFmt_JSON)
         {
-            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(content);
+            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(content, ipt_fast);
             if (name && *name)
                 appendXMLOpenTag(xml, name);
             toXML(convertPT, xml, 0, 0);
@@ -1068,7 +1068,7 @@ public:
         StringBuffer json;
         if (mlFmt==MarkupFmt_XML)
         {
-            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(StringBuffer("<Control>").append(content).append("</Control>"));
+            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(StringBuffer("<Control>").append(content).append("</Control>"), ipt_fast);
             toJSON(convertPT, json, 0, 0);
             content = json.str();
         }
@@ -1179,7 +1179,7 @@ public:
         StringBuffer xml;
         if (mlFmt==MarkupFmt_JSON)
         {
-            Owned<IPropertyTree> convertPT = createPTreeFromJSONString(content);
+            Owned<IPropertyTree> convertPT = createPTreeFromJSONString(content, ipt_fast);
             toXML(convertPT, xml, 0, 0);
             content = xml.str();
         }
@@ -1723,7 +1723,7 @@ readAnother:
                     client->setHttpMode(queryName, false, httpHelper);
 
                 bool aclupdate = strieq(queryName, "aclupdate"); //ugly
-                byte iptFlags = aclupdate ? ipt_caseInsensitive : 0;
+                byte iptFlags = aclupdate ? ipt_caseInsensitive|ipt_fast : ipt_fast;
 
                 createQueryPTree(queryPT, httpHelper, rawText, iptFlags, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces), queryName);
 
@@ -1768,7 +1768,7 @@ readAnother:
                 readFlags |= (whitespace == WhiteSpaceHandling::Strip ? ptr_ignoreWhiteSpace : ptr_none);
                 try
                 {
-                    createQueryPTree(queryPT, httpHelper, rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)readFlags, queryName);
+                    createQueryPTree(queryPT, httpHelper, rawText.str(), ipt_caseInsensitive|ipt_fast, (PTreeReaderOptions)readFlags, queryName);
                 }
                 catch (IException *E)
                 {
@@ -1840,7 +1840,7 @@ readAnother:
                             Owned<IPropertyTreeIterator> reqIter = queryPT->getElements(reqIterString.str());
                             ForEach(*reqIter)
                             {
-                                IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive);
+                                IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive|ipt_fast);
                                 Owned<IPropertyTreeIterator> iter = reqIter->query().getElements("*");
                                 ForEach(*iter)
                                 {
@@ -1851,7 +1851,7 @@ readAnother:
                         }
                         else
                         {
-                            IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive);
+                            IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive|ipt_fast);
                             Owned<IPropertyTreeIterator> iter = queryPT->getElements("*");
                             ForEach(*iter)
                             {

roxie/ccd/ccdquery.cpp

@@ -170,9 +170,9 @@ IPropertyTree * addXrefInfo(IPropertyTree &reply, const char *section, const cha
     VStringBuffer xpath("%s[@name='%s']", section, name);
     if (!reply.hasProp(xpath))
     {
-        IPropertyTree *info = createPTree(section, 0);
+        IPropertyTree *info = reply.addPropTree(section);
         info->setProp("@name", name);
-        return reply.addPropTree(section, info);
+        return info;
     }
     return NULL;
 }
@@ -236,7 +236,7 @@ public:
         CriticalBlock b(onceCrit);
         if (!onceContext)
         {
-            onceContext.setown(createPTree());
+            onceContext.setown(createPTree(ipt_lowmem));
             onceResultStore.setown(createDeserializedResultStore());
             Owned <IRoxieServerContext> ctx = createOnceServerContext(factory, logctx);
             onceManager.set(&ctx->queryRowManager());
@@ -1297,7 +1297,7 @@ public:
 
     void getGraphStats(StringBuffer &reply, const IPropertyTree &thisGraph) const
     {
-        Owned<IPropertyTree> graph = createPTreeFromIPT(&thisGraph);
+        Owned<IPropertyTree> graph = createPTreeFromIPT(&thisGraph, ipt_lowmem);
         Owned<IPropertyTreeIterator> edges = graph->getElements(".//edge");
         ForEach(*edges)
         {
@@ -1325,7 +1325,7 @@ public:
     virtual IPropertyTree* cloneQueryXGMML() const
     {
         assertex(dll && dll->queryWorkUnit());
-        Owned<IPropertyTree> tree = createPTree("Query");
+        Owned<IPropertyTree> tree = createPTree("Query", ipt_lowmem);
         Owned<IConstWUGraphIterator> graphs = &dll->queryWorkUnit()->getGraphs(GraphTypeActivities);
         SCMStringBuffer graphNameStr;
         ForEach(*graphs)
@@ -1333,12 +1333,9 @@ public:
             graphs->query().getName(graphNameStr);
             const char *graphName = graphNameStr.s.str();
             Owned<IPropertyTree> graphXgmml = graphs->query().getXGMMLTree(false);
-            IPropertyTree *newGraph = createPTree();
+            IPropertyTree *newGraph = tree->addPropTree("Graph");
             newGraph->setProp("@id", graphName);
-            IPropertyTree *newXGMML = createPTree();
-            newXGMML->addPropTree("graph", graphXgmml.getLink());
-            newGraph->addPropTree("xgmml", newXGMML);
-            tree->addPropTree("Graph", newGraph);
+            newGraph->addPropTree("xgmml")->addPropTree("graph", graphXgmml.getLink());
         }
         return tree.getClear();
     }
@@ -1383,7 +1380,7 @@ public:
     }
     virtual void getQueryInfo(StringBuffer &reply, bool full, IArrayOf<IQueryFactory> *slaveQueries, const IRoxieContextLogger &logctx) const
     {
-        Owned<IPropertyTree> xref = createPTree("Query", 0);
+        Owned<IPropertyTree> xref = createPTree("Query", ipt_fast);
         xref->setProp("@id", id);
         if (suspended())
         {

roxie/ccd/ccdserver.cpp

@@ -3202,7 +3202,7 @@ void throwRemoteException(IMessageUnpackCursor *extra)
     {
         char *xml = (char *) extra->getNext(*rowlen);
         ReleaseRoxieRow(rowlen);
-        Owned<IPropertyTree> p = createPTreeFromXMLString(xml);
+        Owned<IPropertyTree> p = createPTreeFromXMLString(xml, ipt_fast);
         ReleaseRoxieRow(xml);
         unsigned code = p->getPropInt("Code", 0);
         const char *msg = p->queryProp("Message");
@@ -11940,7 +11940,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
                 throw MakeStringException(0, "Invalid name %s in user metadata for index %s (not legal XML element name)", name.str(), fname.get());
             }
             if(!metadata)
-                metadata.setown(createPTree("metadata"));
+                metadata.setown(createPTree("metadata", ipt_fast));
             metadata->setProp(name.str(), value.str());
         }
     }
@@ -11948,7 +11948,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
     void buildLayoutMetadata(Owned<IPropertyTree> & metadata)
     {
         if(!metadata)
-            metadata.setown(createPTree("metadata"));
+            metadata.setown(createPTree("metadata", ipt_fast));
         metadata->setProp("_record_ECL", helper.queryRecordECL());
 
         void * layoutMetaBuff;
@@ -12101,7 +12101,7 @@ public:
         //properties of the first file part.
         Owned<IPropertyTree> attrs;
         if(clusterHandler)
-            attrs.setown(createPTree("Part"));  // clusterHandler is going to set attributes
+            attrs.setown(createPTree("Part", ipt_fast));  // clusterHandler is going to set attributes
         else
         {
             // add cluster
@@ -15570,7 +15570,7 @@ public:
     {
         if (numUses > 1)
         {
-            Owned<IPropertyTree> splitterNode = createPTree();
+            Owned<IPropertyTree> splitterNode = createPTree(ipt_fast);
             factory.setown(createRoxieServerThroughSpillActivityFactory(sourceAct->queryFactory()->queryQueryFactory(), createGraphOutputSplitter, numUses, *splitterNode));
             IRoxieServerActivity *splitter = factory->createActivity(ctx, NULL);
             splitter->onCreate(NULL);

roxie/ccd/ccdsnmp.cpp

@@ -953,7 +953,7 @@ public:
     }
     static IPropertyTree *getAllQueryStats(bool includeQueries, time_t from, time_t to)
     {
-        Owned<IPTree> result = createPTree("QueryStats");
+        Owned<IPTree> result = createPTree("QueryStats", ipt_fast);
         if (includeQueries)
         {
             SpinBlock b(queryStatsCrit);
@@ -994,7 +994,7 @@ public:
     {
         time_t timeNow;
         time(&timeNow);
-        Owned<IPropertyTree> result = createPTree("Query");
+        Owned<IPropertyTree> result = createPTree("Query", ipt_fast);
         result->setProp("@id", queryName);
         if (expirySeconds && difftime(timeNow, from) <= expirySeconds)
         {

roxie/ccd/ccdstate.cpp

@@ -1374,13 +1374,13 @@ public:
             {
                 StringBuffer freply;
                 serverManager->getStats(queryId, graphName, freply, logctx);
-                Owned<IPropertyTree> stats = createPTreeFromXMLString(freply.str());
+                Owned<IPropertyTree> stats = createPTreeFromXMLString(freply.str(), ipt_fast);
                 for (unsigned channel = 0; channel < numChannels; channel++)
                     if (slaveManagers->item(channel))
                     {
                         StringBuffer sreply;
                         slaveManagers->item(channel)->getStats(queryId, graphName, sreply, logctx);
-                        Owned<IPropertyTree> cstats = createPTreeFromXMLString(sreply.str());
+                        Owned<IPropertyTree> cstats = createPTreeFromXMLString(sreply.str(), ipt_fast);
                         mergeStats(stats, cstats, 1);
                     }
                 toXML(stats, reply);
@@ -1523,7 +1523,7 @@ public:
     virtual void load(bool forceReload)
     {
         hash64_t newHash = numChannels;
-        Owned<IPropertyTree> newQuerySet = createPTree("QuerySet");
+        Owned<IPropertyTree> newQuerySet = createPTree("QuerySet", ipt_lowmem);
         newQuerySet->setProp("@name", "_standalone");
         newQuerySet->addPropTree("Query", standaloneDll.getLink());
         Owned<CRoxieSlaveQuerySetManagerSet> newSlaveManagers = new CRoxieSlaveQuerySetManagerSet(numChannels, querySet);
@@ -1559,7 +1559,7 @@ public:
     : stateHash(0), daliHelper(_daliHelper)
     {
         Owned<IPropertyTree> standAloneDllTree;
-        standAloneDllTree.setown(createPTree("Query"));
+        standAloneDllTree.setown(createPTree("Query", ipt_lowmem));
         standAloneDllTree->setProp("@id", "roxie");
         standAloneDllTree->setProp("@dll", standAloneDll->queryDll()->queryName());
         Owned<CRoxieQueryPackageManager> qpm = new CStandaloneQueryPackageManager(numChannels, querySet, LINK(&queryEmptyRoxiePackageMap()), standAloneDllTree.getClear());

system/jlib/jdebug.cpp

@@ -2612,6 +2612,8 @@ public:
         if(!mode) return;
         unsigned memused=0;
         unsigned memtot=0;
+        str.appendf("LPT=%u ", queryNumLocalTrees());
+        str.appendf("APT=%u ", queryNumAtomTrees());
         if(mode & PerfMonProcMem)
         {
             if (!outofhandles)

system/jlib/jptree.cpp

@@ -120,23 +120,25 @@ MODULE_INIT(INIT_PRIORITY_JPTREE)
     AttrStrUnionWithTable::roNameTable = new RONameTable;
     initializeRoTable();
 #endif
-	keyTable = new AtomRefTable;
-	keyTableNC = new AtomRefTable(true);
-	attrHT = new CAttrValHashTable;
+    keyTable = new AtomRefTable;
+    keyTableNC = new AtomRefTable(true);
+    attrHT = new CAttrValHashTable;
     return true;
 }
 
 MODULE_EXIT()
 {
     nullPTreeIterator->Release();
-	delete attrHT;
-	keyTable->Release();
-	keyTableNC->Release();
-	free(freelist);
-	freelist = NULL;
+    delete attrHT;
+    keyTable->Release();
+    keyTableNC->Release();
+#ifdef USE_READONLY_ATOMTABLE
+    delete AttrStrUnionWithTable::roNameTable;
+#endif
+    free(freelist);
+    freelist = NULL;
 }
 
-
 static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
 {
     IPropertyTree *l = (IPropertyTree *) *ll;
@@ -2908,14 +2910,22 @@ AttrValue *PTree::getNextAttribute(AttrValue *cur) const
 
 // LocalPTree
 
-LocalPTree::LocalPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags, _value, _children)
+static RelaxedAtomic<unsigned> numLocalTrees;
+unsigned queryNumLocalTrees()
+{
+    return numLocalTrees;
+}
+
+LocalPTree::LocalPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_fast, _value, _children)
 {
     if (_name)
         setName(_name);
+    numLocalTrees++;
 }
 
 LocalPTree::~LocalPTree()
 {
+    numLocalTrees--;
     name.destroy();
     if (!attrs)
         return;
@@ -2998,14 +3008,22 @@ std::atomic<__int64> AttrStrAtom::maxsize { 0 };
 
 ///////////////////
 
-CAtomPTree::CAtomPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags, _value, _children)
+static RelaxedAtomic<unsigned> numAtomTrees;
+unsigned queryNumAtomTrees()
+{
+    return numAtomTrees;
+}
+
+CAtomPTree::CAtomPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_lowmem, _value, _children)
 {
+    numAtomTrees++;
     if (_name)
         setName(_name);
 }
 
 CAtomPTree::~CAtomPTree()
 {
+    numAtomTrees--;
     bool nc = isnocase();
     HashKeyElement *name_ptr = name.getPtr();
     if (name_ptr)
@@ -3143,11 +3161,19 @@ void CAtomPTree::setAttribute(const char *key, const char *val)
     {
         if (streq(v->value.get(), val))
             return;
-        CriticalBlock block(hashcrit);
-        if (v->value.isPtr())
-            attrHT->removeval(v->value.getPtr());
+        AttrStr * goer = v->value.getPtr();
         if (!v->value.set(val))
+        {
+            CriticalBlock block(hashcrit);
+            if (goer)
+                attrHT->removeval(goer);
             v->value.setPtr(attrHT->addval(val));
+        }
+        else if (goer)
+        {
+            CriticalBlock block(hashcrit);
+            attrHT->removeval(goer);
+        }
     }
     else
     {

system/jlib/jptree.hpp

@@ -61,6 +61,10 @@ typedef unsigned IPTIteratorCodes;
 #define iptiter_remote 0x02
 #define iptiter_remoteget 0x06
 #define iptiter_remotegetbranch 0x0e
+
+extern jlib_decl unsigned queryNumLocalTrees();
+extern jlib_decl unsigned queryNumAtomTrees();
+
 interface jlib_decl IPropertyTree : extends serializable
 {
     virtual bool hasProp(const char *xpath) const = 0;
@@ -98,6 +102,9 @@ interface jlib_decl IPropertyTree : extends serializable
     virtual IPropertyTree *setPropTree(const char *xpath, IPropertyTree *val) = 0;
     virtual IPropertyTree *addPropTree(const char *xpath, IPropertyTree *val) = 0;
 
+    virtual IPropertyTree *setPropTree(const char *xpath) = 0;
+    virtual IPropertyTree *addPropTree(const char *xpath) = 0;
+
     virtual bool removeProp(const char *xpath) = 0;
     virtual bool removeTree(IPropertyTree *child) = 0;
     virtual aindex_t queryChildIndex(IPropertyTree *child) = 0;
@@ -215,7 +222,6 @@ jlib_decl IPropertyTree *createPTreeFromXMLString(const char *xml, byte flags=ip
 jlib_decl IPropertyTree *createPTreeFromXMLString(unsigned len, const char *xml, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromXMLFile(const char *filename, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromIPT(const IPropertyTree *srcTree, ipt_flags flags=ipt_none);
-
 jlib_decl IPropertyTree *createPTreeFromJSONString(const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 

system/jlib/jptree.ipp

@@ -384,7 +384,6 @@ struct PtrStrUnion
         {
 #ifdef LITTLE_ENDIAN
             char flag;
-#endif
             union
             {
                 char chars[sizeof(PTR *)-1];
@@ -392,10 +391,18 @@ struct PtrStrUnion
                 {
                     int8_t idx1;
                     int16_t idx2;
-                    int32_t idx4;
                 };
             };
-#ifndef LITTLE_ENDIAN
+#else
+            union
+            {
+                char chars[sizeof(PTR *)-1];
+                struct
+                {
+                    int16_t idx2;
+                    int8_t idx1;
+                };
+            };
             char flag;
 #endif
         };
@@ -426,7 +433,7 @@ struct PtrStrUnion
     {
         if (key)
         {
-            size32_t l = strlen(key);
+            size32_t l = strnlen(key, sizeof(PTR *));  // technically sizeof(PTR)-1 would do, but I suspect 8 bytes is actually more optimal to search than 7
             if (l <= sizeof(PTR *)-2)
             {
                 flag=1;
@@ -487,7 +494,7 @@ struct AttrStrUnionWithTable : public AttrStrUnion
     inline const char *get() const
     {
         if (!isPtr() && flag==3)
-            return roNameTable->getIndex(idx4)->str_DO_NOT_USE_DIRECTLY;  // Should probably rename this back now!
+            return roNameTable->getIndex(idx2)->str_DO_NOT_USE_DIRECTLY;  // Should probably rename this back now!
         return AttrStrUnion::get();
     }
     bool set(const char *key)
@@ -499,8 +506,9 @@ struct AttrStrUnionWithTable : public AttrStrUnion
             unsigned idx = roNameTable->findIndex(key, AttrStrC::getHash(key));
             if (idx != (unsigned) -1)
             {
+                assert(idx <= 0xffff);
                 flag = 3;
-                idx4 = idx;
+                idx2 = idx;
                 return true;
             }
         }
@@ -597,6 +605,8 @@ public:
     virtual IPropertyTree *queryBranch(const char *xpath) const override { return queryPropTree(xpath); }
     virtual IPropertyTree *setPropTree(const char *xpath, IPropertyTree *val) override;
     virtual IPropertyTree *addPropTree(const char *xpath, IPropertyTree *val) override;
+    virtual IPropertyTree *setPropTree(const char *xpath) override { return setPropTree(xpath, create()); }
+    virtual IPropertyTree *addPropTree(const char *xpath) override { return addPropTree(xpath, create()); }
     virtual bool removeTree(IPropertyTree *child) override;
     virtual bool removeProp(const char *xpath) override;
     virtual aindex_t queryChildIndex(IPropertyTree *child) override;
@@ -745,10 +755,10 @@ public:
     const char *queryName() const override;
     virtual unsigned queryHash() const override
     {
-        const char *name = queryName();
-        assert(name);
-        size32_t nl = strlen(name);
-        return isnocase() ? hashnc((const byte *)name, nl, 0): hashc((const byte *)name, nl, 0);
+        const char *myname = queryName();
+        assert(myname);
+        size32_t nl = strlen(myname);
+        return isnocase() ? hashnc((const byte *)myname, nl, 0): hashc((const byte *)myname, nl, 0);
     }
     virtual void setName(const char *_name) override;
     virtual bool isEquivalent(IPropertyTree *tree) const override { return (nullptr != QUERYINTERFACE(tree, LocalPTree)); }

system/jlib/jutil.cpp

@@ -2366,7 +2366,7 @@ IPropertyTree *getHPCCEnvironment()
         {
             Owned<IFileIO> fileio = file->open(IFOread);
             if (fileio)
-                return createPTree(*fileio);
+                return createPTree(*fileio, ipt_lowmem);
         }
     }
     return NULL;

testing/unittests/unittests.cpp

@@ -295,6 +295,7 @@ class PtreeThreadingTest : public CppUnit::TestFixture
     }
     void _testContention(byte flags)
     {
+        enum ContentionMode { max_contention, some_contention, min_contention, some_control, min_control };
         class casyncfor: public CAsyncFor
         {
             volatile int v;
@@ -303,12 +304,12 @@ class PtreeThreadingTest : public CppUnit::TestFixture
                 v++;
             }
             byte flags = ipt_none;
-            int mode = 0;
+            ContentionMode mode = max_contention;
             int iterations = 0;
             const char *desc = nullptr;
 
         public:
-            casyncfor(const char *_desc, byte _flags, int _mode, int _iter)
+            casyncfor(const char *_desc, byte _flags, ContentionMode _mode, int _iter)
             : flags(_flags), mode(_mode), iterations(_iter), desc(_desc)
             {
             };
@@ -319,14 +320,14 @@ class PtreeThreadingTest : public CppUnit::TestFixture
                 unsigned elapsed = msTick()-start;
                 double looptime = (elapsed * 1.0) / (iterations*num);
                 if (mode < 3)
-                    DBGLOG("%s (%s) test completed in %d ms (%f ms/iter)", desc, flags & ipt_fast ? "fast" : "lowmem", elapsed, looptime-overhead);
+                    DBGLOG("%s (%s) test completed in %u ms (%f ms/iter)", desc, flags & ipt_fast ? "fast" : "lowmem", elapsed, looptime-overhead);
                 return looptime;
             }
             void Do(unsigned i)
             {
                 for (unsigned i = 0; i < iterations; i++)
                 {
-                    Owned<IPropertyTree> p = mode >= 3 ? nullptr : createPTreeFromXMLString(false ? "<helloxx there='1'/>" :
+                    Owned<IPropertyTree> p = mode >= some_control ? nullptr : createPTreeFromXMLString(
                             "<W_LOCAL buildVersion='community_6.0.0-trunk0Debug[heads/cass-wu-part3-0-g10b954-dirty]'"
                             "         cloneable='1'"
                             "         clusterName=''"
@@ -528,21 +529,21 @@ class PtreeThreadingTest : public CppUnit::TestFixture
                     , flags);
                     switch(mode)
                     {
-                    case 1: case 3: for (int j = 0; j < 100000; j++) donothing(); break;
-                    case 2: case 4: for (int j = 0; j < 1000000; j++) donothing(); break;
+                    case some_contention: case some_control: for (int j = 0; j < 100000; j++) donothing(); break;
+                    case min_contention: case min_control: for (int j = 0; j < 1000000; j++) donothing(); break;
                     }
                 }
             }
-        } max("maxContention",flags,0,1000),
-          some("someContention",flags,1,200),
-          min("minContention",flags,2,200),
-          csome("control some",flags,3,200),
-          cmin("control min",flags,4,200),
-          seq("single",flags,0,1000);
-          max.For(8,8);
-          some.For(8,8,csome.For(8,8));
-          min.For(8,8,cmin.For(8,8));
-          seq.For(8,1);
+        } max("maxContention",flags,max_contention,1000),
+          some("someContention",flags,some_contention,200),
+          min("minContention",flags,min_contention,200),
+          csome("control some",flags,some_control,200),
+          cmin("control min",flags,min_control,200),
+          seq("single",flags,max_contention,1000);
+        max.For(8,8);
+        some.For(8,8,csome.For(8,8));
+        min.For(8,8,cmin.For(8,8));
+        seq.For(8,1);
     }
 };