Merge pull request #10007 from richardkchapman/lowmem-ptree

HPCC-17625 JPTree memory usage and contention issues

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/environment/environment.cpp

@@ -463,14 +463,14 @@ public:
 
     virtual IEnvironment * loadLocalEnvironmentFile(const char * filename)
     {
-        Owned<IPropertyTree> ptree = createPTreeFromXMLFile(filename);
+        Owned<IPropertyTree> ptree = createPTreeFromXMLFile(filename, ipt_lowmem);
         Owned<CLocalEnvironment> pLocalEnv = new CLocalEnvironment(nullptr, ptree);
         return new CLockedEnvironment(pLocalEnv);
     }
 
     virtual IEnvironment * loadLocalEnvironment(const char * xml)
     {
-        Owned<IPropertyTree> ptree = createPTreeFromXMLString(xml);
+        Owned<IPropertyTree> ptree = createPTreeFromXMLString(xml, ipt_lowmem);
         Owned<CLocalEnvironment> pLocalEnv = new CLocalEnvironment(nullptr, ptree);
         return new CLockedEnvironment(pLocalEnv);
     }
@@ -1400,7 +1400,7 @@ void CLocalEnvironment::preload()
 
 void CLocalEnvironment::setXML(const char *xml)
 {
-    Owned<IPropertyTree> newRoot = createPTreeFromXMLString(xml);
+    Owned<IPropertyTree> newRoot = createPTreeFromXMLString(xml, ipt_lowmem);
     synchronized procedure(safeCache);
     Owned<IPropertyTreeIterator> it = p->getElements("*");
     ForEach(*it)
@@ -1709,7 +1709,7 @@ CConstDropZoneServerInfoIterator::CConstDropZoneServerInfoIterator(const IConstD
     if (0 != dropZoneMachineName.length())
     {
         // Create a ServerList for legacy element.
-        Owned<IPropertyTree> legacyServerList = createPTree();
+        Owned<IPropertyTree> legacyServerList = createPTree(ipt_lowmem);
 
         Owned<IConstMachineInfo> machineInfo = constEnv->getMachine(dropZoneMachineName.str());
         if (machineInfo)
@@ -1719,10 +1719,9 @@ CConstDropZoneServerInfoIterator::CConstDropZoneServerInfoIterator(const IConstD
 
             // Create a single ServerList record related to @computer
             //<ServerList name="ServerList" server="<IP_of_@computer>"/>
-            Owned<IPropertyTree> newRecord = createPTree();
+            IPropertyTree *newRecord = legacyServerList->addPropTree("ServerList");
             newRecord->setProp("@name", "ServerList");
             newRecord->setProp("@server", dropZoneMachineNetAddress.str());
-            legacyServerList->addPropTree("ServerList",newRecord.getClear());
 
             maxIndex = 1;
         }

common/thorhelper/roxiedebug.cpp

@@ -1763,7 +1763,7 @@ void CBaseServerDebugContext::debugPrint(IXmlWriter *output, const char *edgeId,
 #if 1
     CommonXmlWriter dummyOutput(0, 0);
     activityCtx->printEdge(&dummyOutput, startRow, numRows); // MORE - need to suppress the <Edge> if we want backward compatibility!
-    Owned<IPropertyTree> result = createPTreeFromXMLString(dummyOutput.str());
+    Owned<IPropertyTree> result = createPTreeFromXMLString(dummyOutput.str(),ipt_fast);
     if (result)
     {
         Owned<IAttributeIterator> attributes = result->getAttributes();

common/thorhelper/thorxmlread.cpp

@@ -564,7 +564,7 @@ void XmlSetColumnProvider::readUtf8X(size32_t & len, char * & target, const char
 IDataVal & CXmlToRawTransformer::transform(IDataVal & result, size32_t len, const void * text, bool isDataSet)
 {
     // MORE - should redo using a pull parser sometime
-    Owned<IPropertyTree> root = createPTreeFromXMLString(len, (const char *)text, ipt_none, xmlReadFlags);
+    Owned<IPropertyTree> root = createPTreeFromXMLString(len, (const char *)text, ipt_fast, xmlReadFlags);
     return transformTree(result, *root, isDataSet);
 }
 
@@ -595,7 +595,7 @@ IDataVal & CXmlToRawTransformer::transformTree(IDataVal & result, IPropertyTree
                     try
                     {
                         decodedXML.append("<root>").append(body).append("</root>");
-                        decodedTree.setown(createPTreeFromXMLString(decodedXML.str(), ipt_caseInsensitive));
+                        decodedTree.setown(createPTreeFromXMLString(decodedXML.str(), ipt_caseInsensitive|ipt_fast));
                         rows.setown(decodedTree->getElements("Row"));
                     }
                     catch (IException *E)
@@ -645,7 +645,7 @@ IDataVal & CXmlToRawTransformer::transformTree(IDataVal & result, IPropertyTree
 
 size32_t createRowFromXml(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace)
 {
-    Owned<IPropertyTree> root = createPTreeFromXMLString(size, utf8, ipt_none, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
+    Owned<IPropertyTree> root = createPTreeFromXMLString(size, utf8, ipt_fast, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
     if (!root)
     {
         throwError(THORCERR_InvalidXmlFromXml);
@@ -666,7 +666,7 @@ const void * createRowFromXml(IEngineRowAllocator * rowAllocator, size32_t len,
 
 size32_t createRowFromJson(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace)
 {
-    Owned<IPropertyTree> root = createPTreeFromJSONString(size, utf8, ipt_none, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
+    Owned<IPropertyTree> root = createPTreeFromJSONString(size, utf8, ipt_fast, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
     if (!root)
     {
         throwError(THORCERR_InvalidJsonFromJson);

common/workunit/package.cpp

@@ -39,7 +39,7 @@ CPackageNode::CPackageNode(IPropertyTree *p)
     if (p)
         node.set(p);
     else
-        node.setown(createPTree("HpccPackages"));
+        node.setown(createPTree("HpccPackages", ipt_lowmem));
     StringBuffer xml;
     toXML(node, xml, 0, XML_SortTags);
     hash = rtlHash64Data(xml.length(), xml.str(), 9994410);
@@ -158,7 +158,7 @@ IHpccPackageMap *createPackageMapFromPtree(IPropertyTree *t, const char *queryse
 
 IHpccPackageMap *createPackageMapFromXml(const char *xml, const char *queryset, const char *id)
 {
-    Owned<IPropertyTree> t = createPTreeFromXMLString(xml);
+    Owned<IPropertyTree> t = createPTreeFromXMLString(xml, ipt_lowmem);
     return createPackageMapFromPtree(t, queryset, id);
 }
 

common/workunit/pkgimpl.hpp

@@ -450,7 +450,7 @@ public:
                 Owned<ISimpleSuperFileEnquiry> ssfe = pkg->resolveSuperFile(rf.getLogicalName());
                 if (ssfe && ssfe->numSubFiles()>0)
                 {
-                    IPropertyTree *superInfo = fileInfo->addPropTree("SuperFile", createPTree());
+                    IPropertyTree *superInfo = fileInfo->addPropTree("SuperFile");
                     superInfo->setProp("@name", rf.getLogicalName());
                     unsigned count = ssfe->numSubFiles();
                     while (count--)
@@ -495,7 +495,7 @@ public:
                     referencedPackages.setValue(baseId, true);
             }
         }
-        Owned<IPropertyTree> tempQuerySet=createPTree();
+        Owned<IPropertyTree> tempQuerySet=createPTree(ipt_fast);
         Owned<IPropertyTreeIterator> queries;
         if (queriesToCheck.length())
         {

common/workunit/workflow.cpp

@@ -126,7 +126,7 @@ public:
     CWorkflowItem(IPropertyTree & _tree) { tree.setown(&_tree); }
     CWorkflowItem(IPropertyTree * ptree, unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
     {
-        tree.setown(LINK(ptree->addPropTree("Item", createPTree())));
+        tree.setown(LINK(ptree->addPropTree("Item")));
         tree->setPropInt("@wfid", wfid);
         setEnum(tree, "@type", type, wftypes);
         setEnum(tree, "@mode", mode, wfmodes);
@@ -136,7 +136,7 @@ public:
         {
             tree->setPropInt("@recovery", recovery);
             tree->setPropInt("@retriesAllowed", retriesAllowed);
-            tree->addPropTree("Dependency", createPTree())->setPropInt("@wfid", recovery);
+            tree->addPropTree("Dependency")->setPropInt("@wfid", recovery);
         }
         if(contingencyFor) tree->setPropInt("@contingencyFor", contingencyFor);
         reset();
@@ -165,11 +165,11 @@ public:
     virtual bool         queryPersistRefresh() const { return tree->getPropBool("@persistRefresh", true); }
     virtual IStringVal & getCriticalName(IStringVal & val) const { val.set(tree->queryProp("@criticalName")); return val; }
     virtual IStringVal & queryCluster(IStringVal & val) const { val.set(tree->queryProp("@cluster")); return val; }
-    virtual void         setScheduledNow() { tree->setPropTree("Schedule", createPTree()); setEnum(tree, "@state", WFStateReqd, wfstates); }
-    virtual void         setScheduledOn(char const * name, char const * text) { IPropertyTree * stree = createPTree(); stree->setProp("@name", name); stree->setProp("@text", text); tree->setPropTree("Schedule", createPTree())->setPropTree("Event", stree); setEnum(tree, "@state", WFStateWait, wfstates); }
+    virtual void         setScheduledNow() { tree->setPropTree("Schedule"); setEnum(tree, "@state", WFStateReqd, wfstates); }
+    virtual void         setScheduledOn(char const * name, char const * text) { IPropertyTree * stree =  tree->setPropTree("Schedule")->setPropTree("Event"); stree->setProp("@name", name); stree->setProp("@text", text);; setEnum(tree, "@state", WFStateWait, wfstates); }
     virtual void         setSchedulePriority(unsigned priority) { assertex(tree->hasProp("Schedule")); tree->setPropInt("Schedule/@priority", priority); }
     virtual void         setScheduleCount(unsigned count) { assertex(tree->hasProp("Schedule")); tree->setPropInt("Schedule/@count", count); tree->setPropInt("Schedule/@countRemaining", count); }
-    virtual void         addDependency(unsigned wfid) { tree->addPropTree("Dependency", createPTree())->setPropInt("@wfid", wfid); }
+    virtual void         addDependency(unsigned wfid) { tree->addPropTree("Dependency")->setPropInt("@wfid", wfid); }
     virtual void         setPersistInfo(char const * name, unsigned wfid, int numPersistInstances, bool refresh)
     {
         tree->setProp("@persistName", name);

common/workunit/workunit.cpp

@@ -196,17 +196,16 @@ void CWuGraphStats::beforeDispose()
     StringBuffer tag;
     tag.append("sg").append(id);
 
-    IPropertyTree * subgraph = createPTree(tag);
-    subgraph->setProp("@c", queryCreatorTypeName(creatorType));
-    subgraph->setProp("@creator", creator);
-    subgraph->setPropInt("@minActivity", minActivity);
-    subgraph->setPropInt("@maxActivity", maxActivity);
 
     //Replace the particular subgraph statistics added by this creator
     StringBuffer qualified(tag);
     qualified.append("[@creator='").append(creator).append("']");
     progress->removeProp(qualified);
-    subgraph = progress->addPropTree(tag, subgraph);
+    IPropertyTree * subgraph = progress->addPropTree(tag);
+    subgraph->setProp("@c", queryCreatorTypeName(creatorType));
+    subgraph->setProp("@creator", creator);
+    subgraph->setPropInt("@minActivity", minActivity);
+    subgraph->setPropInt("@maxActivity", maxActivity);
     subgraph->setPropBin("Stats", compressed.length(), compressed.toByteArray());
     if (!progress->getPropBool("@stats", false))
         progress->setPropBool("@stats", true);
@@ -292,7 +291,7 @@ protected:
                 throwUnexpected();
             }
 
-            IPropertyTree * next = curTarget->addPropTree(tag, createPTree());
+            IPropertyTree * next = curTarget->addPropTree(tag);
             next->setProp("@id", id);
             expandProcessTreeFromStats(rootTarget, next, &cur);
         }
@@ -1145,7 +1144,7 @@ public:
         IPropertyTree *node = progress->queryPropTree(path.str());
         if (!node)
         {
-            node = progress->addPropTree("node", createPTree());
+            node = progress->addPropTree("node");
             node->setPropInt64("@id", nodeId);
         }
         node->setPropInt("@_state", (unsigned)state);
@@ -1153,7 +1152,7 @@ public:
         {
             case WUGraphRunning:
             {
-                IPropertyTree *running = conn->queryRoot()->setPropTree("Running", createPTree());
+                IPropertyTree *running = conn->queryRoot()->setPropTree("Running");
                 running->setProp("@graph", graphName);
                 running->setPropInt64("@subId", nodeId);
                 break;
@@ -3913,7 +3912,7 @@ void CLocalWorkUnit::loadXML(const char *xml)
     CriticalBlock block(crit);
     clearCached(true);
     assertex(xml);
-    p.setown(createPTreeFromXMLString(xml));
+    p.setown(createPTreeFromXMLString(xml,ipt_lowmem));
 }
 
 void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
@@ -5269,7 +5268,7 @@ void CLocalWorkUnit::setWarningSeverity(unsigned code, ErrorSeverity severity)
     if (!mapping)
     {
         IPropertyTree * onWarnings = ensurePTree(p, "OnWarnings");
-        mapping = onWarnings->addPropTree("OnWarning", createPTree());
+        mapping = onWarnings->addPropTree("OnWarning");
         mapping->setPropInt("@code", code);
     }
 
@@ -5604,7 +5603,7 @@ void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned
     if (!p->hasProp(xpath))
     {
         IPropertyTree *node = ensurePTree(p, processType.str());
-        node = node->addPropTree(instance, createPTree());
+        node = node->addPropTree(instance);
         node->setProp("@log", log);
         node->setPropInt("@pid", pid);
     }
@@ -5888,7 +5887,7 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
     CriticalBlock block(crit);
     IPropertyTree * stats = p->queryPropTree("Statistics");
     if (!stats)
-        stats = p->addPropTree("Statistics", createPTree("Statistics"));
+        stats = p->addPropTree("Statistics");
 
     IPropertyTree * statTree = NULL;
     if (mergeAction != StatsMergeAppend)
@@ -5900,7 +5899,7 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
 
     if (!statTree)
     {
-        statTree = stats->addPropTree("Statistic", createPTree("Statistic"));
+        statTree = stats->addPropTree("Statistic");
         statTree->setProp("@creator", creator);
         statTree->setProp("@scope", scope);
         statTree->setProp("@kind", kindName);
@@ -5992,9 +5991,9 @@ IWUPlugin* CLocalWorkUnit::updatePluginByName(const char *qname)
     if (existing)
         return (IWUPlugin *) existing;
     if (!plugins.length())
-        p->addPropTree("Plugins", createPTree("Plugins"));
+        p->addPropTree("Plugins");
     IPropertyTree *pl = p->queryPropTree("Plugins");
-    IPropertyTree *s = pl->addPropTree("Plugin", createPTree("Plugin"));
+    IPropertyTree *s = pl->addPropTree("Plugin");
     s->Link();
     IWUPlugin* q = new CLocalWUPlugin(s); 
     q->Link();
@@ -6028,9 +6027,9 @@ IWULibrary* CLocalWorkUnit::updateLibraryByName(const char *qname)
     if (existing)
         return (IWULibrary *) existing;
     if (!libraries.length())
-        p->addPropTree("Libraries", createPTree("Libraries"));
+        p->addPropTree("Libraries");
     IPropertyTree *pl = p->queryPropTree("Libraries");
-    IPropertyTree *s = pl->addPropTree("Library", createPTree("Library"));
+    IPropertyTree *s = pl->addPropTree("Library");
     s->Link();
     IWULibrary* q = new CLocalWULibrary(s); 
     q->Link();
@@ -6092,9 +6091,9 @@ IWUException* CLocalWorkUnit::createException()
     loadExceptions();
 
     if (!exceptions.length())
-        p->addPropTree("Exceptions", createPTree("Exceptions"));
+        p->addPropTree("Exceptions");
     IPropertyTree *r = p->queryPropTree("Exceptions");
-    IPropertyTree *s = r->addPropTree("Exception", createPTree("Exception"));
+    IPropertyTree *s = r->addPropTree("Exception");
     s->setPropInt("@sequence", exceptions.ordinality());
     IWUException* q = new CLocalWUException(LINK(s)); 
     exceptions.append(*LINK(q));
@@ -6132,7 +6131,7 @@ IWUWebServicesInfo* CLocalWorkUnit::updateWebServicesInfo(bool create)
         if (!s)
         {
             if (create)
-                s = p->addPropTree("WebServicesInfo", createPTreeFromXMLString("<WebServicesInfo />"));
+                s = p->addPropTree("WebServicesInfo");
             else
                 return NULL;
         }
@@ -6231,9 +6230,9 @@ IWUResult* CLocalWorkUnit::createResult()
     // For this to be legally called, we must have the write-able interface. So we are already locked for write.
     loadResults();
     if (!results.length())
-        p->addPropTree("Results", createPTree("Results"));
+        p->addPropTree("Results");
     IPropertyTree *r = p->queryPropTree("Results");
-    IPropertyTree *s = r->addPropTree("Result", createPTree());
+    IPropertyTree *s = r->addPropTree("Result");
 
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
@@ -6381,10 +6380,8 @@ IWUResult* CLocalWorkUnit::updateTemporaryByName(const char *qname)
     IConstWUResult *existing = getTemporaryByName(qname);
     if (existing)
         return (IWUResult *) existing;
-    if (!temporaries.length())
-        p->addPropTree("Temporaries", createPTree("Temporaries"));
-    IPropertyTree *vars = p->queryPropTree("Temporaries");
-    IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
+    IPropertyTree *vars = (temporaries.length()) ? p->queryPropTree("Temporaries") : p->addPropTree("Temporaries");
+    IPropertyTree *s = vars->addPropTree("Variable");
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
     q->Link();
@@ -6401,9 +6398,9 @@ IWUResult* CLocalWorkUnit::updateVariableByName(const char *qname)
     if (existing)
         return (IWUResult *) existing;
     if (!variables.length())
-        p->addPropTree("Variables", createPTree("Variables"));
+        p->addPropTree("Variables");
     IPropertyTree *vars = p->queryPropTree("Variables");
-    IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
+    IPropertyTree *s = vars->addPropTree("Variable");
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
     q->Link();
@@ -6488,9 +6485,7 @@ static void _noteFileRead(IDistributedFile *file, IPropertyTree *filesRead)
                 IDistributedFile &file = iter->query();
                 StringBuffer fname;
                 file.getLogicalName(fname);
-                Owned<IPropertyTree> subfile = createPTree();
-                subfile->setProp("@name", fname.str());
-                fileTree->addPropTree("Subfile", subfile.getClear());
+                fileTree->addPropTree("Subfile")->setProp("@name", fname.str());
                 _noteFileRead(&file, filesRead);
             }
         }
@@ -6509,7 +6504,7 @@ void CLocalWorkUnit::noteFileRead(IDistributedFile *file)
         CriticalBlock block(crit);
         IPropertyTree *files = p->queryPropTree("FilesRead");
         if (!files)
-            files = p->addPropTree("FilesRead", createPTree());
+            files = p->addPropTree("FilesRead");
         _noteFileRead(file, files);
     }
 }
@@ -6553,7 +6548,7 @@ void CLocalWorkUnit::addFile(const char *fileName, StringArray *clusters, unsign
     CriticalBlock block(crit);
     IPropertyTree *files = p->queryPropTree("Files");
     if (!files)
-        files = p->addPropTree("Files", createPTree());
+        files = p->addPropTree("Files");
     if (!clusters)
         ::addFile(files, fileName, NULL, usageCount, fileKind, graphOwner);
     else
@@ -6726,7 +6721,7 @@ void CLocalWorkUnit::addDiskUsageStats(__int64 _avgNodeUsage, unsigned _minNode,
         maxNodeUsage = stats->getPropInt64("@maxNodeUsage");
     else
     {
-        stats = p->addPropTree("DiskUsageStats", createPTree());
+        stats = p->addPropTree("DiskUsageStats");
         maxNodeUsage = 0;
     }
 
@@ -7115,9 +7110,9 @@ void CLocalWorkUnit::createGraph(const char * name, const char *label, WUGraphTy
 {
     CriticalBlock block(crit);
     if (!graphs.length())
-        p->addPropTree("Graphs", createPTree("Graphs"));
+        p->addPropTree("Graphs");
     IPropertyTree *r = p->queryPropTree("Graphs");
-    IPropertyTree *s = r->addPropTree("Graph", createPTree());
+    IPropertyTree *s = r->addPropTree("Graph");
     CLocalWUGraph *q = new CLocalWUGraph(*this, LINK(s));
     q->setName(name);
     q->setLabel(label);
@@ -7171,13 +7166,13 @@ void CLocalWUGraph::setLabel(const char *str)
 
 void CLocalWUGraph::setXGMML(const char *str)
 {
-    setXGMMLTree(createPTreeFromXMLString(str));
+    setXGMMLTree(createPTreeFromXMLString(str,ipt_lowmem));
 }
 
 void CLocalWUGraph::setXGMMLTree(IPropertyTree *_graph)
 {
     assertex(strcmp(_graph->queryName(), "graph")==0);
-    IPropertyTree *xgmml = p->setPropTree("xgmml", createPTree());
+    IPropertyTree *xgmml = p->setPropTree("xgmml");
     MemoryBuffer mb;
     _graph->serialize(mb);
     // Note - we could compress further but that would introduce compatibility concerns, so don't bother
@@ -7194,7 +7189,7 @@ static void expandAttributes(IPropertyTree & targetNode, IPropertyTree & progres
         const char *aName = aIter->queryName()+1;
         if (0 != stricmp("id", aName)) // "id" reserved.
         {
-            IPropertyTree *att = targetNode.addPropTree("att", createPTree());
+            IPropertyTree *att = targetNode.addPropTree("att");
             att->setProp("@name", aName);
             att->setProp("@value", aIter->queryValue());
         }
@@ -7233,7 +7228,7 @@ void CLocalWUGraph::mergeProgress(IPropertyTree &rootNode, IPropertyTree &progre
                     ForEach (*iter)
                     {
                         IPropertyTree &t = iter->query();
-                        IPropertyTree *att = graphEdge->addPropTree("att", createPTree());
+                        IPropertyTree *att = graphEdge->addPropTree("att");
                         att->setProp("@name", t.queryName());
                         att->setProp("@value", t.queryProp(NULL));
                     }
@@ -7276,7 +7271,7 @@ IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
         // daliadmin can retrospectively compress existing graphs, so need to check for all versions
         MemoryBuffer mb;
         if (p->getPropBin("xgmml/graphBin", mb))
-            graph.setown(createPTree(mb));
+            graph.setown(createPTree(mb, ipt_lowmem));
         else
             graph.setown(p->getBranch("xgmml/graph"));
         if (!graph)
@@ -7286,7 +7281,7 @@ IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
         return graph.getLink();
     else
     {
-        Owned<IPropertyTree> copy = createPTreeFromIPT(graph);
+        Owned<IPropertyTree> copy = createPTreeFromIPT(graph, ipt_lowmem);
         Owned<IConstWUGraphProgress> progress = owner.getGraphProgress(p->queryProp("@name"));
         if (progress)
         {
@@ -7421,7 +7416,7 @@ IStringVal& CLocalWUQuery::getQueryShortText(IStringVal &str) const
         text = p->queryProp("Text");
         if (isArchiveQuery(text))
         {
-            Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive);
+            Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
             const char * path = xml->queryProp("Query/@attributePath");
             if (path)
             {
@@ -7496,7 +7491,7 @@ void CLocalWUQuery::setQueryText(const char *text)
     bool isArchive = isArchiveQuery(text);
     if (isArchive)
     {
-        Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive);
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
         const char * path = xml->queryProp("Query/@attributePath");
         if (path)
         {
@@ -7527,9 +7522,9 @@ void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const
     CriticalBlock block(crit);
     loadAssociated();
     if (!associated.length())
-        p->addPropTree("Associated", createPTree("Associated"));
+        p->addPropTree("Associated");
     IPropertyTree *pl = p->queryPropTree("Associated");
-    IPropertyTree *s = pl->addPropTree("File", createPTree("File"));
+    IPropertyTree *s = pl->addPropTree("File");
     setEnum(s, "@type", type, queryFileTypes);
     s->setProp("@filename", name);
     s->setProp("@ip", ip);
@@ -9042,10 +9037,10 @@ extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(const char *xml)
 {
     Owned<CLocalWorkUnit> cw = new CLocalWorkUnit((ISecManager *) NULL, NULL);
     if (xml)
-        cw->loadPTree(createPTreeFromXMLString(xml));
+        cw->loadPTree(createPTreeFromXMLString(xml, ipt_lowmem));
     else
     {
-        Owned<IPropertyTree> p = createPTree("W_LOCAL");
+        Owned<IPropertyTree> p = createPTree("W_LOCAL", ipt_lowmem);
         p->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
         cw->loadPTree(p.getClear());
     }
@@ -9317,7 +9312,7 @@ IWorkflowItem * CLocalWorkUnit::addWorkflowItem(unsigned wfid, WFType type, WFMo
     workflowIteratorCached = false;
     IPropertyTree * s = p->queryPropTree("Workflow");
     if(!s)
-        s = p->addPropTree("Workflow", createPTree("Workflow"));
+        s = p->addPropTree("Workflow");
     return createWorkflowItem(s, wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor);
 }
 
@@ -9329,7 +9324,7 @@ IWorkflowItemIterator * CLocalWorkUnit::updateWorkflowItems()
     {
         IPropertyTree * s = p->queryPropTree("Workflow");
         if(!s)
-            s = p->addPropTree("Workflow", createPTree("Workflow"));
+            s = p->addPropTree("Workflow");
         workflowIterator.setown(createWorkflowItemIterator(s)); 
         workflowIteratorCached = true;
     }
@@ -9520,7 +9515,7 @@ unsigned CLocalWorkUnit::addLocalFileUpload(LocalFileUploadType type, char const
     CriticalBlock block(crit);
     IPropertyTree * s = p->queryPropTree("LocalFileUploads");
     if(!s)
-        s = p->addPropTree("LocalFileUploads", createPTree());
+        s = p->addPropTree("LocalFileUploads");
     unsigned id = s->numChildren();
     Owned<CLocalFileUpload> upload = new CLocalFileUpload(id, type, source, destination, eventTag);
     s->addPropTree("LocalFileUpload", upload->getTree());
@@ -10031,12 +10026,10 @@ public:
     {
         assertex(baseconn);
         Owned<IPropertyTree> root = baseconn->getRoot();
-        ensurePTree(root, "EventQueue");
-        Owned<IPropertyTree> eventQueue = root->getPropTree("EventQueue");
-        Owned<IPropertyTree> eventItem = createPTree();
+        IPropertyTree *eventQueue = ensurePTree(root, "EventQueue");
+        IPropertyTree *eventItem = eventQueue->addPropTree("Item");
         eventItem->setProp("@name", name);
         eventItem->setProp("@text", text);
-        eventQueue->addPropTree("Item", eventItem.getLink());
     }
 
     virtual void remove()
@@ -10246,7 +10239,7 @@ IPropertyTree * addNamedQuery(IPropertyTree * queryRegistry, const char * name,
 
     StringBuffer id;
     id.append(lcName).append(".").append(seq);
-    IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive);
+    IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive|ipt_lowmem);
     newEntry->setProp("@name", lcName);
     newEntry->setProp("@wuid", wuid);
     newEntry->setProp("@dll", dll);
@@ -10306,9 +10299,8 @@ void setQueryAlias(IPropertyTree * queryRegistry, const char * name, const char
     IPropertyTree * match = queryRegistry->queryPropTree(xpath);
     if (!match)
     {
-        IPropertyTree * newEntry = createPTree("Alias");
-        newEntry->setProp("@name", lcName);
-        match = queryRegistry->addPropTree("Alias", newEntry);
+        match = queryRegistry->addPropTree("Alias");
+        match->setProp("@name", lcName);
     }
     match->setProp("@id", value);
 }

common/workunit/wujobq.cpp

@@ -113,13 +113,6 @@ public:
         item->setProp("@enqueuedt",dts.str());
     }
 
-    IPropertyTree *createBranch(CJobQueueItem)
-    {
-        IPropertyTree *item = createPTree("Item");
-        assignBranch(item,this);
-        return item;
-    }
-
     const char *queryWUID()
     {
         return wu.get();
@@ -893,7 +886,7 @@ public:
                             StringBuffer cpath;
                             cpath.appendf("Queue[@name=\"%s\"]",qd->qname.get());
                             if (!proot->hasProp(cpath.str())) {
-                                IPropertyTree *pt = proot->addPropTree("Queue",createPTree("Queue"));
+                                IPropertyTree *pt = proot->addPropTree("Queue");
                                 pt->setProp("@name",qd->qname.get());
                                 pt->setProp("@state","active");
                                 pt->setPropInt("@count", 0);
@@ -1087,8 +1080,7 @@ public:
         IPropertyTree *ret = qd.root->queryPropTree(path.str());
         if (!ret)
         {
-            ret = createPTree("Client");
-            ret = qd.root->addPropTree("Client",ret);
+            ret = qd.root->addPropTree("Client");
             ret->setPropInt64("@session",sessionid);
             StringBuffer eps;
             ret->setProp("@node",queryMyNode()->endpoint().getUrlStr(eps).str());

dali/base/dacsds.cpp

@@ -1105,13 +1105,13 @@ IPropertyTree *CClientRemoteTree::collateData()
     {
         ChangeTree(IPropertyTree *donor=NULL) { ptree = LINK(donor); }
         ~ChangeTree() { ::Release(ptree); }
-        inline void createTree() { assertex(!ptree); ptree = createPTree(RESERVED_CHANGE_NODE); }
+        inline void createTree() { assertex(!ptree); ptree = createPTree(RESERVED_CHANGE_NODE, ipt_fast); }
         inline IPropertyTree *queryTree() { return ptree; }
         inline IPropertyTree *getTree() { return LINK(ptree); }
         inline IPropertyTree *queryCreateTree()
         {
             if (!ptree)
-                ptree = createPTree(RESERVED_CHANGE_NODE);
+                ptree = createPTree(RESERVED_CHANGE_NODE, ipt_fast);
             return ptree;
         }
     private:
@@ -1126,10 +1126,10 @@ IPropertyTree *CClientRemoteTree::collateData()
         Owned<IAttributeIterator> iter = getAttributes();
         if (iter->count())
         {
-            IPropertyTree *t = createPTree();
+            IPropertyTree *t = ct.queryTree()->addPropTree(ATTRCHANGE_TAG);
             ForEach(*iter)
                 t->setProp(iter->queryName(), queryProp(iter->queryName()));
-            ct.queryTree()->addPropTree(ATTRCHANGE_TAG, t);
+
         }
         ct.queryTree()->setPropBool("@new", true);
     }
@@ -1142,10 +1142,9 @@ IPropertyTree *CClientRemoteTree::collateData()
             {
                 ct.queryTree()->removeTree(ac);
                 Owned<IAttributeIterator> iter = ac->getAttributes();
-                IPropertyTree *t = createPTree();
+                IPropertyTree *t = ct.queryTree()->addPropTree(ATTRCHANGE_TAG);
                 ForEach(*iter)
                     t->setProp(iter->queryName(), queryProp(iter->queryName()));
-                ct.queryTree()->addPropTree(ATTRCHANGE_TAG, t);
             }
         }
     }
@@ -1945,9 +1944,9 @@ IPropertyTree &CClientSDSManager::queryProperties() const
         default:
             assertex(false);
     }
-    properties = createPTree(mb);
+    properties = createPTree(mb, ipt_lowmem);
     if (!properties->hasProp("Client"))
-        properties->setPropTree("Client", createPTree());
+        properties->setPropTree("Client");
     return *properties;
 }
 
@@ -1968,7 +1967,7 @@ IPropertyTree *CClientSDSManager::getXPaths(__int64 serverId, const char *xpath,
     switch (replyMsg)
     {
         case DAMP_SDSREPLY_OK:
-            return createPTree(mb);
+            return createPTree(mb, ipt_lowmem);
         case DAMP_SDSREPLY_EMPTY:
             return NULL;
         case DAMP_SDSREPLY_ERROR:
@@ -2007,7 +2006,7 @@ IPropertyTreeIterator *CClientSDSManager::getXPathsSortLimit(const char *baseXPa
         default:
             throwUnexpected();
     }
-    Owned<IPropertyTree> matchTree = createPTree(mb);
+    Owned<IPropertyTree> matchTree = createPTree(mb, ipt_lowmem);
     Owned<CRemoteConnection> conn = (CRemoteConnection *)connect(baseXPath, myProcessSession(), RTM_LOCK_READ, INFINITE);
     if (!conn)
         return createNullPTreeIterator();
@@ -2076,7 +2075,7 @@ IPropertyTreeIterator *CClientSDSManager::getElementsRaw(const char *xpath, INod
             mb.read(count);
             for (c=0; c<count; c++)
             {
-                Owned<IPropertyTree> item = createPTree(mb);
+                Owned<IPropertyTree> item = createPTree(mb, ipt_lowmem);
                 resultIterator->array.append(*LINK(item));
             }
             return LINK(resultIterator);

dali/base/dasds.ipp

@@ -109,7 +109,7 @@ class ChangeInfo : public CInterfaceOf<IInterface>
 {
     DECL_NAMEDCOUNT;
 public:
-    ChangeInfo(IPropertyTree &_owner) : owner(&_owner) { INIT_NAMEDCOUNT; tree.setown(createPTree(RESERVED_CHANGE_NODE)); }
+    ChangeInfo(IPropertyTree &_owner) : owner(&_owner) { INIT_NAMEDCOUNT; tree.setown(createPTree(RESERVED_CHANGE_NODE, ipt_fast)); }
     const IPropertyTree *queryOwner() const { return owner; }
     const void *queryFindParam() const { return &owner; }
 public: // data
@@ -276,26 +276,25 @@ public:
     void registerRenamed(IPropertyTree &owner, const char *newName, const char *oldName, unsigned pos, __int64 id)
     {
         ChangeInfo *changes = queryCreateChangeInfo(owner);
-        IPropertyTree *t = createPTree();
+        IPropertyTree *t = changes->tree->addPropTree(RENAME_TAG);
         t->setProp("@from", oldName);
         t->setProp("@to", newName);
         t->setPropInt64("@id", id);
 #ifdef SIBLING_MOVEMENT_CHECK
         t->setProp("@pos", pos);
 #endif
-        changes->tree->addPropTree(RENAME_TAG, t);
+
     }
 
     void registerDeleted(IPropertyTree &owner, const char *name, unsigned pos, __int64 id)
     {
         ChangeInfo *changes = queryCreateChangeInfo(owner);
-        IPropertyTree *t = createPTree();
+        IPropertyTree *t = changes->tree->addPropTree(DELETE_TAG);
         t->setProp("@name", name);
         t->setPropInt64("@id", id);
 #ifdef SIBLING_MOVEMENT_CHECK
         t->setPropInt("@pos", pos+1);
 #endif
-        changes->tree->addPropTree(DELETE_TAG, t);
     }
 
     virtual void registerAttrChange(IPropertyTree &owner, const char *attr)
@@ -305,7 +304,7 @@ public:
         if (t) t->removeProp(attr);
         t = changes->tree->queryPropTree("AC");
         if (!t)
-            t = changes->tree->addPropTree("AC", createPTree());
+            t = changes->tree->addPropTree("AC");
         t->setProp(attr, "");
     }
 
@@ -316,7 +315,7 @@ public:
         if (t) t->removeProp(attr);
         t = changes->tree->queryPropTree("AD");
         if (!t)
-            t = changes->tree->addPropTree("AD", createPTree());
+            t = changes->tree->addPropTree("AD");
         t->addProp(attr, "");
     }
 
@@ -325,7 +324,7 @@ public:
         ChangeInfo *changes = queryCreateChangeInfo(owner);
         IPropertyTree *t = changes->tree->queryPropTree(APPEND_TAG);
         if (!t)
-            t = changes->tree->setPropTree(APPEND_TAG, createPTree());
+            t = changes->tree->setPropTree(APPEND_TAG);
         t->setPropInt(NULL, l);
     }
 

roxie/ccd/ccdactivities.cpp

@@ -81,7 +81,7 @@ extern void putStatsValue(IPropertyTree *node, const char *statName, const char
         IPropertyTree *att = node->queryPropTree(xpath.str());
         if (!att)
         {
-            att = node->addPropTree("att", createPTree());
+            att = node->addPropTree("att");
             att->setProp("@name", statName);
         }
         att->setProp("@type", statType);
@@ -245,15 +245,13 @@ protected:
             const char *id = strchr(levelx, '[');
             if (!id)
             {
-                child = createPTree(levelx);
-                parent->addPropTree(levelx, child);
+                child = parent->addPropTree(levelx);
             }
             else
             {
                 StringBuffer elem;
                 elem.append(id-levelx, levelx);
-                child = createPTree(elem);
-                parent->addPropTree(elem, child);
+                child = parent->addPropTree(elem);
                 for (;;)
                 {
                     StringBuffer attr, val;

roxie/ccd/ccdcontext.cpp

@@ -2018,7 +2018,7 @@ protected:
             {
                 CriticalBlock b(contextCrit);
                 if (!persists)
-                    persists = createPTree();
+                    persists = createPTree(ipt_fast);
                 return *persists;
             }
         case ResultSequenceOnce:
@@ -2031,14 +2031,14 @@ protected:
             {
                 CriticalBlock b(contextCrit);
                 if (!temporaries)
-                    temporaries = createPTree();
+                    temporaries = createPTree(ipt_fast);
                 return *temporaries;
             }
         default:
             {
                 CriticalBlock b(contextCrit);
                 if (!rereadResults)
-                    rereadResults = createPTree();
+                    rereadResults = createPTree(ipt_fast);
                 return *rereadResults;
             }
         }
@@ -2687,8 +2687,8 @@ protected:
         if (workUnit->getXmlParams(wuParams, false).length())
         {
             // Merge in params from WU. Ones on command line take precedence though...
-            Owned<IPropertyTree> wuParamTree = createPTreeFromXMLString(wuParams.str(), ipt_caseInsensitive);
-            Owned<IPropertyTreeIterator> params = wuParamTree ->getElements("*");
+            Owned<IPropertyTree> wuParamTree = createPTreeFromXMLString(wuParams.str(), ipt_caseInsensitive|ipt_fast);
+            Owned<IPropertyTreeIterator> params = wuParamTree->getElements("*");
             ForEach(*params)
             {
                 IPropertyTree &param = params->query();
@@ -2735,7 +2735,7 @@ public:
         init();
         rowManager->setMemoryLimit(options.memoryLimit);
         workflow.setown(_factory->createWorkflowMachine(workUnit, true, logctx));
-        context.setown(createPTree(ipt_caseInsensitive));
+        context.setown(createPTree(ipt_caseInsensitive|ipt_fast));
     }
 
     CRoxieServerContext(IConstWorkUnit *_workUnit, const IQueryFactory *_factory, const ContextLogger &_logctx)
@@ -2745,7 +2745,7 @@ public:
         workUnit.set(_workUnit);
         rowManager->setMemoryLimit(options.memoryLimit);
         workflow.setown(_factory->createWorkflowMachine(workUnit, false, logctx));
-        context.setown(createPTree(ipt_caseInsensitive));
+        context.setown(createPTree(ipt_caseInsensitive|ipt_fast));
 
         //MORE: Use various debug settings to override settings:
         rowManager->setActivityTracking(workUnit->getDebugValueBool("traceRoxiePeakMemory", false));
@@ -3206,7 +3206,7 @@ public:
         else
         {
             if (!val)
-                val = ctx.addPropTree(name, createPTree());
+                val = ctx.addPropTree(name, createPTree(ipt_fast));
             val->setProp("@format", "deserialized");
             val->setPropInt("@id", resultStore.addResult(count, data, meta));
         }
@@ -3317,7 +3317,7 @@ public:
     virtual void setResultXml(const char *name, unsigned sequence, const char *xml)
     {
         CriticalBlock b(contextCrit);
-        useContext(sequence).setPropTree(name, createPTreeFromXMLString(xml, ipt_caseInsensitive));
+        useContext(sequence).setPropTree(name, createPTreeFromXMLString(xml, ipt_caseInsensitive|ipt_fast));
     }
 
     virtual void setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val)

roxie/ccd/ccddali.cpp

@@ -241,11 +241,11 @@ private:
 
     static void initCache()
     {
-        IPropertyTree *tree = createPTree("Roxie");
-        tree->addPropTree("QuerySets", createPTree("QuerySets"));
-        tree->addPropTree("PackageSets", createPTree("PackageSets"));
-        tree->addPropTree("PackageMaps", createPTree("PackageMaps"));
-        tree->addPropTree("Files", createPTree("Files"));
+        IPropertyTree *tree = createPTree("Roxie", ipt_lowmem);
+        tree->addPropTree("QuerySets");
+        tree->addPropTree("PackageSets");
+        tree->addPropTree("PackageMaps");
+        tree->addPropTree("Files");
         cache.setown(tree);
     }
 
@@ -256,7 +256,7 @@ private:
             StringBuffer cacheFileName(queryDirectory);
             cacheFileName.append(roxieStateName);
             if (checkFileExists(cacheFileName))
-                cache.setown(createPTreeFromXMLFile(cacheFileName));
+                cache.setown(createPTreeFromXMLFile(cacheFileName, ipt_lowmem));
             else
                 initCache();
         }
@@ -293,7 +293,7 @@ private:
                 if (conn)
                 {
                     Owned <IPropertyTree> daliTree = conn->getRoot();
-                    localTree.setown(createPTreeFromIPT(daliTree));
+                    localTree.setown(createPTreeFromIPT(daliTree, ipt_lowmem));
                 }
                 writeCache(xpath, path, localTree);
                 return localTree.getClear();
@@ -442,7 +442,7 @@ public:
         Owned<IPropertyTree> ret = loadDaliTree("QuerySets/QuerySet", id);
         if (!ret)
         {
-            ret.setown(createPTree("QuerySet"));
+            ret.setown(createPTree("QuerySet", ipt_lowmem));
             ret->setProp("@id", id);
         }
         return ret.getClear();
@@ -453,7 +453,7 @@ public:
         Owned<IPropertyTree> ret = loadDaliTree("PackageSets", NULL);
         if (!ret)
         {
-            ret.setown(createPTree("PackageSets"));
+            ret.setown(createPTree("PackageSets", ipt_lowmem));
         }
         return ret.getClear();
     }
@@ -476,7 +476,7 @@ public:
         Owned<IPropertyTree> ret = loadDaliTree("PackageMaps/PackageMap", id);
         if (!ret)
         {
-            ret.setown(createPTree("PackageMap"));
+            ret.setown(createPTree("PackageMap", ipt_lowmem));
             ret->setProp("@id", id);
         }
         return ret.getClear();

roxie/ccd/ccddebug.cpp

@@ -410,7 +410,7 @@ public:
                                 IPropertyTree *att = edge.queryPropTree("att[@name=\"_roxieStarted\"]");
                                 if (!att)
                                 {
-                                    att = edge.addPropTree("att", createPTree());
+                                    att = edge.addPropTree("att");
                                     att->setProp("@name", "_roxieStarted");
                                 }
                                 else
@@ -440,7 +440,7 @@ public:
                                     IPropertyTree *att = edge.queryPropTree("att[@name=\"_roxieStarted\"]");
                                     if (!att)
                                     {
-                                        att = edge.addPropTree("att", createPTree());
+                                        att = edge.addPropTree("att");
                                         att->setProp("@name", "_roxieStarted");
                                     }
                                     else
@@ -1586,7 +1586,7 @@ public:
                 DebugRequestLookupActivityByEdgeId request(proxyId, edgeId);
                 CommonXmlWriter reply(0);
                 sendProxyRequest(&reply, request);
-                Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str());
+                Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str(), ipt_fast);
                 if (response)
                 {
                     memsize_t proxyId = (memsize_t) response->getPropInt64("@proxyId", 0);
@@ -1679,7 +1679,7 @@ public:
         reply.outputBeginNested("Counts", true);
         sendProxyRequest(&reply, request);
         reply.outputEndNested("Counts"); // strange way to do it...
-        Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str());
+        Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str(), ipt_fast);
         if (response)
         {
             Owned<IPropertyTreeIterator> edges = response->getElements("edge");

roxie/ccd/ccdfile.cpp

@@ -1795,7 +1795,7 @@ public:
                     addFile(sub.queryLogicalName(), fDesc.getClear(), remoteFDesc.getClear());
                 }
                 // We have to clone the properties since we don't want to keep the superfile locked
-                properties.setown(createPTreeFromIPT(&dFile->queryAttributes()));
+                properties.setown(createPTreeFromIPT(&dFile->queryAttributes(), ipt_lowmem));
                 if (!isDynamic && !lockSuperFiles)
                 {
                     notifier.setown(daliHelper->getSuperFileSubscription(lfn, this));
@@ -2205,7 +2205,7 @@ public:
         assertex(file->exists());
         offset_t size = file->size();
         Owned<IFileDescriptor> fdesc = createFileDescriptor();
-        Owned<IPropertyTree> pp = createPTree("Part");
+        Owned<IPropertyTree> pp = createPTree("Part", ipt_lowmem);
         pp->setPropInt64("@size",size);
         pp->setPropBool("@local", true);
         fdesc->setPart(0, queryMyNode(), localFileName, pp);
@@ -2372,7 +2372,7 @@ public:
                 bool propertiesPresent;
                 serverData.read(propertiesPresent);
                 if (propertiesPresent)
-                    properties.setown(createPTree(serverData));
+                    properties.setown(createPTree(serverData, ipt_lowmem));
             }
             else
                 throw MakeStringException(ROXIE_CALLBACK_ERROR, "Failed to get response from server for dynamic file callback");

roxie/ccd/ccdlistener.cpp

@@ -173,7 +173,7 @@ public:
         StringBuffer lockQuery;
         lockQuery.appendf("<control:childlock thisEndpoint='%d' parent='%d'/>", activeIdxes.item(idx), myEndpoint);
         doChildQuery(idx, lockQuery.str(), lockReply);
-        Owned<IPropertyTree> lockResult = createPTreeFromXMLString(lockReply.str(), ipt_caseInsensitive);
+        Owned<IPropertyTree> lockResult = createPTreeFromXMLString(lockReply.str(), ipt_caseInsensitive|ipt_fast);
         int lockCount = lockResult->getPropInt("Lock", 0);
         if (lockCount)
         {
@@ -340,7 +340,7 @@ public:
 
     void doLockChild(const char *queryText, StringBuffer &reply)
     {
-        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText);
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText,ipt_fast);
         doLockChild(xml, queryText, reply);
     }
 
@@ -395,7 +395,7 @@ public:
 
     void doControlQuery(SocketEndpoint &ep, const char *queryText, StringBuffer &reply)
     {
-        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText); // control queries are case sensitive
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText,ipt_fast); // control queries are case sensitive
         doControlQuery(ep, xml, queryText, reply);
     }
 
@@ -413,7 +413,7 @@ public:
             mergeType=CascadeMergeQueries;
         Owned<IPropertyTree> mergedReply;
         if (mergeType!=CascadeMergeNone)
-            mergedReply.setown(createPTree("Endpoint"));
+            mergedReply.setown(createPTree("Endpoint",ipt_fast));
 
         class casyncfor: public CAsyncFor
         {
@@ -444,7 +444,7 @@ public:
                 {
                     StringBuffer childReply;
                     parent->doChildQuery(i, queryText, childReply);
-                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(childReply);
+                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(childReply,ipt_fast);
                     if (!replyXML)
                     {
                         StringBuffer err;
@@ -490,7 +490,7 @@ public:
                 CriticalBlock cb(crit);
                 if (mergedReply)
                 {
-                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(myReply);
+                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(myReply,ipt_fast);
                     if (mergeType == CascadeMergeStats)
                         mergeStats(mergedReply, replyXML);
                     else if (mergeType == CascadeMergeQueries)

roxie/ccd/ccdmain.cpp

@@ -282,9 +282,7 @@ void getAccessList(const char *aclName, const IPropertyTree *topology, IProperty
     xpath.append("ACL[@name='").append(aclName).append("']");
     if (aclInfo->queryPropTree(xpath))
         throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - recursive ACL definition of %s", aclName);
-    Owned<IPropertyTree> X = createPTree("ACL");
-    X->setProp("@name", aclName);
-    aclInfo->addPropTree("ACL", X.getClear());
+    aclInfo->addPropTree("ACL")->setProp("@name", aclName);
 
     Owned<IPropertyTree> acl = topology->getPropTree(xpath.str());
     if (!acl)
@@ -314,10 +312,9 @@ void addSlaveChannel(unsigned channel, unsigned level)
     xpath.appendf("RoxieSlaveProcess[@channel=\"%d\"]", channel);
     if (ccdChannels->hasProp(xpath.str()))
         throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - channel %d repeated", channel);
-    IPropertyTree *ci = createPTree("RoxieSlaveProcess");
+    IPropertyTree *ci = ccdChannels->addPropTree("RoxieSlaveProcess");
     ci->setPropInt("@channel", channel);
     ci->setPropInt("@subChannel", numSlaves[channel]);
-    ccdChannels->addPropTree("RoxieSlaveProcess", ci);
 }
 
 void addChannel(unsigned nodeNumber, unsigned channel, unsigned level)
@@ -499,7 +496,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
     Thread::setDefaultStackSize(0x10000);   // NB under windows requires linker setting (/stack:)
 #endif
     srand( (unsigned)time( NULL ) );
-    ccdChannels = createPTree("Channels");
+    ccdChannels = createPTree("Channels", ipt_lowmem);
 
     char currentDirectory[_MAX_DIR];
     if (!getcwd(currentDirectory, sizeof(currentDirectory)))
@@ -524,7 +521,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         if (checkFileExists(topologyFile.str()))
         {
             DBGLOG("Loading topology file %s", topologyFile.str());
-            topology = createPTreeFromXMLFile(topologyFile.str());
+            topology = createPTreeFromXMLFile(topologyFile.str(), ipt_lowmem);
             saveTopology();
         }
         else
@@ -539,6 +536,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 " <RoxieFarmProcess/>"
                 " <RoxieServerProcess netAddress='.'/>"
                 "</RoxieTopology>"
+                , ipt_lowmem
                 );
             int port = globals->getPropInt("--port", 9876);
             topology->setPropInt("RoxieFarmProcess/@port", port);
@@ -1178,7 +1176,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 const char *aclName = roxieFarm.queryProp("@aclName");
                 if (aclName && *aclName)
                 {
-                    Owned<IPropertyTree> aclInfo = createPTree("AccessInfo");
+                    Owned<IPropertyTree> aclInfo = createPTree("AccessInfo", ipt_lowmem);
                     getAccessList(aclName, topology, aclInfo);
                     Owned<IPropertyTreeIterator> accesses = aclInfo->getElements("Access");
                     ForEach(*accesses)
@@ -1203,7 +1201,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 roxieServer->start();
             }
             writeSentinelFile(sentinelFile);
-            DBGLOG("Waiting for queries");
+            DBGLOG("Waiting for queries LPT=%u APT=%u", queryNumLocalTrees(), queryNumAtomTrees());
             if (pingInterval)
                 startPingTimer();
             LocalIAbortHandler abortHandler(waiter);

roxie/ccd/ccdprotocol.cpp

@@ -863,7 +863,7 @@ public:
             return;
         if (fmt==MarkupFmt_JSON)
         {
-            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(content);
+            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(content, ipt_fast);
             if (name && *name)
                 appendXMLOpenTag(xml, name);
             toXML(convertPT, xml, 0, 0);
@@ -1115,7 +1115,7 @@ public:
         StringBuffer json;
         if (mlFmt==MarkupFmt_XML)
         {
-            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(StringBuffer("<Control>").append(content).append("</Control>"));
+            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(StringBuffer("<Control>").append(content).append("</Control>"), ipt_fast);
             toJSON(convertPT, json, 0, 0);
             content = json.str();
         }
@@ -1226,7 +1226,7 @@ public:
         StringBuffer xml;
         if (mlFmt==MarkupFmt_JSON)
         {
-            Owned<IPropertyTree> convertPT = createPTreeFromJSONString(content);
+            Owned<IPropertyTree> convertPT = createPTreeFromJSONString(content, ipt_fast);
             toXML(convertPT, xml, 0, 0);
             content = xml.str();
         }
@@ -1770,7 +1770,7 @@ readAnother:
                     client->setHttpMode(queryName, false, httpHelper);
 
                 bool aclupdate = strieq(queryName, "aclupdate"); //ugly
-                byte iptFlags = aclupdate ? ipt_caseInsensitive : 0;
+                byte iptFlags = aclupdate ? ipt_caseInsensitive|ipt_fast : ipt_fast;
 
                 createQueryPTree(queryPT, httpHelper, rawText, iptFlags, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces), queryName);
 
@@ -1815,7 +1815,7 @@ readAnother:
                 readFlags |= (whitespace == WhiteSpaceHandling::Strip ? ptr_ignoreWhiteSpace : ptr_none);
                 try
                 {
-                    createQueryPTree(queryPT, httpHelper, rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)readFlags, queryName);
+                    createQueryPTree(queryPT, httpHelper, rawText.str(), ipt_caseInsensitive|ipt_fast, (PTreeReaderOptions)readFlags, queryName);
                 }
                 catch (IException *E)
                 {
@@ -1887,7 +1887,7 @@ readAnother:
                             Owned<IPropertyTreeIterator> reqIter = queryPT->getElements(reqIterString.str());
                             ForEach(*reqIter)
                             {
-                                IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive);
+                                IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive|ipt_fast);
                                 Owned<IPropertyTreeIterator> iter = reqIter->query().getElements("*");
                                 ForEach(*iter)
                                 {
@@ -1898,7 +1898,7 @@ readAnother:
                         }
                         else
                         {
-                            IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive);
+                            IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive|ipt_fast);
                             Owned<IPropertyTreeIterator> iter = queryPT->getElements("*");
                             ForEach(*iter)
                             {

roxie/ccd/ccdquery.cpp

@@ -170,9 +170,9 @@ IPropertyTree * addXrefInfo(IPropertyTree &reply, const char *section, const cha
     VStringBuffer xpath("%s[@name='%s']", section, name);
     if (!reply.hasProp(xpath))
     {
-        IPropertyTree *info = createPTree(section, 0);
+        IPropertyTree *info = reply.addPropTree(section);
         info->setProp("@name", name);
-        return reply.addPropTree(section, info);
+        return info;
     }
     return NULL;
 }
@@ -236,7 +236,7 @@ public:
         CriticalBlock b(onceCrit);
         if (!onceContext)
         {
-            onceContext.setown(createPTree());
+            onceContext.setown(createPTree(ipt_lowmem));
             onceResultStore.setown(createDeserializedResultStore());
             Owned <IRoxieServerContext> ctx = createOnceServerContext(factory, logctx);
             onceManager.set(&ctx->queryRowManager());
@@ -1297,7 +1297,7 @@ public:
 
     void getGraphStats(StringBuffer &reply, const IPropertyTree &thisGraph) const
     {
-        Owned<IPropertyTree> graph = createPTreeFromIPT(&thisGraph);
+        Owned<IPropertyTree> graph = createPTreeFromIPT(&thisGraph, ipt_lowmem);
         Owned<IPropertyTreeIterator> edges = graph->getElements(".//edge");
         ForEach(*edges)
         {
@@ -1325,7 +1325,7 @@ public:
     virtual IPropertyTree* cloneQueryXGMML() const
     {
         assertex(dll && dll->queryWorkUnit());
-        Owned<IPropertyTree> tree = createPTree("Query");
+        Owned<IPropertyTree> tree = createPTree("Query", ipt_lowmem);
         Owned<IConstWUGraphIterator> graphs = &dll->queryWorkUnit()->getGraphs(GraphTypeActivities);
         SCMStringBuffer graphNameStr;
         ForEach(*graphs)
@@ -1333,12 +1333,9 @@ public:
             graphs->query().getName(graphNameStr);
             const char *graphName = graphNameStr.s.str();
             Owned<IPropertyTree> graphXgmml = graphs->query().getXGMMLTree(false);
-            IPropertyTree *newGraph = createPTree();
+            IPropertyTree *newGraph = tree->addPropTree("Graph");
             newGraph->setProp("@id", graphName);
-            IPropertyTree *newXGMML = createPTree();
-            newXGMML->addPropTree("graph", graphXgmml.getLink());
-            newGraph->addPropTree("xgmml", newXGMML);
-            tree->addPropTree("Graph", newGraph);
+            newGraph->addPropTree("xgmml")->addPropTree("graph", graphXgmml.getLink());
         }
         return tree.getClear();
     }
@@ -1383,7 +1380,7 @@ public:
     }
     virtual void getQueryInfo(StringBuffer &reply, bool full, IArrayOf<IQueryFactory> *slaveQueries, const IRoxieContextLogger &logctx) const
     {
-        Owned<IPropertyTree> xref = createPTree("Query", 0);
+        Owned<IPropertyTree> xref = createPTree("Query", ipt_fast);
         xref->setProp("@id", id);
         if (suspended())
         {

roxie/ccd/ccdserver.cpp

@@ -3202,7 +3202,7 @@ void throwRemoteException(IMessageUnpackCursor *extra)
     {
         char *xml = (char *) extra->getNext(*rowlen);
         ReleaseRoxieRow(rowlen);
-        Owned<IPropertyTree> p = createPTreeFromXMLString(xml);
+        Owned<IPropertyTree> p = createPTreeFromXMLString(xml, ipt_fast);
         ReleaseRoxieRow(xml);
         unsigned code = p->getPropInt("Code", 0);
         const char *msg = p->queryProp("Message");
@@ -11940,7 +11940,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
                 throw MakeStringException(0, "Invalid name %s in user metadata for index %s (not legal XML element name)", name.str(), fname.get());
             }
             if(!metadata)
-                metadata.setown(createPTree("metadata"));
+                metadata.setown(createPTree("metadata", ipt_fast));
             metadata->setProp(name.str(), value.str());
         }
     }
@@ -11948,7 +11948,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
     void buildLayoutMetadata(Owned<IPropertyTree> & metadata)
     {
         if(!metadata)
-            metadata.setown(createPTree("metadata"));
+            metadata.setown(createPTree("metadata", ipt_fast));
         metadata->setProp("_record_ECL", helper.queryRecordECL());
 
         void * layoutMetaBuff;
@@ -12101,7 +12101,7 @@ public:
         //properties of the first file part.
         Owned<IPropertyTree> attrs;
         if(clusterHandler)
-            attrs.setown(createPTree("Part"));  // clusterHandler is going to set attributes
+            attrs.setown(createPTree("Part", ipt_fast));  // clusterHandler is going to set attributes
         else
         {
             // add cluster
@@ -15570,7 +15570,7 @@ public:
     {
         if (numUses > 1)
         {
-            Owned<IPropertyTree> splitterNode = createPTree();
+            Owned<IPropertyTree> splitterNode = createPTree(ipt_fast);
             factory.setown(createRoxieServerThroughSpillActivityFactory(sourceAct->queryFactory()->queryQueryFactory(), createGraphOutputSplitter, numUses, *splitterNode));
             IRoxieServerActivity *splitter = factory->createActivity(ctx, NULL);
             splitter->onCreate(NULL);

roxie/ccd/ccdsnmp.cpp

@@ -953,7 +953,7 @@ public:
     }
     static IPropertyTree *getAllQueryStats(bool includeQueries, time_t from, time_t to)
     {
-        Owned<IPTree> result = createPTree("QueryStats");
+        Owned<IPTree> result = createPTree("QueryStats", ipt_fast);
         if (includeQueries)
         {
             SpinBlock b(queryStatsCrit);
@@ -994,7 +994,7 @@ public:
     {
         time_t timeNow;
         time(&timeNow);
-        Owned<IPropertyTree> result = createPTree("Query");
+        Owned<IPropertyTree> result = createPTree("Query", ipt_fast);
         result->setProp("@id", queryName);
         if (expirySeconds && difftime(timeNow, from) <= expirySeconds)
         {

roxie/ccd/ccdstate.cpp

@@ -1374,13 +1374,13 @@ public:
             {
                 StringBuffer freply;
                 serverManager->getStats(queryId, graphName, freply, logctx);
-                Owned<IPropertyTree> stats = createPTreeFromXMLString(freply.str());
+                Owned<IPropertyTree> stats = createPTreeFromXMLString(freply.str(), ipt_fast);
                 for (unsigned channel = 0; channel < numChannels; channel++)
                     if (slaveManagers->item(channel))
                     {
                         StringBuffer sreply;
                         slaveManagers->item(channel)->getStats(queryId, graphName, sreply, logctx);
-                        Owned<IPropertyTree> cstats = createPTreeFromXMLString(sreply.str());
+                        Owned<IPropertyTree> cstats = createPTreeFromXMLString(sreply.str(), ipt_fast);
                         mergeStats(stats, cstats, 1);
                     }
                 toXML(stats, reply);
@@ -1523,7 +1523,7 @@ public:
     virtual void load(bool forceReload)
     {
         hash64_t newHash = numChannels;
-        Owned<IPropertyTree> newQuerySet = createPTree("QuerySet");
+        Owned<IPropertyTree> newQuerySet = createPTree("QuerySet", ipt_lowmem);
         newQuerySet->setProp("@name", "_standalone");
         newQuerySet->addPropTree("Query", standaloneDll.getLink());
         Owned<CRoxieSlaveQuerySetManagerSet> newSlaveManagers = new CRoxieSlaveQuerySetManagerSet(numChannels, querySet);
@@ -1559,7 +1559,7 @@ public:
     : stateHash(0), daliHelper(_daliHelper)
     {
         Owned<IPropertyTree> standAloneDllTree;
-        standAloneDllTree.setown(createPTree("Query"));
+        standAloneDllTree.setown(createPTree("Query", ipt_lowmem));
         standAloneDllTree->setProp("@id", "roxie");
         standAloneDllTree->setProp("@dll", standAloneDll->queryDll()->queryName());
         Owned<CRoxieQueryPackageManager> qpm = new CStandaloneQueryPackageManager(numChannels, querySet, LINK(&queryEmptyRoxiePackageMap()), standAloneDllTree.getClear());

system/jlib/jdebug.cpp

@@ -2612,6 +2612,8 @@ public:
         if(!mode) return;
         unsigned memused=0;
         unsigned memtot=0;
+        str.appendf("LPT=%u ", queryNumLocalTrees());
+        str.appendf("APT=%u ", queryNumAtomTrees());
         if(mode & PerfMonProcMem)
         {
             if (!outofhandles)

system/jlib/jhash.hpp

@@ -518,6 +518,23 @@ public:
         return c;
     }
 
+    unsigned findIndex(const char *key, unsigned h)
+    {
+        unsigned i=h%htn;
+        while (table[i]) {
+            if ((table[i]->hash==h)&&table[i]->eq(key))
+                return i;
+            if (++i==htn)
+                i = 0;
+        }
+        return (unsigned) -1;
+    }
+
+    C *getIndex(unsigned v) const
+    {
+        assert(v != (unsigned)-1 && v < htn);
+        return table[v];
+    }
 
     void remove(C *c)
     {

system/jlib/jptree-attrs.hpp

@@ -0,0 +1,81 @@
+    "@accessed",
+    "@activity",
+    "@agentSession",
+    "@buildVersion",
+    "@checkSum",
+    "@cloneable",
+    "@cluster",
+    "@clusterName",
+    "@codeVersion",
+    "@columnMapping",
+    "@connected",
+    "@created",
+    "@creator",
+    "@csvSeparate",
+    "@csvTerminate",
+    "@defaultBaseDir",
+    "@defaultReplicateDir",
+    "@description",
+    "@directory",
+    "@dllname",
+    "@eclVersion",
+    "@enqueuedt",
+    "@expireDays",
+    "@fetchEntire",
+    "@fieldwidth",
+    "@fileCrc",
+    "@filename",
+    "@footerLength",
+    "@format",
+    "@formatCrc",
+    "@hasArchive",
+    "@headerLength",
+    "@interleaved",
+    "@isArchive",
+    "@isClone",
+    "@isLibrary",
+    "@isScalar",
+    "@jobName",
+    "@libCount",
+    "@localValue",
+    "@mapFlags",
+    "@maxActivity",
+    "@minActivity",
+    "@modified",
+    "@numclusters",
+    "@numparts",
+    "@numsubfiles",
+    "@offset",
+    "@partmask",
+    "@persistent",
+    "@prevenqueuedt",
+    "@prevpriority",
+    "@prevwuid",
+    "@priority",
+    "@publishedBy",
+    "@recordCount",
+    "@recordSize",
+    "@recordSizeEntry",
+    "@redundancy",
+    "@rowLimit",
+    "@rowTag",
+    "@sequence",
+    "@serverId",
+    "@session",
+    "@severity",
+    "@source",
+    "@stateEx",
+    "@status",
+    "@submitID",
+    "@table:date-value",
+    "@table:number-columns-repeated",
+    "@table:style-name",
+    "@table:value",
+    "@table:value-type",
+    "@target",
+    "@totalThorTime",
+    "@transformerEntry",
+    "@waiting",
+    "@workunit",
+    "@wuidVersion",
+    "@xmlns:xsi",

system/jlib/jptree.cpp

@@ -70,34 +70,75 @@ IPropertyTreeIterator *createNullPTreeIterator() { return LINK(nullPTreeIterator
 
 //===================================================================
 
+#ifdef USE_READONLY_ATOMTABLE
+RONameTable *AttrStrUnionWithTable::roNameTable = nullptr;
+#endif
 static AtomRefTable *keyTable = nullptr;
 static AtomRefTable *keyTableNC = nullptr;
+
 static CriticalSection hashcrit;
 static CAttrValHashTable *attrHT = nullptr;
 static AttrValue **freelist = nullptr;
 static unsigned freelistmax = 0;
 static CLargeMemoryAllocator freeallocator((memsize_t)-1, 0x1000*sizeof(AttrValue), true);
 
+#ifdef USE_READONLY_ATOMTABLE
+static const char * roAttributes[] =
+{
+#include "jptree-attrs.hpp"    // potentially auto-generated
+    nullptr
+};
+
+void initializeRoTable()
+{
+    for (const char **attr = roAttributes; *attr; attr++)
+    {
+        AttrStrUnionWithTable::roNameTable->find(*attr, true);
+    }
+#ifdef TRACE_ATOM_SIZE
+    // If you are wanting an idea of the savings from use of the RO hash table, it may be useful to reset
+    // the counts here. But it's more correct to actually leave them in place.
+    //AttrStrAtom::totsize = 0;
+    //AttrStrAtom::maxsize = 0;
+#endif
+#ifdef _DEBUG
+    for (const char **a = roAttributes; *a; a++)
+    {
+        // sanity check
+        unsigned idx = AttrStrUnionWithTable::roNameTable->findIndex(*a,AttrStrC::getHash(*a));
+        AttrStrC *val = AttrStrUnionWithTable::roNameTable->getIndex(idx);
+        assert(val && val->eq(*a));
+    }
+#endif
+}
+#endif
+
 MODULE_INIT(INIT_PRIORITY_JPTREE)
 {
     nullPTreeIterator = new NullPTreeIterator;
-	keyTable = new AtomRefTable;
-	keyTableNC = new AtomRefTable(true);
-	attrHT = new CAttrValHashTable;
+#ifdef USE_READONLY_ATOMTABLE
+    AttrStrUnionWithTable::roNameTable = new RONameTable;
+    initializeRoTable();
+#endif
+    keyTable = new AtomRefTable;
+    keyTableNC = new AtomRefTable(true);
+    attrHT = new CAttrValHashTable;
     return true;
 }
 
 MODULE_EXIT()
 {
     nullPTreeIterator->Release();
-	delete attrHT;
-	keyTable->Release();
-	keyTableNC->Release();
-	free(freelist);
-	freelist = NULL;
+    delete attrHT;
+    keyTable->Release();
+    keyTableNC->Release();
+#ifdef USE_READONLY_ATOMTABLE
+    delete AttrStrUnionWithTable::roNameTable;
+#endif
+    free(freelist);
+    freelist = NULL;
 }
 
-
 static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
 {
     IPropertyTree *l = (IPropertyTree *) *ll;
@@ -110,7 +151,7 @@ static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
 unsigned ChildMap::getHashFromElement(const void *e) const
 {
     PTree &elem = (PTree &) (*(IPropertyTree *)e);
-    return elem.queryKey()->queryHash();
+    return elem.queryHash();
 }
 
 unsigned ChildMap::numChildren()
@@ -1735,11 +1776,11 @@ IAttributeIterator *PTree::getAttributes(bool sorted) const
         virtual bool isValid() override { return cur ? true : false; }
         virtual const char *queryName() const override
         {
-            return cur->key->get();
+            return cur->key.get();
         }
         virtual const char *queryValue() const override
         {
-            return cur->value->get();
+            return cur->value.get();
         }
         virtual StringBuffer &getValue(StringBuffer &out) override
         {
@@ -1761,7 +1802,7 @@ IAttributeIterator *PTree::getAttributes(bool sorted) const
 
         static int compareAttrs(AttrValue * const *ll, AttrValue * const *rr)
         {
-            return stricmp((*ll)->key->get(), (*rr)->key->get());
+            return stricmp((*ll)->key.get(), (*rr)->key.get());
         };
 
         CSortedAttributeIterator(const PTree *_parent) : cur(NULL), iter(NULL), parent(_parent)
@@ -1804,12 +1845,12 @@ IAttributeIterator *PTree::getAttributes(bool sorted) const
         virtual const char *queryName() const override
         {
             assertex(cur);
-            return cur->key->get();
+            return cur->key.get();
         }
         virtual const char *queryValue() const override
         {
             assertex(cur);
-            return cur->value->get();
+            return cur->value.get();
         }
         virtual StringBuffer &getValue(StringBuffer &out) override
         {
@@ -2822,7 +2863,7 @@ AttrValue *PTree::findAttribute(const char *key) const
     {
         while (a-- != attrs)
         {
-            if (strieq(a->key->get(), key))
+            if (strieq(a->key.get(), key))
                 return a;
         }
     }
@@ -2830,7 +2871,7 @@ AttrValue *PTree::findAttribute(const char *key) const
     {
         while (a-- != attrs)
         {
-            if (streq(a->key->get(), key))
+            if (streq(a->key.get(), key))
                 return a;
         }
     }
@@ -2841,7 +2882,7 @@ const char *PTree::getAttributeValue(const char *key) const
 {
     AttrValue *e = findAttribute(key);
     if (e)
-        return e->value->get();
+        return e->value.get();
     return nullptr;
 }
 
@@ -2869,36 +2910,48 @@ AttrValue *PTree::getNextAttribute(AttrValue *cur) const
 
 // LocalPTree
 
-LocalPTree::LocalPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags, _value, _children)
+static RelaxedAtomic<unsigned> numLocalTrees;
+unsigned queryNumLocalTrees()
+{
+    return numLocalTrees;
+}
+
+LocalPTree::LocalPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_fast, _value, _children)
 {
     if (_name)
         setName(_name);
+    numLocalTrees++;
 }
 
 LocalPTree::~LocalPTree()
 {
-    if (name)
-        free(name);
+    numLocalTrees--;
+    name.destroy();
     if (!attrs)
         return;
     AttrValue *a = attrs+numAttrs;
     while (a--!=attrs)
     {
-        free(a->key);
-        free(a->value);
+        a->key.destroy();
+        a->value.destroy();
     }
     free(attrs);
 }
 
+const char *LocalPTree::queryName() const
+{
+    return name.get();
+}
+
 void LocalPTree::setName(const char *_name)
 {
-    HashKeyElement *oname = name;
-    if (!_name)
-        name = nullptr;
-    else
-        name = AtomRefTable::createKeyElement(_name, isnocase());
+    if (_name==name.get())
+        return;
+    AttrStr *oname = name.getPtr();  // Don't free until after we copy - they could overlap
+    if (!name.set(_name))
+        name.setPtr(AttrStr::create(_name));
     if (oname)
-        free(oname);
+        AttrStr::destroy(oname);
 }
 
 bool LocalPTree::removeAttribute(const char *key)
@@ -2908,8 +2961,8 @@ bool LocalPTree::removeAttribute(const char *key)
         return false;
     numAttrs--;
     unsigned pos = del-attrs;
-    free(del->key);
-    free(del->value);
+    del->key.destroy();
+    del->value.destroy();
     memmove(attrs+pos, attrs+pos+1, (numAttrs-pos)*sizeof(AttrValue));
     return true;
 }
@@ -2923,41 +2976,66 @@ void LocalPTree::setAttribute(const char *key, const char *val)
     if (!val)
         val = "";  // cannot have NULL value
     AttrValue *v = findAttribute(key);
+    AttrStr *goer = nullptr;
     if (v)
     {
-        if (streq(v->value->get(), val))
+        if (streq(v->value.get(), val))
             return;
-        AttrStrC::destroy((AttrStrC *)v->value);
-        v->value = AttrStrC::create(val);
+        goer = v->value.getPtr();
     }
     else
     {
         attrs = (AttrValue *)realloc(attrs, (numAttrs+1)*sizeof(AttrValue));
-        if (isnocase())
-            attrs[numAttrs].key = AttrStrNC::create(key);
-        else
-            attrs[numAttrs].key = AttrStrC::create(key);
-        attrs[numAttrs].value = AttrStrC::create(val);
-        numAttrs++;
+        v = &attrs[numAttrs++];
+        if (!v->key.set(key))
+            v->key.setPtr(isnocase() ? AttrStr::createNC(key) : AttrStr::create(key));
     }
+    if (!v->value.set(val))
+        v->value.setPtr(AttrStr::create(val));
+    if (goer)
+        AttrStr::destroy(goer);
 }
 
+#ifdef TRACE_STRING_SIZE
+std::atomic<__int64> AttrStr::totsize { 0 };
+std::atomic<__int64> AttrStr::maxsize { 0 };
+#endif
+
+#ifdef TRACE_ATOM_SIZE
+std::atomic<__int64> AttrStrAtom::totsize { 0 };
+std::atomic<__int64> AttrStrAtom::maxsize { 0 };
+#endif
 
 ///////////////////
 
-CAtomPTree::CAtomPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags, _value, _children)
+static RelaxedAtomic<unsigned> numAtomTrees;
+unsigned queryNumAtomTrees()
+{
+    return numAtomTrees;
+}
+
+CAtomPTree::CAtomPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_lowmem, _value, _children)
 {
+    numAtomTrees++;
     if (_name)
         setName(_name);
 }
 
 CAtomPTree::~CAtomPTree()
 {
+    numAtomTrees--;
     bool nc = isnocase();
-    if (name)
+    HashKeyElement *name_ptr = name.getPtr();
+    if (name_ptr)
     {
         AtomRefTable *kT = nc?keyTableNC:keyTable;
-        kT->releaseKey(name);
+#ifdef TRACE_ATOM_SIZE
+        size_t gosize = sizeof(HashKeyElement)+strlen(name_ptr->get())+1;
+        if (kT->releaseKey(name_ptr))
+            AttrStrAtom::totsize -= gosize;
+#else
+        kT->releaseKey(name_ptr);
+#endif
     }
     if (!attrs)
         return;
@@ -2966,8 +3044,10 @@ CAtomPTree::~CAtomPTree()
         CriticalBlock block(hashcrit);
         while (a--!=attrs)
         {
-            attrHT->removekey(a->key, nc);
-            attrHT->removeval(a->value);
+            if (a->key.isPtr())
+                attrHT->removekey(a->key.getPtr(), nc);
+            if (a->value.isPtr())
+                attrHT->removeval(a->value.getPtr());
         }
         freeAttrArray(attrs, numAttrs);
     }
@@ -2976,17 +3056,65 @@ CAtomPTree::~CAtomPTree()
 void CAtomPTree::setName(const char *_name)
 {
     AtomRefTable *kT = isnocase()?keyTableNC:keyTable;
-    HashKeyElement *oname = name;
+    HashKeyElement *oname = name.getPtr(); // NOTE - don't release yet as could overlap source name
     if (!_name)
-        name = nullptr;
+        name.setPtr(nullptr);
     else
     {
         if (!validateXMLTag(_name))
             throw MakeIPTException(PTreeExcpt_InvalidTagName, ": %s", _name);
-        name = kT->queryCreate(_name);
+        if (!name.set(_name))
+        {
+#ifdef TRACE_ALL_ATOM
+            DBGLOG("TRACE_ALL_ATOM: %s", _name);
+#endif
+#ifdef TRACE_ATOM_SIZE
+            bool didCreate;
+            name.setPtr(kT->queryCreate(_name, didCreate));
+            if (didCreate)
+            {
+                AttrStrAtom::totsize += sizeof(HashKeyElement)+strlen(_name)+1;
+                if (AttrStrAtom::totsize > AttrStrAtom::maxsize)
+                {
+                    AttrStrAtom::maxsize.store(AttrStrAtom::totsize);
+                    DBGLOG("TRACE_ATOM_SIZE: total size now %" I64F "d", AttrStrAtom::maxsize.load());
+                }
+            }
+#else
+            name.setPtr(kT->queryCreate(_name));
+#endif
+        }
     }
     if (oname)
+    {
+#ifdef TRACE_ATOM_SIZE
+        size_t gosize = sizeof(HashKeyElement)+strlen(oname->get())+1;
+        if (kT->releaseKey(oname))
+            AttrStrAtom::totsize -= gosize;
+#else
         kT->releaseKey(oname);
+#endif
+    }
+}
+
+const char *CAtomPTree::queryName() const
+{
+    return name.get();
+}
+
+unsigned CAtomPTree::queryHash() const
+{
+    if (name.isPtr())
+    {
+        assert(name.getPtr());
+        return name.getPtr()->queryHash();
+    }
+    else
+    {
+        const char *_name = name.get();
+        size32_t nl = strlen(_name);
+        return isnocase() ? hashnc((const byte *) _name, nl, 0): hashc((const byte *) _name, nl, 0);
+    }
 }
 
 AttrValue *CAtomPTree::newAttrArray(unsigned n)
@@ -3031,19 +3159,32 @@ void CAtomPTree::setAttribute(const char *key, const char *val)
     AttrValue *v = findAttribute(key);
     if (v)
     {
-        if (streq(v->value->get(), val))
+        if (streq(v->value.get(), val))
             return;
-        CriticalBlock block(hashcrit);
-        attrHT->removeval(v->value);
-        v->value = attrHT->addval(val);
+        AttrStr * goer = v->value.getPtr();
+        if (!v->value.set(val))
+        {
+            CriticalBlock block(hashcrit);
+            if (goer)
+                attrHT->removeval(goer);
+            v->value.setPtr(attrHT->addval(val));
+        }
+        else if (goer)
+        {
+            CriticalBlock block(hashcrit);
+            attrHT->removeval(goer);
+        }
     }
     else
     {
         CriticalBlock block(hashcrit);
         AttrValue *newattrs = newAttrArray(numAttrs+1);
         memcpy(newattrs, attrs, numAttrs*sizeof(AttrValue));
-        newattrs[numAttrs].key = attrHT->addkey(key, isnocase());
-        newattrs[numAttrs].value = attrHT->addval(val);
+        v = &newattrs[numAttrs];
+        if (!v->key.set(key))
+            v->key.setPtr(attrHT->addkey(key, isnocase()));
+        if (!v->value.set(val))
+            v->value.setPtr(attrHT->addval(val));
         freeAttrArray(attrs, numAttrs);
         numAttrs++;
         attrs = newattrs;
@@ -3057,8 +3198,10 @@ bool CAtomPTree::removeAttribute(const char *key)
         return false;
     numAttrs--;
     CriticalBlock block(hashcrit);
-    attrHT->removekey(del->key, isnocase());
-    attrHT->removeval(del->value);
+    if (del->key.isPtr())
+        attrHT->removekey(del->key.getPtr(), isnocase());
+    if (del->value.isPtr())
+        attrHT->removeval(del->value.getPtr());
     AttrValue *newattrs = newAttrArray(numAttrs);
     if (newattrs)
     {
@@ -3438,9 +3581,9 @@ IPropertyTreeIterator *PTStackIterator::popFromStack(StringAttr &path)
 
 // factory methods
 
-IPropertyTree *createPTree(MemoryBuffer &src)
+IPropertyTree *createPTree(MemoryBuffer &src, byte flags)
 {
-    IPropertyTree *tree = new DEFAULT_PTREE_TYPE();
+    IPropertyTree *tree = createPTree(nullptr, flags);
     tree->deserialize(src);
     return tree;
 }
@@ -6070,8 +6213,8 @@ jlib_decl void testJdocCompare()
 
 #endif
 
-
-class COrderedPTree : public DEFAULT_PTREE_TYPE
+template <class BASE_PTREE>
+class COrderedPTree : public BASE_PTREE
 {
     template <class BASECHILDMAP>
     class jlib_decl COrderedChildMap : public BASECHILDMAP
@@ -6137,43 +6280,54 @@ class COrderedPTree : public DEFAULT_PTREE_TYPE
         }
     };
 public:
-    COrderedPTree(const char *name=NULL, byte flags=ipt_none, IPTArrayValue *value=NULL, ChildMap *children=NULL)
-        : DEFAULT_PTREE_TYPE(name, flags|ipt_ordered, value, children) { }
+    typedef COrderedPTree<BASE_PTREE> SELF;
+    COrderedPTree<BASE_PTREE>(const char *name=NULL, byte flags=ipt_none, IPTArrayValue *value=NULL, ChildMap *children=NULL)
+        : BASE_PTREE(name, flags|ipt_ordered, value, children) { }
 
-    virtual bool isEquivalent(IPropertyTree *tree) const override { return (NULL != QUERYINTERFACE(tree, COrderedPTree)); }
+    virtual bool isEquivalent(IPropertyTree *tree) const override { return (NULL != QUERYINTERFACE(tree, COrderedPTree<BASE_PTREE>)); }
     virtual IPropertyTree *create(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL, bool existing=false) override
     {
-        return new COrderedPTree(name, flags, value, children);
+        return new COrderedPTree<BASE_PTREE>(name, SELF::flags, value, children);
     }
     virtual IPropertyTree *create(MemoryBuffer &mb) override
     {
-        IPropertyTree *tree = new COrderedPTree();
+        IPropertyTree *tree = new COrderedPTree<BASE_PTREE>();
         tree->deserialize(mb);
         return tree;
     }
     virtual void createChildMap() override
     {
-        if (isnocase())
-            children = new COrderedChildMap<ChildMapNC>();
+        if (SELF::isnocase())
+            SELF::children = new COrderedChildMap<ChildMapNC>();
         else
-            children = new COrderedChildMap<ChildMap>();
+            SELF::children = new COrderedChildMap<ChildMap>();
     }
 };
 
 IPropertyTree *createPTree(byte flags)
 {
-    if (flags & ipt_ordered)
-        return new COrderedPTree(NULL, flags);
-    else
-        return new DEFAULT_PTREE_TYPE(NULL, flags);
+    return createPTree(NULL, flags);
 }
 
 IPropertyTree *createPTree(const char *name, byte flags)
 {
-    if (flags & ipt_ordered)
-        return new COrderedPTree(name, flags);
-    else
+    switch (flags & (ipt_ordered|ipt_fast|ipt_lowmem))
+    {
+    case ipt_ordered|ipt_fast:
+        return new COrderedPTree<LocalPTree>(name, flags);
+    case ipt_ordered|ipt_lowmem:
+        return new COrderedPTree<CAtomPTree>(name, flags);
+    case ipt_ordered:
+        return new COrderedPTree<DEFAULT_PTREE_TYPE>(name, flags);
+    case ipt_fast:
+        return new LocalPTree(name, flags);
+    case ipt_lowmem:
+        return new CAtomPTree(name, flags);
+    case 0:
         return new DEFAULT_PTREE_TYPE(name, flags);
+    default:
+        throwUnexpectedX("Invalid flags - ipt_fast and ipt_lowmem should not be specified together");
+    }
 }
 
 typedef enum _ptElementType

system/jlib/jptree.hpp

@@ -61,6 +61,10 @@ typedef unsigned IPTIteratorCodes;
 #define iptiter_remote 0x02
 #define iptiter_remoteget 0x06
 #define iptiter_remotegetbranch 0x0e
+
+extern jlib_decl unsigned queryNumLocalTrees();
+extern jlib_decl unsigned queryNumAtomTrees();
+
 interface jlib_decl IPropertyTree : extends serializable
 {
     virtual bool hasProp(const char *xpath) const = 0;
@@ -98,6 +102,9 @@ interface jlib_decl IPropertyTree : extends serializable
     virtual IPropertyTree *setPropTree(const char *xpath, IPropertyTree *val) = 0;
     virtual IPropertyTree *addPropTree(const char *xpath, IPropertyTree *val) = 0;
 
+    virtual IPropertyTree *setPropTree(const char *xpath) = 0;
+    virtual IPropertyTree *addPropTree(const char *xpath) = 0;
+
     virtual bool removeProp(const char *xpath) = 0;
     virtual bool removeTree(IPropertyTree *child) = 0;
     virtual aindex_t queryChildIndex(IPropertyTree *child) = 0;
@@ -170,9 +177,19 @@ interface IPTreeNodeCreator : extends IInterface
     virtual IPropertyTree *create(const char *tag) = 0;
 };
 
-// NB ipt_ext4 - used by SDS
-// NB ipt_ext5 - used by SDS
-enum ipt_flags { ipt_none=0x00, ipt_caseInsensitive=0x01, ipt_binary=0x02, ipt_ordered=0x04, ipt_ext1=0x08, ipt_ext2=16, ipt_ext3=32, ipt_ext4=64, ipt_ext5=128 };
+enum ipt_flags
+{
+    ipt_none=0x00,
+    ipt_caseInsensitive=0x01,
+    ipt_binary  = 0x02,
+    ipt_ordered = 0x04,   // Preserve element ordering
+    ipt_fast    = 0x08,   // Prioritize speed over low memory usage
+    ipt_lowmem  = 0x10,   // Prioritize low memory usage over speed
+    ipt_ext3    = 0x20,   // Unused
+    ipt_ext4    = 0x40,   // Used internally in Dali
+    ipt_ext5    = 0x80    // Used internally in Dali
+};
+
 jlib_decl IPTreeMaker *createPTreeMaker(byte flags=ipt_none, IPropertyTree *root=NULL, IPTreeNodeCreator *nodeCreator=NULL);
 jlib_decl IPTreeMaker *createRootLessPTreeMaker(byte flags=ipt_none, IPropertyTree *root=NULL, IPTreeNodeCreator *nodeCreator=NULL);
 jlib_decl IPTreeReader *createXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions=ptr_ignoreWhiteSpace, size32_t bufSize=0);
@@ -194,7 +211,7 @@ jlib_decl void synchronizePTree(IPropertyTree *target, IPropertyTree *source);
 jlib_decl IPropertyTree *ensurePTree(IPropertyTree *root, const char *xpath);
 jlib_decl bool areMatchingPTrees(IPropertyTree * left, IPropertyTree * right);
 
-jlib_decl IPropertyTree *createPTree(MemoryBuffer &src);
+jlib_decl IPropertyTree *createPTree(MemoryBuffer &src, byte flags=ipt_none);
 
 jlib_decl IPropertyTree *createPTree(byte flags=ipt_none);
 jlib_decl IPropertyTree *createPTree(const char *name, byte flags=ipt_none);
@@ -205,7 +222,6 @@ jlib_decl IPropertyTree *createPTreeFromXMLString(const char *xml, byte flags=ip
 jlib_decl IPropertyTree *createPTreeFromXMLString(unsigned len, const char *xml, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromXMLFile(const char *filename, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromIPT(const IPropertyTree *srcTree, ipt_flags flags=ipt_none);
-
 jlib_decl IPropertyTree *createPTreeFromJSONString(const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 

system/jlib/jptree.ipp

@@ -28,6 +28,7 @@
 
 #include "jptree.hpp"
 #include "jbuff.hpp"
+#include "jlog.hpp"
 
 #define ANE_APPEND -1
 #define ANE_SET -2
@@ -205,7 +206,7 @@ public:
     virtual const void *queryValueRaw() const override { return get(); }
     virtual size32_t queryValueRawSize() const override { return (size32_t)length(); }
 
-// serilizable
+// serializable
     virtual void serialize(MemoryBuffer &tgt) override;
     virtual void deserialize(MemoryBuffer &src) override;
 
@@ -217,18 +218,313 @@ private:
 #define IptFlagSet(fs, f) (fs |= (f))
 #define IptFlagClr(fs, f) (fs &= (~f))
 
+// NOTE - hairy code alert!
+// In order to keep code common between atom and local versions of ptree, we store the atom-specific information BEFORE the this pointer of AttrStr
+// (see class AttrStrAtom below).
+// This requires some care - in particular must use the right method to destroy the objects, and must not add any virtual methods to either class
+
+//#define TRACE_STRING_SIZE
+//#define TRACE_ATOM_SIZE
+//#define TRACE_ALL_STRING
+//#define TRACE_ALL_ATOM
+
 struct AttrStr
 {
+    const char *get() const
+    {
+        return str_DO_NOT_USE_DIRECTLY;
+    }
+    char str_DO_NOT_USE_DIRECTLY[1];  // Actually [n] - null terminated
+
+    static AttrStr *create(const char *k)
+    {
+        size32_t kl = k ? strlen(k) : 0;
+#ifdef TRACE_ALL_STRING
+        DBGLOG("TRACE_ALL_STRING: %s", k);
+#endif
+#ifdef TRACE_STRING_SIZE
+        totsize += kl+1;
+        if (totsize > maxsize)
+        {
+            maxsize.store(totsize);
+            DBGLOG("TRACE_STRING_SIZE: total size now %" I64F "d", maxsize.load());
+        }
+#endif
+        AttrStr *ret = (AttrStr *) malloc(kl+1);
+        memcpy(ret->str_DO_NOT_USE_DIRECTLY, k, kl);
+        ret->str_DO_NOT_USE_DIRECTLY[kl] = 0;
+        return ret;
+    }
+    static inline AttrStr *createNC(const char *k)
+    {
+        // If we started to use a static hash table for common values, we would probably want to use a different one here for case-insensitive matches
+        return create(k);
+    }
+
+    static void destroy(AttrStr *a)
+    {
+#ifdef TRACE_STRING_SIZE
+        totsize -= strlen(a->str_DO_NOT_USE_DIRECTLY)+1;
+#endif
+        free(a);
+    }
+
+#ifdef TRACE_STRING_SIZE
+    static std::atomic<__int64> totsize;
+    static std::atomic<__int64> maxsize;
+#endif
+};
+
+
+// In order to keep code common between atom and local versions of ptree, we store the atom-specific information BEFORE the this pointer of AttrStr
+// This requires some care - in particular must use the right method to destroy the objects, and must not add any virtual methods to either class
+// Note that memory usage is significant as we create literally millions of these objects
+
+typedef unsigned hashfunc( const unsigned char *k, unsigned length, unsigned initval);
+
+struct AttrStrAtom
+{
     unsigned hash;
     unsigned short linkcount;
-    char str[1];
-    const char *get() const { return str; }
+    char str_DO_NOT_USE_DIRECTLY[1];  // Actually N
+
+    static AttrStrAtom *create(const char *k, size32_t kl, hashfunc _hash)
+    {
+#ifdef TRACE_ALL_ATOM
+        DBGLOG("TRACE_ALL_ATOM: %s", k);
+#endif
+#ifdef TRACE_ATOM_SIZE
+        totsize += sizeof(AttrStrAtom)+kl+1;
+        if (totsize > maxsize)
+        {
+            maxsize.store(totsize);
+            DBGLOG("TRACE_ATOM_SIZE: total size now %" I64F "d", maxsize.load());
+        }
+#endif
+        AttrStrAtom *ret = (AttrStrAtom *) malloc(offsetof(AttrStrAtom, str_DO_NOT_USE_DIRECTLY)+kl+1);
+        memcpy(ret->str_DO_NOT_USE_DIRECTLY, k, kl);
+        ret->str_DO_NOT_USE_DIRECTLY[kl] = 0;
+        ret->hash = _hash((const unsigned char *) k, kl, 17);
+        ret->linkcount = 0;
+        return ret;
+    }
+    static void destroy(AttrStrAtom *a)
+    {
+#ifdef TRACE_ATOM_SIZE
+        totsize -= sizeof(AttrStrAtom)+strlen(a->str_DO_NOT_USE_DIRECTLY)+1;
+#endif
+        free(a);
+    }
+
+    AttrStr *toAttrStr()
+    {
+        return (AttrStr *) &str_DO_NOT_USE_DIRECTLY;
+    }
+    static AttrStrAtom *toAtom(AttrStr *a)
+    {
+        return (AttrStrAtom *)(&a->str_DO_NOT_USE_DIRECTLY - offsetof(AttrStrAtom, str_DO_NOT_USE_DIRECTLY));
+    }
+#ifdef TRACE_ATOM_SIZE
+    static std::atomic<__int64> totsize;
+    static std::atomic<__int64> maxsize;
+#endif
+ };
+
+struct AttrStrC : public AttrStrAtom
+{
+    static inline unsigned getHash(const char *k)
+    {
+        return hashc((const byte *)k, strlen(k), 17);
+    }
+    inline bool eq(const char *k)
+    {
+        return streq(k,str_DO_NOT_USE_DIRECTLY);
+    }
+    static AttrStrC *create(const char *k)
+    {
+        size32_t kl = k ? strlen(k) : 0;
+        return (AttrStrC *) AttrStrAtom::create(k, kl, hashc);
+    }
+};
+
+struct AttrStrNC : public AttrStrAtom
+{
+    static inline unsigned getHash(const char *k)
+    {
+        return hashnc((const byte *)k, strlen(k), 17);
+    }
+    inline bool eq(const char *k)
+    {
+        return strieq(k,str_DO_NOT_USE_DIRECTLY);
+    }
+    static AttrStrNC *create(const char *k)
+    {
+        size32_t kl = k ? strlen(k) : 0;
+        return (AttrStrNC *) AttrStrAtom::create(k, kl, hashnc);
+    }
+};
+
+typedef CMinHashTable<AttrStrC> RONameTable;
+
+// NOTE - hairy code alert!
+// To save on storage (and contention) we store short string values in same slot as the pointer to longer
+// ones would occupy. This relies on the assumption that the pointers you want to store are always AT LEAST
+// 2-byte aligned. This should be the case on anything coming from malloc on any modern architecture.
+
+#define USE_STRUNION
+
+template<class PTR>
+struct PtrStrUnion
+{
+#ifdef USE_STRUNION
+    union
+    {
+        PTR *ptr;
+        struct
+        {
+#ifdef LITTLE_ENDIAN
+            char flag;
+            union
+            {
+                char chars[sizeof(PTR *)-1];
+                struct
+                {
+                    int8_t idx1;
+                    int16_t idx2;
+                };
+            };
+#else
+            union
+            {
+                char chars[sizeof(PTR *)-1];
+                struct
+                {
+                    int16_t idx2;
+                    int8_t idx1;
+                };
+            };
+            char flag;
+#endif
+        };
+    };
+    inline PtrStrUnion<PTR>() : ptr(nullptr) {}
+    inline bool isPtr() const
+    {
+        return (flag&1) == 0;
+    }
+    inline const char *get() const
+    {
+        if (!isPtr())
+        {
+            assert(flag==1);
+            return chars;
+        }
+        else if (ptr)
+            return ptr->get();
+        else
+            return nullptr;
+    }
+    inline void destroy()
+    {
+        if (isPtr() && ptr)
+            PTR::destroy(ptr);
+    }
+    bool set(const char *key)
+    {
+        if (key)
+        {
+            size32_t l = strnlen(key, sizeof(PTR *));  // technically sizeof(PTR)-1 would do, but I suspect 8 bytes is actually more optimal to search than 7
+            if (l <= sizeof(PTR *)-2)
+            {
+                flag=1;
+                memmove(chars, key, l);  // Technically, they could overlap
+                chars[l]=0;
+                return true;
+            }
+        }
+        return false;
+    }
+    inline void setPtr(PTR *a)
+    {
+        ptr = a;
+        assert(isPtr());
+    }
+#else
+    PTR *ptr = nullptr;
+    inline bool isPtr()
+    {
+        return true;
+    }
+    inline const char *get()
+    {
+        if (ptr)
+            return ptr->get();
+        else
+            return nullptr;
+    }
+    inline void destroy()
+    {
+        if (ptr)
+            PTR::destroy(ptr);
+    }
+    bool set(const char *key)
+    {
+        return false;
+    }
+    inline void setPtr(PTR *a)
+    {
+        ptr = a;
+    }
+#endif
+    inline PTR *getPtr() const
+    {
+        return isPtr() ? ptr : nullptr;
+    }
+};
+
+#ifdef USE_STRUNION
+#define USE_READONLY_ATOMTABLE
+#endif
+
+typedef PtrStrUnion<AttrStr> AttrStrUnion;
+
+#ifdef USE_READONLY_ATOMTABLE
+struct AttrStrUnionWithTable : public AttrStrUnion
+{
+    inline const char *get() const
+    {
+        if (!isPtr() && flag==3)
+            return roNameTable->getIndex(idx2)->str_DO_NOT_USE_DIRECTLY;  // Should probably rename this back now!
+        return AttrStrUnion::get();
+    }
+    bool set(const char *key)
+    {
+        if (AttrStrUnion::set(key))
+            return true;
+        if (key && key[0]=='@')
+        {
+            unsigned idx = roNameTable->findIndex(key, AttrStrC::getHash(key));
+            if (idx != (unsigned) -1)
+            {
+                assert(idx <= 0xffff);
+                flag = 3;
+                idx2 = idx;
+                return true;
+            }
+        }
+        return false;
+    }
+    static RONameTable *roNameTable;
 };
+#else
+typedef AttrStrUnion AttrStrUnionWithTable;
+
+#endif
 
 struct AttrValue
 {
-    AttrStr *key;
-    AttrStr *value;
+    AttrStrUnionWithTable key;
+    AttrStrUnion value;
 };
 
 
@@ -244,11 +540,7 @@ public:
     ~PTree();
     virtual void beforeDispose() override { }
 
-    const char *queryName() const
-    {
-        return name?name->get():nullptr;
-    }
-    HashKeyElement *queryKey() const { return name; }
+    virtual unsigned queryHash() const = 0;
     IPropertyTree *queryParent() { return parent; }
     IPropertyTree *queryChild(unsigned index);
     ChildMap *queryChildren() { return children; }
@@ -313,6 +605,8 @@ public:
     virtual IPropertyTree *queryBranch(const char *xpath) const override { return queryPropTree(xpath); }
     virtual IPropertyTree *setPropTree(const char *xpath, IPropertyTree *val) override;
     virtual IPropertyTree *addPropTree(const char *xpath, IPropertyTree *val) override;
+    virtual IPropertyTree *setPropTree(const char *xpath) override { return setPropTree(xpath, create()); }
+    virtual IPropertyTree *addPropTree(const char *xpath) override { return addPropTree(xpath, create()); }
     virtual bool removeTree(IPropertyTree *child) override;
     virtual bool removeProp(const char *xpath) override;
     virtual aindex_t queryChildIndex(IPropertyTree *child) override;
@@ -366,63 +660,10 @@ protected: // data
     IPropertyTree *parent = nullptr; // ! currently only used if tree embedded into array, used to locate position.
     ChildMap *children;   // set by constructor
     IPTArrayValue *value; // set by constructor
-    HashKeyElement *name = nullptr;
     AttrValue *attrs = nullptr;
 };
 
 
-struct AttrStrC : public AttrStr
-{
-    static inline unsigned getHash(const char *k)
-    {
-        return hashc((const byte *)k, strlen(k), 17);
-    }
-    inline bool eq(const char *k)
-    {
-        return streq(k,str);
-    }
-    static AttrStrC *create(const char *k)
-    {
-        size32_t kl = (k?strlen(k):0);
-        AttrStrC *ret = (AttrStrC *)malloc(sizeof(AttrStrC)+kl);
-        memcpy(ret->str, k, kl);
-        ret->str[kl] = 0;
-        ret->hash = hashc((const byte *)k, kl, 17);
-        ret->linkcount = 0;
-        return ret;
-    }
-    static void destroy(AttrStrC *a)
-    {
-        free(a);
-    }
-};
-
-struct AttrStrNC : public AttrStr
-{
-    static inline unsigned getHash(const char *k)
-    {
-        return hashnc((const byte *)k, strlen(k), 17);
-    }
-    inline bool eq(const char *k)
-    {
-        return strieq(k,str);
-    }
-    static AttrStrNC *create(const char *k)
-    {
-        size32_t kl = (k?strlen(k):0);
-        AttrStrNC *ret = (AttrStrNC *)malloc(sizeof(AttrStrNC)+kl);
-        memcpy(ret->str,k,kl);
-        ret->str[kl] = 0;
-        ret->hash = hashnc((const byte *)k, kl, 17);
-        ret->linkcount = 0;
-        return ret;
-    }
-    static void destroy(AttrStrNC *a)
-    {
-        free(a);
-    }
-};
-
 class CAttrValHashTable
 {
     CMinHashTable<AttrStrC>  htc;
@@ -431,24 +672,25 @@ class CAttrValHashTable
 public:
     inline AttrStr *addkey(const char *v,bool nc)
     {
-        AttrStr * ret;
+        AttrStrAtom * ret;
         if (nc)
             ret = htnc.find(v,true);
         else
             ret = htc.find(v,true);
         if (ret->linkcount!=(unsigned short)-1)
             ret->linkcount++;
-        return ret;
+        return ret->toAttrStr();
     }
     inline AttrStr *addval(const char *v)
     {
-        AttrStr * ret = htv.find(v,true);
+        AttrStrAtom * ret = htv.find(v,true);
         if (ret->linkcount!=(unsigned short)-1)
             ret->linkcount++;
-        return ret;
+        return ret->toAttrStr();
     }
-    inline void removekey(AttrStr *a,bool nc)
+    inline void removekey(AttrStr *_a,bool nc)
     {
+        AttrStrAtom *a = AttrStrAtom::toAtom(_a);
         if (a->linkcount!=(unsigned short)-1)
         {
             if (--(a->linkcount)==0)
@@ -460,8 +702,9 @@ public:
             }
         }
     }
-    inline void removeval(AttrStr *a)
+    inline void removeval(AttrStr *_a)
     {
+        AttrStrAtom *a = AttrStrAtom::toAtom(_a);
         if (a->linkcount!=(unsigned short)-1)
             if (--(a->linkcount)==0)
                 htv.remove((AttrStrC *)a);
@@ -473,13 +716,15 @@ class jlib_decl CAtomPTree : public PTree
 {
     AttrValue *newAttrArray(unsigned n);
     void freeAttrArray(AttrValue *a, unsigned n);
-
+    PtrStrUnion<HashKeyElement> name;
 protected:
     virtual void setAttribute(const char *attr, const char *val) override;
     virtual bool removeAttribute(const char *k) override;
 public:
     CAtomPTree(const char *name=nullptr, byte flags=ipt_none, IPTArrayValue *value=nullptr, ChildMap *children=nullptr);
     ~CAtomPTree();
+    const char *queryName() const override;
+    virtual unsigned queryHash() const override;
     virtual void setName(const char *_name) override;
     virtual bool isEquivalent(IPropertyTree *tree) const override { return (nullptr != QUERYINTERFACE(tree, CAtomPTree)); }
     virtual IPropertyTree *create(const char *name=nullptr, IPTArrayValue *value=nullptr, ChildMap *children=nullptr, bool existing=false) override
@@ -502,10 +747,19 @@ class jlib_decl LocalPTree : public PTree
 protected:
     virtual void setAttribute(const char *attr, const char *val) override;
     virtual bool removeAttribute(const char *k) override;
+    AttrStrUnion name;
 public:
     LocalPTree(const char *name=nullptr, byte flags=ipt_none, IPTArrayValue *value=nullptr, ChildMap *children=nullptr);
     ~LocalPTree();
 
+    const char *queryName() const override;
+    virtual unsigned queryHash() const override
+    {
+        const char *myname = queryName();
+        assert(myname);
+        size32_t nl = strlen(myname);
+        return isnocase() ? hashnc((const byte *)myname, nl, 0): hashc((const byte *)myname, nl, 0);
+    }
     virtual void setName(const char *_name) override;
     virtual bool isEquivalent(IPropertyTree *tree) const override { return (nullptr != QUERYINTERFACE(tree, LocalPTree)); }
     virtual IPropertyTree *create(const char *name=nullptr, IPTArrayValue *value=nullptr, ChildMap *children=nullptr, bool existing=false) override

system/jlib/jsuperhash.hpp

@@ -72,11 +72,12 @@ protected:
     bool             removeExact(void * et);
     inline void      setCache(unsigned v) const { cache = v; }
 
+    inline unsigned  doFind(const void * findParam) const
+      { return doFind(getHashFromFindParam(findParam), findParam); }
+
 private:
     bool             doAdd(void *, bool);
     void             doDeleteElement(unsigned);
-    inline unsigned  doFind(const void * findParam) const
-      { return doFind(getHashFromFindParam(findParam), findParam); }
     unsigned         doFind(unsigned, const void *) const;
     unsigned         doFindElement(unsigned, const void *) const;
     unsigned         doFindNew(unsigned) const;
@@ -530,20 +531,38 @@ public:
         return key;
     }
 
+    inline HashKeyElement *queryCreate(const char *_key, bool &didCreate)
+    {
+        CriticalBlock b(crit);
+        HashKeyElement *key = find(*_key);
+        if (key)
+        {
+            didCreate = false;
+            key->linkCount++;
+        }
+        else
+        {
+            didCreate = true;
+            key = createKeyElement(_key);
+        }
+        return key;
+    }
+
     inline void linkKey(const char *key)
     {
         queryCreate(key);
     }
 
-    inline void releaseKey(HashKeyElement *key)
+    inline bool releaseKey(HashKeyElement *key)
     {
         CriticalBlock b(crit);
         if (0 == key->linkCount)
         {
             verifyex(removeExact(key));
-            return;
+            return true;
         }
         --key->linkCount;
+        return false;
     }
 
 protected:

system/jlib/jutil.cpp

@@ -2378,7 +2378,7 @@ IPropertyTree *getHPCCEnvironment()
         {
             Owned<IFileIO> fileio = file->open(IFOread);
             if (fileio)
-                return createPTree(*fileio);
+                return createPTree(*fileio, ipt_lowmem);
         }
     }
     return NULL;

testing/unittests/unittests.cpp

@@ -102,11 +102,13 @@ void loadDlls(IArray &objects, const char * libDirectory)
     {
         const char *thisDll = libFiles->query().queryFilename();
         if (!strstr(thisDll, "javaembed"))  // Bit of a hack, but loading this if java not present terminates...
-        {
-            LoadedObject *loaded = loadDll(thisDll);
-            if (loaded)
-                objects.append(*loaded);
-        }
+            if (!strstr(thisDll, "py2embed"))      // These two clash, so ...
+                if (!strstr(thisDll, "py3embed"))  // ... best to load neither...
+                {
+                    LoadedObject *loaded = loadDll(thisDll);
+                    if (loaded)
+                        objects.append(*loaded);
+                }
     }
 }
 
@@ -280,6 +282,274 @@ class InternalStatisticsTest : public CppUnit::TestFixture
 CPPUNIT_TEST_SUITE_REGISTRATION( InternalStatisticsTest );
 CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( InternalStatisticsTest, "StatisticsTest" );
 
+class PtreeThreadingTest : public CppUnit::TestFixture
+{
+    CPPUNIT_TEST_SUITE( PtreeThreadingTest  );
+        CPPUNIT_TEST(testContention);
+    CPPUNIT_TEST_SUITE_END();
+
+    void testContention()
+    {
+        _testContention(ipt_lowmem);
+        _testContention(ipt_fast);
+    }
+    void _testContention(byte flags)
+    {
+        enum ContentionMode { max_contention, some_contention, min_contention, some_control, min_control };
+        class casyncfor: public CAsyncFor
+        {
+            volatile int v;
+            void donothing()
+            {
+                v++;
+            }
+            byte flags = ipt_none;
+            ContentionMode mode = max_contention;
+            int iterations = 0;
+            const char *desc = nullptr;
+
+        public:
+            casyncfor(const char *_desc, byte _flags, ContentionMode _mode, int _iter)
+            : flags(_flags), mode(_mode), iterations(_iter), desc(_desc)
+            {
+            };
+            double For(unsigned num, unsigned maxatonce, double overhead = 0.0)
+            {
+                unsigned start = msTick();
+                CAsyncFor::For(num, maxatonce);
+                unsigned elapsed = msTick()-start;
+                double looptime = (elapsed * 1.0) / (iterations*num);
+                if (mode < 3)
+                    DBGLOG("%s (%s) test completed in %u ms (%f ms/iter)", desc, flags & ipt_fast ? "fast" : "lowmem", elapsed, looptime-overhead);
+                return looptime;
+            }
+            void Do(unsigned i)
+            {
+                for (unsigned i = 0; i < iterations; i++)
+                {
+                    Owned<IPropertyTree> p = mode >= some_control ? nullptr : createPTreeFromXMLString(
+                            "<W_LOCAL buildVersion='community_6.0.0-trunk0Debug[heads/cass-wu-part3-0-g10b954-dirty]'"
+                            "         cloneable='1'"
+                            "         clusterName=''"
+                            "         codeVersion='158'"
+                            "         eclVersion='6.0.0'"
+                            "         hash='2796091347'"
+                            "         state='completed'"
+                            "         xmlns:xsi='http://www.w3.org/1999/XMLSchema-instance'>"
+                            " <Debug>"
+                            "  <debugquery>1</debugquery>"
+                            "  <expandpersistinputdependencies>1</expandpersistinputdependencies>"
+                            "  <savecpptempfiles>1</savecpptempfiles>"
+                            "  <saveecltempfiles>1</saveecltempfiles>"
+                            "  <spanmultiplecpp>0</spanmultiplecpp>"
+                            "  <standaloneexe>1</standaloneexe>"
+                            "  <targetclustertype>hthor</targetclustertype>"
+                            " </Debug>"
+                            " <FilesRead>"
+                            "  <File name='myfile' useCount='2' cluster = 'mycluster'/>"
+                            "  <File name='mysuperfile' useCount='2' cluster = 'mycluster'>"
+                            "   <Subfile name='myfile'/>"
+                            "  </File>"
+                            "</FilesRead>"
+                            " <Graphs>"
+                            "  <Graph name='graph1' type='activities'>"
+                            "   <xgmml>"
+                            "    <graph wfid='2'>"
+                            "     <node id='1'>"
+                            "      <att>"
+                            "       <graph>"
+                            "        <att name='rootGraph' value='1'/>"
+                            "        <edge id='2_0' source='2' target='3'/>"
+                            "        <edge id='3_0' source='3' target='4'/>"
+                            "        <edge id='4_0' source='4' target='5'/>"
+                            "        <node id='2' label='Inline Row&#10;{1}'>"
+                            "         <att name='definition' value='./sets.ecl(2,13)'/>"
+                            "         <att name='_kind' value='148'/>"
+                            "         <att name='ecl' value='ROW(TRANSFORM({ integer8 v },SELF.v := 1;));&#10;'/>"
+                            "         <att name='recordSize' value='8'/>"
+                            "         <att name='predictedCount' value='1'/>"
+                            "        </node>"
+                            "        <node id='3' label='Filter'>"
+                            "         <att name='definition' value='./sets.ecl(3,15)'/>"
+                            "         <att name='_kind' value='5'/>"
+                            "         <att name='ecl' value='FILTER(v = STORED(&apos;one&apos;));&#10;'/>"
+                            "         <att name='recordSize' value='8'/>"
+                            "         <att name='predictedCount' value='0..?[disk]'/>"
+                            "        </node>"
+                            "        <node id='4' label='Count'>"
+                            "         <att name='_kind' value='125'/>"
+                            "         <att name='ecl' value='TABLE({ integer8 value := COUNT(group) });&#10;'/>"
+                            "         <att name='recordSize' value='8'/>"
+                            "         <att name='predictedCount' value='1'/>"
+                            "        </node>"
+                            "        <node id='5' label='Store&#10;Internal(&apos;wf2&apos;)'>"
+                            "         <att name='_kind' value='22'/>"
+                            "         <att name='ecl' value='extractresult(value, named(&apos;wf2&apos;));&#10;'/>"
+                            "         <att name='recordSize' value='8'/>"
+                            "        </node>"
+                            "       </graph>"
+                            "      </att>"
+                            "     </node>"
+                            "    </graph>"
+                            "   </xgmml>"
+                            "  </Graph>"
+                            "  <Graph name='graph2' type='activities'>"
+                            "   <xgmml>"
+                            "    <graph wfid='3'>"
+                            "     <node id='6'>"
+                            "      <att>"
+                            "       <graph>"
+                            "        <att name='rootGraph' value='1'/>"
+                            "        <edge id='7_0' source='7' target='8'/>"
+                            "        <edge id='8_0' source='8' target='9'/>"
+                            "        <node id='7' label='Inline Row&#10;{1}'>"
+                            "         <att name='definition' value='./sets.ecl(2,13)'/>"
+                            "         <att name='_kind' value='148'/>"
+                            "         <att name='ecl' value='ROW(TRANSFORM({ integer8 v },SELF.v := 1;));&#10;'/>"
+                            "         <att name='recordSize' value='8'/>"
+                            "         <att name='predictedCount' value='1'/>"
+                            "        </node>"
+                            "        <node id='8' label='Filter'>"
+                            "         <att name='definition' value='./sets.ecl(5,1)'/>"
+                            "         <att name='_kind' value='5'/>"
+                            "         <att name='ecl' value='FILTER(v = INTERNAL(&apos;wf2&apos;));&#10;'/>"
+                            "         <att name='recordSize' value='8'/>"
+                            "         <att name='predictedCount' value='0..?[disk]'/>"
+                            "        </node>"
+                            "        <node id='9' label='Output&#10;Result #1'>"
+                            "         <att name='definition' value='./sets.ecl(1,1)'/>"
+                            "         <att name='name' value='sets'/>"
+                            "         <att name='definition' value='./sets.ecl(5,1)'/>"
+                            "         <att name='_kind' value='16'/>"
+                            "         <att name='ecl' value='OUTPUT(..., workunit);&#10;'/>"
+                            "         <att name='recordSize' value='8'/>"
+                            "        </node>"
+                            "       </graph>"
+                            "      </att>"
+                            "     </node>"
+                            "    </graph>"
+                            "   </xgmml>"
+                            "  </Graph>"
+                            " </Graphs>"
+                            " <Query fetchEntire='1'>"
+                            "  <Associated>"
+                            "   <File desc='a.out.cpp'"
+                            "         filename='/Users/rchapman/HPCC-Platform/ossd/a.out.cpp'"
+                            "         ip='192.168.2.203'"
+                            "         type='cpp'/>"
+                            "  </Associated>"
+                            " </Query>"
+                            " <Results>"
+                            "  <Result isScalar='0'"
+                            "          name='Result 1'"
+                            "          recordSizeEntry='mf1'"
+                            "          rowLimit='-1'"
+                            "          sequence='0'"
+                            "          status='calculated'>"
+                            "   <rowCount>1</rowCount>"
+                            "   <SchemaRaw xsi:type='SOAP-ENC:base64'>"
+                            "    dgABCAEAGBAAAAB7IGludGVnZXI4IHYgfTsK   </SchemaRaw>"
+                            "   <totalRowCount>1</totalRowCount>"
+                            "   <Value xsi:type='SOAP-ENC:base64'>"
+                            "    AQAAAAAAAAA=   </Value>"
+                            "  </Result>"
+                            " </Results>"
+                            " <State>completed</State>"
+                            " <Statistics>"
+                            "  <Statistic c='eclcc'"
+                            "             count='1'"
+                            "             creator='eclcc'"
+                            "             kind='TimeElapsed'"
+                            "             s='compile'"
+                            "             scope='compile:parseTime'"
+                            "             ts='1431603789722535'"
+                            "             unit='ns'"
+                            "             value='805622'/>"
+                            "  <Statistic c='unknown'"
+                            "             count='1'"
+                            "             creator='unknownRichards-iMac.local'"
+                            "             kind='WhenQueryStarted'"
+                            "             s='global'"
+                            "             scope='workunit'"
+                            "             ts='1431603790007020'"
+                            "             unit='ts'"
+                            "             value='1431603790007001'/>"
+                            "  <Statistic c='unknown'"
+                            "             count='1'"
+                            "             creator='unknownRichards-iMac.local'"
+                            "             desc='Graph graph1'"
+                            "             kind='TimeElapsed'"
+                            "             s='graph'"
+                            "             scope='graph1'"
+                            "             ts='1431603790007912'"
+                            "             unit='ns'"
+                            "             value='0'/>"
+                            " </Statistics>"
+                            " <Temporaries>"
+                            "  <Variable name='wf2' status='calculated'>"
+                            "   <rowCount>1</rowCount>"
+                            "   <totalRowCount>1</totalRowCount>"
+                            "   <Value xsi:type='SOAP-ENC:base64'>"
+                            "    AQAAAAAAAAA=   </Value>"
+                            "  </Variable>"
+                            " </Temporaries>"
+                            " <Tracing>"
+                            "  <EclAgentBuild>community_6.0.0-trunk0Debug[heads/cass-wu-part3-0-g10b954-dirty]</EclAgentBuild>"
+                            " </Tracing>"
+                            " <Variables>"
+                            "  <Variable name='one' sequence='-1' status='calculated'>"
+                            "   <rowCount>1</rowCount>"
+                            "   <SchemaRaw xsi:type='SOAP-ENC:base64'>"
+                            "    b25lAAEIAQAYAAAAAA==   </SchemaRaw>"
+                            "   <totalRowCount>1</totalRowCount>"
+                            "   <Value xsi:type='SOAP-ENC:base64'>"
+                            "    AQAAAAAAAAA=   </Value>"
+                            "  </Variable>"
+                            " </Variables>"
+                            " <Workflow>"
+                            "  <Item mode='normal'"
+                            "        state='done'"
+                            "        type='normal'"
+                            "        wfid='1'/>"
+                            "  <Item mode='normal'"
+                            "        state='done'"
+                            "        type='normal'"
+                            "        wfid='2'>"
+                            "   <Dependency wfid='1'/>"
+                            "  </Item>"
+                            "  <Item mode='normal'"
+                            "        state='done'"
+                            "        type='normal'"
+                            "        wfid='3'>"
+                            "   <Dependency wfid='2'/>"
+                            "   <Schedule/>"
+                            "  </Item>"
+                            " </Workflow>"
+                            "</W_LOCAL>"
+                    , flags);
+                    switch(mode)
+                    {
+                    case some_contention: case some_control: for (int j = 0; j < 100000; j++) donothing(); break;
+                    case min_contention: case min_control: for (int j = 0; j < 1000000; j++) donothing(); break;
+                    }
+                }
+            }
+        } max("maxContention",flags,max_contention,1000),
+          some("someContention",flags,some_contention,200),
+          min("minContention",flags,min_contention,200),
+          csome("control some",flags,some_control,200),
+          cmin("control min",flags,min_control,200),
+          seq("single",flags,max_contention,1000);
+        max.For(8,8);
+        some.For(8,8,csome.For(8,8));
+        min.For(8,8,cmin.For(8,8));
+        seq.For(8,1);
+    }
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION( PtreeThreadingTest );
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( PtreeThreadingTest, "PtreeThreadingTest" );
+
 //MORE: This can't be included in jlib because of the dll dependency
 class StringBufferTest : public CppUnit::TestFixture
 {