Merge branch 'candidate-6.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.gitmodules

@@ -52,3 +52,6 @@
 [submodule "system/tbb_sm/tbb"]
     path = system/tbb_sm/tbb
     url = https://github.com/hpcc-systems/tbb.git
+[submodule "plugins/sqs/aws-sdk-cpp"]
+	path = plugins/sqs/aws-sdk-cpp
+	url = https://github.com/hpcc-systems/aws-sdk-cpp

cmake_modules/commonSetup.cmake

@@ -146,6 +146,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     PY2EMBED
     PY3EMBED
     REDIS
+    SQS
     MYSQLEMBED
     JAVAEMBED
     SQLITE3EMBED

common/environment/environment.cpp

@@ -463,14 +463,14 @@ public:
 
     virtual IEnvironment * loadLocalEnvironmentFile(const char * filename)
     {
-        Owned<IPropertyTree> ptree = createPTreeFromXMLFile(filename);
+        Owned<IPropertyTree> ptree = createPTreeFromXMLFile(filename, ipt_lowmem);
         Owned<CLocalEnvironment> pLocalEnv = new CLocalEnvironment(nullptr, ptree);
         return new CLockedEnvironment(pLocalEnv);
     }
 
     virtual IEnvironment * loadLocalEnvironment(const char * xml)
     {
-        Owned<IPropertyTree> ptree = createPTreeFromXMLString(xml);
+        Owned<IPropertyTree> ptree = createPTreeFromXMLString(xml, ipt_lowmem);
         Owned<CLocalEnvironment> pLocalEnv = new CLocalEnvironment(nullptr, ptree);
         return new CLockedEnvironment(pLocalEnv);
     }
@@ -1400,7 +1400,7 @@ void CLocalEnvironment::preload()
 
 void CLocalEnvironment::setXML(const char *xml)
 {
-    Owned<IPropertyTree> newRoot = createPTreeFromXMLString(xml);
+    Owned<IPropertyTree> newRoot = createPTreeFromXMLString(xml, ipt_lowmem);
     synchronized procedure(safeCache);
     Owned<IPropertyTreeIterator> it = p->getElements("*");
     ForEach(*it)
@@ -1709,7 +1709,7 @@ CConstDropZoneServerInfoIterator::CConstDropZoneServerInfoIterator(const IConstD
     if (0 != dropZoneMachineName.length())
     {
         // Create a ServerList for legacy element.
-        Owned<IPropertyTree> legacyServerList = createPTree();
+        Owned<IPropertyTree> legacyServerList = createPTree(ipt_lowmem);
 
         Owned<IConstMachineInfo> machineInfo = constEnv->getMachine(dropZoneMachineName.str());
         if (machineInfo)
@@ -1719,10 +1719,9 @@ CConstDropZoneServerInfoIterator::CConstDropZoneServerInfoIterator(const IConstD
 
             // Create a single ServerList record related to @computer
             //<ServerList name="ServerList" server="<IP_of_@computer>"/>
-            Owned<IPropertyTree> newRecord = createPTree();
+            IPropertyTree *newRecord = legacyServerList->addPropTree("ServerList");
             newRecord->setProp("@name", "ServerList");
             newRecord->setProp("@server", dropZoneMachineNetAddress.str());
-            legacyServerList->addPropTree("ServerList",newRecord.getClear());
 
             maxIndex = 1;
         }

common/thorhelper/roxiedebug.cpp

@@ -1763,7 +1763,7 @@ void CBaseServerDebugContext::debugPrint(IXmlWriter *output, const char *edgeId,
 #if 1
     CommonXmlWriter dummyOutput(0, 0);
     activityCtx->printEdge(&dummyOutput, startRow, numRows); // MORE - need to suppress the <Edge> if we want backward compatibility!
-    Owned<IPropertyTree> result = createPTreeFromXMLString(dummyOutput.str());
+    Owned<IPropertyTree> result = createPTreeFromXMLString(dummyOutput.str(),ipt_fast);
     if (result)
     {
         Owned<IAttributeIterator> attributes = result->getAttributes();

common/thorhelper/thorcommon.cpp

@@ -1869,7 +1869,7 @@ void setAutoAffinity(unsigned curProcess, unsigned processPerMachine, const char
 
     numa_bitmask_free(available_nodes);
 
-    DBGLOG("Affinity: Max cpus(%u) nodes(%u) actual nodes(%u)", maxcpus, numa_max_node()+1, numNumaNodes);
+    DBGLOG("Affinity: Max cpus(%u) nodes(%u) actual nodes(%u), processes(%u)", maxcpus, numa_max_node()+1, numNumaNodes, processPerMachine);
 #else
     //On very old versions of numa assume that all nodes are present
     for (unsigned i=0; i<=numa_max_node(); i++)
@@ -1881,25 +1881,52 @@ void setAutoAffinity(unsigned curProcess, unsigned processPerMachine, const char
     if (numNumaNodes <= 1)
         return;
 
-    //MORE: If processPerMachine < numNumaNodes we may want to associate with > 1 node.
-    unsigned curNode = curProcess % numNumaNodes;
+    unsigned firstNode = 0;
+    unsigned numNodes = 1;
+    if (processPerMachine >= numNumaNodes)
+    {
+        firstNode = curProcess % numNumaNodes;
+    }
+    else
+    {
+        firstNode = (curProcess * numNumaNodes) / processPerMachine;
+        unsigned nextNode = ((curProcess+1) *  numNumaNodes) / processPerMachine;
+        numNodes = nextNode - firstNode;
+    }
+
+    if ((processPerMachine % numNumaNodes) != 0)
+        DBGLOG("Affinity: %u processes will not be evenly balanced over %u numa nodes", processPerMachine, numNumaNodes);
 
 #if defined(LIBNUMA_API_VERSION) && (LIBNUMA_API_VERSION>=2)
+    //This code assumes the nodes are sensibly ordered (e.g., nodes on the same socket are next to each other), and
+    //only works well when number of processes is a multiple of the number of numa nodes.  A full solution would look
+    //at distances.
     struct bitmask * cpus = numa_allocate_cpumask();
-    numa_node_to_cpus(numaMap[curNode], cpus);
+    struct bitmask * nodeMask = numa_allocate_cpumask();
+    for (unsigned node=0; node < numNodes; node++)
+    {
+        numa_node_to_cpus(numaMap[firstNode+node], nodeMask);
+        //Shame there is no inbuilt union operation.
+        for (unsigned cpu=0; cpu < maxcpus; cpu++)
+        {
+            if (numa_bitmask_isbitset(nodeMask, cpu))
+                numa_bitmask_setbit(cpus, cpu);
+        }
+    }
     bool ok = (numa_sched_setaffinity(0, cpus) == 0);
+    numa_bitmask_free(nodeMask);
     numa_bitmask_free(cpus);
 #else
     cpu_set_t cpus;
     CPU_ZERO(&cpus);
-    numa_node_to_cpus(numaMap[curNode], (unsigned long *) &cpus, sizeof (cpus));
+    numa_node_to_cpus(numaMap[firstNode], (unsigned long *) &cpus, sizeof (cpus));
     bool ok = sched_setaffinity (0, sizeof(cpus), &cpus) != 0;
 #endif
 
     if (!ok)
-        throw makeStringExceptionV(1, "Failed to set affinity to numa node %u (id:%u)", curNode, numaMap[curNode]);
+        throw makeStringExceptionV(1, "Failed to set affinity to numa node %u (id:%u)", firstNode, numaMap[firstNode]);
 
-    DBGLOG("Process bound to numa node %u (id:%u) of %u", curNode, numaMap[curNode], numNumaNodes);
+    DBGLOG("Process bound to numa node %u..%u (id:%u) of %u", firstNode, firstNode + numNodes - 1, numaMap[firstNode], numNumaNodes);
 #endif
     clearAffinityCache();
 }

common/thorhelper/thorxmlread.cpp

@@ -564,7 +564,7 @@ void XmlSetColumnProvider::readUtf8X(size32_t & len, char * & target, const char
 IDataVal & CXmlToRawTransformer::transform(IDataVal & result, size32_t len, const void * text, bool isDataSet)
 {
     // MORE - should redo using a pull parser sometime
-    Owned<IPropertyTree> root = createPTreeFromXMLString(len, (const char *)text, ipt_none, xmlReadFlags);
+    Owned<IPropertyTree> root = createPTreeFromXMLString(len, (const char *)text, ipt_fast, xmlReadFlags);
     return transformTree(result, *root, isDataSet);
 }
 
@@ -595,7 +595,7 @@ IDataVal & CXmlToRawTransformer::transformTree(IDataVal & result, IPropertyTree
                     try
                     {
                         decodedXML.append("<root>").append(body).append("</root>");
-                        decodedTree.setown(createPTreeFromXMLString(decodedXML.str(), ipt_caseInsensitive));
+                        decodedTree.setown(createPTreeFromXMLString(decodedXML.str(), ipt_caseInsensitive|ipt_fast));
                         rows.setown(decodedTree->getElements("Row"));
                     }
                     catch (IException *E)
@@ -645,7 +645,7 @@ IDataVal & CXmlToRawTransformer::transformTree(IDataVal & result, IPropertyTree
 
 size32_t createRowFromXml(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace)
 {
-    Owned<IPropertyTree> root = createPTreeFromXMLString(size, utf8, ipt_none, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
+    Owned<IPropertyTree> root = createPTreeFromXMLString(size, utf8, ipt_fast, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
     if (!root)
     {
         throwError(THORCERR_InvalidXmlFromXml);
@@ -666,7 +666,7 @@ const void * createRowFromXml(IEngineRowAllocator * rowAllocator, size32_t len,
 
 size32_t createRowFromJson(ARowBuilder & rowBuilder, size32_t size, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace)
 {
-    Owned<IPropertyTree> root = createPTreeFromJSONString(size, utf8, ipt_none, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
+    Owned<IPropertyTree> root = createPTreeFromJSONString(size, utf8, ipt_fast, stripWhitespace ? ptr_ignoreWhiteSpace : ptr_none);
     if (!root)
     {
         throwError(THORCERR_InvalidJsonFromJson);

common/workunit/package.cpp

@@ -39,7 +39,7 @@ CPackageNode::CPackageNode(IPropertyTree *p)
     if (p)
         node.set(p);
     else
-        node.setown(createPTree("HpccPackages"));
+        node.setown(createPTree("HpccPackages", ipt_lowmem));
     StringBuffer xml;
     toXML(node, xml, 0, XML_SortTags);
     hash = rtlHash64Data(xml.length(), xml.str(), 9994410);
@@ -158,7 +158,7 @@ IHpccPackageMap *createPackageMapFromPtree(IPropertyTree *t, const char *queryse
 
 IHpccPackageMap *createPackageMapFromXml(const char *xml, const char *queryset, const char *id)
 {
-    Owned<IPropertyTree> t = createPTreeFromXMLString(xml);
+    Owned<IPropertyTree> t = createPTreeFromXMLString(xml, ipt_lowmem);
     return createPackageMapFromPtree(t, queryset, id);
 }
 

common/workunit/pkgimpl.hpp

@@ -450,7 +450,7 @@ public:
                 Owned<ISimpleSuperFileEnquiry> ssfe = pkg->resolveSuperFile(rf.getLogicalName());
                 if (ssfe && ssfe->numSubFiles()>0)
                 {
-                    IPropertyTree *superInfo = fileInfo->addPropTree("SuperFile", createPTree());
+                    IPropertyTree *superInfo = fileInfo->addPropTree("SuperFile");
                     superInfo->setProp("@name", rf.getLogicalName());
                     unsigned count = ssfe->numSubFiles();
                     while (count--)
@@ -495,7 +495,7 @@ public:
                     referencedPackages.setValue(baseId, true);
             }
         }
-        Owned<IPropertyTree> tempQuerySet=createPTree();
+        Owned<IPropertyTree> tempQuerySet=createPTree(ipt_fast);
         Owned<IPropertyTreeIterator> queries;
         if (queriesToCheck.length())
         {

common/workunit/workflow.cpp

@@ -126,7 +126,7 @@ public:
     CWorkflowItem(IPropertyTree & _tree) { tree.setown(&_tree); }
     CWorkflowItem(IPropertyTree * ptree, unsigned wfid, WFType type, WFMode mode, unsigned success, unsigned failure, unsigned recovery, unsigned retriesAllowed, unsigned contingencyFor)
     {
-        tree.setown(LINK(ptree->addPropTree("Item", createPTree())));
+        tree.setown(LINK(ptree->addPropTree("Item")));
         tree->setPropInt("@wfid", wfid);
         setEnum(tree, "@type", type, wftypes);
         setEnum(tree, "@mode", mode, wfmodes);
@@ -136,7 +136,7 @@ public:
         {
             tree->setPropInt("@recovery", recovery);
             tree->setPropInt("@retriesAllowed", retriesAllowed);
-            tree->addPropTree("Dependency", createPTree())->setPropInt("@wfid", recovery);
+            tree->addPropTree("Dependency")->setPropInt("@wfid", recovery);
         }
         if(contingencyFor) tree->setPropInt("@contingencyFor", contingencyFor);
         reset();
@@ -165,11 +165,11 @@ public:
     virtual bool         queryPersistRefresh() const { return tree->getPropBool("@persistRefresh", true); }
     virtual IStringVal & getCriticalName(IStringVal & val) const { val.set(tree->queryProp("@criticalName")); return val; }
     virtual IStringVal & queryCluster(IStringVal & val) const { val.set(tree->queryProp("@cluster")); return val; }
-    virtual void         setScheduledNow() { tree->setPropTree("Schedule", createPTree()); setEnum(tree, "@state", WFStateReqd, wfstates); }
-    virtual void         setScheduledOn(char const * name, char const * text) { IPropertyTree * stree = createPTree(); stree->setProp("@name", name); stree->setProp("@text", text); tree->setPropTree("Schedule", createPTree())->setPropTree("Event", stree); setEnum(tree, "@state", WFStateWait, wfstates); }
+    virtual void         setScheduledNow() { tree->setPropTree("Schedule"); setEnum(tree, "@state", WFStateReqd, wfstates); }
+    virtual void         setScheduledOn(char const * name, char const * text) { IPropertyTree * stree =  tree->setPropTree("Schedule")->setPropTree("Event"); stree->setProp("@name", name); stree->setProp("@text", text);; setEnum(tree, "@state", WFStateWait, wfstates); }
     virtual void         setSchedulePriority(unsigned priority) { assertex(tree->hasProp("Schedule")); tree->setPropInt("Schedule/@priority", priority); }
     virtual void         setScheduleCount(unsigned count) { assertex(tree->hasProp("Schedule")); tree->setPropInt("Schedule/@count", count); tree->setPropInt("Schedule/@countRemaining", count); }
-    virtual void         addDependency(unsigned wfid) { tree->addPropTree("Dependency", createPTree())->setPropInt("@wfid", wfid); }
+    virtual void         addDependency(unsigned wfid) { tree->addPropTree("Dependency")->setPropInt("@wfid", wfid); }
     virtual void         setPersistInfo(char const * name, unsigned wfid, int numPersistInstances, bool refresh)
     {
         tree->setProp("@persistName", name);

common/workunit/workunit.cpp

@@ -196,17 +196,16 @@ void CWuGraphStats::beforeDispose()
     StringBuffer tag;
     tag.append("sg").append(id);
 
-    IPropertyTree * subgraph = createPTree(tag);
-    subgraph->setProp("@c", queryCreatorTypeName(creatorType));
-    subgraph->setProp("@creator", creator);
-    subgraph->setPropInt("@minActivity", minActivity);
-    subgraph->setPropInt("@maxActivity", maxActivity);
 
     //Replace the particular subgraph statistics added by this creator
     StringBuffer qualified(tag);
     qualified.append("[@creator='").append(creator).append("']");
     progress->removeProp(qualified);
-    subgraph = progress->addPropTree(tag, subgraph);
+    IPropertyTree * subgraph = progress->addPropTree(tag);
+    subgraph->setProp("@c", queryCreatorTypeName(creatorType));
+    subgraph->setProp("@creator", creator);
+    subgraph->setPropInt("@minActivity", minActivity);
+    subgraph->setPropInt("@maxActivity", maxActivity);
     subgraph->setPropBin("Stats", compressed.length(), compressed.toByteArray());
     if (!progress->getPropBool("@stats", false))
         progress->setPropBool("@stats", true);
@@ -292,7 +291,7 @@ protected:
                 throwUnexpected();
             }
 
-            IPropertyTree * next = curTarget->addPropTree(tag, createPTree());
+            IPropertyTree * next = curTarget->addPropTree(tag);
             next->setProp("@id", id);
             expandProcessTreeFromStats(rootTarget, next, &cur);
         }
@@ -1145,7 +1144,7 @@ public:
         IPropertyTree *node = progress->queryPropTree(path.str());
         if (!node)
         {
-            node = progress->addPropTree("node", createPTree());
+            node = progress->addPropTree("node");
             node->setPropInt64("@id", nodeId);
         }
         node->setPropInt("@_state", (unsigned)state);
@@ -1153,7 +1152,7 @@ public:
         {
             case WUGraphRunning:
             {
-                IPropertyTree *running = conn->queryRoot()->setPropTree("Running", createPTree());
+                IPropertyTree *running = conn->queryRoot()->setPropTree("Running");
                 running->setProp("@graph", graphName);
                 running->setPropInt64("@subId", nodeId);
                 break;
@@ -3913,7 +3912,7 @@ void CLocalWorkUnit::loadXML(const char *xml)
     CriticalBlock block(crit);
     clearCached(true);
     assertex(xml);
-    p.setown(createPTreeFromXMLString(xml));
+    p.setown(createPTreeFromXMLString(xml,ipt_lowmem));
 }
 
 void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
@@ -5269,7 +5268,7 @@ void CLocalWorkUnit::setWarningSeverity(unsigned code, ErrorSeverity severity)
     if (!mapping)
     {
         IPropertyTree * onWarnings = ensurePTree(p, "OnWarnings");
-        mapping = onWarnings->addPropTree("OnWarning", createPTree());
+        mapping = onWarnings->addPropTree("OnWarning");
         mapping->setPropInt("@code", code);
     }
 
@@ -5604,7 +5603,7 @@ void CLocalWorkUnit::addProcess(const char *type, const char *instance, unsigned
     if (!p->hasProp(xpath))
     {
         IPropertyTree *node = ensurePTree(p, processType.str());
-        node = node->addPropTree(instance, createPTree());
+        node = node->addPropTree(instance);
         node->setProp("@log", log);
         node->setPropInt("@pid", pid);
     }
@@ -5888,7 +5887,7 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
     CriticalBlock block(crit);
     IPropertyTree * stats = p->queryPropTree("Statistics");
     if (!stats)
-        stats = p->addPropTree("Statistics", createPTree("Statistics"));
+        stats = p->addPropTree("Statistics");
 
     IPropertyTree * statTree = NULL;
     if (mergeAction != StatsMergeAppend)
@@ -5900,7 +5899,7 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
 
     if (!statTree)
     {
-        statTree = stats->addPropTree("Statistic", createPTree("Statistic"));
+        statTree = stats->addPropTree("Statistic");
         statTree->setProp("@creator", creator);
         statTree->setProp("@scope", scope);
         statTree->setProp("@kind", kindName);
@@ -5992,9 +5991,9 @@ IWUPlugin* CLocalWorkUnit::updatePluginByName(const char *qname)
     if (existing)
         return (IWUPlugin *) existing;
     if (!plugins.length())
-        p->addPropTree("Plugins", createPTree("Plugins"));
+        p->addPropTree("Plugins");
     IPropertyTree *pl = p->queryPropTree("Plugins");
-    IPropertyTree *s = pl->addPropTree("Plugin", createPTree("Plugin"));
+    IPropertyTree *s = pl->addPropTree("Plugin");
     s->Link();
     IWUPlugin* q = new CLocalWUPlugin(s); 
     q->Link();
@@ -6028,9 +6027,9 @@ IWULibrary* CLocalWorkUnit::updateLibraryByName(const char *qname)
     if (existing)
         return (IWULibrary *) existing;
     if (!libraries.length())
-        p->addPropTree("Libraries", createPTree("Libraries"));
+        p->addPropTree("Libraries");
     IPropertyTree *pl = p->queryPropTree("Libraries");
-    IPropertyTree *s = pl->addPropTree("Library", createPTree("Library"));
+    IPropertyTree *s = pl->addPropTree("Library");
     s->Link();
     IWULibrary* q = new CLocalWULibrary(s); 
     q->Link();
@@ -6092,9 +6091,9 @@ IWUException* CLocalWorkUnit::createException()
     loadExceptions();
 
     if (!exceptions.length())
-        p->addPropTree("Exceptions", createPTree("Exceptions"));
+        p->addPropTree("Exceptions");
     IPropertyTree *r = p->queryPropTree("Exceptions");
-    IPropertyTree *s = r->addPropTree("Exception", createPTree("Exception"));
+    IPropertyTree *s = r->addPropTree("Exception");
     s->setPropInt("@sequence", exceptions.ordinality());
     IWUException* q = new CLocalWUException(LINK(s)); 
     exceptions.append(*LINK(q));
@@ -6132,7 +6131,7 @@ IWUWebServicesInfo* CLocalWorkUnit::updateWebServicesInfo(bool create)
         if (!s)
         {
             if (create)
-                s = p->addPropTree("WebServicesInfo", createPTreeFromXMLString("<WebServicesInfo />"));
+                s = p->addPropTree("WebServicesInfo");
             else
                 return NULL;
         }
@@ -6231,9 +6230,9 @@ IWUResult* CLocalWorkUnit::createResult()
     // For this to be legally called, we must have the write-able interface. So we are already locked for write.
     loadResults();
     if (!results.length())
-        p->addPropTree("Results", createPTree("Results"));
+        p->addPropTree("Results");
     IPropertyTree *r = p->queryPropTree("Results");
-    IPropertyTree *s = r->addPropTree("Result", createPTree());
+    IPropertyTree *s = r->addPropTree("Result");
 
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
@@ -6381,10 +6380,8 @@ IWUResult* CLocalWorkUnit::updateTemporaryByName(const char *qname)
     IConstWUResult *existing = getTemporaryByName(qname);
     if (existing)
         return (IWUResult *) existing;
-    if (!temporaries.length())
-        p->addPropTree("Temporaries", createPTree("Temporaries"));
-    IPropertyTree *vars = p->queryPropTree("Temporaries");
-    IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
+    IPropertyTree *vars = (temporaries.length()) ? p->queryPropTree("Temporaries") : p->addPropTree("Temporaries");
+    IPropertyTree *s = vars->addPropTree("Variable");
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
     q->Link();
@@ -6401,9 +6398,9 @@ IWUResult* CLocalWorkUnit::updateVariableByName(const char *qname)
     if (existing)
         return (IWUResult *) existing;
     if (!variables.length())
-        p->addPropTree("Variables", createPTree("Variables"));
+        p->addPropTree("Variables");
     IPropertyTree *vars = p->queryPropTree("Variables");
-    IPropertyTree *s = vars->addPropTree("Variable", createPTree("Variable"));
+    IPropertyTree *s = vars->addPropTree("Variable");
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
     q->Link();
@@ -6488,9 +6485,7 @@ static void _noteFileRead(IDistributedFile *file, IPropertyTree *filesRead)
                 IDistributedFile &file = iter->query();
                 StringBuffer fname;
                 file.getLogicalName(fname);
-                Owned<IPropertyTree> subfile = createPTree();
-                subfile->setProp("@name", fname.str());
-                fileTree->addPropTree("Subfile", subfile.getClear());
+                fileTree->addPropTree("Subfile")->setProp("@name", fname.str());
                 _noteFileRead(&file, filesRead);
             }
         }
@@ -6509,7 +6504,7 @@ void CLocalWorkUnit::noteFileRead(IDistributedFile *file)
         CriticalBlock block(crit);
         IPropertyTree *files = p->queryPropTree("FilesRead");
         if (!files)
-            files = p->addPropTree("FilesRead", createPTree());
+            files = p->addPropTree("FilesRead");
         _noteFileRead(file, files);
     }
 }
@@ -6553,7 +6548,7 @@ void CLocalWorkUnit::addFile(const char *fileName, StringArray *clusters, unsign
     CriticalBlock block(crit);
     IPropertyTree *files = p->queryPropTree("Files");
     if (!files)
-        files = p->addPropTree("Files", createPTree());
+        files = p->addPropTree("Files");
     if (!clusters)
         ::addFile(files, fileName, NULL, usageCount, fileKind, graphOwner);
     else
@@ -6726,7 +6721,7 @@ void CLocalWorkUnit::addDiskUsageStats(__int64 _avgNodeUsage, unsigned _minNode,
         maxNodeUsage = stats->getPropInt64("@maxNodeUsage");
     else
     {
-        stats = p->addPropTree("DiskUsageStats", createPTree());
+        stats = p->addPropTree("DiskUsageStats");
         maxNodeUsage = 0;
     }
 
@@ -7115,9 +7110,9 @@ void CLocalWorkUnit::createGraph(const char * name, const char *label, WUGraphTy
 {
     CriticalBlock block(crit);
     if (!graphs.length())
-        p->addPropTree("Graphs", createPTree("Graphs"));
+        p->addPropTree("Graphs");
     IPropertyTree *r = p->queryPropTree("Graphs");
-    IPropertyTree *s = r->addPropTree("Graph", createPTree());
+    IPropertyTree *s = r->addPropTree("Graph");
     CLocalWUGraph *q = new CLocalWUGraph(*this, LINK(s));
     q->setName(name);
     q->setLabel(label);
@@ -7171,13 +7166,13 @@ void CLocalWUGraph::setLabel(const char *str)
 
 void CLocalWUGraph::setXGMML(const char *str)
 {
-    setXGMMLTree(createPTreeFromXMLString(str));
+    setXGMMLTree(createPTreeFromXMLString(str,ipt_lowmem));
 }
 
 void CLocalWUGraph::setXGMMLTree(IPropertyTree *_graph)
 {
     assertex(strcmp(_graph->queryName(), "graph")==0);
-    IPropertyTree *xgmml = p->setPropTree("xgmml", createPTree());
+    IPropertyTree *xgmml = p->setPropTree("xgmml");
     MemoryBuffer mb;
     _graph->serialize(mb);
     // Note - we could compress further but that would introduce compatibility concerns, so don't bother
@@ -7194,7 +7189,7 @@ static void expandAttributes(IPropertyTree & targetNode, IPropertyTree & progres
         const char *aName = aIter->queryName()+1;
         if (0 != stricmp("id", aName)) // "id" reserved.
         {
-            IPropertyTree *att = targetNode.addPropTree("att", createPTree());
+            IPropertyTree *att = targetNode.addPropTree("att");
             att->setProp("@name", aName);
             att->setProp("@value", aIter->queryValue());
         }
@@ -7233,7 +7228,7 @@ void CLocalWUGraph::mergeProgress(IPropertyTree &rootNode, IPropertyTree &progre
                     ForEach (*iter)
                     {
                         IPropertyTree &t = iter->query();
-                        IPropertyTree *att = graphEdge->addPropTree("att", createPTree());
+                        IPropertyTree *att = graphEdge->addPropTree("att");
                         att->setProp("@name", t.queryName());
                         att->setProp("@value", t.queryProp(NULL));
                     }
@@ -7276,7 +7271,7 @@ IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
         // daliadmin can retrospectively compress existing graphs, so need to check for all versions
         MemoryBuffer mb;
         if (p->getPropBin("xgmml/graphBin", mb))
-            graph.setown(createPTree(mb));
+            graph.setown(createPTree(mb, ipt_lowmem));
         else
             graph.setown(p->getBranch("xgmml/graph"));
         if (!graph)
@@ -7286,7 +7281,7 @@ IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
         return graph.getLink();
     else
     {
-        Owned<IPropertyTree> copy = createPTreeFromIPT(graph);
+        Owned<IPropertyTree> copy = createPTreeFromIPT(graph, ipt_lowmem);
         Owned<IConstWUGraphProgress> progress = owner.getGraphProgress(p->queryProp("@name"));
         if (progress)
         {
@@ -7421,7 +7416,7 @@ IStringVal& CLocalWUQuery::getQueryShortText(IStringVal &str) const
         text = p->queryProp("Text");
         if (isArchiveQuery(text))
         {
-            Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive);
+            Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
             const char * path = xml->queryProp("Query/@attributePath");
             if (path)
             {
@@ -7496,7 +7491,7 @@ void CLocalWUQuery::setQueryText(const char *text)
     bool isArchive = isArchiveQuery(text);
     if (isArchive)
     {
-        Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive);
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(text, ipt_caseInsensitive|ipt_lowmem);
         const char * path = xml->queryProp("Query/@attributePath");
         if (path)
         {
@@ -7527,9 +7522,9 @@ void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const
     CriticalBlock block(crit);
     loadAssociated();
     if (!associated.length())
-        p->addPropTree("Associated", createPTree("Associated"));
+        p->addPropTree("Associated");
     IPropertyTree *pl = p->queryPropTree("Associated");
-    IPropertyTree *s = pl->addPropTree("File", createPTree("File"));
+    IPropertyTree *s = pl->addPropTree("File");
     setEnum(s, "@type", type, queryFileTypes);
     s->setProp("@filename", name);
     s->setProp("@ip", ip);
@@ -9042,10 +9037,10 @@ extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(const char *xml)
 {
     Owned<CLocalWorkUnit> cw = new CLocalWorkUnit((ISecManager *) NULL, NULL);
     if (xml)
-        cw->loadPTree(createPTreeFromXMLString(xml));
+        cw->loadPTree(createPTreeFromXMLString(xml, ipt_lowmem));
     else
     {
-        Owned<IPropertyTree> p = createPTree("W_LOCAL");
+        Owned<IPropertyTree> p = createPTree("W_LOCAL", ipt_lowmem);
         p->setProp("@xmlns:xsi", "http://www.w3.org/1999/XMLSchema-instance");
         cw->loadPTree(p.getClear());
     }
@@ -9317,7 +9312,7 @@ IWorkflowItem * CLocalWorkUnit::addWorkflowItem(unsigned wfid, WFType type, WFMo
     workflowIteratorCached = false;
     IPropertyTree * s = p->queryPropTree("Workflow");
     if(!s)
-        s = p->addPropTree("Workflow", createPTree("Workflow"));
+        s = p->addPropTree("Workflow");
     return createWorkflowItem(s, wfid, type, mode, success, failure, recovery, retriesAllowed, contingencyFor);
 }
 
@@ -9329,7 +9324,7 @@ IWorkflowItemIterator * CLocalWorkUnit::updateWorkflowItems()
     {
         IPropertyTree * s = p->queryPropTree("Workflow");
         if(!s)
-            s = p->addPropTree("Workflow", createPTree("Workflow"));
+            s = p->addPropTree("Workflow");
         workflowIterator.setown(createWorkflowItemIterator(s)); 
         workflowIteratorCached = true;
     }
@@ -9520,7 +9515,7 @@ unsigned CLocalWorkUnit::addLocalFileUpload(LocalFileUploadType type, char const
     CriticalBlock block(crit);
     IPropertyTree * s = p->queryPropTree("LocalFileUploads");
     if(!s)
-        s = p->addPropTree("LocalFileUploads", createPTree());
+        s = p->addPropTree("LocalFileUploads");
     unsigned id = s->numChildren();
     Owned<CLocalFileUpload> upload = new CLocalFileUpload(id, type, source, destination, eventTag);
     s->addPropTree("LocalFileUpload", upload->getTree());
@@ -10031,12 +10026,10 @@ public:
     {
         assertex(baseconn);
         Owned<IPropertyTree> root = baseconn->getRoot();
-        ensurePTree(root, "EventQueue");
-        Owned<IPropertyTree> eventQueue = root->getPropTree("EventQueue");
-        Owned<IPropertyTree> eventItem = createPTree();
+        IPropertyTree *eventQueue = ensurePTree(root, "EventQueue");
+        IPropertyTree *eventItem = eventQueue->addPropTree("Item");
         eventItem->setProp("@name", name);
         eventItem->setProp("@text", text);
-        eventQueue->addPropTree("Item", eventItem.getLink());
     }
 
     virtual void remove()
@@ -10246,7 +10239,7 @@ IPropertyTree * addNamedQuery(IPropertyTree * queryRegistry, const char * name,
 
     StringBuffer id;
     id.append(lcName).append(".").append(seq);
-    IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive);
+    IPropertyTree * newEntry = createPTree("Query", ipt_caseInsensitive|ipt_lowmem);
     newEntry->setProp("@name", lcName);
     newEntry->setProp("@wuid", wuid);
     newEntry->setProp("@dll", dll);
@@ -10306,9 +10299,8 @@ void setQueryAlias(IPropertyTree * queryRegistry, const char * name, const char
     IPropertyTree * match = queryRegistry->queryPropTree(xpath);
     if (!match)
     {
-        IPropertyTree * newEntry = createPTree("Alias");
-        newEntry->setProp("@name", lcName);
-        match = queryRegistry->addPropTree("Alias", newEntry);
+        match = queryRegistry->addPropTree("Alias");
+        match->setProp("@name", lcName);
     }
     match->setProp("@id", value);
 }

common/workunit/wujobq.cpp

@@ -113,13 +113,6 @@ public:
         item->setProp("@enqueuedt",dts.str());
     }
 
-    IPropertyTree *createBranch(CJobQueueItem)
-    {
-        IPropertyTree *item = createPTree("Item");
-        assignBranch(item,this);
-        return item;
-    }
-
     const char *queryWUID()
     {
         return wu.get();
@@ -893,7 +886,7 @@ public:
                             StringBuffer cpath;
                             cpath.appendf("Queue[@name=\"%s\"]",qd->qname.get());
                             if (!proot->hasProp(cpath.str())) {
-                                IPropertyTree *pt = proot->addPropTree("Queue",createPTree("Queue"));
+                                IPropertyTree *pt = proot->addPropTree("Queue");
                                 pt->setProp("@name",qd->qname.get());
                                 pt->setProp("@state","active");
                                 pt->setPropInt("@count", 0);
@@ -1087,8 +1080,7 @@ public:
         IPropertyTree *ret = qd.root->queryPropTree(path.str());
         if (!ret)
         {
-            ret = createPTree("Client");
-            ret = qd.root->addPropTree("Client",ret);
+            ret = qd.root->addPropTree("Client");
             ret->setPropInt64("@session",sessionid);
             StringBuffer eps;
             ret->setProp("@node",queryMyNode()->endpoint().getUrlStr(eps).str());

configuration/configurator/schemas/SchemaAppInfo.cpp

@@ -56,6 +56,8 @@ CAppInfo* CAppInfo::load(CXSDNodeBase* pParentNode, const IPropertyTree *pSchema
     strXPathDocID.append("/").append(TAG_DOC_ID);
     StringBuffer strXPathDocLineBreak(xpath);
     strXPathDocLineBreak.append("/").append(TAG_DOC_USE_LINE_BREAK);
+    StringBuffer strXPathRequired(xpath);
+    strXPathRequired.append("/").append(TAG_REQUIRED);
 
     StringBuffer strViewType;
     StringBuffer strColIndex;
@@ -68,6 +70,7 @@ CAppInfo* CAppInfo::load(CXSDNodeBase* pParentNode, const IPropertyTree *pSchema
     StringBuffer strViewChildNodes;
     StringBuffer strXPath;
     StringBuffer strDocTableID;
+    StringBuffer strRequired;
     bool bDocLineBreak = false;
 
     if (pSchemaRoot->queryPropTree(strXPathViewType.str()) != NULL)
@@ -94,8 +97,10 @@ CAppInfo* CAppInfo::load(CXSDNodeBase* pParentNode, const IPropertyTree *pSchema
         strDocTableID.append(pSchemaRoot->queryPropTree(strXPathDocID.str())->queryProp(""));
     if (pSchemaRoot->queryPropTree(strXPathDocLineBreak.str()) != NULL)
         bDocLineBreak = true;
+    if (pSchemaRoot->queryPropTree(strXPathRequired.str()) != NULL)
+        strRequired.append(pSchemaRoot->queryPropTree(strXPathRequired.str())->queryProp(""));
 
-    CAppInfo *pAppInfo = new CAppInfo(pParentNode, strViewType.str(),  strColIndex.str(), strToolTip.str(), strTitle.str(), strWidth.str(), strAutoGenForWizard.str(), strAutoGenDefaultValue.str(), NULL, strViewChildNodes.str(), strXPath.str(), strDocTableID.str(), bDocLineBreak);
+    CAppInfo *pAppInfo = new CAppInfo(pParentNode, strViewType.str(),  strColIndex.str(), strToolTip.str(), strTitle.str(), strWidth.str(), strAutoGenForWizard.str(), strAutoGenDefaultValue.str(), NULL, strViewChildNodes.str(), strXPath.str(), strDocTableID.str(), bDocLineBreak, strRequired.str());
     pAppInfo->setXSDXPath(xpath);
 
     return pAppInfo;
@@ -118,6 +123,7 @@ void CAppInfo::dump(::std::ostream &cout, unsigned int offset) const
     QUICK_OUT(cout, ViewChildNodes, offset);
     QUICK_OUT(cout, XPath, offset);
     QUICK_OUT(cout, XSDXPath, offset);
+    QUICK_OUT(cout, Required, offset);
 
     quickOutFooter(cout, XSD_APP_INFO_STR, offset);
 }

configuration/configurator/schemas/SchemaAppInfo.hpp

@@ -44,6 +44,7 @@ public:
     GETTERSETTER(ViewChildNodes)
     GETTERSETTER(XPath)
     GETTERSETTER(DocTableID)
+    GETTERSETTER(Required)
 
     virtual void dump(::std::ostream &cout, unsigned int offset = 0) const;
     virtual void getDocumentation(StringBuffer &strDoc) const;
@@ -58,10 +59,11 @@ public:
 protected:
 
     CAppInfo(CXSDNodeBase* pParentNode, const char *pViewType = NULL, const char *pColIndex = NULL, const char* pToolTip = NULL, const char* pTitle = NULL, const char* pWidth = NULL, const char* pAutoGenForWizard = NULL,\
-             const char* pAutoGenDefaultValue = NULL, const char* pAutoGenDefaultForMultiNode = NULL, const char* pViewChildNodes = NULL, const char* pXPath = NULL, const char* pDocTableID = NULL, bool bDocLineBreak = false)\
+             const char* pAutoGenDefaultValue = NULL, const char* pAutoGenDefaultForMultiNode = NULL, const char* pViewChildNodes = NULL, const char* pXPath = NULL, const char* pDocTableID = NULL, bool bDocLineBreak = false, \
+			 const char* pRequired = NULL ) \
         : CXSDNodeBase::CXSDNodeBase(pParentNode, XSD_APP_INFO), m_strViewType(pViewType), m_strColIndex(pColIndex), m_strToolTip(pToolTip), m_strTitle(pTitle), m_strWidth(pWidth), m_strAutoGenForWizard(pAutoGenForWizard),\
-          m_strAutoGenDefaultValue(pAutoGenDefaultValue), m_strAutoGenDefaultValueForMultiNode(pAutoGenDefaultForMultiNode), m_strViewChildNodes(pViewChildNodes), m_strXPath(pXPath), m_strDocTableID(pDocTableID),\
-          m_bDocLineBreak(bDocLineBreak)
+          m_strAutoGenDefaultValue(pAutoGenDefaultValue), m_strAutoGenDefaultValueForMultiNode(pAutoGenDefaultForMultiNode), m_strViewChildNodes(pViewChildNodes), m_strXPath(pXPath), m_strDocTableID(pDocTableID), \
+          m_bDocLineBreak(bDocLineBreak), m_strRequired(pRequired)
     {
     }
 

configuration/configurator/schemas/SchemaAttributes.cpp

@@ -105,6 +105,10 @@ void CAttribute::getDocumentation(StringBuffer &strDoc) const
             return; // HIDDEN
         else
             pToolTip = pAppInfo->getToolTip();
+
+        const char* pReq = pAppInfo->getRequired();
+        if (pReq != nullptr && stricmp("True", pReq) == 0)
+            pRequired = TAG_REQUIRED;
     }
 
     strDoc.appendf("<%s>\n", DM_TABLE_ROW);

configuration/configurator/schemas/SchemaCommon.hpp

@@ -248,6 +248,7 @@ static const char* TAG_AUTOGENDEFAULTVALUEFORMULTINODE("autogendefaultformultino
 static const char* TAG_XPATH("xpath");
 static const char* TAG_DOC_ID("docid");
 static const char* TAG_DOC_USE_LINE_BREAK("docuselinebreak");
+static const char* TAG_REQUIRED("required");
 static const char* TAG_UNBOUNDED("unbounded");
 
 #define TAG_OPTIONAL                   "optional"

dali/base/dacsds.cpp

@@ -749,16 +749,19 @@ ChildMap *CClientRemoteTree::_checkChildren()
 
 IPropertyTree *CClientRemoteTree::ownPTree(IPropertyTree *tree)
 {
-    // if taking ownership of an orphaned clientremote tree need to reset it's attributes.
-    if ((connection.queryStateChanges()) && isEquivalent(tree) && (!QUERYINTERFACE(tree, CClientRemoteTree)->IsShared()))
+    // if taking ownership of an orphaned clientremote tree need to reset its attributes.
+    if ((connection.queryStateChanges()) && isEquivalent(tree))
     {
-        CClientRemoteTree *_tree = QUERYINTERFACE(tree, CClientRemoteTree);
-        if (_tree->queryServerId())
-            ((CClientRemoteTree *)tree)->resetState(CPS_Changed, true);
-        return tree;
+        CClientRemoteTree * remoteTree = static_cast<CClientRemoteTree *>(tree);
+        if (!remoteTree->IsShared())
+        {
+            if (remoteTree->queryServerId())
+                remoteTree->resetState(CPS_Changed, true);
+            return tree;
+        }
     }
-    else
-        return PARENT::ownPTree(tree);
+
+    return PARENT::ownPTree(tree);
 }
 
 IPropertyTree *CClientRemoteTree::create(const char *name, IPTArrayValue *value, ChildMap *children, bool existing)
@@ -1105,13 +1108,13 @@ IPropertyTree *CClientRemoteTree::collateData()
     {
         ChangeTree(IPropertyTree *donor=NULL) { ptree = LINK(donor); }
         ~ChangeTree() { ::Release(ptree); }
-        inline void createTree() { assertex(!ptree); ptree = createPTree(RESERVED_CHANGE_NODE); }
+        inline void createTree() { assertex(!ptree); ptree = createPTree(RESERVED_CHANGE_NODE, ipt_fast); }
         inline IPropertyTree *queryTree() { return ptree; }
         inline IPropertyTree *getTree() { return LINK(ptree); }
         inline IPropertyTree *queryCreateTree()
         {
             if (!ptree)
-                ptree = createPTree(RESERVED_CHANGE_NODE);
+                ptree = createPTree(RESERVED_CHANGE_NODE, ipt_fast);
             return ptree;
         }
     private:
@@ -1126,10 +1129,10 @@ IPropertyTree *CClientRemoteTree::collateData()
         Owned<IAttributeIterator> iter = getAttributes();
         if (iter->count())
         {
-            IPropertyTree *t = createPTree();
+            IPropertyTree *t = ct.queryTree()->addPropTree(ATTRCHANGE_TAG);
             ForEach(*iter)
                 t->setProp(iter->queryName(), queryProp(iter->queryName()));
-            ct.queryTree()->addPropTree(ATTRCHANGE_TAG, t);
+
         }
         ct.queryTree()->setPropBool("@new", true);
     }
@@ -1142,10 +1145,9 @@ IPropertyTree *CClientRemoteTree::collateData()
             {
                 ct.queryTree()->removeTree(ac);
                 Owned<IAttributeIterator> iter = ac->getAttributes();
-                IPropertyTree *t = createPTree();
+                IPropertyTree *t = ct.queryTree()->addPropTree(ATTRCHANGE_TAG);
                 ForEach(*iter)
                     t->setProp(iter->queryName(), queryProp(iter->queryName()));
-                ct.queryTree()->addPropTree(ATTRCHANGE_TAG, t);
             }
         }
     }
@@ -1544,7 +1546,7 @@ IPropertyTreeIterator *CClientSDSManager::getElements(CRemoteConnection &connect
             unsigned count;
             mb.read(count);
             CDisableFetchChangeBlock block(connection);
-            Owned<CPTArrayIterator> iter = new CPTArrayIterator();
+            Owned<DaliPTArrayIterator> iter = new DaliPTArrayIterator();
             while (count--)
             {
                 CClientRemoteTree *tree = new CClientRemoteTree(connection);
@@ -1945,9 +1947,9 @@ IPropertyTree &CClientSDSManager::queryProperties() const
         default:
             assertex(false);
     }
-    properties = createPTree(mb);
+    properties = createPTree(mb, ipt_lowmem);
     if (!properties->hasProp("Client"))
-        properties->setPropTree("Client", createPTree());
+        properties->setPropTree("Client");
     return *properties;
 }
 
@@ -1968,7 +1970,7 @@ IPropertyTree *CClientSDSManager::getXPaths(__int64 serverId, const char *xpath,
     switch (replyMsg)
     {
         case DAMP_SDSREPLY_OK:
-            return createPTree(mb);
+            return createPTree(mb, ipt_lowmem);
         case DAMP_SDSREPLY_EMPTY:
             return NULL;
         case DAMP_SDSREPLY_ERROR:
@@ -2007,7 +2009,7 @@ IPropertyTreeIterator *CClientSDSManager::getXPathsSortLimit(const char *baseXPa
         default:
             throwUnexpected();
     }
-    Owned<IPropertyTree> matchTree = createPTree(mb);
+    Owned<IPropertyTree> matchTree = createPTree(mb, ipt_lowmem);
     Owned<CRemoteConnection> conn = (CRemoteConnection *)connect(baseXPath, myProcessSession(), RTM_LOCK_READ, INFINITE);
     if (!conn)
         return createNullPTreeIterator();
@@ -2071,12 +2073,12 @@ IPropertyTreeIterator *CClientSDSManager::getElementsRaw(const char *xpath, INod
     {
         case DAMP_SDSREPLY_OK:
         {
-            Owned<CPTArrayIterator> resultIterator = new CPTArrayIterator;
+            Owned<DaliPTArrayIterator> resultIterator = new DaliPTArrayIterator;
             unsigned count, c;
             mb.read(count);
             for (c=0; c<count; c++)
             {
-                Owned<IPropertyTree> item = createPTree(mb);
+                Owned<IPropertyTree> item = createPTree(mb, ipt_lowmem);
                 resultIterator->array.append(*LINK(item));
             }
             return LINK(resultIterator);

dali/base/dadfs.cpp

@@ -11235,6 +11235,37 @@ bool decodeChildGroupName(const char *gname,StringBuffer &parentname, StringBuff
     return true;
 }
 
+/* given a list of group offsets (positions), create a compact representation of the range
+ * compatible with the group range syntax, e.g. mygroup[1-5,8-10] or mygroup[1,5,10]
+ */
+StringBuffer &encodeChildGroupRange(UnsignedArray &positions, StringBuffer &rangeText)
+{
+    unsigned items = positions.ordinality();
+    if (0 == items)
+        return rangeText;
+    unsigned start = positions.item(0);
+    unsigned last = start;
+    rangeText.append('[');
+    unsigned p=1;
+    while (true)
+    {
+        unsigned pos = p==items ? NotFound : positions.item(p++);
+        if ((pos != last+1))
+        {
+            if (last-start>0)
+                rangeText.append(start).append('-').append(last);
+            else
+                rangeText.append(last);
+            if (NotFound == pos)
+                break;
+            rangeText.append(',');
+            start = pos;
+        }
+        last = pos;
+    }
+    return rangeText.append(']');
+}
+
 class CLightWeightSuperFileConn: implements ISimpleSuperFileEnquiry, public CInterface
 {
     CFileLock lock;

dali/base/dadfs.hpp

@@ -725,6 +725,7 @@ interface INamedGroupStore: implements IGroupResolver
 extern da_decl INamedGroupStore  &queryNamedGroupStore();
 
 extern da_decl bool decodeChildGroupName(const char *gname,StringBuffer &parentname, StringBuffer &range);
+extern da_decl StringBuffer &encodeChildGroupRange(UnsignedArray &positions, StringBuffer &rangeText);
 
 
 // ==MISC========================================================================================================

dali/base/dasds.cpp

@@ -7042,7 +7042,7 @@ IPropertyTreeIterator *CCovenSDSManager::getElements(CRemoteConnection &connecti
     Owned<CLCReadLockBlock> lockBlock = new CLCReadLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__);
     CDisableFetchChangeBlock block(connection);
     Owned<CServerRemoteTree> serverConnRoot = (CServerRemoteTree *)getRegisteredTree(((CClientRemoteTree *)connection.queryRoot())->queryServerId());
-    Owned<CPTArrayIterator> elements = new CPTArrayIterator();
+    Owned<DaliPTArrayIterator> elements = new DaliPTArrayIterator();
     Owned<IPropertyTreeIterator> iter = serverConnRoot->getElements(xpath);
     ForEach (*iter)
     {

dali/base/dasds.ipp

@@ -109,7 +109,7 @@ class ChangeInfo : public CInterfaceOf<IInterface>
 {
     DECL_NAMEDCOUNT;
 public:
-    ChangeInfo(IPropertyTree &_owner) : owner(&_owner) { INIT_NAMEDCOUNT; tree.setown(createPTree(RESERVED_CHANGE_NODE)); }
+    ChangeInfo(IPropertyTree &_owner) : owner(&_owner) { INIT_NAMEDCOUNT; tree.setown(createPTree(RESERVED_CHANGE_NODE, ipt_fast)); }
     const IPropertyTree *queryOwner() const { return owner; }
     const void *queryFindParam() const { return &owner; }
 public: // data
@@ -276,26 +276,25 @@ public:
     void registerRenamed(IPropertyTree &owner, const char *newName, const char *oldName, unsigned pos, __int64 id)
     {
         ChangeInfo *changes = queryCreateChangeInfo(owner);
-        IPropertyTree *t = createPTree();
+        IPropertyTree *t = changes->tree->addPropTree(RENAME_TAG);
         t->setProp("@from", oldName);
         t->setProp("@to", newName);
         t->setPropInt64("@id", id);
 #ifdef SIBLING_MOVEMENT_CHECK
         t->setProp("@pos", pos);
 #endif
-        changes->tree->addPropTree(RENAME_TAG, t);
+
     }
 
     void registerDeleted(IPropertyTree &owner, const char *name, unsigned pos, __int64 id)
     {
         ChangeInfo *changes = queryCreateChangeInfo(owner);
-        IPropertyTree *t = createPTree();
+        IPropertyTree *t = changes->tree->addPropTree(DELETE_TAG);
         t->setProp("@name", name);
         t->setPropInt64("@id", id);
 #ifdef SIBLING_MOVEMENT_CHECK
         t->setPropInt("@pos", pos+1);
 #endif
-        changes->tree->addPropTree(DELETE_TAG, t);
     }
 
     virtual void registerAttrChange(IPropertyTree &owner, const char *attr)
@@ -305,7 +304,7 @@ public:
         if (t) t->removeProp(attr);
         t = changes->tree->queryPropTree("AC");
         if (!t)
-            t = changes->tree->addPropTree("AC", createPTree());
+            t = changes->tree->addPropTree("AC");
         t->setProp(attr, "");
     }
 
@@ -316,7 +315,7 @@ public:
         if (t) t->removeProp(attr);
         t = changes->tree->queryPropTree("AD");
         if (!t)
-            t = changes->tree->addPropTree("AD", createPTree());
+            t = changes->tree->addPropTree("AD");
         t->addProp(attr, "");
     }
 
@@ -325,7 +324,7 @@ public:
         ChangeInfo *changes = queryCreateChangeInfo(owner);
         IPropertyTree *t = changes->tree->queryPropTree(APPEND_TAG);
         if (!t)
-            t = changes->tree->setPropTree(APPEND_TAG, createPTree());
+            t = changes->tree->setPropTree(APPEND_TAG);
         t->setPropInt(NULL, l);
     }
 
@@ -535,11 +534,11 @@ protected:
     CConnectionHashTable connections;
 };
 
-class CPTArrayIterator : public CArrayIteratorOf<IPropertyTree, IPropertyTreeIterator>
+class DaliPTArrayIterator : public CArrayIteratorOf<IPropertyTree, IPropertyTreeIterator>
 {
     DECL_NAMEDCOUNT;
 public:
-    CPTArrayIterator() : CArrayIteratorOf<IPropertyTree, IPropertyTreeIterator>(array) { INIT_NAMEDCOUNT; }
+    DaliPTArrayIterator() : CArrayIteratorOf<IPropertyTree, IPropertyTreeIterator>(array) { INIT_NAMEDCOUNT; }
     IArrayOf<IPropertyTree> array;
 };
 

dali/daliadmin/daliadmin.cpp

@@ -102,7 +102,7 @@ void usage(const char *exe)
   printf("  dfscompratio <logicalname>      -- returns compression ratio of file\n");
   printf("  dfsscopes <mask>                -- lists logical scopes (mask = * for all)\n");
   printf("  cleanscopes                     -- remove empty scopes\n");
-  printf("  dfsreplication <clustermask> <logicalnamemask> <redundancy-count> -- set redundancy for files matching mask, on specified clusters only\n");
+  printf("  dfsreplication <clustermask> <logicalnamemask> <redundancy-count> [dryrun] -- set redundancy for files matching mask, on specified clusters only\n");
   printf("  holdlock <logicalfile> <read|write> -- hold a lock to the logical-file until a key is pressed");
   printf("\n");
   printf("Workunit commands:\n");
@@ -129,6 +129,7 @@ void usage(const char *exe)
   printf("  wuidcompress <wildcard> <type>  --  scan workunits that match <wildcard> and compress resources of <type>\n");
   printf("  wuiddecompress <wildcard> <type> --  scan workunits that match <wildcard> and decompress resources of <type>\n");
   printf("  xmlsize <filename> [<percentage>] --  analyse size usage in xml file, display individual items above 'percentage' \n");
+  printf("  migratefiles <src-group> <target-group> [<filemask>] [dryrun] [createmaps] [listonly] [verbose]\n");
   printf("\n");
   printf("Common options\n");
   printf("  server=<dali-server-ip>         -- server ip\n");
@@ -1830,6 +1831,7 @@ static void dfsreplication(const char *clusterMask, const char *lfnMask, unsigne
 
     const char *basePath = "/Files";
     const char *propToSet = "@redundancy";
+    const char *defVal = "1"; // default reduncancy value, attribute not set/stored if equal to default.
     StringBuffer value;
     value.append(redundancy);
 
@@ -1847,7 +1849,7 @@ static void dfsreplication(const char *clusterMask, const char *lfnMask, unsigne
         {
             IPropertyTree &cluster = clusterIter->query();
             const char *oldValue = cluster.queryProp(propToSet);
-            if (!oldValue || !streq(value, oldValue))
+            if ((!oldValue && !streq(value, defVal)) || (oldValue && !streq(value, oldValue)))
             {
                 const char *fileName = file.queryProp("OrigName");
                 const char *clusterName = cluster.queryProp("@name");
@@ -1856,7 +1858,12 @@ static void dfsreplication(const char *clusterMask, const char *lfnMask, unsigne
                     msg.appendf(" [old value = %s]", oldValue);
                 PROGLOG("%s", msg.str());
                 if (!dryRun)
-                    cluster.setProp(propToSet, value);
+                {
+                    if (!streq(value, defVal))
+                        cluster.setProp(propToSet, value);
+                    else
+                        cluster.removeProp(propToSet);
+                }
             }
         }
     }
@@ -2808,6 +2815,285 @@ static void validateStore(bool fix, bool deleteFiles, bool verbose)
 
 //=============================================================================
 
+static void migrateFiles(const char *srcGroup, const char *tgtGroup, const char *filemask, const char *_options)
+{
+    if (strieq(srcGroup, tgtGroup))
+        throw makeStringExceptionV(0, "source and target cluster groups cannot be the same! cluster = %s", srcGroup);
+
+    enum class mg_options : unsigned { nop, createmaps=1, listonly=2, dryrun=4, verbose=8};
+
+    StringArray options;
+    options.appendList(_options, ",");
+    mg_options opts = mg_options::nop;
+    ForEachItemIn(o, options)
+    {
+        const char *opt = options.item(o);
+        if (strieq("CREATEMAPS", opt))
+            opts = (mg_options)((unsigned)opts | (unsigned)mg_options::createmaps);
+        else if (strieq("LISTONLY", opt))
+            opts = (mg_options)((unsigned)opts | (unsigned)mg_options::listonly);
+        else if (strieq("DRYRUN", opt))
+            opts = (mg_options)((unsigned)opts | (unsigned)mg_options::dryrun);
+        else if (strieq("VERBOSE", opt))
+            opts = (mg_options)((unsigned)opts | (unsigned)mg_options::verbose);
+        else
+            WARNLOG("Unknown option: %s", opt);
+    }
+
+    /*
+     * CMatchScanner scans logical files, looking for files that are in the source group
+     * and matching against the logical file names against filemask.
+     * Then (depending on options) manipulates the meta data to point to new target group
+     * and outputs a file per node of the source group, with a list of all matching
+     * physical files in the format: srcIP,dstIP,physical file
+     */
+    class CMatchScanner : public CSDSFileScanner
+    {
+        StringAttr srcGroup, tgtGroup;
+        mg_options options;
+        StringBuffer tgtClusterGroupText;
+        Owned<IGroup> srcClusterGroup, tgtClusterGroup;
+        IPointerArrayOf<IFileIOStream> fileLists;
+        unsigned matchingFiles = 0;
+        Linked<IRemoteConnection> conn;
+        StringAttr filemask;
+        bool wild = false;
+        unsigned srcClusterSize = 0;
+        unsigned tgtClusterSize = 0;
+
+        bool mgOpt(mg_options o)
+        {
+            return ((unsigned)o & (unsigned)options);
+        }
+        IFileIOStream *getFileIOStream(unsigned p)
+        {
+            while (fileLists.ordinality()<=p)
+                fileLists.append(nullptr);
+
+            Linked<IFileIOStream> stream = fileLists.item(p);
+            if (nullptr == stream)
+            {
+                VStringBuffer filePartList("fileparts%u_%s_%u.lst", GetCurrentProcessId(), srcGroup.get(), p);
+                Owned<IFile> iFile = createIFile(filePartList);
+                Owned<IFileIO> iFileIO = iFile->open(IFOcreate);
+                if (!iFileIO)
+                    throw makeStringExceptionV(0, "Failed to open: %s", filePartList.str());
+                stream.setown(createBufferedIOStream(iFileIO));
+                fileLists.replace(stream.getLink(), p);
+            }
+            return stream.getClear();
+        }
+        unsigned find(IGroup *group, const IpAddress &ip) const
+        {
+            unsigned c = group->ordinality();
+            for (unsigned i=0; i<c; i++)
+            {
+                const IpAddress &nodeIP = group->queryNode(i).endpoint();
+                if (ip.ipequals(nodeIP))
+                    return i;
+            }
+            return NotFound;
+        }
+    public:
+        CMatchScanner(const char *_srcGroup, const char *_tgtGroup, mg_options _options) : srcGroup(_srcGroup), tgtGroup(_tgtGroup), options(_options)
+        {
+            srcClusterGroup.setown(queryNamedGroupStore().lookup(srcGroup));
+            if (!srcClusterGroup)
+                throw makeStringExceptionV(0, "Could not find source cluster group: %s", _srcGroup);
+            tgtClusterGroup.setown(queryNamedGroupStore().lookup(tgtGroup));
+            if (!tgtClusterGroup)
+                throw makeStringExceptionV(0, "Could not find target cluster group: %s", _tgtGroup);
+
+            srcClusterSize = srcClusterGroup->ordinality();
+            tgtClusterSize = tgtClusterGroup->ordinality();
+            if (tgtClusterSize>srcClusterSize)
+                throw makeStringExceptionV(0, "Unsupported - target cluster is wider than source (target size=%u, source size=%u", tgtClusterSize, srcClusterSize);
+            if (0 != (srcClusterSize%tgtClusterSize))
+                throw makeStringExceptionV(0, "Unsupported - target cluster must be a factor of source cluster size (target size=%u, source size=%u", tgtClusterSize, srcClusterSize);
+
+            tgtClusterGroup->getText(tgtClusterGroupText);
+        }
+        virtual bool checkFileOk(IPropertyTree &file, const char *filename) override
+        {
+            const char *group = file.queryProp("@group");
+            if (!group)
+            {
+                if (mgOpt(mg_options::verbose))
+                    PROGLOG("No group defined - filename=%s, mask=%s, srcGroup=%s", filename, filemask.get(), srcGroup.get());
+                return false;
+            }
+            else if (nullptr == strstr(file.queryProp("@group"), srcGroup)) // crude match, could be rejected in processFile
+            {
+                if (mgOpt(mg_options::verbose))
+                    PROGLOG("GROUP-MISMATCH - filename=%s, mask=%s, srcGroup=%s, file group=%s", filename, filemask.get(), srcGroup.get(), group);
+                return false;
+            }
+            else if (wild)
+            {
+                if (WildMatch(filename, filemask, false))
+                {
+                    if (mgOpt(mg_options::verbose))
+                        PROGLOG("WILD-MISMATCH - filename=%s, mask=%s, srcGroup=%s, file group=%s", filename, filemask.get(), srcGroup.get(), group);
+                    return true;
+                }
+            }
+            else if (strieq(filename, filemask))
+                return true;
+            if (mgOpt(mg_options::verbose))
+                PROGLOG("EXACT-MISMATCH - filename=%s, mask=%s, srcGroup=%s, file group=%s", filename, filemask.get(), srcGroup.get(), group);
+            return false;
+        }
+        virtual bool checkScopeOk(const char *scopename) override
+        {
+            if (mgOpt(mg_options::verbose))
+                PROGLOG("Processing scope %s", scopename);
+            return true;
+        }
+        virtual void processFile(IPropertyTree &root, StringBuffer &name) override
+        {
+            try
+            {
+                bool doCommit = false;
+                StringBuffer _tgtClusterGroupText = tgtClusterGroupText;
+
+                Owned<IFileDescriptor> fileDesc = deserializeFileDescriptorTree(&root, &queryNamedGroupStore());
+                unsigned numClusters = fileDesc->numClusters();
+                for (unsigned clusterNum=0; clusterNum<numClusters; clusterNum++)
+                {
+                    StringBuffer srcFileGroup;
+                    fileDesc->getClusterGroupName(clusterNum, srcFileGroup);
+
+                    StringBuffer srcFileGroupName, srcFileGroupRange;
+                    if (!decodeChildGroupName(srcFileGroup, srcFileGroupName, srcFileGroupRange))
+                        srcFileGroupName.append(srcFileGroup);
+                    if (streq(srcFileGroupName, srcGroup))
+                    {
+                        IGroup *srcFileClusterGroup = fileDesc->queryClusterGroup(clusterNum);
+                        unsigned srcFileClusterGroupWidth = srcFileClusterGroup->ordinality();
+
+                        StringBuffer _tgtGroup(tgtGroup);
+                        unsigned groupOffset = NotFound;
+                        if (srcFileGroupRange.length())
+                        {
+                            SocketEndpointArray epas;
+                            UnsignedArray dstPositions;
+                            Owned<INodeIterator> nodeIter = srcFileClusterGroup->getIterator();
+                            ForEach(*nodeIter)
+                            {
+                                const IpAddress &ip = nodeIter->query().endpoint();
+                                unsigned srcRelPos = find(srcClusterGroup, ip);
+                                if (NotFound == groupOffset)
+                                    groupOffset = srcRelPos;
+                                unsigned dstRelPos = srcRelPos % tgtClusterSize;
+                                dstPositions.append(dstRelPos);
+                            }
+                            StringBuffer rangeText;
+                            encodeChildGroupRange(dstPositions, rangeText);
+                            _tgtGroup.append(rangeText);
+                        }
+                        else
+                            groupOffset = 0;
+                        unsigned numParts = fileDesc->numParts();
+                        PROGLOG("Processing file %s (width=%u), cluster group=%s (%u of %u), new group = %s", name.str(), numParts, srcFileGroup.str(), clusterNum+1, numClusters, _tgtGroup.str());
+                        if (!mgOpt(mg_options::listonly))
+                        {
+                            if (!mgOpt(mg_options::dryrun))
+                            {
+                                doCommit = true;
+                                VStringBuffer clusterXPath("Cluster[%u]", clusterNum+1);
+                                IPropertyTree *cluster = root.queryPropTree(clusterXPath);
+                                root.setProp("@group", _tgtGroup);
+                                if (cluster)
+                                    cluster->setProp("@name", _tgtGroup);
+                                else
+                                    WARNLOG("No Cluster found for file: %s", name.str());
+                            }
+                            if (mgOpt(mg_options::createmaps))
+                            {
+                                for (unsigned partNum=0; partNum<numParts; partNum++)
+                                {
+                                    unsigned r = partNum % srcFileClusterGroupWidth;
+                                    const SocketEndpoint &srcEp = srcFileClusterGroup->queryNode(r).endpoint();
+                                    unsigned relPos = find(srcClusterGroup, srcEp);
+                                    unsigned dstPos = (partNum+groupOffset) % tgtClusterSize;
+                                    const SocketEndpoint &tgtEp = tgtClusterGroup->queryNode(dstPos).endpoint();
+
+                                    // output srcIP, dstIP, path/file-part-name >> script<N>.lst
+
+                                    Owned<IFileIOStream> iFileIOStream = getFileIOStream(relPos+1);
+
+                                    StringBuffer outputLine;
+                                    srcEp.getIpText(outputLine);
+                                    outputLine.append(",");
+                                    tgtEp.getIpText(outputLine);
+                                    outputLine.append(",");
+
+                                    IPartDescriptor *part = fileDesc->queryPart(partNum);
+                                    StringBuffer filePath;
+                                    part->getPath(filePath);
+
+                                    outputLine.append(filePath);
+                                    outputLine.newline();
+
+                                    iFileIOStream->write(outputLine.length(), outputLine.str());
+                                }
+                            }
+                        }
+                    }
+                }
+                ++matchingFiles;
+                if (doCommit)
+                    conn->commit(); // NB: the scanner rolls back any changes, mainly to reduce cost/exposure to previously lazy fetched scope branches
+            }
+            catch (IException *e)
+            {
+                VStringBuffer errorMsg("Failed to process file : %s", name.str());
+                EXCLOG(e, errorMsg.str());
+                e->Release();
+            }
+        }
+        unsigned scan(IRemoteConnection *_conn, const char *_filemask, bool includefiles=true, bool includesuper=false)
+        {
+            filemask.set(_filemask);
+            conn.set(_conn);
+            wild = containsWildcard(_filemask);
+            CSDSFileScanner::scan(_conn, includefiles, includesuper);
+            return matchingFiles;
+        }
+    } scanner(srcGroup, tgtGroup, opts);
+
+    IUserDescriptor *user = nullptr;
+    Owned<IRemoteConnection> conn = querySDS().connect("/Files", myProcessSession(), 0, 100000);
+    bool success=false;
+    unsigned matchingFiles=0;
+    try
+    {
+        matchingFiles = scanner.scan(conn, filemask, true, false);
+        success=true;
+    }
+    catch (IException *e)
+    {
+        EXCLOG(e, nullptr);
+        e->Release();
+    }
+    if (!success)
+    {
+        WARNLOG("Failed to make changes");
+        conn->rollback();
+    }
+    else if ((unsigned)opts & (unsigned)mg_options::dryrun)
+    {
+        conn->rollback();
+        WARNLOG("Dry-run, no changes committed. %u files matched", matchingFiles);
+    }
+    else
+        PROGLOG("Committed changes: %u files changed", matchingFiles);
+}
+
+
+//=============================================================================
+
+
 
 void testThorRunningWUs()
 {
@@ -3215,6 +3501,24 @@ int main(int argc, char* argv[])
                             dumpStats(params.item(1), params.item(2), params.item(3), params.item(4), params.item(5), params.item(6), nullptr, csv);
                         }
                     }
+                    else if (stricmp(cmd, "migratefiles") == 0)
+                    {
+                        CHECKPARAMS(2, 7);
+                        const char *srcGroup = params.item(1);
+                        const char *dstGroup = params.item(2);
+                        const char *filemask = "*";
+                        StringBuffer options;
+                        if (params.isItem(3))
+                        {
+                            filemask = params.item(3);
+                            unsigned arg=4;
+                            StringArray optArray;
+                            while (arg<params.ordinality())
+                                optArray.append(params.item(arg++));
+                            optArray.getString(options, ",");
+                        }
+                        migrateFiles(srcGroup, dstGroup, filemask, options);
+                    }
                     else
                         ERRLOG("Unknown command %s",cmd);
                 }

dali/dfuplus/dfuplus.cpp

@@ -1675,22 +1675,24 @@ int CDfuPlusHelper::erasehistory()
 
     progress("\nErase history of '%s' with%s backup.\n", lfn, (backup ? "" : "out"));
 
-    Owned<IClientEraseHistoryRequest> req = dfuclient->createEraseHistoryRequest();
-    req->setName(lfn);
+    if (backup)
+    {
+        // Get and backup file history before erased.
+        // If any problem happens during the backup the history remain intact.
+        Owned<IClientListHistoryRequest> req = dfuclient->createListHistoryRequest();
+        req->setName(lfn);
 
-    Owned<IClientEraseHistoryResponse> resp = dfuclient->EraseHistory(req);
+        Owned<IClientListHistoryResponse> resp = dfuclient->ListHistory(req);
 
-    const IMultiException* excep = &resp->getExceptions();
-    if (excep != nullptr && excep->ordinality() > 0)
-    {
-        StringBuffer errmsg;
-        excep->errorMessage(errmsg);
-        error("%s\n", errmsg.str());
-        return -1;
-    }
+        const IMultiException* excep = &resp->getExceptions();
+        if (excep != nullptr && excep->ordinality() > 0)
+        {
+            StringBuffer errmsg;
+            excep->errorMessage(errmsg);
+            error("%s\n", errmsg.str());
+            return -1;
+        }
 
-    if (backup)
-    {
         IArrayOf<IConstHistory>& arrHistory = resp->getHistory();
         if (0 == arrHistory.length())
         {
@@ -1705,20 +1707,36 @@ int CDfuPlusHelper::erasehistory()
 
         int ofile = open(dstxml, _O_WRONLY | _O_CREAT | _O_TRUNC, _S_IREAD | _S_IWRITE);
         if(ofile == -1)
-            throw MakeStringException(-1, "can't open file %s\n", dstxml);
+            throw MakeStringException(-1, "can't open file %s, erase history cancelled.\n", dstxml);
 
         ssize_t written = write(ofile, xmlDump.str(), xmlDump.length());
         if (written < 0)
-            throw MakeStringException(-1, "can't write to file %s\n", dstxml);
+            throw MakeStringException(-1, "can't write to file %s, erase history cancelled.\n", dstxml);
 
         if (written != xmlDump.length())
-            throw MakeStringException(-1, "truncated write to file %s\n", dstxml);
+            throw MakeStringException(-1, "truncated write to file %s, erase history cancelled.\n", dstxml);
 
         close(ofile);
 
         info("History written into %s.\n",dstxml);
     }
 
+    Owned<IClientEraseHistoryRequest> req = dfuclient->createEraseHistoryRequest();
+    req->setName(lfn);
+
+    Owned<IClientEraseHistoryResponse> resp = dfuclient->EraseHistory(req);
+
+    const IMultiException* excep = &resp->getExceptions();
+    if (excep != nullptr && excep->ordinality() > 0)
+    {
+        StringBuffer errmsg;
+        excep->errorMessage(errmsg);
+        error("%s\n", errmsg.str());
+        return -1;
+    }
+
+    info("History erased.\n");
+
     return 0;
 }
 

deployment/deployutils/deployutils.cpp

@@ -332,6 +332,14 @@ void addItem(StringBuffer& jsStrBuf,
              const char* extra, 
              short ctrlType)
 {
+  // Control types:
+  // 1 - text entry
+  // 2 - ?
+  // 3 - true/false radio buttons
+  // 4 - drop menu
+  // 5 - password entry
+  // 6 - multiline text
+  // 7-? ?
   StringBuffer sbAttr("Attributes");
 
   jsStrBuf.appendf("var attr%s%s = {};", attrName, tabName);
@@ -513,6 +521,10 @@ public:
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, TAG_PORT, "", 0, 1, "", 1);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, TAG_REQARRAYTHREADS, "", 0, 1, "", 1);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, "aclName", "", 0, 1, "|'#$process/ACL'", 4);
+        addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, "protocol", "", 0, 1, "|new Array('ssl','native')", 4);
+        addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, "passphrase", "", 0, 1, "", 5);
+        addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, "certificateFileName", "", 0, 1, "", 1);
+        addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_FARM, "privateKeyFileName", "", 0, 1, "", 1);
 
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_ONLY_SLAVE, TAG_NAME, "", 0, 1, "", 0);
         addItem(jsStrBuf, m_pEnv.get(), XML_TAG_ROXIE_ONLY_SLAVE, TAG_COMPUTER, "", 0, 1, "", 0);
@@ -540,6 +552,10 @@ public:
           m_colIndex.appendf("colIndex['numThreads%s']=%d;", serverStr, index++);
           m_colIndex.appendf("colIndex['requestArrayThreads%s']=%d;", serverStr, index++);
           m_colIndex.appendf("colIndex['aclName%s']=%d;", serverStr, index++);
+          m_colIndex.appendf("colIndex['protocol%s']=%d;", serverStr, index++);
+          m_colIndex.appendf("colIndex['passphrase%s']=%d;", serverStr, index++);
+          m_colIndex.appendf("colIndex['certificateFileName%s']=%d;", serverStr, index++);
+          m_colIndex.appendf("colIndex['privateKeyFileName%s']=%d;", serverStr, index++);
 
           index = 0;
           const char* agentStr = "Agents";
@@ -1654,6 +1670,26 @@ public:
             }
           }
         }
+        else if(!strcmp(type,"serverListType"))
+        {
+          if(m_wizard)
+          {
+            StringBuffer buildSetName, ipAddr;
+            tempPath.clear().appendf("./Programs/Build/BuildSet[%s=\"%s\"]",XML_ATTR_PROCESS_NAME,m_compName.str());
+            IPropertyTree* pCompTree = m_pEnv->queryPropTree(tempPath.str());
+            if(pCompTree)
+            {
+              buildSetName.append(pCompTree->queryProp(XML_ATTR_NAME));
+              CInstDetails* pInst  = m_wizard->getServerIPMap(compName, buildSetName,m_pEnv);
+              if( pInst )
+              {
+                StringArray& ipArray = pInst->getIpAssigned();
+                if(ipArray.length())
+                  wizDefVal.clear().append(ipArray.item(0));
+              }
+            }
+          }
+        }
         else if(!strcmp(type,"xs:string"))
         {
           StringBuffer nameOfComp;

docs/ECLProgrammersGuide/PRG_Mods/CodeSign.xml

@@ -75,7 +75,8 @@
 
     <para><emphasis role="bold">Cluster</emphasis></para>
 
-    <para>Specify the cluster for which this rule applies.</para>
+    <para>Specify the cluster for which this rule applies. If cluster is left
+    blank, the restriction applies to all clusters in the environment.</para>
 
     <para><emphasis role="bold">Value</emphasis></para>
 
@@ -106,6 +107,14 @@
 
             <entry>Allow/Deny an external function (SERVICE)</entry>
           </row>
+
+          <row>
+            <entry><emphasis>datafile</emphasis></entry>
+
+            <entry>(Valid only for --allowedsigned). This specifies that
+            access to data is only allowed if the code has been signed and the
+            key is present. </entry>
+          </row>
         </tbody>
       </tgroup>
     </informaltable>

docs/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml

@@ -78,6 +78,13 @@
               </row>
 
               <row>
+                <entry>results</entry>
+
+                <entry>returns the full results of a given WUID in XML
+                format.</entry>
+              </row>
+
+              <row>
                 <entry>activate</entry>
 
                 <entry>Activate a published query</entry>
@@ -220,6 +227,7 @@ ECL_USER_NAME
 ECL_PASSWORD
 ECL_WAIT_TIMEOUT
 ECL_RESULT_LIMIT
+ECLCC_PATH
 </programlisting>
 
           <para></para>
@@ -1053,6 +1061,109 @@ ecl run thor findperson.ecl -I C:\MyECL\
           </informaltable></para>
       </sect2>
 
+      <sect2 id="CT_CLI_SyntaxECLResult" role="brk">
+        <title>ecl results </title>
+
+        <para><emphasis role="bold">ecl results &lt;wuid&gt; [--noroot]
+        [--exception-level=<emphasis>&lt;value&gt;</emphasis>]
+        </emphasis></para>
+
+        <para><emphasis role="bold">Examples</emphasis>:</para>
+
+        <programlisting>ecl results W20170519-142920 
+ecl results W20170519-142920 --noroot --exception-level=error
+</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="1" frame="all" rowsep="1">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec />
+
+              <tbody>
+                <row>
+                  <entry>ecl results</entry>
+
+                  <entry>returns the full results of a given WUID in XML
+                  format.</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Arguments</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>wuid</entry>
+
+                  <entry>The workunit from which to return results.</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>--noroot</entry>
+
+                  <entry>Suppresses the &lt;Result&gt; root tag in the XML
+                  returned.</entry>
+                </row>
+
+                <row>
+                  <entry>--exception-level</entry>
+
+                  <entry>Sets the minimum severity for reporting exceptions.
+                  Possible severity levels are <emphasis
+                  role="bold">info</emphasis>, <emphasis
+                  role="bold">warning</emphasis>, or <emphasis
+                  role="bold">error</emphasis>. The default is info which
+                  returns all exceptions.</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+
+                <row>
+                  <entry>-s, --server</entry>
+
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
+                </row>
+
+                <row>
+                  <entry>--port</entry>
+
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
+                </row>
+
+                <row>
+                  <entry>-ssl</entry>
+
+                  <entry>Use SSL to secure the connection to the
+                  server.</entry>
+                </row>
+
+                <row>
+                  <entry>-u, --username</entry>
+
+                  <entry>The username (if necessary)</entry>
+                </row>
+
+                <row>
+                  <entry>-pw, --password</entry>
+
+                  <entry>The password (if necessary)</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
       <sect2 id="CT_CLI_SyntaxECLActivate" role="brk">
         <title>ecl activate</title>
 
@@ -2954,7 +3065,7 @@ ecl packagemap validate roxie --active</programlisting>
         <para><emphasis role="bold">ecl packagemap copy &lt;path&gt;
         &lt;target&gt; </emphasis></para>
 
-        <para>Copies a packagemap from one target to another. </para>
+        <para>Copies a packagemap from one target to another.</para>
 
         <para>Examples:</para>
 
@@ -2982,8 +3093,8 @@ ecl packagemap copy //192.168.0.100:8010/roxie/MyPkg roxie2
                 <row>
                   <entry>copy</entry>
 
-                  <entry>Copies a packagemap from one target to another
-                  </entry>
+                  <entry>Copies a packagemap from one target to
+                  another</entry>
                 </row>
 
                 <row>
@@ -2993,7 +3104,7 @@ ecl packagemap copy //192.168.0.100:8010/roxie/MyPkg roxie2
                 <row>
                   <entry>path</entry>
 
-                  <entry>Path to the source packagemap to copy. </entry>
+                  <entry>Path to the source packagemap to copy.</entry>
                 </row>
 
                 <row>
@@ -3027,39 +3138,39 @@ ecl packagemap copy //192.168.0.100:8010/roxie/MyPkg roxie2
                 </row>
 
                 <row>
-                  <entry>--source-process </entry>
+                  <entry>--source-process</entry>
 
                   <entry>Process cluster to copy files from</entry>
                 </row>
 
                 <row>
-                  <entry>--preload-all </entry>
+                  <entry>--preload-all</entry>
 
                   <entry>Set preload files option for all packages</entry>
                 </row>
 
                 <row>
-                  <entry>--replace </entry>
+                  <entry>--replace</entry>
 
-                  <entry>Replace existing packagmap </entry>
+                  <entry>Replace existing packagmap</entry>
                 </row>
 
                 <row>
-                  <entry>--update-super-files </entry>
+                  <entry>--update-super-files</entry>
 
                   <entry>Update local DFS superfiles if remote Dali has
                   changed</entry>
                 </row>
 
                 <row>
-                  <entry>--update-clone-from </entry>
+                  <entry>--update-clone-from</entry>
 
                   <entry>Update local clone from location if remote Dali has
                   changed</entry>
                 </row>
 
                 <row>
-                  <entry>--dont-append-cluster </entry>
+                  <entry>--dont-append-cluster</entry>
 
                   <entry>Only use to avoid locking issues due to adding
                   cluster to file</entry>
@@ -3110,11 +3221,11 @@ ecl packagemap copy //192.168.0.100:8010/roxie/MyPkg roxie2
 
         <itemizedlist>
           <listitem>
-            <para>remote packagemap: //IP:PORT/Target/PackageMapId </para>
+            <para>remote packagemap: //IP:PORT/Target/PackageMapId</para>
           </listitem>
 
           <listitem>
-            <para>local packagemap: target/PackageMapId </para>
+            <para>local packagemap: target/PackageMapId</para>
           </listitem>
         </itemizedlist>
       </sect2>

docs/HPCCSystemAdmin/SA-Mods/CassandraWUServer.xml

@@ -270,13 +270,18 @@
               <listitem>
                 <?dbfo keep-together="always"?>
 
-                <para>Add the following options on the Options tab.<graphic
+                <para>Add options on the Options tab.<graphic
                 fileref="../../images/Cass3.jpg"
                 vendor="configmgrSS" /></para>
 
-                <para><variablelist>
+                <xi:include href="../../UsingConfigManager/UsingConfigManager.xml"
+                            xpointer="xpointer(//*[@id='daliplugoptionstable'])"
+                            xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+                <para>
+                  <variablelist>
                     <varlistentry>
-                      <term>Note: </term>
+                      <term>Note:</term>
 
                       <listitem>
                         <para>User and password are only needed if your
@@ -284,7 +289,8 @@
                         credentials.</para>
                       </listitem>
                     </varlistentry>
-                  </variablelist></para>
+                  </variablelist>
+                </para>
               </listitem>
 
               <listitem>

docs/UsingConfigManager/UsingConfigManager.xml

@@ -810,6 +810,120 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
         </sect3>
       </sect2>
 
+      <sect2 id="DaliServerPlugin" role="brk">
+        <title>DaliServerPlugin</title>
+
+        <para>DaliServerPlugin allows you to add plugin functionality to a
+        Dail server.</para>
+
+        <sect3>
+          <title>DaliServerPlugin attributes</title>
+
+          <para>This section describes the DaliServerPlugin attributes.</para>
+
+          <para>
+            <graphic fileref="images/Cass2.jpg" vendor="configmgrSS" />
+          </para>
+
+          <!--configMgr-daplugin-Include-->
+
+          <para>
+            <xi:include href="XMLGeneration/xml/daliplugin.xsd.mod.xml"
+                        xpointer="xpointer(//*[@id='daplug.t'])"
+                        xmlns:xi="http://www.w3.org/2001/XInclude" />
+          </para>
+        </sect3>
+
+        <sect3>
+          <title>DaliServerPlugin Options</title>
+
+          <para>This section describes the DaliServerPlugin options</para>
+
+          <para>These options are available for the DaliServerplugin when
+          configuring a Casandra server. See the System Administrator's Guide
+          for more details about configuring a Cassandra server as a system
+          datastore.</para>
+
+          <para>
+            <informaltable colsep="1" frame="all" id="daliplugoptionstable"
+                           rowsep="1">
+              <tgroup cols="2">
+                <colspec align="left" colwidth="125.55pt" />
+
+                <colspec />
+
+                <tbody>
+                  <row>
+                    <entry>randomWuidSuffix</entry>
+
+                    <entry>An integer value indicating how many randomized
+                    digits to append to workunits. Set this if you need to
+                    create workunits at a high rate to reduce the risk of
+                    collisions (which would slow down the process of creating
+                    a new unique workunit id).</entry>
+                  </row>
+
+                  <row>
+                    <entry>traceLevel</entry>
+
+                    <entry>An integer value indicating how much tracing to
+                    output from Cassandra workunit operations. Set to zero or
+                    do not set in normal usage.</entry>
+                  </row>
+
+                  <row>
+                    <entry>partitions</entry>
+
+                    <entry>An integer value indicating how many ways to
+                    partition the data on a Cassandra cluster. The default is
+                    2. The value only takes effect when a new Cassandra
+                    workunit repository is created. Larger values permit
+                    scaling to a more distributed store but at the expense of
+                    some overhead on smaller stores where the scaling is not
+                    needed.</entry>
+                  </row>
+
+                  <row>
+                    <entry>prefixsize</entry>
+
+                    <entry>
+                      <para>An integer value specifying the minimum number of
+                      characters that must be provided when wildcard searching
+                      in the repository. Larger values will be more efficient
+                      but also more restrictive on users. The default is 2. As
+                      with partitions, this value only takes effect when a new
+                      Cassandra workunit repository is created.</para>
+                    </entry>
+                  </row>
+
+                  <row>
+                    <entry>keyspace</entry>
+
+                    <entry>The name of the Cassandra keyspace to use for the
+                    HPCC data store. The default is
+                    <emphasis>hpcc</emphasis>.</entry>
+                  </row>
+
+                  <row>
+                    <entry>user</entry>
+
+                    <entry>The username to use if the Cassandra server is
+                    configured to require credentials.</entry>
+                  </row>
+
+                  <row>
+                    <entry>password</entry>
+
+                    <entry>The password to use if the Cassandra server is
+                    configured to require credentials.</entry>
+                  </row>
+                </tbody>
+              </tgroup>
+            </informaltable>
+          </para>
+        </sect3>
+      </sect2>
+
       <sect2 id="Dafilesrv-Process" role="brk">
         <title>Dafilesrv Process</title>
 
@@ -1050,34 +1164,138 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
       <sect2 id="Drop-Zone" role="brk">
         <title>Drop Zone</title>
 
-        <sect3>
-          <title>DropZone Attributes</title>
+        <para>A Drop Zone (or landing zone) is a location where files can be
+        transferred to or from your HPCC system. The drop zone is a logical
+        combination of a path and one or more servers.</para>
 
-          <orderedlist>
-            <listitem>
-              <para>Select Drop Zone in the Navigator panel on the left
-              side.</para>
-            </listitem>
+        <para>Multiple drop zones allow you to configure different top level
+        folders for one or more servers. Multiple servers for a single drop
+        zone provides a logical grouping of distinct locations. Multiple drop
+        zones are useful to allow different permissions for users or
+        groups.</para>
 
-            <listitem>
-              <para>Select the Attributes tab.</para>
-            </listitem>
+        <para>To add a drop zone:</para>
 
+        <para>
+          <orderedlist>
             <listitem>
-              <para>In the Value column of the Computer row, choose a node
-              from the drop list as shown below:<graphic
-              fileref="images/CM-img06.jpg" vendor="configmgrSS" /></para>
+              <para>Right-click on the Navigator panel on the left side and
+              choose <emphasis role="bold">New Components</emphasis></para>
             </listitem>
 
             <listitem>
-              <para>Click the <inlinegraphic fileref="images/GS-img01c.png" />
-              disk icon to save</para>
+              <para>Select <emphasis role="bold">Drop Zone</emphasis> </para>
             </listitem>
           </orderedlist>
-        </sect3>
+        </para>
 
         <sect3>
-          <title>DropZone Notes</title>
+          <title>Drop Zone Attributes</title>
+
+          <para>You can change the configuration of your drop zone using the
+          attributes tab. If you have multiple drop zones, select the drop
+          zone to configure from the Navigator panel on the left side.</para>
+
+          <para>To change the drop zone attributes:</para>
+
+          <para>
+            <orderedlist>
+              <listitem>
+                <para>On the <emphasis role="bold">Attributes</emphasis> tab,
+                select the Attribute to modify.</para>
+              </listitem>
+
+              <listitem>
+                <para>Double-click on the value on the right side of the
+                attribute table for the value you wish to modify. </para>
+
+                <para>For example, select the <emphasis
+                role="bold">name</emphasis> attribute, double click on the
+                <emphasis role="bold">value</emphasis> column and provide the
+                drop zone with a more meaningful name.</para>
+              </listitem>
+
+              <listitem>
+                <para>Click the disk icon to save.</para>
+              </listitem>
+            </orderedlist>
+          </para>
+
+          <para>
+            <graphic fileref="images/CM-img06.jpg" vendor="configmgrSS" />
+          </para>
+
+          <para>
+            <xi:include href="XMLGeneration/xml/dropzone.xsd.mod.xml"
+                        xpointer="xpointer(//*[@id='dz.t1'])"
+                        xmlns:xi="http://www.w3.org/2001/XInclude" />
+          </para>
+
+          <!-- /*Attribute table***replace with dynamically generated table*/
+          <informaltable>
+            <tgroup cols="2">
+              <tbody>
+                <row>
+                  <entry>description</entry>
+
+                  <entry>a description</entry>
+                </row>
+
+                <row>
+                  <entry>directory</entry>
+
+                  <entry>The directory for the drop zone</entry>
+                </row>
+
+                <row>
+                  <entry>name</entry>
+
+                  <entry>The name for the drop zone</entry>
+                </row>
+
+                <row>
+                  <entry>umask</entry>
+
+                  <entry>umask</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable>
+
+
+-->
+        </sect3>
+
+        <sect3 id="DropZoneServerList">
+          <title>Drop Zone Server List</title>
+
+          <para>This tab allows you to add any servers that you wish to
+          configure as a part of the selected drop zone.</para>
+
+          <para>To add a server to the current drop zone: <orderedlist>
+              <listitem>
+                <para>Select the <emphasis role="bold">Drop Zone</emphasis> to
+                configure from the Navigator panel on the left side.</para>
+              </listitem>
+
+              <listitem>
+                <para>Select the <emphasis role="bold">Server List</emphasis>
+                tab, right-click on the Server Address field and choose
+                <emphasis role="bold">Add</emphasis>.</para>
+              </listitem>
+
+              <listitem>
+                <para>Enter the hostname or IP address of the server.</para>
+              </listitem>
+
+              <listitem>
+                <para>Click the disk icon to save.</para>
+              </listitem>
+            </orderedlist></para>
+        </sect3>
+
+        <sect3 id="DropZoneNotes">
+          <title>Drop Zone Notes</title>
 
           <para>This tab allows you to add any notes pertinent to the
           component's configuration. This can be useful to keep a record of

ecl/eclcc/eclcc.cpp

@@ -397,6 +397,7 @@ protected:
     bool optGenerateHeader = false;
     bool optShowPaths = false;
     bool optNoSourcePath = false;
+    bool optFastSyntax = false;
     mutable bool daliConnected = false;
     mutable bool disconnectReported = false;
     int argc;
@@ -1139,6 +1140,8 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     {
         //Minimize the scope of the parse context to reduce lifetime of cached items.
         HqlParseContext parseCtx(instance.dataServer, this, instance.archive);
+        if (optFastSyntax)
+            parseCtx.setFastSyntax();
         if (optMaxErrors > 0)
             parseCtx.maxErrors = optMaxErrors;
         parseCtx.unsuppressImmediateSyntaxErrors = optUnsuppressImmediateSyntaxErrors;
@@ -2183,6 +2186,9 @@ int EclCC::parseCommandLineOptions(int argc, const char* argv[])
         {
             debugOptions.append(tempArg);
         }
+        else if (iter.matchFlag(optFastSyntax, "--fastsyntax"))
+        {
+        }
         else if (iter.matchFlag(tempBool, "-g") || iter.matchFlag(tempBool, "--debug"))
         {
             if (tempBool)

ecl/hql/hqlattr.cpp

@@ -2085,9 +2085,9 @@ static bool increasesRowSize(IHqlExpression * newRecord, IHqlExpression * oldRec
 // Does this operation decrease the size of the row?  False negatives preferred.
 bool reducesRowSize(IHqlExpression * expr)
 {
-    OwnedHqlExpr newRecord = getSerializedForm(expr->queryRecord(), diskAtom);
-    OwnedHqlExpr oldRecord = getSerializedForm(expr->queryChild(0)->queryRecord(), diskAtom);
-    return increasesRowSize(oldRecord, newRecord, false);
+    OwnedHqlExpr projectedRecord = getSerializedForm(expr->queryRecord(), diskAtom);
+    OwnedHqlExpr previousRecord = getSerializedForm(expr->queryChild(0)->queryRecord(), diskAtom);
+    return increasesRowSize(previousRecord, projectedRecord, false);
 }
 
 // Does this operation increase the size of the row?  False negatives preferred.

ecl/hql/hqlexpr.cpp

@@ -9095,7 +9095,7 @@ IHqlExpression * createDelayedReference(node_operator op, IHqlExpression * modul
     else
         ret.setown(createValue(op, attr->getType(), args));
 
-    if (attr->isScope())
+    if (attr->isScope() || attr->getOperator() == no_enum)
     {
         if (attr->getOperator() != no_funcdef)
             ret.setown(createDelayedScope(ret.getClear()));
@@ -9298,6 +9298,26 @@ no_purevirtual - a member that has no associated definition.
 
 */
 
+bool definesMacro(IHqlExpression * expr)
+{
+    IHqlScope * scope = expr->queryScope();
+    HqlExprArray syms;
+    scope->getSymbols(syms);
+    ForEachItemIn(i, syms)
+    {
+        //HACK - needs more work
+        IHqlExpression & symbol = syms.item(i);
+        if (symbol.isMacro())
+            return true;
+        if (symbol.isScope())
+        {
+            //MORE: Need to ensure that this symbol is defined, and walk the children
+        }
+
+    }
+    return false;
+}
+
 bool canBeDelayed(IHqlExpression * expr)
 {
     switch (expr->getOperator())
@@ -9309,7 +9329,18 @@ bool canBeDelayed(IHqlExpression * expr)
     case no_record:             // LEFT has problems because queryRecord() is the unbound funcdef
     case no_keyindex:           // BUILD() and other functions a reliant on this being expanded out
     case no_newkeyindex:
+    case no_forwardscope:
+    case no_type:
         return false;
+    case no_remotescope:
+    case no_scope:
+    case no_privatescope:
+    case no_virtualscope:
+    {
+        if (definesMacro(expr))
+            return false;
+        return true;
+    }
     case no_funcdef:
         return canBeDelayed(expr->queryChild(0));
     }
@@ -10045,13 +10076,21 @@ CHqlDelayedScope::CHqlDelayedScope(HqlExprArray &_ownedOperands)
  : CHqlExpressionWithTables(no_delayedscope), type(nullptr)
 {
     setOperands(_ownedOperands); // after type is initialized
-    type = queryChild(0)->queryType();
+    IHqlExpression * arg0 = queryChild(0);
+    if (arg0->getOperator() == no_delayedselect)
+        arg0 = arg0->queryChild(2);
+    type = arg0->queryType();
 
     ITypeInfo * scopeType = type;
     if (scopeType->getTypeCode() == type_function)
         scopeType = scopeType->queryChildType();
 
     typeScope = ::queryScope(scopeType);
+    if (!typeScope)
+    {
+        typeScope = arg0->queryScope();
+        type = typeScope->queryExpression()->queryType();
+    }
     assertex(typeScope);
 
     if (!hasAttribute(_virtualSeq_Atom))
@@ -10600,6 +10639,9 @@ IHqlExpression * CHqlDelayedScopeCall::lookupSymbol(IIdAtom * searchName, unsign
     if (lookupFlags & LSFignoreBase)
         return match.getClear();
 
+    if (!canBeVirtual(match) && match->isFullyBound())
+        return match.getClear();
+
     return createDelayedReference(no_delayedselect, this, match, lookupFlags, ctx);
 }
 
@@ -12208,6 +12250,16 @@ void expandDelayedFunctionCalls(IErrorReceiver * errors, HqlExprArray & exprs)
     replaceArray(exprs, target);
 }
 
+IHqlExpression * expandDelayedFunctionCalls(IErrorReceiver * errors, IHqlExpression * expr)
+{
+    HqlExprArray functionCache;
+    CallExpansionContext ctx;
+    ctx.functionCache = &functionCache;
+    ctx.errors = errors;
+    CallExpandTransformer binder(ctx);
+
+    return binder.transform(expr);
+}
 
 extern IHqlExpression * createReboundFunction(IHqlExpression *func, HqlExprArray &actuals)
 {

ecl/hql/hqlexpr.hpp

@@ -33,14 +33,6 @@
 //It nearly works - but there are still some examples which have problems - primarily libraries, old parameter syntax, enums and other issues.
 //There may also problems with queryRecord() which needs to really be replaced with recordof(x), especially if "templates" are delayed expanded.
 //To work properly it may require many of the transformations in hqlgram2.cpp to be moved to after the expansion.  (E.g., BUILD)
-//#define DELAY_CALL_EXPANSION
-
-#ifdef DELAY_CALL_EXPANSION
-#define DEFAULT_EXPAND_CALL false
-#else
-#define DEFAULT_EXPAND_CALL true
-#endif
-
 
 #define GATHER_HIDDEN_SELECTORS
 
@@ -345,7 +337,7 @@ enum node_operator : unsigned short {
         no_datasetfromdictionary,
         no_delayedscope,
         no_assertconcrete,
-        no_unboundselect,
+        no_unboundselect,               // A symbol selected from a module derived from a parameter
         no_id,
         no_orderedactionlist,
         no_dataset_from_transform,
@@ -870,7 +862,7 @@ public:
     HqlParseContext(IEclRepository * _eclRepository, ICodegenContextCallback *_codegenCtx, IPropertyTree * _archive)
     : archive(_archive), eclRepository(_eclRepository), codegenCtx(_codegenCtx)
     {
-        expandCallsWhenBound = DEFAULT_EXPAND_CALL;
+        expandCallsWhenBound = true;
         ignoreUnknownImport = false;
         ignoreSignatures = false;
         _clear(metaState);
@@ -895,6 +887,7 @@ public:
     inline IEclRepository * queryRepository() const { return eclRepository; }
     inline bool isAborting() const { return aborting; }
     inline void setAborting() { aborting = true; }
+    inline void setFastSyntax() { expandCallsWhenBound = false; }
 
 public:
     Linked<IPropertyTree> archive;
@@ -1285,6 +1278,7 @@ extern HQL_API IHqlExpression * createTypeTransfer(IHqlExpression * expr, ITypeI
 extern HQL_API IHqlExpression * cloneFieldMangleName(IHqlExpression * expr);
 extern HQL_API IHqlExpression * expandOutOfLineFunctionCall(IHqlExpression * expr);
 extern HQL_API void expandDelayedFunctionCalls(IErrorReceiver * errors, HqlExprArray & exprs);
+extern HQL_API IHqlExpression * expandDelayedFunctionCalls(IErrorReceiver * errors, IHqlExpression * expr);
 
 extern HQL_API IHqlExpression *createQuoted(const char * name, ITypeInfo *type);
 extern HQL_API IHqlExpression *createVariable(const char * name, ITypeInfo *type);

ecl/hql/hqlfold.cpp

@@ -3767,6 +3767,12 @@ IHqlExpression * foldConstantOperator(IHqlExpression * expr, unsigned foldOption
             OwnedHqlExpr folded = expandOutOfLineFunctionCall(expr);
             if ((folded != expr) && folded->isConstant())
                 return folded.getClear();
+            if (foldOptions & HFOforcefold)
+            {
+                OwnedHqlExpr transformed = expandDelayedFunctionCalls(nullptr, expr);
+                if (expr != transformed)
+                    return transformed.getClear();
+            }
             break;
         }
     case no_trim:

ecl/hql/hqlgram.y

@@ -4256,14 +4256,18 @@ recordDef
     | RECORDOF '(' goodObject ')'
                         {
                             OwnedHqlExpr ds = $3.getExpr();
-                            IHqlExpression * record = queryOriginalRecord(ds);
+                            LinkedHqlExpr record = queryOriginalRecord(ds);
                             if (!record)
                             {
                                 parser->reportError(ERR_EXPECTED, $3, "The argument does not have a associated record");
-                                record = queryNullRecord();
+                                record.set(queryNullRecord());
                             }
                             else if (ds->isFunction() && !record->isFullyBound())
-                                parser->reportError(ERR_EXPECTED, $1, "RECORDOF(function-definition), result record depends on the function parameters");
+                            {
+                                record.setown(getUnadornedRecordOrField(record));
+                                if (!record->isFullyBound())
+                                    parser->reportError(ERR_EXPECTED, $1, "RECORDOF(function-definition), result record depends on the function parameters");
+                            }
 
                             $$.setExpr(LINK(record));
                             $$.setPosition($1);
@@ -9282,7 +9286,7 @@ simpleDataSet
                                 {
                                     if (compareDs)
                                     {
-                                        if (isGrouped(cur) != isGrouped(compareDs))
+                                        if (parser->lookupCtx.queryParseContext().expandCallsWhenBound && (isGrouped(cur) != isGrouped(compareDs)))
                                             parser->reportError(ERR_GROUPING_MISMATCH, $1, "Branches of the condition have different grouping");
                                         OwnedHqlExpr mapped = parser->checkEnsureRecordsMatch(compareDs, cur, $5.pos, false);
                                         if (mapped != cur)

ecl/hql/hqlgram2.cpp

@@ -2616,6 +2616,9 @@ void HqlGram::addField(const attribute &errpos, IIdAtom * name, ITypeInfo *_type
             fieldType.set(defaultIntegralType);
         }
         break;
+    case type_enumerated:
+        fieldType.set(fieldType->queryChildType());
+        break;
     case type_decimal:
         if (fieldType->getSize() == UNKNOWN_LENGTH)
         {
@@ -3520,9 +3523,10 @@ IHqlExpression *HqlGram::lookupSymbol(IIdAtom * searchName, const attribute& err
         if (dotScope) 
         {
             IHqlExpression *ret = NULL;
-            if (dotScope->getOperator() == no_enum)
+            IHqlScope * scope = dotScope->queryScope();
+            if (scope)
             {
-                ret = dotScope->queryScope()->lookupSymbol(searchName, LSFrequired, lookupCtx);
+                ret = scope->lookupSymbol(searchName, LSFrequired, lookupCtx);
             }
             else
             {
@@ -6244,7 +6248,13 @@ void HqlGram::checkFormals(IIdAtom * name, HqlExprArray& parms, HqlExprArray& de
         // check default value
         if (isMacro)
         {
-            IHqlExpression* def = &defaults.item(idx);
+            LinkedHqlExpr def = &defaults.item(idx);
+            if (!def->isConstant() && !lookupCtx.queryParseContext().expandCallsWhenBound)
+            {
+                OwnedHqlExpr expanded = expandDelayedFunctionCalls(nullptr, def);
+                defaults.replace(*LINK(expanded), idx);
+                def.set(expanded);
+            }
             
             if ((def->getOperator() != no_omitted) && !def->isConstant()) 
             {
@@ -6326,27 +6336,13 @@ IHqlExpression *HqlGram::bindParameters(const attribute & errpos, IHqlExpression
             {
                 if (requireLateBind(function, actuals))
                 {
-                    IHqlExpression * ret = NULL;
-                    if (!expandCallsWhenBound)
-                    {
-                        HqlExprArray args;
-                        args.append(*LINK(body));
-                        unwindChildren(args, function, 1);
-                        OwnedHqlExpr newFunction = createFunctionDefinition(function->queryId(), args);
-                        OwnedHqlExpr boundExpr = createBoundFunction(this, newFunction, actuals, lookupCtx.functionCache, expandCallsWhenBound);
+                    //A function with virtual dataset parameters - must be expanded now
+                    const bool expandCallsWhenBound = true;
+                    OwnedHqlExpr boundExpr = createBoundFunction(this, function, actuals, lookupCtx.functionCache, expandCallsWhenBound);
                         
-                        // get rid of the wrapper
-                        //assertex(boundExpr->getOperator()==no_template_context);
-                        ret = LINK(boundExpr);//->queryChild(0));
-                    }
-                    else
-                    {
-                        OwnedHqlExpr boundExpr = createBoundFunction(this, function, actuals, lookupCtx.functionCache, expandCallsWhenBound);
-                        
-                        // get rid of the wrapper
-                        assertex(boundExpr->getOperator()==no_template_context);
-                        ret = LINK(boundExpr->queryChild(0));
-                    }
+                    // get rid of the wrapper
+                    assertex(boundExpr->getOperator()==no_template_context);
+                    IHqlExpression * ret = LINK(boundExpr->queryChild(0));
 
                     IHqlExpression * formals = function->queryChild(1);
                     // bind fields
@@ -7194,8 +7190,8 @@ void HqlGram::checkOutputRecord(attribute & errpos, bool outerLevel)
     OwnedHqlExpr record = errpos.getExpr();
     bool allConstant = true;
     errpos.setExpr(checkOutputRecord(record, errpos, allConstant, outerLevel));
-    if (allConstant && (record->getOperator() != no_null) && (record->numChildren() != 0))
-        reportWarning(CategoryUnusual, WRN_OUTPUT_ALL_CONSTANT,errpos.pos,"All values for OUTPUT are constant - is this the intention?");
+    if (allConstant && (record->getOperator() != no_null) && (record->numChildren() != 0) && record->isFullyBound())
+        reportWarning(CategoryUnusual, WRN_OUTPUT_ALL_CONSTANT, errpos.pos, "All values for OUTPUT are constant - is this the intention?");
 }
 
 void HqlGram::checkSoapRecord(attribute & errpos)
@@ -8834,7 +8830,7 @@ void HqlGram::ensureMapToRecordsMatch(OwnedHqlExpr & defaultExpr, HqlExprArray &
         }
     }
 
-    if (groupingDiffers)
+    if (lookupCtx.queryParseContext().expandCallsWhenBound && groupingDiffers)
         reportError(ERR_GROUPING_MISMATCH, errpos, "Branches of the condition have different grouping");
 }
 
@@ -8951,7 +8947,7 @@ void HqlGram::checkMergeInputSorted(attribute &atr, bool isLocal)
     
     if (isGrouped(expr) && appearsToBeSorted(expr, false, false))
         reportWarning(CategoryUnexpected, WRN_MERGE_NOT_SORTED, atr.pos, "Input to MERGE is only sorted with the group");
-    else
+    else if (expr->isFullyBound())
         reportWarning(CategoryUnexpected, WRN_MERGE_NOT_SORTED, atr.pos, "Input to MERGE doesn't appear to be sorted");
 }
 
@@ -9145,7 +9141,7 @@ IHqlExpression * HqlGram::processIfProduction(attribute & condAttr, attribute &
     if (left->queryRecord() && falseAttr)
         right.setown(checkEnsureRecordsMatch(left, right, falseAttr->pos, false));
 
-    if (isGrouped(left) != isGrouped(right))
+    if (lookupCtx.queryParseContext().expandCallsWhenBound && (isGrouped(left) != isGrouped(right)))
         reportError(ERR_GROUPING_MISMATCH, trueAttr, "Branches of the condition have different grouping");
 
     if (cond->isConstant())

ecl/hql/hqlthql.cpp

@@ -23,6 +23,7 @@
 #include "hqlutil.hpp"
 #include "hqlmeta.hpp"
 #include "workunit.hpp"
+#include "hqlerrors.hpp"
 
 //The following constants can be uncommented to increase the level of detail which is added to the processed graphs
 //E.g. generated when -fl used in hqltest
@@ -448,7 +449,16 @@ StringBuffer &HqltHql::callEclFunction(StringBuffer &s, IHqlExpression * expr, b
 {
     assertex(expr->isNamedSymbol());
     IHqlExpression * funcdef = expr->queryFunctionDefinition();
-    assertex(funcdef->getOperator() == no_funcdef || funcdef->getOperator() == no_internalselect);
+    switch (funcdef->getOperator())
+    {
+    case no_funcdef:
+    case no_internalselect:
+    case no_delayedselect:
+        break;
+    default:
+        throw makeStringExceptionV(ERR_INTERNALEXCEPTION, "Internal: Unexpected function definition %s", getOpString(funcdef->getOperator()));
+    }
+
     IHqlExpression * formals = queryFunctionParameters(funcdef);
 
     s.append('(');

ecl/hql/hqltrans.cpp

@@ -319,7 +319,7 @@ unsigned activityHidesSelectorGetNumNonHidden(IHqlExpression * expr, IHqlExpress
         case childdataset_dataset:
         case childdataset_datasetleft:
         case childdataset_top_left_right:
-            if (expr->queryChild(0)->queryBody() == selector)
+            if (expr->queryChild(0)->queryNormalizedSelector() == selector)
                 return 1;
             break;
         }
@@ -1060,6 +1060,7 @@ QuickExpressionReplacer::QuickExpressionReplacer()
 
 void QuickExpressionReplacer::setMapping(IHqlExpression * oldValue, IHqlExpression * newValue)
 {
+    assertex(oldValue);
     for (;;)
     {
         oldValue->setTransformExtra(newValue);
@@ -1970,7 +1971,7 @@ IHqlExpression * NewHqlTransformer::transformCall(IHqlExpression * expr)
     bool same = transformChildren(body, args);
     if (same && (oldFuncdef == newFuncdef))
         return LINK(expr);
-    OwnedHqlExpr newCall = createBoundFunction(NULL, newFuncdef, args, NULL, DEFAULT_EXPAND_CALL);
+    OwnedHqlExpr newCall = createBoundFunction(NULL, newFuncdef, args, NULL, false);
     return expr->cloneAllAnnotations(newCall);
 }
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10830,6 +10830,7 @@ void HqlCppTranslator::addSchemaField(IHqlExpression *field, MemoryBuffer &schem
     switch(schemaType->getTypeCode())
     {
     case type_alien:
+    case type_enumerated:
         schemaType.set(schemaType->queryChildType());
         break;
     case type_bitfield:

ecl/hqlcpp/hqliproj.cpp

@@ -2911,7 +2911,7 @@ IHqlExpression * ImplicitProjectTransformer::createTransformed(IHqlExpression *
                         OwnedHqlExpr newBody = replaceChild(body, 0, newBodyCode);
                         OwnedHqlExpr newFuncdef = replaceChild(funcdef, 0, newBody);
                         unwindChildren(args, expr, 0);
-                        transformed.setown(createBoundFunction(NULL, newFuncdef, args, NULL, DEFAULT_EXPAND_CALL));
+                        transformed.setown(createBoundFunction(NULL, newFuncdef, args, NULL, false));
 
                         logChange("Auto project embed", expr, complexExtra->outputFields);
                         return transformed.getClear();

ecl/hqlcpp/hqlttcpp.cpp

@@ -11595,6 +11595,15 @@ void HqlTreeNormalizer::analyseExpr(IHqlExpression * expr)
             analyseChildren(body);
             break;
         }
+    case no_if:
+        if (expr->isDataset())
+        {
+            IHqlExpression * left = expr->queryChild(1);
+            IHqlExpression * right = expr->queryChild(2);
+            if (right && (isGrouped(left) != isGrouped(right)))
+                translator.reportError(expr, ERR_GROUPING_MISMATCH, "Branches of the condition have different grouping");
+            break;
+        }
     }
 
     Parent::analyseExpr(body);

ecl/regress/ifdataset_err.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+namesTable1 := dataset('x',namesRecord,FLAT);
+namesTable2 := group(namesTable1, surname);
+p := IF(count(namesTable1) > 100, namesTable1, namesTable2);
+output(p);

esp/files/scripts/configmgr/configmgr.js

@@ -751,6 +751,18 @@ function initItemForRoxiePorts(item) {
   item.aclName = "";
   item.aclName_extra = "";
   item.aclName_ctrlType = 0;
+  item.protocol = "";
+  item.protocol_extra = "";
+  item.protocol_ctrlType = 0;
+  item.passphrase = "";
+  item.passphrase_extra = "";
+  item.passphrase_ctrlType = 0;
+  item.certificateFileName = "";
+  item.certificateFileName_extra = "";
+  item.certificateFileName_ctrlType = 0;
+  item.privateKeyFileName = "";
+  item.privateKeyFileName_extra = "";
+  item.privateKeyFileName_ctrlType = 0;
   item.process = "";
   item.process_extra = "";
   item.process_ctrlType = 0;

esp/smc/SMCLib/TpWrapper.cpp

@@ -1751,6 +1751,7 @@ void CTpWrapper::appendTpDropZone(double clientVersion, IConstEnvironment* const
             ipAddr.ipset(server.str());
             ipAddr.getIpText(networkAddress);
             machine->setNetaddress(networkAddress.str());
+            machine->setConfigNetaddress(server.str());
         }
         if (directory.length() > 0)
         {

esp/src/eclwatch/DFUQueryWidget.js

@@ -216,7 +216,7 @@ define([
             var list = this.arrayToList(selection, "Name");
             if (confirm(this.i18n.DeleteSelectedFiles + "\n" + list)) {
                 var context = this;
-                WsDfu.DFUArrayAction(selection, this.i18n.Delete).then(function (response) {
+                WsDfu.DFUArrayAction(selection, "Delete").then(function (response) {
                     context.refreshGrid(true);
                 });
             }

esp/src/eclwatch/FileHistoryWidget.js

@@ -0,0 +1,129 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+define([
+    "dojo/_base/declare",
+    "dojo/_base/lang",
+    "dojo/i18n",
+    "dojo/i18n!./nls/hpcc",
+    "dojo/_base/array",
+
+    "dijit/registry",
+
+    "dijit/form/Button",
+    "dijit/ToolbarSeparator",
+
+    "hpcc/GridDetailsWidget",
+    "hpcc/WsDfu",
+    "hpcc/ESPUtil"
+], function (declare, lang, i18n, nlsHPCC, arrayUtil,
+                registry, Button, ToolbarSeparator,
+                GridDetailsWidget, WsDfu, ESPUtil) {
+    return declare("FileHistoryWidget", [GridDetailsWidget], {
+        i18n: nlsHPCC,
+        gridTitle: nlsHPCC.History,
+        idProperty: "Name",
+
+        init: function (params) {
+            if (this.inherited(arguments))
+                return;
+            this._refreshActionState();
+            this.refreshGrid();
+            this.initTab();
+        },
+
+        _onRefresh: function (event) {
+            this.refreshGrid();
+        },
+
+        createGrid: function (domID) {
+            var context = this;
+
+            this.openButton = registry.byId(this.id + "Open");
+
+            this.eraseHistory = new Button({
+                label: context.i18n.EraseHistory,
+                onClick: function () { context._onErase(); }
+            }).placeAt(this.openButton, "after");
+
+            var tmpSplitter = new ToolbarSeparator().placeAt(this.eraseHistory.domNode, "before");
+
+            dojo.destroy(this.openButton);
+
+            var retVal = new declare([ESPUtil.Grid(true, true)])({
+                store: this.store,
+                columns: {
+                    Name: {label: this.i18n.Name, width:70, sortable: false},
+                    IP: {label: this.i18n.IP, width: 30, sortable: false},
+                    Operation: {label: this.i18n.Operation, width: 30, sortable: false},
+                    Owner: {label: this.i18n.Owner, width: 30, sortable: false},
+                    Path: {label: this.i18n.Path, width: 70, sortable: false},
+                    Timestamp: {label: this.i18n.TimeStamp, width: 30, sortable: false},
+                    Workunit: {label: this.i18n.Workunit, width: 30, sortable: false}
+                }
+            }, domID);
+
+            return retVal;
+        },
+
+        _onErase: function (event) {
+            var context = this;
+            if (confirm(this.i18n.EraseHistoryQ + "\n" + this.params.Name + "?")) {
+                WsDfu.EraseHistory({
+                    request: {
+                        Name: context.params.Name
+                    }
+                }).then(function (response) {
+                    if (response) {
+                        context.refreshGrid();
+                    }
+                });
+            }
+        },
+
+        refreshGrid: function () {
+            var context = this;
+
+            WsDfu.ListHistory({
+                request: {
+                    Name: context.params.Name
+                }
+            }).then(function (response) {
+                var results = [];
+                var newRows = [];
+                if (lang.exists("ListHistoryResponse.History.Origin", response)) {
+                    results = response.ListHistoryResponse.History.Origin;
+                }
+
+                if (results.length) {
+                    arrayUtil.forEach(results, function (row, idx) {
+                       newRows.push({
+                            Name: row.Name,
+                            IP: row.IP,
+                            Operation: row.Operation,
+                            Owner: row.Owner,
+                            Path: row.Path,
+                            Timestamp: row.Timestamp,
+                            Workunit: row.Workunit
+                        });
+                    });
+                }
+
+                context.store.setData(newRows);
+                context.grid.set("query", {});
+            });
+        }
+    });
+});

esp/src/eclwatch/FileSpray.js

@@ -203,8 +203,9 @@ define([
                 arrayUtil.forEach(parent.TpMachines.TpMachine, function (item, idx) {
                     children.push({
                          calculatedID: item.Netaddress,
-                         displayName: item.Netaddress,
+                         displayName: item.ConfigNetaddress !== item.Netaddress ? item.ConfigNetaddress + " [" + item.Netaddress + "]" : item.ConfigNetaddress,
                          NetAddress: item.Netaddress,
+                         ConfigNetaddress: item.ConfigNetaddress,
                          type: "machine",
                          isMachine: true,
                          isDir: false,

esp/src/eclwatch/LFDetailsWidget.js

@@ -47,6 +47,7 @@ define([
     "hpcc/ESPDFUWorkunit",
     "hpcc/FileBelongsToWidget",
     "hpcc/FileSpray",
+    "hpcc/FileHistoryWidget",
 
     "dojo/text!../templates/LFDetailsWidget.html",
 
@@ -60,7 +61,7 @@ define([
 
 ], function (exports, declare, lang, i18n, nlsHPCC, arrayUtil, dom, domAttr, domClass, domForm, query,
                 BorderContainer, TabContainer, ContentPane, Toolbar, TooltipDialog, Form, SimpleTextarea, TextBox, Button, DropDownButton, TitlePane, registry,
-                _TabContainerWidget, DelayLoadWidget, TargetSelectWidget, TargetComboBoxWidget, ESPLogicalFile, ESPDFUWorkunit, FileBelongsToWidget, FileSpray,
+                _TabContainerWidget, DelayLoadWidget, TargetSelectWidget, TargetComboBoxWidget, ESPLogicalFile, ESPDFUWorkunit, FileBelongsToWidget, FileSpray, FileHistoryWidget,
                 template) {
     exports.fixCircularDependency = declare("LFDetailsWidget", [_TabContainerWidget], {
         templateString: template,
@@ -83,6 +84,7 @@ define([
         workunitWidget: null,
         dfuWorkunitWidget: null,
         fileBelongsTo: null,
+        fileHistoryWidget: null,
 
         logicalFile: null,
         prevState: "",
@@ -102,6 +104,7 @@ define([
             this.queriesWidget = registry.byId(this.id + "_Queries");
             this.workunitWidget = registry.byId(this.id + "_Workunit");
             this.dfuWorkunitWidget = registry.byId(this.id + "_DFUWorkunit");
+            this.fileHistoryWidget = registry.byId(this.id + "_FileHistory");
             this.copyTargetSelect = registry.byId(this.id + "CopyTargetSelect");
             this.desprayTargetSelect = registry.byId(this.id + "DesprayTargetSelect");
             this.desprayTooltiopDialog = registry.byId(this.id + "DesprayTooltipDialog");
@@ -299,6 +302,10 @@ define([
                         NodeGroup: this.logicalFile.NodeGroup,
                         Name: this.logicalFile.Name
                     });
+                 } else if (currSel.id == this.fileHistoryWidget.id) {
+                    this.fileHistoryWidget.init({
+                        Name: this.logicalFile.Name
+                    });
                  } else {
                     currSel.init(currSel.params);
                 }
@@ -411,4 +418,4 @@ define([
             this.setDisabled(this.id + "ReplicateDropDown", !this.logicalFile.CanReplicateFlag || this.logicalFile.ReplicateFlag === false);
         }
     });
-});
+});

esp/src/eclwatch/WsDfu.js

@@ -171,6 +171,14 @@ define([
             return ESPRequest.send("WsDfu", "DFUSpace", params);
         },
 
+        ListHistory: function (params) {
+            return ESPRequest.send("WsDfu", "ListHistory", params);
+        },
+
+        EraseHistory: function (params) {
+            return ESPRequest.send("WsDfu", "EraseHistory", params);
+        },
+
         DFUInfo: function (params) {
             return ESPRequest.send("WsDfu", "DFUInfo", params).then(function (response) {
                 if (lang.exists("Exceptions.Exception", response)) {
@@ -220,4 +228,3 @@ define([
 
     return self;
 });
-

esp/src/eclwatch/nls/es/hpcc.js

@@ -175,6 +175,8 @@ define(
     EnglishQ: "Ingles?",
     EnterAPercentage: "Entre un porcentaje",
     EnterAPercentageOrMB: "Entre un porcentaje o MB",
+    EraseHistory: "Borrar Historia",
+    EraseHistoryQ: "Borrar historia de archivo:",
     Error: "Error",
     Errorparsingserverresult: "Error analizando los resultados del servidor",
     Errors: "Errores",
@@ -396,6 +398,7 @@ define(
     OpenSafeMode: "Abrir (modo seguro)",
     OpenSource: "Código Abierto",
     OpenTreeMode: "Abrir (vista en arbol)",
+    Operation: "Operación",
     Operations: "Operaciones",
     Options: "Opciones",
     OriginalFile: "Archivo Original",
@@ -632,6 +635,7 @@ define(
     ThorNetworkAddress: "Dirección de red de Thor",
     ThorProcess: "Proceso de Thor",
     Time: "Tiempo",
+    TimeStamp: "Marca de hora",
     Timers: "Cronómetros",
     TimeSeconds: "Tiempo (Segundos)",
     TimeStarted: "Tiempo empezado",
@@ -667,6 +671,7 @@ define(
     title_HPCCPlatformOps: "ECL Watch - Operaciones",
     title_HPCCPlatformRoxie: "ECL Watch - Roxie",
     title_HPCCPlatformServicesPlugin: "ECL Watch - Plugins",
+    title_History: "Historia",
     title_Inputs: "Entradas",
     title_LFDetails: "Detalles de archivos lógicos",
     title_LibrariesUsed: "Librerias Usadas",
@@ -784,4 +789,4 @@ define(
     ZoomAll: "Aumentar todo",
     ZoomWidth: "Aumentar al ancho"
 })
-);
+);

esp/src/eclwatch/nls/hpcc.js

@@ -181,6 +181,8 @@ define({root:
     EnglishQ: "English?",
     EnterAPercentage: "Enter a percentage",
     EnterAPercentageOrMB: "Enter A Percentage or MB",
+    EraseHistory: "Erase History",
+    EraseHistoryQ: "Erase history for:",
     Error: "Error",
     Errorparsingserverresult: "Error parsing server result",
     Errors: "Error(s)",
@@ -401,6 +403,7 @@ define({root:
     OpenLegacyMode: "Open (legacy)",
     OpenNativeMode: "Open (native)",
     OpenSource: "Open Source",
+    Operation: "Operation",
     Operations: "Operations",
     Options: "Options",
     OriginalFile: "Original File",
@@ -641,6 +644,7 @@ define({root:
     ThorMasterAddress: "Thor Master Address",
     ThorProcess: "Thor Process",
     Time: "Time",
+    TimeStamp: "Time Stamp",
     TimeSeconds: "Time (Seconds)",
     TimeStarted: "Time Started",
     TimeStopped: "Time Stopped",
@@ -676,6 +680,7 @@ define({root:
     title_HPCCPlatformOps: "ECL Watch - Operations",
     title_HPCCPlatformRoxie: "ECL Watch - Roxie",
     title_HPCCPlatformServicesPlugin: "ECL Watch - Plugins",
+    title_History: "History",
     title_Inputs: "Inputs",
     title_Log: "Log File",
     title_LFDetails: "Logical File Details",

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -210,6 +210,8 @@
             </div>
             <div id="${id}_DFUWorkunit" title="${i18n.Workunit}" data-dojo-props="delayWidget: 'DFUWUDetailsWidget'" data-dojo-type="DelayLoadWidget">
             </div>
+            <div id="${id}_FileHistory" title="${i18n.History}" data-dojo-props="delayWidget: 'FileHistoryWidget'" data-dojo-type="DelayLoadWidget">
+            </div>
         </div>
     </div>
 </div>

initfiles/componentfiles/configxml/RoxieTopology.xsl

@@ -160,7 +160,7 @@
             @XSL_PLUGIN_DEFINITION@
             <xsl:for-each select="RoxieFarmProcess">
                 <xsl:element name="RoxieFarmProcess">
-                    <xsl:copy-of select="@*[name()!='name' and name()!='level']"/>
+                    <xsl:copy-of select="@*[name()!='level']"/>
                 </xsl:element>
             </xsl:for-each>
             <xsl:for-each select="RoxieServerProcess">

initfiles/componentfiles/configxml/daliplugin.xsd

@@ -24,6 +24,13 @@
             <xs:documentation>Describes a Dali Server Plugin</xs:documentation>
         </xs:annotation>
         <xs:complexType>
+        <!--Doc Id code-->
+          <xs:annotation>
+				    <xs:appinfo>
+				      <docid>daplug.t</docid>
+				    </xs:appinfo>
+          </xs:annotation>
+        <!--End Doc Id code-->
             <xs:attribute name="build" type="buildType" use="required">
               <xs:annotation>
                 <xs:appinfo>

initfiles/componentfiles/configxml/dropzone.xsd.in

@@ -27,6 +27,12 @@
             </xs:appinfo>
         </xs:annotation>
         <xs:complexType>
+           <!--DOC-Autobuild-code-->
+              <xs:annotation>
+		<xs:appinfo>
+		   <docid>dz.t1</docid>
+	    	</xs:appinfo>
+              </xs:annotation>	    	
             <xs:sequence>
                 <xs:element name="Notes" maxOccurs="unbounded">
                     <xs:annotation>
@@ -99,12 +105,13 @@
                             <xs:element name="ServerListEntry" type="xs:string" minOccurs="0" maxOccurs="1"/>
                         </xs:sequence>
                     </xs:complexType>
-                    <xs:attribute name="server" type="string" use="required">
+                    <xs:attribute name="server" type="serverListType" use="required">
                         <xs:annotation>
                             <xs:appinfo>
                                 <title>Server Address</title>
                                 <tooltip>Server Address to associate with this DropZone</tooltip>
                                 <width>120</width>
+                                <autogenforwizard>1</autogenforwizard>
                             </xs:appinfo>
                         </xs:annotation>
                     </xs:attribute>

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -77,6 +77,38 @@
                 </xs:appinfo>
               </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="protocol" type="xs:string" use="optional" default="native">
+              <xs:annotation>
+                <xs:appinfo>
+                  <tooltip>Protocol to use</tooltip>
+                  <title>Protocol</title>
+                </xs:appinfo>
+              </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="passphrase" type="xs:string" use="optional">
+              <xs:annotation>
+                <xs:appinfo>
+                  <tooltip>Pass phrase for cert</tooltip>
+                  <title>PassPhrase</title>
+                </xs:appinfo>
+              </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="certificateFileName" type="xs:string" use="optional">
+              <xs:annotation>
+                <xs:appinfo>
+                  <tooltip>Path to certificate filename</tooltip>
+                  <title>CertFile</title>
+                </xs:appinfo>
+              </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="privateKeyFileName" type="xs:string" use="optional">
+              <xs:annotation>
+                <xs:appinfo>
+                  <tooltip>Path to private key filename</tooltip>
+                  <title>CertFile</title>
+                </xs:appinfo>
+              </xs:annotation>
+            </xs:attribute>
           </xs:complexType>
         </xs:element>
 

plugins/CMakeLists.txt

@@ -38,4 +38,4 @@ add_subdirectory (redis)
 add_subdirectory (kafka)
 add_subdirectory (exampleplugin)
 add_subdirectory (couchbase)
-add_subdirectory(sqs)
+add_subdirectory (sqs)

plugins/sqs/CMakeLists.txt

@@ -6,62 +6,67 @@ project(sqs)
 
 
 if(SQS)
-   ADD_PLUGIN(sqs)
-   if(MAKE_SQS)
-    # Locate the AWS SDK for C++ package.
-    # Requires that you build with:
-    # -Daws-sdk-cpp_DIR=/path/to/sdk_build
-    # or export/set:
-    # CMAKE_PREFIX_PATH=/path/to/sdk_build
-    find_package(aws-sdk-cpp)
-    if(!aws-sdk-cpp_FOUND) 
-         message(FATAL_ERROR
-          "AWS SDK not found and required that you build with: 
-		-Daws-sdk-cpp_DIR=/path/to/sdk_build 
-           or export/set:
-		CMAKE_PREFIX_PATH=/path/to/sdk_buid")
-    endif()	
+    ADD_PLUGIN(sqs)
+    set(AWS_SDK_CPP_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp)
+    include(ExternalProject)
+    ExternalProject_Add(
+        aws-sdk-cpp
+        SOURCE_DIR ${AWS_SDK_CPP_SOURCE_DIR}
+        BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-cpp
+        BUILD_COMMAND $(MAKE) LDFLAGS=-Wl,-rpath-link,${LIB_PATH} aws-cpp-sdk-sqs
+        INSTALL_COMMAND "")
+    add_library(aws-cpp-sdk-core SHARED IMPORTED)
+    add_library(aws-cpp-sdk-sqs SHARED IMPORTED)
+    set_property(TARGET aws-cpp-sdk-core
+        PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-cpp/aws-cpp-sdk-core/libaws-cpp-sdk-core.so)
+    set_property(TARGET aws-cpp-sdk-sqs
+        PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-cpp/aws-cpp-sdk-sqs/libaws-cpp-sdk-sqs.so)
+    add_dependencies(aws-cpp-sdk-core aws-sdk-cpp)
+    add_dependencies(aws-cpp-sdk-sqs aws-sdk-cpp)
 
-    # Link to the SDK shared libraries.
     add_definitions(-DUSE_IMPORT_EXPORT)
-#    add_subdirectory(doc)
-    include_directories (BEFORE ${INCLUDE_DIR})
-    set(
-            SRCS
-            sqs.h
-            sqs.cpp)
 
-   include_directories(
-            ./../../system/include
-            ./../../rtl/eclrtl
-            ./../../rtl/include
-            ./../../common/deftype
-            ./../../system/jlib
-            ${PROJECT_BINARY_DIR}/include
-            ${CMAKE_BINARY_DIR} )
+    set(SRCS
+        sqs.h
+        sqs.cpp)
 
-       add_definitions(-D_USRDLL -DECL_SQS_EXPORTS)
-        HPCC_ADD_LIBRARY(sqs SHARED ${SRCS})
+    include_directories(
+        ./../../system/include
+        ./../../rtl/eclrtl
+        ./../../rtl/include
+        ./../../common/deftype
+        ./../../system/jlib
+        ${AWS_SDK_CPP_SOURCE_DIR}/aws-cpp-sdk-core/include
+        ${AWS_SDK_CPP_SOURCE_DIR}/aws-cpp-sdk-sqs/include
+        ${CMAKE_BINARY_DIR})
 
-        if(${CMAKE_VERSION} VERSION_LESS "2.8.9")
-            message(WARNING "Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-        elseif(NOT APPLE)
-            set_target_properties(sqs PROPERTIES NO_SONAME 1)
-        endif()
+    add_definitions(-D_USRDLL -DECL_SQS_EXPORTS)
+    HPCC_ADD_LIBRARY(sqs SHARED ${SRCS})
 
-        install(
-            TARGETS sqs
-            DESTINATION plugins)
+    if(NOT APPLE)
+        set_target_properties(sqs PROPERTIES NO_SONAME 1)
+    endif()
 
+    install(
+        TARGETS sqs
+        DESTINATION plugins)
 
-        target_link_libraries(
-            sqs 
-            aws-cpp-sdk-sqs
-            eclrtl
-            jlib
-            ${ZLIB_LIBRARIES})
+    install(CODE "set(ENV{LD_LIBRARY_PATH} \"\$ENV{LD_LIBRARY_PATH}:${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/aws-sdk-cpp/aws-cpp-sdk-core:${PROJECT_BINARY_DIR}/aws-sdk-cpp/aws-cpp-sdk-sqs\")")
+    install(FILES
+        ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp/LICENSE.txt
+        DESTINATION "."
+        RENAME aws-sdk-cpp-LICENSE.txt)
+    install(PROGRAMS
+        ${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-cpp/aws-cpp-sdk-core/libaws-cpp-sdk-core.so
+        ${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-cpp/aws-cpp-sdk-sqs/libaws-cpp-sdk-sqs.so
+        DESTINATION lib)
 
-   endif()
+    target_link_libraries(
+        sqs
+        aws-cpp-sdk-sqs
+        eclrtl
+        jlib
+        ${ZLIB_LIBRARIES})
 endif()
 
 if(PLATFORM OR CLIENTTOOLS_ONLY)

plugins/sqs/aws-sdk-cpp

@@ -0,0 +1 @@
+Subproject commit f2f7c74390f02d2682b1fc36a7b077aab22ed37a

plugins/sqs/cmake_install.cmake

@@ -1,44 +0,0 @@
-# Install script for directory: /hpcc/HPCC-Platform/plugins/sqs
-
-# Set the install prefix
-if(NOT DEFINED CMAKE_INSTALL_PREFIX)
-  set(CMAKE_INSTALL_PREFIX "/usr/local")
-endif()
-string(REGEX REPLACE "/$" "" CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}")
-
-# Set the install configuration name.
-if(NOT DEFINED CMAKE_INSTALL_CONFIG_NAME)
-  if(BUILD_TYPE)
-    string(REGEX REPLACE "^[^A-Za-z0-9_]+" ""
-           CMAKE_INSTALL_CONFIG_NAME "${BUILD_TYPE}")
-  else()
-    set(CMAKE_INSTALL_CONFIG_NAME "")
-  endif()
-  message(STATUS "Install configuration: \"${CMAKE_INSTALL_CONFIG_NAME}\"")
-endif()
-
-# Set the component getting installed.
-if(NOT CMAKE_INSTALL_COMPONENT)
-  if(COMPONENT)
-    message(STATUS "Install component: \"${COMPONENT}\"")
-    set(CMAKE_INSTALL_COMPONENT "${COMPONENT}")
-  else()
-    set(CMAKE_INSTALL_COMPONENT)
-  endif()
-endif()
-
-# Install shared libraries without execute permission?
-if(NOT DEFINED CMAKE_INSTALL_SO_NO_EXE)
-  set(CMAKE_INSTALL_SO_NO_EXE "1")
-endif()
-
-if(CMAKE_INSTALL_COMPONENT)
-  set(CMAKE_INSTALL_MANIFEST "install_manifest_${CMAKE_INSTALL_COMPONENT}.txt")
-else()
-  set(CMAKE_INSTALL_MANIFEST "install_manifest.txt")
-endif()
-
-string(REPLACE ";" "\n" CMAKE_INSTALL_MANIFEST_CONTENT
-       "${CMAKE_INSTALL_MANIFEST_FILES}")
-file(WRITE "/hpcc/HPCC-Platform/plugins/sqs/${CMAKE_INSTALL_MANIFEST}"
-     "${CMAKE_INSTALL_MANIFEST_CONTENT}")

plugins/sqs/sqs.pc

@@ -1,11 +0,0 @@
-prefix=/usr
-exec_prefix=${prefix}
-libdir=${exec_prefix}/lib
-includedir=${prefix}/include
-
-Name: SQSHPCC
-Description: This is a plugin for HPCC allowing to dialog with Amazon's SQS 
-Version: 0.0.1
-Requires:
-Libs: -L${libdir} -lssl -lcrypto -lz
-Cflags: -I${includedir}

roxie/ccd/CMakeLists.txt

@@ -88,6 +88,7 @@ include_directories (
          ${CMAKE_BINARY_DIR}/oss
          ${HPCC_SOURCE_DIR}/dali/ft
          ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/system/security/securesocket
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DCCD_EXPORTS -DSTARTQUERY_EXPORTS )

roxie/ccd/ccdactivities.cpp

@@ -81,7 +81,7 @@ extern void putStatsValue(IPropertyTree *node, const char *statName, const char
         IPropertyTree *att = node->queryPropTree(xpath.str());
         if (!att)
         {
-            att = node->addPropTree("att", createPTree());
+            att = node->addPropTree("att");
             att->setProp("@name", statName);
         }
         att->setProp("@type", statType);
@@ -245,15 +245,13 @@ protected:
             const char *id = strchr(levelx, '[');
             if (!id)
             {
-                child = createPTree(levelx);
-                parent->addPropTree(levelx, child);
+                child = parent->addPropTree(levelx);
             }
             else
             {
                 StringBuffer elem;
                 elem.append(id-levelx, levelx);
-                child = createPTree(elem);
-                parent->addPropTree(elem, child);
+                child = parent->addPropTree(elem);
                 for (;;)
                 {
                     StringBuffer attr, val;

roxie/ccd/ccdcontext.cpp

@@ -2021,7 +2021,7 @@ protected:
             {
                 CriticalBlock b(contextCrit);
                 if (!persists)
-                    persists = createPTree();
+                    persists = createPTree(ipt_fast);
                 return *persists;
             }
         case ResultSequenceOnce:
@@ -2034,14 +2034,14 @@ protected:
             {
                 CriticalBlock b(contextCrit);
                 if (!temporaries)
-                    temporaries = createPTree();
+                    temporaries = createPTree(ipt_fast);
                 return *temporaries;
             }
         default:
             {
                 CriticalBlock b(contextCrit);
                 if (!rereadResults)
-                    rereadResults = createPTree();
+                    rereadResults = createPTree(ipt_fast);
                 return *rereadResults;
             }
         }
@@ -2690,8 +2690,8 @@ protected:
         if (workUnit->getXmlParams(wuParams, false).length())
         {
             // Merge in params from WU. Ones on command line take precedence though...
-            Owned<IPropertyTree> wuParamTree = createPTreeFromXMLString(wuParams.str(), ipt_caseInsensitive);
-            Owned<IPropertyTreeIterator> params = wuParamTree ->getElements("*");
+            Owned<IPropertyTree> wuParamTree = createPTreeFromXMLString(wuParams.str(), ipt_caseInsensitive|ipt_fast);
+            Owned<IPropertyTreeIterator> params = wuParamTree->getElements("*");
             ForEach(*params)
             {
                 IPropertyTree &param = params->query();
@@ -2738,7 +2738,7 @@ public:
         init();
         rowManager->setMemoryLimit(options.memoryLimit);
         workflow.setown(_factory->createWorkflowMachine(workUnit, true, logctx));
-        context.setown(createPTree(ipt_caseInsensitive));
+        context.setown(createPTree(ipt_caseInsensitive|ipt_fast));
     }
 
     CRoxieServerContext(IConstWorkUnit *_workUnit, const IQueryFactory *_factory, const ContextLogger &_logctx)
@@ -2748,7 +2748,7 @@ public:
         workUnit.set(_workUnit);
         rowManager->setMemoryLimit(options.memoryLimit);
         workflow.setown(_factory->createWorkflowMachine(workUnit, false, logctx));
-        context.setown(createPTree(ipt_caseInsensitive));
+        context.setown(createPTree(ipt_caseInsensitive|ipt_fast));
 
         //MORE: Use various debug settings to override settings:
         rowManager->setActivityTracking(workUnit->getDebugValueBool("traceRoxiePeakMemory", false));
@@ -3207,7 +3207,7 @@ public:
         else
         {
             if (!val)
-                val = ctx.addPropTree(name, createPTree());
+                val = ctx.addPropTree(name, createPTree(ipt_fast));
             val->setProp("@format", "deserialized");
             val->setPropInt("@id", resultStore.addResult(count, data, meta));
         }
@@ -3318,7 +3318,7 @@ public:
     virtual void setResultXml(const char *name, unsigned sequence, const char *xml)
     {
         CriticalBlock b(contextCrit);
-        useContext(sequence).setPropTree(name, createPTreeFromXMLString(xml, ipt_caseInsensitive));
+        useContext(sequence).setPropTree(name, createPTreeFromXMLString(xml, ipt_caseInsensitive|ipt_fast));
     }
 
     virtual void setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val)

roxie/ccd/ccddali.cpp

@@ -241,11 +241,11 @@ private:
 
     static void initCache()
     {
-        IPropertyTree *tree = createPTree("Roxie");
-        tree->addPropTree("QuerySets", createPTree("QuerySets"));
-        tree->addPropTree("PackageSets", createPTree("PackageSets"));
-        tree->addPropTree("PackageMaps", createPTree("PackageMaps"));
-        tree->addPropTree("Files", createPTree("Files"));
+        IPropertyTree *tree = createPTree("Roxie", ipt_lowmem);
+        tree->addPropTree("QuerySets");
+        tree->addPropTree("PackageSets");
+        tree->addPropTree("PackageMaps");
+        tree->addPropTree("Files");
         cache.setown(tree);
     }
 
@@ -256,7 +256,7 @@ private:
             StringBuffer cacheFileName(queryDirectory);
             cacheFileName.append(roxieStateName);
             if (checkFileExists(cacheFileName))
-                cache.setown(createPTreeFromXMLFile(cacheFileName));
+                cache.setown(createPTreeFromXMLFile(cacheFileName, ipt_lowmem));
             else
                 initCache();
         }
@@ -293,7 +293,7 @@ private:
                 if (conn)
                 {
                     Owned <IPropertyTree> daliTree = conn->getRoot();
-                    localTree.setown(createPTreeFromIPT(daliTree));
+                    localTree.setown(createPTreeFromIPT(daliTree, ipt_lowmem));
                 }
                 writeCache(xpath, path, localTree);
                 return localTree.getClear();
@@ -442,7 +442,7 @@ public:
         Owned<IPropertyTree> ret = loadDaliTree("QuerySets/QuerySet", id);
         if (!ret)
         {
-            ret.setown(createPTree("QuerySet"));
+            ret.setown(createPTree("QuerySet", ipt_lowmem));
             ret->setProp("@id", id);
         }
         return ret.getClear();
@@ -453,7 +453,7 @@ public:
         Owned<IPropertyTree> ret = loadDaliTree("PackageSets", NULL);
         if (!ret)
         {
-            ret.setown(createPTree("PackageSets"));
+            ret.setown(createPTree("PackageSets", ipt_lowmem));
         }
         return ret.getClear();
     }
@@ -476,7 +476,7 @@ public:
         Owned<IPropertyTree> ret = loadDaliTree("PackageMaps/PackageMap", id);
         if (!ret)
         {
-            ret.setown(createPTree("PackageMap"));
+            ret.setown(createPTree("PackageMap", ipt_lowmem));
             ret->setProp("@id", id);
         }
         return ret.getClear();

roxie/ccd/ccddebug.cpp

@@ -410,7 +410,7 @@ public:
                                 IPropertyTree *att = edge.queryPropTree("att[@name=\"_roxieStarted\"]");
                                 if (!att)
                                 {
-                                    att = edge.addPropTree("att", createPTree());
+                                    att = edge.addPropTree("att");
                                     att->setProp("@name", "_roxieStarted");
                                 }
                                 else
@@ -440,7 +440,7 @@ public:
                                     IPropertyTree *att = edge.queryPropTree("att[@name=\"_roxieStarted\"]");
                                     if (!att)
                                     {
-                                        att = edge.addPropTree("att", createPTree());
+                                        att = edge.addPropTree("att");
                                         att->setProp("@name", "_roxieStarted");
                                     }
                                     else
@@ -1586,7 +1586,7 @@ public:
                 DebugRequestLookupActivityByEdgeId request(proxyId, edgeId);
                 CommonXmlWriter reply(0);
                 sendProxyRequest(&reply, request);
-                Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str());
+                Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str(), ipt_fast);
                 if (response)
                 {
                     memsize_t proxyId = (memsize_t) response->getPropInt64("@proxyId", 0);
@@ -1679,7 +1679,7 @@ public:
         reply.outputBeginNested("Counts", true);
         sendProxyRequest(&reply, request);
         reply.outputEndNested("Counts"); // strange way to do it...
-        Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str());
+        Owned<IPropertyTree> response = createPTreeFromXMLString(reply.str(), ipt_fast);
         if (response)
         {
             Owned<IPropertyTreeIterator> edges = response->getElements("edge");

roxie/ccd/ccdfile.cpp

@@ -389,22 +389,22 @@ public:
     virtual const char *queryFilename() { return logical->queryFilename(); }
     virtual bool isAlive() const { return CInterface::isAlive(); }
 
-    virtual IMemoryMappedFile *queryMappedFile()
+    virtual IMemoryMappedFile *getMappedFile() override
     {
         CriticalBlock b(crit);
         if (mmapped)
-            return mmapped;
+            return mmapped.getLink();
         if (!remote)
         {
             mmapped.setown(logical->openMemoryMapped());
-            return mmapped;
+            return mmapped.getLink();
         }
-        return NULL;
+        return nullptr;
     }
 
-    virtual IFileIO *queryFileIO()
+    virtual IFileIO *getFileIO() override
     {
-        return this;
+        return LINK(this);
     }
 
 
@@ -1795,7 +1795,7 @@ public:
                     addFile(sub.queryLogicalName(), fDesc.getClear(), remoteFDesc.getClear());
                 }
                 // We have to clone the properties since we don't want to keep the superfile locked
-                properties.setown(createPTreeFromIPT(&dFile->queryAttributes()));
+                properties.setown(createPTreeFromIPT(&dFile->queryAttributes(), ipt_lowmem));
                 if (!isDynamic && !lockSuperFiles)
                 {
                     notifier.setown(daliHelper->getSuperFileSubscription(lfn, this));
@@ -2205,7 +2205,7 @@ public:
         assertex(file->exists());
         offset_t size = file->size();
         Owned<IFileDescriptor> fdesc = createFileDescriptor();
-        Owned<IPropertyTree> pp = createPTree("Part");
+        Owned<IPropertyTree> pp = createPTree("Part", ipt_lowmem);
         pp->setPropInt64("@size",size);
         pp->setPropBool("@local", true);
         fdesc->setPart(0, queryMyNode(), localFileName, pp);
@@ -2372,7 +2372,7 @@ public:
                 bool propertiesPresent;
                 serverData.read(propertiesPresent);
                 if (propertiesPresent)
-                    properties.setown(createPTree(serverData));
+                    properties.setown(createPTree(serverData, ipt_lowmem));
             }
             else
                 throw MakeStringException(ROXIE_CALLBACK_ERROR, "Failed to get response from server for dynamic file callback");

roxie/ccd/ccdfile.hpp

@@ -48,7 +48,7 @@ interface ILazyFileIO : extends IFileIO
     virtual void close() = 0;
     virtual void setCopying(bool copying) = 0;
     virtual bool isCopying() const = 0;
-    virtual IMemoryMappedFile *queryMappedFile() = 0;
+    virtual IMemoryMappedFile *getMappedFile() = 0;
 
     virtual void setCache(const IRoxieFileCache *) = 0;
     virtual void removeCache(const IRoxieFileCache *) = 0;

roxie/ccd/ccdlistener.cpp

@@ -173,7 +173,7 @@ public:
         StringBuffer lockQuery;
         lockQuery.appendf("<control:childlock thisEndpoint='%d' parent='%d'/>", activeIdxes.item(idx), myEndpoint);
         doChildQuery(idx, lockQuery.str(), lockReply);
-        Owned<IPropertyTree> lockResult = createPTreeFromXMLString(lockReply.str(), ipt_caseInsensitive);
+        Owned<IPropertyTree> lockResult = createPTreeFromXMLString(lockReply.str(), ipt_caseInsensitive|ipt_fast);
         int lockCount = lockResult->getPropInt("Lock", 0);
         if (lockCount)
         {
@@ -340,7 +340,7 @@ public:
 
     void doLockChild(const char *queryText, StringBuffer &reply)
     {
-        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText);
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText,ipt_fast);
         doLockChild(xml, queryText, reply);
     }
 
@@ -395,7 +395,7 @@ public:
 
     void doControlQuery(SocketEndpoint &ep, const char *queryText, StringBuffer &reply)
     {
-        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText); // control queries are case sensitive
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(queryText,ipt_fast); // control queries are case sensitive
         doControlQuery(ep, xml, queryText, reply);
     }
 
@@ -413,7 +413,7 @@ public:
             mergeType=CascadeMergeQueries;
         Owned<IPropertyTree> mergedReply;
         if (mergeType!=CascadeMergeNone)
-            mergedReply.setown(createPTree("Endpoint"));
+            mergedReply.setown(createPTree("Endpoint",ipt_fast));
 
         class casyncfor: public CAsyncFor
         {
@@ -444,7 +444,7 @@ public:
                 {
                     StringBuffer childReply;
                     parent->doChildQuery(i, queryText, childReply);
-                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(childReply);
+                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(childReply,ipt_fast);
                     if (!replyXML)
                     {
                         StringBuffer err;
@@ -490,7 +490,7 @@ public:
                 CriticalBlock cb(crit);
                 if (mergedReply)
                 {
-                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(myReply);
+                    Owned<IPropertyTree> replyXML = createPTreeFromXMLString(myReply,ipt_fast);
                     if (mergeType == CascadeMergeStats)
                         mergeStats(mergedReply, replyXML);
                     else if (mergeType == CascadeMergeQueries)

roxie/ccd/ccdmain.cpp

@@ -282,9 +282,7 @@ void getAccessList(const char *aclName, const IPropertyTree *topology, IProperty
     xpath.append("ACL[@name='").append(aclName).append("']");
     if (aclInfo->queryPropTree(xpath))
         throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - recursive ACL definition of %s", aclName);
-    Owned<IPropertyTree> X = createPTree("ACL");
-    X->setProp("@name", aclName);
-    aclInfo->addPropTree("ACL", X.getClear());
+    aclInfo->addPropTree("ACL")->setProp("@name", aclName);
 
     Owned<IPropertyTree> acl = topology->getPropTree(xpath.str());
     if (!acl)
@@ -314,10 +312,9 @@ void addSlaveChannel(unsigned channel, unsigned level)
     xpath.appendf("RoxieSlaveProcess[@channel=\"%d\"]", channel);
     if (ccdChannels->hasProp(xpath.str()))
         throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - channel %d repeated", channel);
-    IPropertyTree *ci = createPTree("RoxieSlaveProcess");
+    IPropertyTree *ci = ccdChannels->addPropTree("RoxieSlaveProcess");
     ci->setPropInt("@channel", channel);
     ci->setPropInt("@subChannel", numSlaves[channel]);
-    ccdChannels->addPropTree("RoxieSlaveProcess", ci);
 }
 
 void addChannel(unsigned nodeNumber, unsigned channel, unsigned level)
@@ -499,7 +496,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
     Thread::setDefaultStackSize(0x10000);   // NB under windows requires linker setting (/stack:)
 #endif
     srand( (unsigned)time( NULL ) );
-    ccdChannels = createPTree("Channels");
+    ccdChannels = createPTree("Channels", ipt_lowmem);
 
     char currentDirectory[_MAX_DIR];
     if (!getcwd(currentDirectory, sizeof(currentDirectory)))
@@ -524,7 +521,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         if (checkFileExists(topologyFile.str()))
         {
             DBGLOG("Loading topology file %s", topologyFile.str());
-            topology = createPTreeFromXMLFile(topologyFile.str());
+            topology = createPTreeFromXMLFile(topologyFile.str(), ipt_lowmem);
             saveTopology();
         }
         else
@@ -539,6 +536,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 " <RoxieFarmProcess/>"
                 " <RoxieServerProcess netAddress='.'/>"
                 "</RoxieTopology>"
+                , ipt_lowmem
                 );
             int port = globals->getPropInt("--port", 9876);
             topology->setPropInt("RoxieFarmProcess/@port", port);
@@ -1043,6 +1041,8 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         if (!localSlave)
             openMulticastSocket();
 
+        StringBuffer certFileName;
+        StringBuffer keyFileName;
         setDaliServixSocketCaching(true);  // enable daliservix caching
         loadPlugins();
         createDelayedReleaser();
@@ -1118,6 +1118,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 unsigned numThreads = roxieFarm.getPropInt("@numThreads", numServerThreads);
                 unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
                 unsigned requestArrayThreads = roxieFarm.getPropInt("@requestArrayThreads", 5);
+                // NOTE: farmer name [@name=] is not copied into topology
                 const IpAddress &ip = getNodeAddress(myNodeIndex);
                 if (!roxiePort)
                 {
@@ -1129,10 +1130,44 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 if (port)
                 {
                     const char *protocol = roxieFarm.queryProp("@protocol");
+                    const char *passPhrase = nullptr;
+                    const char *certFile = nullptr;
+                    const char *keyFile = nullptr;
+                    if (protocol && streq(protocol, "ssl"))
+                    {
+#ifdef _USE_OPENSSL
+                        certFile = roxieFarm.queryProp("@certificateFileName");
+                        if (!certFile)
+                            throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFileName tag", port);
+                        if (isAbsolutePath(certFile))
+                            certFileName.append(certFile);
+                        else
+                            certFileName.append(codeDirectory.str()).append(certFile);
+                        if (!checkFileExists(certFileName.str()))
+                            throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFile (%s)", port, certFileName.str());
+
+                        keyFile =  roxieFarm.queryProp("@privateKeyFileName");
+                        if (!keyFile)
+                            throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFileName tag", port);
+                        if (isAbsolutePath(keyFile))
+                            keyFileName.append(keyFile);
+                        else
+                            keyFileName.append(codeDirectory.str()).append(keyFile);
+                        if (!checkFileExists(keyFileName.str()))
+                            throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFile (%s)", port, keyFileName.str());
+
+                        passPhrase = roxieFarm.queryProp("@passphrase");
+                        if (isEmptyString(passPhrase))
+                            passPhrase = nullptr;
+#else
+                        WARNLOG("Skipping Roxie SSL Farm Listener on port %d : OpenSSL disabled in build", port);
+                        continue;
+#endif
+                    }
                     const char *soname =  roxieFarm.queryProp("@so");
                     const char *config  = roxieFarm.queryProp("@config");
                     Owned<IHpccProtocolPlugin> protocolPlugin = ensureProtocolPlugin(*protocolCtx, soname);
-                    roxieServer.setown(protocolPlugin->createListener(protocol ? protocol : "native", createRoxieProtocolMsgSink(ip, port, numThreads, suspended), port, listenQueue, config));
+                    roxieServer.setown(protocolPlugin->createListener(protocol ? protocol : "native", createRoxieProtocolMsgSink(ip, port, numThreads, suspended), port, listenQueue, config, certFileName.str(), keyFileName.str(), passPhrase));
                 }
                 else
                     roxieServer.setown(createRoxieWorkUnitListener(numThreads, suspended));
@@ -1141,7 +1176,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 const char *aclName = roxieFarm.queryProp("@aclName");
                 if (aclName && *aclName)
                 {
-                    Owned<IPropertyTree> aclInfo = createPTree("AccessInfo");
+                    Owned<IPropertyTree> aclInfo = createPTree("AccessInfo", ipt_lowmem);
                     getAccessList(aclName, topology, aclInfo);
                     Owned<IPropertyTreeIterator> accesses = aclInfo->getElements("Access");
                     ForEach(*accesses)
@@ -1166,7 +1201,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 roxieServer->start();
             }
             writeSentinelFile(sentinelFile);
-            DBGLOG("Waiting for queries");
+            DBGLOG("Waiting for queries LPT=%u APT=%u", queryNumLocalTrees(), queryNumAtomTrees());
             if (pingInterval)
                 startPingTimer();
             LocalIAbortHandler abortHandler(waiter);

roxie/ccd/ccdprotocol.cpp

@@ -22,10 +22,11 @@
 #include "roxie.hpp"
 #include "roxiehelper.hpp"
 #include "ccdprotocol.hpp"
+#include "securesocket.hpp"
 
 //================================================================================================================================
 
-IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue);
+IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *certFile, const char *keyFile, const char *passPhrase);
 
 class CHpccProtocolPlugin : implements IHpccProtocolPlugin, public CInterface
 {
@@ -54,9 +55,9 @@ public:
         trapTooManyActiveQueries = ctx.ctxGetPropBool("@trapTooManyActiveQueries", true);
         numRequestArrayThreads = ctx.ctxGetPropInt("@requestArrayThreads", 5);
     }
-    IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config)
+    IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const char *certFile=nullptr, const char *keyFile=nullptr, const char *passPhrase=nullptr)
     {
-        return createProtocolListener(protocol, sink, port, listenQueue);
+        return createProtocolListener(protocol, sink, port, listenQueue, certFile, keyFile, passPhrase);
     }
 public:
     StringArray targetNames;
@@ -218,14 +219,23 @@ class ProtocolSocketListener : public ProtocolListener
     unsigned listenQueue;
     Owned<ISocket> socket;
     SocketEndpoint ep;
+    const char *protocol;
+    const char *certFile;
+    const char *keyFile;
+    const char *passPhrase;
+    Owned<ISecureSocketContext> secureContext;
 
 public:
-    ProtocolSocketListener(IHpccProtocolMsgSink *_sink, unsigned _port, unsigned _listenQueue)
+    ProtocolSocketListener(IHpccProtocolMsgSink *_sink, unsigned _port, unsigned _listenQueue, const char *_protocol, const char *_certFile, const char *_keyFile, const char *_passPhrase)
       : ProtocolListener(_sink)
     {
         port = _port;
         listenQueue = _listenQueue;
         ep.set(port, queryHostIP());
+        protocol = _protocol;
+        certFile = _certFile;
+        keyFile = _keyFile;
+        passPhrase = _passPhrase;
     }
 
     IHpccProtocolMsgSink *queryMsgSink()
@@ -269,11 +279,48 @@ public:
         started.signal();
         while (running)
         {
-            ISocket *client = socket->accept(true);
+            Owned<ISocket> client = socket->accept(true);
+            Owned<ISecureSocket> ssock;
             if (client)
             {
+                if (protocol && streq(protocol, "ssl"))
+                {
+#ifdef _USE_OPENSSL
+                    try
+                    {
+                        if (!secureContext)
+                            secureContext.setown(createSecureSocketContextEx(certFile, keyFile, passPhrase, ServerSocket));
+                        ssock.setown(secureContext->createSecureSocket(client.getClear()));
+                        int status = ssock->secure_accept();
+                        if (status < 0)
+                        {
+                            // secure_accept may also DBGLOG() errors ...
+                            WARNLOG("ProtocolSocketListener failure to establish secure connection");
+                            continue;
+                        }
+                    }
+                    catch (IException *E)
+                    {
+                        StringBuffer s;
+                        E->errorMessage(s);
+                        WARNLOG("%s", s.str());
+                        E->Release();
+                        continue;
+                    }
+                    catch (...)
+                    {
+                        StringBuffer s;
+                        WARNLOG("ProtocolSocketListener failure to establish secure connection");
+                        continue;
+                    }
+                    client.setown(ssock.getClear());
+#else
+                    WARNLOG("ProtocolSocketListener failure to establish secure connection: OpenSSL disabled in build");
+                    continue;
+#endif
+                }
                 client->set_linger(-1);
-                pool->start(client);
+                pool->start(client.getClear());
             }
         }
         DBGLOG("ProtocolSocketListener closed query socket");
@@ -816,7 +863,7 @@ public:
             return;
         if (fmt==MarkupFmt_JSON)
         {
-            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(content);
+            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(content, ipt_fast);
             if (name && *name)
                 appendXMLOpenTag(xml, name);
             toXML(convertPT, xml, 0, 0);
@@ -1068,7 +1115,7 @@ public:
         StringBuffer json;
         if (mlFmt==MarkupFmt_XML)
         {
-            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(StringBuffer("<Control>").append(content).append("</Control>"));
+            Owned<IPropertyTree> convertPT = createPTreeFromXMLString(StringBuffer("<Control>").append(content).append("</Control>"), ipt_fast);
             toJSON(convertPT, json, 0, 0);
             content = json.str();
         }
@@ -1179,7 +1226,7 @@ public:
         StringBuffer xml;
         if (mlFmt==MarkupFmt_JSON)
         {
-            Owned<IPropertyTree> convertPT = createPTreeFromJSONString(content);
+            Owned<IPropertyTree> convertPT = createPTreeFromJSONString(content, ipt_fast);
             toXML(convertPT, xml, 0, 0);
             content = xml.str();
         }
@@ -1723,7 +1770,7 @@ readAnother:
                     client->setHttpMode(queryName, false, httpHelper);
 
                 bool aclupdate = strieq(queryName, "aclupdate"); //ugly
-                byte iptFlags = aclupdate ? ipt_caseInsensitive : 0;
+                byte iptFlags = aclupdate ? ipt_caseInsensitive|ipt_fast : ipt_fast;
 
                 createQueryPTree(queryPT, httpHelper, rawText, iptFlags, (PTreeReaderOptions)(ptr_ignoreWhiteSpace|ptr_ignoreNameSpaces), queryName);
 
@@ -1768,7 +1815,7 @@ readAnother:
                 readFlags |= (whitespace == WhiteSpaceHandling::Strip ? ptr_ignoreWhiteSpace : ptr_none);
                 try
                 {
-                    createQueryPTree(queryPT, httpHelper, rawText.str(), ipt_caseInsensitive, (PTreeReaderOptions)readFlags, queryName);
+                    createQueryPTree(queryPT, httpHelper, rawText.str(), ipt_caseInsensitive|ipt_fast, (PTreeReaderOptions)readFlags, queryName);
                 }
                 catch (IException *E)
                 {
@@ -1840,7 +1887,7 @@ readAnother:
                             Owned<IPropertyTreeIterator> reqIter = queryPT->getElements(reqIterString.str());
                             ForEach(*reqIter)
                             {
-                                IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive);
+                                IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive|ipt_fast);
                                 Owned<IPropertyTreeIterator> iter = reqIter->query().getElements("*");
                                 ForEach(*iter)
                                 {
@@ -1851,7 +1898,7 @@ readAnother:
                         }
                         else
                         {
-                            IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive);
+                            IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive|ipt_fast);
                             Owned<IPropertyTreeIterator> iter = queryPT->getElements("*");
                             ForEach(*iter)
                             {
@@ -2022,11 +2069,11 @@ void ProtocolSocketListener::runOnce(const char *query)
     p->runOnce(query);
 }
 
-IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue)
+IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *certFile=nullptr, const char *keyFile=nullptr, const char *passPhrase=nullptr)
 {
     if (traceLevel)
         DBGLOG("Creating Roxie socket listener, protocol %s, pool size %d, listen queue %d%s", protocol, sink->getPoolSize(), listenQueue, sink->getIsSuspended() ? " SUSPENDED":"");
-    return new ProtocolSocketListener(sink, port, listenQueue);
+    return new ProtocolSocketListener(sink, port, listenQueue, protocol, certFile, keyFile, passPhrase);
 }
 
 extern IHpccProtocolPlugin *loadHpccProtocolPlugin(IHpccProtocolPluginContext *ctx, IActiveQueryLimiterFactory *_limiterFactory)

roxie/ccd/ccdquery.cpp

@@ -170,9 +170,9 @@ IPropertyTree * addXrefInfo(IPropertyTree &reply, const char *section, const cha
     VStringBuffer xpath("%s[@name='%s']", section, name);
     if (!reply.hasProp(xpath))
     {
-        IPropertyTree *info = createPTree(section, 0);
+        IPropertyTree *info = reply.addPropTree(section);
         info->setProp("@name", name);
-        return reply.addPropTree(section, info);
+        return info;
     }
     return NULL;
 }
@@ -236,7 +236,7 @@ public:
         CriticalBlock b(onceCrit);
         if (!onceContext)
         {
-            onceContext.setown(createPTree());
+            onceContext.setown(createPTree(ipt_lowmem));
             onceResultStore.setown(createDeserializedResultStore());
             Owned <IRoxieServerContext> ctx = createOnceServerContext(factory, logctx);
             onceManager.set(&ctx->queryRowManager());
@@ -1297,7 +1297,7 @@ public:
 
     void getGraphStats(StringBuffer &reply, const IPropertyTree &thisGraph) const
     {
-        Owned<IPropertyTree> graph = createPTreeFromIPT(&thisGraph);
+        Owned<IPropertyTree> graph = createPTreeFromIPT(&thisGraph, ipt_lowmem);
         Owned<IPropertyTreeIterator> edges = graph->getElements(".//edge");
         ForEach(*edges)
         {
@@ -1325,7 +1325,7 @@ public:
     virtual IPropertyTree* cloneQueryXGMML() const
     {
         assertex(dll && dll->queryWorkUnit());
-        Owned<IPropertyTree> tree = createPTree("Query");
+        Owned<IPropertyTree> tree = createPTree("Query", ipt_lowmem);
         Owned<IConstWUGraphIterator> graphs = &dll->queryWorkUnit()->getGraphs(GraphTypeActivities);
         SCMStringBuffer graphNameStr;
         ForEach(*graphs)
@@ -1333,12 +1333,9 @@ public:
             graphs->query().getName(graphNameStr);
             const char *graphName = graphNameStr.s.str();
             Owned<IPropertyTree> graphXgmml = graphs->query().getXGMMLTree(false);
-            IPropertyTree *newGraph = createPTree();
+            IPropertyTree *newGraph = tree->addPropTree("Graph");
             newGraph->setProp("@id", graphName);
-            IPropertyTree *newXGMML = createPTree();
-            newXGMML->addPropTree("graph", graphXgmml.getLink());
-            newGraph->addPropTree("xgmml", newXGMML);
-            tree->addPropTree("Graph", newGraph);
+            newGraph->addPropTree("xgmml")->addPropTree("graph", graphXgmml.getLink());
         }
         return tree.getClear();
     }
@@ -1383,7 +1380,7 @@ public:
     }
     virtual void getQueryInfo(StringBuffer &reply, bool full, IArrayOf<IQueryFactory> *slaveQueries, const IRoxieContextLogger &logctx) const
     {
-        Owned<IPropertyTree> xref = createPTree("Query", 0);
+        Owned<IPropertyTree> xref = createPTree("Query", ipt_fast);
         xref->setProp("@id", id);
         if (suspended())
         {

roxie/ccd/ccdserver.cpp

@@ -3198,7 +3198,7 @@ void throwRemoteException(IMessageUnpackCursor *extra)
     {
         char *xml = (char *) extra->getNext(*rowlen);
         ReleaseRoxieRow(rowlen);
-        Owned<IPropertyTree> p = createPTreeFromXMLString(xml);
+        Owned<IPropertyTree> p = createPTreeFromXMLString(xml, ipt_fast);
         ReleaseRoxieRow(xml);
         unsigned code = p->getPropInt("Code", 0);
         const char *msg = p->queryProp("Message");
@@ -12022,7 +12022,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
                 throw MakeStringException(0, "Invalid name %s in user metadata for index %s (not legal XML element name)", name.str(), fname.get());
             }
             if(!metadata)
-                metadata.setown(createPTree("metadata"));
+                metadata.setown(createPTree("metadata", ipt_fast));
             metadata->setProp(name.str(), value.str());
         }
     }
@@ -12030,7 +12030,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
     void buildLayoutMetadata(Owned<IPropertyTree> & metadata)
     {
         if(!metadata)
-            metadata.setown(createPTree("metadata"));
+            metadata.setown(createPTree("metadata", ipt_fast));
         metadata->setProp("_record_ECL", helper.queryRecordECL());
 
         void * layoutMetaBuff;
@@ -12183,7 +12183,7 @@ public:
         //properties of the first file part.
         Owned<IPropertyTree> attrs;
         if(clusterHandler)
-            attrs.setown(createPTree("Part"));  // clusterHandler is going to set attributes
+            attrs.setown(createPTree("Part", ipt_fast));  // clusterHandler is going to set attributes
         else
         {
             // add cluster
@@ -15652,7 +15652,7 @@ public:
     {
         if (numUses > 1)
         {
-            Owned<IPropertyTree> splitterNode = createPTree();
+            Owned<IPropertyTree> splitterNode = createPTree(ipt_fast);
             factory.setown(createRoxieServerThroughSpillActivityFactory(sourceAct->queryFactory()->queryQueryFactory(), createGraphOutputSplitter, numUses, *splitterNode));
             IRoxieServerActivity *splitter = factory->createActivity(ctx, NULL);
             splitter->onCreate(NULL);
@@ -21705,6 +21705,7 @@ public:
 
     virtual const void *nextRow()
     {
+        ActivityTimer t(totalCycles, timeActivities);
         if (eof)
             return NULL;
         else if (useRemote())
@@ -21917,6 +21918,7 @@ public:
 
     virtual const void *nextRow()
     {
+        ActivityTimer t(totalCycles, timeActivities);
         if (eof)
             return NULL;
         else if (useRemote())
@@ -22026,6 +22028,7 @@ public:
 
     virtual const void *nextRow()
     {
+        ActivityTimer t(totalCycles, timeActivities);
         if (eof)
             return NULL;
         else if (useRemote())
@@ -22133,6 +22136,7 @@ public:
 
     virtual const void *nextRow()
     {
+        ActivityTimer t(totalCycles, timeActivities);
         if (eof)
             return NULL;
         else if (useRemote())

roxie/ccd/ccdsnmp.cpp

@@ -953,7 +953,7 @@ public:
     }
     static IPropertyTree *getAllQueryStats(bool includeQueries, time_t from, time_t to)
     {
-        Owned<IPTree> result = createPTree("QueryStats");
+        Owned<IPTree> result = createPTree("QueryStats", ipt_fast);
         if (includeQueries)
         {
             SpinBlock b(queryStatsCrit);
@@ -994,7 +994,7 @@ public:
     {
         time_t timeNow;
         time(&timeNow);
-        Owned<IPropertyTree> result = createPTree("Query");
+        Owned<IPropertyTree> result = createPTree("Query", ipt_fast);
         result->setProp("@id", queryName);
         if (expirySeconds && difftime(timeNow, from) <= expirySeconds)
         {

roxie/ccd/ccdstate.cpp

@@ -1374,13 +1374,13 @@ public:
             {
                 StringBuffer freply;
                 serverManager->getStats(queryId, graphName, freply, logctx);
-                Owned<IPropertyTree> stats = createPTreeFromXMLString(freply.str());
+                Owned<IPropertyTree> stats = createPTreeFromXMLString(freply.str(), ipt_fast);
                 for (unsigned channel = 0; channel < numChannels; channel++)
                     if (slaveManagers->item(channel))
                     {
                         StringBuffer sreply;
                         slaveManagers->item(channel)->getStats(queryId, graphName, sreply, logctx);
-                        Owned<IPropertyTree> cstats = createPTreeFromXMLString(sreply.str());
+                        Owned<IPropertyTree> cstats = createPTreeFromXMLString(sreply.str(), ipt_fast);
                         mergeStats(stats, cstats, 1);
                     }
                 toXML(stats, reply);
@@ -1523,7 +1523,7 @@ public:
     virtual void load(bool forceReload)
     {
         hash64_t newHash = numChannels;
-        Owned<IPropertyTree> newQuerySet = createPTree("QuerySet");
+        Owned<IPropertyTree> newQuerySet = createPTree("QuerySet", ipt_lowmem);
         newQuerySet->setProp("@name", "_standalone");
         newQuerySet->addPropTree("Query", standaloneDll.getLink());
         Owned<CRoxieSlaveQuerySetManagerSet> newSlaveManagers = new CRoxieSlaveQuerySetManagerSet(numChannels, querySet);
@@ -1559,7 +1559,7 @@ public:
     : stateHash(0), daliHelper(_daliHelper)
     {
         Owned<IPropertyTree> standAloneDllTree;
-        standAloneDllTree.setown(createPTree("Query"));
+        standAloneDllTree.setown(createPTree("Query", ipt_lowmem));
         standAloneDllTree->setProp("@id", "roxie");
         standAloneDllTree->setProp("@dll", standAloneDll->queryDll()->queryName());
         Owned<CRoxieQueryPackageManager> qpm = new CStandaloneQueryPackageManager(numChannels, querySet, LINK(&queryEmptyRoxiePackageMap()), standAloneDllTree.getClear());

roxie/ccd/hpccprotocol.hpp

@@ -136,7 +136,7 @@ interface IActiveQueryLimiterFactory : extends IInterface
 
 interface IHpccProtocolPlugin : extends IInterface
 {
-    virtual IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config)=0;
+    virtual IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const char *certFile=nullptr, const char *keyFile=nullptr, const char *passPhrase=nullptr)=0;
 };
 
 extern IHpccProtocolPlugin *loadHpccProtocolPlugin(IHpccProtocolPluginContext *ctx, IActiveQueryLimiterFactory *limiterFactory);

roxie/roxiepipe/CMakeLists.txt

@@ -29,12 +29,18 @@ set (    SRCS
          roxiepipe.cpp 
     )
 
-include_directories ( 
+include_directories (
          ./../../system/include 
          ./../../system/jlib 
          ./../../roxie/roxie
     )
 
+IF (USE_OPENSSL)
+    include_directories (
+         ./../../system/security/securesocket
+    )
+ENDIF()
+
 ADD_DEFINITIONS( -D_CONSOLE )
 
 HPCC_ADD_EXECUTABLE ( roxiepipe ${SRCS} )
@@ -43,3 +49,9 @@ target_link_libraries ( roxiepipe
          jlib
     )
 
+IF (USE_OPENSSL)
+    target_link_libraries ( roxiepipe
+         securesocket
+    )
+ENDIF()
+

roxie/roxiepipe/roxiepipe.cpp

@@ -25,6 +25,9 @@
 #include "jmisc.hpp"
 #include "jqueue.tpp"
 #include "roxie.hpp"
+#ifdef _USE_OPENSSL
+# include "securesocket.hpp"
+#endif
 
 static int in_width = 0;
 static int out_width = 1;
@@ -40,11 +43,12 @@ static int readTimeout = 300;
 static Mutex readMutex;
 static Mutex writeMutex;
 static StringBuffer hosts;
-static ISmartSocketFactory *smartSocketFactory;
+static ISmartSocketFactory *smartSocketFactory = nullptr;
 static bool Aborting = false;
 static StringBuffer fatalError;
 static CriticalSection fatalErrorSect;
 
+static bool useSSL = false;
 
 interface IReceivedRoxieException : extends IException
 {
@@ -69,7 +73,6 @@ private:
 };
 
 
-
 class RoxieThread : public Thread
 {
 private:
@@ -445,6 +448,15 @@ public:
     }
 };
 
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    return true;
+}
+
+MODULE_EXIT()
+{
+    ::Release(smartSocketFactory);
+}
 
 int main(int argc, char *argv[])
 {
@@ -571,6 +583,15 @@ int main(int argc, char *argv[])
             DebugBreak();
         }
 #endif
+        else if (stricmp(argv[i], "-ssl") == 0)
+        {
+#ifdef _USE_OPENSSL
+            useSSL = true;
+#else
+            fatalError.append("-ssl argument not supported : OpenSSL disabled in build");
+            break;
+#endif
+        }
         else
         {
             fatalError.appendf("Unknown/unexpected parameter %s", argv[i]);
@@ -622,7 +643,12 @@ int main(int argc, char *argv[])
 
             try
             {
-                smartSocketFactory = createSmartSocketFactory(hosts.str(), retryMode);
+#ifdef _USE_OPENSSL
+                if (useSSL)
+                    smartSocketFactory = createSecureSmartSocketFactory(hosts.str(), retryMode);
+                else
+#endif
+                    smartSocketFactory = createSmartSocketFactory(hosts.str(), retryMode);
             }
             catch (ISmartSocketException *e)
             {

rtl/eclrtl/rtlkey.cpp

@@ -533,9 +533,9 @@ public:
     virtual IKeySegmentMonitor *combine(const IKeySegmentMonitor *with) const { throwUnexpected(); };
     virtual bool isSimple() const { return true; }
 
-    virtual MemoryBuffer &serialize(MemoryBuffer &mb)
+    virtual MemoryBuffer &serialize(MemoryBuffer &mb) const override
     {
-        return CKeySegmentMonitor::serialize(mb);
+        CKeySegmentMonitor::serialize(mb);
         if (val) 
             mb.append((bool)true).append(size,val);
         else

system/jhtree/jhtree.cpp

@@ -260,6 +260,21 @@ void SegMonitorList::recalculateCache()
     cachedLRS = _lastRealSeg();
 }
 
+void SegMonitorList::deserialize(MemoryBuffer &mb)
+{
+    unsigned num;
+    mb.read(num);
+    while (num--)
+        append(deserializeKeySegmentMonitor(mb));
+}
+
+void SegMonitorList::serialize(MemoryBuffer &mb) const
+{
+    mb.append((unsigned) ordinality());
+    ForEachItemIn(idx, segMonitors)
+        segMonitors.item(idx).serialize(mb);
+}
+
 // interface IIndexReadContext
 void SegMonitorList::append(IKeySegmentMonitor *segment)
 {
@@ -935,6 +950,11 @@ public:
         activitySegs->setMergeBarrier(offset); 
     }
 
+    virtual void deserializeSegmentMonitors(MemoryBuffer &mb) override
+    {
+        segs.deserialize(mb);
+    }
+
     virtual void finishSegmentMonitors()
     {
         if(transformSegs)
@@ -1975,7 +1995,8 @@ class CLazyKeyIndex : implements IKeyIndex, public CInterface
 {
     StringAttr keyfile;
     unsigned crc; 
-    Linked<IDelayedFile> iFileIO;
+    Linked<IDelayedFile> delayedFile;
+    Owned<IFileIO> iFileIO;
     Owned<IKeyIndex> realKey;
     CriticalSection c;
     bool isTLK;
@@ -1986,11 +2007,14 @@ class CLazyKeyIndex : implements IKeyIndex, public CInterface
         CriticalBlock b(c);
         if (!realKey)
         {
-            IMemoryMappedFile *mapped = useMemoryMappedIndexes ? iFileIO->queryMappedFile() : 0;
+            Owned<IMemoryMappedFile> mapped = useMemoryMappedIndexes ? delayedFile->getMappedFile() : nullptr;
             if (mapped)
                 realKey.setown(queryKeyStore()->load(keyfile, crc, mapped, isTLK, preloadAllowed));
             else
-                realKey.setown(queryKeyStore()->load(keyfile, crc, iFileIO->queryFileIO(), isTLK, preloadAllowed));
+            {
+                iFileIO.setown(delayedFile->getFileIO());
+                realKey.setown(queryKeyStore()->load(keyfile, crc, iFileIO, isTLK, preloadAllowed));
+            }
             if (!realKey)
             {
                 DBGLOG("Lazy key file %s could not be opened", keyfile.get());
@@ -2002,8 +2026,8 @@ class CLazyKeyIndex : implements IKeyIndex, public CInterface
 
 public:
     IMPLEMENT_IINTERFACE;
-    CLazyKeyIndex(const char *_keyfile, unsigned _crc, IDelayedFile *_iFileIO, bool _isTLK, bool _preloadAllowed)
-        : keyfile(_keyfile), crc(_crc), iFileIO(_iFileIO), isTLK(_isTLK), preloadAllowed(_preloadAllowed)
+    CLazyKeyIndex(const char *_keyfile, unsigned _crc, IDelayedFile *_delayedFile, bool _isTLK, bool _preloadAllowed)
+        : keyfile(_keyfile), crc(_crc), delayedFile(_delayedFile), isTLK(_isTLK), preloadAllowed(_preloadAllowed)
     {}
 
     virtual bool IsShared() const { return CInterface::IsShared(); }
@@ -2027,7 +2051,7 @@ public:
     virtual offset_t queryMetadataHead() { return checkOpen().queryMetadataHead(); }
     virtual IPropertyTree * getMetadata() { return checkOpen().getMetadata(); }
     virtual unsigned getNodeSize() { return checkOpen().getNodeSize(); }
-    virtual const IFileIO *queryFileIO() const override { return iFileIO->queryFileIO(); }
+    virtual const IFileIO *queryFileIO() const override { return iFileIO; } // NB: if not yet opened, will be null
 };
 
 extern jhtree_decl IKeyIndex *createKeyIndex(const char *keyfile, unsigned crc, IFileIO &iFileIO, bool isTLK, bool preloadAllowed)

system/jhtree/jhtree.hpp

@@ -30,8 +30,8 @@
 
 interface jhtree_decl IDelayedFile : public IInterface
 {
-    virtual IMemoryMappedFile *queryMappedFile() = 0;
-    virtual IFileIO *queryFileIO() = 0;
+    virtual IMemoryMappedFile *getMappedFile() = 0;
+    virtual IFileIO *getFileIO() = 0;
 };
 
 interface jhtree_decl IKeyCursor : public IInterface
@@ -178,6 +178,8 @@ public:
     void checkSize(size32_t keyedSize, char const * keyname);
     void recalculateCache();
     void finish();
+    void deserialize(MemoryBuffer &mb);
+    void serialize(MemoryBuffer &mb) const;
 
     // interface IIndexReadContext
     virtual void append(IKeySegmentMonitor *segment);
@@ -213,6 +215,7 @@ interface IKeyManager : public IInterface, extends IIndexReadContext
     virtual void resetCounts() = 0;
 
     virtual void setLayoutTranslator(IRecordLayoutTranslator * trans) = 0;
+    virtual void deserializeSegmentMonitors(MemoryBuffer &mb) = 0;
     virtual void finishSegmentMonitors() = 0;
 
     virtual bool lookupSkip(const void *seek, size32_t seekGEOffset, size32_t seeklen) = 0;

system/jlib/jbuff.hpp

@@ -249,31 +249,48 @@ private:
     
 };
 
-// Utility class, to back patch a size into current position
-class jlib_decl DelayedSizeMarker
+// Utility class, to back patch a scalar into current position
+template <class CLASS>
+class jlib_decl DelayedMarker
 {
+protected:
     MemoryBuffer &mb;
     size32_t pos;
 public:
-    DelayedSizeMarker(MemoryBuffer &_mb) : mb(_mb)
+    DelayedMarker(MemoryBuffer &_mb) : mb(_mb)
     {
         restart();
     }
+    inline void write(CLASS a)
+    {
+        mb.writeEndianDirect(pos, sizeof(a), &a);
+    }
+    // resets position marker and writes CLASS # bytes to be filled subsequently by write()
+    inline void restart()
+    {
+        pos = mb.length();
+        mb.appendBytes(0, sizeof(CLASS));
+    }
+};
+
+// Utility class, to back patch a size into current position
+class jlib_decl DelayedSizeMarker : private DelayedMarker<size32_t>
+{
+    typedef DelayedMarker<size32_t> PARENT;
+public:
+    DelayedSizeMarker(MemoryBuffer &mb) : PARENT(mb)
+    {
+    }
     inline void write()
     {
         size32_t sz = size();
-        mb.writeDirect(pos, sizeof(sz), &sz);
+        PARENT::write(sz);
     }
     inline size32_t size() const
     {
         return (size32_t)(mb.length() - (pos + sizeof(size32_t)));
     }
-    // resets position marker and writes another size to be filled subsequently by write()
-    inline void restart()
-    {
-        pos = mb.length();
-        mb.append((size32_t)0);
-    }
+    inline void restart() { PARENT::restart(); }
 };
 
 interface jlib_decl serializable : extends IInterface

system/jlib/jdebug.cpp

@@ -2612,6 +2612,8 @@ public:
         if(!mode) return;
         unsigned memused=0;
         unsigned memtot=0;
+        str.appendf("LPT=%u ", queryNumLocalTrees());
+        str.appendf("APT=%u ", queryNumAtomTrees());
         if(mode & PerfMonProcMem)
         {
             if (!outofhandles)

system/jlib/jhash.hpp

@@ -518,6 +518,23 @@ public:
         return c;
     }
 
+    unsigned findIndex(const char *key, unsigned h)
+    {
+        unsigned i=h%htn;
+        while (table[i]) {
+            if ((table[i]->hash==h)&&table[i]->eq(key))
+                return i;
+            if (++i==htn)
+                i = 0;
+        }
+        return (unsigned) -1;
+    }
+
+    C *getIndex(unsigned v) const
+    {
+        assert(v != (unsigned)-1 && v < htn);
+        return table[v];
+    }
 
     void remove(C *c)
     {

system/jlib/jptree-attrs.hpp

@@ -0,0 +1,81 @@
+    "@accessed",
+    "@activity",
+    "@agentSession",
+    "@buildVersion",
+    "@checkSum",
+    "@cloneable",
+    "@cluster",
+    "@clusterName",
+    "@codeVersion",
+    "@columnMapping",
+    "@connected",
+    "@created",
+    "@creator",
+    "@csvSeparate",
+    "@csvTerminate",
+    "@defaultBaseDir",
+    "@defaultReplicateDir",
+    "@description",
+    "@directory",
+    "@dllname",
+    "@eclVersion",
+    "@enqueuedt",
+    "@expireDays",
+    "@fetchEntire",
+    "@fieldwidth",
+    "@fileCrc",
+    "@filename",
+    "@footerLength",
+    "@format",
+    "@formatCrc",
+    "@hasArchive",
+    "@headerLength",
+    "@interleaved",
+    "@isArchive",
+    "@isClone",
+    "@isLibrary",
+    "@isScalar",
+    "@jobName",
+    "@libCount",
+    "@localValue",
+    "@mapFlags",
+    "@maxActivity",
+    "@minActivity",
+    "@modified",
+    "@numclusters",
+    "@numparts",
+    "@numsubfiles",
+    "@offset",
+    "@partmask",
+    "@persistent",
+    "@prevenqueuedt",
+    "@prevpriority",
+    "@prevwuid",
+    "@priority",
+    "@publishedBy",
+    "@recordCount",
+    "@recordSize",
+    "@recordSizeEntry",
+    "@redundancy",
+    "@rowLimit",
+    "@rowTag",
+    "@sequence",
+    "@serverId",
+    "@session",
+    "@severity",
+    "@source",
+    "@stateEx",
+    "@status",
+    "@submitID",
+    "@table:date-value",
+    "@table:number-columns-repeated",
+    "@table:style-name",
+    "@table:value",
+    "@table:value-type",
+    "@target",
+    "@totalThorTime",
+    "@transformerEntry",
+    "@waiting",
+    "@workunit",
+    "@wuidVersion",
+    "@xmlns:xsi",

system/jlib/jptree.cpp

@@ -70,34 +70,75 @@ IPropertyTreeIterator *createNullPTreeIterator() { return LINK(nullPTreeIterator
 
 //===================================================================
 
+#ifdef USE_READONLY_ATOMTABLE
+RONameTable *AttrStrUnionWithTable::roNameTable = nullptr;
+#endif
 static AtomRefTable *keyTable = nullptr;
 static AtomRefTable *keyTableNC = nullptr;
+
 static CriticalSection hashcrit;
 static CAttrValHashTable *attrHT = nullptr;
 static AttrValue **freelist = nullptr;
 static unsigned freelistmax = 0;
 static CLargeMemoryAllocator freeallocator((memsize_t)-1, 0x1000*sizeof(AttrValue), true);
 
+#ifdef USE_READONLY_ATOMTABLE
+static const char * roAttributes[] =
+{
+#include "jptree-attrs.hpp"    // potentially auto-generated
+    nullptr
+};
+
+void initializeRoTable()
+{
+    for (const char **attr = roAttributes; *attr; attr++)
+    {
+        AttrStrUnionWithTable::roNameTable->find(*attr, true);
+    }
+#ifdef TRACE_ATOM_SIZE
+    // If you are wanting an idea of the savings from use of the RO hash table, it may be useful to reset
+    // the counts here. But it's more correct to actually leave them in place.
+    //AttrStrAtom::totsize = 0;
+    //AttrStrAtom::maxsize = 0;
+#endif
+#ifdef _DEBUG
+    for (const char **a = roAttributes; *a; a++)
+    {
+        // sanity check
+        unsigned idx = AttrStrUnionWithTable::roNameTable->findIndex(*a,AttrStrC::getHash(*a));
+        AttrStrC *val = AttrStrUnionWithTable::roNameTable->getIndex(idx);
+        assert(val && val->eq(*a));
+    }
+#endif
+}
+#endif
+
 MODULE_INIT(INIT_PRIORITY_JPTREE)
 {
     nullPTreeIterator = new NullPTreeIterator;
-	keyTable = new AtomRefTable;
-	keyTableNC = new AtomRefTable(true);
-	attrHT = new CAttrValHashTable;
+#ifdef USE_READONLY_ATOMTABLE
+    AttrStrUnionWithTable::roNameTable = new RONameTable;
+    initializeRoTable();
+#endif
+    keyTable = new AtomRefTable;
+    keyTableNC = new AtomRefTable(true);
+    attrHT = new CAttrValHashTable;
     return true;
 }
 
 MODULE_EXIT()
 {
     nullPTreeIterator->Release();
-	delete attrHT;
-	keyTable->Release();
-	keyTableNC->Release();
-	free(freelist);
-	freelist = NULL;
+    delete attrHT;
+    keyTable->Release();
+    keyTableNC->Release();
+#ifdef USE_READONLY_ATOMTABLE
+    delete AttrStrUnionWithTable::roNameTable;
+#endif
+    free(freelist);
+    freelist = NULL;
 }
 
-
 static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
 {
     IPropertyTree *l = (IPropertyTree *) *ll;
@@ -110,7 +151,7 @@ static int comparePropTrees(IInterface * const *ll, IInterface * const *rr)
 unsigned ChildMap::getHashFromElement(const void *e) const
 {
     PTree &elem = (PTree &) (*(IPropertyTree *)e);
-    return elem.queryKey()->queryHash();
+    return elem.queryHash();
 }
 
 unsigned ChildMap::numChildren()
@@ -1735,11 +1776,11 @@ IAttributeIterator *PTree::getAttributes(bool sorted) const
         virtual bool isValid() override { return cur ? true : false; }
         virtual const char *queryName() const override
         {
-            return cur->key->get();
+            return cur->key.get();
         }
         virtual const char *queryValue() const override
         {
-            return cur->value->get();
+            return cur->value.get();
         }
         virtual StringBuffer &getValue(StringBuffer &out) override
         {
@@ -1761,7 +1802,7 @@ IAttributeIterator *PTree::getAttributes(bool sorted) const
 
         static int compareAttrs(AttrValue * const *ll, AttrValue * const *rr)
         {
-            return stricmp((*ll)->key->get(), (*rr)->key->get());
+            return stricmp((*ll)->key.get(), (*rr)->key.get());
         };
 
         CSortedAttributeIterator(const PTree *_parent) : cur(NULL), iter(NULL), parent(_parent)
@@ -1804,12 +1845,12 @@ IAttributeIterator *PTree::getAttributes(bool sorted) const
         virtual const char *queryName() const override
         {
             assertex(cur);
-            return cur->key->get();
+            return cur->key.get();
         }
         virtual const char *queryValue() const override
         {
             assertex(cur);
-            return cur->value->get();
+            return cur->value.get();
         }
         virtual StringBuffer &getValue(StringBuffer &out) override
         {
@@ -2822,7 +2863,7 @@ AttrValue *PTree::findAttribute(const char *key) const
     {
         while (a-- != attrs)
         {
-            if (strieq(a->key->get(), key))
+            if (strieq(a->key.get(), key))
                 return a;
         }
     }
@@ -2830,7 +2871,7 @@ AttrValue *PTree::findAttribute(const char *key) const
     {
         while (a-- != attrs)
         {
-            if (streq(a->key->get(), key))
+            if (streq(a->key.get(), key))
                 return a;
         }
     }
@@ -2841,7 +2882,7 @@ const char *PTree::getAttributeValue(const char *key) const
 {
     AttrValue *e = findAttribute(key);
     if (e)
-        return e->value->get();
+        return e->value.get();
     return nullptr;
 }
 
@@ -2869,36 +2910,48 @@ AttrValue *PTree::getNextAttribute(AttrValue *cur) const
 
 // LocalPTree
 
-LocalPTree::LocalPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags, _value, _children)
+static RelaxedAtomic<unsigned> numLocalTrees;
+unsigned queryNumLocalTrees()
+{
+    return numLocalTrees;
+}
+
+LocalPTree::LocalPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_fast, _value, _children)
 {
     if (_name)
         setName(_name);
+    numLocalTrees++;
 }
 
 LocalPTree::~LocalPTree()
 {
-    if (name)
-        free(name);
+    numLocalTrees--;
+    name.destroy();
     if (!attrs)
         return;
     AttrValue *a = attrs+numAttrs;
     while (a--!=attrs)
     {
-        free(a->key);
-        free(a->value);
+        a->key.destroy();
+        a->value.destroy();
     }
     free(attrs);
 }
 
+const char *LocalPTree::queryName() const
+{
+    return name.get();
+}
+
 void LocalPTree::setName(const char *_name)
 {
-    HashKeyElement *oname = name;
-    if (!_name)
-        name = nullptr;
-    else
-        name = AtomRefTable::createKeyElement(_name, isnocase());
+    if (_name==name.get())
+        return;
+    AttrStr *oname = name.getPtr();  // Don't free until after we copy - they could overlap
+    if (!name.set(_name))
+        name.setPtr(AttrStr::create(_name));
     if (oname)
-        free(oname);
+        AttrStr::destroy(oname);
 }
 
 bool LocalPTree::removeAttribute(const char *key)
@@ -2908,8 +2961,8 @@ bool LocalPTree::removeAttribute(const char *key)
         return false;
     numAttrs--;
     unsigned pos = del-attrs;
-    free(del->key);
-    free(del->value);
+    del->key.destroy();
+    del->value.destroy();
     memmove(attrs+pos, attrs+pos+1, (numAttrs-pos)*sizeof(AttrValue));
     return true;
 }
@@ -2923,41 +2976,66 @@ void LocalPTree::setAttribute(const char *key, const char *val)
     if (!val)
         val = "";  // cannot have NULL value
     AttrValue *v = findAttribute(key);
+    AttrStr *goer = nullptr;
     if (v)
     {
-        if (streq(v->value->get(), val))
+        if (streq(v->value.get(), val))
             return;
-        AttrStrC::destroy((AttrStrC *)v->value);
-        v->value = AttrStrC::create(val);
+        goer = v->value.getPtr();
     }
     else
     {
         attrs = (AttrValue *)realloc(attrs, (numAttrs+1)*sizeof(AttrValue));
-        if (isnocase())
-            attrs[numAttrs].key = AttrStrNC::create(key);
-        else
-            attrs[numAttrs].key = AttrStrC::create(key);
-        attrs[numAttrs].value = AttrStrC::create(val);
-        numAttrs++;
+        v = &attrs[numAttrs++];
+        if (!v->key.set(key))
+            v->key.setPtr(isnocase() ? AttrStr::createNC(key) : AttrStr::create(key));
     }
+    if (!v->value.set(val))
+        v->value.setPtr(AttrStr::create(val));
+    if (goer)
+        AttrStr::destroy(goer);
 }
 
+#ifdef TRACE_STRING_SIZE
+std::atomic<__int64> AttrStr::totsize { 0 };
+std::atomic<__int64> AttrStr::maxsize { 0 };
+#endif
+
+#ifdef TRACE_ATOM_SIZE
+std::atomic<__int64> AttrStrAtom::totsize { 0 };
+std::atomic<__int64> AttrStrAtom::maxsize { 0 };
+#endif
 
 ///////////////////
 
-CAtomPTree::CAtomPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags, _value, _children)
+static RelaxedAtomic<unsigned> numAtomTrees;
+unsigned queryNumAtomTrees()
+{
+    return numAtomTrees;
+}
+
+CAtomPTree::CAtomPTree(const char *_name, byte _flags, IPTArrayValue *_value, ChildMap *_children) : PTree(_flags|ipt_lowmem, _value, _children)
 {
+    numAtomTrees++;
     if (_name)
         setName(_name);
 }
 
 CAtomPTree::~CAtomPTree()
 {
+    numAtomTrees--;
     bool nc = isnocase();
-    if (name)
+    HashKeyElement *name_ptr = name.getPtr();
+    if (name_ptr)
     {
         AtomRefTable *kT = nc?keyTableNC:keyTable;
-        kT->releaseKey(name);
+#ifdef TRACE_ATOM_SIZE
+        size_t gosize = sizeof(HashKeyElement)+strlen(name_ptr->get())+1;
+        if (kT->releaseKey(name_ptr))
+            AttrStrAtom::totsize -= gosize;
+#else
+        kT->releaseKey(name_ptr);
+#endif
     }
     if (!attrs)
         return;
@@ -2966,8 +3044,10 @@ CAtomPTree::~CAtomPTree()
         CriticalBlock block(hashcrit);
         while (a--!=attrs)
         {
-            attrHT->removekey(a->key, nc);
-            attrHT->removeval(a->value);
+            if (a->key.isPtr())
+                attrHT->removekey(a->key.getPtr(), nc);
+            if (a->value.isPtr())
+                attrHT->removeval(a->value.getPtr());
         }
         freeAttrArray(attrs, numAttrs);
     }
@@ -2976,17 +3056,65 @@ CAtomPTree::~CAtomPTree()
 void CAtomPTree::setName(const char *_name)
 {
     AtomRefTable *kT = isnocase()?keyTableNC:keyTable;
-    HashKeyElement *oname = name;
+    HashKeyElement *oname = name.getPtr(); // NOTE - don't release yet as could overlap source name
     if (!_name)
-        name = nullptr;
+        name.setPtr(nullptr);
     else
     {
         if (!validateXMLTag(_name))
             throw MakeIPTException(PTreeExcpt_InvalidTagName, ": %s", _name);
-        name = kT->queryCreate(_name);
+        if (!name.set(_name))
+        {
+#ifdef TRACE_ALL_ATOM
+            DBGLOG("TRACE_ALL_ATOM: %s", _name);
+#endif
+#ifdef TRACE_ATOM_SIZE
+            bool didCreate;
+            name.setPtr(kT->queryCreate(_name, didCreate));
+            if (didCreate)
+            {
+                AttrStrAtom::totsize += sizeof(HashKeyElement)+strlen(_name)+1;
+                if (AttrStrAtom::totsize > AttrStrAtom::maxsize)
+                {
+                    AttrStrAtom::maxsize.store(AttrStrAtom::totsize);
+                    DBGLOG("TRACE_ATOM_SIZE: total size now %" I64F "d", AttrStrAtom::maxsize.load());
+                }
+            }
+#else
+            name.setPtr(kT->queryCreate(_name));
+#endif
+        }
     }
     if (oname)
+    {
+#ifdef TRACE_ATOM_SIZE
+        size_t gosize = sizeof(HashKeyElement)+strlen(oname->get())+1;
+        if (kT->releaseKey(oname))
+            AttrStrAtom::totsize -= gosize;
+#else
         kT->releaseKey(oname);
+#endif
+    }
+}
+
+const char *CAtomPTree::queryName() const
+{
+    return name.get();
+}
+
+unsigned CAtomPTree::queryHash() const
+{
+    if (name.isPtr())
+    {
+        assert(name.getPtr());
+        return name.getPtr()->queryHash();
+    }
+    else
+    {
+        const char *_name = name.get();
+        size32_t nl = strlen(_name);
+        return isnocase() ? hashnc((const byte *) _name, nl, 0): hashc((const byte *) _name, nl, 0);
+    }
 }
 
 AttrValue *CAtomPTree::newAttrArray(unsigned n)
@@ -3031,19 +3159,32 @@ void CAtomPTree::setAttribute(const char *key, const char *val)
     AttrValue *v = findAttribute(key);
     if (v)
     {
-        if (streq(v->value->get(), val))
+        if (streq(v->value.get(), val))
             return;
-        CriticalBlock block(hashcrit);
-        attrHT->removeval(v->value);
-        v->value = attrHT->addval(val);
+        AttrStr * goer = v->value.getPtr();
+        if (!v->value.set(val))
+        {
+            CriticalBlock block(hashcrit);
+            if (goer)
+                attrHT->removeval(goer);
+            v->value.setPtr(attrHT->addval(val));
+        }
+        else if (goer)
+        {
+            CriticalBlock block(hashcrit);
+            attrHT->removeval(goer);
+        }
     }
     else
     {
         CriticalBlock block(hashcrit);
         AttrValue *newattrs = newAttrArray(numAttrs+1);
         memcpy(newattrs, attrs, numAttrs*sizeof(AttrValue));
-        newattrs[numAttrs].key = attrHT->addkey(key, isnocase());
-        newattrs[numAttrs].value = attrHT->addval(val);
+        v = &newattrs[numAttrs];
+        if (!v->key.set(key))
+            v->key.setPtr(attrHT->addkey(key, isnocase()));
+        if (!v->value.set(val))
+            v->value.setPtr(attrHT->addval(val));
         freeAttrArray(attrs, numAttrs);
         numAttrs++;
         attrs = newattrs;
@@ -3057,8 +3198,10 @@ bool CAtomPTree::removeAttribute(const char *key)
         return false;
     numAttrs--;
     CriticalBlock block(hashcrit);
-    attrHT->removekey(del->key, isnocase());
-    attrHT->removeval(del->value);
+    if (del->key.isPtr())
+        attrHT->removekey(del->key.getPtr(), isnocase());
+    if (del->value.isPtr())
+        attrHT->removeval(del->value.getPtr());
     AttrValue *newattrs = newAttrArray(numAttrs);
     if (newattrs)
     {
@@ -3438,9 +3581,9 @@ IPropertyTreeIterator *PTStackIterator::popFromStack(StringAttr &path)
 
 // factory methods
 
-IPropertyTree *createPTree(MemoryBuffer &src)
+IPropertyTree *createPTree(MemoryBuffer &src, byte flags)
 {
-    IPropertyTree *tree = new DEFAULT_PTREE_TYPE();
+    IPropertyTree *tree = createPTree(nullptr, flags);
     tree->deserialize(src);
     return tree;
 }
@@ -6070,8 +6213,8 @@ jlib_decl void testJdocCompare()
 
 #endif
 
-
-class COrderedPTree : public DEFAULT_PTREE_TYPE
+template <class BASE_PTREE>
+class COrderedPTree : public BASE_PTREE
 {
     template <class BASECHILDMAP>
     class jlib_decl COrderedChildMap : public BASECHILDMAP
@@ -6137,43 +6280,54 @@ class COrderedPTree : public DEFAULT_PTREE_TYPE
         }
     };
 public:
-    COrderedPTree(const char *name=NULL, byte flags=ipt_none, IPTArrayValue *value=NULL, ChildMap *children=NULL)
-        : DEFAULT_PTREE_TYPE(name, flags|ipt_ordered, value, children) { }
+    typedef COrderedPTree<BASE_PTREE> SELF;
+    COrderedPTree<BASE_PTREE>(const char *name=NULL, byte flags=ipt_none, IPTArrayValue *value=NULL, ChildMap *children=NULL)
+        : BASE_PTREE(name, flags|ipt_ordered, value, children) { }
 
-    virtual bool isEquivalent(IPropertyTree *tree) const override { return (NULL != QUERYINTERFACE(tree, COrderedPTree)); }
+    virtual bool isEquivalent(IPropertyTree *tree) const override { return (NULL != QUERYINTERFACE(tree, COrderedPTree<BASE_PTREE>)); }
     virtual IPropertyTree *create(const char *name=NULL, IPTArrayValue *value=NULL, ChildMap *children=NULL, bool existing=false) override
     {
-        return new COrderedPTree(name, flags, value, children);
+        return new COrderedPTree<BASE_PTREE>(name, SELF::flags, value, children);
     }
     virtual IPropertyTree *create(MemoryBuffer &mb) override
     {
-        IPropertyTree *tree = new COrderedPTree();
+        IPropertyTree *tree = new COrderedPTree<BASE_PTREE>();
         tree->deserialize(mb);
         return tree;
     }
     virtual void createChildMap() override
     {
-        if (isnocase())
-            children = new COrderedChildMap<ChildMapNC>();
+        if (SELF::isnocase())
+            SELF::children = new COrderedChildMap<ChildMapNC>();
         else
-            children = new COrderedChildMap<ChildMap>();
+            SELF::children = new COrderedChildMap<ChildMap>();
     }
 };
 
 IPropertyTree *createPTree(byte flags)
 {
-    if (flags & ipt_ordered)
-        return new COrderedPTree(NULL, flags);
-    else
-        return new DEFAULT_PTREE_TYPE(NULL, flags);
+    return createPTree(NULL, flags);
 }
 
 IPropertyTree *createPTree(const char *name, byte flags)
 {
-    if (flags & ipt_ordered)
-        return new COrderedPTree(name, flags);
-    else
+    switch (flags & (ipt_ordered|ipt_fast|ipt_lowmem))
+    {
+    case ipt_ordered|ipt_fast:
+        return new COrderedPTree<LocalPTree>(name, flags);
+    case ipt_ordered|ipt_lowmem:
+        return new COrderedPTree<CAtomPTree>(name, flags);
+    case ipt_ordered:
+        return new COrderedPTree<DEFAULT_PTREE_TYPE>(name, flags);
+    case ipt_fast:
+        return new LocalPTree(name, flags);
+    case ipt_lowmem:
+        return new CAtomPTree(name, flags);
+    case 0:
         return new DEFAULT_PTREE_TYPE(name, flags);
+    default:
+        throwUnexpectedX("Invalid flags - ipt_fast and ipt_lowmem should not be specified together");
+    }
 }
 
 typedef enum _ptElementType

system/jlib/jptree.hpp

@@ -61,6 +61,10 @@ typedef unsigned IPTIteratorCodes;
 #define iptiter_remote 0x02
 #define iptiter_remoteget 0x06
 #define iptiter_remotegetbranch 0x0e
+
+extern jlib_decl unsigned queryNumLocalTrees();
+extern jlib_decl unsigned queryNumAtomTrees();
+
 interface jlib_decl IPropertyTree : extends serializable
 {
     virtual bool hasProp(const char *xpath) const = 0;
@@ -98,6 +102,9 @@ interface jlib_decl IPropertyTree : extends serializable
     virtual IPropertyTree *setPropTree(const char *xpath, IPropertyTree *val) = 0;
     virtual IPropertyTree *addPropTree(const char *xpath, IPropertyTree *val) = 0;
 
+    virtual IPropertyTree *setPropTree(const char *xpath) = 0;
+    virtual IPropertyTree *addPropTree(const char *xpath) = 0;
+
     virtual bool removeProp(const char *xpath) = 0;
     virtual bool removeTree(IPropertyTree *child) = 0;
     virtual aindex_t queryChildIndex(IPropertyTree *child) = 0;
@@ -170,9 +177,19 @@ interface IPTreeNodeCreator : extends IInterface
     virtual IPropertyTree *create(const char *tag) = 0;
 };
 
-// NB ipt_ext4 - used by SDS
-// NB ipt_ext5 - used by SDS
-enum ipt_flags { ipt_none=0x00, ipt_caseInsensitive=0x01, ipt_binary=0x02, ipt_ordered=0x04, ipt_ext1=0x08, ipt_ext2=16, ipt_ext3=32, ipt_ext4=64, ipt_ext5=128 };
+enum ipt_flags
+{
+    ipt_none=0x00,
+    ipt_caseInsensitive=0x01,
+    ipt_binary  = 0x02,
+    ipt_ordered = 0x04,   // Preserve element ordering
+    ipt_fast    = 0x08,   // Prioritize speed over low memory usage
+    ipt_lowmem  = 0x10,   // Prioritize low memory usage over speed
+    ipt_ext3    = 0x20,   // Unused
+    ipt_ext4    = 0x40,   // Used internally in Dali
+    ipt_ext5    = 0x80    // Used internally in Dali
+};
+
 jlib_decl IPTreeMaker *createPTreeMaker(byte flags=ipt_none, IPropertyTree *root=NULL, IPTreeNodeCreator *nodeCreator=NULL);
 jlib_decl IPTreeMaker *createRootLessPTreeMaker(byte flags=ipt_none, IPropertyTree *root=NULL, IPTreeNodeCreator *nodeCreator=NULL);
 jlib_decl IPTreeReader *createXMLStreamReader(ISimpleReadStream &stream, IPTreeNotifyEvent &iEvent, PTreeReaderOptions xmlReaderOptions=ptr_ignoreWhiteSpace, size32_t bufSize=0);
@@ -194,7 +211,7 @@ jlib_decl void synchronizePTree(IPropertyTree *target, IPropertyTree *source);
 jlib_decl IPropertyTree *ensurePTree(IPropertyTree *root, const char *xpath);
 jlib_decl bool areMatchingPTrees(IPropertyTree * left, IPropertyTree * right);
 
-jlib_decl IPropertyTree *createPTree(MemoryBuffer &src);
+jlib_decl IPropertyTree *createPTree(MemoryBuffer &src, byte flags=ipt_none);
 
 jlib_decl IPropertyTree *createPTree(byte flags=ipt_none);
 jlib_decl IPropertyTree *createPTree(const char *name, byte flags=ipt_none);
@@ -205,7 +222,6 @@ jlib_decl IPropertyTree *createPTreeFromXMLString(const char *xml, byte flags=ip
 jlib_decl IPropertyTree *createPTreeFromXMLString(unsigned len, const char *xml, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromXMLFile(const char *filename, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromIPT(const IPropertyTree *srcTree, ipt_flags flags=ipt_none);
-
 jlib_decl IPropertyTree *createPTreeFromJSONString(const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 

system/jlib/jptree.ipp

@@ -28,6 +28,7 @@
 
 #include "jptree.hpp"
 #include "jbuff.hpp"
+#include "jlog.hpp"
 
 #define ANE_APPEND -1
 #define ANE_SET -2
@@ -205,7 +206,7 @@ public:
     virtual const void *queryValueRaw() const override { return get(); }
     virtual size32_t queryValueRawSize() const override { return (size32_t)length(); }
 
-// serilizable
+// serializable
     virtual void serialize(MemoryBuffer &tgt) override;
     virtual void deserialize(MemoryBuffer &src) override;
 
@@ -217,18 +218,313 @@ private:
 #define IptFlagSet(fs, f) (fs |= (f))
 #define IptFlagClr(fs, f) (fs &= (~f))
 
+// NOTE - hairy code alert!
+// In order to keep code common between atom and local versions of ptree, we store the atom-specific information BEFORE the this pointer of AttrStr
+// (see class AttrStrAtom below).
+// This requires some care - in particular must use the right method to destroy the objects, and must not add any virtual methods to either class
+
+//#define TRACE_STRING_SIZE
+//#define TRACE_ATOM_SIZE
+//#define TRACE_ALL_STRING
+//#define TRACE_ALL_ATOM
+
 struct AttrStr
 {
+    const char *get() const
+    {
+        return str_DO_NOT_USE_DIRECTLY;
+    }
+    char str_DO_NOT_USE_DIRECTLY[1];  // Actually [n] - null terminated
+
+    static AttrStr *create(const char *k)
+    {
+        size32_t kl = k ? strlen(k) : 0;
+#ifdef TRACE_ALL_STRING
+        DBGLOG("TRACE_ALL_STRING: %s", k);
+#endif
+#ifdef TRACE_STRING_SIZE
+        totsize += kl+1;
+        if (totsize > maxsize)
+        {
+            maxsize.store(totsize);
+            DBGLOG("TRACE_STRING_SIZE: total size now %" I64F "d", maxsize.load());
+        }
+#endif
+        AttrStr *ret = (AttrStr *) malloc(kl+1);
+        memcpy(ret->str_DO_NOT_USE_DIRECTLY, k, kl);
+        ret->str_DO_NOT_USE_DIRECTLY[kl] = 0;
+        return ret;
+    }
+    static inline AttrStr *createNC(const char *k)
+    {
+        // If we started to use a static hash table for common values, we would probably want to use a different one here for case-insensitive matches
+        return create(k);
+    }
+
+    static void destroy(AttrStr *a)
+    {
+#ifdef TRACE_STRING_SIZE
+        totsize -= strlen(a->str_DO_NOT_USE_DIRECTLY)+1;
+#endif
+        free(a);
+    }
+
+#ifdef TRACE_STRING_SIZE
+    static std::atomic<__int64> totsize;
+    static std::atomic<__int64> maxsize;
+#endif
+};
+
+
+// In order to keep code common between atom and local versions of ptree, we store the atom-specific information BEFORE the this pointer of AttrStr
+// This requires some care - in particular must use the right method to destroy the objects, and must not add any virtual methods to either class
+// Note that memory usage is significant as we create literally millions of these objects
+
+typedef unsigned hashfunc( const unsigned char *k, unsigned length, unsigned initval);
+
+struct AttrStrAtom
+{
     unsigned hash;
     unsigned short linkcount;
-    char str[1];
-    const char *get() const { return str; }
+    char str_DO_NOT_USE_DIRECTLY[1];  // Actually N
+
+    static AttrStrAtom *create(const char *k, size32_t kl, hashfunc _hash)
+    {
+#ifdef TRACE_ALL_ATOM
+        DBGLOG("TRACE_ALL_ATOM: %s", k);
+#endif
+#ifdef TRACE_ATOM_SIZE
+        totsize += sizeof(AttrStrAtom)+kl+1;
+        if (totsize > maxsize)
+        {
+            maxsize.store(totsize);
+            DBGLOG("TRACE_ATOM_SIZE: total size now %" I64F "d", maxsize.load());
+        }
+#endif
+        AttrStrAtom *ret = (AttrStrAtom *) malloc(offsetof(AttrStrAtom, str_DO_NOT_USE_DIRECTLY)+kl+1);
+        memcpy(ret->str_DO_NOT_USE_DIRECTLY, k, kl);
+        ret->str_DO_NOT_USE_DIRECTLY[kl] = 0;
+        ret->hash = _hash((const unsigned char *) k, kl, 17);
+        ret->linkcount = 0;
+        return ret;
+    }
+    static void destroy(AttrStrAtom *a)
+    {
+#ifdef TRACE_ATOM_SIZE
+        totsize -= sizeof(AttrStrAtom)+strlen(a->str_DO_NOT_USE_DIRECTLY)+1;
+#endif
+        free(a);
+    }
+
+    AttrStr *toAttrStr()
+    {
+        return (AttrStr *) &str_DO_NOT_USE_DIRECTLY;
+    }
+    static AttrStrAtom *toAtom(AttrStr *a)
+    {
+        return (AttrStrAtom *)(&a->str_DO_NOT_USE_DIRECTLY - offsetof(AttrStrAtom, str_DO_NOT_USE_DIRECTLY));
+    }
+#ifdef TRACE_ATOM_SIZE
+    static std::atomic<__int64> totsize;
+    static std::atomic<__int64> maxsize;
+#endif
+ };
+
+struct AttrStrC : public AttrStrAtom
+{
+    static inline unsigned getHash(const char *k)
+    {
+        return hashc((const byte *)k, strlen(k), 17);
+    }
+    inline bool eq(const char *k)
+    {
+        return streq(k,str_DO_NOT_USE_DIRECTLY);
+    }
+    static AttrStrC *create(const char *k)
+    {
+        size32_t kl = k ? strlen(k) : 0;
+        return (AttrStrC *) AttrStrAtom::create(k, kl, hashc);
+    }
+};
+
+struct AttrStrNC : public AttrStrAtom
+{
+    static inline unsigned getHash(const char *k)
+    {
+        return hashnc((const byte *)k, strlen(k), 17);
+    }
+    inline bool eq(const char *k)
+    {
+        return strieq(k,str_DO_NOT_USE_DIRECTLY);
+    }
+    static AttrStrNC *create(const char *k)
+    {
+        size32_t kl = k ? strlen(k) : 0;
+        return (AttrStrNC *) AttrStrAtom::create(k, kl, hashnc);
+    }
+};
+
+typedef CMinHashTable<AttrStrC> RONameTable;
+
+// NOTE - hairy code alert!
+// To save on storage (and contention) we store short string values in same slot as the pointer to longer
+// ones would occupy. This relies on the assumption that the pointers you want to store are always AT LEAST
+// 2-byte aligned. This should be the case on anything coming from malloc on any modern architecture.
+
+#define USE_STRUNION
+
+template<class PTR>
+struct PtrStrUnion
+{
+#ifdef USE_STRUNION
+    union
+    {
+        PTR *ptr;
+        struct
+        {
+#ifdef LITTLE_ENDIAN
+            char flag;
+            union
+            {
+                char chars[sizeof(PTR *)-1];
+                struct
+                {
+                    int8_t idx1;
+                    int16_t idx2;
+                };
+            };
+#else
+            union
+            {
+                char chars[sizeof(PTR *)-1];
+                struct
+                {
+                    int16_t idx2;
+                    int8_t idx1;
+                };
+            };
+            char flag;
+#endif
+        };
+    };
+    inline PtrStrUnion<PTR>() : ptr(nullptr) {}
+    inline bool isPtr() const
+    {
+        return (flag&1) == 0;
+    }
+    inline const char *get() const
+    {
+        if (!isPtr())
+        {
+            assert(flag==1);
+            return chars;
+        }
+        else if (ptr)
+            return ptr->get();
+        else
+            return nullptr;
+    }
+    inline void destroy()
+    {
+        if (isPtr() && ptr)
+            PTR::destroy(ptr);
+    }
+    bool set(const char *key)
+    {
+        if (key)
+        {
+            size32_t l = strnlen(key, sizeof(PTR *));  // technically sizeof(PTR)-1 would do, but I suspect 8 bytes is actually more optimal to search than 7
+            if (l <= sizeof(PTR *)-2)
+            {
+                flag=1;
+                memmove(chars, key, l);  // Technically, they could overlap
+                chars[l]=0;
+                return true;
+            }
+        }
+        return false;
+    }
+    inline void setPtr(PTR *a)
+    {
+        ptr = a;
+        assert(isPtr());
+    }
+#else
+    PTR *ptr = nullptr;
+    inline bool isPtr()
+    {
+        return true;
+    }
+    inline const char *get()
+    {
+        if (ptr)
+            return ptr->get();
+        else
+            return nullptr;
+    }
+    inline void destroy()
+    {
+        if (ptr)
+            PTR::destroy(ptr);
+    }
+    bool set(const char *key)
+    {
+        return false;
+    }
+    inline void setPtr(PTR *a)
+    {
+        ptr = a;
+    }
+#endif
+    inline PTR *getPtr() const
+    {
+        return isPtr() ? ptr : nullptr;
+    }
+};
+
+#ifdef USE_STRUNION
+#define USE_READONLY_ATOMTABLE
+#endif
+
+typedef PtrStrUnion<AttrStr> AttrStrUnion;
+
+#ifdef USE_READONLY_ATOMTABLE
+struct AttrStrUnionWithTable : public AttrStrUnion
+{
+    inline const char *get() const
+    {
+        if (!isPtr() && flag==3)
+            return roNameTable->getIndex(idx2)->str_DO_NOT_USE_DIRECTLY;  // Should probably rename this back now!
+        return AttrStrUnion::get();
+    }
+    bool set(const char *key)
+    {
+        if (AttrStrUnion::set(key))
+            return true;
+        if (key && key[0]=='@')
+        {
+            unsigned idx = roNameTable->findIndex(key, AttrStrC::getHash(key));
+            if (idx != (unsigned) -1)
+            {
+                assert(idx <= 0xffff);
+                flag = 3;
+                idx2 = idx;
+                return true;
+            }
+        }
+        return false;
+    }
+    static RONameTable *roNameTable;
 };
+#else
+typedef AttrStrUnion AttrStrUnionWithTable;
+
+#endif
 
 struct AttrValue
 {
-    AttrStr *key;
-    AttrStr *value;
+    AttrStrUnionWithTable key;
+    AttrStrUnion value;
 };
 
 
@@ -244,11 +540,7 @@ public:
     ~PTree();
     virtual void beforeDispose() override { }
 
-    const char *queryName() const
-    {
-        return name?name->get():nullptr;
-    }
-    HashKeyElement *queryKey() const { return name; }
+    virtual unsigned queryHash() const = 0;
     IPropertyTree *queryParent() { return parent; }
     IPropertyTree *queryChild(unsigned index);
     ChildMap *queryChildren() { return children; }
@@ -313,6 +605,8 @@ public:
     virtual IPropertyTree *queryBranch(const char *xpath) const override { return queryPropTree(xpath); }
     virtual IPropertyTree *setPropTree(const char *xpath, IPropertyTree *val) override;
     virtual IPropertyTree *addPropTree(const char *xpath, IPropertyTree *val) override;
+    virtual IPropertyTree *setPropTree(const char *xpath) override { return setPropTree(xpath, create()); }
+    virtual IPropertyTree *addPropTree(const char *xpath) override { return addPropTree(xpath, create()); }
     virtual bool removeTree(IPropertyTree *child) override;
     virtual bool removeProp(const char *xpath) override;
     virtual aindex_t queryChildIndex(IPropertyTree *child) override;
@@ -366,63 +660,10 @@ protected: // data
     IPropertyTree *parent = nullptr; // ! currently only used if tree embedded into array, used to locate position.
     ChildMap *children;   // set by constructor
     IPTArrayValue *value; // set by constructor
-    HashKeyElement *name = nullptr;
     AttrValue *attrs = nullptr;
 };
 
 
-struct AttrStrC : public AttrStr
-{
-    static inline unsigned getHash(const char *k)
-    {
-        return hashc((const byte *)k, strlen(k), 17);
-    }
-    inline bool eq(const char *k)
-    {
-        return streq(k,str);
-    }
-    static AttrStrC *create(const char *k)
-    {
-        size32_t kl = (k?strlen(k):0);
-        AttrStrC *ret = (AttrStrC *)malloc(sizeof(AttrStrC)+kl);
-        memcpy(ret->str, k, kl);
-        ret->str[kl] = 0;
-        ret->hash = hashc((const byte *)k, kl, 17);
-        ret->linkcount = 0;
-        return ret;
-    }
-    static void destroy(AttrStrC *a)
-    {
-        free(a);
-    }
-};
-
-struct AttrStrNC : public AttrStr
-{
-    static inline unsigned getHash(const char *k)
-    {
-        return hashnc((const byte *)k, strlen(k), 17);
-    }
-    inline bool eq(const char *k)
-    {
-        return strieq(k,str);
-    }
-    static AttrStrNC *create(const char *k)
-    {
-        size32_t kl = (k?strlen(k):0);
-        AttrStrNC *ret = (AttrStrNC *)malloc(sizeof(AttrStrNC)+kl);
-        memcpy(ret->str,k,kl);
-        ret->str[kl] = 0;
-        ret->hash = hashnc((const byte *)k, kl, 17);
-        ret->linkcount = 0;
-        return ret;
-    }
-    static void destroy(AttrStrNC *a)
-    {
-        free(a);
-    }
-};
-
 class CAttrValHashTable
 {
     CMinHashTable<AttrStrC>  htc;
@@ -431,24 +672,25 @@ class CAttrValHashTable
 public:
     inline AttrStr *addkey(const char *v,bool nc)
     {
-        AttrStr * ret;
+        AttrStrAtom * ret;
         if (nc)
             ret = htnc.find(v,true);
         else
             ret = htc.find(v,true);
         if (ret->linkcount!=(unsigned short)-1)
             ret->linkcount++;
-        return ret;
+        return ret->toAttrStr();
     }
     inline AttrStr *addval(const char *v)
     {
-        AttrStr * ret = htv.find(v,true);
+        AttrStrAtom * ret = htv.find(v,true);
         if (ret->linkcount!=(unsigned short)-1)
             ret->linkcount++;
-        return ret;
+        return ret->toAttrStr();
     }
-    inline void removekey(AttrStr *a,bool nc)
+    inline void removekey(AttrStr *_a,bool nc)
     {
+        AttrStrAtom *a = AttrStrAtom::toAtom(_a);
         if (a->linkcount!=(unsigned short)-1)
         {
             if (--(a->linkcount)==0)
@@ -460,8 +702,9 @@ public:
             }
         }
     }
-    inline void removeval(AttrStr *a)
+    inline void removeval(AttrStr *_a)
     {
+        AttrStrAtom *a = AttrStrAtom::toAtom(_a);
         if (a->linkcount!=(unsigned short)-1)
             if (--(a->linkcount)==0)
                 htv.remove((AttrStrC *)a);
@@ -473,13 +716,15 @@ class jlib_decl CAtomPTree : public PTree
 {
     AttrValue *newAttrArray(unsigned n);
     void freeAttrArray(AttrValue *a, unsigned n);
-
+    PtrStrUnion<HashKeyElement> name;
 protected:
     virtual void setAttribute(const char *attr, const char *val) override;
     virtual bool removeAttribute(const char *k) override;
 public:
     CAtomPTree(const char *name=nullptr, byte flags=ipt_none, IPTArrayValue *value=nullptr, ChildMap *children=nullptr);
     ~CAtomPTree();
+    const char *queryName() const override;
+    virtual unsigned queryHash() const override;
     virtual void setName(const char *_name) override;
     virtual bool isEquivalent(IPropertyTree *tree) const override { return (nullptr != QUERYINTERFACE(tree, CAtomPTree)); }
     virtual IPropertyTree *create(const char *name=nullptr, IPTArrayValue *value=nullptr, ChildMap *children=nullptr, bool existing=false) override
@@ -502,10 +747,19 @@ class jlib_decl LocalPTree : public PTree
 protected:
     virtual void setAttribute(const char *attr, const char *val) override;
     virtual bool removeAttribute(const char *k) override;
+    AttrStrUnion name;
 public:
     LocalPTree(const char *name=nullptr, byte flags=ipt_none, IPTArrayValue *value=nullptr, ChildMap *children=nullptr);
     ~LocalPTree();
 
+    const char *queryName() const override;
+    virtual unsigned queryHash() const override
+    {
+        const char *myname = queryName();
+        assert(myname);
+        size32_t nl = strlen(myname);
+        return isnocase() ? hashnc((const byte *)myname, nl, 0): hashc((const byte *)myname, nl, 0);
+    }
     virtual void setName(const char *_name) override;
     virtual bool isEquivalent(IPropertyTree *tree) const override { return (nullptr != QUERYINTERFACE(tree, LocalPTree)); }
     virtual IPropertyTree *create(const char *name=nullptr, IPTArrayValue *value=nullptr, ChildMap *children=nullptr, bool existing=false) override

system/jlib/jsmartsock.cpp

@@ -95,33 +95,7 @@ private:
 };
 
 
-class jlib_decl CSmartSocket: implements ISmartSocket, public CInterface
-{
-    ISocket *sock;
-    SocketEndpoint ep;
-    CSmartSocketFactory *factory;
-
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CSmartSocket(ISocket *_sock, SocketEndpoint &_ep, CSmartSocketFactory *_factory);
-    ~CSmartSocket();
-
-    // ISmartSocket
-    ISocket *querySocket() { return (sock); }
-
-    // subset of ISocket
-    void read(void* buf, size32_t min_size, size32_t max_size, size32_t &size_read,
-                        unsigned timeout = WAIT_FOREVER);
-    void read(void* buf, size32_t size);
-
-    size32_t write(void const* buf, size32_t size);
-
-    void close();
-};
-
-
-CSmartSocket::CSmartSocket(ISocket *_sock, SocketEndpoint &_ep, CSmartSocketFactory *_factory) : sock(_sock), ep(_ep), factory(_factory)
+CSmartSocket::CSmartSocket(ISocket *_sock, SocketEndpoint &_ep, ISmartSocketFactory *_factory) : sock(_sock), ep(_ep), factory(_factory)
 {
 };
 
@@ -305,14 +279,13 @@ SocketEndpoint& CSmartSocketFactory::nextEndpoint()
     return (ss->ep);
 }
 
-ISmartSocket *CSmartSocketFactory::connect_timeout( unsigned timeoutms)
+ISocket *CSmartSocketFactory::connect_sock(unsigned timeoutms, SmartSocketEndpoint *&ss, SocketEndpoint &ep)
 {
-    SmartSocketEndpoint *ss = nextSmartEndpoint();
+    ss = nextSmartEndpoint();
     if (!ss)
         throw createSmartSocketException(0, "smartsocket failed to get nextEndpoint");
 
-    ISocket *sock = NULL;
-    SocketEndpoint ep;
+    ISocket *sock = nullptr;
     try 
     {
         {
@@ -324,12 +297,10 @@ ISmartSocket *CSmartSocketFactory::connect_timeout( unsigned timeoutms)
             sock = ISocket::connect_timeout(ep, timeoutms);
         else
             sock = ISocket::connect(ep);
-
-        return new CSmartSocket(sock, ep, this);
     }
     catch (IException *e)
     {
-        StringBuffer s("CSmartSocketFactory::connect ");
+        StringBuffer s("CSmartSocketFactory::connect_sock ");
         ep.getUrlStr(s);
         EXCLOG(e,s.str());
         ss->status=false;
@@ -337,6 +308,15 @@ ISmartSocket *CSmartSocketFactory::connect_timeout( unsigned timeoutms)
             sock->Release();
         throw;
     }
+    return sock;
+}
+
+ISmartSocket *CSmartSocketFactory::connect_timeout(unsigned timeoutms)
+{
+    SocketEndpoint ep;
+    SmartSocketEndpoint *ss = nullptr;
+    Owned<ISocket> sock = connect_sock(timeoutms, ss, ep);
+    return new CSmartSocket(sock.getClear(), ep, this);
 }
 
 ISmartSocket *CSmartSocketFactory::connect()

system/jlib/jsmartsock.hpp

@@ -21,7 +21,6 @@
 
 #include "jsocket.hpp"
 
-
 interface jlib_decl ISmartSocket : extends IInterface
 {
     // unique to ISmartSocket
@@ -69,6 +68,7 @@ interface jlib_thrown_decl ISmartSocketException : extends IException
 
 jlib_decl ISmartSocketFactory *createSmartSocketFactory(const char *_socklist, bool _retry = false, unsigned _retryInterval = 60, unsigned _dnsInterval = (unsigned) -1);
 
+jlib_decl ISmartSocketException *createSmartSocketException(int errorCode, const char *msg);
 
 #endif
 

system/jlib/jsmartsock.ipp

@@ -84,7 +84,8 @@ public:
 
     ISmartSocket *connect();
 
-    ISmartSocket *connect_timeout( unsigned timeoutms = 0);
+    ISocket *connect_sock(unsigned timeoutms, SmartSocketEndpoint *&ss, SocketEndpoint &ep);
+    virtual ISmartSocket *connect_timeout(unsigned timeoutms = 0);
 
     ISmartSocket *connectNextAvailableSocket();
 
@@ -117,5 +118,28 @@ private:
     StringAttr msg;
 };
 
+class jlib_decl CSmartSocket: public CSimpleInterfaceOf<ISmartSocket>
+{
+public:
+    ISocket *sock;
+    SocketEndpoint ep;
+    ISmartSocketFactory *factory;
+
+    CSmartSocket(ISocket *_sock, SocketEndpoint &_ep, ISmartSocketFactory *_factory);
+    virtual ~CSmartSocket();
+
+    // ISmartSocket
+    virtual ISocket *querySocket() override { return sock; }
+
+    // subset of ISocket
+    virtual void read(void* buf, size32_t min_size, size32_t max_size, size32_t &size_read,
+                        unsigned timeout = WAIT_FOREVER) override;
+    virtual void read(void* buf, size32_t size) override;
+
+    virtual size32_t write(void const* buf, size32_t size) override;
+
+    virtual void close() override;
+};
+
 #endif
 

system/jlib/jsuperhash.hpp

@@ -72,11 +72,12 @@ protected:
     bool             removeExact(void * et);
     inline void      setCache(unsigned v) const { cache = v; }
 
+    inline unsigned  doFind(const void * findParam) const
+      { return doFind(getHashFromFindParam(findParam), findParam); }
+
 private:
     bool             doAdd(void *, bool);
     void             doDeleteElement(unsigned);
-    inline unsigned  doFind(const void * findParam) const
-      { return doFind(getHashFromFindParam(findParam), findParam); }
     unsigned         doFind(unsigned, const void *) const;
     unsigned         doFindElement(unsigned, const void *) const;
     unsigned         doFindNew(unsigned) const;
@@ -530,20 +531,38 @@ public:
         return key;
     }
 
+    inline HashKeyElement *queryCreate(const char *_key, bool &didCreate)
+    {
+        CriticalBlock b(crit);
+        HashKeyElement *key = find(*_key);
+        if (key)
+        {
+            didCreate = false;
+            key->linkCount++;
+        }
+        else
+        {
+            didCreate = true;
+            key = createKeyElement(_key);
+        }
+        return key;
+    }
+
     inline void linkKey(const char *key)
     {
         queryCreate(key);
     }
 
-    inline void releaseKey(HashKeyElement *key)
+    inline bool releaseKey(HashKeyElement *key)
     {
         CriticalBlock b(crit);
         if (0 == key->linkCount)
         {
             verifyex(removeExact(key));
-            return;
+            return true;
         }
         --key->linkCount;
+        return false;
     }
 
 protected:

system/jlib/jutil.cpp

@@ -1490,6 +1490,18 @@ void StringArray::appendListUniq(const char *list, const char *delim)
     DelimToStringArray(list, *this, delim, true);
 }
 
+StringBuffer &StringArray::getString(StringBuffer &ret, const char *delim)
+{
+    ForEachItemIn(i, *this)
+    {
+        const char *v = item(i);
+        ret.append(v);
+        if (i+1 != ordinality())
+            ret.append(delim);
+    }
+    return ret;
+}
+
 void StringArray::sortAscii(bool nocase)
 {
     PARENT::sort(nocase ? CCmp::compareNC : CCmp::compare);
@@ -2366,7 +2378,7 @@ IPropertyTree *getHPCCEnvironment()
         {
             Owned<IFileIO> fileio = file->open(IFOread);
             if (fileio)
-                return createPTree(*fileio);
+                return createPTree(*fileio, ipt_lowmem);
         }
     }
     return NULL;

system/jlib/jutil.hpp

@@ -220,6 +220,7 @@ public:
     void appendList(const char *list, const char *delim);
     // Appends a list in a string delimited by 'delim' without duplicates
     void appendListUniq(const char *list, const char *delim);
+    StringBuffer &getString(StringBuffer &ret, const char *delim); // get CSV string of array contents
     void sortAscii(bool nocase=false);
     void sortAsciiReverse(bool nocase=false);
     void sortCompare(int (*compare)(const char * const * l, const char * const * r));

system/security/securesocket/securesocket.cpp

@@ -58,6 +58,7 @@
 #include <openssl/conf.h>
 #include <openssl/x509v3.h>
 
+#include "jsmartsock.ipp"
 #include "securesocket.hpp"
 
 Owned<ISecureSocketContext> server_securesocket_context;
@@ -164,12 +165,15 @@ public:
     virtual void   read(void* buf, size32_t size)
     {
         size32_t size_read;
-        readTimeout(buf, size, size, size_read, 0, false);
+        // MCK - this was:
+        // readTimeout(buf, size, size, size_read, 0, false);
+        // but that is essentially a non-blocking read() and we want a blocking read() ...
+        readTimeout(buf, 0, size, size_read, WAIT_FOREVER, false);
     }
 
     virtual size32_t get_max_send_size()
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::get_max_send_size: not implemented");
     }
 
     //
@@ -177,7 +181,7 @@ public:
     //
     virtual ISocket* accept(bool allowcancel=false) // not needed for UDP
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::accept: not implemented");
     }
 
     //
@@ -185,7 +189,7 @@ public:
     // 
     virtual int wait_write(unsigned timeout)
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::wait_write: not implemented");
     }
 
     //
@@ -194,14 +198,14 @@ public:
     //
     virtual bool set_nonblock(bool on) // returns old state
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::set_nonblock: not implemented");
     }
 
     // enable 'nagling' - small packet coalescing (implies delayed transmission)
     //
     virtual bool set_nagle(bool on) // returns old state
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::set_nagle: not implemented");
     }
 
 
@@ -209,7 +213,7 @@ public:
     //
     virtual void set_linger(int lingersecs)  
     {
-        throw MakeStringException(-1, "not implemented");
+        m_socket->set_linger(lingersecs);
     }
 
 
@@ -218,7 +222,7 @@ public:
     //
     virtual void  cancel_accept() // not needed for UDP
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::cancel_accept: not implemented");
     }
 
     //
@@ -258,12 +262,12 @@ public:
     //
     virtual bool connectionless() // true if accept need not be called (i.e. UDP)
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::connectionless: not implemented");
     }
 
     virtual void set_return_addr(int port,const char *name) // used for UDP servers only
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::set_return_addr: not implemented");
     }
 
     // Block functions 
@@ -274,7 +278,7 @@ public:
                             unsigned timeout=0 // timeout in msecs (0 for no timeout)
                   ) 
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::set_block_mode: not implemented");
     }
 
 
@@ -284,12 +288,12 @@ public:
                             size32_t sz          // size to send (0 for eof)
                   )
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::send_block: not implemented");
     }
 
     virtual size32_t receive_block_size ()     // get size of next block (always must call receive_block after) 
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::receive_block_size: not implemented");
     }
 
     virtual size32_t receive_block(
@@ -298,7 +302,7 @@ public:
                                                // if less than block size truncates block
                   )
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::receive_block: not implemented");
     }
 
     virtual void  close()
@@ -321,59 +325,59 @@ public:
 
     virtual size32_t write_multiple(unsigned num,const void **buf, size32_t *size)
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::write_multiple: not implemented");
     }
 
     virtual size32_t get_send_buffer_size() // get OS send buffer
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::get_send_buffer_size: not implemented");
     }
 
     void set_send_buffer_size(size32_t sz)  // set OS send buffer size
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::set_send_buffer_size: not implemented");
     }
 
     bool join_multicast_group(SocketEndpoint &ep)   // for udp multicast
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::join_multicast_group: not implemented");
         return false;
     }
 
     bool leave_multicast_group(SocketEndpoint &ep)  // for udp multicast
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::leave_multicast_group: not implemented");
         return false;
     }
 
     void set_ttl(unsigned _ttl)   // set ttl
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::set_ttl: not implemented");
     }
 
     size32_t get_receive_buffer_size()  // get OS send buffer
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::get_receive_buffer_size: not implemented");
     }
 
     void set_receive_buffer_size(size32_t sz)   // set OS send buffer size
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::set_receive_buffer_size: not implemented");
     }
 
     virtual void set_keep_alive(bool set) // set option SO_KEEPALIVE
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::set_keep_alive: not implemented");
     }
 
     virtual size32_t udp_write_to(const SocketEndpoint &ep, void const* buf, size32_t size)
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::udp_write_to: not implemented");
     }
 
     virtual bool check_connection()
     {
-        throw MakeStringException(-1, "not implemented");
+        throw MakeStringException(-1, "CSecureSocket::check_connection: not implemented");
     }
 
     virtual bool isSecure() const override
@@ -1641,3 +1645,39 @@ SECURESOCKET_API int signCertificate(const char* csr, const char* ca_certificate
 }
 
 }
+
+class CSecureSmartSocketFactory : public CSmartSocketFactory
+{
+public:
+    Owned<ISecureSocketContext> secureContext;
+
+    CSecureSmartSocketFactory(const char *_socklist, bool _retry, unsigned _retryInterval, unsigned _dnsInterval) : CSmartSocketFactory(_socklist, _retry, _retryInterval, _dnsInterval)
+    {
+        secureContext.setown(createSecureSocketContext(ClientSocket));
+    }
+
+    virtual ISmartSocket *connect_timeout(unsigned timeoutms) override
+    {
+        SocketEndpoint ep;
+        SmartSocketEndpoint *ss = nullptr;
+        Owned<ISecureSocket> ssock;
+        Owned<ISocket> sock = connect_sock(timeoutms, ss, ep);
+        try
+        {
+            ssock.setown(secureContext->createSecureSocket(sock.getClear()));
+            // secure_connect may also DBGLOG() errors ...
+            ssock->secure_connect();
+        }
+        catch (IException *e)
+        {
+            ss->status = false;
+            throw;
+        }
+        return new CSmartSocket(ssock.getClear(), ep, this);
+    }
+};
+
+ISmartSocketFactory *createSecureSmartSocketFactory(const char *_socklist, bool _retry, unsigned _retryInterval, unsigned _dnsInterval)
+{
+    return new CSecureSmartSocketFactory(_socklist, _retry, _retryInterval, _dnsInterval);
+}

system/security/securesocket/securesocket.hpp

@@ -30,6 +30,7 @@
 
 #include "jsocket.hpp"
 #include "jptree.hpp"
+#include "jsmartsock.hpp"
 
 #define SSLIB "securesocket"
 
@@ -89,5 +90,7 @@ SECURESOCKET_API ICertificate *createCertificate();
 SECURESOCKET_API int signCertificate(const char* csr, const char* ca_certificate, const char* ca_privkey, const char* ca_passphrase, int days, StringBuffer& certificate);
 };
 
+SECURESOCKET_API ISmartSocketFactory *createSecureSmartSocketFactory(const char *_socklist, bool _retry = false, unsigned _retryInterval = 60, unsigned _dnsInterval = (unsigned) -1);
+
 #endif
 

testing/regress/ecl/badcatch.ecl

@@ -0,0 +1,41 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+MyRec := RECORD
+    STRING50 Value1;
+    unsigned Value2;
+END;
+
+MyRec t1(unsigned c) := transform
+  SELF.value1 := 'X';
+  SELF.value2 := c;
+END;
+
+ds := DATASET(100000, t1(COUNTER));
+
+MyRec FailTransform := transform
+  self.value1 := FAILMESSAGE[1..17];
+  self.value2 := FAILCODE
+END;
+
+splitds := nofold(ds(Value1 != 'f'));
+limited := LIMIT(splitds, 2);
+
+recovered := CATCH(limited, SKIP);
+
+count(splitds);
+count(recovered);

testing/regress/ecl/catch3.ecl

@@ -0,0 +1,70 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+
+MyRec := RECORD
+    STRING50 Value1;
+    unsigned Value2;
+END;
+
+ds := DATASET([
+           {'C',1},
+           {'C',2},
+           {'C',3},
+           {'C',4},
+           {'C',5},
+     {'X',1},
+     {'A',1}],MyRec);
+
+MyRec FailTransform := transform
+  self.value1 := FAILMESSAGE[1..17];
+  self.value2 := FAILCODE
+END;
+
+limited := LIMIT(ds, 2);
+
+recovered := CATCH(limited, SKIP);
+
+recovered2 := CATCH(limited, onfail(FailTransform));
+
+recovered3 := CATCH(CATCH(limited, FAIL(1, 'Failed, dude')), onfail(FailTransform));
+
+OUTPUT(recovered);
+OUTPUT(recovered2);
+OUTPUT(recovered3);
+
+// What about exceptions in child queries
+
+MyRec childXform(MyRec l, unsigned lim) := TRANSFORM
+    SELF.value2 := (SORT(LIMIT(ds(value1=l.value1), lim), value2))[1].value2;
+    SELF := l;
+    END;
+
+
+failingChild := project(ds,childxform(LEFT, 2));
+passingChild := project(ds,childxform(LEFT, 20));
+
+output(CATCH(failingChild, onfail(FailTransform)));
+output(CATCH(passingChild, onfail(FailTransform)));
+
+// What about exceptions in dependencies?
+
+Value2Max := MAX(limited, value2);
+
+output(CATCH(ds(value2 = value2Max), onfail(FailTransform)));