Merge pull request #3925 from jakesmith/hpcc-8715

HPCC-8715 - Compress graphs and handle graphs/results more efficiently.

Reviewed-By: Gavin Halliday <gavin.halliday@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -509,8 +509,8 @@ template <>  struct CachedTags<CLocalWUAppValue, IConstWUAppValue>
 
 class CLocalWorkUnit : public CInterface, implements IConstWorkUnit , implements ISDSSubscription, implements IExtendedWUInterface
 {
-    friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str);
-    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags);
+    friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool decodeGraphs);
+    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs);
 
     // NOTE - order is important - we need to construct connection before p and (especially) destruct after p
     Owned<IRemoteConnection> connection;
@@ -558,6 +558,7 @@ public:
     CLocalWorkUnit(IRemoteConnection *_conn, IPropertyTree* root, ISecManager *secmgr, ISecUser *secuser);
     ~CLocalWorkUnit();
     CLocalWorkUnit(const char *dummyWuid, const char *parentWuid, ISecManager *secmgr, ISecUser *secuser);
+    IPropertyTree *getUnpackedTree() const;
 
     ISecManager *querySecMgr(){return secMgr.get();}
     ISecUser *querySecUser(){return secUser.get();}
@@ -1629,15 +1630,18 @@ public:
 
 class CLocalWUGraph : public CInterface, implements IWUGraph
 {
+    const CLocalWorkUnit &owner;
     Owned<IPropertyTree> p;
+    mutable Owned<IPropertyTree> graph; // cached copy of graph xgmml
     mutable Linked<IConstWUGraphProgress> progress;
     StringAttr wuid;
+    unsigned wuidVersion;
 
     void mergeProgress(IPropertyTree &tree, IPropertyTree &progressTree, const unsigned &progressV) const;
 
 public:
     IMPLEMENT_IINTERFACE;
-    CLocalWUGraph(IPropertyTree *p, const char *wuid);
+    CLocalWUGraph(const CLocalWorkUnit &owner, IPropertyTree *p);
 
     virtual IStringVal & getXGMML(IStringVal & ret, bool mergeProgress) const;
     virtual IStringVal & getDOT(IStringVal & ret) const;
@@ -1646,12 +1650,14 @@ public:
     virtual IStringVal & getTypeName(IStringVal & ret) const;
     virtual WUGraphType getType() const;
     virtual IPropertyTree * getXGMMLTree(bool mergeProgress) const;
+    virtual IPropertyTree * getXGMMLTreeRaw() const;
     virtual bool isValid() const;
 
     virtual void setName(const char *str);
+    virtual void setLabel(const char *str);
     virtual void setType(WUGraphType type);
     virtual void setXGMML(const char *str);
-    virtual void setXGMMLTree(IPropertyTree * tree);
+    virtual void setXGMMLTree(IPropertyTree * tree, bool compress=true);
 };
 
 class CLocalWUActivity : public CInterface, implements IWUActivity
@@ -1888,7 +1894,7 @@ public:
     }
 };
 
-#define WUID_VERSION 1 // recorded in each wuid created, useful for bkwd compat. checks
+#define WUID_VERSION 2 // recorded in each wuid created, useful for bkwd compat. checks
 
 class CWorkUnitFactory : public CInterface, implements IWorkUnitFactory, implements IDaliClientShutdown
 {
@@ -2801,7 +2807,7 @@ bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerr
         return false;
 
     StringBuffer buf;
-    exportWorkUnitToXML(this, buf);
+    exportWorkUnitToXML(this, buf, false);
 
     StringBuffer extraWorkUnitXML;
     StringBuffer xpath("/GraphProgress/");
@@ -3028,7 +3034,7 @@ void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
 {
     CriticalBlock block(crit);
     StringBuffer x;
-    tgt.append(exportWorkUnitToXML(this, x).str());
+    tgt.append(exportWorkUnitToXML(this, x, false).str());
 }
 
 void CLocalWorkUnit::deserialize(MemoryBuffer &src)
@@ -5394,7 +5400,7 @@ IWUResult* CLocalWorkUnit::createResult()
     if (!results.length())
         p->addPropTree("Results", createPTree("Results"));
     IPropertyTree *r = p->queryPropTree("Results");
-    IPropertyTree *s = r->addPropTree("Result", createPTreeFromXMLString("<Result fetchEntire='1'/>"));
+    IPropertyTree *s = r->addPropTree("Result", createPTree());
 
     s->Link();
     IWUResult* q = new CLocalWUResult(s); 
@@ -5883,6 +5889,28 @@ bool CLocalWorkUnit::switchThorQueue(const char *cluster, IQueueSwitcher *qs)
 
 //=================================================================================================
 
+IPropertyTree *CLocalWorkUnit::getUnpackedTree() const
+{
+    Owned<IPropertyTree> ret = createPTreeFromIPT(p);
+    Owned<IConstWUGraphIterator> graphIter = &getGraphs(GraphTypeAny);
+    ForEach(*graphIter)
+    {
+        IConstWUGraph &graph  = graphIter->query();
+        Owned<IPropertyTree> graphTree = graph.getXGMMLTree(false);
+        SCMStringBuffer gName;
+        graph.getName(gName);
+        StringBuffer xpath("Graphs/Graph[@name=\"");
+        xpath.append(gName.s).append("\"]/xgmml");
+        IPropertyTree *xgmml = ret->queryPropTree(xpath.str());
+        if (xgmml) // don't know of any reason it shouldn't exist
+        {
+            xgmml->removeProp("graphBin");
+            xgmml->setPropTree("graph", graphTree.getClear());
+        }
+    }
+    return ret.getClear();
+}
+
 void CLocalWorkUnit::loadGraphs() const
 {
     CriticalBlock block(crit);
@@ -5895,18 +5923,15 @@ void CLocalWorkUnit::loadGraphs() const
         }
         else
             cachedGraphs.set(p->queryPropTree("Graphs"));
-        if (cachedGraphs.get()) {
-            Owned<IPropertyTreeIterator> r = cachedGraphs->getElements("Graph");
-            for (r->first(); r->isValid(); r->next())
+        if (cachedGraphs.get())
+        {
+            Owned<IPropertyTreeIterator> iter = cachedGraphs->getElements("Graph");
+            ForEach(*iter)
             {
-                IPropertyTree *rp = &r->query();
-                rp->Link();
-
-                graphs.append(*new CLocalWUGraph(rp, p->queryName()));
+                IPropertyTree &graph = iter->query();
+                graphs.append(*new CLocalWUGraph(*this, LINK(&graph)));
             }
-
         }
-        
     }
 }
 
@@ -5919,8 +5944,12 @@ mapEnums graphTypes[] = {
    { GraphTypeSize,  NULL },
 };
 
-CLocalWUGraph::CLocalWUGraph(IPropertyTree *props, const char *_wuid) : p(props), wuid(_wuid)
+CLocalWUGraph::CLocalWUGraph(const CLocalWorkUnit &_owner, IPropertyTree *props) : p(props), owner(_owner)
 {
+    SCMStringBuffer str;
+    owner.getWuid(str);
+    wuid.set(str.s.str());
+    wuidVersion = owner.getWuidVersion();
 }
 
 IStringVal& CLocalWUGraph::getName(IStringVal &str) const
@@ -5931,9 +5960,17 @@ IStringVal& CLocalWUGraph::getName(IStringVal &str) const
 
 IStringVal& CLocalWUGraph::getLabel(IStringVal &str) const
 {
-    Owned<IPropertyTree> xgmml = getXGMMLTree(false);
-    str.set(xgmml->queryProp("@label"));
-    return str;
+    if (wuidVersion >= 2)
+    {
+        str.set(p->queryProp("@label"));
+        return str;
+    }
+    else
+    {
+        Owned<IPropertyTree> xgmml = getXGMMLTree(false);
+        str.set(xgmml->queryProp("@label"));
+        return str;
+    }
 }
 
 
@@ -6073,7 +6110,7 @@ IConstWUGraphIterator& CLocalWorkUnit::getGraphs(WUGraphType type) const
             }
         public:
             IMPLEMENT_IINTERFACE;
-            CConstWUGraphIterator(IConstWUGraphIterator *_base,WUGraphType _type)   
+            CConstWUGraphIterator(IConstWUGraphIterator *_base,WUGraphType _type)
                 : base(_base)
             {
                 type = _type;
@@ -6086,9 +6123,9 @@ IConstWUGraphIterator& CLocalWorkUnit::getGraphs(WUGraphType type) const
                     return true;
                 return next();
             }
-            bool next() 
+            bool next()
             {
-                while (base->next()) 
+                while (base->next())
                     if (match())
                         return true;
                 return false;
@@ -6134,9 +6171,9 @@ IWUGraph* CLocalWorkUnit::createGraph()
     if (!graphs.length())
         p->addPropTree("Graphs", createPTree("Graphs"));
     IPropertyTree *r = p->queryPropTree("Graphs");
-    IPropertyTree *s = r->addPropTree("Graph", createPTreeFromXMLString("<Graph fetchEntire='1'/>"));
+    IPropertyTree *s = r->addPropTree("Graph", createPTree());
     s->Link();
-    IWUGraph* q = new CLocalWUGraph(s, p->queryName());
+    IWUGraph* q = new CLocalWUGraph(*this, s);
     q->Link();
     graphs.append(*q);
     return q;
@@ -6149,7 +6186,7 @@ IWUGraph * CLocalWorkUnit::updateGraph(const char * name)
     IConstWUGraph *existing = getGraph(name);
     if (existing)
         return (IWUGraph *) existing;
-    IWUGraph * q = createGraph(); 
+    IWUGraph * q = createGraph();
     q->setName(name);
     return q;
 }
@@ -6172,17 +6209,29 @@ void CLocalWUGraph::setName(const char *str)
     progress.setown(new CConstGraphProgress(wuid, str));
 }
 
+void CLocalWUGraph::setLabel(const char *str)
+{
+    p->setProp("@label", str);
+}
+
 void CLocalWUGraph::setXGMML(const char *str)
 {
     setXGMMLTree(createPTreeFromXMLString(str));
 }
 
-void CLocalWUGraph::setXGMMLTree(IPropertyTree *graph)
+void CLocalWUGraph::setXGMMLTree(IPropertyTree *_graph, bool compress)
 {
-    assertex(strcmp(graph->queryName(), "graph")==0);
-    IPropertyTree *xgmml = createPTree("xgmml");
-    xgmml->setPropTree("graph", graph);
-    p->setPropTree("xgmml", xgmml);
+    assertex(strcmp(_graph->queryName(), "graph")==0);
+    IPropertyTree *xgmml = p->setPropTree("xgmml", createPTree());
+    if (compress)
+    {
+        MemoryBuffer mb;
+        _graph->serialize(mb);
+        xgmml->setPropBin("graphBin", mb.length(), mb.toByteArray());
+        graph.setown(_graph);
+    }
+    else
+        xgmml->setPropTree("graph", _graph);
 }
 
 void CLocalWUGraph::mergeProgress(IPropertyTree &rootNode, IPropertyTree &progressTree, const unsigned &progressV) const
@@ -6264,15 +6313,30 @@ void CLocalWUGraph::mergeProgress(IPropertyTree &rootNode, IPropertyTree &progre
         mergeProgress(iter->query(), progressTree, progressV);
 }
 
+IPropertyTree * CLocalWUGraph::getXGMMLTreeRaw() const
+{
+    return p->getPropTree("xgmml");
+}
+
 IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
 {
+    if (!graph)
+    {
+        // NB: although graphBin introduced in wuidVersion==2,
+        // daliadmin can retrospectively compress existing graphs, so need to check for all versions
+        MemoryBuffer mb;
+        if (p->getPropBin("xgmml/graphBin", mb))
+            graph.setown(createPTree(mb));
+        else
+            graph.setown(p->getBranch("xgmml/graph"));
+        if (!graph)
+            return NULL;
+    }
     if (!doMergeProgress)
-        return p->getPropTree("xgmml/graph");
+        return graph.getLink();
     else
     {
-        IPropertyTree *src = p->queryPropTree("xgmml/graph");
-        if (!src) return NULL;
-        Owned<IPropertyTree> copy = createPTreeFromIPT(src);
+        Owned<IPropertyTree> copy = createPTreeFromIPT(graph);
         Owned<IConstWUGraphProgress> _progress;
         if (progress) _progress.set(progress);
         else
@@ -6283,13 +6347,14 @@ IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
         Owned<IPropertyTreeIterator> nodeIterator = copy->getElements("node");
         ForEach (*nodeIterator)
             mergeProgress(nodeIterator->query(), *progressTree, progressV);
-        return LINK(copy);
+        return copy.getClear();
     }
 }
 
 bool CLocalWUGraph::isValid() const
 {
-    return p->hasProp("xgmml/graph/node");
+    // JCSMORE - I can't really see why this is necessary, a graph cannot be empty.
+    return p->hasProp("xgmml/graph/node") || p->hasProp("xgmml/graphBin");
 }
 
 WUGraphType CLocalWUGraph::getType() const
@@ -7833,7 +7898,7 @@ extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit()
     return ret;
 }
 
-extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str)
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack)
 {
     const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
     if (!w)
@@ -7843,20 +7908,27 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
             w = wl->c;
     }
     if (w)
-        toXML(w->p, str, 0, XML_Format|XML_SortTags);
+    {
+        Linked<IPropertyTree> p;
+        if (unpack)
+            p.setown(w->getUnpackedTree());
+        else
+            p.set(w->p);
+        toXML(p, str, 0, XML_Format|XML_SortTags);
+    }
     else
         str.append("Unrecognized workunit format");
     return str;
 }
 
-extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str)
+extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack)
 {
     StringBuffer x;
-    str.set(exportWorkUnitToXML(wu,x).str());
+    str.set(exportWorkUnitToXML(wu,x,unpack).str());
     return str;
 }
 
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags)
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack)
 {
     const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
     if (!w)
@@ -7866,7 +7938,14 @@ extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const
             w = wl->c;
     }
     if (w)
-        saveXML(filename, w->p, 0, XML_Format|XML_SortTags|extraXmlFlags);
+    {
+        Linked<IPropertyTree> p;
+        if (unpack)
+            p.setown(w->getUnpackedTree());
+        else
+            p.set(w->p);
+        saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);
+    }
 }
 
 

common/workunit/workunit.hpp

@@ -207,6 +207,7 @@ interface IConstWUGraph : extends IInterface
     virtual IStringVal & getTypeName(IStringVal & ret) const = 0;
     virtual WUGraphType getType() const = 0;
     virtual IPropertyTree * getXGMMLTree(bool mergeProgress) const = 0;
+    virtual IPropertyTree * getXGMMLTreeRaw() const = 0;
     virtual bool isValid() const = 0;
 };
 
@@ -214,8 +215,9 @@ interface IConstWUGraph : extends IInterface
 interface IWUGraph : extends IConstWUGraph
 {
     virtual void setXGMML(const char * text) = 0;
-    virtual void setXGMMLTree(IPropertyTree * tree) = 0;
+    virtual void setXGMMLTree(IPropertyTree * tree, bool compress=true) = 0;
     virtual void setName(const char * name) = 0;
+    virtual void setLabel(const char * name) = 0;
     virtual void setType(WUGraphType type) = 0;
 };
 
@@ -1168,9 +1170,9 @@ extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory();
 extern WORKUNIT_API IWorkUnitFactory * getSecWorkUnitFactory(ISecManager &secmgr, ISecUser &secuser);
 extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser);
 extern WORKUNIT_API ILocalWorkUnit* createLocalWorkUnit();
-extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str);
-extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str);
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags);
+extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack);
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack);
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password);
 extern WORKUNIT_API void abortWorkUnit(const char *wuid);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser);

dali/daliadmin/CMakeLists.txt

@@ -37,6 +37,7 @@ include_directories (
          ./../../system/include 
          ./../../system/jlib 
          ./../ft 
+         ./../../common/workunit
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )
@@ -48,5 +49,6 @@ target_link_libraries ( daliadmin
          mp 
          remote 
          dalibase 
+         workunit
     )
 

dali/daliadmin/daliadmin.cpp

@@ -43,6 +43,8 @@
 
 #include "rmtfile.hpp"
 
+#include "workunit.hpp"
+
 #ifdef _WIN32
 #include <conio.h>
 #else
@@ -111,6 +113,8 @@ void usage(const char *exe)
   printf("  getxref <destxmlfile>           -- get all XREF information\n");
   printf("  dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
   printf("  unlock <xpath or logicalfile>   --  unlocks either matching xpath(s) or matching logical file(s), can contain wildcards\n");
+  printf("  wuidcompress <wildcard> <type>  --  scan workunits that match <wildcard> and compress resources of <type>\n");
+  printf("  wuiddecompress <wildcard> <type> --  scan workunits that match <wildcard> and decompress resources of <type>\n");
   printf("\n");
   printf("Common options\n");
   printf("  server=<dali-server-ip>         -- server ip\n");
@@ -2059,6 +2063,53 @@ static void unlock(const char *pattern)
     }
 }
 
+static void wuidCompress(const char *match, const char *type, bool compress)
+{
+    if (0 != stricmp("graph", type))
+    {
+        WARNLOG("Currently, only type=='graph' supported.");
+        return;
+    }
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IConstWorkUnitIterator> iter = factory->getWorkUnitsByXPath(match);
+    ForEach(*iter)
+    {
+        IConstWorkUnit &wuid = iter->query();
+
+        StringArray graphNames;
+        Owned<IConstWUGraphIterator> graphIter = &wuid.getGraphs(GraphTypeAny);
+        ForEach(*graphIter)
+        {
+            SCMStringBuffer graphName;
+            IConstWUGraph &graph = graphIter->query();
+            Owned<IPropertyTree> xgmml = graph.getXGMMLTreeRaw();
+            if (compress != xgmml->hasProp("graphBin"))
+            {
+                graph.getName(graphName);
+                graphNames.append(graphName.s.str());
+            }
+        }
+
+        if (graphNames.ordinality())
+        {
+            SCMStringBuffer wuidName;
+            wuid.getWuid(wuidName);
+            StringAttr msg;
+            msg.set(compress?"Compressing":"Uncompressing");
+            PROGLOG("%s graphs for workunit: %s", msg.get(), wuidName.s.str());
+            Owned<IWorkUnit> wWuid = &wuid.lock();
+            ForEachItemIn(n, graphNames)
+            {
+                Owned<IWUGraph> wGraph = wWuid->updateGraph(graphNames.item(n));
+                PROGLOG("%s graph: %s", msg.get(), graphNames.item(n));
+                // get/set - will convert to/from new format (binary compress blob)
+                Owned<IPropertyTree> xgmml = wGraph->getXGMMLTree(false);
+                wGraph->setXGMMLTree(xgmml.getClear(), compress);
+            }
+        }
+    }
+}
+
 
 //=============================================================================
 
@@ -2388,7 +2439,14 @@ int main(int argc, char* argv[])
                 CHECKPARAMS(1,1);
                 unlock(params.item(1));
             }
-
+            else if (stricmp(cmd,"wuidCompress")==0) {
+                CHECKPARAMS(2,2);
+                wuidCompress(params.item(1), params.item(2), true);
+            }
+            else if (stricmp(cmd,"wuidDecompress")==0) {
+                CHECKPARAMS(2,2);
+                wuidCompress(params.item(1), params.item(2), false);
+            }
             else 
                 ERRLOG("Unknown command %s",cmd);
         }

ecl/eclcc/eclcc.cpp

@@ -1240,7 +1240,7 @@ void EclCC::generateOutput(EclCompileInstance & instance)
         else
             xmlFilename.append(DEFAULT_OUTPUTNAME);
         xmlFilename.append(".xml");
-        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0);
+        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0, true);
     }
 }
 
@@ -1759,7 +1759,7 @@ void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
             dbglogTransformStats(true);
             if (info.wu &&
                 (info.wu->getDebugValueBool("generatePartialOutputOnError", false) || info.errs->errCount() == 0))
-                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64);
+                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64, true);
         }
     }
     catch (IException * e)

ecl/hqlcpp/hqlcpp.ipp

@@ -474,11 +474,14 @@ typedef CIArrayOf<BuildCtx> BuildCtxArray;
 struct GeneratedGraphInfo : public CInterface
 {
 public:
-    GeneratedGraphInfo(const char * _name, IPropertyTree * _graph) : name(_name), graph(_graph) {}
+    GeneratedGraphInfo(const char * _name, const char *_label) : name(_name), label(_label)
+    {
+        xgmml.setown(createPTree("graph"));
+    }
 
 public:
-    StringAttr name;
-    Linked<IPropertyTree> graph;
+    StringAttr name, label;
+    Owned<IPropertyTree> xgmml;
 };
 
 enum SubGraphType { SubGraphRoot, SubGraphRemote, SubGraphChild, SubGraphLoop };
@@ -1833,7 +1836,7 @@ public:
     //MORE: At some point the global getUniqueId() should be killed so there are only local references.
     inline unsigned __int64 getUniqueId() { return ::getUniqueId(); } //{ return ++nextUid; }
     inline StringBuffer & getUniqueId(StringBuffer & target) { return appendUniqueId(target, getUniqueId()); }
-    inline unsigned curGraphSequence() const { return activeGraphName ? graphSeqNumber : 0; }
+    inline unsigned curGraphSequence() const { return activeGraph ? graphSeqNumber : 0; }
 
 public:
     void traceExpression(const char * title, IHqlExpression * expr, unsigned level=500);
@@ -1866,12 +1869,11 @@ protected:
     ClusterType         targetClusterType;
     bool contextAvailable;
     unsigned maxSequence;
-    StringAttr          activeGraphName;
     unsigned            startCursorSet;
     bool                requireTable;
     BuildCtx *          activeGraphCtx;
     HqlExprArray        metas;
-    Owned<IPropertyTree> graph;
+    Owned<GeneratedGraphInfo> activeGraph;
     unsigned            graphSeqNumber;
     StringAttr          graphLabel;
     NlpParseContext *   nlpParse;               // Not linked so it can try and stay opaque.

ecl/hqlcpp/hqlecl.cpp

@@ -320,7 +320,7 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
 void HqlDllGenerator::addWorkUnitAsResource()
 {
     SCMStringBuffer wuXML;
-    exportWorkUnitToXML(wu, wuXML);
+    exportWorkUnitToXML(wu, wuXML, false);
     code->addCompressResource("WORKUNIT", wuXML.length(), wuXML.str(), NULL, 1000);
 }
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -805,7 +805,7 @@ protected:
 
 void HqlCppTranslator::optimizeBuildActionList(BuildCtx & ctx, IHqlExpression * exprs)
 {
-    if ((exprs->getOperator() != no_actionlist) || !graph)
+    if ((exprs->getOperator() != no_actionlist) || !activeGraph)
     {
         buildStmt(ctx, exprs);
         return;
@@ -5597,8 +5597,9 @@ bool HqlCppTranslator::buildCpp(IHqlCppInstance & _code, HqlQueryContext & query
         {
             GeneratedGraphInfo & cur = graphs.item(i2);
             Owned<IWUGraph> wug = wu()->updateGraph(cur.name);
-            wug->setXGMMLTree(cur.graph.getClear());
+            wug->setXGMMLTree(cur.xgmml.getClear());
             wug->setType(GraphTypeActivities);
+            wug->setLabel(cur.label);
         }
 
         code->processIncludes();
@@ -6834,7 +6835,7 @@ void HqlCppTranslator::addDependency(BuildCtx & ctx, ABoundActivity * element, A
 #if 0
     StringBuffer edgeText;
     edgeText.append("edge[@id=\").append(idText).append("\"]");
-    if (graph->hasProp(edgePath))
+    if (activeGraph->xgmml->hasProp(edgePath))
         return;
 #endif
 
@@ -6878,7 +6879,7 @@ void HqlCppTranslator::addDependency(BuildCtx & ctx, ABoundActivity * element, A
 
     addGraphAttributeInt(edge, "_sourceActivity", sourceActivity->queryActivityId());
     addGraphAttributeInt(edge, "_targetActivity", sinkActivity->queryActivityId());
-    graph->addPropTree("edge", edge);
+    activeGraph->xgmml->addPropTree("edge", edge);
 }
 
 void HqlCppTranslator::buildClearRecord(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * record, int direction)
@@ -8749,7 +8750,7 @@ unsigned HqlCppTranslator::doBuildThorChildSubGraph(BuildCtx & ctx, IHqlExpressi
             graphId = activeSubgraph->graphId;
     }
     else
-        node = graph->addPropTree("node", node);
+        node = activeGraph->xgmml->addPropTree("node", node);
 
     node->setPropInt("@id", thisId);
 
@@ -8810,7 +8811,7 @@ unsigned HqlCppTranslator::doBuildThorSubGraph(BuildCtx & ctx, IHqlExpression *
     BuildCtx graphctx(ctx);
     graphctx.addGroup();
 
-    bool needToCreateGraph = !graph;
+    bool needToCreateGraph = !activeGraph;
     if (!graphTag && outputLibraryId)
         graphTag = outputLibraryId;
     if (needToCreateGraph)
@@ -8827,7 +8828,7 @@ unsigned HqlCppTranslator::doBuildThorSubGraph(BuildCtx & ctx, IHqlExpression *
 
 void HqlCppTranslator::beginGraph(const char * _graphName)
 {
-    if (activeGraphName)
+    if (activeGraph)
         throwError(HQLERR_NestedThorNodes);
 
     graphSeqNumber++;
@@ -8836,30 +8837,23 @@ void HqlCppTranslator::beginGraph(const char * _graphName)
         graphName.append("graph").append(graphSeqNumber);
     else
         graphName.append(_graphName);
-    activeGraphName.set(graphName.str());
 
-    graph.setown(createPTree("graph"));
-    if (graphLabel)
-    {
-        graph->setProp("@label", graphLabel);
-        graphLabel.clear();
-    }
+    activeGraph.setown(new GeneratedGraphInfo(graphName, graphLabel));
+    graphLabel.clear();
+
     if (insideLibrary())
-        graph->setPropBool("@library", true);
+        activeGraph->xgmml->setPropBool("@library", true);
 }
 
 
 void HqlCppTranslator::endGraph()
 {
-    graphs.append(* new GeneratedGraphInfo(activeGraphName, graph));
-    graph.clear();
-    activeGraphName.set(NULL);
+    graphs.append(*activeGraph.getClear());
 }
 
 void HqlCppTranslator::clearGraph()
 {
-    graph.clear();
-    activeGraphName.clear();
+    activeGraph.clear();
 }
 
 
@@ -9029,7 +9023,7 @@ void HqlCppTranslator::doBuildThorGraph(BuildCtx & ctx, IHqlExpression * expr)
             Owned<SubGraphInfo> graphInfo;
             if (graphTag)
             {
-                graphInfo.setown(new SubGraphInfo(graph, 0, 0, graphTag, SubGraphRoot));
+                graphInfo.setown(new SubGraphInfo(activeGraph->xgmml, 0, 0, graphTag, SubGraphRoot));
                 graphctx.associate(*graphInfo);
             }
 
@@ -9040,7 +9034,7 @@ void HqlCppTranslator::doBuildThorGraph(BuildCtx & ctx, IHqlExpression * expr)
             graphctx.removeAssociation(graphInfo);
 
             HqlExprArray args;
-            args.append(*createConstant(activeGraphName));
+            args.append(*createConstant(activeGraph->name));
             args.append(*createConstant(targetThor()));
             args.append(*createConstant(0));
             args.append(*createValue(no_nullptr, makeReferenceModifier(makeRowType(queryNullRecord()->getType()))));
@@ -17520,7 +17514,7 @@ void HqlCppTranslator::buildActivityFramework(ActivityInstance * instance)
 
 void HqlCppTranslator::buildActivityFramework(ActivityInstance * instance, bool alwaysExecuted)
 {
-    instance->createGraphNode(graph, alwaysExecuted);
+    instance->createGraphNode(activeGraph->xgmml, alwaysExecuted);
     if (options.trackDuplicateActivities)
     {
         IHqlExpression * search = instance->dataset;

ecl/wutest/wutest.cpp

@@ -70,7 +70,7 @@ bool dump(IConstWorkUnit &w, IProperties *globals)
     else if (stricmp(action, "dump")==0)
     {
         SCMStringBuffer xml;
-        exportWorkUnitToXML(&w, xml);
+        exportWorkUnitToXML(&w, xml, true);
         printf("%s\n", xml.str());
     }
     else if (stricmp(action, "temporaries")==0)

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1874,7 +1874,7 @@ void WsWuInfo::getWorkunitXml(const char* plainText, MemoryBuffer& buf)
         header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><?xml-stylesheet href=\"../esp/xslt/xmlformatter.xsl\" type=\"text/xsl\"?>";
 
     SCMStringBuffer xml;
-    exportWorkUnitToXML(cw, xml);
+    exportWorkUnitToXML(cw, xml, true);
 
     buf.append(strlen(header), header);
     buf.append(xml.length(), xml.str());

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3125,7 +3125,7 @@ bool CWsWorkunitsEx::onWUExport(IEspContext &context, IEspWUExportRequest &req,
         {
             Owned<IConstWorkUnit> cw = factory->openWorkUnit(it->c_str(), false);
             if (cw)
-                exportWorkUnitToXML(cw, xml);
+                exportWorkUnitToXML(cw, xml, true);
         }
         xml.append("</Workunits>");