HPCC-12490 Add getGraphsMeta() to explicitly get the meta info

getGraphsMeta retrieves the workunits graph meta data (name,
type, label, state) without getting the graph data.

The state comes from the graph progress, all graphs being
requested now, use a single connection to pull the state from
the progress. Previously it connected to each graph progress
node independently in series. No graph progress data is pulled
in the process.

The create graph process is changes slightly, to explicitly
pass in the xgmml and an odd 'IConstWUGraph::isValid()' is
removed, which had the negative effect of pulling a lot of the
graph data early.

Signed-off-by: Jake Smith <jake.smith@lexisnexis.com>

common/workunit/workunit.cpp

@@ -248,6 +248,13 @@ public:
         formatVersion = 0;
         progress = NULL;
     }
+    CConstGraphProgress(const char *_wuid, const char *_graphName, IPropertyTree *allProgress) : wuid(_wuid), graphName(_graphName)
+    {
+        progress = allProgress->queryPropTree(graphName);
+        formatVersion = progress->getPropInt("@format");
+        connectedWrite = false; // should never be
+        connected = true;
+    }
     void connect()
     {
         conn.clear();
@@ -261,6 +268,8 @@ public:
     void lockWrite()
     {
         if (connectedWrite) return;
+        if (!rootPath.length())
+            throw MakeStringException(WUERR_GraphProgressWriteUnsupported, "Writing to graph progress unsupported in this context");
         // JCSMORE - look at using changeMode here.
         if (conn)
             conn.clear();
@@ -603,6 +612,7 @@ public:
     virtual unsigned getTimerCount() const;
     virtual unsigned getApplicationValueCount() const;
     virtual IConstWUGraphIterator & getGraphs(WUGraphType type) const;
+    virtual IConstWUGraphMetaIterator & getGraphsMeta(WUGraphType type) const;
     virtual IConstWUGraph * getGraph(const char *name) const;
     virtual IConstWUGraphProgress * getGraphProgress(const char * name) const;
     virtual IStringVal & getJobName(IStringVal & str) const;
@@ -730,6 +740,7 @@ public:
     void deschedule();
     unsigned addLocalFileUpload(LocalFileUploadType type, char const * source, char const * destination, char const * eventTag);
     IWUResult * updateGlobalByName(const char * name);
+    IWUGraph * createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml);
     IWUGraph * updateGraph(const char * name);
     IWUQuery * updateQuery();
     IWUWebServicesInfo* updateWebServicesInfo(bool create);
@@ -974,6 +985,8 @@ public:
             { return c->getApplicationValueCount(); }
     virtual IConstWUGraphIterator & getGraphs(WUGraphType type) const
             { return c->getGraphs(type); }
+    virtual IConstWUGraphMetaIterator & getGraphsMeta(WUGraphType type) const
+            { return c->getGraphsMeta(type); }
     virtual IConstWUGraph * getGraph(const char *name) const
             { return c->getGraph(name); }
     virtual IConstWUGraphProgress * getGraphProgress(const char * name) const
@@ -1205,6 +1218,8 @@ public:
             { return c->addLocalFileUpload(type, source, destination, eventTag); }
     virtual IWUResult * updateGlobalByName(const char * name)
             { return c->updateGlobalByName(name); }
+    virtual IWUGraph * createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml)
+            { return c->createGraph(name, type, xgmml); }
     virtual IWUGraph * updateGraph(const char * name) 
             { return c->updateGraph(name); }
     virtual IWUQuery * updateQuery()
@@ -1651,9 +1666,9 @@ public:
     virtual IStringVal & getLabel(IStringVal & ret) const;
     virtual IStringVal & getTypeName(IStringVal & ret) const;
     virtual WUGraphType getType() const;
+    virtual WUGraphState getState() const;
     virtual IPropertyTree * getXGMMLTree(bool mergeProgress) const;
     virtual IPropertyTree * getXGMMLTreeRaw() const;
-    virtual bool isValid() const;
 
     virtual void setName(const char *str);
     virtual void setLabel(const char *str);
@@ -6475,6 +6490,14 @@ IStringVal& CLocalWUGraph::getLabel(IStringVal &str) const
     }
 }
 
+WUGraphState CLocalWUGraph::getState() const
+{
+    Owned<IConstWUGraphProgress> graphProgress = owner.getGraphProgress(p->queryProp("@name"));
+    if (!graphProgress)
+        return WUGraphUnknown;
+    return graphProgress->queryGraphState();
+}
+
 
 IStringVal& CLocalWUGraph::getXGMML(IStringVal &str, bool mergeProgress) const
 {
@@ -6596,6 +6619,112 @@ void CLocalWorkUnit::setHash(unsigned __int64 hash)
     p->setPropInt64("@hash", hash);
 }
 
+IConstWUGraphMetaIterator& CLocalWorkUnit::getGraphsMeta(WUGraphType type) const
+{
+    /* NB: this method should be 'cheap', loadGraphs() creates IConstWUGraph interfaces to the graphs
+     * it does not actually pull the graph data. We only use IConstWUGraphMeta here, which never probes the xgmml
+     * This method also connects to the graph progress (/GraphProgress/<wuid>) using a single connection, in order
+     * to get state information, it does not pull all the progress data.
+     */
+
+    CriticalBlock block(crit);
+    loadGraphs();
+
+    class CConstWUGraphMetaIterator: public CInterface, implements IConstWUGraphMetaIterator, implements IConstWUGraphMeta
+    {
+        StringAttr wuid;
+        WUGraphType type;
+        Owned<IConstWUGraphIterator> graphIter;
+        IConstWUGraph *curGraph;
+        Owned<IConstWUGraphProgress> curGraphProgress;
+        Linked<IPropertyTree> graphProgress;
+        Owned<IRemoteConnection> progressConn;
+        bool match()
+        {
+            return (GraphTypeAny == type) || (type == graphIter->query().getType());
+        }
+
+        void setCurrent(IConstWUGraph &graph)
+        {
+            curGraph = &graph;
+            SCMStringBuffer graphName;
+            curGraph->getName(graphName);
+            if (progressConn->queryRoot())
+                curGraphProgress.setown(new CConstGraphProgress(wuid, graphName.str(), progressConn->queryRoot()));
+        }
+    public:
+        IMPLEMENT_IINTERFACE;
+        CConstWUGraphMetaIterator(const char *_wuid, IConstWUGraphIterator *_graphIter, WUGraphType _type)
+            : wuid(_wuid), graphIter(_graphIter), type(_type)
+        {
+            curGraph = NULL;
+            StringBuffer progressPath;
+            progressPath.append("/GraphProgress/").append(wuid);
+            progressConn.setown(querySDS().connect(progressPath.str(), myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT));
+        }
+        virtual bool first()
+        {
+            curGraph = NULL;
+            curGraphProgress.clear();
+            if (!graphIter->first())
+                return false;
+            if (match())
+            {
+                setCurrent(graphIter->query());
+                return true;
+            }
+            return next();
+        }
+        virtual bool next()
+        {
+            while (graphIter->next())
+            {
+                if (match())
+                {
+                    setCurrent(graphIter->query());
+                    return true;
+                }
+            }
+            curGraph = NULL;
+            curGraphProgress.clear();
+            return false;
+        }
+        virtual bool isValid()
+        {
+            return NULL != curGraph;
+        }
+        virtual IConstWUGraphMeta & query()
+        {
+            return *this;
+        }
+        // IConstWUGraphMeta
+        virtual IStringVal & getName(IStringVal & ret) const
+        {
+            return curGraph->getName(ret);
+        }
+        virtual IStringVal & getLabel(IStringVal & ret) const
+        {
+            return curGraph->getLabel(ret);
+        }
+        virtual IStringVal & getTypeName(IStringVal & ret) const
+        {
+            return curGraph->getTypeName(ret);
+        }
+        virtual WUGraphType getType() const
+        {
+            return curGraph->getType();
+        }
+        virtual WUGraphState getState() const
+        {
+            if (!curGraphProgress)
+                return WUGraphUnknown;
+            return curGraphProgress->queryGraphState();
+        }
+    };
+    IConstWUGraphIterator *graphIter = new CArrayIteratorOf<IConstWUGraph,IConstWUGraphIterator> (graphs, 0, (IConstWorkUnit *) this);
+    return * new CConstWUGraphMetaIterator(p->queryName(), graphIter, type);
+}
+
 IConstWUGraphIterator& CLocalWorkUnit::getGraphs(WUGraphType type) const
 {
     CriticalBlock block(crit);
@@ -6670,7 +6799,7 @@ IWUGraph* CLocalWorkUnit::createGraph()
     CriticalBlock block(crit);
     ensureGraphsUnpacked();
     loadGraphs();
-    if (!graphs.length())
+    if (!graphs.ordinality())
         p->addPropTree("Graphs", createPTree("Graphs"));
     IPropertyTree *r = p->queryPropTree("Graphs");
     IPropertyTree *s = r->addPropTree("Graph", createPTree());
@@ -6681,18 +6810,33 @@ IWUGraph* CLocalWorkUnit::createGraph()
     return q;
 }
 
-IWUGraph * CLocalWorkUnit::updateGraph(const char * name)
+IWUGraph * CLocalWorkUnit::createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml)
 {
     CriticalBlock block(crit);
     ensureGraphsUnpacked();
-    IConstWUGraph *existing = getGraph(name);
+    Linked<IConstWUGraph> existing = getGraph(name);
     if (existing)
-        return (IWUGraph *) existing;
-    IWUGraph * q = createGraph();
+        throwUnexpected();
+
+    if (!graphs.length())
+        p->addPropTree("Graphs", createPTree("Graphs"));
+    IPropertyTree *r = p->queryPropTree("Graphs");
+    IPropertyTree *s = r->addPropTree("Graph", createPTree());
+    IWUGraph* q = new CLocalWUGraph(*this, LINK(s));
+    graphs.append(*LINK(q));
     q->setName(name);
+    q->setXGMMLTree(xgmml);
+    q->setType(type);
     return q;
 }
 
+IWUGraph * CLocalWorkUnit::updateGraph(const char * name)
+{
+    CriticalBlock block(crit);
+    ensureGraphsUnpacked();
+    return (IWUGraph *)getGraph(name);
+}
+
 IConstWUGraphProgress *CLocalWorkUnit::getGraphProgress(const char *name) const
 {
     CriticalBlock block(crit);
@@ -6853,12 +6997,6 @@ IPropertyTree * CLocalWUGraph::getXGMMLTree(bool doMergeProgress) const
     }
 }
 
-bool CLocalWUGraph::isValid() const
-{
-    // JCSMORE - I can't really see why this is necessary, a graph cannot be empty.
-    return p->hasProp("xgmml/graph/node") || p->hasProp("xgmml/graphBin");
-}
-
 WUGraphType CLocalWUGraph::getType() const
 {
     return (WUGraphType) getEnum(p, "@type", graphTypes);

common/workunit/workunit.hpp

@@ -198,20 +198,33 @@ interface IXmlToRawTransformer;
 interface IPropertyTree;
 interface IPropertyTreeIterator;
 
-interface IConstWUGraph : extends IInterface
+
+enum WUGraphState
+{
+    WUGraphUnknown = 0,
+    WUGraphComplete = 1,
+    WUGraphRunning = 2,
+    WUGraphFailed = 3,
+    WUGraphPaused = 4
+};
+
+interface IConstWUGraphMeta : extends IInterface
 {
-    virtual IStringVal & getXGMML(IStringVal & ret, bool mergeProgress) const = 0;
-    virtual IStringVal & getDOT(IStringVal & ret) const = 0;
     virtual IStringVal & getName(IStringVal & ret) const = 0;
     virtual IStringVal & getLabel(IStringVal & ret) const = 0;
     virtual IStringVal & getTypeName(IStringVal & ret) const = 0;
     virtual WUGraphType getType() const = 0;
+    virtual WUGraphState getState() const = 0;
+};
+
+interface IConstWUGraph : extends IConstWUGraphMeta
+{
+    virtual IStringVal & getXGMML(IStringVal & ret, bool mergeProgress) const = 0;
+    virtual IStringVal & getDOT(IStringVal & ret) const = 0;
     virtual IPropertyTree * getXGMMLTree(bool mergeProgress) const = 0;
     virtual IPropertyTree * getXGMMLTreeRaw() const = 0;
-    virtual bool isValid() const = 0;
 };
 
-
 interface IWUGraph : extends IConstWUGraph
 {
     virtual void setXGMML(const char * text) = 0;
@@ -227,6 +240,11 @@ interface IConstWUGraphIterator : extends IScmIterator
     virtual IConstWUGraph & query() = 0;
 };
 
+interface IConstWUGraphMetaIterator : extends IScmIterator
+{
+    virtual IConstWUGraphMeta & query() = 0;
+};
+
 
 //! IWUResult
 enum
@@ -737,16 +755,6 @@ enum WUSubscribeOptions
 
 interface IWUGraphProgress;
 interface IPropertyTree;
-enum WUGraphState
-{
-    WUGraphUnknown = 0,
-    WUGraphComplete = 1,
-    WUGraphRunning = 2,
-    WUGraphFailed = 3,
-    WUGraphPaused = 4
-};
-
-
 
 enum WUFileKind
 {
@@ -849,6 +857,7 @@ interface IConstWorkUnit : extends IInterface
     virtual unsigned getExceptionCount() const = 0;
     virtual IConstWUExceptionIterator & getExceptions() const = 0;
     virtual IConstWUResult * getGlobalByName(const char * name) const = 0;
+    virtual IConstWUGraphMetaIterator & getGraphsMeta(WUGraphType type) const = 0;
     virtual IConstWUGraphIterator & getGraphs(WUGraphType type) const = 0;
     virtual IConstWUGraph * getGraph(const char * name) const = 0;
     virtual IConstWUGraphProgress * getGraphProgress(const char * name) const = 0;
@@ -975,6 +984,7 @@ interface IWorkUnit : extends IConstWorkUnit
     virtual void deschedule() = 0;
     virtual unsigned addLocalFileUpload(LocalFileUploadType type, const char * source, const char * destination, const char * eventTag) = 0;
     virtual IWUResult * updateGlobalByName(const char * name) = 0;
+    virtual IWUGraph * createGraph(const char * name, WUGraphType type, IPropertyTree *xgmml) = 0;
     virtual IWUGraph * updateGraph(const char * name) = 0;
     virtual IWUQuery * updateQuery() = 0;
     virtual IWUWebServicesInfo * updateWebServicesInfo(bool create) = 0;

common/workunit/wuerror.hpp

@@ -47,6 +47,7 @@
 #define WUERR_PackageAlreadyExists              5022
 #define WUERR_MismatchClusterType               5023
 #define WUERR_InvalidDll                        5024
-#define WUERR_WorkunitPublished                 5005
+#define WUERR_WorkunitPublished                 5005 // that's bizarre!
+#define WUERR_GraphProgressWriteUnsupported     5025
 
 #endif

ecl/hqlcpp/hqlgraph.cpp

@@ -260,9 +260,7 @@ void LogicalGraphCreator::createLogicalGraph(HqlExprArray & exprs)
         createRootGraphActivity(&exprs.item(i));
 //  endSubGraph();
 
-    Owned<IWUGraph> wug = wu->updateGraph("Logical");
-    wug->setXGMMLTree(graph.getClear());
-    wug->setType(GraphTypeEcl);
+    Owned<IWUGraph> wug = wu->createGraph("Logical", GraphTypeEcl, graph.getClear());
 }
 
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5666,9 +5666,7 @@ bool HqlCppTranslator::buildCpp(IHqlCppInstance & _code, HqlQueryContext & query
         ForEachItemIn(i2, graphs)
         {
             GeneratedGraphInfo & cur = graphs.item(i2);
-            Owned<IWUGraph> wug = wu()->updateGraph(cur.name);
-            wug->setXGMMLTree(cur.xgmml.getClear());
-            wug->setType(GraphTypeActivities);
+            Owned<IWUGraph> wug = wu()->createGraph(cur.name, GraphTypeActivities, cur.xgmml.getClear());
             wug->setLabel(cur.label);
         }
 

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -707,40 +707,31 @@ void WsWuInfo::getGraphInfo(IEspECLWorkunit &info, unsigned flags)
         bool running = (!(st==WUStateFailed || st==WUStateAborted || st==WUStateCompleted) && cw->getRunningGraph(runningGraph,id));
 
         IArrayOf<IEspECLGraph> graphs;
-        Owned<IConstWUGraphIterator> it = &cw->getGraphs(GraphTypeAny);
+        Owned<IConstWUGraphMetaIterator> it = &cw->getGraphsMeta(GraphTypeAny);
         ForEach(*it)
         {
-            IConstWUGraph &graph = it->query();
-            if(!graph.isValid())
-                continue;
+            IConstWUGraphMeta &graph = it->query();
 
             SCMStringBuffer name, label, type;
             graph.getName(name);
             graph.getLabel(label);
 
             graph.getTypeName(type);
+            WUGraphState graphState = graph.getState();
 
             Owned<IEspECLGraph> g= createECLGraph("","");
             g->setName(name.str());
             g->setLabel(label.str());
             g->setType(type.str());
-            if(running && strcmp(name.str(),runningGraph.str())==0)
+            if (WUGraphComplete == graphState)
+                g->setComplete(true);
+            else if (running && (WUGraphRunning == graphState))
             {
                 g->setRunning(true);
                 g->setRunningId(id);
             }
-
-            Owned<IConstWUGraphProgress> progress = cw->getGraphProgress(name.str());
-            if (progress)
-            {
-                WUGraphState graphstate= progress->queryGraphState();
-                if (graphstate == WUGraphComplete)
-                    g->setComplete(true);
-                if (version > 1.13 && graphstate == WUGraphFailed)
-                {
-                    g->setFailed(true);
-                }
-            }
+            else if (version > 1.13 && (WUGraphFailed == graphState))
+                g->setFailed(true);
             graphs.append(*g.getLink());
         }
         info.setGraphs(graphs);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3496,63 +3496,68 @@ bool CWsWorkunitsEx::onWUGetGraph(IEspContext& context, IEspWUGetGraphRequest& r
 
         WUGraphIDType id;
         SCMStringBuffer runningGraph;
-        bool running= (isRunning(*cw) && cw->getRunningGraph(runningGraph,id));
+        bool running = (isRunning(*cw) && cw->getRunningGraph(runningGraph,id));
 
         IArrayOf<IEspECLGraphEx> graphs;
-        Owned<IConstWUGraphIterator> it = &cw->getGraphs(GraphTypeAny);
-        ForEach(*it)
-        {
-            IConstWUGraph &graph = it->query();
-            if(!graph.isValid())
-                continue;
 
+        Owned<IConstWUGraphIterator> it;
+        IConstWUGraph *graph = NULL;
+        if (isEmpty(req.getGraphName())) // JCS->GS - is this really required??
+        {
+            it.setown(&cw->getGraphs(GraphTypeAny));
+            if (it->first())
+                graph = &it->query();
+        }
+        else
+            graph = cw->getGraph(req.getGraphName());
+        while (graph)
+        {
             SCMStringBuffer name, label, type;
-            graph.getName(name);
-            graph.getLabel(label);
-            graph.getTypeName(type);
+            graph->getName(name);
+            graph->getLabel(label);
+            graph->getTypeName(type);
 
-            if(isEmpty(req.getGraphName()) || strieq(name.str(), req.getGraphName()))
+            Owned<IEspECLGraphEx> g = createECLGraphEx("","");
+            g->setName(name.str());
+            g->setLabel(label.str());
+            g->setType(type.str());
+            WUGraphState graphState = graph->getState();
+
+            if (running && (WUGraphRunning == graphState))
             {
-                Owned<IEspECLGraphEx> g = createECLGraphEx("","");
-                g->setName(name.str());
-                g->setLabel(label.str());
-                g->setType(type.str());
-                if(running && streq(name.str(), runningGraph.str()))
-                {
-                    g->setRunning(true);
-                    g->setRunningId(id);
-                }
+                g->setRunning(true);
+                g->setRunningId(id);
+            }
+            else if (context.getClientVersion() > 1.20)
+            {
+                if (WUGraphComplete == graphState)
+                    g->setComplete(true);
+                else if (WUGraphFailed == graphState)
+                    g->setFailed(true);
+            }
 
-                Owned<IPropertyTree> xgmml = graph.getXGMMLTree(true);
+            Owned<IPropertyTree> xgmml = graph->getXGMMLTree(true);
 
-                // New functionality, if a subgraph id is specified and we only want to load the xgmml for that subgraph
-                // then we need to conditionally pull a propertytree from the xgmml graph one and use that for the xgmml.
+            // New functionality, if a subgraph id is specified and we only want to load the xgmml for that subgraph
+            // then we need to conditionally pull a propertytree from the xgmml graph one and use that for the xgmml.
 
-                StringBuffer xml;
-                if (notEmpty(req.getSubGraphId()))
-                {
-                    VStringBuffer xpath("//node[@id='%s']", req.getSubGraphId());
-                    toXML(xgmml->queryPropTree(xpath.str()), xml);
-                }
-                else
-                    toXML(xgmml, xml);
+            //JCSMORE this should be part of the API and therefore allow *only* the subtree to be pulled from the backend.
 
-                g->setGraph(xml.str());
-
-                if (context.getClientVersion() > 1.20)
-                {
-                    Owned<IConstWUGraphProgress> progress = cw->getGraphProgress(name.str());
-                    if (progress)
-                    {
-                        WUGraphState graphstate= progress->queryGraphState();
-                        if (graphstate == WUGraphComplete)
-                            g->setComplete(true);
-                        else if (graphstate == WUGraphFailed)
-                            g->setFailed(true);
-                    }
-                }
-                graphs.append(*g.getClear());
+            StringBuffer xml;
+            if (notEmpty(req.getSubGraphId()))
+            {
+                VStringBuffer xpath("//node[@id='%s']", req.getSubGraphId());
+                toXML(xgmml->queryPropTree(xpath.str()), xml);
             }
+            else
+                toXML(xgmml, xml);
+
+            g->setGraph(xml.str());
+
+            graphs.append(*g.getClear());
+            if (!it || !it->next())
+                break;
+            graph = &it->query();
         }
         resp.setGraphs(graphs);
     }

esp/smc/SMCLib/WUXMLInfo.cpp

@@ -178,31 +178,29 @@ bool CWUXMLInfo::buildXmlWuidInfo(IConstWorkUnit &wu, StringBuffer& wuStructure,
 bool CWUXMLInfo::buildXmlGraphList(IConstWorkUnit &wu,IPropertyTree& XMLStructure)
 {
     try {
-      SCMStringBuffer buf;
+        SCMStringBuffer buf;
 
         IPropertyTree* resultTree = XMLStructure.addPropTree("WUGraphs", createPTree(ipt_caseInsensitive));
         Owned<IConstWUGraphIterator> graphs = &wu.getGraphs(GraphTypeAny);
         ForEach(*graphs)
         {
             IConstWUGraph &graph = graphs->query();
-            if(!graph.isValid())
-                continue;
-            IPropertyTree * p   =   resultTree->addPropTree("WUGraph", createPTree(ipt_caseInsensitive));
-            p->setProp("Wuid",      wu.getWuid(buf).str());
-         buf.clear();
-            p->setProp("Name",      graph.getName(buf).str());
-         buf.clear();
-         // MORE? (debugging)
-            p->setPropInt("Running",    (((wu.getState() == WUStateRunning)||(wu.getState() == WUStateDebugPaused)||(wu.getState() == WUStateDebugRunning)) ? 1 : 0));
+            IPropertyTree * p = resultTree->addPropTree("WUGraph", createPTree(ipt_caseInsensitive));
+            p->setProp("Wuid", wu.getWuid(buf).str());
+            buf.clear();
+            p->setProp("Name", graph.getName(buf).str());
+            buf.clear();
+            // MORE? (debugging)
+            p->setPropInt("Running", (((wu.getState() == WUStateRunning)||(wu.getState() == WUStateDebugPaused)||(wu.getState() == WUStateDebugRunning)) ? 1 : 0));
         }
     }
-    catch(IException* e){   
-      StringBuffer msg;
-      e->errorMessage(msg);
+    catch(IException* e) {
+        StringBuffer msg;
+        e->errorMessage(msg);
         WARNLOG("%s", msg.str());
         e->Release();
     }
-    catch(...){
+    catch(...) {
         WARNLOG("Unknown Exception caught within CWUXMLInfo::buildXmlGraphList");
     }
     return true;