Merge pull request #6516 from ghalliday/issue12369

HPCC-12369 Fix stats for graphs by naming the graph consistently

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclagent/agentctx.hpp

@@ -91,7 +91,7 @@ struct IAgentContext : extends IGlobalCodeContext
     virtual IRecordLayoutTranslatorCache * queryRecordLayoutTranslatorCache() const = 0;
     virtual void addWuException(const char * text, unsigned code, unsigned severity, char const * source) = 0;
 
-    virtual IHThorGraphResults * executeLibraryGraph(const char * libraryName, unsigned expectedInterfaceHash, unsigned activityId, bool embedded, const byte * parentExtract) = 0;
+    virtual IHThorGraphResults * executeLibraryGraph(const char * libraryName, unsigned expectedInterfaceHash, unsigned activityId, const char * embeddedGraphName, const byte * parentExtract) = 0;
     virtual bool getWorkunitResultFilename(StringBuffer & diskFilename, const char * wuid, const char * name, int seq) = 0;
     virtual IHThorGraphResults * createGraphLoopResults() = 0;
     virtual void outputFormattedResult(const char *name, unsigned sequence, bool close) = 0;

ecl/eclagent/eclagent.cpp

@@ -1655,13 +1655,13 @@ EclAgentQueryLibrary * EclAgent::queryEclLibrary(const char * libraryName, unsig
     return NULL;
 }
 
-EclAgentQueryLibrary * EclAgent::loadEclLibrary(const char * libraryName, unsigned expectedInterfaceHash, bool embedded)
+EclAgentQueryLibrary * EclAgent::loadEclLibrary(const char * libraryName, unsigned expectedInterfaceHash, const char * embeddedGraphName)
 {
     Linked<EclAgentQueryLibrary> library = queryEclLibrary(libraryName, expectedInterfaceHash);
     if (!library)
     {
         const char * graphName = "graph1";
-        if (embedded)
+        if (embeddedGraphName)
         {
             library.setown(new EclAgentQueryLibrary);
             library->wu.set(queryWorkUnit());
@@ -1669,7 +1669,7 @@ EclAgentQueryLibrary * EclAgent::loadEclLibrary(const char * libraryName, unsign
             library->name.set(libraryName);
             queryLibraries.append(*LINK(library));
 
-            graphName = libraryName;
+            graphName = embeddedGraphName;
         }
         else
         {

ecl/eclagent/eclagent.ipp

@@ -206,9 +206,9 @@ public:
     {
         ctx->addWuException(text, code, severity, source);
     }
-    virtual IHThorGraphResults * executeLibraryGraph(const char * libraryName, unsigned expectedInterfaceHash, unsigned activityId, bool embedded, const byte * parentExtract)
+    virtual IHThorGraphResults * executeLibraryGraph(const char * libraryName, unsigned expectedInterfaceHash, unsigned activityId, const char * embeddedGraphName, const byte * parentExtract)
     {
-        return ctx->executeLibraryGraph(libraryName, expectedInterfaceHash, activityId, embedded, parentExtract);
+        return ctx->executeLibraryGraph(libraryName, expectedInterfaceHash, activityId, embeddedGraphName, parentExtract);
     }
     virtual bool getWorkunitResultFilename(StringBuffer & diskFilename, const char * wuid, const char * name, int seq)
     {
@@ -403,7 +403,7 @@ private:
     ILoadedDllEntry * loadWorkUnitDll(IConstWorkUnit * wu);
     IConstWorkUnit * resolveLibrary(const char * libraryName, unsigned expectedInterfaceHash);
     EclAgentQueryLibrary * queryEclLibrary(const char * libraryName, unsigned expectedInterfaceHash);
-    EclAgentQueryLibrary * loadEclLibrary(const char * libraryName, unsigned expectedInterfaceHash, bool embedded);
+    EclAgentQueryLibrary * loadEclLibrary(const char * libraryName, unsigned expectedInterfaceHash, const char * embeddedGraphName);
     virtual bool getWorkunitResultFilename(StringBuffer & diskFilename, const char * wuid, const char * name, int seq);
     virtual IDebuggableContext *queryDebugContext() const { return debugContext; };
 
@@ -528,7 +528,7 @@ public:
 
     virtual void executeThorGraph(const char * graphName);
     virtual void executeGraph(const char * graphName, bool realThor, size32_t parentExtractSize, const void * parentExtract);
-    virtual IHThorGraphResults * executeLibraryGraph(const char * libraryName, unsigned expectedInterfaceHash, unsigned activityId, bool embedded, const byte * parentExtract);
+    virtual IHThorGraphResults * executeLibraryGraph(const char * libraryName, unsigned expectedInterfaceHash, unsigned activityId, const char * embeddedGraphName, const byte * parentExtract);
     virtual IThorChildGraph * resolveChildQuery(__int64 subgraphId, IHThorArg * colocal);
     virtual IEclGraphResults * resolveLocalQuery(__int64 activityId);
 

ecl/eclagent/eclgraph.cpp

@@ -1286,6 +1286,10 @@ EclSubGraph * EclGraph::idToGraph(unsigned id)
 
 void EclGraph::updateLibraryProgress()
 {
+    //Check for old format embedded graph names, and don't update the stats if not the correct format
+    if (!MATCHES_CONST_PREFIX(queryGraphName(), GraphScopePrefix))
+        return;
+
     Owned<IConstWUGraphProgress> graphProgress = getGraphProgress();
     ForEachItemIn(idx, graphs)
     {
@@ -1820,13 +1824,13 @@ void EclAgent::executeGraph(const char * graphName, bool realThor, size32_t pare
     }
 }
 
-IHThorGraphResults * EclAgent::executeLibraryGraph(const char * libraryName, unsigned expectedInterfaceHash, unsigned activityId, bool embedded, const byte * parentExtract)
+IHThorGraphResults * EclAgent::executeLibraryGraph(const char * libraryName, unsigned expectedInterfaceHash, unsigned activityId, const char * embeddedGraphName, const byte * parentExtract)
 {
     Linked<EclGraph> savedGraph = activeGraph.get();
 
     try
     {
-        EclAgentQueryLibrary * library = loadEclLibrary(libraryName, expectedInterfaceHash, embedded);
+        EclAgentQueryLibrary * library = loadEclLibrary(libraryName, expectedInterfaceHash, embeddedGraphName);
 
         Owned<IHThorGraphResults> libraryResults = new GraphResults;
 

ecl/hqlcpp/hqlcpp.ipp

@@ -1055,7 +1055,7 @@ public:
     }
 
     void updateClusterType();
-    bool buildCode(HqlQueryContext & query, const char * embeddedLibraryName, bool isEmbeddedLibrary);
+    bool buildCode(HqlQueryContext & query, const char * embeddedLibraryName, const char * embeddedGraphName);
 
     inline StringBuffer & generateExprCpp(StringBuffer & out, IHqlExpression * expr)
     {

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1824,6 +1824,14 @@ void ActivityInstance::addAttributeBool(const char * name, bool value)
     addGraphAttributeBool(graphNode, name, value);
 }
 
+void ActivityInstance::addAttribute(const char * name, IHqlExpression * expr)
+{
+    StringBuffer temp;
+    IValue * value = expr->queryValue();
+    assertex(value);
+    value->getStringValue(temp);
+    addGraphAttribute(graphNode, name, temp);
+}
 
 void ActivityInstance::addLocationAttribute(IHqlExpression * location)
 {
@@ -5734,15 +5742,15 @@ void dumpActivityCounts()
 
 
 
-bool HqlCppTranslator::buildCode(HqlQueryContext & query, const char * embeddedLibraryName, bool isEmbeddedLibrary)
+bool HqlCppTranslator::buildCode(HqlQueryContext & query, const char * embeddedLibraryName, const char * embeddedGraphName)
 {
     cycle_t startCycles = get_cycles_now();
     WorkflowArray workflow;
-    bool ok = prepareToGenerate(query, workflow, isEmbeddedLibrary);
+    bool ok = prepareToGenerate(query, workflow, (embeddedLibraryName != NULL));
     if (ok)
     {
         //This is done late so that pickBestEngine has decided which engine we are definitely targeting.
-        if (!isEmbeddedLibrary)
+        if (!embeddedLibraryName)
             updateClusterType();
 
         noteFinishedTiming("compile:tree transform", startCycles);
@@ -5778,7 +5786,7 @@ bool HqlCppTranslator::buildCode(HqlQueryContext & query, const char * embeddedL
                 }
 
                 BuildCtx ctx(*code, goAtom);
-                buildLibraryGraph(ctx, graph, embeddedLibraryName);
+                buildLibraryGraph(ctx, graph, embeddedGraphName);
             }
             else
                 buildWorkflow(workflow);
@@ -5829,10 +5837,15 @@ bool HqlCppTranslator::buildCpp(IHqlCppInstance & _code, HqlQueryContext & query
 
             StringBuffer internalLibraryName;
             name->queryValue()->getStringValue(internalLibraryName);
+            //Use a graph number that couldn't possibly clash with any graphs generated by the main query
+            StringBuffer embeddedGraphName;
+            embeddedGraphName.append("graph").append(EMBEDDED_GRAPH_DELTA+i+1);
+
             overrideOptionsForLibrary();
             HqlQueryContext libraryQuery;
             libraryQuery.expr.set(definition);
-            if (!buildCode(libraryQuery, internalLibraryName.str(), true))
+
+            if (!buildCode(libraryQuery, internalLibraryName.str(), embeddedGraphName.str()))
                 return false;
         }
 
@@ -5840,7 +5853,7 @@ bool HqlCppTranslator::buildCpp(IHqlCppInstance & _code, HqlQueryContext & query
             overrideOptionsForLibrary();
         else
             overrideOptionsForQuery();
-        if (!buildCode(query, NULL, false))
+        if (!buildCode(query, NULL, NULL))
             return false;
 
         //Return early if iteratively generating the field usage statistics

ecl/hqlcpp/hqlhtcpp.ipp

@@ -146,6 +146,7 @@ public:
     void buildMetaMember();
 
     void addAttribute(const char * name, const char * value);
+    void addAttribute(const char * name, IHqlExpression * expr);
     void addAttributeInt(const char * name, __int64 value);
     void addAttributeBool(const char * name, bool value);
     void addLocationAttribute(IHqlExpression * location);

ecl/hqlcpp/hqllib.cpp

@@ -306,14 +306,18 @@ public:
                 if (internalExpr)
                 {
                     IHqlExpression * nameExpr = moduleExpr->queryAttribute(nameAtom);
-                    if (!matchedInternalLibraries.contains(*nameExpr))
+                    unsigned sequence = matchedInternalLibraries.find(*nameExpr);
+                    if (sequence == NotFound)
                     {
+                        sequence = internalLibraries.ordinality();
                         internalLibraries.append(*transformEmbeddedLibrary(scopeFunc));
                         matchedInternalLibraries.append(*LINK(nameExpr));
                     }
                     //remove the parameter from the internal attribute from the module that is being called.
                     //so save in subsequent translation
-                    OwnedHqlExpr newModuleExpr = replaceOwnedAttribute(moduleExpr, createAttribute(internalAtom));
+                    VStringBuffer graphName("graph%u", EMBEDDED_GRAPH_DELTA+(sequence+1));
+                    OwnedHqlExpr noInternalExpr = removeAttribute(moduleExpr, internalAtom);
+                    OwnedHqlExpr newModuleExpr = appendOwnedOperand(noInternalExpr, createAttribute(embeddedAtom, createConstant(graphName)));
                     OwnedHqlExpr newScopeFunc = replaceChild(scopeFunc, 0, newModuleExpr);
                     HqlExprArray instanceArgs;
                     unwindChildren(instanceArgs, expr);
@@ -517,6 +521,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityLibraryInstance(BuildCtx & ctx
     IHqlExpression * module = moduleFunction->queryChild(0);
     assertex(module->getOperator() == no_libraryscope);
     IHqlExpression * nameAttr = module->queryAttribute(nameAtom);
+    IHqlExpression * embeddedAttr = module->queryAttribute(embeddedAtom);
     OwnedHqlExpr name = foldHqlExpression(nameAttr->queryChild(0));
     IValue * nameValue = name->queryValue();
     IHqlExpression * originalName = queryAttributeChild(module, _original_Atom, 0);
@@ -554,9 +559,12 @@ ABoundActivity * HqlCppTranslator::doBuildActivityLibraryInstance(BuildCtx & ctx
         Owned<IWULibrary> wulib = wu()->updateLibraryByName(libraryName.str());
     }
 
+
     instance->addAttributeInt("_interfaceHash", library->getInterfaceHash());
-    instance->addAttributeBool("embedded", module->hasAttribute(internalAtom));
+    instance->addAttributeBool("embedded", (embeddedAttr != NULL));
     instance->addAttributeInt("_maxOutputs", library->outputs.ordinality());
+    if (embeddedAttr)
+        instance->addAttribute("graph", embeddedAttr->queryChild(0));
     if (!targetHThor())
         instance->addAttributeInt("_graphid", nextActivityId());            // reserve an id...
 

ecl/hqlcpp/hqllib.ipp

@@ -20,6 +20,8 @@
 #include "hqlhtcpp.ipp"
 #include "hqlecl.hpp"
 
+#define EMBEDDED_GRAPH_DELTA 1000000000
+
 class HqlCppLibrary : public CInterface
 {
 public:

ecl/hthor/hthor.cpp

@@ -9554,7 +9554,13 @@ CHThorLibraryCallActivity::CHThorLibraryCallActivity(IAgentContext &_agent, unsi
 {
     libraryName.set(node->queryProp("att[@name=\"libname\"]/@value"));
     interfaceHash = node->getPropInt("att[@name=\"_interfaceHash\"]/@value", 0);
-    embedded = node->getPropBool("att[@name=\"embedded\"]/@value", false) ;
+    bool embedded = node->getPropBool("att[@name=\"embedded\"]/@value", false) ;
+    if (embedded)
+    {
+        embeddedGraphName.set(node->queryProp("att[@name=\"graph\"]/@value"));
+        if (!embeddedGraphName)
+            embeddedGraphName.set(libraryName);
+    }
 
     Owned<IPropertyTreeIterator> iter = node->getElements("att[@name=\"_outputUsed\"]");
     ForEach(*iter)
@@ -9576,7 +9582,7 @@ IHThorGraphResult * CHThorLibraryCallActivity::getResultRows(unsigned whichOutpu
         if (libraryName.length() == 0)
             libraryName.setown(helper.getLibraryName());
         helper.createParentExtract(extractBuilder);
-        results.setown(agent.executeLibraryGraph(libraryName, interfaceHash, activityId, embedded, extractBuilder.getbytes()));
+        results.setown(agent.executeLibraryGraph(libraryName, interfaceHash, activityId, embeddedGraphName, extractBuilder.getbytes()));
     }
 
     return results->queryResult(whichOutput);

ecl/hthor/hthor.ipp

@@ -2691,8 +2691,8 @@ class CHThorLibraryCallActivity : public CHThorSimpleActivityBase
     Owned<IHThorGraphResults> results;
     ActivityState state;
     StringAttr libraryName;
+    StringAttr embeddedGraphName;
     unsigned interfaceHash;
-    bool embedded;
 
     CIArrayOf<LibraryCallOutput> outputs;
     Owned<IHThorBoundLoopGraph> libraryGraph;

roxie/ccd/ccdcontext.cpp

@@ -1379,7 +1379,7 @@ public:
     {
         if (extra.embedded)
         {
-            return factory->lookupGraph(extra.libraryName, probeManager, *this, parentActivity);
+            return factory->lookupGraph(extra.embeddedGraphName, probeManager, *this, parentActivity);
         }
         else
         {

roxie/ccd/ccdquery.cpp

@@ -804,6 +804,12 @@ protected:
                 extra.libraryName.set(node.queryProp("att[@name=\"libname\"]/@value"));
                 extra.interfaceHash = node.getPropInt("att[@name=\"_interfaceHash\"]/@value", 0);
                 extra.embedded = node.getPropBool("att[@name=\"embedded\"]/@value", false) ;
+                if (extra.embedded)
+                {
+                    extra.embeddedGraphName.set(node.queryProp("att[@name=\"graph\"]/@value"));
+                    if (!extra.embeddedGraphName)
+                        extra.embeddedGraphName.set(extra.libraryName);
+                }
 
                 Owned<IPropertyTreeIterator> iter = node.getElements("att[@name=\"_outputUsed\"]");
                 ForEach(*iter)

roxie/ccd/ccdserver.hpp

@@ -79,6 +79,7 @@ public:
     unsigned graphid;
 
     StringAttr libraryName;
+    StringAttr embeddedGraphName;
     unsigned interfaceHash;
     bool embedded;
 };