Merge pull request #12226 from ghalliday/issue21553

HPCC-21553 Reduce number of calls that traverse subgraph stats information

Reviewed-By: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -2911,8 +2911,6 @@ void WuScopeFilter::finishedFilter()
             if (!(properties & (PTattributes|PThints)))
                 sourceFlags &= ~(SSFsearchGraph|SSFsearchWorkflow);
         }
-
-
     }
 
     //Optimize sources if they haven't been explicitly specified
@@ -2931,22 +2929,11 @@ void WuScopeFilter::finishedFilter()
         if (!(properties & (PTattributes|PThints)))
             sourceFlags &= ~(SSFsearchGraph);
 
-        setDepth(1, 1);    // MORE - will become 2 once scopes are modified to include the workflow id.
+        setDepth(2, 2);
     }
     else if (matchOnly(SSTsubgraph))
     {
         sourceFlags &= ~(SSFsearchWorkflow);
-
-        //subgraph timings are stored globally - otherwise need to look in the graph stats.
-        if (!(properties & (PTattributes|PThints)))
-        {
-            if (desiredStats.ordinality() == 1)
-            {
-                StatisticKind stat = (StatisticKind)desiredStats.item(0);
-                if (stat == StTimeElapsed || stat == StWhenStarted)
-                    sourceFlags &= ~(SSFsearchGraphStats|SSFsearchGraph);
-            }
-        }
     }
     else if (matchOnly(SSTcompilestage))
     {
@@ -2959,10 +2946,32 @@ void WuScopeFilter::finishedFilter()
         sourceFlags &= ~(SSFsearchGlobalStats|SSFsearchWorkflow);
     }
 
-    // Everything stored in the graphs stats has a depth of 2 or more - so ignore if there are not matches in that depth
-    if ((include.nestedDepth == 0) && (scopeFilter.compareDepth(2) > 0))
+    // Everything stored in the graphs stats has a depth of 3 or more - so ignore if there are not matches in that depth
+    if (include.nestedDepth == 0)
     {
-        sourceFlags &= ~(SSFsearchGraphStats);
+        if (scopeFilter.compareDepth(3) > 0)    // 3 is larger than any scope in the filter
+            sourceFlags &= ~(SSFsearchGraphStats);
+        else if (scopeFilter.compareDepth(3) == 0)    // 3 matches the maximum scope in the filter
+        {
+            //Subgraph WhenStarted and TimeElapsed are stored globally.  If that is all that is required
+            //then do not include the subgraph timings
+            if (desiredStats.ordinality())
+            {
+                bool needGraphStats = false;
+                ForEachItemIn(i, desiredStats)
+                {
+                    unsigned stat = desiredStats.item(i);
+                    if ((stat != StWhenStarted) && (stat != StTimeElapsed))
+                        needGraphStats = true;
+                }
+                if (!needGraphStats)
+                {
+                    sourceFlags &= ~(SSFsearchGraphStats);
+                    if (matchOnly(SSTsubgraph))
+                        sourceFlags &= SSFsearchGlobalStats;
+                }
+            }
+        }
     }
 
     if (scopeFilter.compareDepth(1) < 0)

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -1556,7 +1556,6 @@ bool CWsWorkunitsEx::onWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEsp
             {
                 //The access is checked here because getArchivedWUInfo() has its own access check.
                 ensureWsWorkunitAccess(context, wuid.str(), SecAccess_Read);
-                PROGLOG("WUInfo: %s", wuid.str());
 
                 unsigned long flags=0;
                 if (req.getTruncateEclTo64k())
@@ -1596,6 +1595,8 @@ bool CWsWorkunitsEx::onWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEsp
                 if (req.getIncludeTotalClusterTime())
                     flags|=WUINFO_IncludeTotalClusterTime;
 
+                PROGLOG("WUInfo: %s %lx", wuid.str(), flags);
+
                 WsWuInfo winfo(context, wuid.str());
                 winfo.getInfo(resp.updateWorkunit(), flags);
 

esp/src/src/Timings.ts

@@ -33,17 +33,17 @@ export class Timings {
         .baseUrl("")
         .request({
             ScopeFilter: {
-                MaxDepth: 2,
+                MaxDepth: 3,
                 ScopeTypes: []
             },
             NestedFilter: {
-                Depth: 999999,
+                Depth: 0,
                 ScopeTypes: []
             },
             PropertiesToReturn: {
-                AllProperties: true,
+                AllProperties: false,
                 AllStatistics: true,
-                AllHints: true,
+                AllHints: false,
                 Properties: ["WhenStarted", "TimeElapsed"]
             },
             ScopeOptions: {
@@ -156,13 +156,13 @@ export class Timings {
                     ScopeTypes: []
                 },
                 NestedFilter: {
-                    Depth: 999999,
+                    Depth: 0,
                     ScopeTypes: []
                 },
                 PropertiesToReturn: {
-                    AllProperties: true,
+                    AllProperties: false,
                     AllStatistics: true,
-                    AllHints: true,
+                    AllHints: false,
                     Properties: []
                 },
                 ScopeOptions: {