HPCC-12646 Improve stats filtering and remove duplicate elapsed time

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/workunit/workunit.cpp

@@ -568,6 +568,37 @@ class ExtractedStatistic : public CInterfaceOf<IConstWUStatistic>
 public:
     virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const
     {
+        if (!description && createDefault)
+        {
+            switch (kind)
+            {
+            case StTimeElapsed:
+                {
+                    if (scopeType != SSTsubgraph)
+                        break;
+                    //Create a default description for a root subgraph
+                    const char * colon = strchr(scope, ':');
+                    if (!colon)
+                        break;
+
+                    const char * subgraph = colon+1;
+                    //Check for nested subgraph
+                    if (strchr(subgraph, ':'))
+                        break;
+
+                    assertex(strncmp(subgraph, SubGraphScopePrefix, strlen(SubGraphScopePrefix)) == 0);
+                    StringAttr graphname;
+                    graphname.set(scope, colon - scope);
+                    unsigned subId = atoi(subgraph + strlen(SubGraphScopePrefix));
+
+                    StringBuffer desc;
+                    formatGraphTimerLabel(desc, graphname, 0, subId);
+                    str.set(desc);
+                    return str;
+                }
+            }
+        }
+
         str.set(description);
         return str;
     }
@@ -821,11 +852,20 @@ protected:
             IStatisticCollection * curCollection = &collections.tos();
             if (childIterators.ordinality() < collections.ordinality())
             {
-                //Start iterating the children for the current collection
-                childIterators.append(curCollection->getScopes(NULL));
-                if (!childIterators.tos().first())
+                if (!filter || filter->recurseChildScopes(curStat->scopeType, curStat->scope))
+                {
+                    //Start iterating the children for the current collection
+                    childIterators.append(curCollection->getScopes(NULL));
+                    if (!childIterators.tos().first())
+                    {
+                        finishCollection();
+                        continue;
+                    }
+                }
+                else
                 {
-                    finishCollection();
+                    //Don't walk the child scopes
+                    collections.pop();
                     continue;
                 }
             }
@@ -6091,8 +6131,6 @@ IConstWUStatisticIterator& CLocalWorkUnit::getStatistics(const IStatisticsFilter
 
     statistics.load(p,"Statistics/*");
     Owned<IConstWUStatisticIterator> localStats = new WorkUnitStatisticsIterator(statistics, 0, (IConstWorkUnit *) this, filter);
-    if (filter && !filter->queryMergeSources())
-        return *localStats.getClear();
 
     const char * wuid = p->queryName();
     Owned<IConstWUStatisticIterator> graphStats = new CConstGraphProgressStatisticsIterator(wuid, filter);

ecl/eclagent/eclgraph.cpp

@@ -989,19 +989,6 @@ void EclSubGraph::execute(const byte * parentExtract)
     {
         updateProgress();
         cleanupActivities();
-
-        {
-            unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now()-startGraphCycles);
-
-            Owned<IWorkUnit> wu(agent->updateWorkUnit());
-            StringBuffer timerText;
-            formatGraphTimerLabel(timerText, parent.queryGraphName(), seqNo+1, id);
-
-            //graphn: id
-            StringBuffer wuScope;
-            formatGraphTimerScope(wuScope, parent.queryGraphName(), seqNo+1, id);
-            updateWorkunitTimeStat(wu, SSTsubgraph, wuScope, StTimeElapsed, timerText.str(), elapsed);
-        }
     }
     agent->updateWULogfile();//Update workunit logfile name in case of rollover
 }

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5877,7 +5877,7 @@ public:
 
     virtual void report(const char * scope, const char * description, const __int64 totaltime, const __int64 maxtime, const unsigned count)
     {
-        StatisticScopeType scopeType = SSTsection; // MORE?
+        StatisticScopeType scopeType = SSTcompilestage;
         StatisticKind kind = StTimeElapsed;
         wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, description, totaltime, count, maxtime, StatsMergeReplace);
     }

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -380,7 +380,6 @@ void WsWuInfo::getTimers(IEspECLWorkunit &info, unsigned flags)
 
         IArrayOf<IEspECLTimer> timers;
         StatisticsFilter filter;
-        filter.setMergeSources(false);
         filter.setScopeDepth(1, 2);
         filter.setMeasure(SMeasureTimeNs);
         Owned<IConstWUStatisticIterator> it = &cw->getStatistics(&filter);
@@ -448,7 +447,6 @@ unsigned WsWuInfo::getTimerCount()
     {
         //This filter must match the filter in the function above, otherwise it will be inconsistent
         StatisticsFilter filter;
-        filter.setMergeSources(false);
         filter.setScopeDepth(1, 2);
         filter.setMeasure(SMeasureTimeNs);
         Owned<IConstWUStatisticIterator> it = &cw->getStatistics(&filter);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -4267,8 +4267,6 @@ bool CWsWorkunitsEx::onWUGetStats(IEspContext &context, IEspWUGetStatsRequest &r
             filter.setScopeDepth(req.getMinScopeDepth(), req.getMaxScopeDepth());
         else if (!req.getMinScopeDepth_isNull())
             filter.setScopeDepth(req.getMinScopeDepth());
-        if (!req.getIncludeGraphs_isNull())
-            filter.setMergeSources(req.getIncludeGraphs());
 
         bool createDescriptions = false;
         if (!req.getCreateDescriptions_isNull())

plugins/workunitservices/workunitservices.cpp

@@ -723,7 +723,6 @@ WORKUNITSERVICES_API void wsWorkunitTimings( ICodeContext *ctx, size32_t & __len
     if (wu)
     {
         StatisticsFilter filter;
-        filter.setMergeSources(false);
         filter.setScopeDepth(1, 2);
         filter.setMeasure(SMeasureTimeNs);
 
@@ -823,7 +822,7 @@ WORKUNITSERVICES_API IRowStream * wsWorkunitStatistics( ICodeContext *ctx, IEngi
     //Filter needs to be allocated because the iterator outlasts it.
     Owned<StatisticsFilter> filter = new StatisticsFilter(filterText);
     if (!includeActivities)
-        filter->setMergeSources(false);
+        filter->setScopeDepth(1, 2);
     Owned<IConstWUStatisticIterator> stats = &wu->getStatistics(filter);
     return new StreamedStatistics(wu, allocator, stats);
 }

system/jlib/jstats.cpp

@@ -1526,6 +1526,16 @@ bool ScopedItemFilter::match(const char * search) const
 }
 
 
+bool ScopedItemFilter::recurseChildScopes(const char * curScope) const
+{
+    if (maxDepth == 0 || !curScope)
+        return true;
+
+    if (queryStatisticsDepth(curScope) >= maxDepth)
+        return false;
+    return true;
+}
+
 void ScopedItemFilter::set(const char * _value)
 {
     if (_value && *_value && !streq(_value, "*") )
@@ -1602,7 +1612,6 @@ StatisticsFilter::StatisticsFilter(StatisticCreatorType _creatorType, const char
 
 void StatisticsFilter::init()
 {
-    mergeSources = true;
     creatorType = SCTall;
     scopeType = SSTall;
     measure = SMeasureAll;
@@ -1626,6 +1635,12 @@ bool StatisticsFilter::matches(StatisticCreatorType curCreatorType, const char *
     return true;
 }
 
+bool StatisticsFilter::recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const
+{
+    return scopeFilter.recurseChildScopes(curScope);
+}
+
+
 void StatisticsFilter::set(const char * creatorTypeText, const char * scopeTypeText, const char * kindText)
 {
     StatisticCreatorType creatorType = queryCreatorType(creatorTypeText);
@@ -1693,11 +1708,6 @@ void StatisticsFilter::setMeasure(StatisticMeasure _measure)
     measure = _measure;
 }
 
-void StatisticsFilter::setMergeSources(bool _value)
-{
-    mergeSources = _value;
-}
-
 void StatisticsFilter::setKind(StatisticKind _kind)
 {
     kind = _kind;

system/jlib/jstats.h

@@ -286,8 +286,7 @@ interface IStatisticsFilter : public IInterface
 {
 public:
     virtual bool matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind) const = 0;
-    //These are a bit arbitrary...
-    virtual bool queryMergeSources() const = 0;
+    virtual bool recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const = 0;
     virtual const char * queryScope() const = 0;
 
 };
@@ -345,6 +344,7 @@ public:
 
     bool match(const char * search) const;
     bool matchDepth(unsigned low, unsigned high) const;
+    bool recurseChildScopes(const char * curScope) const;
 
     const char * queryValue() const { return value ? value.get() : "*"; }
 
@@ -370,7 +370,7 @@ public:
     StatisticsFilter(StatisticCreatorType _creatorType, const char * _creator, StatisticScopeType _scopeType, const char * _scope, StatisticMeasure _measure, StatisticKind _kind);
 
     virtual bool matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind) const;
-    virtual bool queryMergeSources() const { return mergeSources && scopeFilter.matchDepth(2,0); }
+    virtual bool recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const;
     virtual const char * queryScope() const { return scopeFilter.queryValue(); }
 
     void set(const char * _creatorTypeText, const char * _scopeTypeText, const char * _kindText);
@@ -387,7 +387,6 @@ public:
     void setKind(StatisticKind _kind);
     void setKind(const char * _kind);
     void setMeasure(StatisticMeasure _measure);
-    void setMergeSources(bool _value);      // set to false for legacy timing semantics
 
 protected:
     void init();
@@ -399,7 +398,6 @@ protected:
     StatisticKind kind;
     ScopedItemFilter creatorFilter;
     ScopedItemFilter scopeFilter;
-    bool mergeSources;
 };
 
 //---------------------------------------------------------------------------------------------------------------------