Merge pull request #15242 from richardkchapman/hpcc26339

HPCC-26339 Suppress misleading Roxie global stats

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>
Merged-by: Gavin Halliday <ghalliday@hpccsystems.com>

roxie/ccd/ccd.hpp

@@ -552,6 +552,8 @@ public:
 extern void putStatsValue(IPropertyTree *node, const char *statName, const char *statType, unsigned __int64 val);
 extern void putStatsValue(StringBuffer &reply, const char *statName, const char *statType, unsigned __int64 val);
 
+extern const StatisticsMapping globalStatistics;
+
 class ContextLogger : implements IRoxieContextLogger, public CInterface
 {
 protected:
@@ -576,7 +578,7 @@ private:
 public:
     IMPLEMENT_IINTERFACE;
 
-    ContextLogger() : stats(allStatistics)
+    ContextLogger() : stats(globalStatistics, true)
     {
         ctxTraceLevel = traceLevel;
         intercept = false;

roxie/ccd/ccdserver.cpp

@@ -406,6 +406,25 @@ static const StatisticsMapping groupStatistics({ StNumGroups, StNumGroupMax }, a
 static const StatisticsMapping sortStatistics({ StTimeSortElapsed }, actStatistics);
 static const StatisticsMapping indexWriteStatistics({ StNumDuplicateKeys }, actStatistics);
 
+// These ones get accumulated and reported in COMPLETE: line (and workunit). Excludes ones that are not sensible to sum across activities
+
+extern const StatisticsMapping globalStatistics({StWhenFirstRow, StTimeLocalExecute, StSizeMaxRowSize,
+                                                 StNumRowsProcessed, StNumSlaves, StNumStarts, StNumStops, StNumStrands,
+                                                 StNumScansPerRow, StNumAllocations, StNumAllocationScans,
+                                                 StTimeFirstExecute, StCycleLocalExecuteCycles,
+                                                 StNumAtmostTriggered,
+                                                 StNumServerCacheHits, StNumIndexSeeks, StNumIndexScans, StNumIndexWildSeeks,
+                                                 StNumIndexSkips, StNumIndexNullSkips, StNumIndexMerges, StNumIndexMergeCompares,
+                                                 StNumPreFiltered, StNumPostFiltered, StNumIndexAccepted, StNumIndexRejected,
+                                                 StNumIndexRowsRead, StNumDiskRowsRead, StNumDiskSeeks, StNumDiskAccepted,
+                                                 StNumBlobCacheHits, StNumLeafCacheHits, StNumNodeCacheHits,
+                                                 StNumBlobCacheAdds, StNumLeafCacheAdds, StNumNodeCacheAdds,
+                                                 StNumDiskRejected, StSizeAgentReply, StTimeAgentWait,
+                                                 StTimeSoapcall,
+                                                 StNumGroups,
+                                                 StTimeSortElapsed,
+                                                 StNumDuplicateKeys});
+
 //=================================================================================
 
 extern SinkMode getSinkMode(const char *val)

system/jlib/jstats.h

@@ -510,12 +510,18 @@ class CNestedRuntimeStatisticMap;
 class jlib_decl CRuntimeStatisticCollection
 {
 public:
-    CRuntimeStatisticCollection(const StatisticsMapping & _mapping) : mapping(_mapping)
-    {
+    CRuntimeStatisticCollection(const StatisticsMapping & _mapping, bool _ignoreUnknown = false) : mapping(_mapping)
+#ifdef _DEBUG
+    ,ignoreUnknown(_ignoreUnknown)
+#endif
+{
         unsigned num = mapping.numStatistics();
         values = new CRuntimeStatistic[num+1]; // extra entry is to gather unexpected stats
     }
     CRuntimeStatisticCollection(const CRuntimeStatisticCollection & _other) : mapping(_other.mapping)
+#ifdef _DEBUG
+    , ignoreUnknown(_other.ignoreUnknown)
+#endif
     {
         unsigned num = mapping.numStatistics();
         values = new CRuntimeStatistic[num+1];
@@ -527,13 +533,19 @@ public:
     inline CRuntimeStatistic & queryStatistic(StatisticKind kind)
     {
         unsigned index = queryMapping().getIndex(kind);
-        dbgassertex(index < mapping.numStatistics());
+#ifdef _DEBUG
+        if (!ignoreUnknown)
+            dbgassertex(index < mapping.numStatistics());
+#endif
         return values[index];
     }
     inline const CRuntimeStatistic & queryStatistic(StatisticKind kind) const
     {
         unsigned index = queryMapping().getIndex(kind);
-        dbgassertex(index < mapping.numStatistics());
+#ifdef _DEBUG
+        if (!ignoreUnknown)
+            dbgassertex(index < mapping.numStatistics());
+#endif
         return values[index];
     }
 
@@ -602,6 +614,9 @@ protected:
     CRuntimeStatistic * values;
     std::atomic<CNestedRuntimeStatisticMap *> nested {nullptr};
     static CriticalSection nestlock;
+#ifdef _DEBUG
+    bool ignoreUnknown = false;
+#endif
 };
 
 //NB: Serialize and deserialize are not currently implemented.