Merge pull request #8904 from ghalliday/issue15845

HPCC-15845 Add support for average/sd on query stats

Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

roxie/ccd/ccdquery.hpp

@@ -232,6 +232,8 @@ protected:
     UnsignedArray childQueryIndexes;
     CachedOutputMetaData meta;
     mutable CRuntimeStatisticCollection mystats;
+    // MORE: Could be CRuntimeSummaryStatisticCollection to include derived stats, but stats are currently converted
+    // to IPropertyTrees.  Would need to serialize/deserialize and then merge/derived so that they merged properly
 
 public:
     CActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
@@ -261,13 +263,7 @@ public:
 
     virtual void getNodeProgressInfo(IPropertyTree &node) const
     {
-        ForEachItemIn(i, mystats)
-        {
-            StatisticKind kind = mystats.getKind(i);
-            unsigned __int64 value = mystats.getStatisticValue(kind);
-            if (value)
-                putStatsValue(&node, queryStatisticName(kind), "sum", value);
-        }
+        mystats.getNodeProgressInfo(node);
     }
 
     virtual void resetNodeProgressInfo()

system/jlib/jstatcodes.h

@@ -196,7 +196,8 @@ enum StatisticKind
     StNodeMin                           = 0x70000,  // the node containing the minimum
     StNodeMax                           = 0x80000,  // the node containing the maximum
     StDeltaX                            = 0x90000,  // a difference in the value of X
-    StNextModifier                      = 0xa0000,
+    StStdDevX                           = 0xa0000,  // standard deviation in the value of X
+    StNextModifier                      = 0xb0000,
 
 };
 

system/jlib/jstats.cpp

@@ -23,6 +23,7 @@
 #include "jlog.hpp"
 #include "jregexp.hpp"
 #include "jfile.hpp"
+#include <math.h>
 
 #ifdef _WIN32
 #include <sys/timeb.h>
@@ -454,11 +455,14 @@ extern jlib_decl StatsMergeAction queryMergeMode(StatisticKind kind)
 
 #define NAMES(x, y) \
     BASE_NAMES(x, y) \
-    #x "Delta" # y
+    #x "Delta" # y, \
+    #x "StdDev" #y,
 
-#define TIMENAMES(x, y) \
+
+#define WHENNAMES(x, y) \
     BASE_NAMES(x, y) \
-    "TimeDelta" # y
+    "TimeDelta" # y, \
+    "TimeStdDev" # y,
 
 #define BASE_TAGS(x, y) \
     "@" #x "Min" # y,  \
@@ -474,13 +478,15 @@ extern jlib_decl StatsMergeAction queryMergeMode(StatisticKind kind)
 #define TAGS(x, y) \
     "@" #x #y, \
     BASE_TAGS(x, y) \
-    "@" #x "Delta" # y
+    "@" #x "Delta" # y, \
+    "@" #x "StdDev" # y,
 
 //Define the tags for time items.
-#define TIMETAGS(x, y) \
+#define WHENTAGS(x, y) \
     "@" #x #y, \
     BASE_TAGS(x, y) \
-    "@TimeDelta" # y
+    "@TimeDelta" # y, \
+    "@TimeStdDev" # y,
 
 #define CORESTAT(x, y, m)     St##x##y, m, St##x##y, { NAMES(x, y) }, { TAGS(x, y) }
 #define STAT(x, y, m)         CORESTAT(x, y, m)
@@ -489,7 +495,7 @@ extern jlib_decl StatsMergeAction queryMergeMode(StatisticKind kind)
 
 //These are the macros to use to define the different entries in the stats meta table
 #define TIMESTAT(y) STAT(Time, y, SMeasureTimeNs)
-#define WHENSTAT(y) St##When##y, SMeasureTimestampUs, St##When##y, { TIMENAMES(When, y) }, { TIMETAGS(When, y) }
+#define WHENSTAT(y) St##When##y, SMeasureTimestampUs, St##When##y, { WHENNAMES(When, y) }, { WHENTAGS(When, y) }
 #define NUMSTAT(y) STAT(Num, y, SMeasureCount)
 #define SIZESTAT(y) STAT(Size, y, SMeasureSize)
 #define LOADSTAT(y) STAT(Load, y, SMeasureLoad)
@@ -607,6 +613,7 @@ StatisticMeasure queryMeasure(StatisticKind kind)
     case StNodeMax:
         return SMeasureNode;
     case StDeltaX:
+    case StStdDevX:
         {
             StatisticMeasure measure = queryMeasure((StatisticKind)(kind & StKindMask));
             switch (measure)
@@ -654,6 +661,27 @@ unsigned __int64 convertMeasure(StatisticKind from, StatisticKind to, unsigned _
 }
 
 
+static double convertSquareMeasure(StatisticMeasure from, StatisticMeasure to, double value)
+{
+    if (from == to)
+        return value;
+    const unsigned __int64 largeValue = 1000000000;
+    double scale;
+    if ((from == SMeasureCycle) && (to == SMeasureTimeNs))
+        scale = (double)cycle_to_nanosec(largeValue) / (double)largeValue;
+    else if ((from == SMeasureTimeNs) && (to == SMeasureCycle))
+        scale = (double)nanosec_to_cycle(largeValue) / (double)largeValue;
+    else
+        throwUnexpected();
+    return value * scale * scale;
+}
+
+static double convertSquareMeasure(StatisticKind from, StatisticKind to, double value)
+{
+    return convertSquareMeasure(queryMeasure(from), queryMeasure(to), value);
+}
+
+
 StatisticKind querySerializedKind(StatisticKind kind)
 {
     StatisticKind rawkind = (StatisticKind)(kind & StKindMask);
@@ -1471,6 +1499,84 @@ extern IStatisticGatherer * createStatisticsGatherer(StatisticCreatorType creato
 
 //--------------------------------------------------------------------------------------------------------------------
 
+extern IPropertyTree * selectTreeStat(IPropertyTree *node, const char *statName, const char *statType)
+{
+    StringBuffer xpath;
+    xpath.appendf("att[@name='%s']", statName);
+    IPropertyTree *att = node->queryPropTree(xpath.str());
+    if (!att)
+    {
+        att = node->addPropTree("att", createPTree());
+        att->setProp("@name", statName);
+        att->setProp("@type", statType);
+    }
+    return att;
+}
+
+extern void putStatsTreeValue(IPropertyTree *node, const char *statName, const char *statType, unsigned __int64 val)
+{
+    if (val)
+        selectTreeStat(node, statName, statType)->setPropInt64("@value", val);
+}
+
+class TreeNodeStatisticGatherer : public CInterfaceOf<IStatisticGatherer>
+{
+public:
+    TreeNodeStatisticGatherer(IPropertyTree & root) : node(&root) {}
+
+    virtual void beginScope(const StatsScopeId & id)
+    {
+        StringBuffer temp;
+        id.getScopeText(temp);
+        beginScope(temp.str());
+    }
+    virtual void beginSubGraphScope(unsigned id) { throwUnexpected(); }
+    virtual void beginActivityScope(unsigned id) { throwUnexpected(); }
+    virtual void beginEdgeScope(unsigned id, unsigned oid) { throwUnexpected(); }
+    virtual void endScope()
+    { 
+        node = &stack.popGet();
+    }
+    virtual void addStatistic(StatisticKind kind, unsigned __int64 value)
+    {
+        putStatsTreeValue(node, queryStatisticName(kind), "sum", value);
+    }
+
+    virtual void updateStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
+    {
+        if (value)
+        {
+            IPropertyTree * stat = selectTreeStat(node, queryStatisticName(kind), "sum");
+            unsigned __int64 newValue = mergeStatisticValue(stat->getPropInt64("@value"), value, mergeAction);
+            stat->setPropInt64("@value", newValue);
+        }
+    }
+
+    virtual IStatisticCollection * getResult() { throwUnexpected(); }
+
+protected:
+    void beginScope(const char * id)
+    {
+        stack.append(*node);
+
+        StringBuffer xpath;
+        xpath.appendf("scope[@name='%s']", id);
+        IPropertyTree *att = node->queryPropTree(xpath.str());
+        if (!att)
+        {
+            att = node->addPropTree("scope", createPTree());
+            att->setProp("@name", id);
+        }
+        node = att;
+    }
+
+protected:
+    IPropertyTree * node;
+    ICopyArrayOf<IPropertyTree> stack;
+};
+
+//--------------------------------------------------------------------------------------------------------------------
+
 void CRuntimeStatistic::merge(unsigned __int64 otherValue, StatsMergeAction mergeAction)
 {
     value = mergeStatisticValue(value, otherValue, mergeAction);
@@ -1514,7 +1620,7 @@ void CRuntimeStatisticCollection::mergeStatistic(StatisticKind kind, unsigned __
 CRuntimeStatisticCollection & CRuntimeStatisticCollection::registerNested(const StatsScopeId & scope, const StatisticsMapping & mapping)
 {
     ensureNested();
-    return nested->addNested(scope, mapping);
+    return nested->addNested(scope, mapping).queryStats();
 }
 
 void CRuntimeStatisticCollection::rollupStatistics(unsigned numTargets, IContextLogger * const * targets) const
@@ -1637,6 +1743,12 @@ void CRuntimeStatisticCollection::deserializeMerge(MemoryBuffer& in)
     }
 }
 
+void CRuntimeStatisticCollection::getNodeProgressInfo(IPropertyTree &node) const
+{
+    TreeNodeStatisticGatherer gatherer(node);
+    recordStatistics(gatherer);
+}
+
 bool CRuntimeStatisticCollection::serialize(MemoryBuffer& out) const
 {
     unsigned numValid = 0;
@@ -1673,6 +1785,125 @@ bool CRuntimeStatisticCollection::serialize(MemoryBuffer& out) const
 }
 
 
+//---------------------------------------------------------------------------------------------------------------------
+
+void CRuntimeSummaryStatisticCollection::DerivedStats::mergeStatistic(unsigned __int64 value, unsigned node)
+{
+    if (count == 0)
+    {
+        min = value;
+        max = value;
+        minNode = node;
+        maxNode = node;
+    }
+    else
+    {
+        if (value < min)
+        {
+            min = value;
+            minNode = node;
+        }
+        if (value > max)
+        {
+            max = value;
+            maxNode = node;
+        }
+    }
+    count++;
+    double dvalue = (double)value;
+    sumSquares += dvalue * dvalue;
+}
+
+CRuntimeSummaryStatisticCollection::CRuntimeSummaryStatisticCollection(const StatisticsMapping & _mapping) : CRuntimeStatisticCollection(_mapping)
+{
+    derived = new DerivedStats[ordinality()+1];
+}
+
+CRuntimeSummaryStatisticCollection::~CRuntimeSummaryStatisticCollection()
+{
+    delete[] derived;
+}
+
+void CRuntimeSummaryStatisticCollection::ensureNested()
+{
+    if (!nested)
+        nested = new CNestedSummaryRuntimeStatisticMap;
+}
+
+void CRuntimeSummaryStatisticCollection::mergeStatistic(StatisticKind kind, unsigned __int64 value)
+{
+    CRuntimeStatisticCollection::mergeStatistic(kind, value);
+    unsigned index = queryMapping().getIndex(kind);
+    derived[index].mergeStatistic(value, 0);
+}
+
+static bool isSignificantSkew(StatisticKind kind, unsigned __int64 range, unsigned __int64 count)
+{
+    //MORE: Could get more sophisticated!
+    return range > 1;
+}
+
+void CRuntimeSummaryStatisticCollection::recordStatistics(IStatisticGatherer & target) const
+{
+    CRuntimeStatisticCollection::recordStatistics(target);
+    for (unsigned i = 0; i < ordinality(); i++)
+    {
+        DerivedStats & cur = derived[i];
+        if (cur.count)
+        {
+            StatisticKind kind = getKind(i);
+            StatisticKind serialKind= querySerializedKind(kind);
+
+            unsigned __int64 minValue = convertMeasure(kind, serialKind, cur.min);
+            unsigned __int64 maxValue = convertMeasure(kind, serialKind, cur.max);
+            if (minValue != maxValue)
+            {
+                double sum = (double)convertMeasure(kind, serialKind, values[i].get());
+                //Sum of squares needs to be translated twice
+                double sumSquares = convertSquareMeasure(kind, serialKind, cur.sumSquares);
+                double mean = (double)(sum / cur.count);
+                double variance = (sumSquares - sum * mean) / cur.count;
+                double stdDev = sqrt(variance);
+                double maxSkew = (10000.0 * ((maxValue-mean)/mean));
+                double minSkew = (10000.0 * ((mean-minValue)/mean));
+                unsigned __int64 range = maxValue - minValue;
+
+                target.addStatistic((StatisticKind)(serialKind|StMinX), minValue);
+                target.addStatistic((StatisticKind)(serialKind|StMaxX), maxValue);
+                target.addStatistic((StatisticKind)(serialKind|StAvgX), (unsigned __int64)mean);
+                target.addStatistic((StatisticKind)(serialKind|StDeltaX), range);
+                target.addStatistic((StatisticKind)(serialKind|StStdDevX), (unsigned __int64)stdDev);
+                //If all nodes are the same then we re actually merging results from multiple runs
+                //if the range is less than the count then
+                if ((cur.minNode != cur.maxNode) && isSignificantSkew(serialKind, range, cur.count))
+                {
+                    target.addStatistic((StatisticKind)(serialKind|StSkewMin), (unsigned __int64)minSkew);
+                    target.addStatistic((StatisticKind)(serialKind|StSkewMax), (unsigned __int64)maxSkew);
+                    target.addStatistic((StatisticKind)(serialKind|StNodeMin), cur.minNode);
+                    target.addStatistic((StatisticKind)(serialKind|StNodeMax), cur.maxNode);
+                }
+            }
+            else if (cur.count != 1)
+                target.addStatistic((StatisticKind)(serialKind|StAvgX), minValue);
+        }
+    }
+}
+
+bool CRuntimeSummaryStatisticCollection::serialize(MemoryBuffer & out) const
+{
+    UNIMPLEMENTED; // NB: Need to convert sum squares twice.
+}
+
+void CRuntimeSummaryStatisticCollection::deserialize(MemoryBuffer & in)
+{
+    UNIMPLEMENTED;
+}
+
+void CRuntimeSummaryStatisticCollection::deserializeMerge(MemoryBuffer& in)
+{
+    UNIMPLEMENTED;
+}
+
 //---------------------------------------------------
 
 bool CNestedRuntimeStatisticCollection::matches(const StatsScopeId & otherScope) const
@@ -1681,17 +1912,32 @@ bool CNestedRuntimeStatisticCollection::matches(const StatsScopeId & otherScope)
 }
 
 //NOTE: When deserializing, the scope is deserialized by the caller, and the correct target selected
-//which is why there is no corresponding deserialize() method
+//which is why there is no corresponding deserialize() ofthe scope at this point
+void CNestedRuntimeStatisticCollection::deserialize(MemoryBuffer & in)
+{
+    stats->deserialize(in);
+}
+
+void CNestedRuntimeStatisticCollection::deserializeMerge(MemoryBuffer& in)
+{
+    stats->deserialize(in);
+}
+
+void CNestedRuntimeStatisticCollection::merge(const CNestedRuntimeStatisticCollection & other)
+{
+    stats->merge(other.queryStats());
+}
+
 bool CNestedRuntimeStatisticCollection::serialize(MemoryBuffer& out) const
 {
     scope.serialize(out);
-    return CRuntimeStatisticCollection::serialize(out);
+    return stats->serialize(out);
 }
 
 void CNestedRuntimeStatisticCollection::recordStatistics(IStatisticGatherer & target) const
 {
     target.beginScope(scope);
-    CRuntimeStatisticCollection::recordStatistics(target);
+    stats->recordStatistics(target);
     target.endScope();
 }
 
@@ -1699,7 +1945,7 @@ StringBuffer & CNestedRuntimeStatisticCollection::toStr(StringBuffer &str) const
 {
     str.append(' ');
     scope.getScopeText(str).append("={");
-    CRuntimeStatisticCollection::toStr(str);
+    stats->toStr(str);
     return str.append(" }");
 }
 
@@ -1707,13 +1953,13 @@ StringBuffer & CNestedRuntimeStatisticCollection::toXML(StringBuffer &str) const
 {
     str.append("<Scope id=\"");
     scope.getScopeText(str).append("\">");
-    CRuntimeStatisticCollection::toXML(str);
+    stats->toXML(str);
     return str.append("</Scope>");
 }
 
 //---------------------------------------------------
 
-CRuntimeStatisticCollection & CNestedRuntimeStatisticMap::addNested(const StatsScopeId & scope, const StatisticsMapping & mapping)
+CNestedRuntimeStatisticCollection & CNestedRuntimeStatisticMap::addNested(const StatsScopeId & scope, const StatisticsMapping & mapping)
 {
     ForEachItemIn(i, map)
     {
@@ -1721,7 +1967,7 @@ CRuntimeStatisticCollection & CNestedRuntimeStatisticMap::addNested(const StatsS
         if (cur.matches(scope))
             return cur;
     }
-    CNestedRuntimeStatisticCollection * stats = new CNestedRuntimeStatisticCollection(scope, mapping);
+    CNestedRuntimeStatisticCollection * stats = new CNestedRuntimeStatisticCollection(scope, createStats(mapping));
     map.append(*stats);
     return *stats;
 }
@@ -1737,7 +1983,7 @@ void CNestedRuntimeStatisticMap::deserialize(MemoryBuffer& in)
         scope.deserialize(in, currentStatisticsVersion);
 
         //Use allStatistics as the default mapping if it hasn't already been added.
-        CRuntimeStatisticCollection & child = addNested(scope, allStatistics);
+        CNestedRuntimeStatisticCollection & child = addNested(scope, allStatistics);
         child.deserialize(in);
     }
 }
@@ -1752,7 +1998,7 @@ void CNestedRuntimeStatisticMap::deserializeMerge(MemoryBuffer& in)
         scope.deserialize(in, currentStatisticsVersion);
 
         //Use allStatistics as the default mapping if it hasn't already been added.
-        CRuntimeStatisticCollection & child = addNested(scope, allStatistics);
+        CNestedRuntimeStatisticCollection & child = addNested(scope, allStatistics);
         child.deserializeMerge(in);
     }
 }
@@ -1762,7 +2008,7 @@ void CNestedRuntimeStatisticMap::merge(const CNestedRuntimeStatisticMap & other)
     ForEachItemIn(i, other.map)
     {
         CNestedRuntimeStatisticCollection & cur = other.map.item(i);
-        CRuntimeStatisticCollection & target = addNested(cur.scope, cur.queryMapping());
+        CNestedRuntimeStatisticCollection & target = addNested(cur.scope, cur.queryMapping());
         target.merge(cur);
     }
 }
@@ -1799,6 +2045,16 @@ StringBuffer & CNestedRuntimeStatisticMap::toXML(StringBuffer &str) const
     return str;
 }
 
+CRuntimeStatisticCollection * CNestedRuntimeStatisticMap::createStats(const StatisticsMapping & mapping)
+{
+    return new CRuntimeStatisticCollection(mapping);
+}
+
+CRuntimeStatisticCollection * CNestedSummaryRuntimeStatisticMap::createStats(const StatisticsMapping & mapping)
+{
+    return new CRuntimeSummaryStatisticCollection(mapping);
+}
+
 //---------------------------------------------------
 
 bool ScopedItemFilter::matchDepth(unsigned low, unsigned high) const

system/jlib/jstats.h

@@ -330,7 +330,7 @@ public:
         for (unsigned i=0; i <= num; i++)
             values[i].set(_other.values[i].get());
     }
-    ~CRuntimeStatisticCollection();
+    virtual ~CRuntimeStatisticCollection();
 
     inline CRuntimeStatistic & queryStatistic(StatisticKind kind)
     {
@@ -353,11 +353,7 @@ public:
     {
         queryStatistic(kind).addAtomic(value);
     }
-    void mergeStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
-    {
-        queryStatistic(kind).merge(value, mergeAction);
-    }
-    void mergeStatistic(StatisticKind kind, unsigned __int64 value);
+    virtual void mergeStatistic(StatisticKind kind, unsigned __int64 value);
     void setStatistic(StatisticKind kind, unsigned __int64 value)
     {
         queryStatistic(kind).set(value);
@@ -384,54 +380,102 @@ public:
     void rollupStatistics(IContextLogger * target) { rollupStatistics(1, &target); }
     void rollupStatistics(unsigned num, IContextLogger * const * targets) const;
 
-    void recordStatistics(IStatisticGatherer & target) const;
+    virtual void recordStatistics(IStatisticGatherer & target) const;
+    void getNodeProgressInfo(IPropertyTree &node) const;
 
     // Print out collected stats to string
     StringBuffer &toStr(StringBuffer &str) const;
     // Print out collected stats to string as XML
     StringBuffer &toXML(StringBuffer &str) const;
     // Serialize/deserialize
-    bool serialize(MemoryBuffer & out) const;  // Returns true if any non-zero
-    void deserialize(MemoryBuffer & in);
-    void deserializeMerge(MemoryBuffer& in);
+    virtual bool serialize(MemoryBuffer & out) const;  // Returns true if any non-zero
+    virtual void deserialize(MemoryBuffer & in);
+    virtual void deserializeMerge(MemoryBuffer& in);
+
 
 protected:
-    void ensureNested();
+    virtual void ensureNested();
     void reportIgnoredStats() const;
+    void mergeStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
+    {
+        queryStatistic(kind).merge(value, mergeAction);
+    }
     const CRuntimeStatistic & queryUnknownStatistic() const { return values[mapping.numStatistics()]; }
 
-private:
+protected:
     const StatisticsMapping & mapping;
     CRuntimeStatistic * values;
     CNestedRuntimeStatisticMap * nested = nullptr;
 };
 
-class CNestedRuntimeStatisticCollection : public CRuntimeStatisticCollection, public CInterface
+//NB: Serialize and deserialize are not currently implemented.
+class jlib_decl CRuntimeSummaryStatisticCollection : public CRuntimeStatisticCollection
 {
 public:
-    CNestedRuntimeStatisticCollection(const StatsScopeId & _scope, const StatisticsMapping & _mapping)
-    : CRuntimeStatisticCollection(_mapping), scope(_scope)
+    CRuntimeSummaryStatisticCollection(const StatisticsMapping & _mapping);
+    ~CRuntimeSummaryStatisticCollection();
+
+    virtual void mergeStatistic(StatisticKind kind, unsigned __int64 value) override;
+    virtual void recordStatistics(IStatisticGatherer & target) const override;
+    virtual bool serialize(MemoryBuffer & out) const override;  // Returns true if any non-zero
+    virtual void deserialize(MemoryBuffer & in) override;
+    virtual void deserializeMerge(MemoryBuffer& in) override;
+
+protected:
+    struct DerivedStats
     {
-    }
-    CNestedRuntimeStatisticCollection(const CNestedRuntimeStatisticCollection & _other)
-    : CRuntimeStatisticCollection(_other), scope(_other.scope)
+    public:
+        void mergeStatistic(unsigned __int64 value, unsigned node);
+    public:
+        unsigned __int64 max = 0;
+        unsigned __int64 min = 0;
+        unsigned __int64 count = 0;
+        double sumSquares = 0;
+        unsigned minNode = 0;
+        unsigned maxNode = 0;
+    };
+
+protected:
+    virtual void ensureNested() override;
+
+protected:
+    DerivedStats * derived;
+};
+
+class CNestedRuntimeStatisticCollection : public CInterface
+{
+public:
+    CNestedRuntimeStatisticCollection(const StatsScopeId & _scope, CRuntimeStatisticCollection * _stats)
+    : scope(_scope), stats(_stats)
     {
     }
+    CNestedRuntimeStatisticCollection(const CNestedRuntimeStatisticCollection & _other) = delete;
+    ~CNestedRuntimeStatisticCollection() { delete stats; }
+
     bool matches(const StatsScopeId & otherScope) const;
+    inline const StatisticsMapping & queryMapping() const { return stats->queryMapping(); };
+    inline CRuntimeStatisticCollection & queryStats() { return *stats; }
+    inline const CRuntimeStatisticCollection & queryStats() const { return *stats; }
+
     bool serialize(MemoryBuffer & out) const;  // Returns true if any non-zero
     void deserialize(MemoryBuffer & in);
+    void deserializeMerge(MemoryBuffer& in);
+    void merge(const CNestedRuntimeStatisticCollection & other);
     void recordStatistics(IStatisticGatherer & target) const;
     StringBuffer & toStr(StringBuffer &str) const;
     StringBuffer & toXML(StringBuffer &str) const;
 
 public:
     StatsScopeId scope;
+    CRuntimeStatisticCollection * stats;
 };
 
 class CNestedRuntimeStatisticMap
 {
 public:
-    CRuntimeStatisticCollection & addNested(const StatsScopeId & scope, const StatisticsMapping & mapping);
+    virtual ~CNestedRuntimeStatisticMap() = default;
+
+    CNestedRuntimeStatisticCollection & addNested(const StatsScopeId & scope, const StatisticsMapping & mapping);
 
     bool serialize(MemoryBuffer & out) const;  // Returns true if any non-zero
     void deserialize(MemoryBuffer & in);
@@ -442,8 +486,16 @@ public:
     StringBuffer & toXML(StringBuffer &str) const;
 
 protected:
+    virtual CRuntimeStatisticCollection * createStats(const StatisticsMapping & mapping);
+
+protected:
     CIArrayOf<CNestedRuntimeStatisticCollection> map;
+};
 
+class CNestedSummaryRuntimeStatisticMap : public CNestedRuntimeStatisticMap
+{
+protected:
+    virtual CRuntimeStatisticCollection * createStats(const StatisticsMapping & mapping) override;
 };
 
 //---------------------------------------------------------------------------------------------------------------------