Merge pull request #6890 from ghalliday/issue12885

HPCC-12885 Introduce backward compatible statistics

Reviewed-By:Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -442,6 +442,30 @@ private:
             unsigned __int64 value;
             collection.getStatistic(kind, value, i);
             formatStatistic(formattedValue.clear(), value, kind);
+
+                //Until 6.0 generate the backward compatible tag name
+            const char * legacyTreeTag = queryLegacyTreeTag(kind);
+            if (legacyTreeTag)
+            {
+                StatisticMeasure measure = queryMeasure(kind);
+                if (measure == SMeasureSkew)
+                {
+                    //Minimum stats were always output as +ve numbers
+                    if (queryStatsVariant(kind) == StSkewMin)
+                        value = -value;
+
+                    target->setPropInt64(legacyTreeTag, value/100);
+                }
+                else if (measure == SMeasureTimeNs)
+                {
+                    //Legacy timings are in ms => scale
+                    target->setPropInt64(legacyTreeTag, value/1000000);
+                }
+                else
+                    target->setProp(legacyTreeTag, formattedValue);
+            }
+
+            //Unconditionally output in the new format.
             target->setProp(queryTreeTag(kind), formattedValue);
         }
     }

system/jlib/jstats.cpp

@@ -396,7 +396,7 @@ extern jlib_decl StatsMergeAction queryMergeMode(StatisticKind kind)
     "@" #x "Avg" # y, \
     "@Skew" # y, \
     "@SkewMin" # y, \
-    "SkewMax" # y, \
+    "@SkewMax" # y, \
     "@NodeMin" # y, \
     "@NodeMax" # y,
 
@@ -406,26 +406,34 @@ extern jlib_decl StatsMergeAction queryMergeMode(StatisticKind kind)
     BASE_TAGS(x, y) \
     "@" #x "Delta" # y
 
-//Define tags when the default needs to be overriden
-#define XTAGS(x, y, dft) \
-    dft, \
-    BASE_TAGS(x, y) \
-    #x "Delta" # y
-
 //Define the tags for time items.
 #define TIMETAGS(x, y) \
     "@" #x #y, \
     BASE_TAGS(x, y) \
     "@TimeDelta" # y
 
-#define STAT(x, y, m) St##x##y, m, { NAMES(x, y) }, { TAGS(x, y) }
-#define TAGSTAT(x, y, m, dft) St##x##y, m, { NAMES(x, y) }, { XTAGS(x, y, dft) }
+#define LEGACYTAGS(dft)    \
+        dft, \
+        NULL, \
+        NULL, \
+        NULL, \
+        NULL, \
+        NULL, \
+        NULL, \
+        NULL, \
+        NULL, \
+        NULL
+
+#define CORESTAT(x, y, m)     St##x##y, m, { NAMES(x, y) }, { TAGS(x, y) }
+#define STAT(x, y, m)         CORESTAT(x, y, m), { LEGACYTAGS(NULL) }
+#define TAGSTAT(x, y, m, dft) St##x##y, m, { NAMES(x, y) }, { TAGS(x, y) }, { LEGACYTAGS(dft) }
+
 
 //--------------------------------------------------------------------------------------------------------------------
 
 //These are the macros to use to define the different entries in the stats meta table
 #define TIMESTAT(y) STAT(Time, y, SMeasureTimeNs)
-#define WHENSTAT(y) St##When##y, SMeasureTimestampUs, { TIMENAMES(When, y) }, { TIMETAGS(When, y) }
+#define WHENSTAT(y) St##When##y, SMeasureTimestampUs, { TIMENAMES(When, y) }, { TIMETAGS(When, y) }, { LEGACYTAGS(NULL) }
 #define NUMSTAT(y) STAT(Num, y, SMeasureCount)
 #define SIZESTAT(y) STAT(Size, y, SMeasureSize)
 #define LOADSTAT(y) STAT(Load, y, SMeasureLoad)
@@ -447,6 +455,7 @@ public:
     StatisticMeasure measure;
     const char * names[StNextModifier/StVariantScale];
     const char * tags[StNextModifier/StVariantScale];
+    const char * legacytags[StNextModifier/StVariantScale];
 };
 
 static const StatisticMeta statsMetaData[StMax] = {
@@ -461,13 +470,13 @@ static const StatisticMeta statsMetaData[StMax] = {
     { WHENSTAT(Compiled) },
     { WHENSTAT(WorkunitModified) },
     { TIMESTAT(Elapsed) },
-    { TIMESTAT2(LocalExecute, "@localTime") },
+    { CORESTAT(Time, LocalExecute, SMeasureTimeNs), { "@localTime", "@timeMinMs", "@timeMaxMs" } },
     { TIMESTAT2(TotalExecute, "@totalTime") },
     { TIMESTAT(Remaining) },
     { SIZESTAT(GeneratedCpp) },
     { SIZESTAT(PeakMemory) },
     { SIZESTAT(MaxRowSize) },
-    { NUMSTAT2(RowsProcessed, "@count") },
+    { CORESTAT(Num, RowsProcessed, SMeasureCount), { "@count", "@min", "@max", NULL, "skew", "minskew", "maxskew", NULL, NULL, NULL } },
     { NUMSTAT2(Slaves, "@slaves") },
     { NUMSTAT2(Started, "@started") },
     { NUMSTAT2(Stopped, "@stopped") },
@@ -511,8 +520,8 @@ static const StatisticMeta statsMetaData[StMax] = {
 
 StatisticMeasure queryMeasure(StatisticKind kind)
 {
-    unsigned varient = (kind & ~StKindMask);
-    switch (varient)
+    unsigned variant = queryStatsVariant(kind);
+    switch (variant)
     {
     case StSkew:
     case StSkewMin:
@@ -567,6 +576,15 @@ const char * queryTreeTag(StatisticKind kind)
     return statsMetaData[rawkind].tags[variant];
 }
 
+const char * queryLegacyTreeTag(StatisticKind kind)
+{
+    StatisticKind rawkind = (StatisticKind)(kind & StKindMask);
+    unsigned variant = (kind / StVariantScale);
+    dbgassertex(rawkind >= StKindNone && rawkind < StMax);
+    dbgassertex(variant < (StNextModifier/StVariantScale));
+    return statsMetaData[rawkind].legacytags[variant];
+}
+
 //--------------------------------------------------------------------------------------------------------------------
 
 StatisticKind queryStatisticKind(const char * search)

system/jlib/jstats.h

@@ -181,6 +181,8 @@ enum StatisticKind
 
 };
 
+inline StatisticKind queryStatsVariant(StatisticKind kind) { return (StatisticKind)(kind & ~StKindMask); }
+
 //---------------------------------------------------------------------------------------------------------------------
 
 interface IStatistic : extends IInterface
@@ -571,6 +573,7 @@ extern jlib_decl StatisticMeasure queryMeasure(StatisticKind kind);
 extern jlib_decl const char * queryStatisticName(StatisticKind kind);
 extern jlib_decl void queryLongStatisticName(StringBuffer & out, StatisticKind kind);
 extern jlib_decl const char * queryTreeTag(StatisticKind kind);
+extern jlib_decl const char * queryLegacyTreeTag(StatisticKind kind);
 extern jlib_decl const char * queryCreatorTypeName(StatisticCreatorType sct);
 extern jlib_decl const char * queryScopeTypeName(StatisticScopeType sst);
 extern jlib_decl const char * queryMeasureName(StatisticMeasure measure);