HPCC-16546 Allow statistics to be filtered by value

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/workunit/workunit.cpp

@@ -423,7 +423,7 @@ public:
     }
     virtual bool matches(const IStatisticsFilter * filter) const
     {
-        return filter->matches(creatorType, creator, scopeType, scope, measure, kind);
+        return filter->matches(creatorType, creator, scopeType, scope, measure, kind, value);
     }
 
 public:
@@ -667,7 +667,7 @@ protected:
         IPropertyTree & curSubGraph = subgraphIter->query();
         curStat->creatorType = queryCreatorType(curSubGraph.queryProp("@c"));
         curStat->creator.set(curSubGraph.queryProp("@creator"));
-        return filter->matches(curStat->creatorType, curStat->creator, SSTall, NULL, SMeasureAll, StKindAll);
+        return filter->matches(curStat->creatorType, curStat->creator, SSTall, NULL, SMeasureAll, StKindAll, AnyStatisticValue);
     }
 
     bool checkScope()
@@ -677,7 +677,7 @@ protected:
         IStatisticCollection * collection = &collections.tos();
         curStat->scopeType = collection->queryScopeType();
         collection->getFullScope(curStat->scope.clear());
-        return filter->matches(SCTall, NULL, curStat->scopeType, curStat->scope, SMeasureAll, StKindAll);
+        return filter->matches(SCTall, NULL, curStat->scopeType, curStat->scope, SMeasureAll, StKindAll, AnyStatisticValue);
     }
 
     bool checkStatistic()
@@ -687,7 +687,7 @@ protected:
         curStat->measure = queryMeasure(curStat->kind);
         if (!filter)
             return true;
-        if (!filter->matches(SCTall, NULL, SSTall, NULL, curStat->measure, curStat->kind))
+        if (!filter->matches(SCTall, NULL, SSTall, NULL, curStat->measure, curStat->kind, curStat->value))
             return false;
         return true;
     }
@@ -5781,6 +5781,8 @@ IConstWUStatisticIterator& CLocalWorkUnit::getStatistics(const IStatisticsFilter
 
     statistics.load(p,"Statistics/*");
     Owned<IConstWUStatisticIterator> localStats = new WorkUnitStatisticsIterator(statistics, 0, (IConstWorkUnit *) this, filter);
+    if (!filter->recurseChildScopes(SSTgraph, nullptr))
+        return *localStats.getClear();
 
     const char * wuid = p->queryName();
     Owned<IConstWUStatisticIterator> graphStats = new CConstGraphProgressStatisticsIterator(wuid, filter);
@@ -8734,7 +8736,7 @@ bool CLocalWUStatistic::matches(const IStatisticsFilter * filter) const
         return true;
     const char * creator = p->queryProp("@creator");
     const char * scope = p->queryProp("@scope");
-    return filter->matches(getCreatorType(), creator, getScopeType(), scope, getMeasure(), getKind());
+    return filter->matches(getCreatorType(), creator, getScopeType(), scope, getMeasure(), getKind(), getValue());
 }
 
 //==========================================================================================
@@ -8822,7 +8824,7 @@ bool CLocalWULegacyTiming::matches(const IStatisticsFilter * filter) const
         return true;
     const char * creator = p->queryProp("@creator");
     const char * scope = p->queryProp("@scope");
-    return filter->matches(SCTall, NULL, SSTall, NULL, getMeasure(), getKind());
+    return filter->matches(SCTall, NULL, SSTall, NULL, getMeasure(), getKind(), getValue());
 }
 
 //==========================================================================================

dali/daliadmin/daliadmin.cpp

@@ -2530,12 +2530,8 @@ static const char * checkDash(const char * s)
     return s;
 }
 
-static void dumpStats(IConstWorkUnit * workunit, const char * creatorTypeText, const char * creator, const char * scopeTypeText, const char * scope, const char * kindText, const char * userFilter, bool csv)
+static void dumpStats(IConstWorkUnit * workunit, const StatisticsFilter & filter, bool csv)
 {
-    StatisticsFilter filter(checkDash(creatorTypeText), checkDash(creator), checkDash(scopeTypeText), checkDash(scope), NULL, checkDash(kindText));
-    if (userFilter)
-        filter.setFilter(userFilter);
-
     Owned<IConstWUStatisticIterator> stats = &workunit->getStatistics(&filter);
     if (!csv)
         printf("<Statistics wuid=\"%s\">\n", workunit->queryWuid());
@@ -2591,7 +2587,7 @@ static void dumpStats(IConstWorkUnit * workunit, const char * creatorTypeText, c
                 xml.append(count);
             xml.append(",");
             if (max)
-                xml.append(value);
+                xml.append(max);
             xml.append(",");
             if (ts)
                 formatStatistic(xml, ts, SMeasureTimestampUs);
@@ -2637,6 +2633,10 @@ static void dumpStats(IConstWorkUnit * workunit, const char * creatorTypeText, c
 
 static void dumpStats(const char *wuid, const char * creatorTypeText, const char * creator, const char * scopeTypeText, const char * scope, const char * kindText, const char * userFilter, bool csv)
 {
+    StatisticsFilter filter(checkDash(creatorTypeText), checkDash(creator), checkDash(scopeTypeText), checkDash(scope), NULL, checkDash(kindText));
+    if (userFilter)
+        filter.setFilter(userFilter);
+
     Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
     const char * star = strchr(wuid, '*');
     if (star)
@@ -2652,7 +2652,7 @@ static void dumpStats(const char *wuid, const char * creatorTypeText, const char
         {
             Owned<IConstWorkUnit> workunit = factory->openWorkUnit(iter->query().queryWuid());
             if (workunit)
-                dumpStats(workunit, creatorTypeText, creator, scopeTypeText, scope, kindText, userFilter, csv);
+                dumpStats(workunit, filter, csv);
         }
     }
     else
@@ -2660,7 +2660,7 @@ static void dumpStats(const char *wuid, const char * creatorTypeText, const char
         Owned<IConstWorkUnit> workunit = factory->openWorkUnit(wuid);
         if (!workunit)
             return;
-        dumpStats(workunit, creatorTypeText, creator, scopeTypeText, scope, kindText, userFilter, csv);
+        dumpStats(workunit, filter, csv);
     }
 }
 

ecl/eclplus/DumpHelper.cpp

@@ -17,6 +17,23 @@
 #include "jlib.hpp"
 #include "DumpHelper.ipp"
 
+static bool processExceptions(const IMultiException* excep)
+{
+    if (excep && excep->ordinality())
+    {
+        unsigned i = 0;
+        while (i < excep->ordinality())
+        {
+            StringBuffer msg;
+            excep->item(i).errorMessage(msg);
+            unsigned code = excep->item(i).errorCode();
+            printf("<Error><code>%d</code><message>%s</message></Error>\n", code, msg.str());
+        }
+        return true;
+    }
+    return false;
+}
+
 DumpHelper::DumpHelper(IProperties * _globals, IFormatType * _format) : globals(_globals), format(_format), wuclient(createWorkunitsClient(_globals))
 {
 }
@@ -70,18 +87,8 @@ bool DumpHelper::doit(FILE * fp)
                 return false;
             }
             const IMultiException* excep = &resp->getExceptions();
-            if(excep != NULL && excep->ordinality())
-            {
-                unsigned i = 0;
-                while (i < excep->ordinality())
-                {
-                    StringBuffer msg;
-                    excep->item(i).errorMessage(msg);
-                    unsigned code = excep->item(i).errorCode();
-                    printf("<Error><code>%d</code><message>%s</message></Error>\n", code, msg.str());
-                }
+            if(processExceptions(excep))
                 return false;
-            }
 
             const MemoryBuffer & xmlmem = resp->getThefile();
             StringBuffer xmlbuf;
@@ -124,20 +131,13 @@ bool GraphHelper::doit(FILE * fp)
         req->setName(graph.str());
         Owned<IClientWUProcessGraphResponse> resp = wuclient->WUProcessGraph(req);
         const IMultiException* excep = &resp->getExceptions();
-        if(excep != NULL && excep->ordinality() > 0)
-        {
-            StringBuffer msg;
-            excep->errorMessage(msg);
-            printf("%s\n", msg.str());
+        if(processExceptions(excep))
             return false;
-        }
 
-            StringBuffer graphbuf;
-            graphbuf.append(resp->getTheGraph());
-            fprintf(fp, "%s", graphbuf.str());
-            return true;
-        
-        return false;
+        StringBuffer graphbuf;
+        graphbuf.append(resp->getTheGraph());
+        fprintf(fp, "%s", graphbuf.str());
+        return true;
     }
     else 
     {
@@ -147,3 +147,68 @@ bool GraphHelper::doit(FILE * fp)
 }
 
 
+
+
+StatsHelper::StatsHelper(IProperties * _globals, IFormatType * _format) : globals(_globals), format(_format), wuclient(createWorkunitsClient(_globals))
+{
+}
+
+bool StatsHelper::doit(FILE * fp)
+{
+    Owned<IClientWUGetStatsRequest> req = wuclient->createWUGetStatsRequest();
+    const char* wuid = globals->queryProp("WUID");
+    const char* filter = globals->queryProp("FILTER");
+    if (wuid)
+        req->setWUID(wuid);
+    else
+        req->setWUID("*");
+
+    if (filter)
+        req->setFilter(filter);
+
+    Owned<IClientWUGetStatsResponse> resp = wuclient->WUGetStats(req);
+    const IMultiException* excep = &resp->getExceptions();
+    if(processExceptions(excep))
+        return false;
+
+    IArrayOf<IConstWUStatisticItem> & stats = resp->getStatistics();
+    ForEachItemIn(i, stats)
+    {
+        IConstWUStatisticItem & cur = stats.item(i);
+
+        StringBuffer line;
+        line.append(cur.getWuid());
+        line.append(",");
+        line.append(cur.getCreatorType());
+        line.append(",");
+        line.append(cur.getCreator());
+        line.append(",");
+        line.append(cur.getScopeType());
+        line.append(",");
+        line.append(cur.getScope());
+        line.append(",");
+        line.append(cur.getMeasure());
+        line.append(",");
+        line.append(cur.getKind());
+        line.append(",");
+        line.append(cur.getRawValue());
+        line.append(",");
+        line.append(cur.getValue());
+        line.append(",");
+        if (!cur.getCount_isNull())
+            line.append(cur.getCount());
+        line.append(",");
+        if (!cur.getMax_isNull())
+            line.append(cur.getMax());
+        line.append(",");
+        line.append(cur.getTimeStamp());
+        line.append(",");
+        if (cur.getDescription())
+            line.append('"').append(cur.getDescription()).append('"');
+        printf("%s\n", line.str());
+    }
+
+    return true;
+}
+
+

ecl/eclplus/DumpHelper.ipp

@@ -48,4 +48,17 @@ private:
     Owned<IFormatType> format;
     Owned<IClientWsWorkunits> wuclient;
 };
+
+class StatsHelper : implements IEclPlusHelper
+{
+public:
+    StatsHelper(IProperties * globals, IFormatType * format);
+
+    virtual bool doit(FILE * fp);
+
+private:
+    Owned<IProperties> globals;
+    Owned<IFormatType> format;
+    Owned<IClientWsWorkunits> wuclient;
+};
 #endif // DUMPHELPER_HPP

ecl/eclplus/eclplus.cpp

@@ -137,10 +137,14 @@ IEclPlusHelper * createEclPlusHelper(IProperties * globals)
         {
             helper = new RerunHelper(LINK(globals), format);
         }
+        else if(!stricmp(action, "stats"))
+        {
+            helper = new StatsHelper(LINK(globals), format);
+        }
         else
         {
             ::Release(format);
-            throw MakeStringException(-1, "unknown action");
+            throw MakeStringException(-1, "unknown action '%s'", action);
         }
     }
     else

esp/scm/ws_workunits.ecm

@@ -1703,6 +1703,7 @@ ESPStruct [nil_remove] WUStatisticItem
     int64 RawValue;
     int64 Count;
     int64 Max;
+    [min_ver("1.62")] string Wuid;
 };
 
 ESPrequest [nil_remove] WUGetStatsRequest
@@ -1718,6 +1719,10 @@ ESPrequest [nil_remove] WUGetStatsRequest
     unsigned MaxScopeDepth;
     boolean IncludeGraphs;
     boolean CreateDescriptions;
+    [min_ver("1.62")] int64 MinValue; // Should be uint64 but esdl does not support it.
+    [min_ver("1.62")] int64 MaxValue;
+    [min_ver("1.62")] string Filter; // arbitrary text filter
+    //MORE: Some indication of which fields need to be returned - to reduce the amount of data returned.  Is there a general esp mechanism?
 };
 
 ESPresponse [exceptions_inline] WUGetStatsResponse
@@ -1786,7 +1791,7 @@ ESPresponse [exceptions_inline, nil_remove] WUGetNumFileToCopyResponse
 
 ESPservice [
     auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
-    version("1.61"), default_client_version("1.61"),
+    version("1.62"), default_client_version("1.62"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1599,6 +1599,8 @@ void WsWuInfo::getStats(StatisticsFilter& filter, bool createDescriptions, IArra
 
         Owned<IEspWUStatisticItem> wuStatistic = createWUStatisticItem();
 
+        if (version > 1.61)
+            wuStatistic->setWuid(wuid);
         if (curCreatorType != SCTnone)
             wuStatistic->setCreatorType(queryCreatorTypeName(curCreatorType));
         if (curCreator.length())

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -4795,12 +4795,6 @@ bool CWsWorkunitsEx::onWUGetStats(IEspContext &context, IEspWUGetStatsRequest &r
 {
     try
     {
-        StringBuffer wuid = req.getWUID();
-        WsWuHelpers::checkAndTrimWorkunit("WUInfo", wuid);
-
-        ensureWsWorkunitAccess(context, wuid.str(), SecAccess_Read);
-        PROGLOG("WUGetStats: %s", wuid.str());
-
         const char* creatorType = checkGetStatsInput(req.getCreatorType());
         const char* creator = checkGetStatsInput(req.getCreator());
         const char* scopeType = checkGetStatsInput(req.getScopeType());
@@ -4813,14 +4807,57 @@ bool CWsWorkunitsEx::onWUGetStats(IEspContext &context, IEspWUGetStatsRequest &r
             filter.setScopeDepth(req.getMinScopeDepth(), req.getMaxScopeDepth());
         else if (!req.getMinScopeDepth_isNull())
             filter.setScopeDepth(req.getMinScopeDepth());
+        if (!req.getMinValue_isNull() || !req.getMaxValue_isNull())
+        {
+            unsigned __int64 lowValue = 0;
+            unsigned __int64 highValue = MaxStatisticValue;
+            if (!req.getMinValue_isNull())
+                lowValue = (unsigned __int64)req.getMinValue();
+            if (!req.getMaxValue_isNull())
+                highValue = (unsigned __int64)req.getMaxValue();
+            filter.setValueRange(lowValue, highValue);
+        }
+
+        const char * textFilter = req.getFilter();
+        if (textFilter)
+            filter.setFilter(textFilter);
 
         bool createDescriptions = false;
         if (!req.getCreateDescriptions_isNull())
             createDescriptions = req.getCreateDescriptions();
 
-        WsWuInfo winfo(context, wuid.str());
+        StringBuffer wuid = req.getWUID();
+        PROGLOG("WUGetStats: %s", wuid.str());
+
         IArrayOf<IEspWUStatisticItem> statistics;
-        winfo.getStats(filter, createDescriptions, statistics);
+        if (strchr(wuid, '*'))
+        {
+            WUSortField filters[2];
+            MemoryBuffer filterbuf;
+            filters[0] = WUSFwildwuid;
+            filterbuf.append(wuid.str());
+            filters[1] = WUSFterm;
+            Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
+            Owned<IConstWorkUnitIterator> iter = factory->getWorkUnitsSorted((WUSortField) (WUSFwuid), filters, filterbuf.bufferBase(), 0, INT_MAX, NULL, NULL);
+            ForEach(*iter)
+            {
+                Owned<IConstWorkUnit> workunit = factory->openWorkUnit(iter->query().queryWuid());
+                if (workunit)
+                {
+                    //No need to check for access since the list is already filtered
+                    WsWuInfo winfo(context, workunit->queryWuid());
+                    winfo.getStats(filter, createDescriptions, statistics);
+                }
+            }
+        }
+        else
+        {
+            WsWuHelpers::checkAndTrimWorkunit("WUInfo", wuid);
+            ensureWsWorkunitAccess(context, wuid, SecAccess_Read);
+
+            WsWuInfo winfo(context, wuid);
+            winfo.getStats(filter, createDescriptions, statistics);
+        }
         resp.setStatistics(statistics);
         resp.setWUID(wuid.str());
     }

system/jlib/jstats.cpp

@@ -2283,7 +2283,7 @@ void StatisticsFilter::init()
     kind = StKindAll;
 }
 
-bool StatisticsFilter::matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind) const
+bool StatisticsFilter::matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind, unsigned __int64 value) const
 {
     if ((curCreatorType != SCTall) && (creatorType != SCTall) && (creatorType != curCreatorType))
         return false;
@@ -2297,11 +2297,31 @@ bool StatisticsFilter::matches(StatisticCreatorType curCreatorType, const char *
         return false;
     if (!scopeFilter.match(curScope))
         return false;
+    if (value != MaxStatisticValue)
+    {
+        if ((value < minValue) || (value > maxValue))
+            return false;
+    }
     return true;
 }
 
 bool StatisticsFilter::recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const
 {
+    switch (curScopeType)
+    {
+    case SSTgraph:
+        // A child of a graph will have depth 2 or more
+        if (!scopeFilter.matchDepth(2, (unsigned)-1))
+            return false;
+        break;
+    case SSTsubgraph:
+        // A child of a subgraph will have depth 3 or more
+        if (!scopeFilter.matchDepth(3, (unsigned)-1))
+            return false;
+        break;
+    }
+    if (!curScope)
+        return true;
     return scopeFilter.recurseChildScopes(curScope);
 }
 
@@ -2375,6 +2395,24 @@ void StatisticsFilter::addFilter(const char * filter)
         setScope(value);
     else if (hasPrefix(filter, "scopetype[", false))
         setScopeType(queryScopeType(value));
+    else if (hasPrefix(filter, "value[", false))
+    {
+        //value[exact|low..high] where low and high are optional
+        unsigned __int64 lowValue = 0;
+        unsigned __int64 highValue = MaxStatisticValue;
+        if (isdigit(*value))
+            lowValue = (unsigned __int64)atoi64(value);
+        const char * dotdot = strstr(value, "..");
+        if (dotdot)
+        {
+            unsigned __int64 maxValue = (unsigned __int64)atoi64(dotdot + 2);
+            if (maxValue != 0)
+                highValue = maxValue;
+        }
+        else
+            highValue = lowValue;
+        setValueRange(lowValue, highValue);
+    }
     else
         throw MakeStringException(1, "Unknown stats filter '%s' - expected creator,creatortype,depth,kind,measure,scope,scopetype", filter);
 }
@@ -2425,6 +2463,19 @@ void StatisticsFilter::setScope(const char * _scope)
 void StatisticsFilter::setScopeType(StatisticScopeType _scopeType)
 {
     scopeType = _scopeType;
+    switch (scopeType)
+    {
+    case SSTglobal:
+    case SSTgraph:
+        scopeFilter.setDepth(1);
+        break;
+    case SSTsubgraph:
+        scopeFilter.setDepth(2);
+        break;
+    case SSTactivity:
+        scopeFilter.setDepth(3);
+        break;
+    }
 }
 
 void StatisticsFilter::setMeasure(StatisticMeasure _measure)
@@ -2432,6 +2483,12 @@ void StatisticsFilter::setMeasure(StatisticMeasure _measure)
     measure = _measure;
 }
 
+void StatisticsFilter::setValueRange(unsigned __int64 _minValue, unsigned __int64 _maxValue)
+{
+    minValue = _minValue;
+    maxValue = _maxValue;
+}
+
 void StatisticsFilter::setKind(StatisticKind _kind)
 {
     kind = _kind;

system/jlib/jstats.h

@@ -23,6 +23,9 @@
 
 #include "jstatcodes.h"
 
+const unsigned __int64 MaxStatisticValue = (unsigned __int64)-1;
+const unsigned __int64 AnyStatisticValue = MaxStatisticValue; // Use the maximum value to also represent unknown, since it is unlikely to ever occur.
+
 inline StatisticKind queryStatsVariant(StatisticKind kind) { return (StatisticKind)(kind & ~StKindMask); }
 
 //---------------------------------------------------------------------------------------------------------------------
@@ -116,7 +119,7 @@ public:
 interface IStatisticsFilter : public IInterface
 {
 public:
-    virtual bool matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind) const = 0;
+    virtual bool matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind, unsigned __int64 value) const = 0;
     virtual bool recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const = 0;
     virtual const char * queryScope() const = 0;
 
@@ -190,7 +193,7 @@ public:
     const char * queryValue() const { return value ? value.get() : "*"; }
 
     void set(const char * value);
-    void setDepth(unsigned _minDepth);
+    void setDepth(unsigned _depth);
     void setDepth(unsigned _minDepth, unsigned _maxDepth);
 
 protected:
@@ -210,7 +213,7 @@ public:
     StatisticsFilter(const char * _creatorTypeText, const char * _creator, const char * _scopeTypeText, const char * _scope, const char * _measureText, const char * _kindText);
     StatisticsFilter(StatisticCreatorType _creatorType, const char * _creator, StatisticScopeType _scopeType, const char * _scope, StatisticMeasure _measure, StatisticKind _kind);
 
-    virtual bool matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind) const;
+    virtual bool matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind, unsigned __int64 value) const override;
     virtual bool recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const;
     virtual const char * queryScope() const { return scopeFilter.queryValue(); }
 
@@ -225,6 +228,7 @@ public:
     void setScopeDepth(unsigned _minScopeDepth, unsigned _maxScopeDepth);
     void setScope(const char * _scope);
     void setScopeType(StatisticScopeType _scopeType);
+    void setValueRange(unsigned __int64 minValue, unsigned __int64 _maxValue);
     void setKind(StatisticKind _kind);
     void setKind(const char * _kind);
     void setMeasure(StatisticMeasure _measure);
@@ -240,6 +244,8 @@ protected:
     StatisticKind kind;
     ScopedItemFilter creatorFilter;
     ScopedItemFilter scopeFilter;
+    unsigned __int64 minValue = 0;
+    unsigned __int64 maxValue = (unsigned __int64)(-1);
 };
 
 //---------------------------------------------------------------------------------------------------------------------