Merge pull request #13990 from shamser/issue24450

HPCC-24450 User configurable analyzer options

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/wuanalysis/anacommon.hpp

@@ -23,11 +23,6 @@
 #include "eclhelper.hpp"
 #include "anaerrorcodes.hpp"
 
-#ifdef WUANALYSIS_EXPORTS
-    #define WUANALYSIS_API DECL_EXPORT
-#else
-    #define WUANALYSIS_API DECL_IMPORT
-#endif
 
 interface IWuScope
 {
@@ -82,6 +77,13 @@ private:
     StringBuffer comment;
 };
 
+typedef enum { watOptFirst=0, watOptMinInterestingTime=0, watOptMinInterestingCost, watOptSkewThreshold, watOptMinRowsPerNode, watOptMax } WutOptionType ;
+
+interface IAnalyserOptions
+{
+    virtual stat_type queryOption(WutOptionType opt) const = 0;
+};
+
 extern int compareIssuesCostOrder(CInterface * const * _l, CInterface * const * _r);
 
 #endif

common/wuanalysis/anarule.cpp

@@ -41,16 +41,16 @@ class DistributeSkewRule : public ActivityKindRule
 public:
     DistributeSkewRule() : ActivityKindRule(TAKhashdistribute) {}
 
-    virtual bool check(PerformanceIssue & result, IWuActivity & activity, const WuAnalyseOptions & options) override
+    virtual bool check(PerformanceIssue & result, IWuActivity & activity, const IAnalyserOptions & options) override
     {
         IWuEdge * outputEdge = activity.queryOutput(0);
         if (!outputEdge)
             return false;
         stat_type rowsAvg = outputEdge->getStatRaw(StNumRowsProcessed, StAvgX);
-        if (rowsAvg < rowsThreshold)
+        if (rowsAvg < options.queryOption(watOptMinRowsPerNode))
             return false;
         stat_type rowsMaxSkew = outputEdge->getStatRaw(StNumRowsProcessed, StSkewMax);
-        if (rowsMaxSkew > options.skewThreshold)
+        if (rowsMaxSkew > options.queryOption(watOptSkewThreshold))
         {
             // Use downstream activity time to calculate approximate cost
             IWuActivity * targetActivity = outputEdge->queryTarget();
@@ -70,9 +70,6 @@ public:
         }
         return false;
     }
-
-protected:
-    static const stat_type rowsThreshold = 100;                // avg rows per node.
 };
 
 class IoSkewRule : public AActivityRule
@@ -128,12 +125,11 @@ public:
         return false;
     }
 
-    virtual bool check(PerformanceIssue & result, IWuActivity & activity, const WuAnalyseOptions & options) override
+    virtual bool check(PerformanceIssue & result, IWuActivity & activity, const IAnalyserOptions & options) override
     {
         stat_type ioAvg = activity.getStatRaw(stat, StAvgX);
         stat_type ioMaxSkew = activity.getStatRaw(stat, StSkewMax);
-
-        if (ioMaxSkew > options.skewThreshold)
+        if (ioMaxSkew > options.queryOption(watOptSkewThreshold))
         {
             stat_type timeMaxLocalExecute = activity.getStatRaw(StTimeLocalExecute, StMaxX);
             stat_type timeAvgLocalExecute = activity.getStatRaw(StTimeLocalExecute, StAvgX);

common/wuanalysis/anarule.hpp

@@ -27,7 +27,7 @@ class AActivityRule : public CInterface
 {
 public:
     virtual bool isCandidate(IWuActivity & activity) const = 0;
-    virtual bool check(PerformanceIssue & results, IWuActivity & activity, const WuAnalyseOptions & options) = 0;
+    virtual bool check(PerformanceIssue & results, IWuActivity & activity, const IAnalyserOptions & options) = 0;
     virtual void updateInformation(PerformanceIssue & result,  IWuActivity & activity)
     {
         StringBuffer def;

common/wuanalysis/anawu.cpp

@@ -95,15 +95,101 @@ protected:
 };
 
 //-----------------------------------------------------------------------------------------------------------
+typedef enum {wutOptValueTypeFirst=0, wutOptValueTypeMSec=0, wutOptValueTypeSeconds, wutOptValueTypePercent, wutOptValueTypeCount, wutOptValueTypeMax} WutOptValueType;
+
+struct WuOption
+{
+    WutOptionType option;
+    const char * name;
+    stat_type defaultValue;
+    WutOptValueType type;
+};
+
+constexpr struct WuOption wuOptionsDefaults[watOptMax]
+= { {watOptMinInterestingTime, "minInterestingTime", 1000, wutOptValueTypeMSec},
+    {watOptMinInterestingCost, "minInterestingCost", 30000, wutOptValueTypeMSec},
+    {watOptSkewThreshold, "skewThreshold", 20, wutOptValueTypePercent},
+    {watOptMinRowsPerNode, "minRowsPerNode", 1000, wutOptValueTypeCount} };
+
+constexpr bool checkWuOptionsDefaults(int i = watOptMax)
+{
+    return ((wuOptionsDefaults[i-1].name != nullptr && (wuOptionsDefaults[i-1].option == i-1) && wuOptionsDefaults[i-1].type < wutOptValueTypeMax ) && 
+           (i==1 || checkWuOptionsDefaults(i-1)));
+}
+static_assert(checkWuOptionsDefaults(), "wuOptionsDefaults[] not populated correctly");
+
+class WuAnalyserOptions : public IAnalyserOptions
+{
+public:
+   WuAnalyserOptions()
+    {
+        for (int opt = watOptFirst; opt < watOptMax; opt++)
+            setOptionValue(static_cast<WutOptionType>(opt), wuOptionsDefaults[opt].defaultValue);
+    }
+
+    void setOptionValue(WutOptionType opt, __int64 val)
+    {
+        assertex(opt<watOptMax);
+        switch(wuOptionsDefaults[opt].type)
+        {
+            case wutOptValueTypeMSec:
+                wuOptions[opt] = msecs2StatUnits(val);
+                break;
+            case wutOptValueTypeSeconds:
+                wuOptions[opt] = seconds2StatUnits(val);
+                break;
+            case wutOptValueTypePercent:
+                wuOptions[opt] = statSkewPercent((stat_type)val);
+                break;
+            case wutOptValueTypeCount:
+                wuOptions[opt] = (stat_type) val;
+                break;
+            default:
+                throw MakeStringException(-1, "WuAnalyserOptions::setOptionValue - unknown wuOptionsDefaults[%d].type=%d", (int) opt, (int) wuOptionsDefaults[opt].type);
+        }
+    }
+
+    void applyConfig(IPropertyTree *options)
+    {
+        if (!options) return;
+        for (int opt = watOptFirst; opt < watOptMax; opt++)
+        {
+            StringBuffer wuOptionName("@");
+            wuOptionName.append(wuOptionsDefaults[opt].name);
+            __int64 val =  options->getPropInt64(wuOptionName, -1);
+            if (val!=-1)
+                setOptionValue(static_cast<WutOptionType>(opt), val);
+        }
+    }
+
+    void applyConfig(IConstWorkUnit * wu)
+    {
+        for (int opt = watOptFirst; opt < watOptMax; opt++)
+        {
+            StringBuffer wuOptionName("analyzer_");
+            wuOptionName.append(wuOptionsDefaults[opt].name);
+            __int64 val = wu->getDebugValueInt64(wuOptionName, -1);
+            if (val!=-1)
+                setOptionValue(static_cast<WutOptionType>(opt), val);
+        }
+    }
+    stat_type queryOption(WutOptionType opt) const override { return wuOptions[opt]; }
+private:
+    stat_type wuOptions[watOptMax];
+};
+//-----------------------------------------------------------------------------------------------------------
+
+
 class WorkunitAnalyser
 {
 public:
-    WorkunitAnalyser(WuAnalyseOptions & _options);
+    WorkunitAnalyser();
 
     void check(const char * scope, IWuActivity & activity);
     void analyse(IConstWorkUnit * wu);
     void print();
     void update(IWorkUnit *wu);
+    void applyConfig(IPropertyTree *cfg);
 
 protected:
     void collateWorkunitStats(IConstWorkUnit * workunit, const WuScopeFilter & filter);
@@ -112,7 +198,7 @@ protected:
     CIArrayOf<AActivityRule> rules;
     CIArrayOf<PerformanceIssue> issues;
     WuScope root;
-    WuAnalyseOptions options;
+    WuAnalyserOptions options;
 };
 
 //-----------------------------------------------------------------------------------------------------------
@@ -306,17 +392,23 @@ public:
 };
 
 
+
 //-----------------------------------------------------------------------------------------------------------
-WorkunitAnalyser::WorkunitAnalyser(WuAnalyseOptions & _options) : root("", nullptr), options(_options)
+
+WorkunitAnalyser::WorkunitAnalyser() : root("", nullptr)
 {
     gatherRules(rules);
 }
 
+void WorkunitAnalyser::applyConfig(IPropertyTree *cfg)
+{
+    options.applyConfig(cfg);
+}
+
 void WorkunitAnalyser::check(const char * scope, IWuActivity & activity)
 {
-    if (activity.getStatRaw(StTimeLocalExecute, StMaxX) < options.minInterestingTime)
+    if (activity.getStatRaw(StTimeLocalExecute, StMaxX) < options.queryOption(watOptMinInterestingTime))
         return;
-
     Owned<PerformanceIssue> highestCostIssue;
     ForEachItemIn(i, rules)
     {
@@ -325,7 +417,7 @@ void WorkunitAnalyser::check(const char * scope, IWuActivity & activity)
             Owned<PerformanceIssue> issue (new PerformanceIssue);
             if (rules.item(i).check(*issue, activity, options))
             {
-                if (issue->getCost() >= options.minCost)
+                if (issue->getCost() >= options.queryOption(watOptMinInterestingCost))
                 {
                     if (!highestCostIssue || highestCostIssue->getCost() < issue->getCost())
                         highestCostIssue.setown(issue.getClear());
@@ -344,6 +436,7 @@ void WorkunitAnalyser::check(const char * scope, IWuActivity & activity)
 
 void WorkunitAnalyser::analyse(IConstWorkUnit * wu)
 {
+    options.applyConfig(wu);
     WuScopeFilter filter;
     filter.addOutputProperties(PTstatistics).addOutputProperties(PTattributes);
     filter.finishedFilter();
@@ -407,16 +500,17 @@ WuScope * WorkunitAnalyser::selectFullScope(const char * scope)
 
 //---------------------------------------------------------------------------------------------------------------------
 
-void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, WuAnalyseOptions & options)
+void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options)
 {
-    WorkunitAnalyser analyser(options);
+    WorkunitAnalyser analyser;
+    analyser.applyConfig(options);
     analyser.analyse(wu);
     analyser.update(wu);
 }
 
-void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, WuAnalyseOptions & options, bool updatewu)
+void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, bool updatewu)
 {
-    WorkunitAnalyser analyser(options);
+    WorkunitAnalyser analyser;
     analyser.analyse(wu);
     analyser.print();
     if (updatewu)

common/wuanalysis/anawu.hpp

@@ -18,15 +18,12 @@
 #ifndef ANAWU_HPP
 #define ANAWU_HPP
 
-#include "anacommon.hpp"
-
-struct WuAnalyseOptions
-{
-    stat_type minInterestingTime = msecs2StatUnits(10);// ignore anything under 10 millisecond
-    stat_type minCost = seconds2StatUnits(30);          // only interested in costs of > 30s
-    stat_type skewThreshold = statSkewPercent(20);     // minimum interesting skew measurment
-};
+#ifdef WUANALYSIS_EXPORTS
+    #define WUANALYSIS_API DECL_EXPORT
+#else
+    #define WUANALYSIS_API DECL_IMPORT
+#endif
 
-void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, WuAnalyseOptions & options);
-void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, WuAnalyseOptions & options, bool updatewu);
+void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options);
+void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, bool updatewu);
 #endif

ecl/eclagent/eclagent.cpp

@@ -2001,8 +2001,8 @@ void EclAgent::doProcess()
         {
             if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true)))
             {
-                WuAnalyseOptions options;  // TODO: read options from configuration file
-                analyseWorkunit(w.get(), options);
+                IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions");
+                analyseWorkunit(w.get(), analyzerOptions);
             }
         }
         if(w->queryEventScheduledCount() > 0)

initfiles/componentfiles/configxml/agentexec.xsl

@@ -111,6 +111,7 @@
           <xsl:value-of select="@httpGlobalIdHeader"/>
         </xsl:attribute>
       </xsl:if>
+      <xsl:copy-of select="analyzerOptions"/>
       <xsl:copy-of select="/Environment/Software/Directories"/>  
 
     </agentexec>

tools/wutool/wutool.cpp

@@ -220,8 +220,7 @@ static void process(IConstWorkUnit &w, IProperties *globals, const StringArray &
     }
     else if (stricmp(action, "analyze")==0)
     {
-        WuAnalyseOptions options; // TODO: allow options to be set from from command line parameters
-        analyseAndPrintIssues(&w, options, globals->getPropBool("UPDATEWU"));
+        analyseAndPrintIssues(&w, globals->getPropBool("UPDATEWU"));
     }
     else if (stricmp(action, "dump")==0)
     {