HPCC-11072 Add total time for roxie and hthor

The existing code calculates "Total thor time" for thor
cluster by adding all of graph times togethor. In this
fix, similar to the "Total thor time" for thor, "Total
cluster time" is calculated for both roxie and hthor.
Both "Total thor time" and "Total cluster time" are
reported inside the WsWorkunits.WUINfo response using
new name: "TotalClusterTime".

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

common/workunit/workunit.cpp

@@ -2288,6 +2288,8 @@ mapEnums workunitSortFields[] =
    { WUSFfileread, "FilesRead/File/@name" },
    { WUSFroxiecluster, "RoxieQueryInfo/@roxieClusterName" },
    { WUSFtotalthortime, "Statistics/Statistic[@c='summary'][@creator='thor'][@kind='TimeElapsed']/@value|"
+                        "Statistics/Statistic[@c='summary'][@creator='hthor'][@kind='TimeElapsed']/@value|"
+                        "Statistics/Statistic[@c='summary'][@creator='roxie'][@kind='TimeElapsed']/@value|"
                         "Statistics/Statistic[@desc='Total thor time']/@value|"
                         "Timings/Timing[@name='Total thor time']/@duration"                                 //Use Statistics first. If not found, use Timings
    },
@@ -10583,6 +10585,17 @@ extern WORKUNIT_API void updateWorkunitTimings(IWorkUnit * wu, ITimeReporter *ti
     }
 }
 
+extern WORKUNIT_API void getWorkunitTotalTime(IConstWorkUnit* workunit, const char* creator, unsigned& totalTimeNs, unsigned& totalThisTimeNs)
+{
+    StatisticsFilter summaryTimeFilter(SCTsummary, creator, SSTglobal, GLOBAL_SCOPE, SMeasureTimeNs, StTimeElapsed);
+    Owned<IConstWUStatistic> totalThorTime = getStatistic(workunit, summaryTimeFilter);
+    Owned<IConstWUStatistic> totalThisThorTime = workunit->getStatistic(queryStatisticsComponentName(), GLOBAL_SCOPE, StTimeElapsed);
+    if (totalThorTime)
+        totalTimeNs = totalThorTime->getValue();
+    if (totalThisThorTime)
+        totalThisTimeNs = totalThisThorTime->getValue();
+}
+
 extern WORKUNIT_API void addTimeStamp(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind)
 {
     wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);

common/workunit/workunit.hpp

@@ -1438,6 +1438,7 @@ extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, c
 interface ITimeReporter;
 extern WORKUNIT_API void updateWorkunitTimeStat(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * description, unsigned __int64 value);
 extern WORKUNIT_API void updateWorkunitTimings(IWorkUnit * wu, ITimeReporter *timer);
+extern WORKUNIT_API void getWorkunitTotalTime(IConstWorkUnit* workunit, const char* creator, unsigned& totalTimeNs, unsigned& totalThisTimeNs);
 extern WORKUNIT_API IConstWUStatistic * getStatistic(IConstWorkUnit * wu, const IStatisticsFilter & filter);
 
 extern WORKUNIT_API const char *getTargetClusterComponentName(const char *clustname, const char *processType, StringBuffer &name);

ecl/eclagent/eclgraph.cpp

@@ -1231,7 +1231,15 @@ void EclGraph::execute(const byte * parentExtract)
         StringBuffer description;
         formatGraphTimerLabel(description, queryGraphName(), 0, 0);
 
-        updateWorkunitTimeStat(wu, SSTgraph, queryGraphName(), StTimeElapsed, description.str(), milliToNano(elapsed));
+        unsigned totalTimeNs = 0;
+        unsigned totalThisTimeNs = 0;
+        unsigned elapsedNs = milliToNano(elapsed);
+        const char *totalTimeStr = "Total cluster time";
+        getWorkunitTotalTime(wu, "hthor", totalTimeNs, totalThisTimeNs);
+
+        updateWorkunitTimeStat(wu, SSTgraph, queryGraphName(), StTimeElapsed, description.str(), elapsedNs);
+        updateWorkunitTimeStat(wu, SSTglobal, GLOBAL_SCOPE, StTimeElapsed, NULL, totalThisTimeNs+elapsedNs);
+        wu->setStatistic(SCTsummary, "hthor", SSTglobal, GLOBAL_SCOPE, StTimeElapsed, totalTimeStr, totalTimeNs+elapsedNs, 1, 0, StatsMergeReplace);
     }
 
     if (run)

esp/scm/ws_workunits.ecm

@@ -194,7 +194,8 @@ ESPStruct [nil_remove] ECLWorkunit
     [min_ver("1.30")] bool ThorLCR(false);
     [min_ver("1.01")] int EventSchedule;
     [min_ver("1.02")] bool HaveSubGraphTimings;
-    [min_ver("1.28")] string TotalThorTime;
+    [min_ver("1.28"), depr_ver("1.53")] string TotalThorTime;
+    [min_ver("1.53")] string TotalClusterTime;
 
     ESPstruct ECLQuery Query;
     [min_ver("1.03")] ESParray<ESPstruct ECLHelpFile> Helpers;
@@ -1564,7 +1565,7 @@ ESPresponse [exceptions_inline] WUCheckFeaturesResponse
 };
 
 ESPservice [
-    version("1.52"), default_client_version("1.52"),
+    version("1.53"), default_client_version("1.53"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -392,12 +392,13 @@ void WsWuInfo::getTimers(IEspECLWorkunit &info, unsigned flags)
                 cur.getDescription(name, true);
                 cur.getScope(scope);
 
-                bool isThorTiming = false;
+                bool isThorTiming = false;//Should it be renamed as isClusterTiming?
                 if ((cur.getCreatorType() == SCTsummary) && (cur.getKind() == StTimeElapsed) && streq(scope.str(), GLOBAL_SCOPE))
                 {
                     SCMStringBuffer creator;
                     cur.getCreator(creator);
-                    if (streq(creator.str(), "thor"))
+                    if (streq(creator.str(), "thor") || streq(creator.str(), "hthor") ||
+                        streq(creator.str(), "roxie"))
                         isThorTiming = true;
                 }
                 else if (strieq(name.str(), TOTALTHORTIME)) // legacy
@@ -419,7 +420,10 @@ void WsWuInfo::getTimers(IEspECLWorkunit &info, unsigned flags)
             formatStatistic(totalThorTimeText, totalThorTimeValue, SMeasureTimeNs);
 
             Owned<IEspECLTimer> t= createECLTimer("","");
-            t->setName(TOTALTHORTIME);
+            if (version > 1.52)
+                t->setName(TOTALCLUSTERTIME);
+            else
+                t->setName(TOTALTHORTIME);
             t->setValue(totalThorTimeText.str());
             t->setCount(totalThorTimerCount);
             timers.append(*t.getLink());
@@ -975,7 +979,10 @@ void WsWuInfo::getCommon(IEspECLWorkunit &info, unsigned flags)
         {
             StringBuffer totalThorTimeStr;
             formatDuration(totalThorTimeStr, totalThorTimeMS);
-            info.setTotalThorTime(totalThorTimeStr.str());
+            if (version > 1.52)
+                info.setTotalClusterTime(totalThorTimeStr.str());
+            else
+                info.setTotalThorTime(totalThorTimeStr.str());
         }
     }
 

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -46,6 +46,7 @@ namespace ws_workunits {
 #define    File_ArchiveQuery "ArchiveQuery"
 
 #define    TOTALTHORTIME    "Total thor time"
+#define    TOTALCLUSTERTIME "Total cluster time" //for roxie and hthor
 
 #define    TEMPZIPDIR "tempzipfiles"
 

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -1799,7 +1799,7 @@ void doWUQueryWithSort(IEspContext &context, IEspWUQueryRequest & req, IEspWUQue
             sortorder[0] = WUSFprotected;
         else if (strieq(sortby, "State"))
             sortorder[0] = WUSFstate;
-        else if (strieq(sortby, "ThorTime"))
+        else if (strieq(sortby, "ClusterTime"))
             sortorder[0] = (WUSortField) (WUSFtotalthortime+WUSFnumeric);
         else
             sortorder[0] = WUSFwuid;

roxie/ccd/ccdcontext.cpp

@@ -1424,11 +1424,18 @@ public:
                 graph->abort();
             if (workUnit)
             {
+                unsigned totalTimeNs = 0;
+                unsigned totalThisTimeNs = 0;
+                const char *totalTimeStr = "Total cluster time";
+                getWorkunitTotalTime(workUnit, "roxie", totalTimeNs, totalThisTimeNs);
+
                 const char * graphName = graph->queryName();
                 StringBuffer graphDesc;
                 formatGraphTimerLabel(graphDesc, graphName);
                 WorkunitUpdate progressWorkUnit(&workUnit->lock());
                 updateWorkunitTimeStat(progressWorkUnit, SSTgraph, graphName, StTimeElapsed, graphDesc, elapsedTime);
+                updateWorkunitTimeStat(progressWorkUnit, SSTglobal, GLOBAL_SCOPE, StTimeElapsed, NULL, totalThisTimeNs+elapsedTime);
+                progressWorkUnit->setStatistic(SCTsummary, "roxie", SSTglobal, GLOBAL_SCOPE, StTimeElapsed, totalTimeStr, totalTimeNs+elapsedTime, 1, 0, StatsMergeReplace);
             }
             graph->reset();
             graph.clear();

thorlcr/master/thgraphmanager.cpp

@@ -665,13 +665,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName,
     cycle_t startCycles = get_cycles_now();
     unsigned totalTimeNs = 0;
     unsigned totalThisTimeNs = 0;
-    StatisticsFilter summaryTimeFilter(SCTsummary, "thor", SSTglobal, GLOBAL_SCOPE, SMeasureTimeNs, StTimeElapsed);
-    Owned<IConstWUStatistic> totalThorTime = getStatistic(&workunit, summaryTimeFilter);
-    Owned<IConstWUStatistic> totalThisThorTime = workunit.getStatistic(queryStatisticsComponentName(), GLOBAL_SCOPE, StTimeElapsed);
-    if (totalThorTime)
-        totalTimeNs = totalThorTime->getValue();
-    if (totalThisThorTime)
-        totalThisTimeNs = totalThisThorTime->getValue();
+    getWorkunitTotalTime(&workunit, "thor", totalTimeNs, totalThisTimeNs);
 
     Owned<IConstWUQuery> query = workunit.getQuery(); 
     SCMStringBuffer soName;