Merge pull request #13149 from shamser/issue22950

HPCC-22950 Add the concept of blocked time

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/thorhelper/thorcommon.cpp

@@ -1776,6 +1776,8 @@ void ActivityTimeAccumulator::addStatistics(IStatisticGatherer & builder) const
         builder.addStatistic(StTimeElapsed, elapsed());
         builder.addStatistic(StTimeTotalExecute, cycle_to_nanosec(totalCycles));
         builder.addStatistic(StTimeFirstExecute, latency());
+        if (blockedCycles)
+            builder.addStatistic(StTimeBlocked, cycle_to_nanosec(blockedCycles));
     }
 }
 
@@ -1787,6 +1789,8 @@ void ActivityTimeAccumulator::addStatistics(CRuntimeStatisticCollection & merged
         merged.mergeStatistic(StTimeElapsed, elapsed());
         merged.mergeStatistic(StTimeTotalExecute, cycle_to_nanosec(totalCycles));
         merged.mergeStatistic(StTimeFirstExecute, latency());
+        if (blockedCycles)
+            merged.mergeStatistic(StTimeBlocked, cycle_to_nanosec(blockedCycles));
     }
 }
 
@@ -1809,6 +1813,7 @@ void ActivityTimeAccumulator::merge(const ActivityTimeAccumulator & other)
             if (endCycles < other.endCycles)
                 endCycles = other.endCycles;
             totalCycles += other.totalCycles;
+            blockedCycles += other.blockedCycles;
         }
         else
             *this = other;

common/thorhelper/thorcommon.hpp

@@ -235,6 +235,7 @@ public:
     cycle_t endCycles;   // Wall clock time of last entry to this activity
     unsigned __int64 firstRow; // Timestamp of first row (nanoseconds since epoch)
     cycle_t firstExitCycles;    // Wall clock time of first exit from this activity
+    cycle_t blockedCycles;  // Time spent blocked
 
     // Return the total amount of time (in nanoseconds) spent in this activity (first entry to last exit)
     inline unsigned __int64 elapsed() const { return cycle_to_nanosec(endCycles-startCycles); }
@@ -253,6 +254,7 @@ public:
         endCycles = 0;
         firstRow = 0;
         firstExitCycles = 0;
+        blockedCycles = 0;
     }
 };
 
@@ -323,6 +325,32 @@ public:
         }
     }
 };
+
+class BlockedActivityTimer
+{
+    unsigned __int64 startCycles;
+    ActivityTimeAccumulator &accumulator;
+protected:
+    const bool enabled;
+public:
+    BlockedActivityTimer(ActivityTimeAccumulator &_accumulator, const bool _enabled)
+        : accumulator(_accumulator), enabled(_enabled)
+    {
+        if (enabled)
+            startCycles = get_cycles_now();
+        else
+            startCycles = 0;
+    }
+
+    ~BlockedActivityTimer()
+    {
+        if (enabled)
+        {
+            cycle_t elapsedCycles = get_cycles_now() - startCycles;
+            accumulator.blockedCycles += elapsedCycles;
+        }
+    }
+};
 #else
 struct ActivityTimer
 {
@@ -332,6 +360,10 @@ struct SimpleActivityTimer
 {
     inline SimpleActivityTimer(cycle_t &_accumulator, const bool _enabled) { }
 };
+struct BlockedActivityTimer
+{
+    inline BlockedActivityTimer(ActivityTimeAccumulator &_accumulator, const bool _enabled) { }
+};
 #endif
 
 class THORHELPER_API IndirectCodeContext : implements ICodeContext

system/jlib/jstatcodes.h

@@ -229,6 +229,8 @@ enum StatisticKind
     StCycleTotalCycles,
     StSizeOsDiskRead,
     StSizeOsDiskWrite,
+    StTimeBlocked,
+    StCycleBlockedCycles,
     StMax,
 
     //For any quantity there is potentially the following variants.

system/jlib/jstats.cpp

@@ -877,6 +877,8 @@ static const StatisticMeta statsMetaData[StMax] = {
     { CYCLESTAT(Total) },
     { SIZESTAT(OsDiskRead) },
     { SIZESTAT(OsDiskWrite) },
+    { TIMESTAT(Blocked) },
+    { CYCLESTAT(Blocked) }
 };
 
 

thorlcr/activities/aggregate/thaggregateslave.cpp

@@ -143,7 +143,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eof = false;
     }
@@ -154,7 +154,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon || eof)
             return NULL;
         eof = true;
@@ -240,7 +240,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         aggrowif.setown(createRowInterfaces(helper->queryAggregateRecordSize()));
         partResult.setAllocator(aggrowif->queryRowAllocator()).ensureRow();
@@ -265,7 +265,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (inputStopped) // JCSMORE - this should not be necessary, nextRow() should never be called after stop()
             return NULL;
         OwnedConstThorRow row = inputStream->ungroupedNextRow();

thorlcr/activities/aggregate/thgroupaggregateslave.cpp

@@ -34,14 +34,14 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eof = false;
         ungroupedExistsAggregate = (container.getKind() == TAKexistsaggregate) && !input->isGrouped();
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon || eof)
             return NULL;
         RtlDynamicRowBuilder out(queryRowAllocator());

thorlcr/activities/apply/thapplyslave.cpp

@@ -41,7 +41,7 @@ public:
                 helper->start();
             while(!abortSoon)
             {
-                ActivityTimer t(totalCycles, timeActivities);
+                ActivityTimer t(slaveTimerStats, timeActivities);
                 OwnedConstThorRow r = inputStream->ungroupedNextRow();
                 if (!r)
                     break;

thorlcr/activities/catch/thcatchslave.cpp

@@ -55,12 +55,12 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CCatchSlaveActivityBase::start();
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!eos)
         {
             try
@@ -98,7 +98,7 @@ public:
         {
             try
             {
-                ActivityTimer t(totalCycles, timeActivities);
+                ActivityTimer t(slaveTimerStats, timeActivities);
                 OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
                 if (ret && wasCompleteMatch)
                     dataLinkIncrement();
@@ -206,14 +206,14 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CCatchSlaveActivityBase::start();
         running = gathered = false;
         grouped = input->isGrouped();
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eos)
             return NULL;
         if (!gathered)

thorlcr/activities/choosesets/thchoosesetsslave.cpp

@@ -69,13 +69,13 @@ public:
     LocalChooseSetsActivity(CGraphElementBase *container) : BaseChooseSetsActivity(container) { }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         ActPrintLog("CHOOSESETS: Is Local");
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;)
         {
             OwnedConstThorRow row = inputStream->ungroupedNextRow();
@@ -151,7 +151,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         ActPrintLog("CHOOSESETS: Is Global");
         PARENT::start();
 
@@ -172,7 +172,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (first) 
         {
             first = false;
@@ -283,7 +283,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         ActPrintLog("CHOOSESETS: Is Global");
         if (counts)
         {
@@ -413,7 +413,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (first) 
         {
             first = false;
@@ -492,7 +492,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (first) 
         {
             first = false;

thorlcr/activities/countproject/thcountprojectslave.cpp

@@ -53,14 +53,14 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         ActPrintLog("COUNTPROJECT: Is Local");
         anyThisGroup = false;
         PARENT::start();
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         while (!abortSoon)
         {
             OwnedConstThorRow row(inputStream->nextRow());
@@ -158,7 +158,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         localRecCount = RCUNSET;
         onInputFinishSends = true;
         PARENT::start();
@@ -193,7 +193,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (first) 
         {
             first = false;

thorlcr/activities/csvread/thcsvrslave.cpp

@@ -389,7 +389,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (out)
         {
             OwnedConstThorRow row = out->nextRow();
@@ -412,7 +412,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDiskReadSlaveActivityBase::start();
         if (headerLines)
         {

thorlcr/activities/degroup/thdegroupslave.cpp

@@ -32,13 +32,13 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         if(!input->isGrouped()) ActPrintLog("DEGROUP: Degrouping non-grouped input!");
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon)
         {
             OwnedConstThorRow row = inputStream->ungroupedNextRow();
@@ -58,7 +58,7 @@ public:
     }
     virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon)
         {
             OwnedConstThorRow row = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -621,7 +621,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDiskReadSlaveActivityRecord::start();
         if (helper->getFlags() & TDRlimitskips)
             limit = RCMAX;
@@ -651,7 +651,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (NULL == out) // guard against, but shouldn't happen
             return NULL;
         OwnedConstThorRow ret = out->nextRow();
@@ -782,7 +782,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDiskReadSlaveActivityRecord::start();
         if (helper->getFlags() & TDRlimitskips)
             limit = RCMAX;
@@ -806,7 +806,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!out)
             return NULL;
         OwnedConstThorRow ret = out->nextRow();
@@ -916,7 +916,7 @@ public:
     virtual bool isGrouped() const override { return false; }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDiskReadSlaveActivityRecord::start();
         eoi = hadElement = false;
     }
@@ -930,7 +930,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eoi)
             return NULL;
         RtlDynamicRowBuilder row(allocator);
@@ -1029,7 +1029,7 @@ public:
     virtual bool isGrouped() const override { return false; }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDiskReadSlaveActivityRecord::start();
         stopAfter = (rowcount_t)helper->getChooseNLimit();
         if (!helper->hasFilter())
@@ -1051,7 +1051,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eoi)
             return NULL;
         unsigned __int64 totalCount = 0;
@@ -1150,7 +1150,7 @@ public:
 // IThorDataLink
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDiskReadSlaveActivityRecord::start();
         gathered = eoi = false;
         localAggTable.setown(createRowAggregator(*this, *helper, *helper));
@@ -1172,7 +1172,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eoi)
             return NULL;
         if (!gathered)

thorlcr/activities/enth/thenthslave.cpp

@@ -101,7 +101,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         IHThorEnthArg *helper = static_cast <IHThorEnthArg *> (queryHelper());
         counter = 0;
@@ -143,7 +143,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (localCountReq)
         {
             localCountReq = false;
@@ -237,7 +237,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (first)
         {
             first = false;

thorlcr/activities/external/thexternalslave.cpp

@@ -97,7 +97,7 @@ public:
     CATCH_NEXTROW()
     {
         assertex(rows);
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow row = rows->nextRow();
         if (row)
             dataLinkIncrement();

thorlcr/activities/fetch/thfetchslave.cpp

@@ -369,7 +369,7 @@ public:
 // IThorDataLink impl.
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
 
         if (!keyRowAllocator && fetchBaseHelper->extractAllJoinFields())
@@ -493,7 +493,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon)
             return NULL;
 

thorlcr/activities/filter/thfilterslave.cpp

@@ -64,7 +64,7 @@ public:
     }
     virtual void start() override
     {   
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         matched = 0;
         if (helper->canMatchAny())
             PARENT::start();
@@ -77,7 +77,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         while (!abortSoon)
         {
             OwnedConstThorRow row = inputStream->nextRow();
@@ -109,7 +109,7 @@ public:
     }
     const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         while (!abortSoon)
         {
             OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
@@ -175,7 +175,7 @@ public:
     }
     virtual void start() override
     {   
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         recordCount = 0;
         if (helper->canMatchAny())
             PARENT::start();
@@ -188,7 +188,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         while (!abortSoon)
         {
             OwnedConstThorRow row = inputStream->nextRow();
@@ -256,7 +256,7 @@ public:
     }
     virtual void start() override
     {   
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         if (helper->canMatchAny())
             PARENT::start();
         else
@@ -268,7 +268,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         while (!abortSoon)
         {
             if (groupStream)
@@ -317,7 +317,7 @@ public:
     }
     const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon)
             return NULL;
 

thorlcr/activities/firstn/thfirstnslave.cpp

@@ -48,7 +48,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         stopped = false;
     }
@@ -94,7 +94,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon)
         {
             if (firstget)
@@ -148,7 +148,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon)
         {
             for (;;)
@@ -250,7 +250,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start(); // adds to totalTime (common to local and global firstn)
 
         limit = maxres = RCUNBOUND;
@@ -344,7 +344,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon)
         {
             if (firstget)

thorlcr/activities/funnel/thfunnelslave.cpp

@@ -292,7 +292,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         if (!grouped && parallel)
         {
             //NB starts inputs on each thread
@@ -390,7 +390,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (parallelOutput)
         {
             OwnedConstThorRow row = parallelOutput->nextRow();
@@ -473,7 +473,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         eogNext = false;
         ForEachItemIn(i, inputs)
         {
@@ -493,7 +493,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;)
         {
             bool eog = false;
@@ -573,7 +573,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         curinput = 0;
         eogNext = false;
         ForEachItemIn(i, inputs)
@@ -594,7 +594,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         unsigned n = inputs.ordinality();
         IRowStream *current = queryInputStream(curinput);
         for (;;)
@@ -696,7 +696,7 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         curinput = 0;
         anyThisGroup = anyThisInput = eogNext = false;
         ForEachItemIn(i, inputs)
@@ -718,7 +718,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eoi) 
             return NULL; 
         if (eogNext) { 
@@ -776,7 +776,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
 
         unsigned whichInput = helper->getInputIndex();
         selectedInput = nullptr;
@@ -829,7 +829,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!selectedStream)
             return NULL;
         OwnedConstThorRow ret = selectedStream->nextRow();
@@ -855,7 +855,7 @@ public:
     }
     virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!selectedStream)
             return NULL;
         return selectedStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
@@ -904,7 +904,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         bool selectionIsAll;
         size32_t selectionLen;
         rtlDataAttr selection;

thorlcr/activities/group/thgroupslave.cpp

@@ -75,7 +75,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         ActPrintLog(rolloverEnabled ? "GROUP: is global" : "GROUP: is local");
         PARENT::start();
         eogNext = prevEog = eof = false;
@@ -136,7 +136,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eogNext || eof)
         {
             eogNext = false;

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -2100,7 +2100,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eofin = false;
         instrm.set(inputStream);
@@ -2136,7 +2136,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities); // careful not to call again in derivatives
+        ActivityTimer t(slaveTimerStats, timeActivities); // careful not to call again in derivatives
         if (abortSoon||eofin)
         {
             eofin = true;
@@ -2440,7 +2440,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         bool passthrough;
         Owned<IRowStream> calcStream = partitioner->calc(this, input, inputStream, passthrough);  // may return NULL
@@ -3115,7 +3115,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eos = lastEog = false;
         ThorDataLinkMetaInfo info;
@@ -3150,7 +3150,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eos)
             return NULL;
 
@@ -3789,7 +3789,7 @@ public:
     virtual void start() override
     {
         HashDedupSlaveActivityBase::start();
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         Owned<IThorRowInterfaces> myRowIf = getRowInterfaces(); // avoiding circular link issues
         instrm.setown(distributor->connect(myRowIf, distInput, iHash, iCompare, keepBestCompare));
         distInput = instrm.get();
@@ -3883,7 +3883,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         startAllInputs();
         leftdone = false;
         eof = false;
@@ -3990,7 +3990,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!eof) {
             OwnedConstThorRow row = joinhelper->nextRow();
             if (row) {
@@ -4293,7 +4293,7 @@ IAggregateTable *createRowAggregator(CActivityBase &activity, IHThorHashAggregat
     return new CAggregateHT(activity, extra, helper);
 }
 
-IRowStream *mergeLocalAggs(Owned<IHashDistributor> &distributor, CActivityBase &activity, IHThorRowAggregator &helper, IHThorHashAggregateExtra &helperExtra, IRowStream *localAggStream, mptag_t mptag)
+IRowStream *mergeLocalAggs(Owned<IHashDistributor> &distributor, CSlaveActivity &activity, IHThorRowAggregator &helper, IHThorHashAggregateExtra &helperExtra, IRowStream *localAggStream, mptag_t mptag)
 {
     Owned<IRowStream> strm;
     ICompare *elementComparer = helperExtra.queryCompareElements();
@@ -4313,9 +4313,10 @@ IRowStream *mergeLocalAggs(Owned<IHashDistributor> &distributor, CActivityBase &
         ICompare &cmp;
         IHThorRowAggregator &helper;
         IHashDistributor &distributor;
+        CSlaveActivity &activity;
     public:
-        CAggregatingStream(IHThorRowAggregator &_helper, IEngineRowAllocator &_rowAllocator, ICompare &_cmp, IHashDistributor &_distributor)
-            : helper(_helper), rowAllocator(_rowAllocator), cmp(_cmp), distributor(_distributor), rowBuilder(_rowAllocator)
+        CAggregatingStream(IHThorRowAggregator &_helper, IEngineRowAllocator &_rowAllocator, ICompare &_cmp, IHashDistributor &_distributor, CSlaveActivity &_activity)
+            : helper(_helper), rowAllocator(_rowAllocator), cmp(_cmp), distributor(_distributor), rowBuilder(_rowAllocator), activity(_activity)
         {
         }
         void start(IRowStream *_input)
@@ -4327,7 +4328,11 @@ IRowStream *mergeLocalAggs(Owned<IHashDistributor> &distributor, CActivityBase &
         {
             for (;;)
             {
-                OwnedConstThorRow row = input->nextRow();
+                OwnedConstThorRow row;
+                {
+                    BlockedActivityTimer t(activity.slaveTimerStats, activity.queryTimeActivities());
+                    row.setown(input->nextRow());
+                }
                 if (!row)
                 {
                     if (sz)
@@ -4360,11 +4365,12 @@ IRowStream *mergeLocalAggs(Owned<IHashDistributor> &distributor, CActivityBase &
             rowBuilder.clear();
             input->stop();
             input.clear();
+            BlockedActivityTimer t(activity.slaveTimerStats, activity.queryTimeActivities());
             distributor.disconnect(true);
             distributor.join();
         }
     };
-    CAggregatingStream *mergeStrm = new CAggregatingStream(helper, *rowAllocator, *elementComparer, *distributor.get());
+    CAggregatingStream *mergeStrm = new CAggregatingStream(helper, *rowAllocator, *elementComparer, *distributor.get(), activity);
     mergeStrm->start(strm.getClear());
     return mergeStrm;
 }
@@ -4435,7 +4441,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         doNextGroup(); // or local set if !grouped
         if (!container.queryGrouped())
@@ -4469,7 +4475,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eos) return nullptr;
         const void *next = aggregateStream->nextRow();
         if (next)
@@ -4524,12 +4530,12 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow row = inputStream->ungroupedNextRow();
         if (!row)
             return NULL;

thorlcr/activities/hashdistrib/thhashdistribslave.ipp

@@ -52,7 +52,7 @@ interface IAggregateTable : extends IInterface
     virtual IRowStream *getRowStream(bool sorted) = 0;
 };
 IAggregateTable *createRowAggregator(CActivityBase &activity, IHThorHashAggregateExtra &extra, IHThorRowAggregator &helper);
-IRowStream *mergeLocalAggs(Owned<IHashDistributor> &distributor, CActivityBase &activity, IHThorRowAggregator &helper, IHThorHashAggregateExtra &helperExtra, IRowStream *localAggTable, mptag_t mptag);
+IRowStream *mergeLocalAggs(Owned<IHashDistributor> &distributor, CSlaveActivity &activity, IHThorRowAggregator &helper, IHThorHashAggregateExtra &helperExtra, IRowStream *localAggTable, mptag_t mptag);
 
 activityslaves_decl CActivityBase *createHashDistributeSlave(CGraphElementBase *container);
 activityslaves_decl CActivityBase *createNWayDistributeSlave(CGraphElementBase *container);

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -646,7 +646,7 @@ public:
     // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         keyedProcessed = 0;
         if (!eoi)
@@ -862,7 +862,7 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
 
         needTransform = helper->needTransform();
 
@@ -902,7 +902,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (RCMAX != keyedLimitCount)
         {
             bool limitHit;
@@ -1027,7 +1027,7 @@ public:
     virtual bool isGrouped() const override { return false; }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         localAggTable.setown(createRowAggregator(*this, *helper, *helper));
         localAggTable->init(queryRowAllocator());
@@ -1037,7 +1037,7 @@ public:
 // IRowStream
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (done)
             return NULL;
         if (!gathered)
@@ -1153,7 +1153,7 @@ public:
     virtual bool isGrouped() const override { return false; }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
 
         // NB: initLimits sets up remoteLimit before base start() call, because if parts are remote PARENT::start() will use remoteLimit
         initLimits(helper->getChooseNLimit(), helper->getKeyedLimit(), helper->getRowLimit(), helper->hasMatchFilter());
@@ -1172,7 +1172,7 @@ public:
 // IRowStream
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (done)
             return nullptr;
         done = true;
@@ -1328,7 +1328,7 @@ public:
     virtual bool isGrouped() const override { return false; }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
 
         // NB: initLimits sets up remoteLimit before base start() call, because if parts are remote PARENT::start() will use remoteLimit
         initLimits(helper->getChooseNLimit(), helper->getKeyedLimit(), helper->getRowLimit(), helper->hasMatchFilter());
@@ -1352,7 +1352,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (RCMAX != keyedLimitCount)
         {
             bool limitHit;
@@ -1447,7 +1447,7 @@ public:
     virtual bool isGrouped() const override { return false; }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         hadElement = false;
         done = false;
@@ -1456,7 +1456,7 @@ public:
 // IRowStream
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (done)
             return nullptr;
         done = true;

thorlcr/activities/iterate/thgroupiterateslave.cpp

@@ -40,7 +40,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         anyThisGroup = false;
         eogNext = false;    
@@ -51,7 +51,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;) {
             if(abortSoon)
                 break;          
@@ -124,7 +124,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         RtlDynamicRowBuilder r(rightAllocator);
         size32_t sz = helper->createInitialRight(r);  
@@ -135,7 +135,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;) {
             if(abortSoon)
                 break;          

thorlcr/activities/iterate/thiterateslave.cpp

@@ -77,7 +77,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         if (global) // only want lookahead if global (hence serial)
         {
@@ -119,7 +119,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;) {
             if (eof || abortSoon)
                 break;
@@ -199,7 +199,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;) {
             if (eof || abortSoon)
                 break;
@@ -282,14 +282,14 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eof = !container.queryLocalOrGrouped() && !firstNode();
         count = 0;
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!eof) {
             if (count==0)
                 eof = !helper->first();
@@ -334,13 +334,13 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         dohere = container.queryLocalOrGrouped() || firstNode();
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (dohere)
         {
             OwnedConstThorRow row;
@@ -407,7 +407,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eof || abortSoon)
             return NULL;
         assertex(rows);

thorlcr/activities/join/thjoinslave.cpp

@@ -284,7 +284,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
 
         CAsyncCallStart asyncSecondaryStart(std::bind(&JoinSlaveActivity::startSecondaryInput, this));
         try
@@ -418,7 +418,7 @@ public:
 
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if(joinhelper) 
         {
             OwnedConstThorRow row = joinhelper->nextRow();
@@ -688,7 +688,7 @@ public:
     }
     virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         bool matched = true;
         OwnedConstThorRow next = processor.nextGE(seek, numFields, matched, stepExtra);
         if (next)

thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp

@@ -2093,7 +2093,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         assertex(inputs.ordinality() == 1);
         PARENT::start();
 
@@ -2199,7 +2199,7 @@ public:
 
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon && !eos)
         {
             for (;;)

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -2970,7 +2970,7 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         assertex(inputs.ordinality() == 1);
         PARENT::start();
 
@@ -3016,7 +3016,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow ret;
         while (!abortSoon && !eos)
         {

thorlcr/activities/limit/thlimitslave.cpp

@@ -50,7 +50,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         resultSent = container.queryLocal(); // i.e. local, so don't send result to master
         eos = anyThisGroup = eogNext = false;
@@ -89,7 +89,7 @@ public:
     
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eos)
             return NULL;
         while (!abortSoon && !eogNext)
@@ -131,7 +131,7 @@ public:
     }
     const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
         if (ret)
         {
@@ -238,7 +238,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eof) 
             return NULL;
         if (!limitChecked)

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -1699,7 +1699,7 @@ protected:
     using PARENT::broadcaster;
     using PARENT::inputs;
     using PARENT::queryHelper;
-    using PARENT::totalCycles;
+    using PARENT::slaveTimerStats;
     using PARENT::timeActivities;
     using PARENT::fireException;
     using PARENT::lookupNextRow;
@@ -2734,7 +2734,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!gotRHS)
         {
             getRHS(false);
@@ -3320,7 +3320,7 @@ public:
 // IThorSlaveActivity overloaded methods
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!gotRHS)
             getRHS(false);
         OwnedConstThorRow row = lookupNextRow();

thorlcr/activities/loop/thloopslave.cpp

@@ -273,7 +273,7 @@ public:
 // IThorDataLink
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eof = false;
         helper->createParentExtract(extractBuilder);
@@ -297,7 +297,7 @@ public:
     }
     const void *getNextRow(bool stopping)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon && !eof)
         {
             unsigned emptyIterations = 0;
@@ -477,7 +477,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         executed = false;
         maxIterations = helper->numIterations();
@@ -572,7 +572,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         curRow = 0;
         abortSoon = false;
@@ -598,7 +598,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon)
         {
             OwnedConstThorRow row = resultStream->nextRow();
@@ -654,7 +654,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         lastNull = eoi = false;
         abortSoon = false;
@@ -670,7 +670,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon && !eoi)
             return NULL;
         OwnedConstThorRow row = inputStream->nextRow();
@@ -866,7 +866,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         stopUnselectedInputs();
         if (queryInput(branch))
         {
@@ -890,7 +890,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon)
             return nullptr;
         if (!selectedInputStream)
@@ -922,7 +922,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         branch = helper->getCondition() ? 0 : 1;
         PARENT::start();
     }
@@ -945,7 +945,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         branch = helper->getBranch();
         if (branch >= queryNumInputs())
             branch = queryNumInputs() - 1;
@@ -975,7 +975,7 @@ public:
     virtual void process() override
     {
         processed = THORDATALINK_STARTED;
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         cond = helper->getCondition();
         if (cond)
         {
@@ -1030,7 +1030,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         started = false;
         eos = false;
@@ -1099,7 +1099,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eos = false;
     }
@@ -1149,7 +1149,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         gathered = eos = false;
         aggregated.clear();
@@ -1216,7 +1216,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         ok = false;
         numProcessedLastGroup = getDataLinkCount(); // is this right?
@@ -1296,7 +1296,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         abortSoon = false;
         unsigned sequence = helper->querySequence();
         if ((int)sequence >= 0)
@@ -1318,7 +1318,7 @@ public:
     virtual bool isGrouped() const override { return false; }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon)
         {
             OwnedConstThorRow row = resultStream->nextRow();

thorlcr/activities/merge/thmergeslave.cpp

@@ -283,7 +283,7 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         ForEachItemIn(i, inputs)
         {
             IThorDataLink * input = queryInput(i);
@@ -433,7 +433,7 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         ForEachItemIn(i, inputs)
         {
             IThorDataLink *input = queryInput(i);
@@ -472,7 +472,7 @@ public:
 
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!abortSoon) {
             OwnedConstThorRow row = out->nextRow();
             if (row) {
@@ -564,7 +564,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow ret = merger.nextRow();
         if (ret)
         {
@@ -580,7 +580,7 @@ public:
     }
     virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow ret = merger.nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
         if (ret)
         {

thorlcr/activities/msort/thgroupsortslave.cpp

@@ -54,7 +54,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         unsigned spillPriority = container.queryGrouped() ? SPILL_PRIORITY_GROUPSORT : SPILL_PRIORITY_LARGESORT;
         iLoader.setown(createThorRowLoader(*this, queryRowInterfaces(input), iCompare, unstable ? stableSort_none : stableSort_earlyAlloc, rc_mixed, spillPriority));
@@ -89,7 +89,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon || eoi)
             return NULL;
         OwnedConstThorRow row = out->nextRow();
@@ -140,12 +140,12 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow ret = inputStream->nextRow();
         if (ret && prev && icompare->docompare(prev, ret) > 0)
         {
@@ -164,7 +164,7 @@ public:
     }
     virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
         if (ret && prev && stepCompare->docompare(prev, ret, numFields) > 0)
         {

thorlcr/activities/msort/thmsortslave.cpp

@@ -82,7 +82,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         try
         {
             try
@@ -199,7 +199,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon) 
             return NULL;
         OwnedConstThorRow row = output->nextRow();

thorlcr/activities/normalize/thnormalizeslave.cpp

@@ -48,7 +48,7 @@ public:
     }
     virtual void start() override
     { 
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         numThisRow = 0;
         curRow = 0;
@@ -56,7 +56,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;)
         {
             while (curRow == numThisRow)
@@ -122,14 +122,14 @@ public:
     virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         anyThisGroup = false;
         curChildRow = NULL;
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;) {
             while(!curChildRow) {
                 curRow = 0;
@@ -210,13 +210,13 @@ public:
     virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         anyThisGroup = false;
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;)
         {
             if (!curParent)

thorlcr/activities/nsplitter/thnsplitterslave.cpp

@@ -295,7 +295,7 @@ public:
             else
                 break;
         }
-        ActivityTimer t(totalCycles, queryTimeActivities());
+        ActivityTimer t(slaveTimerStats, queryTimeActivities());
 
         pagedOut = false;
         OwnedConstThorRow row;
@@ -455,7 +455,7 @@ CSplitterOutput::CSplitterOutput(NSplitterSlaveActivity &_activity, unsigned _ou
 // IStartableEngineRowStream
 void CSplitterOutput::start()
 {
-    ActivityTimer s(totalCycles, activity.queryTimeActivities());
+    ActivityTimer s(slaveTimerStats, activity.queryTimeActivities());
     started = true;
     activity.prepareInput();
     if (1 == activity.activeOutputCount)
@@ -475,7 +475,7 @@ void CSplitterOutput::stop()
 
 const void *CSplitterOutput::nextRow()
 {
-    ActivityTimer t(totalCycles, activity.queryTimeActivities());
+    ActivityTimer t(slaveTimerStats, activity.queryTimeActivities());
     if (rec == max) // NB: max will be RCMAX if activeOutputCount == 1
         max = activity.writeahead(max, activity.queryAbortSoon(), writeBlockSem, outIdx);
     const void *row = activity.nextRow(outIdx, rec); // pass ptr to max if need more

thorlcr/activities/null/thnullslave.cpp

@@ -54,12 +54,12 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
     }
     const void * nextRow() override
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         return NULL;
     }
     virtual bool isGrouped() const override
@@ -88,12 +88,12 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
     }
     const void * nextRow() override
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         return inputStream->nextRow();
     }
     virtual bool isGrouped() const override

thorlcr/activities/nullaction/thnullactionslave.cpp

@@ -39,12 +39,12 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         return NULL;
     }
     virtual bool isGrouped() const override { return false; }

thorlcr/activities/parse/thparseslave.cpp

@@ -63,7 +63,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
     }
     virtual void stop() override
@@ -85,7 +85,7 @@ public:
 
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         while (!abortSoon) {
             if (rowIter->isValid()) {
                 anyThisGroup = true;

thorlcr/activities/piperead/thprslave.cpp

@@ -208,7 +208,7 @@ public:
     }
     CATCH_NEXTROW()
     {   
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eof || abortSoon)
             return NULL;
         try
@@ -245,7 +245,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         OwnedRoxieString xmlIteratorPath(helper->getXmlIteratorPath());
         IThorRowInterfaces *_inrowif = needTransform ? inrowif.get() : this;
@@ -378,7 +378,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         OwnedRoxieString xmlIterator(helper->getXmlIteratorPath());
         readTransformer.setown(createReadRowStream(queryRowAllocator(), queryRowDeserializer(), helper->queryXmlTransformer(), helper->queryCsvTransformer(), xmlIterator, flags));
@@ -402,7 +402,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eof || abortSoon)
             return NULL;
         for (;;)

thorlcr/activities/project/thprojectslave.cpp

@@ -35,7 +35,7 @@ public:
     }
     STRAND_CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         for (;;)
         {
             if (parent.queryAbortSoon())
@@ -264,7 +264,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         preload = helper->getLookahead();
         if (!preload)
@@ -283,7 +283,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eof)
             return NULL;
         for (;;)

thorlcr/activities/pull/thpullslave.cpp

@@ -35,14 +35,14 @@ public:
 // IThorDataLink methods
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         if (ensureStartFTLookAhead(0))
             setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), PULL_SMART_BUFFER_SIZE, true, false, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()), false);
     }
     const void * nextRow() override
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow row = inputStream->nextRow();
         if (!row)
             return NULL;

thorlcr/activities/rollup/throllupslave.cpp

@@ -299,7 +299,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDedupRollupBaseActivity::start();
         keepLeft = ddhelper->keepLeft();
         keepBest = ddhelper->keepBest();
@@ -339,7 +339,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eos)
             return NULL;
         checkFirstRow();
@@ -406,7 +406,7 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDedupBaseSlaveActivity::start();
         assertex(1 == numToKeep);
 
@@ -418,7 +418,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eos)
             return NULL;
 
@@ -471,7 +471,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDedupRollupBaseActivity::start();
     }
     virtual void stop() override
@@ -486,7 +486,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eos) return NULL;
         checkFirstRow();
         if (eog())
@@ -559,13 +559,13 @@ public:
     }
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eoi = false;
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (eoi)
             return NULL;
 

thorlcr/activities/sample/thsampleslave.cpp

@@ -36,7 +36,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eogNext = false;
         anyThisGroup = false;
@@ -46,7 +46,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         while(!abortSoon)
         {
             OwnedConstThorRow row = inputStream->nextRow();

thorlcr/activities/selectnth/thselectnthslave.cpp

@@ -81,7 +81,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
 
         lookaheadN = RCMAX;
         startN = 0; // set by initN()
@@ -129,7 +129,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow ret;
         Owned<IException> exception;
         if (first) // only return 1!

thorlcr/activities/selfjoin/thselfjoinslave.cpp

@@ -155,7 +155,7 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         bool hintunsortedoutput = getOptBool(THOROPT_UNSORTED_OUTPUT, (JFreorderable & helper->getJoinFlags()) != 0);
         bool hintparallelmatch = getOptBool(THOROPT_PARALLEL_MATCH, hintunsortedoutput); // i.e. unsorted, implies use parallel by default, otherwise no point
@@ -214,7 +214,7 @@ public:
     
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if(joinhelper) {
             OwnedConstThorRow row = joinhelper->nextRow();
             if (row) {

thorlcr/activities/soapcall/thsoapcallslave.cpp

@@ -53,7 +53,7 @@ public:
     // IThorDataLink methods
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eof = false;
         if (container.queryLocalOrGrouped() || firstNode())
@@ -83,7 +83,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (!eof && wscHelper.get())
         {
             OwnedConstThorRow row = wscHelper->getRow();
@@ -150,7 +150,7 @@ public:
     // IThorDataLink methods
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eof = false;
         wscHelper.setown(createSoapCallHelper(this, queryRowAllocator(), authToken.str(), SCdataset, NULL, container.queryJob().queryContextLogger(), NULL));
@@ -168,7 +168,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow row = wscHelper->getRow();
         if (row)
         {

thorlcr/activities/spill/thspillslave.cpp

@@ -154,7 +154,7 @@ public:
 // IThorDataLink
     virtual void start()
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         uncompressedBytesWritten = 0;
         if (!container.queryJob().queryUseCheckpoints())
@@ -175,7 +175,7 @@ public:
 
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon) 
             return NULL;
 

thorlcr/activities/temptable/thtmptableslave.cpp

@@ -52,7 +52,7 @@ public:
     virtual bool isGrouped() const override { return false; }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         __uint64 numRows = helper->numRows();
         // local when generated from a child query (the range is per node, don't split)
@@ -80,7 +80,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon)
             return NULL;
         while (currentRow < maxRow) {

thorlcr/activities/topn/thtopnslave.cpp

@@ -184,7 +184,7 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         // NB: topNLimit shouldn't be stupid size, resourcing will guarantee this
         __int64 _topNLimit = helper->getLimit();
@@ -211,7 +211,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon || eos)
             return NULL;
         if (NULL == out)

thorlcr/activities/trace/thtraceslave.cpp

@@ -44,7 +44,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         if (traceEnabled && helper->canMatchAny() && queryRowMetaData())
         {
@@ -89,7 +89,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow ret = inputStream->nextRow();
         if (ret)
         {
@@ -105,7 +105,7 @@ public:
     }
     virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
         if (ret)
         {

thorlcr/activities/when/thwhenslave.cpp

@@ -95,7 +95,7 @@ public:
     virtual bool isGrouped() const override { return input->isGrouped(); }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow row(inputStream->nextRow());
         if (!row)
             return NULL;

thorlcr/activities/wuidread/thwuidreadslave.cpp

@@ -71,7 +71,7 @@ public:
     }
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         eogPending = false;
         if (container.queryLocal() || firstNode())
@@ -79,7 +79,7 @@ public:
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon || rowSource.eos())
             return NULL;
 

thorlcr/activities/xmlparse/thxmlparseslave.cpp

@@ -66,14 +66,14 @@ public:
 // IThorDataLink methods
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         PARENT::start();
         anyThisGroup = false;
         eogNext = false;
     }
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         if (abortSoon)
             return NULL;
         if (eogNext)

thorlcr/activities/xmlread/thxmlreadslave.cpp

@@ -237,7 +237,7 @@ public:
 // IThorDataLink
     virtual void start() override
     {
-        ActivityTimer s(totalCycles, timeActivities);
+        ActivityTimer s(slaveTimerStats, timeActivities);
         CDiskReadSlaveActivityBase::start();
         out = createSequentialPartHandler(partHandler, partDescs, false);
     }
@@ -253,7 +253,7 @@ public:
 
     CATCH_NEXTROW()
     {
-        ActivityTimer t(totalCycles, timeActivities);
+        ActivityTimer t(slaveTimerStats, timeActivities);
         OwnedConstThorRow row = out->nextRow();
         if (!row)
             return NULL;

thorlcr/graph/thgraphmaster.cpp

@@ -362,7 +362,8 @@ void CSlaveMessageHandler::threadmain()
 
 //////////////////////
 
-CMasterActivity::CMasterActivity(CGraphElementBase *_container) : CActivityBase(_container), threaded("CMasterActivity", this), timingInfo(_container->queryJob())
+CMasterActivity::CMasterActivity(CGraphElementBase *_container) : CActivityBase(_container), threaded("CMasterActivity", this), timingInfo(_container->queryJob()),
+                                                                  blockedTime(queryJob(), StTimeBlocked)
 {
     notedWarnings = createThreadSafeBitSet();
     mpTag = TAG_NULL;
@@ -515,9 +516,7 @@ void CMasterActivity::reset()
 void CMasterActivity::deserializeStats(unsigned node, MemoryBuffer &mb)
 {
     CriticalBlock b(progressCrit); // don't think needed
-    unsigned __int64 localTimeNs;
-    mb.read(localTimeNs);
-    timingInfo.set(node, localTimeNs);
+    deserializeActivityStats(node, mb);
     rowcount_t count;
     ForEachItemIn(p, progressInfo)
     {
@@ -526,9 +525,19 @@ void CMasterActivity::deserializeStats(unsigned node, MemoryBuffer &mb)
     }
 }
 
+void CMasterActivity::deserializeActivityStats(unsigned node, MemoryBuffer &mb)
+{
+    unsigned __int64 localTimeNs, blockedTimeNs;
+    mb.read(localTimeNs);
+    mb.read(blockedTimeNs);
+    timingInfo.set(node, localTimeNs);
+    blockedTime.set(node, blockedTimeNs);
+}
+
 void CMasterActivity::getActivityStats(IStatisticGatherer & stats)
 {
     timingInfo.getStats(stats);
+    blockedTime.getStats(stats,false);
 }
 
 void CMasterActivity::getEdgeStats(IStatisticGatherer & stats, unsigned idx)

thorlcr/graph/thgraphmaster.ipp

@@ -291,6 +291,7 @@ class graphmaster_decl CMasterActivity : public CActivityBase, implements IThrea
 protected:
     ProgressInfoArray progressInfo;
     CTimingInfo timingInfo;
+    CThorStats blockedTime;
     IBitSet *notedWarnings;
 
     void addReadFile(IDistributedFile *file, bool temp=false);
@@ -303,6 +304,7 @@ public:
     ~CMasterActivity();
 
     virtual void deserializeStats(unsigned node, MemoryBuffer &mb);
+    virtual void deserializeActivityStats(unsigned node, MemoryBuffer &mb);
     virtual void getActivityStats(IStatisticGatherer & stats);
     virtual void getEdgeStats(IStatisticGatherer & stats, unsigned idx);
     virtual void init();

thorlcr/graph/thgraphslave.cpp

@@ -378,7 +378,7 @@ void CSlaveActivity::start()
 
 void CSlaveActivity::startAllInputs()
 {
-    ActivityTimer s(totalCycles, timeActivities);
+    ActivityTimer s(slaveTimerStats, timeActivities);
     ForEachItemIn(i, inputs)
     {
         try { startInput(i); }
@@ -540,16 +540,26 @@ unsigned __int64 CSlaveActivity::queryLocalCycles() const
                 break;
         }
     }
-    unsigned __int64 _totalCycles = queryTotalCycles();
-    if (_totalCycles < inputCycles) // not sure how/if possible, but guard against
+    unsigned __int64 localCycles = queryTotalCycles();
+    if (localCycles < inputCycles) // not sure how/if possible, but guard against
         return 0;
-    return _totalCycles-inputCycles;
+    localCycles -= inputCycles;
+    const unsigned __int64 blockedCycles = queryBlockedCycles();
+    if (localCycles < blockedCycles)
+        return 0;
+    return localCycles-blockedCycles;
+}
+
+void CSlaveActivity::serializeActivityStats(MemoryBuffer &mb) const
+{
+    mb.append((unsigned __int64)cycle_to_nanosec(queryLocalCycles()));
+    mb.append((unsigned __int64)cycle_to_nanosec(queryBlockedCycles()));
 }
 
 void CSlaveActivity::serializeStats(MemoryBuffer &mb)
 {
     CriticalBlock b(crit);
-    mb.append((unsigned __int64)cycle_to_nanosec(queryLocalCycles()));
+    serializeActivityStats(mb);
     ForEachItemIn(i, outputs)
     {
         IThorDataLink *output = queryOutput(i);

thorlcr/graph/thgraphslave.hpp

@@ -41,14 +41,15 @@ interface IStartableEngineRowStream : extends IEngineRowStream
 class COutputTiming
 {
 public:
-    ActivityTimeAccumulator totalCycles;
+    ActivityTimeAccumulator slaveTimerStats;
 
     COutputTiming() { }
 
-    void resetTiming() { totalCycles.reset(); }
-    ActivityTimeAccumulator &getTotalCyclesRef() { return totalCycles; }
-    unsigned __int64 queryTotalCycles() const { return totalCycles.totalCycles; }
-    unsigned __int64 queryEndCycles() const { return totalCycles.endCycles; }
+    void resetTiming() { slaveTimerStats.reset(); }
+    ActivityTimeAccumulator &getTotalCyclesRef() { return slaveTimerStats; }
+    unsigned __int64 queryTotalCycles() const { return slaveTimerStats.totalCycles; }
+    unsigned __int64 queryEndCycles() const { return slaveTimerStats.endCycles; }
+    unsigned __int64 queryBlockedCycles() const { return slaveTimerStats.blockedCycles; }
 };
 
 class CEdgeProgress
@@ -252,6 +253,7 @@ public:
     void stopInput(unsigned index, const char *extra=NULL);
     void stopAllInputs();
     virtual void serializeStats(MemoryBuffer &mb);
+    virtual void serializeActivityStats(MemoryBuffer &mb) const;
     void debugRequest(unsigned edgeIdx, MemoryBuffer &msg);
     bool canStall() const;
     bool isFastThrough() const;
@@ -277,6 +279,7 @@ public:
         return consumerOrdered;
     }
     virtual unsigned __int64 queryTotalCycles() const { return COutputTiming::queryTotalCycles(); }
+    virtual unsigned __int64 queryBlockedCycles() const { return COutputTiming::queryBlockedCycles();}
     virtual unsigned __int64 queryEndCycles() const { return COutputTiming::queryEndCycles(); }
     virtual void debugRequest(MemoryBuffer &msg) override;
 

thorlcr/slave/slave.cpp

@@ -87,7 +87,7 @@ void ProcessSlaveActivity::threadmain()
 
             // set lastCycles to 0 to signal not processing
             unsigned __int64 finalCycles = lastCycles.exchange(0);
-            totalCycles.totalCycles += get_cycles_now()-finalCycles;
+            slaveTimerStats.totalCycles += get_cycles_now()-finalCycles;
         }
         else
             process();
@@ -175,7 +175,7 @@ void ProcessSlaveActivity::serializeStats(MemoryBuffer &mb)
             //Update lastCycles to the current number of cycles - unless it has been set to 0 in the meantime
             //Use std::memory_order_relaxed because there is no requirement for other variables to be synchronized.
             if (lastCycles.compare_exchange_strong(curCycles, nowCycles, std::memory_order_relaxed))
-                totalCycles.totalCycles += nowCycles-curCycles;
+                slaveTimerStats.totalCycles += nowCycles-curCycles;
         }
     }
 #endif