HPCC-15258 Thor stranded support + stranded project

Signed-off-by: Jake Smith <jake.smith@lexisnexis.com>

thorlcr/activities/aggregate/thaggregateslave.cpp

@@ -29,25 +29,24 @@
 #include "thactivityutil.ipp"
 #include "thaggregateslave.ipp"
 
-class AggregateSlaveBase : public CSlaveActivity, public CThorDataLink
+class AggregateSlaveBase : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
 protected:
     bool hadElement, inputStopped;
-    IThorDataLink *input;
 
     void doStopInput()
     {
         if (inputStopped)
             return;
         inputStopped = true;
-        stopInput(input);
+        PARENT::stop();
     }
-    void doStart()
+    virtual void start() override
     {
+        PARENT::start();
         hadElement = false;
         inputStopped = false;
-        input = inputs.item(0);
-        startInput(input);
         if (input->isGrouped())
             ActPrintLog("Grouped mismatch");
     }
@@ -116,12 +115,8 @@ protected:
         queryJobChannel().queryJobComm().send(mb, dst, mpTag);
     }
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    AggregateSlaveBase(CGraphElementBase *_container)
-        : CSlaveActivity(_container), CThorDataLink(this)
+    AggregateSlaveBase(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
-        input = NULL;
         hadElement = inputStopped = false;
     }
     virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
@@ -130,13 +125,14 @@ public:
             mpTag = container.queryJobChannel().deserializeMPTag(data);
         appendOutputLinked(this);
     }
-    virtual bool isGrouped() { return false; }
 };
 
 //
 
 class AggregateSlaveActivity : public AggregateSlaveBase
 {
+    typedef AggregateSlaveBase PARENT;
+
     bool eof;
     IHThorAggregateArg * helper;
 
@@ -152,17 +148,15 @@ public:
         if (firstNode())
             cancelReceiveMsg(1, mpTag);
     }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        doStart();
+        PARENT::start();
         eof = false;
-        dataLinkStart();
     }
-    virtual void stop()
+    virtual void stop() override
     {
         doStopInput();
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -171,7 +165,7 @@ public:
             return NULL;
         eof = true;
 
-        OwnedConstThorRow next = input->ungroupedNextRow();
+        OwnedConstThorRow next = inputStream->ungroupedNextRow();
         RtlDynamicRowBuilder resultcr(queryRowAllocator());
         size32_t sz = helper->clearAggregate(resultcr);         
         if (next)
@@ -182,7 +176,7 @@ public:
             {
                 while (!abortSoon)
                 {
-                    next.setown(input->ungroupedNextRow());
+                    next.setown(inputStream->ungroupedNextRow());
                     if (!next)
                         break;
                     sz = helper->processNext(resultcr, next);
@@ -218,6 +212,8 @@ public:
 
 class ThroughAggregateSlaveActivity : public AggregateSlaveBase
 {
+    typedef AggregateSlaveBase PARENT;
+
     IHThorThroughAggregateArg *helper;
     RtlDynamicRowBuilder partResult;
     size32_t partResultSize;
@@ -233,8 +229,7 @@ class ThroughAggregateSlaveActivity : public AggregateSlaveBase
             OwnedConstThorRow ret = getResult(partrow.getClear());
             sendResult(ret, aggrowif->queryRowSerializer(), 0); // send to master
         }
-        AggregateSlaveBase::doStopInput();
-        dataLinkStop();
+        PARENT::doStopInput();
     }
     void readRest()
     {
@@ -264,13 +259,12 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        doStart();
+        PARENT::start();
         aggrowif.setown(createThorRowInterfaces(queryRowManager(), helper->queryAggregateRecordSize(),queryId(),queryCodeContext()));
         partResult.setAllocator(aggrowif->queryRowAllocator()).ensureRow();
         helper->clearAggregate(partResult);
-        dataLinkStart();
     }
-    virtual void stop()
+    virtual void stop() override
     {
         if (inputStopped) 
             return;
@@ -284,7 +278,7 @@ public:
         ActivityTimer t(totalCycles, timeActivities);
         if (inputStopped)
             return NULL;
-        OwnedConstThorRow row = input->ungroupedNextRow();
+        OwnedConstThorRow row = inputStream->ungroupedNextRow();
         if (!row)
             return NULL;
         process(row);
@@ -293,7 +287,7 @@ public:
     }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
-        inputs.item(0)->getMetaInfo(info);
+        queryInput(0)->getMetaInfo(info);
     }
 };
 

thorlcr/activities/aggregate/thgroupaggregateslave.cpp

@@ -17,45 +17,32 @@
 
 #include "thgroupaggregateslave.ipp"
 
-class GroupAggregateSlaveActivity : public CSlaveActivity, public CThorDataLink
+class GroupAggregateSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
 
-private:
     bool eof, ungroupedExistsAggregate;
     IHThorAggregateArg * helper;
-    IThorDataLink *input;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     GroupAggregateSlaveActivity(CGraphElementBase *_container) 
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     { 
-        input = NULL;
     }
 
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         helper = static_cast <IHThorAggregateArg *> (queryHelper());
     }
 
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         eof = false;
-        input=inputs.item(0);
-        startInput(input);
         ungroupedExistsAggregate = (container.getKind() == TAKexistsaggregate) && !input->isGrouped();
-        dataLinkStart();
-    }
-
-    void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
     }
-
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
@@ -63,7 +50,7 @@ public:
             return NULL;
         RtlDynamicRowBuilder out(queryRowAllocator());
         size32_t sz = helper->clearAggregate(out);
-        OwnedConstThorRow row = input->nextRow();
+        OwnedConstThorRow row = inputStream->nextRow();
         if (row)
         {
             sz = helper->processFirst(out, row);
@@ -72,7 +59,7 @@ public:
             {
                 while (!abortSoon)
                 {
-                    row.setown(input->nextRow());
+                    row.setown(inputStream->nextRow());
                     if (!row)
                         break;
                     sz = helper->processNext(out, row);
@@ -91,14 +78,12 @@ public:
         return out.finalizeRowClear(sz);
     }
 
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.canReduceNumRows = true;
         info.fastThrough = true;
     }
-
-    virtual bool isGrouped() { return false; }
 };
 
 

thorlcr/activities/apply/thapplyslave.cpp

@@ -21,14 +21,12 @@
 class CApplySlaveActivity : public ProcessSlaveActivity
 {
     IHThorApplyArg *helper;
-    IThorDataLink *input;
 
 public:
     CApplySlaveActivity(CGraphElementBase *container) 
         : ProcessSlaveActivity(container)
     { 
         helper = NULL;
-        input = NULL;
     }
 
 // IThorSlaveActivity overloaded methods
@@ -39,9 +37,8 @@ public:
 // IThorSlaveProcess overloaded methods
     virtual void process()
     {
+        start();
         processed = 0;
-        input = inputs.item(0);
-        startInput(input);
         processed = THORDATALINK_STARTED;
         try
         {
@@ -50,7 +47,7 @@ public:
             while(!abortSoon)
             {
                 ActivityTimer t(totalCycles, timeActivities);
-                OwnedConstThorRow r = input->ungroupedNextRow();
+                OwnedConstThorRow r = inputStream->ungroupedNextRow();
                 if (!r)
                     break;
                 helper->apply(r);
@@ -69,7 +66,7 @@ public:
     {
         if (processed & THORDATALINK_STARTED)
         {
-            stopInput(input);
+            stop();
             processed |= THORDATALINK_STOPPED;
         }
     }

thorlcr/activities/catch/thcatchslave.cpp

@@ -22,42 +22,35 @@
 #include "commonext.hpp"
 #include "slave.ipp"
 
-class CCatchSlaveActivityBase : public CSlaveActivity, public CThorDataLink
+class CCatchSlaveActivityBase : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
 protected:
-    Owned<IThorDataLink> input;
     IHThorCatchArg *helper;
     bool eos;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CCatchSlaveActivityBase(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CCatchSlaveActivityBase(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
     }
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         helper = static_cast <IHThorCatchArg *> (queryHelper());
         eos = false;
         appendOutputLinked(this);
     }
-    virtual void start()
+    virtual void start() override
     {
-        input.set(inputs.item(0));
-        startInput(input);
+        PARENT::start();
         eos = false;
-        dataLinkStart();
     }
-    virtual void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
-    }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
 };
 
 class CCatchSlaveActivity : public CCatchSlaveActivityBase, public CThorSteppable
 {
+    typedef CCatchSlaveActivityBase PARENT;
+
 public:
     CCatchSlaveActivity(CGraphElementBase *container) 
         : CCatchSlaveActivityBase(container), CThorSteppable(this)
@@ -75,7 +68,7 @@ public:
         {
             try
             {
-                OwnedConstThorRow row(input->nextRow());
+                OwnedConstThorRow row(inputStream->nextRow());
                 if (!row)
                     return NULL;
                 dataLinkIncrement();
@@ -109,7 +102,7 @@ public:
             try
             {
                 ActivityTimer t(totalCycles, timeActivities);
-                OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
+                OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
                 if (ret && wasCompleteMatch)
                     dataLinkIncrement();
                 return ret.getClear();
@@ -136,20 +129,20 @@ public:
     }
     virtual void resetEOF() 
     { 
-        input->resetEOF();
+        inputStream->resetEOF();
     }
     void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.fastThrough = true;
         info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 // steppable
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CCatchSlaveActivityBase::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        CCatchSlaveActivityBase::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta() { return CThorSteppable::inputStepping; }
 };
@@ -169,10 +162,10 @@ class CSkipCatchSlaveActivity : public CCatchSlaveActivityBase
             running = true;
             while (running)
             {
-                OwnedConstThorRow row = input->nextRow();
+                OwnedConstThorRow row = inputStream->nextRow();
                 if (!row)
                 {
-                    row.setown(input->nextRow());
+                    row.setown(inputStream->nextRow());
                     if (!row)
                         break;
                     else
@@ -204,7 +197,7 @@ public:
         global = !container->queryLocalOrGrouped();
     }
     virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
-    {       
+    {
         CCatchSlaveActivityBase::init(data, slaveData);
         if (global)
         {
@@ -283,7 +276,7 @@ public:
         initMetaInfo(info);
         info.fastThrough = false;
         info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 

thorlcr/activities/choosesets/thchoosesetsslave.cpp

@@ -19,19 +19,18 @@
 #include "thactivityutil.ipp"
 #include "thbufdef.hpp"
 
-class BaseChooseSetsActivity : public CSlaveActivity,  public CThorDataLink
+class BaseChooseSetsActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 protected:
     IHThorChooseSetsArg *helper;
     bool done;
     unsigned numSets;
     unsigned *tallies;
-    Owned<IThorDataLink> input;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    BaseChooseSetsActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    BaseChooseSetsActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = NULL;
         done = false;
@@ -42,14 +41,15 @@ public:
         if (tallies)
             delete [] tallies;
     }
-    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData) override
     {
         mpTag = container.queryJobChannel().deserializeMPTag(data);
         appendOutputLinked(this);
         helper = static_cast <IHThorChooseSetsArg *> (queryHelper());
     }
-    virtual void start()
+    virtual void start() override
     {
+        CSlaveActivity::start();
         numSets = helper->getNumSets();
         if (tallies)
             delete [] tallies;
@@ -58,37 +58,28 @@ public:
         memset(tallies, 0, sizeof(unsigned)*numSets);
         done = helper->setCounts(tallies);
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
 };
 
 
 class LocalChooseSetsActivity : public BaseChooseSetsActivity
 {
+    typedef BaseChooseSetsActivity PARENT;
+
 public:
     LocalChooseSetsActivity(CGraphElementBase *container) : BaseChooseSetsActivity(container) { }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        BaseChooseSetsActivity::start();
+        PARENT::start();
         ActPrintLog("CHOOSESETS: Is Local");
-        input.set(inputs.item(0));
-        startInput(input);
-        dataLinkStart();
-    }
-    virtual void stop()
-    {
-#if THOR_TRACE_LEVEL >= 5
-        ActPrintLog("CHOOSESETS: stop()");
-#endif
-        stopInput(input);
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
         loop
         {
-            OwnedConstThorRow row = input->ungroupedNextRow();
+            OwnedConstThorRow row = inputStream->ungroupedNextRow();
             if(!row || done || abortSoon)
                 break;
 
@@ -109,13 +100,15 @@ public:
         initMetaInfo(info);
         info.fastThrough = true;
         info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 
 
 class ChooseSetsActivity : public BaseChooseSetsActivity
 {
+    typedef BaseChooseSetsActivity PARENT;
+
     bool first;
     bool done;
 
@@ -150,34 +143,27 @@ class ChooseSetsActivity : public BaseChooseSetsActivity
 public:
     ChooseSetsActivity(CGraphElementBase *container) : BaseChooseSetsActivity(container)
     {
-    }   
-    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
+    }
+    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData) override
     {
-        BaseChooseSetsActivity::init(data, slaveData);
+        PARENT::init(data, slaveData);
         SocketEndpoint server;
         server.serialize(slaveData);
     }
-    virtual void start()
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
+    {
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), CHOOSESETS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), false, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
+    }
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         ActPrintLog("CHOOSESETS: Is Global");
-        BaseChooseSetsActivity::start();
+        PARENT::start();
         first = true;
         done = false;
-        input.setown(createDataLinkSmartBuffer(this, inputs.item(0),CHOOSESETS_SMART_BUFFER_SIZE,isSmartBufferSpillNeeded(this),false,RCUNBOUND,NULL,false,&container.queryJob().queryIDiskUsage())); // only allow spill if input can stall
-        startInput(input);
-        dataLinkStart();
     }
-    virtual void stop()
-    {
-#if THOR_TRACE_LEVEL >= 5
-        ActPrintLog("CHOOSESETS: stop()");
-#endif
-        stopInput(input);
-        input.clear();
-        dataLinkStop();
-    }
-    virtual void abort()
+    virtual void abort() override
     {
 #if THOR_TRACE_LEVEL >= 5
         ActPrintLog("CHOOSESETS: abort()");
@@ -202,7 +188,7 @@ public:
         {
             while (!abortSoon)
             {
-                OwnedConstThorRow row = input->ungroupedNextRow();
+                OwnedConstThorRow row = inputStream->ungroupedNextRow();
                 if (!row)
                     break;
                 switch (helper->getRecordAction(row))
@@ -226,7 +212,7 @@ public:
         info.isSequential = true;
         info.canReduceNumRows = true;
         info.canBufferInput = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 
@@ -234,33 +220,25 @@ public:
 //---------------------------------------------------------------------------
 
 class ChooseSetsPlusActivity;
-class InputCounter : public CSimpleInterface, implements IThorDataLink
+class CInputCounter : public CSimpleInterfaceOf<IEngineRowStream>
 {
+    IEngineRowStream *inputStream;
+    ChooseSetsPlusActivity &activity;
 public:
-    InputCounter(ChooseSetsPlusActivity & _activity) : activity(_activity) { }
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface)
-
-    virtual const void *nextRow();
-    virtual void stop();
-    virtual void start();
-    virtual bool isGrouped();
-
-// information routines 
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info);
-    virtual CActivityBase *queryFromActivity();
-    virtual void dataLinkSerialize(MemoryBuffer &mb);
-    unsigned __int64 queryTotalCycles() const;
-    unsigned __int64 queryEndCycles() const;
-    virtual void debugRequest(MemoryBuffer &msg);
-    ChooseSetsPlusActivity & activity;
-    IEngineRowAllocator *queryRowAllocator();
+    CInputCounter(ChooseSetsPlusActivity & _activity) : activity(_activity) { }
+    void setInputStream(IEngineRowStream *_inputStream) { inputStream = _inputStream; }
+
+    virtual const void *nextRow() override;
+    virtual void stop() override;
+    virtual void resetEOF() override { throwUnexpected(); }
 };
 
 
 // A hookling class that counts records as they are read by the smart buffering....
-class ChooseSetsPlusActivity : public CSlaveActivity,  public CThorDataLink, implements ISmartBufferNotify
+class ChooseSetsPlusActivity : public CSlaveActivity, implements ILookAheadStopNotify
 {
-    friend class InputCounter;
+    typedef CSlaveActivity PARENT;
+    friend class CInputCounter;
 protected:
     IHThorChooseSetsExArg * helper;
     bool done;
@@ -270,37 +248,41 @@ protected:
     rowcount_t * priorCounts;
     rowcount_t * totalCounts;
     __int64 * limits;
-    Owned<IThorDataLink> input;
-    InputCounter * inputCounter;
+    Owned<CInputCounter> inputCounter;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    ChooseSetsPlusActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    ChooseSetsPlusActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = NULL;
         counts = NULL;
         priorCounts = NULL;
         totalCounts = NULL;
         limits = NULL;
-        inputCounter = new InputCounter(*this);
+        inputCounter.setown(new CInputCounter(*this));
     }
     ~ChooseSetsPlusActivity()
     {
-        ::Release(inputCounter);
         free(counts);
         free(priorCounts);
         free(totalCounts);
         free(limits);
     }
-    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData) override
     {
         if (!container.queryLocalOrGrouped())
             mpTag = container.queryJobChannel().deserializeMPTag(data);
         appendOutputLinked(this);
         helper = static_cast <IHThorChooseSetsExArg *> (queryHelper());
     }
-    virtual void start()
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
+    {
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        inputCounter->setInputStream(inputStream);
+        setLookAhead(0, createRowStreamLookAhead(this, inputCounter.get(), queryRowInterfaces(input), CHOOSESETSPLUS_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage())); // read all input
+    }
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         ActPrintLog("CHOOSESETS: Is Global");
@@ -319,34 +301,23 @@ public:
         helper->getLimits(limits);
         done = false;
         first = true;
-        input.setown(createDataLinkSmartBuffer(this, inputCounter,CHOOSESETSPLUS_SMART_BUFFER_SIZE,true,false,RCUNBOUND,this,false,&container.queryJob().queryIDiskUsage())); // read all input
-        startInput(input);
-        dataLinkStart();
+        PARENT::start();
     }
-    virtual void stop()
-    {
-#if THOR_TRACE_LEVEL >= 5
-        ActPrintLog("CHOOSESETS: stop()");
-#endif
-        stopInput(input);
-        input.clear();
-        dataLinkStop();
-    }
-    virtual void abort()
+    virtual void abort() override
     {
         CSlaveActivity::abort();
         if (!container.queryLocalOrGrouped())
             cancelReceiveMsg(RANK_ALL, mpTag);
     }
-    virtual bool isGrouped() { return false; }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.buffersInput = true;
         info.isSequential = true;
         info.canReduceNumRows = true;
         info.canBufferInput = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
     void getGlobalCounts()
     {
@@ -358,9 +329,7 @@ public:
         memcpy(totalCounts, msg.readDirect(numSets*sizeof(rowcount_t)), numSets*sizeof(rowcount_t));
         memcpy(priorCounts, msg.readDirect(numSets*sizeof(rowcount_t)), numSets*sizeof(rowcount_t));
     }
-    virtual bool startAsync() { return false; }
-    virtual void onInputStarted(IException *e) { }
-    virtual void onInputFinished(rowcount_t count)
+    virtual void onInputFinished(rowcount_t count) override
     {
         if (container.queryLocalOrGrouped())
             return;
@@ -370,6 +339,27 @@ public:
     }
 };
 
+//////////
+
+const void *CInputCounter::nextRow()
+{
+    OwnedConstThorRow row = inputStream->nextRow();
+    if (row)
+    {
+        unsigned category = activity.helper->getCategory(row);
+        if (category)
+            activity.counts[category-1]++;
+        return row.getClear();
+    }
+    return NULL;
+}
+
+void CInputCounter::stop()
+{
+    inputStream->stop();
+}
+
+//////////
 
 class ChooseSetsLastActivity : public ChooseSetsPlusActivity
 {
@@ -411,7 +401,7 @@ public:
         free(numToSkip);
         free(numToReturn);
     }
-    virtual void start()
+    virtual void start() override
     {
         ChooseSetsPlusActivity::start();
         if (numToSkip)
@@ -436,7 +426,7 @@ public:
         {
             while (!abortSoon)
             {
-                OwnedConstThorRow row = input->ungroupedNextRow();
+                OwnedConstThorRow row = inputStream->ungroupedNextRow();
                 if (!row)
                     break;
                 unsigned category = helper->getCategory(row);
@@ -494,7 +484,7 @@ public:
     {
         free(counter);
     }
-    virtual void start()
+    virtual void start() override
     {
         ChooseSetsPlusActivity::start();
         if (counter)
@@ -515,7 +505,7 @@ public:
         {
             while (!abortSoon)
             {
-                OwnedConstThorRow row = input->ungroupedNextRow();
+                OwnedConstThorRow row = inputStream->ungroupedNextRow();
                 if (!row)
                     break;
                 unsigned category = helper->getCategory(row);
@@ -536,32 +526,8 @@ public:
 };
 
 
-//-----------------------------------------------------------------------------------------------
-
-
-const void *InputCounter::nextRow()
-{
-    OwnedConstThorRow row = activity.inputs.item(0)->nextRow();
-    if (row) {
-        unsigned category = activity.helper->getCategory(row);
-        if (category)
-            activity.counts[category-1]++;
-        return row.getClear();
-    }
-    return NULL;
-}
 
-void InputCounter::stop()                                   { activity.inputs.item(0)->stop(); }
-void InputCounter::start()                                  { activity.inputs.item(0)->start(); }
-bool InputCounter::isGrouped()                              { return activity.inputs.item(0)->isGrouped(); }
-
-// information routines 
-void InputCounter::getMetaInfo(ThorDataLinkMetaInfo &info)  { activity.inputs.item(0)->getMetaInfo(info); }
-CActivityBase *InputCounter::queryFromActivity()            { return activity.inputs.item(0)->queryFromActivity(); }
-void InputCounter::dataLinkSerialize(MemoryBuffer &mb)      { activity.inputs.item(0)->dataLinkSerialize(mb); }
-unsigned __int64 InputCounter::queryTotalCycles() const     { return activity.inputs.item(0)->queryTotalCycles(); }
-unsigned __int64 InputCounter::queryEndCycles() const       { return activity.inputs.item(0)->queryEndCycles(); }
-void InputCounter::debugRequest(MemoryBuffer &msg)          { return activity.inputs.item(0)->debugRequest(msg); }
+//-----------------------------------------------------------------------------------------------
 
 
 //---------------------------------------------------------------------------

thorlcr/activities/countproject/thcountprojectslave.cpp

@@ -19,45 +19,34 @@
 #include "thactivityutil.ipp"
 #include "thbufdef.hpp"
 
-class BaseCountProjectActivity : public CSlaveActivity,  public CThorDataLink, implements ISmartBufferNotify
+class BaseCountProjectActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 protected:
-    IHThorCountProjectArg *helper;
-    rowcount_t count;
-    Owned<IThorDataLink> input;
+    IHThorCountProjectArg *helper = nullptr;
+    rowcount_t count = 0;
 
-    void start()
+    virtual void start() override
     {
+        PARENT::start();
         count = 0;
-        dataLinkStart();
     }
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    BaseCountProjectActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    BaseCountProjectActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
-        helper = NULL;
     }
-    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         helper = static_cast <IHThorCountProjectArg *> (queryHelper());
     }
-    virtual void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
-    }
-    virtual void onInputStarted(IException *)
-    {
-        // not needed
-    }
-    virtual bool startAsync() { return false; }
 };
 
 
 class LocalCountProjectActivity : public BaseCountProjectActivity
 {
+    typedef BaseCountProjectActivity PARENT;
     bool anyThisGroup;
 
 public:
@@ -68,22 +57,20 @@ public:
     {
         ActivityTimer s(totalCycles, timeActivities);
         ActPrintLog("COUNTPROJECT: Is Local");
-        input.set(inputs.item(0));
         anyThisGroup = false;
-        startInput(input);
-        BaseCountProjectActivity::start();
+        PARENT::start();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
         while (!abortSoon)
         {
-            OwnedConstThorRow row(input->nextRow());
+            OwnedConstThorRow row(inputStream->nextRow());
             if (!row)
             {
                 if (anyThisGroup) 
                     break;
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if (!row)
                     break;
                 count = 0;
@@ -101,12 +88,12 @@ public:
         anyThisGroup = false;
         return NULL;        
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
     void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.fastThrough = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
     virtual void onInputFinished(rowcount_t finalCount)
     {
@@ -115,9 +102,9 @@ public:
 };
 
 
-class CountProjectActivity : public BaseCountProjectActivity
+class CountProjectActivity : public BaseCountProjectActivity, implements ILookAheadStopNotify
 {
-private:
+    typedef BaseCountProjectActivity PARENT;
     bool first;
     Semaphore prevRecCountSem;
     rowcount_t prevRecCount, localRecCount;
@@ -162,26 +149,28 @@ public:
     }   
     virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
     {
-        BaseCountProjectActivity::init(data, slaveData);
+        PARENT::init(data, slaveData);
         mpTag = container.queryJobChannel().deserializeMPTag(data);
     }
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
+    {
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), COUNTPROJECT_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage())); // could spot disk write output here?
+    }
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
         ActPrintLog( "COUNTPROJECT: Is Global");
         first = true;
         prevRecCount = 0;
-        startInput(inputs.item(0));
         ThorDataLinkMetaInfo info;
-        inputs.item(0)->getMetaInfo(info);
+        input->getMetaInfo(info);
         localRecCount = (info.totalRowsMin == info.totalRowsMax) ? (rowcount_t)info.totalRowsMax : RCUNSET;
-        input.setown(createDataLinkSmartBuffer(this, inputs.item(0), COUNTPROJECT_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, true, &container.queryJob().queryIDiskUsage())); // could spot disk write output here?
-        input->start();
-        BaseCountProjectActivity::start();
+        PARENT::start();
     }
     virtual void stop()
     {
-        BaseCountProjectActivity::stop();
+        PARENT::stop();
         if (first) // nextRow, therefore getPrevCount()/sendCount() never called
         {
             prevRecCount = count = getPrevCount();
@@ -206,7 +195,7 @@ public:
         }
         while (!abortSoon)
         {
-            OwnedConstThorRow row(input->nextRow()); // NB: lookahead ensures ungrouped
+            OwnedConstThorRow row(inputStream->nextRow()); // NB: lookahead ensures ungrouped
             if (!row) 
                 break;
             RtlDynamicRowBuilder ret(queryRowAllocator());
@@ -219,7 +208,7 @@ public:
         }
         return NULL;
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void onInputFinished(rowcount_t localRecCount)
     {
         if (!haveLocalCount())
@@ -237,7 +226,7 @@ public:
         initMetaInfo(info);
         info.buffersInput = true;
         info.isSequential = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 

thorlcr/activities/csvread/thcsvrslave.cpp

@@ -33,8 +33,10 @@
 #include "csvsplitter.hpp"
 #include "thdiskbaseslave.ipp"
 
-class CCsvReadSlaveActivity : public CDiskReadSlaveActivityBase, public CThorDataLink
+class CCsvReadSlaveActivity : public CDiskReadSlaveActivityBase
 {
+    typedef CDiskReadSlaveActivityBase PARENT;
+
     IHThorCsvReadArg *helper;
     StringAttr csvQuote, csvSeparate, csvTerminate, csvEscape;
     Owned<IRowStream> out;
@@ -303,9 +305,7 @@ class CCsvReadSlaveActivity : public CDiskReadSlaveActivityBase, public CThorDat
         }
     }
 public:
-    IMPLEMENT_IINTERFACE_USING(CDiskReadSlaveActivityBase);
-
-    CCsvReadSlaveActivity(CGraphElementBase *_container) : CDiskReadSlaveActivityBase(_container), CThorDataLink(this)
+    CCsvReadSlaveActivity(CGraphElementBase *_container) : CDiskReadSlaveActivityBase(_container)
     {
         helper = static_cast <IHThorCsvReadArg *> (queryHelper());
         stopAfter = (rowcount_t)helper->getChooseNLimit();
@@ -376,7 +376,7 @@ public:
             gotMeta = true;
             initMetaInfo(cachedMetaInfo);
             cachedMetaInfo.isSource = true;
-            getPartsMetaInfo(cachedMetaInfo, *this, partDescs.ordinality(), partDescs.getArray(), partHandler);
+            getPartsMetaInfo(cachedMetaInfo, partDescs.ordinality(), partDescs.getArray(), partHandler);
             cachedMetaInfo.unknownRowsOutput = true; // at least I don't think we know
         }
         info = cachedMetaInfo;
@@ -422,20 +422,19 @@ public:
             }
         }
         out.setown(createSequentialPartHandler(partHandler, partDescs, false));
-        dataLinkStart();
     }
     virtual void stop()
     {
         sendRemainingHeaderLines();
         out.clear();
-        dataLinkStop();
+        PARENT::stop();
     }
     void abort()
     {
         CDiskReadSlaveActivityBase::abort();
         cancelReceiveMsg(queryJobChannel().queryMyRank()-1, mpTag);
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
 
 friend class CCsvPartHandler;
 };

thorlcr/activities/degroup/thdegroupslave.cpp

@@ -17,46 +17,33 @@
 
 #include "thdegroupslave.ipp"
 
-class CDegroupSlaveActivity : public CSlaveActivity, public CThorDataLink, public CThorSteppable
+class CDegroupSlaveActivity : public CSlaveActivity, public CThorSteppable
 {
-    IThorDataLink *input;
+    typedef CSlaveActivity PARENT;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
     CDegroupSlaveActivity(CGraphElementBase *_container) 
-        : CSlaveActivity(_container) , CThorDataLink(this), CThorSteppable(this)
+        : CSlaveActivity(_container), CThorSteppable(this)
     { 
-        input = NULL; 
     }
-    bool isGrouped() 
-    { 
-        return false; 
-    }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-
-        startInput(input);
+        PARENT::start();
         if(!input->isGrouped()) ActPrintLog("DEGROUP: Degrouping non-grouped input!");
-        dataLinkStart();
-    }
-    void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
     }
     CATCH_NEXTROW()
-    {   
+    {
         ActivityTimer t(totalCycles, timeActivities);
         if (!abortSoon)
         {
-            OwnedConstThorRow row = input->ungroupedNextRow();
+            OwnedConstThorRow row = inputStream->ungroupedNextRow();
             if (row)
             {
                 dataLinkIncrement();
@@ -66,17 +53,17 @@ public:
         }
         return NULL;
     }
-    const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra) override
     {
         try { return nextRowGENoCatch(seek, numFields, wasCompleteMatch, stepExtra); }
         CATCH_NEXTROWX_CATCH;
     }
-    const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         ActivityTimer t(totalCycles, timeActivities);
         if (!abortSoon)
         {
-            OwnedConstThorRow row = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
+            OwnedConstThorRow row = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
             if (row)
             {
                 dataLinkIncrement();
@@ -86,26 +73,30 @@ public:
         }
         return NULL;
     }
-    bool gatherConjunctions(ISteppedConjunctionCollector &collector)
+    virtual bool gatherConjunctions(ISteppedConjunctionCollector &collector) override
     { 
         return input->gatherConjunctions(collector);
     }
-    void resetEOF() 
+    virtual void resetEOF() override
     { 
         abortSoon = false;
-        input->resetEOF(); 
+        inputStream->resetEOF();
+    }
+    virtual bool isGrouped() const override
+    { 
+        return false; 
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.fastThrough = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 // steppable
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CSlaveActivity::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        CSlaveActivity::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta() { return CThorSteppable::inputStepping; }
 };

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -272,8 +272,10 @@ void CDiskRecordPartHandler::close(CRC32 &fileCRC)
 
 /////////////////////////////////////////////////
 
-class CDiskReadSlaveActivity : public CDiskReadSlaveActivityRecord, public CThorDataLink
+class CDiskReadSlaveActivity : public CDiskReadSlaveActivityRecord
 {
+    typedef CDiskReadSlaveActivityRecord PARENT;
+
     class CDiskPartHandler : public CDiskRecordPartHandler
     {
         CDiskReadSlaveActivity &activity;
@@ -397,9 +399,7 @@ public:
 
     IHThorDiskReadArg *helper;
 
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CDiskReadSlaveActivity(CGraphElementBase *_container, IHThorArg *_helper) : CDiskReadSlaveActivityRecord(_container, _helper), CThorDataLink(this)
+    CDiskReadSlaveActivity(CGraphElementBase *_container, IHThorArg *_helper) : CDiskReadSlaveActivityRecord(_container, _helper)
     {
         helper = (IHThorDiskReadArg *)queryHelper();
         unsorted = 0 != (TDRunsorted & helper->getFlags());
@@ -465,7 +465,7 @@ public:
             gotMeta = true;
             initMetaInfo(cachedMetaInfo);
             cachedMetaInfo.isSource = true;
-            getPartsMetaInfo(cachedMetaInfo, *this, partDescs.ordinality(), partDescs.getArray(), partHandler);
+            getPartsMetaInfo(cachedMetaInfo, partDescs.ordinality(), partDescs.getArray(), partHandler);
         }
         info = cachedMetaInfo;
         if (info.totalRowsMin==info.totalRowsMax)
@@ -478,9 +478,8 @@ public:
         ActivityTimer s(totalCycles, timeActivities);
         CDiskReadSlaveActivityRecord::start();
         out = createSequentialPartHandler(partHandler, partDescs, grouped); // **
-        dataLinkStart();
     }
-    virtual bool isGrouped() { return grouped; }
+    virtual bool isGrouped() const override { return grouped; }
 
 // IRowStream
     virtual void stop()
@@ -491,7 +490,7 @@ public:
             out->Release();
             out = NULL;
         }
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -526,8 +525,10 @@ CActivityBase *createDiskReadSlave(CGraphElementBase *container, IHThorArg *help
 // CDiskNormalizeSlave
 //
 
-class CDiskNormalizeSlave : public CDiskReadSlaveActivityRecord, public CThorDataLink
+class CDiskNormalizeSlave : public CDiskReadSlaveActivityRecord
 {
+    typedef CDiskReadSlaveActivityRecord PARENT;
+
     class CNormalizePartHandler : public CDiskRecordPartHandler
     {
         RtlDynamicRowBuilder outBuilder;
@@ -590,10 +591,8 @@ class CDiskNormalizeSlave : public CDiskReadSlaveActivityRecord, public CThorDat
     IRowStream *out;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CDiskNormalizeSlave(CGraphElementBase *_container) 
-        : CDiskReadSlaveActivityRecord(_container), CThorDataLink(this)
+        : CDiskReadSlaveActivityRecord(_container)
     {
         helper = (IHThorDiskNormalizeArg *)queryHelper();
         if (helper->getFlags() & TDRlimitskips)
@@ -624,7 +623,7 @@ public:
             gotMeta = true;
             initMetaInfo(cachedMetaInfo);
             cachedMetaInfo.isSource = true;
-            getPartsMetaInfo(cachedMetaInfo, *this, partDescs.ordinality(), partDescs.getArray(), partHandler);
+            getPartsMetaInfo(cachedMetaInfo, partDescs.ordinality(), partDescs.getArray(), partHandler);
             cachedMetaInfo.unknownRowsOutput = true; // JCSMORE
         }
         info = cachedMetaInfo;
@@ -634,9 +633,8 @@ public:
         ActivityTimer s(totalCycles, timeActivities);
         CDiskReadSlaveActivityRecord::start();
         out = createSequentialPartHandler(partHandler, partDescs, false);
-        dataLinkStart();
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
 
 // IRowStream
     virtual void stop()
@@ -647,7 +645,7 @@ public:
             out->Release();
             out = NULL;
         }
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -713,18 +711,18 @@ public:
     }
 };
 
-class CDiskAggregateSlave : public CDiskReadSlaveActivityRecord, public CThorDataLink
+class CDiskAggregateSlave : public CDiskReadSlaveActivityRecord
 {
+    typedef CDiskReadSlaveActivityRecord PARENT;
+
     IHThorDiskAggregateArg *helper;
     Owned<IEngineRowAllocator> allocator;
     bool eoi, hadElement;
     CPartialResultAggregator aggregator;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CDiskAggregateSlave(CGraphElementBase *_container) 
-        : CDiskReadSlaveActivityRecord(_container), aggregator(*this), CThorDataLink(this)
+        : CDiskReadSlaveActivityRecord(_container), aggregator(*this)
     {
         helper = (IHThorDiskAggregateArg *)queryHelper();
         eoi = false;
@@ -756,13 +754,12 @@ public:
         info.totalRowsMax = 1;
         // MORE TBD
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
         CDiskReadSlaveActivityRecord::start();
         eoi = hadElement = false;
-        dataLinkStart();
     }
 
 // IRowStream
@@ -770,7 +767,7 @@ public:
     {
         if (partHandler)
             partHandler->stop();
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -832,16 +829,16 @@ CActivityBase *createDiskAggregateSlave(CGraphElementBase *container)
 }
 
 
-class CDiskCountSlave : public CDiskReadSlaveActivityRecord, public CThorDataLink
+class CDiskCountSlave : public CDiskReadSlaveActivityRecord
 {
+    typedef CDiskReadSlaveActivityRecord PARENT;
+
     IHThorDiskCountArg *helper;
     rowcount_t stopAfter, preknownTotalCount;
     bool eoi, totalCountKnown;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CDiskCountSlave(CGraphElementBase *_container) : CDiskReadSlaveActivityRecord(_container), CThorDataLink(this)
+    CDiskCountSlave(CGraphElementBase *_container) : CDiskReadSlaveActivityRecord(_container)
     {
         helper = (IHThorDiskCountArg *)queryHelper();
         totalCountKnown = eoi = false;
@@ -876,7 +873,7 @@ public:
         info.isSource = true;
         // MORE TBD
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
@@ -887,7 +884,6 @@ public:
             totalCountKnown = true;
             preknownTotalCount = 0;
         }
-        dataLinkStart();
     }
 
 // IRowStream
@@ -895,7 +891,7 @@ public:
     {
         if (partHandler)
             partHandler->stop();
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -954,8 +950,10 @@ CActivityBase *createDiskCountSlave(CGraphElementBase *container)
 }
 
 class CDiskGroupAggregateSlave 
-  : public CDiskReadSlaveActivityRecord, public CThorDataLink, implements IHThorGroupAggregateCallback
+  : public CDiskReadSlaveActivityRecord, implements IHThorGroupAggregateCallback
 {
+    typedef CDiskReadSlaveActivityRecord PARENT;
+
     IHThorDiskGroupAggregateArg *helper;
     bool gathered, eoi;
     Owned<RowAggregator> localAggTable;
@@ -967,7 +965,7 @@ public:
     IMPLEMENT_IINTERFACE_USING(CDiskReadSlaveActivityRecord);
 
     CDiskGroupAggregateSlave(CGraphElementBase *_container) 
-        : CDiskReadSlaveActivityRecord(_container), CThorDataLink(this)
+        : CDiskReadSlaveActivityRecord(_container)
     {
         helper = (IHThorDiskGroupAggregateArg *)queryHelper();
         merging = false;
@@ -1001,7 +999,6 @@ public:
         gathered = eoi = false;
         localAggTable.setown(new RowAggregator(*helper, *helper));
         localAggTable->start(queryRowAllocator());
-        dataLinkStart();
     }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
@@ -1009,13 +1006,13 @@ public:
         info.isSource = true;
         // MORE TBD
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
 // IRowStream
     virtual void stop()
     {
         if (partHandler)
             partHandler->stop();
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {

thorlcr/activities/diskwrite/thdwslave.cpp

@@ -39,13 +39,13 @@ protected:
 
         while(!abortSoon)
         {       
-            OwnedConstThorRow r = input->nextRow();
+            OwnedConstThorRow r = inputStream->nextRow();
             if (!r.get()) {
                 if (grouped) {
                     if ((processed & THORDATALINK_COUNT_MASK)!=0)
                         out->putRow(NULL);
                 }
-                r.setown(input->nextRow());
+                r.setown(inputStream->nextRow());
                 if (!r.get())
                     break;
             }
@@ -97,7 +97,7 @@ protected:
         }
         while(!abortSoon)
         {
-            OwnedConstThorRow r(input->ungroupedNextRow());
+            OwnedConstThorRow r(inputStream->ungroupedNextRow());
             if (!r) 
                 break;
 

thorlcr/activities/distribution/thdistributionslave.cpp

@@ -23,12 +23,10 @@ class CDistributionSlaveActivity : public ProcessSlaveActivity
     IHThorDistributionArg * helper;
     MemoryAttr ma;
     IDistributionTable * * aggy;                // should this be row?
-    IThorDataLink *input;
 
 public:
     CDistributionSlaveActivity(CGraphElementBase *container) : ProcessSlaveActivity(container)
     {
-        input = NULL;
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
@@ -45,16 +43,15 @@ public:
     }
     void process()
     {
+        start();
         helper->clearAggregate(aggy);
-        input = inputs.item(0);
-        startInput(input);
         processed = THORDATALINK_STARTED;
 
         try
         {
             while (!abortSoon)
             {
-                OwnedConstThorRow row(input->ungroupedNextRow());
+                OwnedConstThorRow row(inputStream->ungroupedNextRow());
                 if (!row)
                     break;
                 helper->process(aggy, row);     
@@ -75,7 +72,7 @@ public:
     {
         if (processed & THORDATALINK_STARTED)
         {
-            stopInput(input);
+            stop();
             processed |= THORDATALINK_STOPPED;
         }
     }   

thorlcr/activities/enth/thenthslave.cpp

@@ -19,14 +19,16 @@
 #include "thactivityutil.ipp"
 #include "thbufdef.hpp"
 
-class BaseEnthActivity : public CSlaveActivity, public CThorDataLink, implements ISmartBufferNotify
+class BaseEnthActivity : public CSlaveActivity, implements ILookAheadStopNotify
 {
+    typedef CSlaveActivity PARENT;
+
+    ThorDataLinkMetaInfo intoMetaInfo;
 protected:
     StringBuffer actStr;
     Semaphore finishedSem;
     rowcount_t counter, localRecCount;
     rowcount_t denominator, numerator;
-    Owned<IThorDataLink> input;
 
     bool haveLocalCount() { return RCUNBOUND != localRecCount; }
     inline bool wanted()
@@ -67,69 +69,58 @@ protected:
     }
     void setLocalCountReq()
     {
-        ThorDataLinkMetaInfo info;
-        input->getMetaInfo(info);
         // Need lookahead _unless_ row count pre-known.
         if (0 == numerator)
             localRecCount = 0;
-        else if (info.totalRowsMin == info.totalRowsMax)
+        else if (intoMetaInfo.totalRowsMin == intoMetaInfo.totalRowsMax)
         {
-            localRecCount = (rowcount_t)info.totalRowsMax;
+            localRecCount = (rowcount_t)intoMetaInfo.totalRowsMax;
             ActPrintLog("%s: row count pre-known to be %" RCPF "d", actStr.str(), localRecCount);
         }
         else
-        {
             localRecCount = RCUNBOUND;
-            input.setown(createDataLinkSmartBuffer(this, input,ENTH_SMART_BUFFER_SIZE,true,false,RCUNBOUND,this,true,&container.queryJob().queryIDiskUsage()));
-            StringBuffer tmpStr(actStr);
-            startInput(input);
-        }
     }
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    BaseEnthActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
-    {
-    }
-    ~BaseEnthActivity()
+    BaseEnthActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
     }
     virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
     {
         appendOutputLinked(this);
     }
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
+    {
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        input->getMetaInfo(intoMetaInfo);
+        // Need lookahead _unless_ row count pre-known.
+        if (numerator && (intoMetaInfo.totalRowsMin != intoMetaInfo.totalRowsMax))
+            setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), ENTH_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage()));
+    }
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         IHThorEnthArg *helper = static_cast <IHThorEnthArg *> (queryHelper());
         counter = 0;
         denominator = validRC(helper->getProportionDenominator());
         numerator = validRC(helper->getProportionNumerator());
-        input.set(inputs.item(0));
-        startInput(input);
-        dataLinkStart();
-    }
-    virtual void stop()
-    {
-        stopInput(input);
-        input.clear();
-        dataLinkStop();
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.buffersInput = true;
         info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, input);
     }
-// ISmartBufferNotify impl.
-    virtual void onInputStarted(IException *) { }
-    virtual bool startAsync() { return false; }
 };
 
 class CLocalEnthSlaveActivity : public BaseEnthActivity
 {
+    typedef BaseEnthActivity PARENT;
+
     bool localCountReq;
 public:
     CLocalEnthSlaveActivity(CGraphElementBase *container) : BaseEnthActivity(container)
@@ -139,7 +130,7 @@ public:
     }
     virtual void start()
     {
-        BaseEnthActivity::start();
+        PARENT::start();
         if (RCUNBOUND == denominator)
         {
             localCountReq = true;
@@ -161,7 +152,7 @@ public:
         }
         while (!abortSoon)
         {
-            OwnedConstThorRow row(input->ungroupedNextRow());
+            OwnedConstThorRow row(inputStream->ungroupedNextRow());
             if (!row)
                 break;
             if (wanted())
@@ -174,7 +165,7 @@ public:
     }
     virtual void abort()
     {
-        BaseEnthActivity::abort();
+        PARENT::abort();
         localRecCount = 0;
         finishedSem.signal();
     }
@@ -188,6 +179,8 @@ public:
 
 class CEnthSlaveActivity : public BaseEnthActivity
 {
+    typedef BaseEnthActivity PARENT;
+
     Semaphore prevRecCountSem;
     rowcount_t prevRecCount;
     bool first;
@@ -224,19 +217,19 @@ public:
     }
     virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
     {
-        BaseEnthActivity::init(data, slaveData);
+        PARENT::init(data, slaveData);
         mpTag = container.queryJobChannel().deserializeMPTag(data);
     }
     virtual void start()
     {
-        BaseEnthActivity::start();
+        PARENT::start();
         prevRecCount = 0;
         first = true;
         setLocalCountReq();
     }
     virtual void abort()
     {
-        BaseEnthActivity::abort();
+        PARENT::abort();
         if (!firstNode())
             cancelReceiveMsg(RANK_ALL, mpTag);
     }
@@ -251,7 +244,7 @@ public:
         }
         while (!abortSoon)
         {
-            OwnedConstThorRow row(input->ungroupedNextRow());
+            OwnedConstThorRow row(inputStream->ungroupedNextRow());
             if (!row)
                 break;
             if (wanted())
@@ -270,7 +263,7 @@ public:
             first = false;
             getPrev();
         }
-        BaseEnthActivity::stop();
+        PARENT::stop();
     }
     virtual void onInputFinished(rowcount_t localRecCount)
     {

thorlcr/activities/fetch/thfetchslave.cpp

@@ -264,8 +264,10 @@ IFetchStream *createFetchStream(CSlaveActivity &owner, IThorRowInterfaces *keyRo
     return new CFetchStream(owner, keyRowIf, fetchRowIf, abortSoon, parts, offsetCount, offsetMapSz, offsetMap, iFetchHandler, tag, eexp);
 }
 
-class CFetchSlaveBase : public CSlaveActivity, public CThorDataLink, implements IFetchHandler
+class CFetchSlaveBase : public CSlaveActivity, implements IFetchHandler
 {
+    typedef CSlaveActivity PARENT;
+
     IRowStream *fetchStreamOut;
     unsigned maxKeyRecSize;
     rowcount_t limit;
@@ -289,7 +291,7 @@ protected:
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CFetchSlaveBase(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CFetchSlaveBase(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         fetchStream = NULL;
         keyIn = NULL;
@@ -304,7 +306,7 @@ public:
         ::Release(fetchStream);
     }
 
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         unsigned numParts;
         data.read(numParts);
@@ -356,9 +358,11 @@ public:
     }
 
 // IThorDataLink impl.
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
+
         class CKeyFieldExtractBase : public CSimpleInterface, implements IRowStream
         {
         protected:
@@ -380,10 +384,6 @@ public:
             virtual void stop() { in.stop(); }
         };
 
-        startInput(inputs.item(0));
-        dataLinkStart();
-
-        IThorDataLink *in = inputs.item(0);
         Owned<IThorRowInterfaces> keyInIf;
         if (indexRowExtractNeeded)
         {
@@ -412,12 +412,12 @@ public:
 
             if (fetchBaseHelper->extractAllJoinFields())
             {
-                keyIn = LINK(in);
-                keyInMeta.set(in->queryFromActivity()->queryRowMetaData());
+                keyIn = LINK(inputStream);
+                keyInMeta.set(input->queryFromActivity()->queryRowMetaData());
             }
             else
             {
-                keyIn = new CKeyFieldExtract(this, *in, *fetchBaseHelper, *fetchContext);
+                keyIn = new CKeyFieldExtract(this, *inputStream, *fetchBaseHelper, *fetchContext);
                 keyInMeta.set(QUERYINTERFACE(fetchBaseHelper->queryExtractedSize(), IOutputMetaData));
             }
             keyInIf.setown(createThorRowInterfaces(queryRowManager(), keyInMeta,queryId(),queryCodeContext()));
@@ -450,7 +450,7 @@ public:
             };
             Owned<IOutputMetaData> fmeta = createFixedSizeMetaData(sizeof(offset_t)); // should be provided by Gavin?
             keyInIf.setown(createThorRowInterfaces(queryRowManager(), fmeta,queryId(),queryCodeContext()));
-            keyIn = new CKeyFPosExtract(keyInIf, this, *in, *fetchBaseHelper, *fetchContext);
+            keyIn = new CKeyFPosExtract(keyInIf, this, *inputStream, *fetchBaseHelper, *fetchContext);
         }
 
         Owned<IThorRowInterfaces> rowIf = createThorRowInterfaces(queryRowManager(), queryRowMetaData(), queryId(), queryCodeContext());
@@ -459,7 +459,7 @@ public:
         fetchStream->start(keyIn);
         initializeFileParts();
     }
-    virtual void stop()
+    virtual void stop() override
     {
         fetchStreamOut->stop();
         dataLinkStop();
@@ -486,8 +486,8 @@ public:
         }
         return NULL;
     }
-    virtual bool isGrouped() { return false; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.canStall = true;

thorlcr/activities/filter/thfilterslave.cpp

@@ -17,50 +17,45 @@
 
 #include "thfilterslave.ipp"
 
-class CFilterSlaveActivityBase : public CSlaveActivity, public CThorDataLink
+class CFilterSlaveActivityBase : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 protected:
-    bool anyThisGroup;
-    IThorDataLink * input;
+    bool anyThisGroup = false;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CFilterSlaveActivityBase(CGraphElementBase *_container) 
-        : CSlaveActivity(_container), CThorDataLink(this)
+    explicit CFilterSlaveActivityBase(CGraphElementBase *_container)
+        : CSlaveActivity(_container)
     {
-        input = NULL;
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
     }
-    void start()
-    {   
-        input = inputs.item(0);
-        anyThisGroup = false;
-        startInput(input);
-        dataLinkStart();
-    }
-    void stop()
+    virtual void start() override
     {
-        stopInput(input);
-        dataLinkStop();
+        PARENT::start();
+        anyThisGroup = false;
     }
-    virtual const void *nextRow()=0;
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual const void *nextRow() override =0;
+
+// IThorDataLink
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.fastThrough = true;
         info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
 };
 
 
 class CFilterSlaveActivity : public CFilterSlaveActivityBase, public CThorSteppable
 {
+    typedef CFilterSlaveActivityBase PARENT;
+
     IHThorFilterArg *helper;
     unsigned matched;
 public:
@@ -70,7 +65,7 @@ public:
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
-        CFilterSlaveActivityBase::init(data,slaveData);
+        PARENT::init(data,slaveData);
         helper = static_cast <IHThorFilterArg *> (queryHelper());
     }
     void start()
@@ -78,14 +73,14 @@ public:
         ActivityTimer s(totalCycles, timeActivities);
         matched = 0;
         abortSoon = !helper->canMatchAny();
-        CFilterSlaveActivityBase::start();
+        PARENT::start();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
         while(!abortSoon)
         {
-            OwnedConstThorRow row = input->nextRow();
+            OwnedConstThorRow row = inputStream->nextRow();
             if (!row)
             {
                 if(anyThisGroup)
@@ -93,7 +88,7 @@ public:
                     anyThisGroup = false;
                     break;
                 }
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if (!row)
                     break;
             }
@@ -117,7 +112,7 @@ public:
         ActivityTimer t(totalCycles, timeActivities);
         while (!abortSoon)
         {
-            OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
+            OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
             if (!ret)
             {
                 abortSoon = true;
@@ -153,19 +148,21 @@ public:
     { 
         abortSoon = !helper->canMatchAny();
         anyThisGroup = false;
-        input->resetEOF(); 
+        inputStream->resetEOF();
     }
 // steppable
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CFilterSlaveActivityBase::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        PARENT::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta() { return CThorSteppable::inputStepping; }
 };
 
 class CFilterProjectSlaveActivity : public CFilterSlaveActivityBase
 {
+    typedef CFilterSlaveActivityBase PARENT;
+
     IHThorFilterProjectArg *helper;
     rowcount_t recordCount;  // NB local (not really used for global)
     Owned<IEngineRowAllocator> allocator;
@@ -176,7 +173,7 @@ public:
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
-        CFilterSlaveActivityBase::init(data,slaveData);
+        PARENT::init(data,slaveData);
         helper = static_cast <IHThorFilterProjectArg *> (queryHelper());
         allocator.set(queryRowAllocator());
     }
@@ -185,19 +182,19 @@ public:
         ActivityTimer s(totalCycles, timeActivities);
         abortSoon = !helper->canMatchAny();
         recordCount = 0;
-        CFilterSlaveActivityBase::start();
+        PARENT::start();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
         while (!abortSoon)
         {
-            OwnedConstThorRow row = input->nextRow();
+            OwnedConstThorRow row = inputStream->nextRow();
             if (!row) {
                 recordCount = 0;
                 if (!anyThisGroup)
                 {
-                    row.setown(input->nextRow());
+                    row.setown(inputStream->nextRow());
                     if (!row)
                     {
                         abortSoon = true;
@@ -235,6 +232,8 @@ public:
 
 class CFilterGroupSlaveActivity : public CFilterSlaveActivityBase, public CThorSteppable
 {
+    typedef CFilterSlaveActivityBase PARENT;
+
     IHThorFilterGroupArg *helper;
     Owned<IThorRowLoader> groupLoader;
     Owned<IRowStream> groupStream;
@@ -253,16 +252,16 @@ public:
             setCompFlag(compType, spillCompInfo);
         }
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
-        CFilterSlaveActivityBase::init(data,slaveData);
+        PARENT::init(data,slaveData);
         helper = (IHThorFilterGroupArg *)queryHelper();
     }
-    void start()
+    virtual void start() override
     {   
         ActivityTimer s(totalCycles, timeActivities);
         abortSoon = !helper->canMatchAny();
-        CFilterSlaveActivityBase::start();
+        PARENT::start();
     }
     CATCH_NEXTROW()
     {
@@ -283,7 +282,7 @@ public:
             CThorExpandingRowArray rows(*this, this);
             try
             {
-                groupLoader->loadGroup(input, abortSoon, &rows);
+                groupLoader->loadGroup(inputStream, abortSoon, &rows);
             }
             catch (IException *e)
             {
@@ -349,18 +348,18 @@ public:
             if (stepExtra.returnMismatches())
             {
                 bool matchedCompletely = true;
-                ret.setown(input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
+                ret.setown(inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
                 if (!wasCompleteMatch)
                     return ret.getClear();
             }
             else
-                ret.setown(input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
+                ret.setown(inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
 #endif
 
         CThorExpandingRowArray rows(*this, this);
         try
         {
-            groupStream.setown(groupLoader->loadGroup(input, abortSoon, &rows));
+            groupStream.setown(groupLoader->loadGroup(inputStream, abortSoon, &rows));
         }
         catch (IException *e)
         {
@@ -388,19 +387,18 @@ public:
     { 
         abortSoon = false;
         groupStream.clear();
-        input->resetEOF(); 
+        inputStream->resetEOF();
     }
-    void stop()
+    virtual void stop() override
     {
+        PARENT::stop();
         groupStream.clear();
-        stopInput(input);
-        dataLinkStop();
     }
 // steppable
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CFilterSlaveActivityBase::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        PARENT::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta() { return CThorSteppable::inputStepping; }
 };

thorlcr/activities/firstn/thfirstnslave.cpp

@@ -25,44 +25,37 @@
 
 #include "thfirstnslave.ipp"
 
-class CFirstNSlaveBase : public CSlaveActivity, public CThorDataLink
+class CFirstNSlaveBase : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 protected:
     rowcount_t limit, skipCount;
-    Owned<IThorDataLink> input;
     bool stopped;
     IHThorFirstNArg *helper;
 
     virtual void doStop()
     {
-        stopInput(input);
-        dataLinkStop();
+        PARENT::stop();
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CFirstNSlaveBase(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CFirstNSlaveBase(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         stopped = true;
         helper = (IHThorFirstNArg *)container.queryHelper();
     }
-    ~CFirstNSlaveBase()
-    {
-    }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         appendOutputLinked(this);
     }
-    void start()
+    virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input.set(inputs.item(0));
-        startInput(input);
+        PARENT::start();
         stopped = false;
-        dataLinkStart();
     }
-    void stop()
+    virtual void stop()
     {
         if (!stopped)
         {
@@ -70,19 +63,20 @@ public:
             stopped = true;
             doStop();
         }
-        input.clear();
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.canReduceNumRows = true;
         info.totalRowsMax = helper->getLimit();
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 
 class CFirstNSlaveLocal : public CFirstNSlaveBase
 {
+    typedef CFirstNSlaveBase PARENT;
+
     bool firstget;
     rowcount_t skipped;
 public:
@@ -91,10 +85,10 @@ public:
     }
 
 // IRowStream overrides
-    virtual bool isGrouped() { return false; }
-    void start()
+    virtual bool isGrouped() const override { return false; }
+    virtual void start()
     {
-        CFirstNSlaveBase::start();
+        PARENT::start();
         skipCount = validRC(helper->numToSkip());
         limit = (rowcount_t)helper->getLimit();
         firstget = true;
@@ -110,7 +104,7 @@ public:
                 firstget = false;
                 while (skipped<skipCount)
                 {
-                    OwnedConstThorRow row = input->ungroupedNextRow();
+                    OwnedConstThorRow row = inputStream->ungroupedNextRow();
                     if (!row)
                     {
                         stop();
@@ -121,7 +115,7 @@ public:
             }
             if (getDataLinkCount() < limit)
             {
-                OwnedConstThorRow row = input->ungroupedNextRow();
+                OwnedConstThorRow row = inputStream->ungroupedNextRow();
                 if (row)
                 {
                     dataLinkIncrement();
@@ -136,6 +130,8 @@ public:
 
 class CFirstNSlaveGrouped : public CFirstNSlaveBase
 {
+    typedef CFirstNSlaveBase PARENT;
+
     unsigned countThisGroup;
 public:
     CFirstNSlaveGrouped(CGraphElementBase *container) : CFirstNSlaveBase(container)
@@ -143,10 +139,10 @@ public:
     }
 
 // IRowStream overrides
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
-    void start()
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
+    virtual void start()
     {
-        CFirstNSlaveBase::start();
+        PARENT::start();
         skipCount = validRC(helper->numToSkip());
         limit = (rowcount_t)helper->getLimit();
         countThisGroup = 0;
@@ -163,7 +159,7 @@ public:
                     unsigned skipped = 0;
                     do
                     {
-                        OwnedConstThorRow row = input->nextRow();
+                        OwnedConstThorRow row = inputStream->nextRow();
                         if (row) 
                             skipped++;
                         else
@@ -180,7 +176,7 @@ public:
                 }
                 if (countThisGroup < limit)
                 {
-                    OwnedConstThorRow row = input->nextRow();
+                    OwnedConstThorRow row = inputStream->nextRow();
                     if (row)
                     {
                         countThisGroup++;
@@ -197,7 +193,7 @@ public:
                 { // consume rest of group
                     loop
                     {
-                        OwnedConstThorRow row = input->nextRow();
+                        OwnedConstThorRow row = inputStream->nextRow();
                         if (!row)
                             break;
                     }
@@ -211,52 +207,55 @@ public:
     }
 };
 
-class CFirstNSlaveGlobal : public CFirstNSlaveBase, implements ISmartBufferNotify
+class CFirstNSlaveGlobal : public CFirstNSlaveBase, implements ILookAheadStopNotify
 {
+    typedef CFirstNSlaveBase PARENT;
+
     Semaphore limitgot;
     CriticalSection crit;
     rowcount_t maxres, skipped, totallimit;
     bool firstget;
     ThorDataLinkMetaInfo inputMeta;
 
-//  ISmartBufferNotify methods used for global firstn only
-
 protected:
     virtual void doStop()
     {
         limitgot.signal(); // JIC not previously signalled by lookahead
         onInputFinished(getDataLinkCount()+skipped);
-        CFirstNSlaveBase::doStop();
+        PARENT::doStop();
     }
 
 public:
     CFirstNSlaveGlobal(CGraphElementBase *container) : CFirstNSlaveBase(container)
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
-        CFirstNSlaveBase::init(data, slaveData);
+        PARENT::init(data, slaveData);
         mpTag = container.queryJobChannel().deserializeMPTag(data);
     }
-    void start()
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
     {
-        CFirstNSlaveBase::start(); // adds to totalTime (common to local and global firstn)
-        ActivityTimer s(totalCycles, timeActivities);
+        PARENT::setInputStream(index, _input, consumerOrdered);
         totallimit = (rowcount_t)helper->getLimit();
+        rowcount_t _skipCount = validRC(helper->numToSkip()); // max
+        rowcount_t maxRead = (totallimit>(RCUNBOUND-_skipCount))?RCUNBOUND:totallimit+_skipCount;
+        setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), FIRSTN_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), false,
+                                          maxRead, this, &container.queryJob().queryIDiskUsage())); // if a very large limit don't bother truncating
+    }
+    virtual void start()
+    {
+        ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start(); // adds to totalTime (common to local and global firstn)
         limit = maxres = RCUNBOUND;
         skipCount = 0;
         skipped = 0;
         firstget = true;
         input->getMetaInfo(inputMeta);
-        rowcount_t _skipCount = validRC(helper->numToSkip()); // max
-        rowcount_t maxRead = (totallimit>(RCUNBOUND-_skipCount))?RCUNBOUND:totallimit+_skipCount;
-        input.setown(createDataLinkSmartBuffer(this, input,FIRSTN_SMART_BUFFER_SIZE,isSmartBufferSpillNeeded(this),false,
-                                          maxRead,this,true,&container.queryJob().queryIDiskUsage())); // if a very large limit don't bother truncating
-        startInput(input);
     }
-    void abort()
+    virtual void abort()
     {
-        CFirstNSlaveBase::abort();
+        PARENT::abort();
         limitgot.signal();
         CriticalBlock b(crit);
         cancelReceiveMsg(RANK_ALL, mpTag);
@@ -334,7 +333,7 @@ public:
                     return NULL;
                 while (skipped<skipCount)
                 {
-                    OwnedConstThorRow row = input->ungroupedNextRow();
+                    OwnedConstThorRow row = inputStream->ungroupedNextRow();
                     if (!row)
                     {
                         stop();
@@ -345,7 +344,7 @@ public:
             }
             if (getDataLinkCount() < limit)
             {
-                OwnedConstThorRow row = input->ungroupedNextRow();
+                OwnedConstThorRow row = inputStream->ungroupedNextRow();
                 if (row)
                 {
                     dataLinkIncrement();
@@ -357,15 +356,13 @@ public:
         return NULL;
     }
 
-    bool isGrouped() { return false; } // need to do different if is!
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; } // need to do different if is!
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
-        CFirstNSlaveBase::getMetaInfo(info);
+        PARENT::getMetaInfo(info);
         info.canBufferInput = true;
     }
-    //  ISmartBufferNotify methods used for global firstn only
-    virtual void onInputStarted(IException *) { } // not needed
-    virtual bool startAsync() { return false; }
+// ILookAheadStopNotify
     virtual void onInputFinished(rowcount_t count)  // count is the total read from input (including skipped)
     {
         // sneaky short circuit

thorlcr/activities/funnel/thfunnelslave.cpp

@@ -34,15 +34,14 @@ class CParallelFunnel : public CSimpleInterface, implements IRowStream
     {
         CThreadedPersistent threaded;
         CParallelFunnel &funnel;
-        Linked<IRowStream> input;
         CriticalSection stopCrit;
         StringAttr idStr;
         unsigned inputIndex;
         rowcount_t readThisInput; // purely for tracing
         bool stopping;
     public:
-        CInputHandler(CParallelFunnel &_funnel, IRowStream *_input, unsigned _inputIndex) 
-            : threaded("CInputHandler", this), funnel(_funnel), input(_input), inputIndex(_inputIndex)
+        CInputHandler(CParallelFunnel &_funnel, unsigned _inputIndex)
+            : threaded("CInputHandler", this), funnel(_funnel), inputIndex(_inputIndex)
         {
             readThisInput = 0;
             StringBuffer s(funnel.idStr);
@@ -74,17 +73,15 @@ class CParallelFunnel : public CSimpleInterface, implements IRowStream
         virtual void main()
         {
             bool started = false;
+            IEngineRowStream *inputStream = nullptr;
             try
             {
-                if (funnel.startInputs)
-                {
-                    IThorDataLink *_input = QUERYINTERFACE(input.get(), IThorDataLink);
-                    _input->start();
-                }
+                funnel.activity.startInput(inputIndex);
                 started = true;
+                inputStream = funnel.activity.queryInputStream(inputIndex);
                 while (!stopping)
                 {
-                    OwnedConstThorRow row = input->ungroupedNextRow();
+                    OwnedConstThorRow row = inputStream->ungroupedNextRow();
                     if (!row) break;
 
                     {
@@ -107,7 +104,7 @@ class CParallelFunnel : public CSimpleInterface, implements IRowStream
                 return;
             try
             {
-                input->stop();
+                inputStream->stop();
             }
             catch (IException *e)
             {
@@ -118,12 +115,11 @@ class CParallelFunnel : public CSimpleInterface, implements IRowStream
         }
     };
 
-    CActivityBase &activity;
+    CSlaveActivity &activity;
     CIArrayOf<CInputHandler> inputHandlers;
     bool startInputs;
     Linked<IException> exception;
     unsigned eoss;
-    IArrayOf<IRowStream> oinstreams;
     StringAttr idStr;
 
     CriticalSection fullCrit, crit;
@@ -150,18 +146,8 @@ class CParallelFunnel : public CSimpleInterface, implements IRowStream
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CParallelFunnel(CActivityBase &_activity, IRowStream **instreams, unsigned numstreams) : activity(_activity)
-    {
-        startInputs = false;
-        unsigned n = 0;
-        while (n<numstreams) oinstreams.append(*LINK(instreams[n++]));
-        init();
-    }
-    CParallelFunnel(CActivityBase &_activity, IThorDataLink **instreams, unsigned numstreams, bool _startInputs) : activity(_activity)
+    CParallelFunnel(CSlaveActivity &_activity) : activity(_activity)
     {
-        startInputs = _startInputs;
-        unsigned n = 0;
-        while (n<numstreams) oinstreams.append(*LINK(instreams[n++]));
         init();
     }
     ~CParallelFunnel()
@@ -179,10 +165,9 @@ public:
         stopped = full = false;
         totSize = 0;
         eoss = 0;
-        unsigned numinputs = oinstreams.ordinality();
         serializer.set(activity.queryRowSerializer());
-        ForEachItemIn(i, oinstreams)
-            inputHandlers.append(* new CInputHandler(*this, &oinstreams.item(i), i));
+        for (unsigned i=0; i<activity.queryNumInputs(); i++)
+            inputHandlers.append(* new CInputHandler(*this, i));
         // because of the way eos reported make sure started afterwards
         ForEachItemIn(j, inputHandlers)
             inputHandlers.item(j).start();
@@ -269,17 +254,6 @@ friend class CInputHandler;
 };
 
 
-IRowStream *createParallelFunnel(CActivityBase &activity, IRowStream **instreams, unsigned numstreams)
-{
-    return new CParallelFunnel(activity, instreams, numstreams);
-}
-
-IRowStream *createParallelFunnel(CActivityBase &activity, IThorDataLink **instreams, unsigned numstreams, bool startInputs)
-{
-    return new CParallelFunnel(activity, instreams, numstreams, startInputs);
-}
-
-
 ///////////////////
 //
 // FunnelSlaveActivity
@@ -287,9 +261,11 @@ IRowStream *createParallelFunnel(CActivityBase &activity, IThorDataLink **instre
 
 //class CParallelFunnel;
 //interface IBitSet;
-class FunnelSlaveActivity : public CSlaveActivity, public CThorDataLink
+class FunnelSlaveActivity : public CSlaveActivity
 {
-    IThorDataLink *current;
+    typedef CSlaveActivity PARENT;
+
+    IRowStream *current;
     unsigned currentMarker;
     bool grouped, *eog, eogNext, parallel;
     rowcount_t readThisInput;
@@ -297,9 +273,7 @@ class FunnelSlaveActivity : public CSlaveActivity, public CThorDataLink
     Owned<IRowStream> parallelOutput;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    FunnelSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    FunnelSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         grouped = false;
         eog = NULL;
@@ -326,7 +300,10 @@ public:
     {
         ActivityTimer s(totalCycles, timeActivities);
         if (!grouped && parallel)
-            parallelOutput.setown(createParallelFunnel(*this, inputs.getArray(), inputs.ordinality(), true));
+        {
+            //NB starts inputs on each thread
+            parallelOutput.setown(new CParallelFunnel(*this));
+        }
         else
         {
             eogNext = false;
@@ -344,14 +321,14 @@ public:
             readThisInput = 0;
             ForEachItemIn(i, inputs)
             {
-                IThorDataLink * input = inputs.item(i);
-                try { startInput(input); }
+                try { startInput(i); }
                 catch (CATCHALL)
                 {
                     ActPrintLog("FUNNEL(%" ACTPF "d): Error staring input %d", container.queryId(), i);
                     throw;
                 }
-                if (!current) current = input;
+                if (!current)
+                    current = queryInputStream(i);
             }
         }
         dataLinkStart();
@@ -368,7 +345,7 @@ public:
             current = NULL;
             unsigned i = stopped;
             for (;i<inputs.ordinality(); i++)
-                stopInput(inputs.item(i));
+                stopInput(i);
             stopped = 0;
         }
         dataLinkStop();
@@ -395,11 +372,11 @@ public:
                 if (currentMarker + 1 < inputs.ordinality())
                 {
                     readThisInput = 0;
+                    stopInput(currentMarker);
+                    ++stopped;
                     currentMarker++;
                     ActPrintLog("FUNNEL: changing to input %d", currentMarker);
-                    ++stopped;
-                    stopInput(current);
-                    current = inputs.item(currentMarker);
+                    current = queryInputStream(currentMarker);
                     // if empty stream, move on (ensuring eog,eog not returned by empty streams)
                     row.setown(current->nextRow());
                     if (row)
@@ -449,11 +426,11 @@ public:
                 if (currentMarker + 1 < inputs.ordinality())
                 {
                     readThisInput = 0;
+                    stopInput(currentMarker);
+                    ++stopped;
                     currentMarker++;
                     ActPrintLog("FUNNEL: changing to input %d", currentMarker);
-                    ++stopped;
-                    stopInput(current);
-                    current = inputs.item(currentMarker);
+                    current = queryInputStream(currentMarker);
                     row.setown(current->ungroupedNextRow());
                     if (row)
                     {
@@ -475,9 +452,9 @@ public:
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
-        calcMetaInfoSize(info, inputs.getArray(), inputs.ordinality());
+        calcMetaInfoSize(info, inputs);
     }
-    virtual bool isGrouped() { return grouped; }
+    virtual bool isGrouped() const override { return grouped; }
 };
 
 /////
@@ -486,7 +463,7 @@ public:
 // CombineSlaveActivity
 //
 
-class CombineSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CombineSlaveActivity : public CSlaveActivity
 {
     IHThorCombineArg *helper;
     bool grouped;
@@ -495,11 +472,8 @@ class CombineSlaveActivity : public CSlaveActivity, public CThorDataLink
     CThorExpandingRowArray rows;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-
     CombineSlaveActivity(CGraphElementBase *_container) 
-        : CSlaveActivity(_container), CThorDataLink(this), rows(*this, this)
+        : CSlaveActivity(_container), rows(*this, this)
     {
         grouped = container.queryGrouped();
     }
@@ -508,18 +482,17 @@ public:
         helper = (IHThorCombineArg *) queryHelper();
         appendOutputLinked(this);
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         init();
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         eogNext = false;
         ForEachItemIn(i, inputs)
         {
-            IThorDataLink * input = inputs.item(i);
-            try { startInput(input); }
+            try { startInput(i); }
             catch (CATCHALL)
             {
                 ActPrintLog("COMBINE(%" ACTPF "d): Error staring input %d", container.queryId(), i);
@@ -528,38 +501,44 @@ public:
         }
         dataLinkStart();
     }
-    void stop()
+    virtual void stop() override
     {
-        for (unsigned i=0;i<inputs.ordinality(); i++)
-            stopInput(inputs.item(i));
+        stopAllInputs();
         dataLinkStop();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
-        loop {
+        loop
+        {
             bool eog = false;
             bool err = false;
             unsigned i;
             unsigned n = inputs.ordinality();
-            for (i=0;i<n;i++) {
-                OwnedConstThorRow row = inputs.item(i)->nextRow();
-                if (row) {
-                    if (eog) {
+            for (i=0;i<n;i++)
+            {
+                OwnedConstThorRow row = queryInputStream(i)->nextRow();
+                if (row)
+                {
+                    if (eog)
+                    {
                         err = true;
                         break;
                     }
                     rows.append(row.getClear());
                 }
-                else {
-                    if (i&&!eog) {
+                else
+                {
+                    if (i&&!eog)
+                    {
                         err = true;
                         break;
                     }
                     eog = true;
                 }
             }
-            if (err) {
+            if (err)
+            {
                 eog = true;
                 rows.kill();
                 throw MakeActivityException(this, -1, "mismatched input row count for Combine");
@@ -569,7 +548,8 @@ public:
             RtlDynamicRowBuilder row(queryRowAllocator());
             size32_t sizeGot = helper->transform(row, rows.ordinality(), rows.getRowArray());
             rows.kill();
-            if (sizeGot) {
+            if (sizeGot)
+            {
                 dataLinkIncrement();
                 return row.finalizeRowClear(sizeGot);
             }
@@ -577,11 +557,11 @@ public:
         rows.kill();
         return NULL;
     }
-    bool isGrouped()
+    virtual bool isGrouped() const override
     {
-        return inputs.item(0)->isGrouped();
+        return queryInput(0)->isGrouped();
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         // TBD I think this should say max out = lhs set.
@@ -592,16 +572,14 @@ public:
 /////
 
 
-class RegroupSlaveActivity : public CSlaveActivity, public CThorDataLink
+class RegroupSlaveActivity : public CSlaveActivity
 {
     IHThorRegroupArg *helper;
     bool grouped;
     bool eogNext;
     unsigned curinput;
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    RegroupSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    RegroupSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         grouped = container.queryGrouped();
     }
@@ -610,19 +588,18 @@ public:
         helper = (IHThorRegroupArg *) queryHelper();
         appendOutputLinked(this);
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         init();
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         curinput = 0;
         eogNext = false;
         ForEachItemIn(i, inputs)
         {
-            IThorDataLink * input = inputs.item(i);
-            try { startInput(input); }
+            try { startInput(i); }
             catch (CATCHALL)
             {
                 ActPrintLog("REGROUP(%" ACTPF "d): Error staring input %d", container.queryId(), i);
@@ -631,42 +608,46 @@ public:
         }
         dataLinkStart();
     }
-    void stop()
+    virtual void stop() override
     {
-        for (unsigned i=0;i<inputs.ordinality(); i++)
-            stopInput(inputs.item(i));
+        stopAllInputs();
         dataLinkStop();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
         unsigned n = inputs.ordinality();
-        loop {
-            if (curinput==n) {
-                curinput = 0;
-                break;
-            }
-            OwnedConstThorRow row = inputs.item(curinput)->nextRow();
-            if (row) {
+        IRowStream *current = queryInputStream(curinput);
+        loop
+        {
+            OwnedConstThorRow row = current->nextRow();
+            if (row)
+            {
                 dataLinkIncrement();
                 return row.getClear();
             }
             curinput++;
+            if (curinput==n)
+            {
+                curinput = 0;
+                break;
+            }
+            current = queryInputStream(curinput);
         }
         return NULL;
     }
 
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
-        calcMetaInfoSize(info, inputs.getArray(), inputs.ordinality());
+        calcMetaInfoSize(info, inputs);
     }
-    virtual bool isGrouped() { return true; }
+    virtual bool isGrouped() const override { return true; }
 };
 
 /////
 
-class NonEmptySlaveActivity : public CSlaveActivity, public CThorDataLink
+class NonEmptySlaveActivity : public CSlaveActivity
 {
     IHThorNonEmptyArg *helper;
     bool eogNext, eoi, anyThisGroup, anyThisInput;
@@ -709,9 +690,7 @@ class NonEmptySlaveActivity : public CSlaveActivity, public CThorDataLink
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    NonEmptySlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    NonEmptySlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = (IHThorNonEmptyArg *) queryHelper();
         sendReceiving = false;
@@ -734,15 +713,14 @@ public:
     }
 
 // IThorDataLink
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         curinput = 0;
         anyThisGroup = anyThisInput = eogNext = false;
         ForEachItemIn(i, inputs)
         {
-            IThorDataLink * input = inputs.item(i);
-            try { startInput(input); }
+            try { startInput(i); }
             catch (CATCHALL)
             {
                 ActPrintLog("NONEMPTY(%" ACTPF "d): Error staring input %d", container.queryId(), i);
@@ -752,10 +730,9 @@ public:
         eoi = 0 == inputs.ordinality();
         dataLinkStart();
     }
-    virtual void stop()
+    virtual void stop() override
     {
-        for (unsigned i=0;i<inputs.ordinality(); i++)
-            stopInput(inputs.item(i));
+        stopAllInputs();
         dataLinkStop();
     }
     CATCH_NEXTROW()
@@ -769,7 +746,7 @@ public:
         }
         loop
         {
-            OwnedConstThorRow row = inputs.item(curinput)->nextRow();
+            OwnedConstThorRow row = queryInputStream(curinput)->nextRow();
             if (row ) {
                 anyThisGroup = true;
                 anyThisInput = true;
@@ -789,136 +766,131 @@ public:
         }
         return NULL;
     }
-    virtual bool isGrouped() { return container.queryGrouped(); }
+    virtual bool isGrouped() const override { return container.queryGrouped(); }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.canReduceNumRows = true;
-        calcMetaInfoSize(info, inputs.getArray(), inputs.ordinality());
+        calcMetaInfoSize(info, inputs);
     }
 };
 
 
-class CNWaySelectActivity : public CSlaveActivity, public CThorDataLink, public CThorSteppable
+class CNWaySelectActivity : public CSlaveActivity, public CThorSteppable
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorNWaySelectArg *helper;
-    IThorDataLink *selectedInput;
+    IThorDataLink *selectedInputITDL = nullptr;
+    IEngineRowStream *selectedStream = nullptr;
+    Owned<IStrandJunction> selectedJunction;
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CNWaySelectActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this), CThorSteppable(this)
+    CNWaySelectActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorSteppable(this)
     {
         helper = (IHThorNWaySelectArg *)queryHelper();
-        selectedInput = NULL;
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         appendOutputLinked(this);
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
 
-        startInput(inputs.item(0));
+        PARENT::start();
 
         unsigned whichInput = helper->getInputIndex();
-        selectedInput = NULL;
+        selectedInputITDL = NULL;
+        selectedStream = NULL;
         if (whichInput--)
         {
             ForEachItemIn(i, inputs)
             {
-                IThorDataLink *cur = inputs.item(i);
-                CActivityBase *activity = cur->queryFromActivity();
-                IThorNWayInput *nWayInput = dynamic_cast<IThorNWayInput *>(activity);
+                IThorDataLink *cur = queryInput(i);
+                IThorNWayInput *nWayInput = dynamic_cast<IThorNWayInput *>(cur);
                 if (nWayInput)
                 {
                     unsigned numRealInputs = nWayInput->numConcreteOutputs();
                     if (whichInput < numRealInputs)
                     {
-                        selectedInput = nWayInput->queryConcreteInput(whichInput);
+                        selectedInputITDL = nWayInput->queryConcreteInput(whichInput);
+                        selectedStream = connectSingleStream(*this, selectedInputITDL, 0, selectedJunction, true);  // Should this be passing whichInput??
                         break;
                     }
                     whichInput -= numRealInputs;
                 }
-                else
-                {
-                    if (whichInput == 0)
-                        selectedInput = cur;
-                    whichInput -= 1;
-                }
-                if (selectedInput)
-                    break;
             }
         }
-        if (selectedInput)
-            selectedInput->start();
-        dataLinkStart();
+        if (selectedInputITDL)
+            selectedInputITDL->start();
+        startJunction(selectedJunction);
     }
-    void stop()
+    virtual void stop() override
     {
-        stopInput(inputs.item(0));
-        if (selectedInput)
-            selectedInput->stop();
-        dataLinkStop();
+        PARENT::stop();
+        if (selectedStream)
+            selectedStream->stop();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
-        if (!selectedInput)
+        if (!selectedStream)
             return NULL;
-        OwnedConstThorRow ret = selectedInput->nextRow();
+        OwnedConstThorRow ret = selectedStream->nextRow();
         if (ret)
             dataLinkIncrement();
         return ret.getClear();
     }
-    bool gatherConjunctions(ISteppedConjunctionCollector &collector)
+    virtual bool gatherConjunctions(ISteppedConjunctionCollector &collector)
     { 
-        if (!selectedInput)
+        if (!selectedStream)
             return false;
-        return selectedInput->gatherConjunctions(collector);
+        return selectedInputITDL->gatherConjunctions(collector);
     }
-    void resetEOF() 
+    virtual void resetEOF()
     { 
-        if (selectedInput)
-            selectedInput->resetEOF(); 
+        if (selectedStream)
+            selectedStream->resetEOF();
     }
-    const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         try { return nextRowGENoCatch(seek, numFields, wasCompleteMatch, stepExtra); }
         CATCH_NEXTROWX_CATCH;
     }
-    const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         ActivityTimer t(totalCycles, timeActivities);
-        if (!selectedInput)
+        if (!selectedStream)
             return NULL;
-        return selectedInput->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
+        return selectedStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
-        if (selectedInput)
-            calcMetaInfoSize(info, selectedInput);
-        else if (!started())
+        if (selectedStream)
+            calcMetaInfoSize(info, selectedInputITDL);
+        else if (!hasStarted())
             info.canStall = true; // unkwown if !started
     }
-    virtual bool isGrouped() { return selectedInput ? selectedInput->isGrouped() : false; }
+    virtual bool isGrouped() const override { return selectedInputITDL ? selectedInputITDL->isGrouped() : false; }
 // steppable
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CSlaveActivity::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        CSlaveActivity::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta()
     {
-        if (selectedInput)
-            return selectedInput->querySteppingMeta();
+        if (selectedInputITDL)
+            return selectedInputITDL->querySteppingMeta();
         return NULL;
     }
 };
 
 
-class CThorNWayInputSlaveActivity : public CSlaveActivity, public CThorDataLink, implements IThorNWayInput
+class CThorNWayInputSlaveActivity : public CSlaveActivity, implements IThorNWayInput
 {
     IHThorNWayInputArg *helper;
     PointerArrayOf<IThorDataLink> selectedInputs;
@@ -927,16 +899,16 @@ class CThorNWayInputSlaveActivity : public CSlaveActivity, public CThorDataLink,
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CThorNWayInputSlaveActivity(CGraphElementBase *container) : CSlaveActivity(container), CThorDataLink(this)
+    CThorNWayInputSlaveActivity(CGraphElementBase *container) : CSlaveActivity(container)
     {
         helper = (IHThorNWayInputArg *)queryHelper();
         grouped = helper->queryOutputMeta()->isGrouped(); // JCSMORE should match graph info, i.e. container.queryGrouped()
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         bool selectionIsAll;
@@ -947,7 +919,7 @@ public:
         if (selectionIsAll)
         {
             ForEachItemIn(i, inputs)
-                selectedInputs.append(inputs.item(i));
+                selectedInputs.append(queryInput(i));
         }
         else
         {
@@ -965,27 +937,25 @@ public:
                 if (!inputs.isItem(nextIndex-1))
                     throw MakeStringException(100, "Index %d in RANGE selection list is out of range", nextIndex);
 
-                selectedInputs.append(inputs.item(nextIndex-1));
+                selectedInputs.append(queryInput(nextIndex-1));
             }
         }
         // NB: Whatever pulls this IThorNWayInput, starts and stops the selectedInputs
-        dataLinkStart();
     }
-    void stop()
+    virtual void stop() override
     {
         // NB: Whatever pulls this IThorNWayInput, starts and stops the selectedInputs
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
         throwUnexpected();
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
-    virtual bool isGrouped() { return grouped; }
+    virtual bool isGrouped() const override { return grouped; }
 // IThorNWayInput impl.
     virtual unsigned numConcreteOutputs() const
     {

thorlcr/activities/funnel/thfunnelslave.ipp

@@ -30,8 +30,6 @@
 
 
 activityslaves_decl CActivityBase *createFunnelSlave(CGraphElementBase *container);
-activityslaves_decl IRowStream *createParallelFunnel(CActivityBase &activity, IRowStream **instreams, unsigned numstreams, const bool &aborted);
-activityslaves_decl IRowStream *createParallelFunnel(CActivityBase &activity, IThorDataLink **instreams, unsigned numstreams, bool startInputs, const bool &aborted);
 activityslaves_decl CActivityBase *createCombineSlave(CGraphElementBase *container);
 activityslaves_decl CActivityBase *createRegroupSlave(CGraphElementBase *container);
 activityslaves_decl CActivityBase *createNonEmptySlave(CGraphElementBase *container);

thorlcr/activities/group/thgroupslave.cpp

@@ -20,15 +20,16 @@
 #include "thgroupslave.ipp"
 
 
-class GroupSlaveActivity : public CSlaveActivity, public CThorDataLink
+class GroupSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorGroupArg * helper;
     bool eogNext, prevEog, eof;
     bool rolloverEnabled, useRollover;
     rowcount_t numGroups;
     rowcount_t numGroupMax;
     rowcount_t startLastGroup;
-    IThorDataLink *input;
     Owned<IRowStream> stream, nextNodeStream;
     OwnedConstThorRow next;
     Owned<IRowServer> rowServer;
@@ -51,10 +52,8 @@ class GroupSlaveActivity : public CSlaveActivity, public CThorDataLink
             return NULL;
     }
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     GroupSlaveActivity(CGraphElementBase *_container)
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     {
         helper = static_cast <IHThorGroupArg *> (queryHelper());
         rolloverEnabled = false;
@@ -72,10 +71,11 @@ public:
             rolloverEnabled = true;
         }
     }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         ActPrintLog(rolloverEnabled ? "GROUP: is global" : "GROUP: is local");
+        PARENT::start();
         eogNext = prevEog = eof = false;
         if (rolloverEnabled)
         {
@@ -85,10 +85,7 @@ public:
 #endif
         }
 
-        input = inputs.item(0);
-        stream.set(input);
-        startInput(input);
-        dataLinkStart();
+        stream.set(inputStream);
         startLastGroup = getDataLinkGlobalCount();
         next.setown(getNext());
 
@@ -124,12 +121,11 @@ public:
             rowServer.setown(createRowServer(this, strm, queryJobChannel().queryJobComm(), mpTag));
         }
     }
-    virtual void stop()
+    virtual void stop() override
     {
         if (nextNodeStream)
             nextNodeStream->stop();
-        stopInput(input);
-        dataLinkStop();
+        PARENT::stop();
     }
     virtual void kill()
     {
@@ -176,7 +172,7 @@ public:
             numGroupMax = numThisGroup;
         numGroups++;
     }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         if (rolloverEnabled)
@@ -184,10 +180,10 @@ public:
             info.isSequential = true;
             info.unknownRowsOutput = true; // don't know how many rolled over
         }
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
-    virtual bool isGrouped() { return true; }
-    void serializeStats(MemoryBuffer &mb)
+    virtual bool isGrouped() const override{ return true; }
+    virtual void serializeStats(MemoryBuffer &mb) override
     {
         CSlaveActivity::serializeStats(mb);
         mb.append(numGroups);

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -1938,31 +1938,25 @@ IHashDistributor *createPullHashDistributor(CActivityBase *activity, ICommunicat
 #endif
 
 
-class HashDistributeSlaveBase : public CSlaveActivity, public CThorDataLink, implements IStopInput
+class HashDistributeSlaveBase : public CSlaveActivity, implements IStopInput
 {
-    IHashDistributor *distributor;
-    IThorDataLink *input;
-    Owned<IRowStream> out;
-    bool inputstopped;
+    typedef CSlaveActivity PARENT;
+
+    IHashDistributor *distributor = nullptr;
+    bool inputstopped = true;
     CriticalSection stopsect;
-    mptag_t mptag;
+    mptag_t mptag = TAG_NULL;
 protected:
+    Owned<IRowStream> out;
     Owned<IRowStream> instrm;
-    IHash *ihash;
-    ICompare *mergecmp;     // if non-null is merge distribute
-    bool eofin;
+    IHash *ihash = nullptr;
+    ICompare *mergecmp = nullptr;     // if non-null is merge distribute
+    bool eofin = false;
+    bool setupDist = true;
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     HashDistributeSlaveBase(CGraphElementBase *_container)
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     {
-        input = NULL;
-        eofin = false;
-        mptag = TAG_NULL;
-        distributor = NULL;
-        mergecmp = NULL;
-        ihash = NULL;
     }
     ~HashDistributeSlaveBase()
     {
@@ -1974,7 +1968,7 @@ public:
             distributor->Release();
         }
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         mptag = container.queryJobChannel().deserializeMPTag(data);
@@ -1989,41 +1983,27 @@ public:
     void stopInput()
     {
         CriticalBlock block(stopsect);  // can be called async by distribute
-        if (!inputstopped) {
-            CSlaveActivity::stopInput(input);
+        if (!inputstopped)
+        {
+            PARENT::stop();
             inputstopped = true;
         }
     }
-    void start(bool passthrough)
+    void doDistSetup()
     {
-        // bit messy
-        eofin = false;
-        if (!instrm.get()) // derived class may override
-        {
-            input = inputs.item(0);
-            startInput(input);
-            inputstopped = false;
-            instrm.set(input);
-            if (passthrough)
-                out.set(instrm);
-        }
-        else if (passthrough)
-        {
-            out.set(instrm);
-        }
-        if (!passthrough)
-        {
-            Owned<IThorRowInterfaces> myRowIf = getRowInterfaces(); // avoiding circular link issues
-            out.setown(distributor->connect(myRowIf, instrm, ihash, mergecmp));
-        }
-        dataLinkStart();
+        Owned<IThorRowInterfaces> myRowIf = getRowInterfaces(); // avoiding circular link issues
+        out.setown(distributor->connect(myRowIf, instrm, ihash, mergecmp));
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        start(false);
+        PARENT::start();
+        eofin = false;
+        instrm.set(inputStream);
+        if (setupDist)
+            doDistSetup();
     }
-    void stop()
+    virtual void stop() override
     {
         ActPrintLog("HASHDISTRIB: stopping");
         if (out)
@@ -2038,14 +2018,13 @@ public:
         }
         stopInput();
         instrm.clear();
-        dataLinkStop();
     }
-    void kill()
+    virtual void kill() override
     {
         ActPrintLog("HASHDISTRIB: kill");
         CSlaveActivity::kill();
     }
-    void abort()
+    virtual void abort() override
     {
         CSlaveActivity::abort();
         if (distributor)
@@ -2054,20 +2033,22 @@ public:
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities); // careful not to call again in derivatives
-        if (abortSoon||eofin) {
+        if (abortSoon||eofin)
+        {
             eofin = true;
             return NULL;
         }
         OwnedConstThorRow row = out->ungroupedNextRow();
-        if (!row.get()) {
+        if (!row.get())
+        {
             eofin =  true;
             return NULL;
         }
         dataLinkIncrement();
         return row.getClear();
     }
-    virtual bool isGrouped() { return false; }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.canStall = true; // currently
@@ -2082,7 +2063,7 @@ class HashDistributeSlaveActivity : public HashDistributeSlaveBase
 {
 public:
     HashDistributeSlaveActivity(CGraphElementBase *container) : HashDistributeSlaveBase(container) { }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         HashDistributeSlaveBase::init(data, slaveData);
         IHThorHashDistributeArg *distribargs = (IHThorHashDistributeArg *)queryHelper();
@@ -2132,8 +2113,6 @@ class CHDRproportional: public CSimpleInterface, implements IHash
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
-
     CHDRproportional(CActivityBase *_activity, IHThorHashDistributeArg *_args,mptag_t _mastertag) : activity(_activity)
     {
         args = _args;
@@ -2157,7 +2136,7 @@ public:
         free(sizes);
     }
 
-    IRowStream *calc(CSlaveActivity *activity, IThorDataLink *in,bool &passthrough)
+    IRowStream *calc(CSlaveActivity *activity, IThorDataLink *in, IEngineRowStream *inputStream, bool &passthrough)
     {
         // first - find size
         serializer.set(activity->queryRowSerializer());
@@ -2179,7 +2158,6 @@ public:
             {
                 ActPrintLogEx(&activity->queryContainer(), thorlog_null, MCwarning, "REDISTRIBUTE size unknown, spilling to disk");
                 MemoryAttr ma;
-                activity->startInput(in);
                 if (activity->getOptBool(THOROPT_COMPRESS_SPILLS, true))
                 {
                     rwFlags |= rw_compress;
@@ -2192,14 +2170,14 @@ public:
                     throw MakeStringException(-1,"Could not created file %s",tempname.str());
                 loop
                 {
-                    const void * row = in->ungroupedNextRow();
+                    const void * row = inputStream->ungroupedNextRow();
                     if (!row)
                         break;
                     out->putRow(row);
                 }
                 out->flush();
                 sz = out->getPosition();
-                activity->stopInput(in);
+                activity->stop();
             }
             ret.setown(createRowStream(tempfile, activity, rwFlags));
         }
@@ -2326,41 +2304,43 @@ public:
             sizes[self] -= rs;
         return self;
     }
-
 };
 
 
 class ReDistributeSlaveActivity : public HashDistributeSlaveBase
 {
+    typedef HashDistributeSlaveBase PARENT;
+
     Owned<CHDRproportional> partitioner;
 public:
     ReDistributeSlaveActivity(CGraphElementBase *container) : HashDistributeSlaveBase(container) { }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         HashDistributeSlaveBase::init(data, slaveData);
         mptag_t tag = container.queryJobChannel().deserializeMPTag(data);
         IHThorHashDistributeArg *distribargs = (IHThorHashDistributeArg *)queryHelper();
         partitioner.setown(new CHDRproportional(this, distribargs,tag));
         ihash = partitioner;
+        setupDist = false;
     }
-
-    void start()
+    virtual void start() override
     {
+        ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         bool passthrough;
-        {
-            ActivityTimer s(totalCycles, timeActivities);
-            instrm.setown(partitioner->calc(this,inputs.item(0),passthrough));  // may return NULL
-        }
-        HashDistributeSlaveBase::start(passthrough);
+        Owned<IRowStream> calcStream = partitioner->calc(this, input, inputStream, passthrough);  // may return NULL
+        if (calcStream)
+            instrm.setown(calcStream.getClear());
+        if (passthrough)
+            out.set(instrm);
+        else
+            doDistSetup();
     }
-
-    void stop()
+    virtual void stop() override
     {
         HashDistributeSlaveBase::stop();
-        if (instrm) {
-            instrm.clear();
-            // should remove here rather than later?
-        }
+        if (instrm)
+            instrm.clear(); // should remove here rather than later?
     }
 };
 
@@ -2598,7 +2578,7 @@ public:
     }
 };
 
-class CBucket : public CSimpleInterface, implements IInterface
+class CBucket : public CSimpleInterface
 {
     HashDedupSlaveActivityBase &owner;
     IThorRowInterfaces *rowIf, *keyIf;
@@ -2614,8 +2594,6 @@ class CBucket : public CSimpleInterface, implements IInterface
 
     void doSpillHashTable();
 public:
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
-
     CBucket(HashDedupSlaveActivityBase &_owner, IThorRowInterfaces *_rowIf, IThorRowInterfaces *_keyIf, IHash *_iRowHash, IHash *_iKeyHash, ICompare *_iCompare, bool _extractKey, unsigned _bucketN, CHashTableRowTable *_htRows);
     bool addKey(const void *key, unsigned hashValue);
     bool addRow(const void *row, unsigned hashValue);
@@ -2773,11 +2751,13 @@ public:
     }
 };
 
-class HashDedupSlaveActivityBase : public CSlaveActivity, public CThorDataLink
+class HashDedupSlaveActivityBase : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 protected:
-    IRowStream *input;      // can be changed
-    IRowStream *initialInput;
+    IRowStream *distInput = nullptr;
+    IRowStream *initialInput = nullptr;
     Owned<IRowStream> currentInput;
     bool inputstopped, eos, lastEog, extractKey, local, isVariable, grouped;
     IHThorHashDedupArg *helper;
@@ -2827,9 +2807,8 @@ public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
     HashDedupSlaveActivityBase(CGraphElementBase *_container, bool _local)
-        : CSlaveActivity(_container), CThorDataLink(this), local(_local)
+        : CSlaveActivity(_container), local(_local)
     {
-        input = initialInput = NULL;
         initialNumBuckets = 0;
         inputstopped = eos = lastEog = extractKey = local = isVariable = grouped = false;
         helper = NULL;
@@ -2887,15 +2866,16 @@ public:
         }
         grouped = container.queryGrouped();
     }
-    void start()
+    virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         inputstopped = false;
         eos = lastEog = false;
-        startInput(inputs.item(0));
         ThorDataLinkMetaInfo info;
-        inputs.item(0)->getMetaInfo(info);
-        initialInput = input = inputs.item(0);
+        queryInput(0)->getMetaInfo(info);
+        distInput = inputStream;
+        initialInput = distInput;
         unsigned div = local ? 1 : queryJob().querySlaves(); // if global, hash values already modulated by # slaves
         bucketHandler.setown(new CBucketHandler(*this, this, keyRowInterfaces, iHash, iKeyHash, rowKeyCompare, extractKey, 0, div));
         initialNumBuckets = container.queryXGMML().getPropInt("hint[@name=\"num_buckets\"]/@value");
@@ -2908,14 +2888,13 @@ public:
         }
         ensureNumHashTables(initialNumBuckets);
         bucketHandler->init(initialNumBuckets);
-        dataLinkStart();
     }
     void stopInput()
     {
         if (!inputstopped)
         {
             SpinBlock b(stopSpin);
-            CSlaveActivity::stopInput(inputs.item(0));
+            PARENT::stop();
             inputstopped = true;
         }
     }
@@ -2938,7 +2917,7 @@ public:
             OwnedConstThorRow row;
             {
                 SpinBlock b(stopSpin);
-                row.setown(grouped?input->nextRow():input->ungroupedNextRow());
+                row.setown(grouped?distInput->nextRow():distInput->ungroupedNextRow());
             }
             if (row)
             {
@@ -2974,7 +2953,7 @@ public:
                                 {
                                     lastEog = true;
                                     // reset for next group
-                                    input = initialInput;
+                                    distInput = initialInput;
                                     bucketHandler.setown(new CBucketHandler(*this, this, keyRowInterfaces, iHash, iKeyHash, rowKeyCompare, extractKey, 0, 1));
                                     ensureNumHashTables(initialNumBuckets); // resets
                                     bucketHandler->init(initialNumBuckets);
@@ -2997,12 +2976,12 @@ public:
                     }
                 }
                 assertex(currentInput);
-                input = currentInput;
+                distInput = currentInput;
             }
         }
     }
 
-    virtual bool isGrouped() { return grouped; }
+    virtual bool isGrouped() const override { return grouped; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info) = 0;
 friend class CBucketHandler;
 friend class CHashTableRowTable;
@@ -3454,7 +3433,6 @@ public:
     {
         ActPrintLog("stopping");
         stopInput();
-        dataLinkStop();
     }
     void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
@@ -3494,21 +3472,21 @@ public:
         CriticalBlock block(stopsect);  // can be called async by distribute
         HashDedupSlaveActivityBase::stopInput();
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         HashDedupSlaveActivityBase::init(data, slaveData);
         mptag = container.queryJobChannel().deserializeMPTag(data);
         distributor = createHashDistributor(this, queryJobChannel().queryJobComm(), mptag, true, this);
     }
-    void start()
+    virtual void start()
     {
         HashDedupSlaveActivityBase::start();
         ActivityTimer s(totalCycles, timeActivities);
         Owned<IThorRowInterfaces> myRowIf = getRowInterfaces(); // avoiding circular link issues
-        instrm.setown(distributor->connect(myRowIf, input, iHash, iCompare));
-        input = instrm.get();
+        instrm.setown(distributor->connect(myRowIf, distInput, iHash, iCompare));
+        distInput = instrm.get();
     }
-    void stop()
+    virtual void stop()
     {
         ActPrintLog("stopping");
         if (instrm)
@@ -3519,15 +3497,14 @@ public:
         distributor->disconnect(true);
         distributor->join();
         stopInput();
-        dataLinkStop();
     }
-    void abort()
+    virtual void abort()
     {
         HashDedupSlaveActivityBase::abort();
         if (distributor)
             distributor->abort();
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.canStall = true;
@@ -3538,10 +3515,14 @@ public:
 //===========================================================================
 
 
-class HashJoinSlaveActivity : public CSlaveActivity, public CThorDataLink, implements IStopInput
+class HashJoinSlaveActivity : public CSlaveActivity, implements IStopInput
 {
-    IThorDataLink *inL;
-    IThorDataLink *inR;
+    typedef CSlaveActivity PARENT;
+
+    IThorDataLink *inL = nullptr;
+    IThorDataLink *inR = nullptr;
+    IEngineRowStream *leftInputStream = nullptr;
+    IEngineRowStream *rightInputStream = nullptr;
     MemoryBuffer ptrbuf;
     IHThorHashJoinArg *joinargs;
     Owned<IJoinHelper> joinhelper;
@@ -3564,7 +3545,7 @@ public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
     HashJoinSlaveActivity(CGraphElementBase *_container)
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     {
         lhsProgressCount = rhsProgressCount = 0;
         mptag = TAG_NULL;
@@ -3576,7 +3557,7 @@ public:
         strmR.clear();
         joinhelper.clear();
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         joinargs = (IHThorHashJoinArg *)queryHelper();
         appendOutputLinked(this);
@@ -3584,19 +3565,18 @@ public:
         mptag2 = container.queryJobChannel().deserializeMPTag(data);
         ActPrintLog("HASHJOIN: init tags %d,%d",(int)mptag,(int)mptag2);
     }
-    void start()
+    virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        startAllInputs();
         inputLstopped = true;
         inputRstopped = true;
         leftdone = false;
         eof = false;
         ActPrintLog("HASHJOIN: starting");
-        inL = inputs.item(0);
-        startInput(inL);
+        inL = queryInput(0);
         inputLstopped = false;
-        inR = inputs.item(1);
-        startInput(inR);
+        inR = queryInput(1);
         inputRstopped = false;
         IHash *ihashL = joinargs->queryHashLeft();
         IHash *ihashR = joinargs->queryHashRight();
@@ -3604,7 +3584,7 @@ public:
         ICompare *icompareR = joinargs->queryCompareRight();
         if (!lhsDistributor)
             lhsDistributor.setown(createHashDistributor(this, queryJobChannel().queryJobComm(), mptag, false, this, "LHS"));
-        Owned<IRowStream> reader = lhsDistributor->connect(queryRowInterfaces(inL), inL, ihashL, icompareL);
+        Owned<IRowStream> reader = lhsDistributor->connect(queryRowInterfaces(inL), leftInputStream, ihashL, icompareL);
         Owned<IThorRowLoader> loaderL = createThorRowLoader(*this, ::queryRowInterfaces(inL), icompareL, stableSort_earlyAlloc, rc_allDisk, SPILL_PRIORITY_HASHJOIN);
         strmL.setown(loaderL->load(reader, abortSoon));
         loaderL.clear();
@@ -3615,7 +3595,7 @@ public:
         leftdone = true;
         if (!rhsDistributor)
             rhsDistributor.setown(createHashDistributor(this, queryJobChannel().queryJobComm(), mptag2, false, this, "RHS"));
-        reader.setown(rhsDistributor->connect(queryRowInterfaces(inR), inR, ihashR, icompareR));
+        reader.setown(rhsDistributor->connect(queryRowInterfaces(inR), rightInputStream, ihashR, icompareR));
         Owned<IThorRowLoader> loaderR = createThorRowLoader(*this, ::queryRowInterfaces(inR), icompareR, stableSort_earlyAlloc, rc_mixed, SPILL_PRIORITY_HASHJOIN);;
         strmR.setown(loaderR->load(reader, abortSoon));
         loaderR.clear();
@@ -3654,20 +3634,22 @@ public:
     void stopInputL()
     {
         CriticalBlock block(stopsect);  // can be called async by distribute
-        if (!inputLstopped) {
-            CSlaveActivity::stopInput(inL);
+        if (!inputLstopped)
+        {
+            PARENT::stopInput(0);
             inputLstopped = true;
         }
     }
     void stopInputR()
     {
         CriticalBlock block(stopsect);  // can be called async by distribute
-        if (!inputRstopped) {
-            CSlaveActivity::stopInput(inR);
+        if (!inputRstopped)
+        {
+            PARENT::stopInput(1);
             inputRstopped = true;
         }
     }
-    void stop()
+    virtual void stop()
     {
         ActPrintLog("HASHJOIN: stopping");
         stopInputL();
@@ -3710,7 +3692,7 @@ public:
         }
         return NULL;
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -3859,10 +3841,11 @@ RowAggregator *mergeLocalAggs(Owned<IHashDistributor> &distributor, CActivityBas
 #pragma warning(push)
 #pragma warning( disable : 4355 ) // 'this' : used in base member initializer list
 #endif
-class CHashAggregateSlave : public CSlaveActivity, public CThorDataLink, implements IHThorRowAggregator
+class CHashAggregateSlave : public CSlaveActivity, implements IHThorRowAggregator
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorHashAggregateArg *helper;
-    IThorDataLink *input;
     mptag_t mptag;
     Owned<RowAggregator> localAggTable;
     bool eos;
@@ -3875,12 +3858,12 @@ class CHashAggregateSlave : public CSlaveActivity, public CThorDataLink, impleme
             localAggTable->start(queryRowAllocator());
             while (!abortSoon)
             {
-                OwnedConstThorRow row = input->nextRow();
+                OwnedConstThorRow row = inputStream->nextRow();
                 if (!row)
                 {
                     if (container.queryGrouped())
                         break;
-                    row.setown(input->nextRow());
+                    row.setown(inputStream->nextRow());
                     if (!row)
                         break;
                 }
@@ -3901,12 +3884,12 @@ public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
     CHashAggregateSlave(CGraphElementBase *_container)
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     {
         mptag = TAG_NULL;
         eos = true;
     }
-    void init(MemoryBuffer & data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
     {
         helper = static_cast <IHThorHashAggregateArg *> (queryHelper());
         appendOutputLinked(this);
@@ -3918,11 +3901,10 @@ public:
         }
         localAggTable.setown(new RowAggregator(*helper, *helper));
     }
-    void start()
+    virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        startInput(input);
+        PARENT::start();
         doNextGroup(); // or local set if !grouped
         if (!container.queryGrouped())
             ActPrintLog("Table before distribution contains %d entries", localAggTable->elementCount());
@@ -3933,16 +3915,14 @@ public:
             ActPrintLog("Table after distribution contains %d entries", localAggTable->elementCount());
         }
         eos = false;
-        dataLinkStart();
     }
-    void stop()
+    virtual void stop()
     {
         ActPrintLog("HASHAGGREGATE: stopping");
         localAggTable->reset();
-        stopInput(input);
-        dataLinkStop();
+        PARENT::stop();
     }
-    void abort()
+    virtual void abort()
     {
         CSlaveActivity::abort();
         if (distributor)
@@ -3968,8 +3948,8 @@ public:
             eos = true;
         return NULL;
     }
-    bool isGrouped() { return false; }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.canStall = true;
@@ -3986,44 +3966,36 @@ public:
 #endif
 
 
-class CHashDistributeSlavedActivity : public CSlaveActivity, public CThorDataLink
+class CHashDistributeSlavedActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHash *ihash;
-    IThorDataLink *input;
     unsigned myNode, nodes;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CHashDistributeSlavedActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CHashDistributeSlavedActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         IHThorHashDistributeArg *distribargs = (IHThorHashDistributeArg *)queryHelper();
         ihash = distribargs->queryHash();
-        input = NULL;
         myNode = queryJobChannel().queryMyRank()-1;
         nodes = container.queryJob().querySlaves();
     }
-    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
     }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        input->start();
-        dataLinkStart();
-    }
-    virtual void stop()
-    {
-        if (input)
-            input->stop();
-        dataLinkStop();
+        PARENT::start();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
-        OwnedConstThorRow row = input->ungroupedNextRow();
+        OwnedConstThorRow row = inputStream->ungroupedNextRow();
         if (!row)
             return NULL;
         if (myNode != (ihash->hash(row.get()) % nodes))
@@ -4041,8 +4013,8 @@ public:
         dataLinkIncrement();
         return row.getClear();
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         if (input)

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -286,8 +286,10 @@ interface IRowStreamStepping : extends IRowStream
     virtual const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra) = 0;
 };
 
-class CIndexReadSlaveActivity : public CIndexReadSlaveBase, public CThorDataLink
+class CIndexReadSlaveActivity : public CIndexReadSlaveBase
 {
+    typedef CIndexReadSlaveBase PARENT;
+
     IHThorIndexReadArg *helper;
     rowcount_t rowLimit, stopAfter;
     bool keyedLimitSkips, first, eoi, needTransform, optimizeSteppedPostFilter, steppingEnabled;
@@ -549,9 +551,7 @@ class CIndexReadSlaveActivity : public CIndexReadSlaveBase, public CThorDataLink
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CIndexReadSlaveActivity(CGraphElementBase *_container) : CIndexReadSlaveBase(_container), CThorDataLink(this)
+    CIndexReadSlaveActivity(CGraphElementBase *_container) : CIndexReadSlaveBase(_container)
     {
         keyedLimitSkips = false;
         first = true;
@@ -626,6 +626,7 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         first = true;
         eoi = false;
         keyedLimit = helperKeyedLimit;
@@ -646,7 +647,6 @@ public:
             keyedLimitCount = 0;            
         else
             eoi = true; // otherwise delayed until calc. in nextRow()
-        dataLinkStart();
     }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
@@ -654,7 +654,7 @@ public:
         info.isSource = true;
         // MORE TBD
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
 
 // IRowStream
     virtual void stop()
@@ -671,7 +671,7 @@ public:
             out->stop();
             out.clear();
         }
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -759,8 +759,10 @@ CActivityBase *createIndexReadSlave(CGraphElementBase *container)
 
 /////////////////////////////////////////////////////////////
 
-class CIndexGroupAggregateSlaveActivity : public CIndexReadSlaveBase, public CThorDataLink, implements IHThorGroupAggregateCallback
+class CIndexGroupAggregateSlaveActivity : public CIndexReadSlaveBase, implements IHThorGroupAggregateCallback
 {
+    typedef CIndexReadSlaveBase PARENT;
+
     IHThorIndexGroupAggregateArg *helper;
     bool gathered, eoi, merging;
     Owned<RowAggregator> localAggTable;
@@ -770,7 +772,7 @@ class CIndexGroupAggregateSlaveActivity : public CIndexReadSlaveBase, public CTh
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CIndexGroupAggregateSlaveActivity(CGraphElementBase *_container) : CIndexReadSlaveBase(_container), CThorDataLink(this)
+    CIndexGroupAggregateSlaveActivity(CGraphElementBase *_container) : CIndexReadSlaveBase(_container)
     {
         helper = (IHThorIndexGroupAggregateArg *)container.queryHelper();
         merging = false;
@@ -781,32 +783,28 @@ public:
         localAggTable->addRow(next);
     }
 // IThorSlaveActivity
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         CIndexReadSlaveBase::init(data, slaveData);
         appendOutputLinked(this);
     }
 // IThorDataLink
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.isSource = true;
         // MORE TBD
     }
-    virtual bool isGrouped() { return false; }
-    virtual void start()
+    virtual bool isGrouped() const override { return false; }
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         localAggTable.setown(new RowAggregator(*helper, *helper));
         localAggTable->start(queryRowAllocator());
         gathered = eoi = false;
-        dataLinkStart();
     }
 // IRowStream
-    virtual void stop()
-    {
-        dataLinkStop();
-    }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
@@ -870,8 +868,10 @@ CActivityBase *createIndexGroupAggregateSlave(CGraphElementBase *container) { re
 /////////////////////////////////////////////////////////////
 
 
-class CIndexCountSlaveActivity : public CIndexReadSlaveBase, public CThorDataLink
+class CIndexCountSlaveActivity : public CIndexReadSlaveBase
 {
+    typedef CIndexReadSlaveBase PARENT;
+
     bool eoi;
     IHThorIndexCountArg *helper;
     rowcount_t choosenLimit;
@@ -879,9 +879,7 @@ class CIndexCountSlaveActivity : public CIndexReadSlaveBase, public CThorDataLin
     bool totalCountKnown;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CIndexCountSlaveActivity(CGraphElementBase *_container) : CIndexReadSlaveBase(_container), CThorDataLink(this)
+    CIndexCountSlaveActivity(CGraphElementBase *_container) : CIndexReadSlaveBase(_container)
     {
         helper = static_cast <IHThorIndexCountArg *> (container.queryHelper());
         preknownTotalCount = 0;
@@ -890,7 +888,7 @@ public:
     }
 
 // IThorSlaveActivity
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         CIndexReadSlaveBase::init(data, slaveData);
         choosenLimit = (rowcount_t)helper->getChooseNLimit();
@@ -898,30 +896,26 @@ public:
     }
 
 // IThorDataLink
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.isSource = true;
         // MORE TBD
     }
-    virtual bool isGrouped() { return false; }
-    virtual void start()
+    virtual bool isGrouped() const override { return false; }
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         eoi = false;
         if (!helper->canMatchAny())
         {
             totalCountKnown = true;
             preknownTotalCount = 0;
         }
-        dataLinkStart();
     }
 
 // IRowStream
-    virtual void stop()
-    {
-        dataLinkStop();
-    }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
@@ -1008,8 +1002,10 @@ CActivityBase *createIndexCountSlave(CGraphElementBase *container)
 }
 
 
-class CIndexNormalizeSlaveActivity : public CIndexReadSlaveBase, public CThorDataLink
+class CIndexNormalizeSlaveActivity : public CIndexReadSlaveBase
 {
+    typedef CIndexReadSlaveBase PARENT;
+
     bool eoi, expanding;
     IHThorIndexNormalizeArg *helper;
     rowcount_t keyedLimit, rowLimit, stopAfter, keyedProcessed, keyedLimitCount;
@@ -1032,9 +1028,7 @@ class CIndexNormalizeSlaveActivity : public CIndexReadSlaveBase, public CThorDat
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CIndexNormalizeSlaveActivity(CGraphElementBase *_container) : CIndexReadSlaveBase(_container), CThorDataLink(this), partHelper(*this)
+    CIndexNormalizeSlaveActivity(CGraphElementBase *_container) : CIndexReadSlaveBase(_container), partHelper(*this)
     {
         helper = (IHThorIndexNormalizeArg *)container.queryHelper();
     }
@@ -1067,10 +1061,11 @@ public:
         info.isSource = true;
         // MORE TBD
     }
-    virtual bool isGrouped() { return false; }
-    virtual void start()
+    virtual bool isGrouped() const override { return false; }
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         keyedLimit = (rowcount_t)helper->getKeyedLimit();
         rowLimit = (rowcount_t)helper->getRowLimit();
         if (helper->getFlags() & TIRlimitskips)
@@ -1097,11 +1092,10 @@ public:
         }
         else
             eoi = true;
-        dataLinkStart();
     }
 
 // IRowStream
-    virtual void stop()
+    virtual void stop() override
     {
         if (RCMAX != keyedLimit)
         {
@@ -1109,7 +1103,7 @@ public:
             if (keyedLimitCount > keyedLimit)
                 helper->onKeyedLimitExceeded(); // should throw exception
         }
-        dataLinkStop();
+        PARENT::stop();
     }
 
     CATCH_NEXTROW()
@@ -1201,8 +1195,10 @@ public:
 
 CActivityBase *createIndexNormalizeSlave(CGraphElementBase *container) { return new CIndexNormalizeSlaveActivity(container); }
 
-class CIndexAggregateSlaveActivity : public CIndexReadSlaveBase, public CThorDataLink
+class CIndexAggregateSlaveActivity : public CIndexReadSlaveBase
 {
+    typedef CIndexReadSlaveBase PARENT;
+
     bool eoi, hadElement;
     IHThorIndexAggregateArg *helper;
     CIndexPartHandlerHelper partHelper;
@@ -1229,23 +1225,21 @@ class CIndexAggregateSlaveActivity : public CIndexReadSlaveBase, public CThorDat
         }
     }
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CIndexAggregateSlaveActivity(CGraphElementBase *_container) 
-        : CIndexReadSlaveBase(_container), CThorDataLink(this), partHelper(*this), aggregator(*this)
+        : CIndexReadSlaveBase(_container), partHelper(*this), aggregator(*this)
     {
         helper = (IHThorIndexAggregateArg *)container.queryHelper();
     }
 
 // IThorSlaveActivity
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         CIndexReadSlaveBase::init(data, slaveData);
         appendOutputLinked(this);
     }
 
 // IThorDataLink
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.isSource = true;
@@ -1253,21 +1247,16 @@ public:
         info.totalRowsMax = 1;
         // MORE TBD
     }
-    virtual bool isGrouped() { return false; }
-    virtual void start()
+    virtual bool isGrouped() const override { return false; }
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         eoi = hadElement = false;
         partn = 0;
-        dataLinkStart();
     }
 
 // IRowStream
-    virtual void stop()
-    {
-        dataLinkStop();
-    }
-
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -33,39 +33,14 @@
 #define FEWWARNCAP 10
 
 
-class CITDL : public CSimpleInterface, public CThorDataLink
-{
-    IThorDataLink *base;
-    Owned<IRowStream> in;
-public:
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
-
-    CITDL(IThorDataLink *_base, IRowStream *_in) : CThorDataLink(_base->queryFromActivity()), in(_in), base(_base)
-    {
-    }
-    virtual const void *nextRow() { return in->nextRow(); }
-    virtual void stop() { in->stop(); }
-// IThorDataLink impl.
-    virtual void start() { }
-    virtual bool isGrouped() { return false; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
-    {
-        // JCSMORE - TBD
-        base->getMetaInfo(info);
-    }
-};
-IThorDataLink *createRowStreamToDataLinkAdapter(IThorDataLink *base, IRowStream *in)
-{
-    return new CITDL(base, in);
-}
-
-class IndexWriteSlaveActivity  : public ProcessSlaveActivity, public ISmartBufferNotify, implements ICopyFileProgress, implements IBlobCreator
+class IndexWriteSlaveActivity : public ProcessSlaveActivity, public ILookAheadStopNotify, implements ICopyFileProgress, implements IBlobCreator
 {
+    typedef ProcessSlaveActivity PARENT;
     StringAttr logicalFilename;
     Owned<IPartDescriptor> partDesc, tlkDesc;
     IHThorIndexWriteArg *helper;
     Owned <IKeyBuilder> builder;
-    Owned<IThorDataLink> input;
+    Owned<IRowStream> myInputStream;
     Owned<IPropertyTree> metadata;
     Linked<IEngineRowAllocator> outRowAllocator;
 
@@ -90,7 +65,7 @@ class IndexWriteSlaveActivity  : public ProcessSlaveActivity, public ISmartBuffe
         if (!inputStopped)
         {
             inputStopped = true;
-            stopInput(input);
+            stop();
         }
     }
     void init()
@@ -120,8 +95,7 @@ public:
         enableTlkPart0 = (0 != container.queryJob().getWorkUnitValueInt("enableTlkPart0", globals->getPropBool("@enableTlkPart0", true)));
         reInit = (0 != (TIWvarfilename & helper->getFlags()));
     }
-
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         isLocal = 0 != (TIWlocal & helper->getFlags());
 
@@ -182,7 +156,11 @@ public:
             maxDiskRecordSize = diskSize->getFixedSize();
         reportOverflow = false;
     }
-
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
+    {
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), INDEXWRITE_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage()));
+    }
     void open(IPartDescriptor &partDesc, bool isTopLevel, bool isVariable)
     {
         StringBuffer partFname;
@@ -299,20 +277,19 @@ public:
     {
         return builder->createBlob(size, (const char *) ptr);
     }
-    void process()
+    virtual void process() override
     {
         ActPrintLog("INDEXWRITE: Start");
         init();
 
+        IRowStream *stream = inputStream;
         ThorDataLinkMetaInfo info;
-        inputs.item(0)->getMetaInfo(info);
+        input->getMetaInfo(info);
         outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize()));
+        start();
         if (refactor)
         {
             assertex(isLocal);
-            input.setown(createDataLinkSmartBuffer(this, inputs.item(0), INDEXWRITE_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, false, &container.queryJob().queryIDiskUsage())); 
-            startInput(input);
-
             if (active)
             {
                 unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0);
@@ -321,7 +298,7 @@ public:
                 unsigned myPart = queryJobChannel().queryMyRank();
 
                 IArrayOf<IRowStream> streams;
-                streams.append(*LINK(input));
+                streams.append(*LINK(stream));
                 --partsPerNode;
 
  // Should this be merging 1,11,21,31 etc.
@@ -334,19 +311,11 @@ public:
                 ICompare *icompare = helper->queryCompare();
                 assertex(icompare);
                 Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter;
-                input.setown(createRowStreamToDataLinkAdapter(inputs.item(0), createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter)));
+                myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter));
+                stream = myInputStream;
             }
             else // serve nodes, creating merged parts
-                rowServer.setown(createRowServer(this, input, queryJobChannel().queryJobComm(), mpTag));
-        }
-        else if (singlePartKey)
-        {
-            input.setown(createDataLinkSmartBuffer(this, inputs.item(0), INDEXWRITE_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, false, &container.queryJob().queryIDiskUsage())); 
-            startInput(input);
-        }
-        else {
-            input.set(inputs.item(0));
-            startInput(input);
+                rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag));
         }
         processed = THORDATALINK_STARTED;
 
@@ -363,7 +332,7 @@ public:
                     open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
                     loop
                     {
-                        OwnedConstThorRow row = input->ungroupedNextRow();
+                        OwnedConstThorRow row = inputStream->ungroupedNextRow();
                         if (!row)
                             break;
                         if (abortSoon) return;
@@ -425,7 +394,7 @@ public:
                         mb.clear();
                         do
                         {
-                            OwnedConstThorRow row = input->ungroupedNextRow();
+                            OwnedConstThorRow row = inputStream->ungroupedNextRow();
                             if (!row) break;
                             serializer->serialize(mbs, (const byte *)row.get());
                         } while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row
@@ -454,7 +423,7 @@ public:
                         receiving = false;
                     do
                     {
-                        OwnedConstThorRow row = input->ungroupedNextRow();
+                        OwnedConstThorRow row = inputStream->ungroupedNextRow();
                         if (!row)
                             break;
                         processRow(row);
@@ -566,30 +535,27 @@ public:
             ActPrintLog("INDEXWRITE: All done");
         }
     }
-
-    void endProcess()
+    virtual void endProcess() override
     {
         if (processed & THORDATALINK_STARTED)
         {
             doStopInput();
             processed |= THORDATALINK_STOPPED;
         }
-        input.clear();
+        inputStream = NULL;
     }
-
-    virtual void abort()
+    virtual void abort() override
     {
-        ProcessSlaveActivity::abort();
+        PARENT::abort();
         cancelReceiveMsg(RANK_ALL, mpTag);
         if (receivingTag2)
             queryJobChannel().queryJobComm().cancel(RANK_ALL, mpTag2);
         if (rowServer)
             rowServer->stop();
     }
-
-    void kill()
+    virtual void kill() override
     {
-        ProcessSlaveActivity::kill();
+        PARENT::kill();
         if (abortSoon)
         {
             if (partDesc)
@@ -598,8 +564,7 @@ public:
                 removeFiles(*tlkDesc);
         }
     }
-
-    void processDone(MemoryBuffer &mb)
+    virtual void processDone(MemoryBuffer &mb) override
     {
         builder.clear();
         if (refactor && !active)
@@ -665,7 +630,6 @@ public:
             fireException(e);
         }
     }
-
     virtual void onInputFinished(rowcount_t finalcount)
     {
         if (!sizeSignalled)
@@ -674,20 +638,9 @@ public:
             ActPrintLog("finished input %" RCPF "d", finalcount);
         }
     }
-
-    virtual bool startAsync()
-    {
-        return false;
-    }
-
-    virtual void onInputStarted(IException *)
-    {
-        // not needed
-    }
-
     virtual void serializeStats(MemoryBuffer &mb)
     {
-        ProcessSlaveActivity::serializeStats(mb);
+        PARENT::serializeStats(mb);
         mb.append(replicateDone);
     }
 

thorlcr/activities/iterate/thgroupiterateslave.cpp

@@ -21,49 +21,36 @@
 #include "thgroupiterateslave.ipp"
 #include "thactivityutil.ipp"
 
-class GroupIterateSlaveActivity : public CSlaveActivity, public CThorDataLink
+class GroupIterateSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
 
-private:
     OwnedConstThorRow prev;
     OwnedConstThorRow defaultLeft;
     IHThorGroupIterateArg * helper;
     rowcount_t count;
     bool eogNext;
     bool anyThisGroup;
-    IThorDataLink *input;
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    GroupIterateSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    GroupIterateSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);   // adding 'me' to outputs array
         helper = static_cast <IHThorGroupIterateArg *> (queryHelper());
     }
-    ~GroupIterateSlaveActivity()
-    {
-    }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         anyThisGroup = false;
         eogNext = false;    
         count = 0;
-        input = inputs.item(0);
-        startInput(input);
-        dataLinkStart();
         RtlDynamicRowBuilder r(queryRowAllocator());
         size32_t sz = helper->createDefault(r);
         defaultLeft.setown(r.finalizeRowClear(sz));
     }
-    void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
-    }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
@@ -79,14 +66,14 @@ public:
                 }
             }
             
-            OwnedConstThorRow row = input->nextRow();
+            OwnedConstThorRow row = inputStream->nextRow();
             if (!row)   {
                 count = 0;
                 if (anyThisGroup) {
                     anyThisGroup = false;
                     break;
                 }
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if (!row)
                     break;
             }
@@ -102,62 +89,54 @@ public:
         }
         return NULL;
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         if (helper->canFilter())
             info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
-    bool isGrouped() 
+    virtual bool isGrouped() const override
     { 
         return true; 
     }
 };
 
 
-class GroupProcessSlaveActivity : public CSlaveActivity, public CThorDataLink
+class GroupProcessSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorProcessArg * helper;
     rowcount_t count;
     bool eogNext;
     bool anyThisGroup;
     OwnedConstThorRow firstright;
     OwnedConstThorRow nextright;
-    IThorDataLink *input;
     Owned<IThorRowInterfaces> rightrowif;
     Owned<IEngineRowAllocator> rightAllocator;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    GroupProcessSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    GroupProcessSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         appendOutputLinked(this);   // adding 'me' to outputs array
         helper = static_cast <IHThorProcessArg *> (queryHelper());
         rightrowif.setown(createThorRowInterfaces(queryRowManager(), helper->queryRightRecordSize(),queryId(),queryCodeContext()));
         rightAllocator.set(rightrowif->queryRowAllocator());
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         RtlDynamicRowBuilder r(rightAllocator);
         size32_t sz = helper->createInitialRight(r);  
         firstright.setown(r.finalizeRowClear(sz));
         anyThisGroup = false;
         count = 0;
         eogNext = false;    
-        input = inputs.item(0);
-        startInput(input);
-        dataLinkStart();
-    }
-    void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -173,14 +152,14 @@ public:
                     break;
                 }
             }
-            OwnedConstThorRow row = input->nextRow();
+            OwnedConstThorRow row = inputStream->nextRow();
             if (!row) {
                 count = 0;
                 if (anyThisGroup) {
                     anyThisGroup = false;
                     break;
                 }
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if (!row)
                     break;
             }
@@ -197,14 +176,14 @@ public:
         }
         return NULL;
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         if (helper->canFilter())
             info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
-    bool isGrouped() 
+    virtual bool isGrouped() const override
     { 
         return true; 
     }

thorlcr/activities/iterate/thiterateslave.cpp

@@ -22,19 +22,18 @@
 
 #include "thiterateslave.ipp"
 
-class IterateSlaveActivityBase : public CSlaveActivity, public CThorDataLink
+class IterateSlaveActivityBase : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     OwnedConstThorRow first;
 protected:
-    Owned<IThorDataLink> input;
     Owned<IThorRowInterfaces> inrowif;
     bool global;
     bool eof, nextPut;
     rowcount_t count;
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    IterateSlaveActivityBase(CGraphElementBase *_container, bool _global) : CSlaveActivity(_container), CThorDataLink(this)
+    IterateSlaveActivityBase(CGraphElementBase *_container, bool _global) : CSlaveActivity(_container)
     {
         global = _global;
     }
@@ -44,6 +43,12 @@ public:
         if (global)
             mpTag = container.queryJobChannel().deserializeMPTag(data);
     }
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
+    {
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        if (global) // only want lookahead if global (hence serial)
+            setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), ENTH_SMART_BUFFER_SIZE, true, false, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
+    }
     const void *getFirst() // for global, not called on 1st slave
     {
         CMessageBuffer msg;
@@ -73,36 +78,21 @@ public:
                 return;
         }
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         count = 0;
         eof = nextPut = false;
-        inrowif.set(::queryRowInterfaces(inputs.item(0)));
-        if (global) // only want lookahead if global (hence serial)
-            input.setown(createDataLinkSmartBuffer(this, inputs.item(0),ITERATE_SMART_BUFFER_SIZE,isSmartBufferSpillNeeded(this),false,RCUNBOUND,NULL,false,&container.queryJob().queryIDiskUsage())); // only allow spill if input can stall
-        else
-            input.set(inputs.item(0));
-        try
-        { 
-            startInput(input); 
-        }
-        catch (IException *e)
-        {
-            ActPrintLog(e,"ITERATE");
-            throw;
-        }
-        dataLinkStart();
+        inrowif.set(::queryRowInterfaces(queryInput(0)));
     }
-    void stop()
+    virtual void stop() override
     {
         if (global)
             putNext(NULL);
-        stopInput(input);
-        input.clear();
-        dataLinkStop();
+        PARENT::stop();
     }
-    bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
 };
 
 class IterateSlaveActivity : public IterateSlaveActivityBase
@@ -116,12 +106,12 @@ public:
         : IterateSlaveActivityBase(_container,_global)
     {
     }
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         helper = static_cast <IHThorIterateArg *> (queryHelper());
         IterateSlaveActivityBase::init(data,slaveData);
     }
-    virtual void start()
+    virtual void start() override
     {
         IterateSlaveActivityBase::start();
         prev.clear();
@@ -149,7 +139,7 @@ public:
                     }
                 }
             }
-            OwnedConstThorRow next = input->ungroupedNextRow();
+            OwnedConstThorRow next = inputStream->ungroupedNextRow();
             if (!next) {
                 putNext(prev); // send to next node if applicable
                 eof = true;
@@ -172,7 +162,7 @@ public:
         info.canBufferInput = true;
         if (helper->canFilter())
             info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 
@@ -234,7 +224,7 @@ public:
                     }
                 }
             }
-            left.setown(input->ungroupedNextRow());
+            left.setown(inputStream->ungroupedNextRow());
             if (!left) {
                 putNext(right); // send to next node 
                 eof = true;
@@ -262,7 +252,7 @@ public:
         info.canBufferInput = true;
         if (helper->canFilter())
             info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 
@@ -279,34 +269,30 @@ CActivityBase *createLocalProcessSlave(CGraphElementBase *container)
 }
 
 
-class CChildIteratorSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CChildIteratorSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     // only for toplevel activity 
     IHThorChildIteratorArg * helper;
     bool eof;
     rowcount_t count;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CChildIteratorSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CChildIteratorSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);   // adding 'me' to outputs array
         helper = static_cast <IHThorChildIteratorArg *> (queryHelper());
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         eof = !container.queryLocalOrGrouped() && !firstNode();
         count = 0;
-        dataLinkStart();
-    }
-    void stop()
-    {
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -329,57 +315,55 @@ public:
         return NULL;
     }
 
-    bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
     }
 };
 
-class CLinkedRawIteratorSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CLinkedRawIteratorSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     // only for toplevel activity 
     IHThorLinkedRawIteratorArg * helper;
     bool dohere, grouped;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CLinkedRawIteratorSlaveActivity(CGraphElementBase *_container) 
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         appendOutputLinked(this);   // adding 'me' to outputs array
         helper = static_cast <IHThorLinkedRawIteratorArg *> (queryHelper());
         grouped = helper->queryOutputMeta()->isGrouped();
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dataLinkStart();
+        PARENT::start();
         dohere = container.queryLocalOrGrouped() || firstNode();
     }
-    void stop()
-    {
-        dataLinkStop();
-    }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
-        if (dohere) {
+        if (dohere)
+        {
             OwnedConstThorRow row;
             row.set(helper->next()); // needs linking allegedly
-            if (row.get()) {
+            if (row.get())
+            {
                 dataLinkIncrement();
                 return row.getClear();
             }
         }
         return NULL;
     }
-    bool isGrouped() { return grouped; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return grouped; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
     }
@@ -397,17 +381,17 @@ CActivityBase *createChildIteratorSlave(CGraphElementBase *container)
 }
 
 
-class CStreamedIteratorSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CStreamedIteratorSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorStreamedIteratorArg *helper;
     Owned<IRowStream> rows;
     bool eof;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CStreamedIteratorSlaveActivity(CGraphElementBase *_container) 
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     {
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
@@ -415,22 +399,22 @@ public:
         appendOutputLinked(this);   // adding 'me' to outputs array
         helper = static_cast <IHThorStreamedIteratorArg *> (queryHelper());
     }
-    virtual void start()
+    virtual void start() override
     {
+        PARENT::start();
         bool isLocal = container.queryLocalData() || container.queryOwner().isLocalChild();
         eof = isLocal ? false : !firstNode();
         if (!eof)
             rows.setown(helper->createInput());
-        dataLinkStart();
     }
-    void stop()
+    virtual void stop() override
     {
         if (rows)
         {
             rows->stop();
             rows.clear();
         }
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -443,8 +427,8 @@ public:
             dataLinkIncrement();
         return next;
     }
-    bool isGrouped() { return false; }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.isSource = true;

thorlcr/activities/join/thjoinslave.cpp

@@ -38,8 +38,15 @@
 #define BUFFERSIZE 0x10000
 #define NUMSLAVEPORTS 2     // actually should be num MP tags
 
-class JoinSlaveActivity : public CSlaveActivity, public CThorDataLink, implements ISmartBufferNotify
+class JoinSlaveActivity : public CSlaveActivity, implements ILookAheadStopNotify
 {
+    typedef CSlaveActivity PARENT;
+
+    unsigned secondaryInputIndex = 0;
+    unsigned primaryInputIndex = 0;
+    IEngineRowStream *rightInputStream = nullptr;
+    IEngineRowStream *primaryInputStream = nullptr;
+    IEngineRowStream *secondaryInputStream = nullptr;
     Owned<IThorDataLink> leftInput, rightInput;
     Owned<IThorDataLink> secondaryInput, primaryInput;
     IHThorJoinBaseArg *helper;
@@ -56,7 +63,6 @@ class JoinSlaveActivity : public CSlaveActivity, public CThorDataLink, implement
     ICompare *primarySecondaryUpperCompare; // if non-null then between join
 
     Owned<IRowStream> leftStream, rightStream;
-    Semaphore secondaryStartSem;
     Owned<IException> secondaryStartException;
 
     bool islocal;
@@ -137,7 +143,7 @@ public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
     JoinSlaveActivity(CGraphElementBase *_container, bool local)
-        : CSlaveActivity(_container), CThorDataLink(this), spillStats(spillStatistics)
+        : CSlaveActivity(_container), spillStats(spillStatistics)
     {
         islocal = local;
         portbase = 0;
@@ -195,13 +201,28 @@ public:
         rightCompare = helper->queryCompareRight();
         leftKeySerializer = helper->querySerializeLeft();
         rightKeySerializer = helper->querySerializeRight();
+        rightpartition = (container.getKind()==TAKjoin)&&((helper->getJoinFlags()&JFpartitionright)!=0);
     }
-    virtual void onInputStarted(IException *except)
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
     {
-        secondaryStartException.set(except);
-        secondaryStartSem.signal();
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        if ((rightpartition && (0 == index)) || (!rightpartition && (1 == index)))
+        {
+            secondaryInputIndex = index;
+            IEngineRowStream *secondaryStream = queryInputStream(secondaryInputIndex);
+            IStartableEngineRowStream *lookAhead = createRowStreamLookAhead(this, secondaryStream, queryRowInterfaces(_input.itdl), JOIN_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(_input.itdl->queryFromActivity()),
+                                                        false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage());
+            setLookAhead(secondaryInputIndex, lookAhead),
+            secondaryInputStream = lookAhead;
+        }
+        else
+        {
+            primaryInputIndex = index;
+            primaryInputStream = queryInputStream(primaryInputIndex);
+        }
+        if (1 == index)
+            rightInputStream = queryInputStream(1);
     }
-    virtual bool startAsync() { return true; }
     virtual void onInputFinished(rowcount_t count)
     {
         ActPrintLog("JOIN: %s input finished, %" RCPF "d rows read", rightpartition?"LHS":"RHS", count);
@@ -229,10 +250,21 @@ public:
         }
     }
 
-    void start()
+    void startSecondaryInput()
+    {
+        try
+        {
+            startInput(secondaryInputIndex);
+        }
+        catch (IException *e)
+        {
+            secondaryStartException.setown(e);
+        }
+
+    }
+    virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        rightpartition = (container.getKind()==TAKjoin)&&((helper->getJoinFlags()&JFpartitionright)!=0);
 
         Linked<IThorRowInterfaces> primaryRowIf, secondaryRowIf;
 
@@ -240,8 +272,8 @@ public:
         bool *secondaryInputStopped, *primaryInputStopped;
         if (rightpartition)
         {
-            primaryInput.set(inputs.item(1));
-            secondaryInput.set(inputs.item(0));
+            primaryInput.set(queryInput(1));
+            secondaryInput.set(queryInput(0));
             secondaryInputStopped = &leftInputStopped;
             primaryInputStopped = &rightInputStopped;
             primaryInputStr.set("R");
@@ -249,34 +281,32 @@ public:
         }
         else
         {
-            primaryInput.set(inputs.item(0));
-            secondaryInput.set(inputs.item(1));
+            primaryInput.set(queryInput(0));
+            secondaryInput.set(queryInput(1));
             secondaryInputStopped = &rightInputStopped;
             primaryInputStopped = &leftInputStopped;
             primaryInputStr.set("L");
             secondaryInputStr.set("R");
         }
         ActPrintLog("JOIN partition: %s", primaryInputStr.get());
-
-        secondaryInput.setown(createDataLinkSmartBuffer(this, secondaryInput, JOIN_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(secondaryInput->queryFromActivity()),
-                                                    false, RCUNBOUND, this, false, &container.queryJob().queryIDiskUsage()));
         ActPrintLog("JOIN: Starting %s then %s", secondaryInputStr.get(), primaryInputStr.get());
-        startInput(secondaryInput);
+
         *secondaryInputStopped = false;
+        CAsyncCallStart asyncSecondaryStart(std::bind(&JoinSlaveActivity::startSecondaryInput, this));
         try
         {
-            startInput(primaryInput);
+            startInput(primaryInputIndex);
             *primaryInputStopped = false;
         }
         catch (IException *e)
         {
             fireException(e);
             barrier->cancel();
-            secondaryStartSem.wait();
+            asyncSecondaryStart.wait();
             stopOtherInput();
             throw;
         }
-        secondaryStartSem.wait();
+        asyncSecondaryStart.wait();
         if (secondaryStartException)
         {
             IException *e=secondaryStartException.getClear();
@@ -309,19 +339,21 @@ public:
         }
         if (!leftStream.get()||!rightStream.get())
             throw MakeActivityException(this, TE_FailedToStartJoinStreams, "Failed to start join streams");
-        joinhelper->init(leftStream, rightStream, ::queryRowAllocator(inputs.item(0)),::queryRowAllocator(inputs.item(1)),::queryRowMetaData(inputs.item(0)));
+        joinhelper->init(leftStream, rightStream, ::queryRowAllocator(queryInput(0)),::queryRowAllocator(queryInput(1)),::queryRowMetaData(queryInput(0)));
     }
     void stopLeftInput()
     {
-        if (!leftInputStopped) {
-            stopInput(leftInput, "(L)");
+        if (!leftInputStopped)
+        {
+            stopInput(0, "(L)");
             leftInputStopped = true;
         }
     }
     void stopRightInput()
     {
-        if (!rightInputStopped) {
-            stopInput(rightInput, "(R)");
+        if (!rightInputStopped)
+        {
+            stopInput(1, "(R)");
             rightInputStopped = true;
         }
     }
@@ -339,13 +371,13 @@ public:
         else
             stopRightInput();
     }
-    void abort()
+    virtual void abort()
     {
         CSlaveActivity::abort();
         if (joinhelper)
             joinhelper->stop();
     }
-    void stop() 
+    virtual void stop()
     {
         stopLeftInput();
         stopRightInput();
@@ -369,7 +401,7 @@ public:
         leftInput.clear();
         rightInput.clear();
     }
-    void reset()
+    virtual void reset()
     {
         if (sorter) return; // JCSMORE loop - shouldn't have to recreate sorter between loop iterations
         if (!islocal && TAG_NULL != mpTagRPC)
@@ -396,7 +428,7 @@ public:
         }
         return NULL;
     }
-    bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -406,6 +438,7 @@ public:
     void dolocaljoin()
     {
         bool isemptylhs = false;
+        IRowStream *leftInputStream = inputStream;
         if (helper->isLeftAlreadyLocallySorted())
         {
             ThorDataLinkMetaInfo info;
@@ -413,14 +446,14 @@ public:
             if (info.totalRowsMax==0) 
                 isemptylhs = true;
             if (rightpartition)
-                leftStream.set(leftInput.get()); // already ungrouped
+                leftStream.set(leftInputStream); // already ungrouped
             else
-                leftStream.setown(createUngroupStream(leftInput));
+                leftStream.setown(createUngroupStream(leftInputStream));
         }
         else
         {
             Owned<IThorRowLoader> iLoaderL = createThorRowLoader(*this, ::queryRowInterfaces(leftInput), leftCompare, stableSort_earlyAlloc, rc_mixed, SPILL_PRIORITY_JOIN);
-            leftStream.setown(iLoaderL->load(leftInput, abortSoon));
+            leftStream.setown(iLoaderL->load(leftInputStream, abortSoon));
             isemptylhs = 0 == iLoaderL->numRows();
             stopLeftInput();
             mergeStats(spillStats, iLoaderL);
@@ -434,14 +467,14 @@ public:
         else if (helper->isRightAlreadyLocallySorted())
         {
             if (rightpartition)
-                rightStream.set(createUngroupStream(rightInput));
+                rightStream.set(createUngroupStream(rightInputStream));
             else
-                rightStream.set(rightInput.get()); // already ungrouped
+                rightStream.set(rightInputStream); // already ungrouped
         }
         else
         {
             Owned<IThorRowLoader> iLoaderR = createThorRowLoader(*this, ::queryRowInterfaces(rightInput), rightCompare, stableSort_earlyAlloc, rc_mixed, SPILL_PRIORITY_JOIN);
-            rightStream.setown(iLoaderR->load(rightInput, abortSoon));
+            rightStream.setown(iLoaderR->load(rightInputStream, abortSoon));
             stopRightInput();
             mergeStats(spillStats, iLoaderR);
         }
@@ -513,12 +546,12 @@ public:
 
         if (noSortPartitionSide())
         {
-            partitionRow.setown(primaryInput->ungroupedNextRow());
-            primaryStream.set(new cRowStreamPlus1Adaptor(primaryInput, partitionRow));
+            partitionRow.setown(primaryInputStream->ungroupedNextRow());
+            primaryStream.set(new cRowStreamPlus1Adaptor(primaryInputStream, partitionRow));
         }
         else
         {
-            sorter->Gather(primaryRowIf, primaryInput, primaryCompare, NULL, NULL, primaryKeySerializer, NULL, false, isUnstable(), abortSoon, NULL);
+            sorter->Gather(primaryRowIf, primaryInputStream, primaryCompare, NULL, NULL, primaryKeySerializer, NULL, false, isUnstable(), abortSoon, NULL);
             stopPartitionInput();
             if (abortSoon)
             {
@@ -543,7 +576,7 @@ public:
             sorter->stopMerge();
         }
         // NB: on secondary sort, the primaryKeySerializer is used
-        sorter->Gather(secondaryRowIf, secondaryInput, secondaryCompare, primarySecondaryCompare, primarySecondaryUpperCompare, primaryKeySerializer, partitionRow, noSortOtherSide(), isUnstable(), abortSoon, primaryRowIf); // primaryKeySerializer *is* correct
+        sorter->Gather(secondaryRowIf, secondaryInputStream, secondaryCompare, primarySecondaryCompare, primarySecondaryUpperCompare, primaryKeySerializer, partitionRow, noSortOtherSide(), isUnstable(), abortSoon, primaryRowIf); // primaryKeySerializer *is* correct
         mergeStats(spillStats, sorter);
         //MORE: Stats from spilling the primaryStream??
         partitionRow.clear();
@@ -592,7 +625,7 @@ public:
 //////////////////////
 
 
-class CMergeJoinSlaveBaseActivity : public CThorNarySlaveActivity, public CThorDataLink, public CThorSteppable
+class CMergeJoinSlaveBaseActivity : public CThorNarySlaveActivity, public CThorSteppable
 {
     IHThorNWayMergeJoinArg *helper;
     Owned<IEngineRowAllocator> inputAllocator, outputAllocator;
@@ -606,30 +639,28 @@ protected:
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CMergeJoinSlaveBaseActivity(CGraphElementBase *container, CMergeJoinProcessor &_processor) : CThorNarySlaveActivity(container), CThorDataLink(this), CThorSteppable(this), processor(_processor)
+    CMergeJoinSlaveBaseActivity(CGraphElementBase *container, CMergeJoinProcessor &_processor) : CThorNarySlaveActivity(container), CThorSteppable(this), processor(_processor)
     {
         helper = (IHThorNWayMergeJoinArg *)queryHelper();
         inputAllocator.setown(getRowAllocator(helper->queryInputMeta()));
         outputAllocator.setown(getRowAllocator(helper->queryOutputMeta()));
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
     }
-    void start()
+    virtual void start() override
     {
         CThorNarySlaveActivity::start();
 
         ForEachItemIn(i1, expandedInputs)
         {
-            IThorDataLink *cur = expandedInputs.item(i1);
-            Owned<CThorSteppedInput> stepInput = new CThorSteppedInput(cur);
+            Owned<CThorSteppedInput> stepInput = new CThorSteppedInput(expandedInputs.item(i1), expandedStreams.item(i1));
             processor.addInput(stepInput);
         }
         processor.beforeProcessing(inputAllocator, outputAllocator);
-        dataLinkStart();
     }
-    virtual void stop()
+    virtual void stop() override
     {
         processor.afterProcessing();
         CThorNarySlaveActivity::stop();
@@ -645,12 +676,12 @@ public:
         }
         return NULL;
     }
-    const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         try { return nextRowGENoCatch(seek, numFields, wasCompleteMatch, stepExtra); }
         CATCH_NEXTROWX_CATCH;
     }
-    const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         ActivityTimer t(totalCycles, timeActivities);
         bool matched = true;
@@ -659,14 +690,14 @@ public:
             dataLinkIncrement();
         return next.getClear();
     }
-    bool isGrouped() { return false; }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.unknownRowsOutput = true;
         info.canBufferInput = true;
     }
-    bool gatherConjunctions(ISteppedConjunctionCollector &collector)
+    virtual bool gatherConjunctions(ISteppedConjunctionCollector &collector)
     {
         return processor.gatherConjunctions(collector);
     }
@@ -675,10 +706,10 @@ public:
         processor.queryResetEOF(); 
     }
 // steppable
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CThorNarySlaveActivity::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        CThorNarySlaveActivity::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta() { return CThorSteppable::inputStepping; }
 };

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -520,8 +520,10 @@ interface IRowStreamSetInput : extends IRowStream
     virtual void setInput(IRowStream *input) = 0;
 };
 
-class CKeyedJoinSlave : public CSlaveActivity, public CThorDataLink, implements IJoinProcessor, implements IJoinGroupNotify
+class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implements IJoinGroupNotify
 {
+    typedef CSlaveActivity PARENT;
+
 #ifdef TRACE_JOINGROUPS
     unsigned groupsPendsNoted, fetchReadBack, groupPendsEnded, doneGroupsDeQueued, wroteToFetchPipe, groupsComplete;
 #endif
@@ -530,7 +532,6 @@ class CKeyedJoinSlave : public CSlaveActivity, public CThorDataLink, implements
     IRowStreamSetInput *resultDistStream;
     CPartDescriptorArray indexParts, dataParts;
     Owned<IKeyIndexSet> tlkKeySet, partKeySet;
-    IThorDataLink *input;
     bool preserveGroups, preserveOrder, eos, inputStopped, needsDiskRead, atMostProvided, remoteDataFiles;
     unsigned joinFlags, abortLimit, parallelLookups, freeQSize, filePartTotal;
     size32_t fixedRecordSize;
@@ -1565,12 +1566,11 @@ class CKeyedJoinSlave : public CSlaveActivity, public CThorDataLink, implements
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CKeyedJoinSlave(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CKeyedJoinSlave(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
 #ifdef TRACE_JOINGROUPS
         groupsPendsNoted = fetchReadBack = groupPendsEnded = doneGroupsDeQueued = wroteToFetchPipe = groupsComplete = 0;
 #endif
-        input = NULL;
         inputHelper = NULL;
         preserveGroups = preserveOrder = eos = false;
         resultDistStream = NULL;
@@ -1747,8 +1747,7 @@ public:
         if (!inputStopped)
         {
             inputStopped = true;
-            CSlaveActivity::stopInput(input, "(LEFT)");
-            input = NULL;
+            PARENT::stop();
         }
     }
     void doAbortLimit(CJoinGroup *jg)
@@ -2040,10 +2039,9 @@ public:
     {
         ActivityTimer s(totalCycles, timeActivities);
         assertex(inputs.ordinality() == 1);
+        PARENT::start();
 
         eos = false;
-        input = inputs.item(0);
-        startInput(input);
         inputHelper = LINK(input->queryFromActivity()->queryContainer().queryHelper());
         inputStopped = false;
         preserveOrder = ((joinFlags & JFreorderable) == 0);
@@ -2051,9 +2049,7 @@ public:
         ActPrintLog("KJ: parallelLookups=%d, freeQSize=%d, preserveGroups=%s, preserveOrder=%s", parallelLookups, freeQSize, preserveGroups?"true":"false", preserveOrder?"true":"false");
 
         pool->setOrdering(preserveGroups, preserveOrder);
-        resultDistStream->setInput(input);
-
-        dataLinkStart();
+        resultDistStream->setInput(inputStream);
     }
     virtual void stop()
     {
@@ -2343,7 +2339,7 @@ public:
         return NULL;
     }
 
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
 
     void getMetaInfo(ThorDataLinkMetaInfo &info)
     {

thorlcr/activities/limit/thlimitslave.cpp

@@ -23,50 +23,47 @@
 #include "thormisc.hpp"
 
 
-class CLimitSlaveActivityBase : public CSlaveActivity, public CThorDataLink
+class CLimitSlaveActivityBase : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 protected:
     rowcount_t rowLimit;
     bool eos, eogNext, stopped, resultSent, anyThisGroup;
-    IThorDataLink *input;
     IHThorLimitArg *helper;
 
     void stopInput(rowcount_t c)
     {
-        if (!stopped) {
+        if (!stopped)
+        {
             stopped = true;
             sendResult(c);
-            CSlaveActivity::stopInput(input);
+            PARENT::stop();
         }
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CLimitSlaveActivityBase(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CLimitSlaveActivityBase(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = (IHThorLimitArg *)queryHelper();
-        input = NULL;       
         resultSent = container.queryLocal(); // i.e. local, so don't send result to master
         eos = stopped = anyThisGroup = eogNext = false;
         rowLimit = RCMAX;
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
 
         if (!container.queryLocal())
             mpTag = container.queryJobChannel().deserializeMPTag(data);
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         resultSent = container.queryLocal(); // i.e. local, so don't send result to master
         eos = stopped = anyThisGroup = eogNext = false;
-        input = inputs.item(0);
-        startInput(input);
         rowLimit = (rowcount_t)helper->getRowLimit();
-        dataLinkStart();
     }
     void sendResult(rowcount_t r)
     {
@@ -76,19 +73,18 @@ public:
         mb.append(r);
         queryJobChannel().queryJobComm().send(mb, 0, mpTag);
     }
-    void stop()
+    virtual void stop() override
     {
         stopInput(getDataLinkCount());
-        dataLinkStop();
     }
-    bool isGrouped() { return inputs.item(0)->isGrouped(); }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.canReduceNumRows = true;
         info.canBufferInput = false;
         info.totalRowsMax = rowLimit;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 
@@ -105,7 +101,7 @@ public:
             return NULL;
         while (!abortSoon && !eogNext)
         {
-            OwnedConstThorRow row = input->nextRow();
+            OwnedConstThorRow row = inputStream->nextRow();
             if (!row)
             {
                 if(anyThisGroup)
@@ -113,7 +109,7 @@ public:
                     anyThisGroup = false;
                     break;
                 }
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if (!row)
                 {
                     eos = true;
@@ -143,7 +139,7 @@ public:
     const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         ActivityTimer t(totalCycles, timeActivities);
-        OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
+        OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
         if (ret)
         {
             if (wasCompleteMatch)
@@ -160,13 +156,13 @@ public:
     void resetEOF() 
     { 
         //Do not reset the rowLimit
-        input->resetEOF(); 
+        inputStream->resetEOF();
     }
 // steppable
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CLimitSlaveActivityBase::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        CLimitSlaveActivityBase::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta() { return CThorSteppable::inputStepping; }
 };
@@ -196,10 +192,10 @@ class CSkipLimitSlaveActivity : public CLimitSlaveActivityBase
         rowcount_t count = 0;
         while (!abortSoon)
         {
-            OwnedConstThorRow row = input->nextRow();
+            OwnedConstThorRow row = inputStream->nextRow();
             if (!row)
             {
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if (!row)
                     break;
                 else
@@ -237,24 +233,19 @@ public:
         rowTransform = _rowTransform;
         helperex = NULL;
     }
-    void stop()
-    {
-        stopInput(0);
-        dataLinkStop();
-    }
     void abort()
     {
         if (!container.queryLocal())
             cancelReceiveMsg(0, mpTag);
         CLimitSlaveActivityBase::abort();
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         CLimitSlaveActivityBase::init(data,slaveData);
         if (rowTransform)
             helperex = static_cast<IHThorLimitTransformExtra *>(queryHelper()->selectInterface(TAIlimittransformextra_1));
     }
-    void start()
+    virtual void start() override
     {
         CLimitSlaveActivityBase::start();
         buf.setown(createOverflowableBuffer(*this, this, true));

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -796,9 +796,10 @@ struct HtEntry { rowidx_t index, count; };
  * and base common functionality for all and lookup varieties
  */
 template <class HTHELPER, class HELPER>
-class CInMemJoinBase : public CSlaveActivity, public CThorDataLink, public CAllOrLookupHelper<HELPER>, implements ISmartBufferNotify, implements IBCastReceive
+class CInMemJoinBase : public CSlaveActivity, public CAllOrLookupHelper<HELPER>, implements ILookAheadStopNotify, implements IBCastReceive
 {
-    Semaphore leftstartsem;
+    typedef CSlaveActivity PARENT;
+
     Owned<IException> leftexception;
 
     bool eos, eog, someSinceEog;
@@ -1283,8 +1284,7 @@ protected:
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CInMemJoinBase(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this),
-        HELPERBASE((HELPER *)queryHelper()), rhs(*this)
+    CInMemJoinBase(CGraphElementBase *_container) : CSlaveActivity(_container), HELPERBASE((HELPER *)queryHelper()), rhs(*this)
     {
         gotRHS = false;
         nextRhsRow = 0;
@@ -1347,6 +1347,17 @@ public:
         }
     }
     HTHELPER *queryTable() { return table; }
+    void startLeftInput()
+    {
+        try
+        {
+            startInput(0);
+        }
+        catch(IException *e)
+        {
+            leftexception.setown(e);
+        }
+    }
 
 // IThorSlaveActivity overloaded methods
     virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
@@ -1373,7 +1384,13 @@ public:
             }
         }
     }
-    virtual void start()
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
+    {
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        if (0 == index)
+            setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), LOOKUPJOINL_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(input->queryFromActivity()), grouped, RCUNBOUND, this, &container.queryJob().queryIDiskUsage()));
+    }
+    virtual void start() override
     {
         assertex(inputs.ordinality() == 2);
 
@@ -1384,8 +1401,8 @@ public:
         leftMatch = false;
         rhsNext = NULL;
         rhsTableLen = 0;
-        leftITDL = inputs.item(0);
-        rightITDL = inputs.item(1);
+        leftITDL = queryInput(0);
+        rightITDL = queryInput(1);
         rightOutputMeta = rightITDL->queryFromActivity()->queryContainer().queryHelper()->queryOutputMeta();
         rightAllocator.setown(rightThorAllocator->getRowAllocator(rightOutputMeta, container.queryId()));
 
@@ -1426,7 +1443,6 @@ public:
         currentHashEntry.index = 0;
         currentHashEntry.count = 0;
 
-        right.set(rightITDL);
         rightSerializer.set(::queryRowSerializer(rightITDL));
         rightDeserializer.set(::queryRowDeserializer(rightITDL));
 
@@ -1438,21 +1454,20 @@ public:
             defaultRight.setown(rr.finalizeRowClear(rrsz));
         }
 
-        leftITDL = createDataLinkSmartBuffer(this,leftITDL,LOOKUPJOINL_SMART_BUFFER_SIZE,isSmartBufferSpillNeeded(leftITDL->queryFromActivity()),grouped,RCUNBOUND,this,false,&container.queryJob().queryIDiskUsage());
-        left.setown(leftITDL);
-        startInput(leftITDL);
-
+        CAsyncCallStart asyncLeftStart(std::bind(&CInMemJoinBase::startLeftInput, this));
         try
         {
-            startInput(rightITDL);
+            startInput(1);
         }
         catch (CATCHALL)
         {
-            leftstartsem.wait();
+            asyncLeftStart.wait();
             left->stop();
             throw;
         }
-        leftstartsem.wait();
+        asyncLeftStart.wait();
+        left.set(inputStream);
+        right.set(queryInputStream(1));
         if (leftexception)
         {
             right->stop();
@@ -1479,12 +1494,12 @@ public:
         clearHT();
         if (right)
         {
-            stopInput(right, "(R)");
+            stopInput(1, "(R)");
             right.clear();
         }
         if (broadcaster)
             broadcaster->reset();
-        stopInput(left, "(L)");
+        stopInput(0, "(L)");
         left.clear();
         dataLinkStop();
     }
@@ -1561,16 +1576,6 @@ public:
         dbgassertex((sendItem==NULL) == stop); // if sendItem==NULL stop must = true, if sendItem != NULL stop must = false;
         rowProcessor->addBlock(sendItem);
     }
-// ISmartBufferNotify
-    virtual void onInputStarted(IException *except)
-    {
-        leftexception.set(except);
-        leftstartsem.signal();
-    }
-    virtual bool startAsync()
-    {
-        return true;
-    }
     virtual void onInputFinished(rowcount_t count)
     {
         ActPrintLog("LHS input finished, %" RCPF "d rows read", count);
@@ -1669,6 +1674,7 @@ protected:
     using PARENT::mySlaveNum;
     using PARENT::tableProxy;
     using PARENT::gatheredRHSNodeStreams;
+    using PARENT::queryInput;
 
     IHash *leftHash, *rightHash;
     ICompare *compareRight, *compareLeftRight;
@@ -2245,7 +2251,7 @@ protected:
 
         Owned<IThorRowLoader> rowLoader = createThorRowLoader(*this, queryRowInterfaces(leftITDL), helper->isLeftAlreadyLocallySorted() ? NULL : compareLeft);
         left.setown(rowLoader->load(left, abortSoon, false));
-        leftITDL = inputs.item(0); // reset
+        leftITDL = queryInput(0); // reset
         ActPrintLog("LHS loaded/sorted");
 
         // rightStream is sorted
@@ -2539,7 +2545,7 @@ public:
         return false;
     }
 // IThorSlaveActivity overloaded methods
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         PARENT::init(data, slaveData);
 
@@ -2555,7 +2561,7 @@ public:
             lhsDistributor.setown(createHashDistributor(this, queryJobChannel().queryJobComm(), lhsDistributeTag, false, NULL, "LHS"));
         }
     }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         PARENT::start();
@@ -2607,7 +2613,7 @@ public:
         dataLinkIncrement();
         return row.getClear();
     }
-    virtual void abort()
+    virtual void abort() override
     {
         PARENT::abort();
         if (rhsDistributor)
@@ -2617,7 +2623,7 @@ public:
         if (joinHelper)
             joinHelper->stop();
     }
-    virtual void stop()
+    virtual void stop() override
     {
         if (isGlobal())
         {
@@ -2644,9 +2650,9 @@ public:
         joinHelper.clear();
         PARENT::stop();
     }
-    virtual bool isGrouped()
+    virtual bool isGrouped() const override
     {
-        return isSmart() ? false : inputs.item(0)->isGrouped();
+        return isSmart() ? false : queryInput(0)->isGrouped();
     }
     virtual void bCastReceive(CSendItem *sendItem, bool stop) // NB: only called on channel 0
     {
@@ -3148,7 +3154,7 @@ public:
         }
         PARENT::stop();
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
 };
 
 

thorlcr/activities/loop/thloopslave.cpp

@@ -28,10 +28,11 @@
 #include "eclrtl_imp.hpp"
 #include "thcompressutil.hpp"
 
-class CLoopSlaveActivityBase : public CSlaveActivity, public CThorDataLink
+class CLoopSlaveActivityBase : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 protected:
-    IThorDataLink *input;
     bool global;
     bool sentEndLooping;
     unsigned maxIterations;
@@ -75,11 +76,8 @@ protected:
         sendLoopingCount(0, 0);
     }
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CLoopSlaveActivityBase(CGraphElementBase *container) : CSlaveActivity(container), CThorDataLink(this)
+    CLoopSlaveActivityBase(CGraphElementBase *container) : CSlaveActivity(container)
     {
-        input = NULL;
         mpTag = TAG_NULL;
         maxEmptyLoopIterations = getOptUInt(THOROPT_LOOP_MAX_EMPTY, 1000);
     }
@@ -96,23 +94,21 @@ public:
         if (!container.queryLocalOrGrouped())
             cancelReceiveMsg(0, mpTag);
     }
-    void dostart()
+    virtual void start() override
     {
+        PARENT::start();
         extractBuilder.clear();
         sentEndLooping = false;
         lastMaxEmpty = false;
         loopCounter = 1;
-        input = inputs.item(0);
-        startInput(input);
     }
     void doStop()
     {
         sendEndLooping();
-        stopInput(input);
-        dataLinkStop();
+        PARENT::stop();
     }
 // IThorDataLink
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -131,6 +127,8 @@ public:
 
 class CLoopSlaveActivity : public CLoopSlaveActivityBase
 {
+    typedef CLoopSlaveActivityBase PARENT;
+
     Owned<IRowStream> curInput;
     Owned<IRowWriterMultiReader> loopPending;
     rowcount_t loopPendingCount;
@@ -270,7 +268,7 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dostart();
+        PARENT::start();
         eof = false;
         helper->createParentExtract(extractBuilder);
         maxIterations = helper->numIterations();
@@ -280,11 +278,10 @@ public:
         finishedLooping = ((container.getKind() == TAKloopcount) && (maxIterations == 0));
         if ((flags & IHThorLoopArg::LFnewloopagain) && !helper->loopFirstTime())
             finishedLooping = true;
-        curInput.set(input);
+        curInput.set(inputStream);
         lastMs = msTick();
 
         ActPrintLog("maxIterations = %d", maxIterations);
-        dataLinkStart();
         nextRowFeeder.setown(new CNextRowFeeder(this));
     }
     void doStop()
@@ -434,6 +431,8 @@ public:
 
 class CGraphLoopSlaveActivity : public CLoopSlaveActivityBase
 {
+    typedef CLoopSlaveActivityBase PARENT;
+
     IHThorGraphLoopArg *helper;
     bool executed;
     unsigned flags;
@@ -454,14 +453,13 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dostart();
+        PARENT::start();
         executed = false;
         maxIterations = helper->numIterations();
         if ((int)maxIterations < 0) maxIterations = 0;
         loopResults.setown(queryGraph().createThorGraphResults(0));
         helper->createParentExtract(extractBuilder);
         ActPrintLog("maxIterations = %d", maxIterations);
-        dataLinkStart();
     }
     CATCH_NEXTROW()
     {
@@ -472,10 +470,10 @@ public:
             Owned<IRowWriter> resultWriter = result->getWriter();
             loop
             {
-                OwnedConstThorRow row = input->nextRow();
+                OwnedConstThorRow row = inputStream->nextRow();
                 if (!row)
                 {
-                    row.setown(input->nextRow());
+                    row.setown(inputStream->nextRow());
                     if (!row)
                         break;
                     resultWriter->putRow(NULL);
@@ -514,21 +512,19 @@ activityslaves_decl CActivityBase *createLoopSlave(CGraphElementBase *container)
 
 /////////////// local result read
 
-class CLocalResultReadActivity : public CSlaveActivity, public CThorDataLink
+class CLocalResultReadActivity : public CSlaveActivity
 {
-    IThorDataLink *input;
+    typedef CSlaveActivity PARENT;
+
     IHThorLocalResultReadArg *helper;
     Owned<IRowStream> resultStream;
     unsigned curRow;
     mptag_t replyTag;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CLocalResultReadActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CLocalResultReadActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = (IHThorLocalResultReadArg *)queryHelper();
-        input = NULL;
         curRow = 0;
         replyTag = queryMPServer().createReplyTag();
     }
@@ -540,6 +536,7 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         curRow = 0;
         abortSoon = false;
         assertex(container.queryResultsGraph());
@@ -549,13 +546,12 @@ public:
         Owned<CGraphBase> graph = queryJobChannel().getGraph(resultGraphId);
         Owned<IThorResult> result = graph->getResult(helper->querySequence(), queryGraph().isLocalChild());
         resultStream.setown(result->getRowStream());
-        dataLinkStart();
     }
     virtual void stop()
     {
         abortSoon = true;
         resultStream.clear();
-        dataLinkStop();
+        PARENT::stop();
     }
     virtual void kill()
     {
@@ -576,7 +572,7 @@ public:
         }
         return NULL;
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -591,8 +587,10 @@ activityslaves_decl CActivityBase *createLocalResultReadSlave(CGraphElementBase
 
 /////////////// local spill write
 
-class CLocalResultSpillActivity : public CSlaveActivity, public CThorDataLink
+class CLocalResultSpillActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorLocalResultSpillArg *helper;
     bool eoi, lastNull;
     Owned<IRowWriter> resultWriter;
@@ -607,9 +605,7 @@ class CLocalResultSpillActivity : public CSlaveActivity, public CThorDataLink
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CLocalResultSpillActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CLocalResultSpillActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = (IHThorLocalResultSpillArg *)queryHelper();
     }
@@ -621,21 +617,20 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         lastNull = eoi = false;
         abortSoon = false;
         assertex(container.queryResultsGraph());
         Owned<CGraphBase> graph = queryJobChannel().getGraph(container.queryResultsGraph()->queryGraphId());
         IThorResult *result = graph->createResult(*this, helper->querySequence(), this, !queryGraph().isLocalChild());  // NB graph owns result
         resultWriter.setown(result->getWriter());
-        startInput(inputs.item(0));
-        dataLinkStart();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
         if (!abortSoon && !eoi)
             return NULL;
-        OwnedConstThorRow row = inputs.item(0)->nextRow();
+        OwnedConstThorRow row = inputStream->nextRow();
         if (!row)
         {
             if (lastNull)
@@ -655,11 +650,10 @@ public:
     }
     virtual void stop()
     {
-        stopInput(inputs.item(0));
         abortSoon = true;
-        dataLinkStop();
+        PARENT::stop();
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -670,18 +664,14 @@ public:
 
 class CLocalResultWriteActivityBase : public ProcessSlaveActivity
 {
-protected:
-    IThorDataLink *input;
 public:
     CLocalResultWriteActivityBase(CGraphElementBase *_container) : ProcessSlaveActivity(_container)
     {
-        input = NULL;
     }
     virtual IThorResult *createResult() = 0;
     virtual void process()
     {
-        input = inputs.item(0);
-        startInput(input);
+        start();
         processed = THORDATALINK_STARTED;
 
         IThorResult *result = createResult();
@@ -689,10 +679,10 @@ public:
         Owned<IRowWriter> resultWriter = result->getWriter();
         loop
         {
-            OwnedConstThorRow nextrec = input->nextRow();
+            OwnedConstThorRow nextrec = inputStream->nextRow();
             if (!nextrec)
             {
-                nextrec.setown(input->nextRow());
+                nextrec.setown(inputStream->nextRow());
                 if (!nextrec)
                     break;
                 resultWriter->putRow(NULL);
@@ -704,7 +694,7 @@ public:
     {
         if (processed & THORDATALINK_STARTED)
         {
-            stopInput(input);
+            stop();
             processed |= THORDATALINK_STOPPED;
         }
     }
@@ -738,27 +728,23 @@ activityslaves_decl CActivityBase *createLocalResultSpillSlave(CGraphElementBase
 class CDictionaryResultWriteActivity : public ProcessSlaveActivity
 {
     IHThorDictionaryResultWriteArg *helper;
-protected:
-    IThorDataLink *input;
 public:
     CDictionaryResultWriteActivity(CGraphElementBase *_container) : ProcessSlaveActivity(_container)
     {
         helper = (IHThorDictionaryResultWriteArg *)queryHelper();
-        input = NULL;
     }
     virtual void process()
     {
-        input = inputs.item(0);
-        startInput(input);
+        start();
         processed = THORDATALINK_STARTED;
 
         RtlLinkedDictionaryBuilder builder(queryRowAllocator(), helper->queryHashLookupInfo());
         loop
         {
-            const void *row = input->nextRow();
+            const void *row = inputStream->nextRow();
             if (!row)
             {
-                row = input->nextRow();
+                row = inputStream->nextRow();
                 if (!row)
                     break;
             }
@@ -781,7 +767,7 @@ public:
     {
         if (processed & THORDATALINK_STARTED)
         {
-            stopInput(input);
+            stop();
             processed |= THORDATALINK_STOPPED;
         }
     }
@@ -800,32 +786,36 @@ activityslaves_decl CActivityBase *createGraphLoopSlave(CGraphElementBase *conta
 
 /////////////
 
-class CConditionalActivity : public CSlaveActivity, public CThorDataLink
+class CConditionalActivity : public CSlaveActivity
 {
-    IThorDataLink *selectedInput;
-public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
+    typedef CSlaveActivity PARENT;
 
-    CConditionalActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    IThorDataLink *selectedInput = NULL;
+    IEngineRowStream *selectInputStream = NULL;
+public:
+    CConditionalActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         appendOutputLinked(this);
-        selectedInput = NULL;
     }
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        selectedInput = container.whichBranch>=inputs.ordinality() ? NULL : inputs.item(container.whichBranch);
+        selectedInput = container.whichBranch>=inputs.ordinality() ? NULL : queryInput(container.whichBranch);
+        selectInputStream = NULL;
         if (selectedInput)
-            startInput(selectedInput);
+        {
+            startInput(container.whichBranch);
+            selectInputStream = queryInputStream(container.whichBranch);
+        }
         dataLinkStart();
     }
     virtual void stop()
     {
-        if (selectedInput)
-            stopInput(selectedInput);
+        if (selectInputStream)
+            stopInput(container.whichBranch);
         abortSoon = true;
         dataLinkStop();
     }
@@ -837,12 +827,12 @@ public:
         if (!selectedInput)
             return NULL;
 
-        OwnedConstThorRow ret = selectedInput->nextRow();
+        OwnedConstThorRow ret = selectInputStream->nextRow();
         if (ret)
             dataLinkIncrement();
         return ret.getClear();
     }
-    virtual bool isGrouped() { return selectedInput?selectedInput->isGrouped():false; }
+    virtual bool isGrouped() const override { return selectedInput?selectedInput->isGrouped():false; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -862,16 +852,16 @@ activityslaves_decl CActivityBase *createCaseSlave(CGraphElementBase *container)
 
 //////////// NewChild acts - move somewhere else..
 
-class CChildNormalizeSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CChildNormalizeSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorChildNormalizeArg *helper;
     Owned<IEngineRowAllocator> allocator;
     bool eos, ok, started;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CChildNormalizeSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CChildNormalizeSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = (IHThorChildNormalizeArg *)queryHelper();
     }
@@ -883,14 +873,10 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         started = false;
         eos = false;
         ok = false;
-        dataLinkStart();
-    }
-    virtual void stop()
-    {
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -924,7 +910,7 @@ public:
         eos = true;
         return NULL;
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -939,15 +925,15 @@ activityslaves_decl CActivityBase *createChildNormalizeSlave(CGraphElementBase *
 
 //=====================================================================================================
 
-class CChildAggregateSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CChildAggregateSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorChildAggregateArg *helper;
     bool eos;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CChildAggregateSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CChildAggregateSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = (IHThorChildAggregateArg *)queryHelper();
     }
@@ -958,12 +944,8 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         eos = false;
-        dataLinkStart();
-    }
-    virtual void stop()
-    {
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -976,7 +958,7 @@ public:
         dataLinkIncrement();
         return ret.finalizeRowClear(sz);
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -990,8 +972,10 @@ activityslaves_decl CActivityBase *createChildAggregateSlave(CGraphElementBase *
 
 //=====================================================================================================
 
-class CChildGroupAggregateActivitySlave : public CSlaveActivity, public CThorDataLink, implements IHThorGroupAggregateCallback
+class CChildGroupAggregateActivitySlave : public CSlaveActivity, implements IHThorGroupAggregateCallback
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorChildGroupAggregateArg *helper;
     bool eos, gathered;
     Owned<IEngineRowAllocator> allocator;
@@ -1000,7 +984,7 @@ class CChildGroupAggregateActivitySlave : public CSlaveActivity, public CThorDat
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CChildGroupAggregateActivitySlave(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CChildGroupAggregateActivitySlave(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = (IHThorChildGroupAggregateArg *)queryHelper();
     }
@@ -1012,15 +996,15 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         gathered = eos = false;
         aggregated.clear();
         aggregated.setown(new RowAggregator(*helper, *helper));
         aggregated->start(queryRowAllocator());
-        dataLinkStart();
     }
     virtual void stop()
     {
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -1040,7 +1024,7 @@ public:
         eos = true;
         return NULL;
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -1060,8 +1044,10 @@ activityslaves_decl CActivityBase *createChildGroupAggregateSlave(CGraphElementB
 
 //=====================================================================================================
 
-class CChildThroughNormalizeSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CChildThroughNormalizeSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorChildThroughNormalizeArg *helper;
     Owned<IEngineRowAllocator> allocator;
     OwnedConstThorRow lastInput;
@@ -1071,9 +1057,7 @@ class CChildThroughNormalizeSlaveActivity : public CSlaveActivity, public CThorD
     bool ok;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CChildThroughNormalizeSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this), nextOutput(NULL)
+    CChildThroughNormalizeSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), nextOutput(NULL)
     {
         helper = (IHThorChildThroughNormalizeArg *)queryHelper();
     }
@@ -1086,21 +1070,14 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         ok = false;
         numProcessedLastGroup = getDataLinkCount(); // is this right?
         lastInput.clear();
         nextOutput.clear();
-        startInput(inputs.item(0));
-        dataLinkStart();
-    }
-    virtual void stop()
-    {
-        stopInput(inputs.item(0));
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
-        IThorDataLink *input = inputs.item(0);
         loop
         {
             if (ok)
@@ -1108,7 +1085,7 @@ public:
 
             while (!ok)
             {
-                lastInput.setown(input->nextRow());
+                lastInput.setown(inputStream->nextRow());
                 if (!lastInput)
                 {
                     if (numProcessedLastGroup != getDataLinkCount()) // is this right?
@@ -1116,7 +1093,7 @@ public:
                         numProcessedLastGroup = getDataLinkCount(); // is this right?
                         return NULL;
                     }
-                    lastInput.setown(input->nextRow());
+                    lastInput.setown(inputStream->nextRow());
                     if (!lastInput)
                         return NULL;
                 }
@@ -1137,7 +1114,7 @@ public:
             } while (ok);
         }
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
@@ -1151,15 +1128,15 @@ activityslaves_decl CActivityBase *createChildThroughNormalizeSlave(CGraphElemen
 
 ///////////
 
-class CGraphLoopResultReadSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CGraphLoopResultReadSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorGraphLoopResultReadArg *helper;
     Owned<IRowStream> resultStream;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CGraphLoopResultReadSlaveActivity(CGraphElementBase *container) : CSlaveActivity(container), CThorDataLink(this)
+    CGraphLoopResultReadSlaveActivity(CGraphElementBase *container) : CSlaveActivity(container)
     {
         helper = (IHThorGraphLoopResultReadArg *)queryHelper();
     }
@@ -1191,7 +1168,7 @@ public:
             abortSoon = true;
         dataLinkStart();
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
@@ -1210,7 +1187,7 @@ public:
     {
         abortSoon = true;
         resultStream.clear();
-        dataLinkStop();
+        PARENT::stop();
     }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
@@ -1299,8 +1276,6 @@ activityslaves_decl CActivityBase *createGraphLoopResultReadSlave(CGraphElementB
 class CGraphLoopResultWriteSlaveActivity : public CLocalResultWriteActivityBase
 {
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CGraphLoopResultWriteSlaveActivity(CGraphElementBase *container) : CLocalResultWriteActivityBase(container)
     {
     }

thorlcr/activities/merge/thmergeslave.cpp

@@ -33,8 +33,10 @@
 
 #define _STABLE_MERGE
 
-class GlobalMergeSlaveActivity : public CSlaveActivity, public CThorDataLink
+class GlobalMergeSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 public:
     IArrayOf<IRowStream> streams; 
     IHThorMergeArg *helper;
@@ -241,9 +243,7 @@ public:
         return createRowStreamMerger(streams.ordinality(), streams.getArray(), helper->queryCompare(), helper->dedup(), linkcounter);
     }
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    GlobalMergeSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    GlobalMergeSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         partitionpos = NULL;
         linkcounter.setown(new CThorRowLinkCounter);
@@ -264,7 +264,7 @@ public:
     }
 
 // IThorSlaveActivity overloaded methods
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         helper = (IHThorMergeArg *)queryHelper();
         appendOutputLinked(this);
@@ -280,24 +280,27 @@ public:
 
 
 // IThorDataLink
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        ForEachItemIn(i, inputs) {
-            IThorDataLink * input = inputs.item(i);
-            try {
-                startInput(input); 
+        ForEachItemIn(i, inputs)
+        {
+            IThorDataLink * input = queryInput(i);
+            try
+            {
+                startInput(i);
             }
-            catch (CATCHALL) {
+            catch (CATCHALL)
+            {
                 ActPrintLog("MERGE(%" ACTPF "d): Error starting input %d", container.queryId(), i);
                 ForEachItemIn(s, streams)
                     streams.item(s).stop();
                 throw;
             }
             if (input->isGrouped())
-                streams.append(*createUngroupStream(input));
+                streams.append(*createUngroupStream(queryInputStream(i)));
             else
-                streams.append(*LINK(input));
+                streams.append(*LINK(queryInputStream(i)));
         }
 #ifndef _STABLE_MERGE
         // shuffle streams otherwise will all be reading in order initially
@@ -341,7 +344,7 @@ public:
         dataLinkStart();
     }
 
-    virtual void stop()
+    virtual void stop() override
     {
         if (out)
             out->stop();
@@ -398,25 +401,23 @@ public:
         return NULL;
     }
 
-    virtual bool isGrouped() { return false; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
-        calcMetaInfoSize(info,inputs.getArray(),inputs.ordinality());
+        calcMetaInfoSize(info, inputs);
     }
 };
 
 
 
-class LocalMergeSlaveActivity : public CSlaveActivity, public CThorDataLink
+class LocalMergeSlaveActivity : public CSlaveActivity
 {
     IArrayOf<IRowStream> streams; 
     Owned<IRowStream> out;
     IHThorMergeArg *helper;
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    LocalMergeSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this) { }
+    LocalMergeSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container) { }
 
 // IThorSlaveActivity overloaded methods
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
@@ -436,10 +437,12 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        ForEachItemIn(i, inputs) {
-            IThorDataLink * input = inputs.item(i);
-            try { 
-                startInput(input); 
+        ForEachItemIn(i, inputs)
+        {
+            IThorDataLink *input = queryInput(i);
+            try
+            {
+                startInput(i);
             }
             catch (CATCHALL) {
                 ActPrintLog("MERGE(%" ACTPF "d): Error starting input %d", container.queryId(), i);
@@ -448,13 +451,12 @@ public:
                 throw;
             }
             if (input->isGrouped())
-                streams.append(*createUngroupStream(input));
+                streams.append(*createUngroupStream(queryInputStream(i)));
             else
-                streams.append(*LINK(input));
+                streams.append(*LINK(queryInputStream(i)));
         }
         Owned<IRowLinkCounter> linkcounter = new CThorRowLinkCounter;
         out.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), helper->queryCompare(), helper->dedup(), linkcounter));
-
         dataLinkStart();
     }
 
@@ -483,22 +485,22 @@ public:
         return NULL;
     }
 
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
-        calcMetaInfoSize(info,inputs.getArray(),inputs.ordinality());
+        calcMetaInfoSize(info, inputs);
     }
 };
 
 
 class CThorStreamMerger : public CStreamMerger
 {
-    IThorDataLink **inputArray;
+    IEngineRowStream **inputArray;
 public:
     CThorStreamMerger() : CStreamMerger(true) {}
 
-    void initInputs(unsigned _numInputs, IThorDataLink ** _inputArray)
+    void initInputs(unsigned _numInputs, IEngineRowStream ** _inputArray)
     {
         CStreamMerger::initInputs(_numInputs);
         inputArray = _inputArray;
@@ -522,17 +524,19 @@ public:
 };
 
 
-class CNWayMergeActivity : public CThorNarySlaveActivity, public CThorDataLink, public CThorSteppable
+class CNWayMergeActivity : public CThorNarySlaveActivity, public CThorSteppable
 {
     IHThorNWayMergeArg *helper;
     CThorStreamMerger merger;
     CSteppingMeta meta;
     bool initializedMeta;
 
+    PointerArrayOf<IEngineRowStream> expandedInputStreams;
+
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CNWayMergeActivity(CGraphElementBase *container) : CThorNarySlaveActivity(container), CThorDataLink(this), CThorSteppable(this)
+    CNWayMergeActivity(CGraphElementBase *container) : CThorNarySlaveActivity(container), CThorSteppable(this)
     {
         helper = (IHThorNWayMergeArg *)queryHelper();
         merger.init(helper->queryCompare(), helper->dedup(), helper->querySteppingMeta()->queryCompare());
@@ -542,23 +546,21 @@ public:
     {
         merger.cleanup();
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
     }
-    void start()
+    virtual void start() override
     {
         CThorNarySlaveActivity::start();
-        merger.initInputs(expandedInputs.length(), expandedInputs.getArray());
-        dataLinkStart();
+        merger.initInputs(expandedStreams.length(), expandedStreams.getArray());
     }
-    void stop()
+    virtual void stop() override
     {
         merger.done();
         CThorNarySlaveActivity::stop();
-        dataLinkStop();
     }
-    void reset()
+    virtual void reset()
     {
         CThorNarySlaveActivity::reset();
         initializedMeta = false;
@@ -574,12 +576,12 @@ public:
         }
         return NULL;
     }
-    const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         try { return nextRowGENoCatch(seek, numFields, wasCompleteMatch, stepExtra); }
         CATCH_NEXTROWX_CATCH;
     }
-    const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         ActivityTimer t(totalCycles, timeActivities);
         OwnedConstThorRow ret = merger.nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
@@ -590,16 +592,16 @@ public:
         }
         return NULL;
     }
-    virtual bool isGrouped() { return false; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
-        calcMetaInfoSize(info,inputs.getArray(),inputs.ordinality());
+        calcMetaInfoSize(info, inputs);
     }
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CThorNarySlaveActivity::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        CThorNarySlaveActivity::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta()
     {

thorlcr/activities/msort/thgroupsortslave.cpp

@@ -30,9 +30,10 @@
 #include "thactivityutil.ipp"
 
 
-class CLocalSortSlaveActivity : public CSlaveActivity, public CThorDataLink 
+class CLocalSortSlaveActivity : public CSlaveActivity 
 {
-    IThorDataLink *input;
+    typedef CSlaveActivity PARENT;
+
     IHThorSortArg *helper;
     ICompare *iCompare;
     Owned<IThorRowLoader> iLoader;
@@ -42,13 +43,11 @@ class CLocalSortSlaveActivity : public CSlaveActivity, public CThorDataLink
     CRuntimeStatisticCollection spillStats;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CLocalSortSlaveActivity(CGraphElementBase *_container)
-        : CSlaveActivity(_container), CThorDataLink(this), spillStats(spillStatistics)
+        : CSlaveActivity(_container), spillStats(spillStatistics)
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         helper = (IHThorSortArg *)queryHelper();
         iCompare = helper->queryCompare();
@@ -56,19 +55,17 @@ public:
         unstable = (algo&&(algo->getAlgorithmFlags()&TAFunstable));
         appendOutputLinked(this);
     }
-    void start()
+    virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dataLinkStart();
-        input = inputs.item(0);
+        PARENT::start();
         unsigned spillPriority = container.queryGrouped() ? SPILL_PRIORITY_GROUPSORT : SPILL_PRIORITY_LARGESORT;
         iLoader.setown(createThorRowLoader(*this, queryRowInterfaces(input), iCompare, unstable ? stableSort_none : stableSort_earlyAlloc, rc_mixed, spillPriority));
-        startInput(input);
         eoi = false;
         if (container.queryGrouped())
-            out.setown(iLoader->loadGroup(input, abortSoon));
+            out.setown(iLoader->loadGroup(inputStream, abortSoon));
         else
-            out.setown(iLoader->load(input, abortSoon));
+            out.setown(iLoader->load(inputStream, abortSoon));
         if (0 == iLoader->numRows())
             eoi = true;
     }
@@ -82,11 +79,10 @@ public:
         mergedStats.serialize(mb);
     }
 
-    void stop()
+    virtual void stop()
     {
         out.clear();
-        stopInput(input);
-        dataLinkStop();
+        PARENT::stop();
 
         //Critical block
         {
@@ -108,7 +104,7 @@ public:
                 eoi = true;
                 return NULL;
             }
-            out.setown(iLoader->loadGroup(input, abortSoon));
+            out.setown(iLoader->loadGroup(inputStream, abortSoon));
             if (0 == iLoader->numRows())
                 eoi = true;
             return NULL; // eog marker
@@ -116,20 +112,21 @@ public:
         dataLinkIncrement();
         return row.getClear();
     }
-    virtual bool isGrouped() { return container.queryGrouped(); }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return container.queryGrouped(); }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.buffersInput = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 
 // Sorted 
 
-class CSortedSlaveActivity : public CSlaveActivity, public CThorDataLink, public CThorSteppable
+class CSortedSlaveActivity : public CSlaveActivity, public CThorSteppable
 {
-    IThorDataLink *input;
+    typedef CSlaveActivity PARENT;
+
     IHThorSortedArg *helper;
     ICompare *icompare;
     OwnedConstThorRow prev; 
@@ -138,33 +135,26 @@ public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
     CSortedSlaveActivity(CGraphElementBase *_container)
-        : CSlaveActivity(_container), CThorDataLink(this), CThorSteppable(this)
+        : CSlaveActivity(_container), CThorSteppable(this)
     {
         helper = (IHThorSortedArg *)queryHelper();
         icompare = helper->queryCompare();
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         helper = (IHThorSortedArg *)queryHelper();
         icompare = helper->queryCompare();
         appendOutputLinked(this);
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dataLinkStart();
-        input = inputs.item(0);
-        startInput(input);
-    }
-    void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
+        PARENT::start();
     }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
-        OwnedConstThorRow ret = input->nextRow();
+        OwnedConstThorRow ret = inputStream->nextRow();
         if (ret && prev && icompare->docompare(prev, ret) > 0)
         {
             // MORE - better to give mismatching rows than indexes?
@@ -175,15 +165,15 @@ public:
             dataLinkIncrement();
         return ret.getClear();
     }
-    const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         try { return nextRowGENoCatch(seek, numFields, wasCompleteMatch, stepExtra); }
         CATCH_NEXTROWX_CATCH;
     }
-    const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         ActivityTimer t(totalCycles, timeActivities);
-        OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
+        OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
         if (ret && prev && stepCompare->docompare(prev, ret, numFields) > 0)
         {
             // MORE - better to give mismatching rows than indexes?
@@ -194,25 +184,25 @@ public:
             dataLinkIncrement();
         return ret.getClear();
     }
-    bool gatherConjunctions(ISteppedConjunctionCollector &collector)
+    virtual bool gatherConjunctions(ISteppedConjunctionCollector &collector)
     { 
         return input->gatherConjunctions(collector);
     }
-    void resetEOF() 
+    virtual void resetEOF()
     { 
-        input->resetEOF(); 
+        inputStream->resetEOF();
     }
-    bool isGrouped() { return false; }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 // steppable
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CSlaveActivity::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        CSlaveActivity::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta() { return CThorSteppable::inputStepping; }
 };

thorlcr/activities/msort/thmsortslave.cpp

@@ -36,9 +36,10 @@
 //
 
 
-class MSortSlaveActivity : public CSlaveActivity, public CThorDataLink
+class MSortSlaveActivity : public CSlaveActivity
 {
-    IThorDataLink *input;
+    typedef CSlaveActivity PARENT;
+
     Owned<IRowStream> output;
     IHThorSortArg *helper;
     Owned<IThorSorter> sorter;
@@ -57,11 +58,8 @@ class MSortSlaveActivity : public CSlaveActivity, public CThorDataLink
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    MSortSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this), spillStats(spillStatistics)
+    MSortSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), spillStats(spillStatistics)
     {
-        input = NULL;
         portbase = 0;
         totalrows = RCUNSET;
     }
@@ -70,7 +68,7 @@ public:
         if (portbase) 
             freePort(portbase,NUMSLAVEPORTS);
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         mpTagRPC = container.queryJobChannel().deserializeMPTag(data);
         mptag_t barrierTag = container.queryJobChannel().deserializeMPTag(data);
@@ -83,14 +81,14 @@ public:
         appendOutputLinked(this);
         server.serialize(slaveData);
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
         try
         {
-            try { 
-                startInput(input); 
+            try
+            {
+                PARENT::start();
             }
             catch (IException *e)
             {
@@ -105,13 +103,12 @@ public:
                 barrier->cancel();
                 throw;
             }
-            dataLinkStart();
             
             Linked<IThorRowInterfaces> rowif = queryRowInterfaces(input);
             Owned<IThorRowInterfaces> auxrowif = createThorRowInterfaces(queryRowManager(), helper->querySortedRecordSize(),queryId(),queryCodeContext());
             sorter->Gather(
                 rowif,
-                input,
+                inputStream,
                 helper->queryCompare(),
                 helper->queryCompareLeftRight(),
                 NULL,helper->querySerialize(),
@@ -120,8 +117,8 @@ public:
                 isUnstable(),
                 abortSoon,
                 auxrowif);
-            stopInput(input);
-            input = NULL;
+
+            PARENT::stop();
             if (abortSoon)
             {
                 ActPrintLogEx(&queryContainer(), thorlog_null, MCwarning, "MSortSlaveActivity::start aborting");
@@ -150,22 +147,22 @@ public:
         ActPrintLog("SORT barrier.1 raised");
         output.setown(sorter->startMerge(totalrows));
     }
-    void stop()
+    virtual void stop() override
     {
-        if (output) {
+        if (output)
+        {
             output->stop();
             output.clear();
         }
         ActPrintLog("SORT waiting barrier.2");
         barrier->wait(false);
         ActPrintLog("SORT barrier.2 raised");
-        if (input)
-            stopInput(input);
+        PARENT::stop();
         sorter->stopMerge();
         ActPrintLog("SORT waiting for merge");
         dataLinkStop();
     }
-    void reset()
+    virtual void reset()
     {
         if (sorter) return; // JCSMORE loop - shouldn't have to recreate sorter between loop iterations
         sorter.setown(CreateThorSorter(this, server,&container.queryJob().queryIDiskUsage(),&queryJobChannel().queryJobComm(),mpTagRPC));
@@ -203,8 +200,8 @@ public:
         return row.getClear();
     }
 
-    virtual bool isGrouped() { return false; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.buffersInput = true;

thorlcr/activities/msort/thsortu.cpp

@@ -148,9 +148,6 @@ class CDualCache: public CSimpleInterface
     unsigned pos2;
     QueueOf<CRollingCacheElem,true> cache;
 public:
-
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
-
     CDualCache()
     {
         strm1 = NULL;

thorlcr/activities/normalize/thnormalizeslave.cpp

@@ -25,10 +25,11 @@
 #include "thexception.hpp"
 
 
-class NormalizeSlaveActivity : public CSlaveActivity, public CThorDataLink
+class NormalizeSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorNormalizeArg * helper;
-    IThorDataLink *input;
     OwnedConstThorRow row;
     unsigned curRow;
     unsigned numThisRow;
@@ -38,32 +39,23 @@ class NormalizeSlaveActivity : public CSlaveActivity, public CThorDataLink
 
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     NormalizeSlaveActivity(CGraphElementBase *_container) 
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         appendOutputLinked(this);
         helper = static_cast <IHThorNormalizeArg *> (queryHelper());
         allocator.set(queryRowAllocator());
     }
-    void start()
+    virtual void start() override
     { 
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         numThisRow = 0;
         curRow = 0;
         anyThisGroup = false;
-        input = inputs.item(0);
-        startInput(input);
-        dataLinkStart();
-    }
-    void stop()
-    { 
-        stopInput(input);
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -74,9 +66,9 @@ public:
             {
                 if (abortSoon) 
                     return NULL;
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if (!row&&!anyThisGroup)
-                    row.setown(input->nextRow());
+                    row.setown(inputStream->nextRow());
                 if(!row) {
                     anyThisGroup = false;
                     return NULL;
@@ -95,8 +87,8 @@ public:
             }
         }
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.unknownRowsOutput = true;
@@ -108,11 +100,12 @@ public:
 ////////////////////
 
 
-class CNormalizeChildSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CNormalizeChildSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorNormalizeChildArg *helper;
     INormalizeChildIterator *cursor;
-    IThorDataLink *input;
     OwnedConstThorRow childBuf;
     void * curChildRow;
     unsigned curRow;
@@ -120,14 +113,12 @@ class CNormalizeChildSlaveActivity : public CSlaveActivity, public CThorDataLink
     Owned<IEngineRowAllocator> allocator;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CNormalizeChildSlaveActivity(CGraphElementBase *_container) 
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     { 
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         helper = static_cast <IHThorNormalizeChildArg *> (queryHelper());
@@ -135,34 +126,27 @@ public:
         cursor = helper->queryIterator();
         allocator.set(queryRowAllocator());
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        startInput(input);
+        PARENT::start();
         anyThisGroup = false;
-        dataLinkStart();
         curChildRow = NULL;
     }
-    void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
-    }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
         loop {
             while(!curChildRow) {
                 curRow = 0;
-                childBuf.setown(input->nextRow());
+                childBuf.setown(inputStream->nextRow());
                 if (!childBuf) {
                     if (anyThisGroup) 
                     {
                         anyThisGroup = false;
                         return NULL;
                     }
-                    childBuf.setown(input->nextRow());
+                    childBuf.setown(inputStream->nextRow());
                     if (!childBuf) // eos
                         return NULL;
                 }
@@ -180,7 +164,7 @@ public:
             }
         }
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.unknownRowsOutput = true;
@@ -188,10 +172,11 @@ public:
     }
 };
 
-class CNormalizeLinkedChildSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CNormalizeLinkedChildSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorNormalizeLinkedChildArg *helper;
-    IThorDataLink *input;
     bool anyThisGroup;
 
     OwnedConstThorRow curParent;
@@ -201,7 +186,7 @@ class CNormalizeLinkedChildSlaveActivity : public CSlaveActivity, public CThorDa
     {
         loop
         {
-            curParent.setown(input->nextRow());
+            curParent.setown(inputStream->nextRow());
             if (!curParent)
             {
                 if (anyThisGroup)
@@ -209,7 +194,7 @@ class CNormalizeLinkedChildSlaveActivity : public CSlaveActivity, public CThorDa
                     anyThisGroup = false;
                     return false;
                 }
-                curParent.setown(input->nextRow());
+                curParent.setown(inputStream->nextRow());
                 if (!curParent)
                     return false;
             }
@@ -221,30 +206,21 @@ class CNormalizeLinkedChildSlaveActivity : public CSlaveActivity, public CThorDa
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CNormalizeLinkedChildSlaveActivity(CGraphElementBase *_container) 
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     { 
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         helper = static_cast <IHThorNormalizeLinkedChildArg *> (queryHelper());
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        startInput(input);
+        PARENT::start();
         anyThisGroup = false;
-        dataLinkStart();
-    }
-    void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -268,7 +244,7 @@ public:
             }
         }
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.unknownRowsOutput = true;

thorlcr/activities/nsplitter/thnsplitterslave.cpp

@@ -25,16 +25,11 @@
 interface ISharedSmartBuffer;
 class NSplitterSlaveActivity;
 
-class CSplitterOutputBase : public CSimpleInterface, implements IRowStream
+class CSplitterOutputBase : public CSimpleInterfaceOf<IStartableEngineRowStream>, public COutputTiming
 {
-protected:
-    ActivityTimeAccumulator totalCycles;
 public:
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
-
-    virtual void start() = 0;
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) = 0;
-    virtual unsigned __int64 queryTotalCycles() const { return totalCycles.totalCycles; }
+// IEngineRowStream
+    virtual void resetEOF() { throwUnexpected(); }
 };
 
 class CSplitterOutput : public CSplitterOutputBase
@@ -46,15 +41,11 @@ class CSplitterOutput : public CSplitterOutputBase
     rowcount_t rec, max;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
-
     CSplitterOutput(NSplitterSlaveActivity &_activity, unsigned output);
 
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info);
-
-    virtual void start();
-    virtual void stop();
-    virtual const void *nextRow();
+    virtual void start() override;
+    virtual void stop() override;
+    virtual const void *nextRow() override;
 };
 
 
@@ -64,6 +55,8 @@ public:
 
 class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBufferCallback
 {
+    typedef CSlaveActivity PARENT;
+
     bool spill;
     bool eofHit;
     bool inputsConfigured;
@@ -72,10 +65,12 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf
     PointerArrayOf<Semaphore> stalledWriters;
     unsigned nstopped;
     rowcount_t recsReady;
-    IThorDataLink *input;
-    bool grouped;
     Owned<IException> startException, writeAheadException;
     Owned<ISharedSmartBuffer> smartBuf;
+    bool inputPrepared = false;
+    bool inputConnected = false;
+    IPointerArrayOf<IThorDataLinkExt> delayInputsList;
+
 
     // NB: CWriter only used by 'balanced' splitter, which blocks write when too far ahead
     class CWriter : public CSimpleInterface, IThreaded
@@ -115,103 +110,118 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf
     class CNullInput : public CSplitterOutputBase
     {
     public:
-        virtual const void *nextRow() { throwUnexpected(); return NULL; }
-        virtual void stop() { throwUnexpected(); }
-        virtual void start() { throwUnexpected(); }
-        virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
-        {
-            ::initMetaInfo(info);
-        }
+        virtual void start() override { throwUnexpected(); }
+        virtual const void *nextRow() override { throwUnexpected(); return NULL; }
+        virtual void stop() override { throwUnexpected(); }
     };
     class CInputWrapper : public CSplitterOutputBase
     {
-        IThorDataLink *input;
+        IRowStream *inputStream = nullptr;
         NSplitterSlaveActivity &activity;
 
     public:
-        CInputWrapper(NSplitterSlaveActivity &_activity, IThorDataLink *_input) : activity(_activity), input(_input) { }
-        virtual const void *nextRow()
+        CInputWrapper(NSplitterSlaveActivity &_activity) : activity(_activity) { }
+        virtual void start() override
         {
-            ActivityTimer t(totalCycles, activity.queryTimeActivities());
-            return input->nextRow();
-        }
-        virtual void stop() { input->stop(); }
-        virtual void start()
-        {
-            ActivityTimer s(totalCycles, activity.queryTimeActivities());
-            input->start();
+            activity.start();
+            inputStream = activity.inputStream;
         }
-        virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+        virtual const void *nextRow() override
         {
-            input->getMetaInfo(info);
+            ActivityTimer t(totalCycles, activity.queryTimeActivities());
+            return inputStream->nextRow();
         }
+        virtual void stop() override { activity.stop(); }
     };
-
-    class CDelayedInput : public CSimpleInterface, public CThorDataLink
+    class CDelayedInput : public CSimpleInterfaceOf<IThorDataLinkExt>, public CEdgeProgress, implements IEngineRowStream
     {
-        Owned<CSplitterOutputBase> input;
+        Owned<CSplitterOutputBase> inputStream;
         Linked<NSplitterSlaveActivity> activity;
         mutable SpinLock processActiveLock;
-
-        unsigned id;
+        unsigned outputIdx = 0;
 
     public:
-        IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
+        IMPLEMENT_IINTERFACE_USING(CSimpleInterfaceOf<IThorDataLinkExt>);
 
-        CDelayedInput(NSplitterSlaveActivity &_activity) : CThorDataLink(&_activity), activity(&_activity), id(0) { }
-        void setInput(CSplitterOutputBase *_input, unsigned _id=0)
+        CDelayedInput(NSplitterSlaveActivity &_activity) : CEdgeProgress(&_activity), activity(&_activity) { }
+        void setInput(CSplitterOutputBase *_inputStream)
         {
             SpinBlock b(processActiveLock);
-            input.setown(_input);
-            id = _id;
+            inputStream.setown(_inputStream);
         }
-        virtual const void *nextRow()
+        const void *nextRow()
         {
-            OwnedConstThorRow row = input->nextRow();
+            OwnedConstThorRow row = inputStream->nextRow();
             if (row)
                 dataLinkIncrement();
             return row.getClear();
         }
-        virtual void stop()
+        void stop()
         {
-            input->stop();
+            inputStream->stop();
             dataLinkStop();
         }
+        void resetEOF()
+        {
+            inputStream->resetEOF();
+        }
     // IThorDataLink impl.
         virtual void start()
         {
             activity->ensureInputsConfigured();
-            input->start();
-            dataLinkStart(id);
+            inputStream->start();
+            dataLinkStart();
         }
-        virtual bool isGrouped() { return activity->inputs.item(0)->isGrouped(); }
-        virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+        virtual CSlaveActivity *queryFromActivity() override { return activity; }
+        virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override { activity->getMetaInfo(info); }
+        virtual void dataLinkSerialize(MemoryBuffer &mb) const { CEdgeProgress::dataLinkSerialize(mb); }
+        virtual bool isGrouped() const { return activity->isGrouped(); }
+        virtual IOutputMetaData * queryOutputMeta() const { return activity->queryOutputMeta(); }
+        virtual unsigned queryOutputIdx() const { return outputIdx; }
+        virtual bool isInputOrdered(bool consumerOrdered) const { return activity->isInputOrdered(consumerOrdered); }
+        virtual void setOutputStream(unsigned index, IEngineRowStream *stream) { activity->setOutputStream(index, stream); }
+        virtual IStrandJunction *getOutputStreams(CActivityBase &ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
         {
-            initMetaInfo(info);
-            if (input)
-                input->getMetaInfo(info);
-            else
-                activity->inputs.item(0)->getMetaInfo(info);
-            info.canStall = !activity->spill;
+            activity->connectInput(consumerOrdered);
+            streams.append(this);
+            return NULL;
         }
-        virtual unsigned __int64 queryTotalCycles() const
+        virtual unsigned __int64 queryTotalCycles() const override
         {
             SpinBlock b(processActiveLock);
-            if (!input)
+            if (!inputStream)
                 return 0;
-            return input->queryTotalCycles();
+            return inputStream->queryTotalCycles();
         }
+        virtual unsigned __int64 queryEndCycles() const
+        {
+            SpinBlock b(processActiveLock);
+            return inputStream->queryEndCycles();
+        }
+        virtual void debugRequest(MemoryBuffer &mb) { activity->debugRequest(mb); }
+    // Stepping methods
+        virtual IInputSteppingMeta *querySteppingMeta() { return NULL; }
+        virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) { return false; }
+    // IThorDataLinkExt
+        virtual void setOutputIdx(unsigned idx) override { outputIdx = idx; }
     };
 
-    IPointerArrayOf<CDelayedInput> delayInputsList;
-
+    void connectInput(bool consumerOrdered)
+    {
+        CriticalBlock block(startLock);
+        bool inputOrdered = isInputOrdered(consumerOrdered);
+        if (!inputConnected)
+        {
+            inputConnected = true;
+            connectInputStreams(inputOrdered);
+        }
+    }
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
     NSplitterSlaveActivity(CGraphElementBase *container) : CSlaveActivity(container), writer(*this)
     {
         spill = false;
-        input = NULL;
         nstopped = 0;
         eofHit = inputsConfigured = writeBlocked = pagedOut = false;
         recsReady = 0;
@@ -240,9 +250,9 @@ public:
             assertex(io);
             ForEachItemIn(o2, delayInputsList)
             {
-                CDelayedInput *delayedInput = delayInputsList.item(o2);
+                CDelayedInput *delayedInput = (CDelayedInput *)delayInputsList.item(o2);
                 if (o2 == o)
-                    delayedInput->setInput(new CInputWrapper(*this, inputs.item(0)));
+                    delayedInput->setInput(new CInputWrapper(*this));
                 else
                     delayedInput->setInput(new CNullInput());
             }
@@ -251,9 +261,9 @@ public:
         {
             ForEachItemIn(o, delayInputsList)
             {
-                CDelayedInput *delayedInput = delayInputsList.item(o);
+                CDelayedInput *delayedInput = (CDelayedInput *)delayInputsList.item(o);
                 if (NULL != container.connectedOutputs.queryItem(o))
-                    delayedInput->setInput(new CSplitterOutput(*this, o), o);
+                    delayedInput->setInput(new CSplitterOutput(*this, o));
                 else
                     delayedInput->setInput(new CNullInput());
             }
@@ -263,8 +273,9 @@ public:
     {
         CSlaveActivity::reset();
         nstopped = 0;
-        grouped = false;
         eofHit = false;
+        inputPrepared = false;
+        inputConnected = false;
         recsReady = 0;
         writeBlocked = false;
         stalledWriters.kill();
@@ -273,7 +284,7 @@ public:
             // ensure old inputs cleared, to avoid being reused before re-setup on subsequent executions
             ForEachItemIn(o, delayInputsList)
             {
-                CDelayedInput *delayedInput = delayInputsList.item(o);
+                CDelayedInput *delayedInput = (CDelayedInput *)delayInputsList.item(o);
                 delayedInput->setInput(NULL);
             }
             inputsConfigured = false;
@@ -297,13 +308,12 @@ public:
     void prepareInput(unsigned output)
     {
         CriticalBlock block(startLock);
-        if (!input)
+        if (!inputPrepared)
         {
-            input = inputs.item(0);
+            inputPrepared = true;
             try
             {
-                startInput(input);
-                grouped = input->isGrouped();
+                PARENT::start();
                 nstopped = container.connectedOutputs.getCount();
                 if (smartBuf)
                     smartBuf->reset();
@@ -374,10 +384,10 @@ public:
                 break;
             try
             {
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if (!row)
                 {
-                    row.setown(input->nextRow());
+                    row.setown(inputStream->nextRow());
                     if (row)
                     {
                         smartBuf->putRow(NULL, this); // may call blocked() (see ISharedSmartBufferCallback impl. below)
@@ -403,8 +413,8 @@ public:
         if (nstopped && --nstopped==0) 
         {
             writer.stop();
-            stopInput(input);
-            input = NULL;
+            PARENT::stop();
+            inputPrepared = false;
         }
     }
     void abort()
@@ -413,16 +423,6 @@ public:
         if (smartBuf)
             smartBuf->cancel();
     }
-    unsigned __int64 queryTotalCycles() const
-    {
-        unsigned __int64 _totalCycles = totalCycles.totalCycles; // more() time
-        ForEachItemIn(o, outputs)
-        {
-            IThorDataLink *delayedInput = outputs.item(o);
-            _totalCycles += delayedInput->queryTotalCycles();
-        }
-        return _totalCycles;
-    }
 // ISharedSmartBufferCallback impl.
     virtual void paged() { pagedOut = true; }
     virtual void blocked()
@@ -440,6 +440,23 @@ public:
                 stalledWriters.popGet()->signal();
         }
     }
+
+// IThorDataLink (for output 0)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
+    {
+        calcMetaInfoSize(info, queryInput(0));
+    }
+    virtual unsigned __int64 queryTotalCycles() const override
+    {
+        unsigned __int64 _totalCycles = PARENT::queryTotalCycles(); // more() time
+        ForEachItemIn(o, outputs)
+        {
+            IThorDataLink *delayedInput = outputs.item(o);
+            _totalCycles += delayedInput->queryTotalCycles();
+        }
+        return _totalCycles;
+    }
+
 friend class CInputWrapper;
 friend class CSplitterOutput;
 friend class CWriter;
@@ -454,7 +471,7 @@ CSplitterOutput::CSplitterOutput(NSplitterSlaveActivity &_activity, unsigned _ou
     rec = max = 0;
 }
 
-// IThorDataLink
+// IStartableEngineRowStream
 void CSplitterOutput::start()
 {
     ActivityTimer s(totalCycles, activity.queryTimeActivities());
@@ -464,6 +481,7 @@ void CSplitterOutput::start()
         throw LINK(activity.startException);
 }
 
+// IEngineRowStream
 void CSplitterOutput::stop()
 { 
     CriticalBlock block(activity.startLock);
@@ -481,12 +499,6 @@ const void *CSplitterOutput::nextRow()
     return row;
 }
 
-
-void CSplitterOutput::getMetaInfo(ThorDataLinkMetaInfo &info)
-{
-    CThorDataLink::calcMetaInfoSize(info, activity.inputs.item(0));
-}
-
 CActivityBase *createNSplitterSlave(CGraphElementBase *container)
 {
     return new NSplitterSlaveActivity(container);

thorlcr/activities/null/thnullslave.cpp

@@ -21,8 +21,6 @@
 class CNullSinkSlaveActivity : public ProcessSlaveActivity
 {
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CNullSinkSlaveActivity(CGraphElementBase *container) : ProcessSlaveActivity(container)
     {
     }
@@ -30,55 +28,47 @@ public:
     virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
     {       
     }
-    virtual void process()
+    virtual void process() override
     {
-        startInput(inputs.item(0));
-        stopInput(inputs.item(0));
+        start();
+        stop();
     }
-    virtual void endProcess()
+    virtual void endProcess() override
     {
     }
 };
 
 
-class CNullSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CNullSlaveActivity : public CSlaveActivity
 {
-public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
+    typedef CSlaveActivity PARENT;
 
-    CNullSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+public:
+    CNullSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         appendOutputLinked(this);
     }
 // IThorSlaveActivity
-    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData) override
     {       
     }
 
 // IThorDataLink
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dataLinkStart();
-    }
-
-    virtual void stop()
-    {
-        dataLinkStop();
+        PARENT::start();
     }
-
-    const void * nextRow() 
+    const void * nextRow() override
     {
         ActivityTimer t(totalCycles, timeActivities);
         return NULL;
     }
-
-    virtual bool isGrouped()
+    virtual bool isGrouped() const override
     {
         return queryHelper()->queryOutputMeta()->isGrouped();
     }
-
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.canReduceNumRows = true; // to 0 in fact
@@ -87,12 +77,12 @@ public:
 };
 
 
-class CThroughSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CThroughSlaveActivity : public CSlaveActivity
 {
-public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
+    typedef CSlaveActivity PARENT;
 
-    CThroughSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+public:
+    CThroughSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         appendOutputLinked(this);
     }
@@ -102,29 +92,23 @@ public:
     }
 
 // IThorDataLink
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        startInput(inputs.item(0));
-        dataLinkStart();
-    }
-    virtual void stop()
-    {
-        stopInput(inputs.item(0));
-        dataLinkStop();
+        PARENT::start();
     }
-    const void * nextRow() 
+    const void * nextRow() override
     {
         ActivityTimer t(totalCycles, timeActivities);
-        return inputs.item(0)->nextRow();
+        return inputStream->nextRow();
     }
-    virtual bool isGrouped()
+    virtual bool isGrouped() const override
     {
-        return inputs.item(0)->isGrouped();
+        return queryInput(0)->isGrouped();
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
-        inputs.item(0)->getMetaInfo(info);
+        queryInput(0)->getMetaInfo(info);
     }
 };
 

thorlcr/activities/nullaction/thnullactionslave.cpp

@@ -24,33 +24,31 @@
 
 #include "thnullactionslave.ipp"
 
-class CNullActionSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CNullActionSlaveActivity : public CSlaveActivity
 {
-public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
+    typedef CSlaveActivity PARENT;
 
-    CNullActionSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this) { }
+public:
+    CNullActionSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container) { }
     ~CNullActionSlaveActivity()
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
     } 
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dataLinkStart();
+        PARENT::start();
     }
-    void stop() { dataLinkStop(); }
-
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
         return NULL;
     }
-    bool isGrouped() { return false; }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.totalRowsMin = info.totalRowsMax = 0;

thorlcr/activities/parse/thparseslave.cpp

@@ -28,10 +28,11 @@
 
 #include "thparseslave.ipp"
 
-class CParseSlaveActivity : public CSlaveActivity, public CThorDataLink, implements IMatchedAction
+class CParseSlaveActivity : public CSlaveActivity, implements IMatchedAction
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorParseArg *helper;
-    IThorDataLink *input;
     OwnedConstThorRow curRow;
     Owned<INlpParseAlgorithm> algorithm;
     Owned<INlpParser> parser;
@@ -44,7 +45,7 @@ class CParseSlaveActivity : public CSlaveActivity, public CThorDataLink, impleme
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CParseSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CParseSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         anyThisGroup = false;
         curSearchTextLen = 0;
@@ -56,7 +57,7 @@ public:
         if (helper->searchTextNeedsFree())
             rtlFree(curSearchText);
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         helper = (IHThorParseArg *)queryHelper();
@@ -66,17 +67,10 @@ public:
         rowIter->first();
         allocator.set(queryRowAllocator());
     } 
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        startInput(input);
-        dataLinkStart();
-    }
-    void stop()
-    { 
-        stopInput(input);
-        dataLinkStop();
+        PARENT::start();
     }
     void processRecord(const void * in)
     {
@@ -101,14 +95,14 @@ public:
                 rowIter->next();
                 return r.getClear();
             }
-            curRow.setown(input->nextRow());
+            curRow.setown(inputStream->nextRow());
 
             if (!curRow) {
                 if (anyThisGroup) {
                     anyThisGroup = false;
                     break;
                 }
-                curRow.setown(input->nextRow());
+                curRow.setown(inputStream->nextRow());
                 if (!curRow)
                     break;
             }
@@ -119,11 +113,11 @@ public:
         
         return NULL;
     }
-    bool isGrouped()
+    virtual bool isGrouped() const override
     { 
-        return inputs.item(0)->isGrouped();
+        return queryInput(0)->isGrouped();
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.unknownRowsOutput = true;

thorlcr/activities/piperead/thprslave.cpp

@@ -156,8 +156,6 @@ protected:
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CPipeSlaveBase(CGraphElementBase *_container)
         : CSlaveActivity(_container)
     {
@@ -176,8 +174,10 @@ public:
 
 //---------------------------------------------------------------------------
 
-class CPipeReadSlaveActivity : public CPipeSlaveBase, public CThorDataLink
+class CPipeReadSlaveActivity : public CPipeSlaveBase
 {
+    typedef CPipeSlaveBase PARENT;
+
 protected:
     IHThorPipeReadArg *helper;
     Owned<IThorRowInterfaces> inrowif;
@@ -186,10 +186,8 @@ protected:
     bool eof;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CPipeSlaveBase);
-
     CPipeReadSlaveActivity(CGraphElementBase *_container) 
-        : CPipeSlaveBase(_container), CThorDataLink(this)
+        : CPipeSlaveBase(_container)
     {
     }
     CATCH_NEXTROW()
@@ -247,27 +245,27 @@ public:
         readTransformer.setown(createReadRowStream(_inrowif->queryRowAllocator(), _inrowif->queryRowDeserializer(), helper->queryXmlTransformer(), helper->queryCsvTransformer(), xmlIteratorPath, flags));
         appendOutputLinked(this);
     }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         eof = false;
         OwnedRoxieString pipeProgram(helper->getPipeProgram());
         openPipe(pipeProgram, "PIPEREAD");
-        dataLinkStart();
     }
-    virtual void stop()
+    virtual void stop() override
     {
         readTrailing();
         verifyPipe();
-        dataLinkStop();
+        PARENT::stop();
     }
     virtual void abort()
     {
         CPipeSlaveBase::abort();
         abortPipe();
     }
-    virtual bool isGrouped() { return false; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.isSource = true;
@@ -305,16 +303,18 @@ public:
     }
 
 protected:
-    CPipeThroughSlaveActivity &     activity;
-    IThorDataLink *                 input;
-    Owned<IException>               exc;
+    CPipeThroughSlaveActivity &activity;
+    IEngineRowStream *inputStream;
+    Owned<IException> exc;
 };
 
 
 //---------------------------------------------------------------------------
 
-class CPipeThroughSlaveActivity : public CPipeSlaveBase, public CThorDataLink
+class CPipeThroughSlaveActivity : public CPipeSlaveBase
 {
+    typedef CPipeSlaveBase PARENT;
+
     friend class PipeWriterThread;
 
     IHThorPipeThroughArg *helper;
@@ -339,10 +339,8 @@ class CPipeThroughSlaveActivity : public CPipeSlaveBase, public CThorDataLink
         writeTransformer->writeTranslatedText(row, pipe);
     }
 public:
-    IMPLEMENT_IINTERFACE_USING(CPipeSlaveBase);
-
     CPipeThroughSlaveActivity(CGraphElementBase *_container)
-        : CPipeSlaveBase(_container), CThorDataLink(this)
+        : CPipeSlaveBase(_container)
     {
         pipeWriter = NULL;
         grouped = false;
@@ -351,7 +349,7 @@ public:
     {
         ::Release(pipeWriter);
     }
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         helper = static_cast <IHThorPipeThroughArg *> (queryHelper());
         flags = helper->getPipeFlags();
@@ -364,15 +362,16 @@ public:
 
         appendOutputLinked(this);
     }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         eof = anyThisGroup = inputExhausted = false;
         firstRead = true;
 
         if (!writeTransformer)
         {
-            writeTransformer.setown(createPipeWriteXformHelper(flags, helper->queryXmlOutput(), helper->queryCsvOutput(), ::queryRowInterfaces(inputs.item(0))->queryRowSerializer()));
+            writeTransformer.setown(createPipeWriteXformHelper(flags, helper->queryXmlOutput(), helper->queryCsvOutput(), ::queryRowInterfaces(queryInput(0))->queryRowSerializer()));
             writeTransformer->ready();
         }
         if (!recreate)
@@ -380,8 +379,6 @@ public:
             OwnedRoxieString pipeProgram(helper->getPipeProgram());
             openPipe(pipeProgram, "PIPETHROUGH");
         }
-        startInput(inputs.item(0));
-        dataLinkStart();
         pipeWriter = new PipeWriterThread(*this);
         pipeWriter->start();
     }
@@ -451,20 +448,20 @@ public:
         }
         return NULL;
     }
-    virtual void stop()
+    virtual void stop() override
     {
         abortSoon = true;
         readTrailing();
         if (recreate)
             pipeVerified.signal();
         Owned<IException> wrexc = pipeWriter->joinExc();
-        stopInput(inputs.item(0));
+        PARENT::stop();
         verifyPipe();
-        dataLinkStop();
         if (wrexc)
             throw wrexc.getClear();
         if (retcode!=0 && !(flags & TPFnofail))
             throw MakeActivityException(this, TE_PipeReturnedFailure, "Process returned %d", retcode);
+        PARENT::stop();
     }
     virtual void kill()
     {
@@ -488,8 +485,8 @@ public:
         pipeOpened.signal();
         abortPipe();
     }
-    virtual bool isGrouped() { return grouped; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return grouped; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.isSource = false;
@@ -502,7 +499,7 @@ public:
 PipeWriterThread::PipeWriterThread(CPipeThroughSlaveActivity & _activity)
    : Thread("PipeWriterThread"), activity(_activity)
 {
-    input = activity.inputs.item(0);
+    inputStream = activity.inputStream;
 }
 
 int PipeWriterThread::run()
@@ -518,7 +515,7 @@ int PipeWriterThread::run()
         {
             if (eos||activity.abortSoon)
                 break;
-            OwnedConstThorRow row = input->ungroupedNextRow();
+            OwnedConstThorRow row = inputStream->ungroupedNextRow();
             if (!row.get())
                 break;
             if (activity.recreate)

thorlcr/activities/pipewrite/thpwslave.cpp

@@ -35,7 +35,6 @@ private:
     Owned<IPipeWriteXformHelper> writeTransformer;
     StringAttr pipeCommand;
     bool pipeOpen;
-    IThorDataLink *input;
 
 public:
     CPipeWriteSlaveActivity(CGraphElementBase *container) : ProcessSlaveActivity(container)
@@ -117,8 +116,7 @@ public:
     }
     void process()
     {
-        input = inputs.item(0);
-        startInput(input);
+        start();
         if (!writeTransformer)
         {
             writeTransformer.setown(createPipeWriteXformHelper(helper->getPipeFlags(), helper->queryXmlOutput(), helper->queryCsvOutput(), ::queryRowInterfaces(input)->queryRowSerializer()));
@@ -145,7 +143,7 @@ public:
     {
         if (processed & THORDATALINK_STARTED)
         {
-            stopInput(input);
+            stop();
             processed |= THORDATALINK_STOPPED;
         }
 
@@ -160,7 +158,7 @@ public:
         ActPrintLog("write");           
         while(!abortSoon)
         {
-            OwnedConstThorRow row = input->ungroupedNextRow();
+            OwnedConstThorRow row = inputStream->ungroupedNextRow();
             if (!row) 
                 break;
             if (recreate)

thorlcr/activities/project/thprojectslave.cpp

@@ -19,92 +19,104 @@
 #include "thprojectslave.ipp"
 #include "eclrtl_imp.hpp"
 
-//  IThorDataLink needs only be implemented once, since there is only one output,
-//  therefore may as well implement it here.
 
-class CProjectSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CProjecStrandProcessor : public CThorStrandProcessor
 {
-    IHThorProjectArg * helper;
-    bool anyThisGroup;
-    IThorDataLink *input;
+    IHThorProjectArg *helper;
     Owned<IEngineRowAllocator> allocator;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CProjectSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this) { }
-
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    explicit CProjecStrandProcessor(CThorStrandedActivity &parent, IEngineRowStream *inputStream, unsigned outputId)
+        : CThorStrandProcessor(parent, inputStream, outputId)
     {
-        appendOutputLinked(this);
         helper = static_cast <IHThorProjectArg *> (queryHelper());
-        anyThisGroup = false;
-        allocator.set(queryRowAllocator());
+        Owned<IRowInterfaces> rowIf = parent.getRowInterfaces();
+        allocator.setown(parent.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RoxieHeapFlags)(roxiemem::RHFpacked|roxiemem::RHFunique)));
     }
-    void start()
-    {
-        ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        startInput(input);
-        dataLinkStart();
-    }
-    void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
-    }
-    CATCH_NEXTROW()
+    STRAND_CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
-        loop {
-            OwnedConstThorRow row = input->nextRow();
-            if (!row && !anyThisGroup)
-                row.setown(input->nextRow());
-            if (!row||abortSoon)
-                break;
-            RtlDynamicRowBuilder ret(allocator);
-            size32_t sz;
-            try {
-                sz = helper->transform(ret, row);
+        loop
+        {
+            if (parent.queryAbortSoon())
+                return nullptr;
+            OwnedConstThorRow in = inputStream->nextRow();
+            if (!in)
+            {
+                if (numProcessedLastGroup == rowsProcessed)
+                    in.setown(inputStream->nextRow());
+                if (!in)
+                {
+                    numProcessedLastGroup = rowsProcessed;
+                    return nullptr;
+                }
             }
-            catch (IException *e) 
-            { 
-                ActPrintLog(e, "In helper->transform()");
-                throw; 
+
+            RtlDynamicRowBuilder rowBuilder(allocator);
+            size32_t outSize;
+            try
+            {
+                outSize = helper->transform(rowBuilder, in);
             }
-            catch (CATCHALL)
-            { 
-                ActPrintLog("PROJECT: Unknown exception in helper->transform()"); 
+            catch (IException *e)
+            {
+                parent.ActPrintLog(e, "In helper->transform()");
                 throw;
             }
-            if (sz) {
-                dataLinkIncrement();
-                anyThisGroup = true;
-                return ret.finalizeRowClear(sz);
+            if (outSize)
+            {
+                rowsProcessed++;
+                return rowBuilder.finalizeRowClear(outSize);
             }
         }
-        anyThisGroup = false;
-        return NULL;
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+};
+
+
+//  IThorDataLink needs only be implemented once, since there is only one output,
+//  therefore may as well implement it here.
+
+class CProjectSlaveActivity : public CThorStrandedActivity
+{
+    IHThorProjectArg *helper = nullptr;
+
+public:
+    explicit CProjectSlaveActivity(CGraphElementBase *_container) : CThorStrandedActivity(_container)
+    {
+        appendOutputLinked(this);
+    }
+
+    virtual CThorStrandProcessor *createStrandProcessor(IEngineRowStream *instream) override
+    {
+        return new CProjecStrandProcessor(*this, instream, 0);
+    }
+    virtual CThorStrandProcessor *createStrandSourceProcessor(bool inputOrdered) override { throwUnexpected(); }
+
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
+    {
+        helper = static_cast <IHThorProjectArg *> (queryHelper());
+    }
+
+// IThorDataLink
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.fastThrough = true; // ish
         if (helper->canFilter())
             info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
-
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
 };
 
 
-class CPrefetchProjectSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CPrefetchProjectSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorPrefetchProjectArg *helper;
     rowcount_t numProcessedLastGroup;
     bool eof;
-    IThorDataLink *input;
     Owned<IEngineRowAllocator> allocator;
     IThorChildGraph *child;
     bool parallel;
@@ -144,7 +156,7 @@ class CPrefetchProjectSlaveActivity : public CSlaveActivity, public CThorDataLin
         ~CPrefetcher() { stop(); }
         PrefetchInfo *pullRecord()
         {
-            OwnedConstThorRow row = parent.input->nextRow();
+            OwnedConstThorRow row = parent.inputStream->nextRow();
             if (row)
             {
                 eog = false;
@@ -236,9 +248,7 @@ class CPrefetchProjectSlaveActivity : public CSlaveActivity, public CThorDataLin
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CPrefetchProjectSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this), prefetcher(*this)
+    CPrefetchProjectSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), prefetcher(*this)
     {
         helper = (IHThorPrefetchProjectArg *) queryHelper();
         parallel = 0 != (helper->getFlags() & PPFparallel);
@@ -247,29 +257,26 @@ public:
             preload = 10; // default
         child = helper->queryChild();
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         allocator.set(queryRowAllocator());
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        startInput(input);
+        PARENT::start();
 
         numProcessedLastGroup = getDataLinkGlobalCount();
         eof = !helper->canMatchAny();
         if (parallel)
             prefetcher.start();
-        dataLinkStart();
     }
-    void stop()
+    virtual void stop() override
     {
         if (parallel)
             prefetcher.stop();
-        stopInput(input);
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -310,14 +317,14 @@ public:
         CSlaveActivity::abort();
         prefetcher.abort();
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         if (helper->canFilter())
             info.canReduceNumRows = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
 };
 
 
@@ -326,7 +333,6 @@ CActivityBase *createPrefetchProjectSlave(CGraphElementBase *container)
     return new CPrefetchProjectSlaveActivity(container);
 }
 
-
 CActivityBase *createProjectSlave(CGraphElementBase *container)
 {
     return new CProjectSlaveActivity(container);

thorlcr/activities/pull/thpullslave.cpp

@@ -21,56 +21,48 @@
 #include "thbufdef.hpp"
 #include "thpullslave.ipp"
 
-class PullSlaveActivity : public CSlaveActivity, public CThorDataLink
+class PullSlaveActivity : public CSlaveActivity
 {
-    Owned<IThorDataLink> input;
+    typedef CSlaveActivity PARENT;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    PullSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
-    {
-    }
-    ~PullSlaveActivity() 
+    PullSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
     }
 
 // IThorSlaveActivity overloaded methods
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
     }
 
 // IThorDataLink methods
-    virtual void start()
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
     {
-        ActivityTimer s(totalCycles, timeActivities);
-        input.setown(createDataLinkSmartBuffer(this,inputs.item(0),PULL_SMART_BUFFER_SIZE,true,false,RCUNBOUND,NULL,false,&container.queryJob().queryIDiskUsage()));
-        startInput(input);
-        dataLinkStart();
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), PULL_SMART_BUFFER_SIZE, true, false, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
     }
-    virtual void stop()
+    virtual void start() override
     {
-        stopInput(input);
-        dataLinkStop();
+        ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
     }
-
-    const void * nextRow()
+    const void * nextRow() override
     {
         ActivityTimer t(totalCycles, timeActivities);
-        OwnedConstThorRow row = input->nextRow();
+        OwnedConstThorRow row = inputStream->nextRow();
         if (!row)
             return NULL;
         dataLinkIncrement();
         return row.getClear();
     }
 
-    virtual bool isGrouped() { return false; } // or input->isGrouped?
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; } // or input->isGrouped?
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.buffersInput = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 

thorlcr/activities/result/thresultslave.cpp

@@ -25,11 +25,8 @@
 class CResultSlaveActivity : public ProcessSlaveActivity
 {
     mptag_t masterMpTag;
-    IThorDataLink *input;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CResultSlaveActivity(CGraphElementBase *container) : ProcessSlaveActivity(container) { }
 
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
@@ -38,14 +35,12 @@ public:
     }
     void process()
     {
+        start();
         processed = 0;
 
-        input = inputs.item(0);
-        startInput(input);
-
         processed = THORDATALINK_STARTED;
 
-        OwnedConstThorRow row = input->ungroupedNextRow();
+        OwnedConstThorRow row = inputStream->ungroupedNextRow();
         CMessageBuffer mb;
         DelayedSizeMarker sizeMark(mb);
         if (row)
@@ -62,7 +57,7 @@ public:
     {
         if (processed & THORDATALINK_STARTED)
         {
-            stopInput(input);
+            stop();
             processed |= THORDATALINK_STOPPED;
         }
     }

thorlcr/activities/rollup/throllupslave.cpp

@@ -27,14 +27,15 @@ class CDedupAllHelper : public CSimpleInterface, implements IRowStream
 {
     CActivityBase *activity;
 
-    unsigned dedupCount;
-    const void ** dedupArray;
-    unsigned dedupIdx;
-    IThorDataLink * in;
-    IHThorDedupArg * helper;
-    bool keepLeft;
-    bool * abort;
-    IStopInput *iStopInput;
+    unsigned dedupCount = 0;
+    const void ** dedupArray = nullptr;
+    unsigned dedupIdx = 0;
+    IThorDataLink * in = nullptr;
+    IEngineRowStream *inputStream = nullptr;
+    IHThorDedupArg * helper = nullptr;
+    bool keepLeft = true;
+    bool * abort = nullptr;
+    IStopInput *iStopInput = nullptr;
 
     Owned<IThorRowLoader> rowLoader;
     CThorExpandingRowArray rows;
@@ -84,25 +85,18 @@ public:
 
     CDedupAllHelper(CActivityBase *_activity) : activity(_activity), rows(*_activity, _activity)
     {
-        in = NULL;
-        helper = NULL;
-        abort = NULL;
-        dedupIdx = dedupCount = 0;
-        dedupArray = NULL;
-        iStopInput = NULL;
-        keepLeft = true;
         rowLoader.setown(createThorRowLoader(*activity, NULL, stableSort_none, rc_allMem));
     }
 
-    void init(IThorDataLink * _in, IHThorDedupArg * _helper, bool _keepLeft, bool * _abort, IStopInput *_iStopInput)
+    void init(IThorDataLink * _in, IEngineRowStream *_inputStream, IHThorDedupArg * _helper, bool _keepLeft, bool * _abort, IStopInput *_iStopInput)
     {
-        in = _in;
+        assertex(_in);
+        inputStream = _inputStream;
         helper = _helper;
         keepLeft = _keepLeft;
         abort = _abort;
         iStopInput = _iStopInput;
 
-        assertex(in);
         assertex(helper);
         assertex(abort);
 
@@ -119,7 +113,7 @@ public:
         // JCSMORE - could do in chunks and merge if > mem
         try
         {
-            groupOp ? rowLoader->loadGroup(in, activity->queryAbortSoon(), &rows) : rowLoader->load(in, activity->queryAbortSoon(), false, &rows);
+            groupOp ? rowLoader->loadGroup(inputStream, activity->queryAbortSoon(), &rows) : rowLoader->load(inputStream, activity->queryAbortSoon(), false, &rows);
         }
         catch (IException *e)
         {
@@ -159,6 +153,8 @@ public:
 
 class CDedupRollupBaseActivity : public CSlaveActivity, implements IStopInput
 {
+    typedef CSlaveActivity PARENT;
+
     bool rollup;
     CriticalSection stopsect;
     Linked<IThorRowInterfaces> rowif;
@@ -169,7 +165,6 @@ protected:
     bool groupOp;
     OwnedConstThorRow kept;
     OwnedConstThorRow keptTransformed; // only used by rollup
-    Owned<IThorDataLink> input;
     bool needFirstRow;
 
     unsigned numKept; // not used by rollup
@@ -184,28 +179,27 @@ public:
     virtual void stopInput()
     {
         CriticalBlock block(stopsect);  // can be called async by distribute
-        if (input)
-        {
-            CSlaveActivity::stopInput(input);
-            input.clear();
-        }
+        PARENT::stop();
     }
-    void start()
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
+    {
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        if (global)
+            setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), rollup?ROLLUP_SMART_BUFFER_SIZE:DEDUP_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), false, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage())); // only allow spill if input can stall
+    }
+    virtual void start()
     {
+        PARENT::start();
         needFirstRow = true;
-        input.set(inputs.item(0));
         rowif.set(queryRowInterfaces(input));
         eogNext = eos = false;
         numKept = 0;
-        if (global)
-            input.setown(createDataLinkSmartBuffer(this, input,rollup?ROLLUP_SMART_BUFFER_SIZE:DEDUP_SMART_BUFFER_SIZE,isSmartBufferSpillNeeded(this),false,RCUNBOUND,NULL,false,&container.queryJob().queryIDiskUsage())); // only allow spill if input can stall
-        startInput(input); 
-    }   
-    void stop()
+    }
+    virtual void stop()
     {
         stopInput();
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         if (global) {
             info.canBufferInput = true;
@@ -214,7 +208,7 @@ public:
         info.canReduceNumRows = true;
     }
 
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         if (global)
             mpTag = container.queryJobChannel().deserializeMPTag(data); // only used for global acts
@@ -245,7 +239,7 @@ public:
                 if (kept)
                     return;
             }
-            kept.setown(input->nextRow());
+            kept.setown(inputStream->nextRow());
             if (!kept && global)
                 putNextKept(); // pass on now
             if (rollup)
@@ -288,7 +282,7 @@ public:
     }
 };
 
-class CDedupBaseSlaveActivity : public CDedupRollupBaseActivity, public CThorDataLink
+class CDedupBaseSlaveActivity : public CDedupRollupBaseActivity
 {
 protected:
     IHThorDedupArg *ddhelper;
@@ -296,13 +290,11 @@ protected:
     unsigned numToKeep;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CDedupBaseSlaveActivity(CGraphElementBase *_container, bool global, bool groupOp)
-        : CDedupRollupBaseActivity(_container, false, global, groupOp), CThorDataLink(this)
+        : CDedupRollupBaseActivity(_container, false, global, groupOp)
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         CDedupRollupBaseActivity::init(data, slaveData);
         appendOutputLinked(this);   // adding 'me' to outputs array
@@ -311,23 +303,17 @@ public:
         numToKeep = ddhelper->numToKeep();
         assertex(keepLeft || numToKeep == 1);
     }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         CDedupRollupBaseActivity::start();
-        dataLinkStart();
     }
-    virtual void stop()
-    {
-        CDedupRollupBaseActivity::stop();
-        dataLinkStop();
-    }
-    virtual bool isGrouped() { return groupOp; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return groupOp; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         CDedupRollupBaseActivity::getMetaInfo(info);
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 
@@ -369,11 +355,11 @@ public:
         OwnedConstThorRow next;
         loop
         {
-            next.setown(input->nextRow());
+            next.setown(inputStream->nextRow());
             if (!next)
             {
                 if (!groupOp)
-                    next.setown(input->nextRow());
+                    next.setown(inputStream->nextRow());
                 if (!next)
                 {
                     if (global&&putNextKept()) // send kept to next node
@@ -416,8 +402,6 @@ class CDedupAllSlaveActivity : public CDedupBaseSlaveActivity
     Owned<CDedupAllHelper> dedupHelper;
     bool lastEog;
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CDedupAllSlaveActivity(CGraphElementBase *_container, bool groupOp)
         : CDedupBaseSlaveActivity(_container, false, groupOp)
     {
@@ -436,7 +420,7 @@ public:
         lastEog = false;
         assertex(!global);      // dedup(),local,all only supported
         dedupHelper.setown(new CDedupAllHelper(this));
-        dedupHelper->init(input, ddhelper, keepLeft, &abortSoon, groupOp?NULL:this);
+        dedupHelper->init(input, inputStream, ddhelper, keepLeft, &abortSoon, groupOp?NULL:this);
         dedupHelper->calcNextDedupAll(groupOp);
     }
     CATCH_NEXTROW()
@@ -464,19 +448,17 @@ public:
     }
 };
 
-class CRollupSlaveActivity : public CDedupRollupBaseActivity, public CThorDataLink
+class CRollupSlaveActivity : public CDedupRollupBaseActivity
 {
 private:
     IHThorRollupArg * ruhelper;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CRollupSlaveActivity(CGraphElementBase *_container, bool global, bool groupOp) 
-        : CDedupRollupBaseActivity(_container, true, global, groupOp), CThorDataLink(this)
+        : CDedupRollupBaseActivity(_container, true, global, groupOp)
     {
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         CDedupRollupBaseActivity::init(data, slaveData);
         appendOutputLinked(this);   // adding 'me' to outputs array
@@ -498,13 +480,12 @@ public:
             return true;
         return false;
     }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         CDedupRollupBaseActivity::start();
-        dataLinkStart();
     }
-    virtual void stop()
+    virtual void stop() override
     {
         if (global && !eos) // if stopped early, ensure next informed
         {
@@ -513,7 +494,6 @@ public:
             putNextKept();
         }
         CDedupRollupBaseActivity::stop();
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -525,11 +505,11 @@ public:
         OwnedConstThorRow next;
         loop
         {
-            next.setown(input->nextRow());
+            next.setown(inputStream->nextRow());
             if (!next)
             {
                 if (!groupOp)
-                    next.setown(input->nextRow());
+                    next.setown(inputStream->nextRow());
                 if (!next)
                 {
                     if (global&&putNextKept()) // send kept to next node
@@ -561,30 +541,28 @@ public:
         }
         return NULL;
     }
-    virtual bool isGrouped() { return groupOp; }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return groupOp; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         CDedupRollupBaseActivity::getMetaInfo(info);
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 
-class CRollupGroupSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CRollupGroupSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorRollupGroupArg *helper;
     Owned<IThorRowLoader> groupLoader;
     bool eoi;
-    IThorDataLink *input;
     CThorExpandingRowArray rows;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CRollupGroupSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this), rows(*this, NULL)
+    CRollupGroupSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), rows(*this, NULL)
     {
         eoi = false;
-        input = NULL;
         helper = NULL;
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
@@ -596,15 +574,8 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
+        PARENT::start();
         eoi = false;
-        startInput(input);
-        dataLinkStart();
-    }
-    virtual void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -616,7 +587,7 @@ public:
         {
             loop
             {
-                groupLoader->loadGroup(input, abortSoon, &rows);
+                groupLoader->loadGroup(inputStream, abortSoon, &rows);
                 unsigned count = rows.ordinality();
                 if (0 == count)
                 {
@@ -642,11 +613,11 @@ public:
             throw checkAndCreateOOMContextException(this, e, "loading group for rollup group", groupLoader->numRows(), inputOutputMeta, groupLoader->probeRow(0));
         }
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
         info.canReduceNumRows = true;
     }
 };

thorlcr/activities/sample/thsampleslave.cpp

@@ -18,62 +18,46 @@
 
 #include "thsampleslave.ipp"
 
-class SampleSlaveActivity : public CSlaveActivity, public CThorDataLink
+class SampleSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
 
     IHThorSampleArg * helper;
     unsigned numSamples, whichSample, numToSkip;
     bool anyThisGroup;
     bool eogNext;
-    IThorDataLink *input;
-
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    SampleSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this) { }
+    SampleSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container) { }
 
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         helper = static_cast <IHThorSampleArg *> (queryHelper());
     }
-
-
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        startInput(input);
+        PARENT::start();
         eogNext = false;
         anyThisGroup = false;
         numSamples = helper->getProportion();
         whichSample = helper->getSampleNumber();
         numToSkip = whichSample ? whichSample - 1 : 0;
-        dataLinkStart();
     }
-
-
-    void stop()
-    {
-        dataLinkStop();
-        stopInput(input);
-    }
-
-
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
         while(!abortSoon)
         {
-            OwnedConstThorRow row = input->nextRow();
+            OwnedConstThorRow row = inputStream->nextRow();
             if(!row)    {
                 numToSkip = whichSample ? whichSample - 1 : 0;
                 if(anyThisGroup) {
                     anyThisGroup = false;           
                     break;
                 }
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if(!row) 
                     break;
             }
@@ -87,17 +71,14 @@ public:
         }
         return NULL;
     }
-
-
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.canReduceNumRows = true;
         info.fastThrough = true;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
-
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
 };
 
 CActivityBase *createSampleSlave(CGraphElementBase *container)

thorlcr/activities/selectnth/thselectnthslave.cpp

@@ -19,12 +19,13 @@
 #include "thactivityutil.ipp"
 #include "thbufdef.hpp"
 
-class CSelectNthSlaveActivity : public CSlaveActivity, public CThorDataLink, implements ISmartBufferNotify
+class CSelectNthSlaveActivity : public CSlaveActivity, implements ILookAheadStopNotify
 {
+    typedef CSlaveActivity PARENT;
+
     bool first, isLocal, seenNth;
     rowcount_t lookaheadN, N, startN;
     bool createDefaultIfFail;
-    Owned<IThorDataLink> input;
     IHThorSelectNArg *helper;
     SpinLock spin;
 
@@ -64,14 +65,11 @@ class CSelectNthSlaveActivity : public CSlaveActivity, public CThorDataLink, imp
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CSelectNthSlaveActivity(CGraphElementBase *_container, bool _isLocal) : CSlaveActivity(_container), CThorDataLink(this)
+    CSelectNthSlaveActivity(CGraphElementBase *_container, bool _isLocal) : CSlaveActivity(_container)
     {
         isLocal = _isLocal;
         createDefaultIfFail = isLocal || lastNode();
     }
-    ~CSelectNthSlaveActivity()
-    {
-    }
 
 // IThorSlaveActivity overloaded methods
     virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
@@ -81,20 +79,22 @@ public:
         appendOutputLinked(this);
         helper = static_cast <IHThorSelectNArg *> (queryHelper());
     }
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
+    {
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        rowcount_t rowN = (rowcount_t)helper->getRowToSelect();
+        if (!isLocal && rowN)
+            setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), SELECTN_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), false, rowN, this, &container.queryJob().queryIDiskUsage()));
+    }
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
 
         lookaheadN = RCMAX;
         startN = 0; // set by initN()
-        rowcount_t rowN = (rowcount_t)helper->getRowToSelect();
-        if (!isLocal && rowN)
-            input.setown(createDataLinkSmartBuffer(this, inputs.item(0), SELECTN_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), false, rowN, this, false, &container.queryJob().queryIDiskUsage()));
-        else
-            input.set(inputs.item(0));
         try
         {
-            startInput(input);
+            PARENT::start();
         }
         catch (IException *e)
         {
@@ -103,16 +103,15 @@ public:
             sendN();
             throw;
         }
-        dataLinkStart();
 
         seenNth = false;
         if (0==helper->getRowToSelect())
         {
             ThorDataLinkMetaInfo info;
-            inputs.item(0)->getMetaInfo(info);
+            queryInput(0)->getMetaInfo(info);
             StringBuffer meta;
-            meta.appendf("META(totalRowsMin=%" I64F "d,totalRowsMax=%" I64F "d,rowsOutput=%" RCPF "d,spilled=%" I64F "d,byteTotal=%" I64F "d)",
-                info.totalRowsMin,info.totalRowsMax,info.rowsOutput,info.spilled,info.byteTotal);
+            meta.appendf("META(totalRowsMin=%" I64F "d,totalRowsMax=%" I64F "d, spilled=%" I64F "d,byteTotal=%" I64F "d)",
+                info.totalRowsMin,info.totalRowsMax,info.spilled,info.byteTotal);
 #if 0                 
             Owned<IThorException> e = MakeActivityWarning(this, -1, "%s", meta.str());
             fireException(e);
@@ -122,11 +121,6 @@ public:
         }
         first = true;
     }
-    virtual void stop()
-    {
-        stopInput(input);
-        dataLinkStop();
-    }
     virtual void abort()
     {
         CSlaveActivity::abort();
@@ -155,7 +149,7 @@ public:
                 {
                     while (!abortSoon)
                     {
-                        ret.setown(input->ungroupedNextRow());
+                        ret.setown(inputStream->ungroupedNextRow());
                         if (!ret)
                             break;
                         N--;
@@ -195,17 +189,15 @@ public:
             dataLinkIncrement();
         return ret.getClear();
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.isSequential = true; 
         info.canReduceNumRows = true; // not sure what selectNth is doing
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
-// ISmartBufferNotify methods used for global selectn only
-    virtual void onInputStarted(IException *) { } // not needed
-    virtual bool startAsync() { return false; }
+// IStartableEngineRowStream methods used for global selectn only
     virtual void onInputFinished(rowcount_t count)
     {
         SpinBlock b(spin);

thorlcr/activities/selfjoin/thselfjoinslave.cpp

@@ -31,8 +31,10 @@
 
 #define NUMSLAVEPORTS 2     // actually should be num MP tags
 
-class SelfJoinSlaveActivity : public CSlaveActivity, public CThorDataLink
+class SelfJoinSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 private:
     Owned<IThorSorter> sorter;
     IHThorJoinArg * helper;
@@ -40,7 +42,6 @@ private:
     bool isLocal;
     bool isLightweight;
     bool inputStopped;
-    IThorDataLink * input;
     Owned<IRowStream> strm;     
     ICompare * compare;
     ISortKeySerializer * keyserializer;
@@ -65,10 +66,9 @@ private:
         ActPrintLog("SELFJOIN: Performing local self-join");
 #endif
         Owned<IThorRowLoader> iLoader = createThorRowLoader(*this, ::queryRowInterfaces(input), compare, isUnstable() ? stableSort_none : stableSort_earlyAlloc, rc_mixed, SPILL_PRIORITY_SELFJOIN);
-        Owned<IRowStream> rs = iLoader->load(input, abortSoon);
+        Owned<IRowStream> rs = iLoader->load(inputStream, abortSoon);
         mergeStats(spillStats, iLoader);  // Not sure of the best policy if rs spills later on.
-        stopInput(input);
-        input = NULL;
+        PARENT::stop();
         return rs.getClear();
     }
 
@@ -77,9 +77,8 @@ private:
 #if THOR_TRACE_LEVEL > 5
         ActPrintLog("SELFJOIN: Performing global self-join");
 #endif
-        sorter->Gather(::queryRowInterfaces(input), input, compare, NULL, NULL, keyserializer, NULL, false, isUnstable(), abortSoon, NULL);
-        stopInput(input);
-        input = NULL;
+        sorter->Gather(::queryRowInterfaces(input), inputStream, compare, NULL, NULL, keyserializer, NULL, false, isUnstable(), abortSoon, NULL);
+        PARENT::stop();
         if(abortSoon)
         {
             barrier->cancel();
@@ -95,20 +94,16 @@ private:
 
     IRowStream * doLightweightSelfJoin()
     {
-        IRowStream *ret = LINK(input);
-        input = NULL;
+        IRowStream *ret = LINK(inputStream);
         return ret;
     }
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     SelfJoinSlaveActivity(CGraphElementBase *_container, bool _isLocal, bool _isLightweight)
-        : CSlaveActivity(_container), CThorDataLink(this), spillStats(spillStatistics)
+        : CSlaveActivity(_container), spillStats(spillStatistics)
     {
         isLocal = _isLocal||_isLightweight;
         isLightweight = _isLightweight;
-        input = NULL;
         portbase = 0;
         compare = NULL;
         keyserializer = NULL;
@@ -123,7 +118,7 @@ public:
     }
 
 // IThorSlaveActivity
-    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer & data, MemoryBuffer &slaveData) override
     {       
         appendOutputLinked(this);
         if(!isLocal)
@@ -160,16 +155,15 @@ public:
     }
 
 // IThorDataLink
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        startInput(input);
-        dataLinkStart();
+        PARENT::start();
         bool hintunsortedoutput = getOptBool(THOROPT_UNSORTED_OUTPUT, (JFreorderable & helper->getJoinFlags()) != 0);
         bool hintparallelmatch = getOptBool(THOROPT_PARALLEL_MATCH, hintunsortedoutput); // i.e. unsorted, implies use parallel by default, otherwise no point
 
-        if (helper->getJoinFlags()&JFlimitedprefixjoin) {
+        if (helper->getJoinFlags()&JFlimitedprefixjoin)
+        {
             CriticalBlock b(joinHelperCrit);
             // use std join helper (less efficient but implements limited prefix)
             joinhelper.setown(createJoinHelper(*this, helper, this, hintparallelmatch, hintunsortedoutput));
@@ -182,7 +176,7 @@ public:
         strm.setown(isLightweight? doLightweightSelfJoin() : (isLocal ? doLocalSelfJoin() : doGlobalSelfJoin()));
         assertex(strm);
 
-        joinhelper->init(strm, NULL, ::queryRowAllocator(inputs.item(0)), ::queryRowAllocator(inputs.item(0)), ::queryRowMetaData(inputs.item(0)));
+        joinhelper->init(strm, NULL, ::queryRowAllocator(queryInput(0)), ::queryRowAllocator(queryInput(0)), ::queryRowMetaData(queryInput(0)));
     }
 
     virtual void abort()
@@ -191,13 +185,8 @@ public:
         if (joinhelper)
             joinhelper->stop();
     }
-    virtual void stop()
+    virtual void stop() override
     {
-        if (input)
-        {
-            stopInput(input);
-            input = NULL;
-        }
         if(!isLocal)
         {
             barrier->wait(false);
@@ -209,7 +198,7 @@ public:
         }
         strm->stop();
         strm.clear();
-        dataLinkStop();
+        PARENT::stop();
     }
     
     CATCH_NEXTROW()
@@ -225,8 +214,8 @@ public:
         return NULL;
     }
 
-    virtual bool isGrouped() { return false; }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return false; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.buffersInput = true; 

thorlcr/activities/soapcall/thsoapcallslave.cpp

@@ -32,8 +32,10 @@ static StringBuffer &buildAuthToken(IUserDescriptor *userDesc, StringBuffer &aut
     return authToken;
 }
 
-class CWscRowCallSlaveActivity : public CSlaveActivity, public CThorDataLink, implements IWSCRowProvider
+class CWscRowCallSlaveActivity : public CSlaveActivity, implements IWSCRowProvider
 {
+    typedef CSlaveActivity PARENT;
+
     bool eof;
     Owned<IWSCHelper> wscHelper;
     StringBuffer authToken;
@@ -41,7 +43,7 @@ class CWscRowCallSlaveActivity : public CSlaveActivity, public CThorDataLink, im
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CWscRowCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this) { }
+    CWscRowCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container) { }
 
     // IThorSlaveActivity overloaded methods
     virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
@@ -53,6 +55,7 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         eof = false;
         if (container.queryLocalOrGrouped() || firstNode())
         {
@@ -69,7 +72,6 @@ public:
             }
             wscHelper->start();
         }
-        dataLinkStart();
     }
     virtual void stop()
     {
@@ -78,7 +80,7 @@ public:
             wscHelper->waitUntilDone();
             wscHelper.clear();
         }
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -95,7 +97,7 @@ public:
         eof = true;
         return NULL;
     }
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void abort()
     {
         CSlaveActivity::abort();
@@ -123,17 +125,18 @@ public:
 
 //---------------------------------------------------------------------------
 
-class SoapDatasetCallSlaveActivity : public CSlaveActivity, public CThorDataLink, implements IWSCRowProvider
+class SoapDatasetCallSlaveActivity : public CSlaveActivity, implements IWSCRowProvider
 {
+    typedef CSlaveActivity PARENT;
+
     bool eof;
     Owned<IWSCHelper> wscHelper;
     CriticalSection crit;
-    IThorDataLink *input;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    SoapDatasetCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this) { }
+    SoapDatasetCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container) { }
 
     // IThorSlaveActivity overloaded methods
     virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
@@ -144,20 +147,17 @@ public:
         wscHelper.setown(createSoapCallHelper(this, queryRowAllocator(), authToken.str(), SCdataset, NULL, queryDummyContextLogger(),NULL));
     }
     // IThorDataLink methods
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         eof = false;
-        input = inputs.item(0);
-        startInput(input);
-        dataLinkStart();
         wscHelper->start();
     }
-    virtual void stop()
+    virtual void stop() override
     {
         eof = true;
-        stopInput(input);
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -171,16 +171,16 @@ public:
         eof = true;
         return NULL;
     }
-    virtual bool isGrouped()
+    virtual bool isGrouped() const override
     {
-        return inputs.item(0)->isGrouped();
+        return queryInput(0)->isGrouped();
     }
-    virtual void abort()
+    virtual void abort() override
     {
         CSlaveActivity::abort();
         wscHelper->abort();
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.unknownRowsOutput = true;
@@ -200,7 +200,7 @@ public:
         CriticalBlock b(crit);
         if (eof)
             return NULL;
-        return input->nextRow();
+        return inputStream->nextRow();
     }
     virtual void releaseRow(const void *r)
     {
@@ -264,7 +264,6 @@ class SoapDatasetActionSlaveActivity : public ProcessSlaveActivity, implements I
 {
     Owned<IWSCHelper> wscHelper;
     CriticalSection crit;
-    IThorDataLink *input;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
@@ -282,10 +281,9 @@ public:
     // IThorSlaveProcess overloaded methods
     virtual void process()
     {
+        start();
         processed = 0;
 
-        input = inputs.item(0);
-        startInput(input);
         processed = THORDATALINK_STARTED;
 
         wscHelper->start();
@@ -298,7 +296,7 @@ public:
     {
         if (processed & THORDATALINK_STARTED)
         {
-            stopInput(input);
+            stop();
             processed |= THORDATALINK_STOPPED;
         }
     }
@@ -322,7 +320,7 @@ public:
         if (abortSoon)
             return NULL;
 
-        const void *row = input->nextRow();
+        const void *row = inputStream->nextRow();
         if (!row) return NULL;
         processed++;
         return row;

thorlcr/activities/spill/thspillslave.cpp

@@ -28,9 +28,10 @@
 #include "thactivityutil.ipp"
 #include "thspillslave.ipp"
 
-class SpillSlaveActivity : public CSlaveActivity, public CThorDataLink
+class SpillSlaveActivity : public CSlaveActivity
 {
-    IThorDataLink *input;
+    typedef CSlaveActivity PARENT;
+
     StringBuffer fileName;
     Owned<IPartDescriptor> partDesc;
     Owned<IExtRowWriter> out;
@@ -43,9 +44,7 @@ class SpillSlaveActivity : public CSlaveActivity, public CThorDataLink
     unsigned usageCount;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    SpillSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    SpillSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         compress = false;
         grouped = false;
@@ -156,14 +155,11 @@ public:
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         uncompressedBytesWritten = 0;
         if (!container.queryJob().queryUseCheckpoints())
             container.queryTempHandler()->registerFile(fileName.str(), container.queryOwner().queryGraphId(), usageCount, true);
-        input = inputs.item(0);
-        startInput(input);
         
-        dataLinkStart();
-
         open();
         hadrow = false;
     }
@@ -171,8 +167,7 @@ public:
     {
         readRest();
         close();
-        stopInput(input);
-        dataLinkStop();
+        PARENT::stop();
     }
 
     CATCH_NEXTROW()
@@ -181,7 +176,7 @@ public:
         if (abortSoon) 
             return NULL;
 
-        OwnedConstThorRow row = grouped?input->nextRow():input->ungroupedNextRow();
+        OwnedConstThorRow row = grouped?inputStream->nextRow():inputStream->ungroupedNextRow();
         if (row) {
             hadrow = true;
             dataLinkIncrement();
@@ -194,12 +189,12 @@ public:
         return NULL;
     }
 
-    virtual bool isGrouped() { return grouped; }
+    virtual bool isGrouped() const override { return grouped; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         initMetaInfo(info);
         info.fastThrough = true; // ish
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 
 };

thorlcr/activities/temptable/thtmptableslave.cpp

@@ -28,8 +28,10 @@
  * up consuming a lot of rows.
  *
  */
-class CInlineTableSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CInlineTableSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
 private:
     IHThorInlineTableArg * helper;
     __uint64 startRow;
@@ -37,26 +39,24 @@ private:
     __uint64 maxRow;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CInlineTableSlaveActivity(CGraphElementBase *_container)
-    : CSlaveActivity(_container), CThorDataLink(this)
+    : CSlaveActivity(_container)
     {
         helper = NULL;
         startRow = 0;
         currentRow = 0;
         maxRow = 0;
     }
-    virtual bool isGrouped() { return false; }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual bool isGrouped() const override { return false; }
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         helper = static_cast <IHThorInlineTableArg *> (queryHelper());
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dataLinkStart();
+        PARENT::start();
         __uint64 numRows = helper->numRows();
         // local when generated from a child query (the range is per node, don't split)
         bool isLocal = container.queryLocalData() || container.queryOwner().isLocalChild();
@@ -81,10 +81,6 @@ public:
         }
         currentRow = startRow;
     }
-    void stop()
-    {
-        dataLinkStop();
-    }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
@@ -101,7 +97,7 @@ public:
         }
         return NULL;
     }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.isSource = true;

thorlcr/activities/thactivityutil.cpp

@@ -50,34 +50,29 @@
 #define JOIN_TIMEOUT (10*60*1000)
 
 
-#ifdef _MSC_VER
-#pragma warning(push)
-#pragma warning( disable : 4355 )
-#endif
-class ThorLookaheadCache: public IThorDataLink, public CSimpleInterface
+
+class CRowStreamLookAhead : public CSimpleInterfaceOf<IStartableEngineRowStream>
 {
     rowcount_t count;
-    Linked<IThorDataLink> in;
+    Linked<IEngineRowStream> inputStream;
+    IThorRowInterfaces *rowIf;
     Owned<ISmartRowBuffer> smartbuf;
     size32_t bufsize;
-    CActivityBase &activity;
+    CSlaveActivity &activity;
     bool allowspill, preserveGrouping;
-    ISmartBufferNotify *notify;
+    ILookAheadStopNotify *notify;
     bool running;
     bool stopped;
     rowcount_t required;
-    Semaphore startsem;
-    bool started;
-    Owned<IException> startexception;
+    Semaphore startSem;
     Owned<IException> getexception;
-    bool asyncstart;
 
-    class Cthread: public Thread
+    class CThread: public Thread
     {
-        ThorLookaheadCache &parent;
+        CRowStreamLookAhead &parent;
     public:
-        Cthread(ThorLookaheadCache &_parent)
-            : Thread("ThorLookaheadCache"), parent(_parent)
+        CThread(CRowStreamLookAhead &_parent)
+            : Thread("CRowStreamLookAhead"), parent(_parent)
         {
         }
         int run()
@@ -87,8 +82,6 @@ class ThorLookaheadCache: public IThorDataLink, public CSimpleInterface
     } thread;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
-
     void doNotify()
     {
         if (notify)
@@ -99,44 +92,26 @@ public:
 
     int run()
     {
-        if (!started) {
-            try {
-                in->start();
-                started = true;
-            }
-            catch(IException * e)
-            {
-                ActPrintLog(&activity, e, "ThorLookaheadCache starting input");
-                startexception.setown(e);
-                if (asyncstart) 
-                    notify->onInputStarted(startexception);
-                running = false;
-                stopped = true;
-                startsem.signal();
-                return 0;
-            }
-        }
-        try {
+        try
+        {
             StringBuffer temp;
             if (allowspill)
                 GetTempName(temp,"lookahd",true);
             assertex(bufsize);
             if (allowspill)
-                smartbuf.setown(createSmartBuffer(&activity, temp.str(), bufsize, queryRowInterfaces(in)));
+                smartbuf.setown(createSmartBuffer(&activity, temp.str(), bufsize, rowIf));
             else
-                smartbuf.setown(createSmartInMemoryBuffer(&activity, queryRowInterfaces(in), bufsize));
-            if (notify) 
-                notify->onInputStarted(NULL);
-            startsem.signal();
+                smartbuf.setown(createSmartInMemoryBuffer(&activity, rowIf, bufsize));
+            startSem.signal();
             IRowWriter *writer = smartbuf->queryWriter();
             if (preserveGrouping)
             {
                 while (required&&running)
                 {
-                    OwnedConstThorRow row = in->nextRow();
+                    OwnedConstThorRow row = inputStream->nextRow();
                     if (!row)
                     {
-                        row.setown(in->nextRow());
+                        row.setown(inputStream->nextRow());
                         if (!row)
                             break;
                         else
@@ -152,7 +127,7 @@ public:
             {
                 while (required&&running)
                 {
-                    OwnedConstThorRow row = in->ungroupedNextRow();
+                    OwnedConstThorRow row = inputStream->ungroupedNextRow();
                     if (!row)
                         break;
                     ++count;
@@ -164,7 +139,8 @@ public:
         }
         catch(IException * e)
         {
-            ActPrintLog(&activity, e, "ThorLookaheadCache get exception");
+            startSem.signal();
+            ActPrintLog(&activity, e, "CRowStreamLookAhead get exception");
             getexception.setown(e);
         }
 
@@ -177,9 +153,9 @@ public:
         class CNotifyThread : implements IThreaded
         {
             CThreaded threaded;
-            ThorLookaheadCache &owner;
+            CRowStreamLookAhead &owner;
         public:
-            CNotifyThread(ThorLookaheadCache &_owner) : threaded("Lookahead-CNotifyThread"), owner(_owner)
+            CNotifyThread(CRowStreamLookAhead &_owner) : threaded("Lookahead-CNotifyThread"), owner(_owner)
             {
                 threaded.init(this);
             }
@@ -202,27 +178,25 @@ public:
         running = false;
         try
         {
-            if (in)
-                in->stop();
+            if (inputStream)
+                inputStream->stop();
         }
         catch(IException * e)
         {
-            ActPrintLog(&activity, e, "ThorLookaheadCache stop exception");
+            ActPrintLog(&activity, e, "CRowStreamLookAhead stop exception");
             if (!getexception.get())
                 getexception.setown(e);
         }
         // NB: Will wait on CNotifyThread to finish before returning
         return 0;
     }
-        
 
-    ThorLookaheadCache(CActivityBase &_activity, IThorDataLink *_in,size32_t _bufsize,bool _allowspill,bool _preserveGrouping, rowcount_t _required,ISmartBufferNotify *_notify, bool _instarted, IDiskUsage *_iDiskUsage)
-        : thread(*this), activity(_activity), in(_in)
+    CRowStreamLookAhead(CSlaveActivity &_activity, IEngineRowStream *_inputStream, IThorRowInterfaces *_rowIf, size32_t _bufsize, bool _allowspill, bool _preserveGrouping, rowcount_t _required, ILookAheadStopNotify *_notify, IDiskUsage *_iDiskUsage)
+        : thread(*this), activity(_activity), inputStream(_inputStream), rowIf(_rowIf)
     {
 #ifdef _FULL_TRACE
-        ActPrintLog(&activity, "ThorLookaheadCache create %x",(unsigned)(memsize_t)this);
+        ActPrintLog(&activity, "CRowStreamLookAhead create %x",(unsigned)(memsize_t)this);
 #endif
-        asyncstart = false;
         allowspill = _allowspill;
         preserveGrouping = _preserveGrouping;
         assertex((unsigned)-1 != _bufsize); // no longer supported
@@ -232,91 +206,67 @@ public:
         required = _required;
         count = 0;
         stopped = true;
-        started = _instarted;
     }
-
-    ~ThorLookaheadCache()
+    ~CRowStreamLookAhead()
     {
         if (!thread.join(1000*60))
-            ActPrintLogEx(&activity.queryContainer(), thorlog_all, MCuserWarning, "ThorLookaheadCache join timedout");
+            ActPrintLogEx(&activity.queryContainer(), thorlog_all, MCuserWarning, "CRowStreamLookAhead join timedout");
+    }
+// IEngineRowStream
+    virtual const void *nextRow() override
+    {
+        OwnedConstThorRow row = smartbuf->nextRow();
+        if (getexception)
+            throw getexception.getClear();
+        if (!row)
+        {
+#ifdef _FULL_TRACE
+            ActPrintLog(&activity, "CRowStreamLookAhead eos %x",(unsigned)(memsize_t)this);
+#endif
+        }
+        return row.getClear();
     }
 
-    void start()
+// IStartableEngineRowStream
+    virtual void start() override
     {
 #ifdef _FULL_TRACE
-        ActPrintLog(&activity, "ThorLookaheadCache start %x",(unsigned)(memsize_t)this);
+        ActPrintLog(&activity, "CRowStreamLookAhead start %x",(unsigned)(memsize_t)this);
 #endif
         stopped = false;
-        asyncstart = notify&&notify->startAsync();
         thread.start();
-        if (!asyncstart) {
-            startsem.wait();
-            if (startexception) 
-                throw startexception.getClear();
-        }
+        startSem.wait();
     }
-
-    void stop()
+// IEngineRowStream
+    virtual void resetEOF() override { throwUnexpected(); }
+// IRowStream
+    virtual void stop() override
     {
 #ifdef _FULL_TRACE
-        ActPrintLog(&activity, "ThorLookaheadCache stop %x",(unsigned)(memsize_t)this);
+        ActPrintLog(&activity, "CRowStreamLookAhead stop %x",(unsigned)(memsize_t)this);
 #endif
-        if (!stopped) {
+        if (!stopped)
+        {
             running = false;
             if (smartbuf)
                 smartbuf->stop(); // just in case blocked
             thread.join();
             stopped = true;
-            if (getexception) 
+            if (getexception)
                 throw getexception.getClear();
         }
     }
-
-    const void *nextRow()
-    {
-        OwnedConstThorRow row = smartbuf->nextRow();
-        if (getexception) 
-            throw getexception.getClear();
-        if (!row) {
-#ifdef _FULL_TRACE
-            ActPrintLog(&activity, "ThorLookaheadCache eos %x",(unsigned)(memsize_t)this);
-#endif
-        }
-        return row.getClear();
-    }
-
-    bool isGrouped() { return preserveGrouping; }
-            
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
-    {
-        memset(&info,0,sizeof(info));
-        in->getMetaInfo(info);
-        // more TBD
-    }
-
-    CActivityBase *queryFromActivity()
-    {
-        return in->queryFromActivity();
-    }
-    void dataLinkSerialize(MemoryBuffer &mb)
-    {
-        // no serialization information (yet)
-    }
-    unsigned __int64 queryTotalCycles() const { return in->queryTotalCycles(); }
-    unsigned __int64 queryEndCycles() const { return in->queryEndCycles(); }
-    virtual void debugRequest(MemoryBuffer &msg) { return in->debugRequest(msg); }
 };
 #ifdef _MSC_VER
 #pragma warning(pop)
 #endif
 
 
-IThorDataLink *createDataLinkSmartBuffer(CActivityBase *activity, IThorDataLink *in, size32_t bufsize, bool allowspill, bool preserveGrouping, rowcount_t maxcount, ISmartBufferNotify *notify, bool instarted, IDiskUsage *iDiskUsage)
+IStartableEngineRowStream *createRowStreamLookAhead(CSlaveActivity *activity, IEngineRowStream *inputStream, IThorRowInterfaces *rowIf, size32_t bufsize, bool allowspill, bool preserveGrouping, rowcount_t maxcount, ILookAheadStopNotify *notify, IDiskUsage *iDiskUsage)
 {
-    return new ThorLookaheadCache(*activity, in,bufsize,allowspill,preserveGrouping,maxcount,notify,instarted,iDiskUsage);
+    return new CRowStreamLookAhead(*activity, inputStream, rowIf, bufsize, allowspill, preserveGrouping, maxcount, notify, iDiskUsage);
 }
 
-
 void initMetaInfo(ThorDataLinkMetaInfo &info)
 {
     memset(&info,0,sizeof(info));
@@ -325,32 +275,25 @@ void initMetaInfo(ThorDataLinkMetaInfo &info)
     info.totalRowsMax = -1; // rely on inputs to set
     info.spilled = (offset_t)-1;
     info.byteTotal = (offset_t)-1;
-    info.rowsOutput = 0;
 }
 
-
-
-
-void CThorDataLink::initMetaInfo(ThorDataLinkMetaInfo &info)
+void calcMetaInfoSize(ThorDataLinkMetaInfo &info, IThorDataLink *link)
 {
-    ::initMetaInfo(info);
-    info.rowsOutput = getDataLinkCount();
-    // more
-}
-
-void CThorDataLink::calcMetaInfoSize(ThorDataLinkMetaInfo &info,IThorDataLink *link)
-{
-    if (!info.unknownRowsOutput&&link&&((info.totalRowsMin<=0)||(info.totalRowsMax<0))) {
+    if (!info.unknownRowsOutput&&link&&((info.totalRowsMin<=0)||(info.totalRowsMax<0)))
+    {
         ThorDataLinkMetaInfo prev;
         link->getMetaInfo(prev);
-        if (info.totalRowsMin<=0) {
+        if (info.totalRowsMin<=0)
+        {
             if (!info.canReduceNumRows)
                 info.totalRowsMin = prev.totalRowsMin;
             else
                 info.totalRowsMin = 0;
         }
-        if (info.totalRowsMax<0) {
-            if (!info.canIncreaseNumRows) {
+        if (info.totalRowsMax<0)
+        {
+            if (!info.canIncreaseNumRows)
+            {
                 info.totalRowsMax = prev.totalRowsMax;
                 if (info.totalRowsMin>info.totalRowsMax)
                     info.totalRowsMax = -1;
@@ -364,26 +307,40 @@ void CThorDataLink::calcMetaInfoSize(ThorDataLinkMetaInfo &info,IThorDataLink *l
 
 }
 
-void CThorDataLink::calcMetaInfoSize(ThorDataLinkMetaInfo &info,IThorDataLink **link,unsigned ninputs)
+void calcMetaInfoSize(ThorDataLinkMetaInfo &info, CThorInputArray &inputs)
 {
-    if (!link||(ninputs<=1)) {
-        calcMetaInfoSize(info,link&&(ninputs==1)?link[0]:NULL);
-        return ;
+    //IThorDataLink **link,unsigned ninputs;
+
+    if (0 == inputs.ordinality())
+    {
+        calcMetaInfoSize(info, nullptr);
+        return;
     }
-    if (!info.unknownRowsOutput) {
+    else if (1 == inputs.ordinality())
+    {
+        calcMetaInfoSize(info, inputs.item(0).itdl);
+        return;
+    }
+    if (!info.unknownRowsOutput)
+    {
         __int64 min=0;
         __int64 max=0;
-        for (unsigned i=0;i<ninputs;i++ ) {
-            if (link[i]) {
+        for (unsigned i=0;i<inputs.ordinality();i++ )
+        {
+            CThorInput &input = inputs.item(i);
+            if (input.itdl)
+            {
                 ThorDataLinkMetaInfo prev;
-                link[i]->getMetaInfo(prev);
-                if (min>=0) {
+                input.itdl->getMetaInfo(prev);
+                if (min>=0)
+                {
                     if (prev.totalRowsMin>=0)
                         min += prev.totalRowsMin;
                     else
                         min = -1;
                 }
-                if (max>=0) {
+                if (max>=0)
+                {
                     if (prev.totalRowsMax>=0)
                         max += prev.totalRowsMax;
                     else
@@ -391,14 +348,17 @@ void CThorDataLink::calcMetaInfoSize(ThorDataLinkMetaInfo &info,IThorDataLink **
                 }
             }
         }
-        if (info.totalRowsMin<=0) {
+        if (info.totalRowsMin<=0)
+        {
             if (!info.canReduceNumRows)
                 info.totalRowsMin = min;
             else
                 info.totalRowsMin = 0;
         }
-        if (info.totalRowsMax<0) {
-            if (!info.canIncreaseNumRows) {
+        if (info.totalRowsMax<0)
+        {
+            if (!info.canIncreaseNumRows)
+            {
                 info.totalRowsMax = max;
                 if (info.totalRowsMin>info.totalRowsMax)
                     info.totalRowsMax = -1;
@@ -409,39 +369,47 @@ void CThorDataLink::calcMetaInfoSize(ThorDataLinkMetaInfo &info,IThorDataLink **
         info.totalRowsMin = 0; // a good bet
 }
 
-void CThorDataLink::calcMetaInfoSize(ThorDataLinkMetaInfo &info, ThorDataLinkMetaInfo *infos,unsigned num)
+void calcMetaInfoSize(ThorDataLinkMetaInfo &info, ThorDataLinkMetaInfo *infos, unsigned num)
 {
-    if (!infos||(num<=1)) {
+    if (!infos||(num<=1))
+    {
         if (1 == num)
             info = infos[0];
         return;
     }
-    if (!info.unknownRowsOutput) {
+    if (!info.unknownRowsOutput)
+    {
         __int64 min=0;
         __int64 max=0;
-        for (unsigned i=0;i<num;i++ ) {
+        for (unsigned i=0;i<num;i++ )
+        {
             ThorDataLinkMetaInfo &prev = infos[i];
-            if (min>=0) {
+            if (min>=0)
+            {
                 if (prev.totalRowsMin>=0)
                     min += prev.totalRowsMin;
                 else
                     min = -1;
             }
-            if (max>=0) {
+            if (max>=0)
+            {
                 if (prev.totalRowsMax>=0)
                     max += prev.totalRowsMax;
                 else
                     max = -1;
             }
         }
-        if (info.totalRowsMin<=0) {
+        if (info.totalRowsMin<=0)
+        {
             if (!info.canReduceNumRows)
                 info.totalRowsMin = min;
             else
                 info.totalRowsMin = 0;
         }
-        if (info.totalRowsMax<0) {
-            if (!info.canIncreaseNumRows) {
+        if (info.totalRowsMax<0)
+        {
+            if (!info.canIncreaseNumRows)
+            {
                 info.totalRowsMax = max;
                 if (info.totalRowsMin>info.totalRowsMax)
                     info.totalRowsMax = -1;

thorlcr/activities/thactivityutil.ipp

@@ -58,7 +58,7 @@ IRowStream *createSequentialPartHandler(CPartHandler *partHandler, IArrayOf<IPar
         }
 
 #define CATCH_NEXTROW() \
-    const void *nextRow() \
+    virtual const void *nextRow() override \
     { \
         try \
         { \
@@ -69,121 +69,18 @@ IRowStream *createSequentialPartHandler(CPartHandler *partHandler, IArrayOf<IPar
     inline const void *nextRowNoCatch() __attribute__((always_inline))
 
 void initMetaInfo(ThorDataLinkMetaInfo &info);
-class CThorDataLink : implements IThorDataLink
-{
-    CActivityBase *owner;
-    rowcount_t count, icount;
-    unsigned outputId;
-    unsigned limit;
-
-protected:
-    inline void dataLinkStart(unsigned _outputId = 0)
-    {
-        outputId = _outputId;
-#ifdef _TESTING
-        ActPrintLog(owner, "ITDL starting for output %d", outputId);
-#endif
-#ifdef _TESTING
-        assertex(!started() || stopped());      // ITDL started twice
-#endif
-        icount = 0;
-//      count = THORDATALINK_STARTED;
-        rowcount_t prevCount = count & THORDATALINK_COUNT_MASK;
-        count = prevCount | THORDATALINK_STARTED;
-    }
-
-    inline void dataLinkStop()
-    {
-#ifdef _TESTING
-        assertex(started());        // ITDL stopped without being started
-#endif
-        count |= THORDATALINK_STOPPED;
-#ifdef _TESTING
-        ActPrintLog(owner, "ITDL output %d stopped, count was %" RCPF "d", outputId, getDataLinkCount());
-#endif
-    }
-
-    inline void dataLinkIncrement()
-    {
-        dataLinkIncrement(1);
-    }
-
-    inline void dataLinkIncrement(rowcount_t v)
-    {
-#ifdef _TESTING
-        assertex(started());
-#ifdef OUTPUT_RECORDSIZE
-        if (count==THORDATALINK_STARTED) {
-            size32_t rsz = queryRowMetaData(this)->getMinRecordSize();
-            ActPrintLog(owner, "Record size %s= %d", queryRowMetaData(this)->isVariableSize()?"(min) ":"",rsz);
-        }   
-#endif
-#endif
-        icount += v;
-        count += v; 
-    }
-
-    inline bool started()
-    {
-        return (count & THORDATALINK_STARTED) ? true : false; 
-    }
-
-    inline bool stopped()
-    {
-        return (count & THORDATALINK_STOPPED) ? true : false;
-    }
-
+void calcMetaInfoSize(ThorDataLinkMetaInfo &info, IThorDataLink *link);
+void calcMetaInfoSize(ThorDataLinkMetaInfo &info, CThorInputArray &inputs);
+void calcMetaInfoSize(ThorDataLinkMetaInfo &info, ThorDataLinkMetaInfo *infos, unsigned num);
 
-public:
-    CThorDataLink(CActivityBase *_owner) : owner(_owner)
-    {
-        icount = count = 0;
-    }
-#ifdef _TESTING
-    ~CThorDataLink()
-    { 
-        if(started()&&!stopped())
-        {
-            ActPrintLog(owner, "ERROR: ITDL was not stopped before destruction");
-            dataLinkStop(); // get some info (even though failed)       
-        }
-    }           
-#endif
-
-    void dataLinkSerialize(MemoryBuffer &mb)
-    {
-        mb.append(count);
-    }
-
-    unsigned __int64 queryTotalCycles() const { return ((CSlaveActivity *)owner)->queryTotalCycles(); }
-    unsigned __int64 queryEndCycles() const  { return ((CSlaveActivity *)owner)->queryEndCycles(); }
-
-    inline rowcount_t getDataLinkGlobalCount()
-    {
-        return (count & THORDATALINK_COUNT_MASK); 
-    } 
-    inline rowcount_t getDataLinkCount()
-    {
-        return icount; 
-    } 
-    virtual void debugRequest(MemoryBuffer &msg) { }
-    CActivityBase *queryFromActivity() { return owner; }
-
-    void initMetaInfo(ThorDataLinkMetaInfo &info); // for derived children to call from getMetaInfo
-    static void calcMetaInfoSize(ThorDataLinkMetaInfo &info,IThorDataLink *input); // for derived children to call from getMetaInfo
-    static void calcMetaInfoSize(ThorDataLinkMetaInfo &info,IThorDataLink **link,unsigned ninputs);
-    static void calcMetaInfoSize(ThorDataLinkMetaInfo &info, ThorDataLinkMetaInfo *infos,unsigned num);
-};
-
-interface ISmartBufferNotify
+interface ILookAheadStopNotify
 {
-    virtual bool startAsync() =0;                       // return true if need to start asynchronously
-    virtual void onInputStarted(IException *e) =0;      // e==NULL if start suceeded, NB only called with exception if Async
-    virtual void onInputFinished(rowcount_t count) =0;
+    virtual void onInputFinished(rowcount_t count) = 0;
 };
-
 interface IDiskUsage;
-IThorDataLink *createDataLinkSmartBuffer(CActivityBase *activity,IThorDataLink *in,size32_t bufsize,bool spillenabled,bool preserveGrouping=true,rowcount_t maxcount=RCUNBOUND,ISmartBufferNotify *notify=NULL, bool inputstarted=false, IDiskUsage *_diskUsage=NULL); //maxcount is maximum rows to read set to RCUNBOUND for all
+IStartableEngineRowStream *createRowStreamLookAhead(CSlaveActivity *activity, IEngineRowStream *inputStream, IThorRowInterfaces *rowIf, size32_t bufsize, bool spillenabled, bool preserveGrouping=true, rowcount_t maxcount=RCUNBOUND, ILookAheadStopNotify *notify=NULL, IDiskUsage *_diskUsage=NULL); //maxcount is maximum rows to read set to RCUNBOUND for all
+
+
 
 bool isSmartBufferSpillNeeded(CActivityBase *act);
 
@@ -199,5 +96,25 @@ void cancelReplicates(CActivityBase *activity, IPartDescriptor &partDesc);
 interface IPartDescriptor;
 IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc, unsigned recordSize, unsigned twFlags, bool &compress, ICompressor *ecomp, ICopyFileProgress *iProgress, bool *aborted, StringBuffer *_locationName=NULL);
 
+class CAsyncCall : implements IThreaded
+{
+    CThreaded threaded;
+    std::function<void()> func;
+public:
+    CAsyncCall(std::function<void()> _func) : threaded("CAsyncCall", this), func(_func) { }
+    void start() { threaded.start(); }
+    void wait() { threaded.join(); }
+// IThreaded
+    virtual void main() { func(); }
+};
+
+class CAsyncCallStart : public CAsyncCall
+{
+public:
+    CAsyncCallStart(std::function<void()> func) : CAsyncCall(func)
+    {
+        start();
+    }
+};
 
 #endif

thorlcr/activities/thdiskbaseslave.cpp

@@ -39,7 +39,7 @@
 
 #include "thdiskbaseslave.ipp"
 
-void getPartsMetaInfo(ThorDataLinkMetaInfo &metaInfo, CThorDataLink &link, unsigned nparts, IPartDescriptor **partDescs, CDiskPartHandlerBase *partHandler)
+void getPartsMetaInfo(ThorDataLinkMetaInfo &metaInfo, unsigned nparts, IPartDescriptor **partDescs, CDiskPartHandlerBase *partHandler)
 {
     ThorDataLinkMetaInfo *metaInfos = new ThorDataLinkMetaInfo[nparts];
     struct ownedMetaInfos
@@ -52,11 +52,11 @@ void getPartsMetaInfo(ThorDataLinkMetaInfo &metaInfo, CThorDataLink &link, unsig
     unsigned p=0;
     for (; p<nparts; p++)
     {
-        link.initMetaInfo(metaInfos[p]);
+        initMetaInfo(metaInfos[p]);
         partHandler->getMetaInfo(metaInfos[p], partDescs[p]);
         sizeTotal += partDescs[p]->queryProperties().getPropInt64("@size");
     }
-    link.calcMetaInfoSize(metaInfo, metaInfos, nparts);
+    calcMetaInfoSize(metaInfo, metaInfos, nparts);
     if (!metaInfo.unknownRowsOutput && !metaInfo.canReduceNumRows && !metaInfo.canIncreaseNumRows)
         metaInfo.byteTotal = sizeTotal;
 }
@@ -251,6 +251,7 @@ const char *CDiskReadSlaveActivityBase::queryLogicalFilename(unsigned index)
 
 void CDiskReadSlaveActivityBase::start()
 {
+    PARENT::start();
     markStart = true;
     diskProgress = 0;
 }
@@ -294,12 +295,18 @@ void CDiskReadSlaveActivityBase::serializeStats(MemoryBuffer &mb)
 
 /////////////////
 
+void CDiskWriteSlaveActivityBase::setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered)
+{
+    PARENT::setInputStream(index, _input, consumerOrdered);
+    if (dlfn.isExternal() && !firstNode())
+        setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
+}
+
 void CDiskWriteSlaveActivityBase::open()
 {
+    start();
     if (dlfn.isExternal() && !firstNode())
     {
-        input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
-        startInput(input);
         if (!rfsQueryParallel)
         {
             ActPrintLog("Blocked, waiting for previous part to complete write");
@@ -312,11 +319,6 @@ void CDiskWriteSlaveActivityBase::open()
             ActPrintLog("Previous write row count = %" RCPF "d", prevRows);
         }
     }
-    else
-    {
-        input.set(inputs.item(0));
-        startInput(input);
-    }
     processed = THORDATALINK_STARTED;
 
     bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
@@ -590,7 +592,7 @@ void CDiskWriteSlaveActivityBase::endProcess()
 {
     if (processed & THORDATALINK_STARTED)
     {
-        stopInput(input);
+        stop();
         processed |= THORDATALINK_STOPPED;
     }
 }

thorlcr/activities/thdiskbaseslave.ipp

@@ -70,12 +70,14 @@ public:
 
 //////////////////////////////////////////////
 
-void getPartsMetaInfo(ThorDataLinkMetaInfo &metaInfo, CThorDataLink &link, unsigned nparts, IPartDescriptor **partDescs, CDiskPartHandlerBase *partHandler);
+void getPartsMetaInfo(ThorDataLinkMetaInfo &metaInfo, unsigned nparts, IPartDescriptor **partDescs, CDiskPartHandlerBase *partHandler);
 
 //////////////////////////////////////////////
 
 class CDiskReadSlaveActivityBase : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     Owned<IThorRowInterfaces> diskRowIf;
 protected:
     StringAttr logicalFilename;
@@ -92,7 +94,7 @@ public:
     CDiskReadSlaveActivityBase(CGraphElementBase *_container);
     const char *queryLogicalFilename(unsigned index);
     IThorRowInterfaces * queryDiskRowInterfaces();
-    void start();
+    virtual void start() override;
 
     
 // IThorSlaveActivity
@@ -104,6 +106,8 @@ friend class CDiskPartHandlerBase;
 
 class CDiskWriteSlaveActivityBase : public ProcessSlaveActivity, implements ICopyFileProgress
 {
+    typedef ProcessSlaveActivity PARENT;
+
 protected:
     IHThorDiskWriteArg *diskHelperBase;
     Owned<IFileIO> outputIO;
@@ -116,7 +120,6 @@ protected:
     offset_t uncompressedBytesWritten;
     unsigned replicateDone;
     Owned<ICompressor> ecomp;
-    Owned<IThorDataLink> input;
     unsigned usageCount;
     CDfsLogicalFileName dlfn;
     StringBuffer tempExternalName;
@@ -138,6 +141,7 @@ public:
     virtual CFPmode onProgress(unsigned __int64 sizeDone, unsigned __int64 totalSize);
 
 // IThorSlaveProcess overloaded methods
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override;
     virtual void kill();
     virtual void process();
     virtual void endProcess();

thorlcr/activities/topn/thtopnslave.cpp

@@ -67,34 +67,32 @@ IRowStream *createFirstNReadSeqVar(IRowStream *input, unsigned limit)
     return new CFirstNReadSeqVar(input, limit);
 }
 
-class TopNSlaveActivity : public CSlaveActivity, public CThorDataLink
+class TopNSlaveActivity : public CSlaveActivity
 {
-    bool eos, eog, global, grouped, inputStopped;
+    typedef CSlaveActivity PARENT;
+
+    bool eos, eog, global, grouped;
     ICompare *compare;
     CThorExpandingRowArray sortedRows;
     Owned<IRowStream> out;
-    IThorDataLink *input;
     IHThorTopNArg *helper;
     rowidx_t topNLimit;
     Owned<IRowServer> rowServer;
     MemoryBuffer topology;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     TopNSlaveActivity(CGraphElementBase *_container, bool _global, bool _grouped)
-        : CSlaveActivity(_container), CThorDataLink(this), global(_global), grouped(_grouped), sortedRows(*this, this)
+        : CSlaveActivity(_container), global(_global), grouped(_grouped), sortedRows(*this, this)
     {
         assertex(!(global && grouped));
         eog = eos = false;
-        inputStopped = true;
     }
     ~TopNSlaveActivity()
     {
         out.clear();
         sortedRows.kill();
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
         appendOutputLinked(this);
         helper = (IHThorTopNArg *) queryHelper();
@@ -176,27 +174,17 @@ public:
         }
         if (global || 0 == topNLimit || 0 == sortedCount)
         {
-            doStopInput();
+            PARENT::stop();
             if (!global || 0 == topNLimit)
                 eos = true;
         }
         return retStream.getClear();
     }
-    void doStopInput()
-    {
-        if (!inputStopped)
-        {
-            inputStopped = true;
-            stopInput(input);
-        }
-    }
 // IThorDataLink
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input = inputs.item(0);
-        startInput(input);
-        inputStopped = false;
+        PARENT::start();
         // NB: topNLimit shouldn't be stupid size, resourcing will guarantee this
         __int64 _topNLimit = helper->getLimit();
         assertex(_topNLimit < RCIDXMAX); // hopefully never this big, but if were must be max-1 for binary insert
@@ -204,23 +192,21 @@ public:
         if (0 == topNLimit)
         {
             eos = true;
-            doStopInput();
+            PARENT::stop();
         }
         else
         {
-            out.setown(getNextSortGroup(input));
+            out.setown(getNextSortGroup(inputStream));
             eos = false;
         }
         eog = false;
-        dataLinkStart();
     }
-    virtual bool isGrouped() { return grouped; }
-    void stop()
+    virtual bool isGrouped() const override { return grouped; }
+    virtual void stop()
     {
         if (out)
             out->stop();
-        doStopInput();
-        dataLinkStop();
+        PARENT::stop();
     }
     CATCH_NEXTROW()
     {
@@ -229,7 +215,7 @@ public:
             return NULL;
         if (NULL == out)
         {
-            out.setown(getNextSortGroup(input));
+            out.setown(getNextSortGroup(inputStream));
             if (NULL == out)
             {
                 eos = true;
@@ -249,7 +235,7 @@ public:
             {
                 if (eog)
                 {
-                    out.setown(getNextSortGroup(input));
+                    out.setown(getNextSortGroup(inputStream));
                     if (NULL == out)
                         eos = true;
                     else
@@ -264,7 +250,7 @@ public:
                 else
                 {
                     eog = true;
-                    out.setown(getNextSortGroup(input));
+                    out.setown(getNextSortGroup(inputStream));
                     if (NULL == out)
                         eos = true;
                 }

thorlcr/activities/trace/thtraceslave.cpp

@@ -20,9 +20,10 @@
 #include "slave.ipp"
 #include "thactivityutil.ipp"
 
-class CTraceSlaveActivity : public CSlaveActivity, public CThorDataLink, public CThorSteppable
+class CTraceSlaveActivity : public CSlaveActivity, public CThorSteppable
 {
-    IThorDataLink *input;
+    typedef CSlaveActivity PARENT;
+
     IHThorTraceArg *helper;
     OwnedRoxieString name;
     unsigned keepLimit;
@@ -34,22 +35,20 @@ public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
     CTraceSlaveActivity(CGraphElementBase *_container)
-        : CSlaveActivity(_container), CThorDataLink(this), CThorSteppable(this),
+        : CSlaveActivity(_container), CThorSteppable(this),
           keepLimit(0), skip(0), sample(0), traceEnabled(false)
     {
         helper = (IHThorTraceArg *) queryHelper();
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         traceEnabled = getOptBool(THOROPT_TRACE_ENABLED, false);
     }
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dataLinkStart();
-        input = inputs.item(0);
-        startInput(input);
+        PARENT::start();
         if (traceEnabled && helper->canMatchAny() && queryRowMetaData())
         {
             keepLimit = helper->getKeepLimit();
@@ -66,11 +65,10 @@ public:
         else
             keepLimit = 0;
     }
-    void stop()
+    virtual void stop() override
     {
         name.clear();
-        stopInput(input);
-        dataLinkStop();
+        PARENT::stop();
     }
     void onTrace(const void *row)
     {
@@ -95,7 +93,7 @@ public:
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
-        OwnedConstThorRow ret = input->nextRow();
+        OwnedConstThorRow ret = inputStream->nextRow();
         if (ret)
         {
             onTrace(ret);
@@ -103,15 +101,15 @@ public:
         }
         return ret.getClear();
     }
-    const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         try { return nextRowGENoCatch(seek, numFields, wasCompleteMatch, stepExtra); }
         CATCH_NEXTROWX_CATCH;
     }
-    const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    virtual const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         ActivityTimer t(totalCycles, timeActivities);
-        OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
+        OwnedConstThorRow ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
         if (ret)
         {
             onTrace(ret);
@@ -119,25 +117,25 @@ public:
         }
         return ret.getClear();
     }
-    bool gatherConjunctions(ISteppedConjunctionCollector &collector)
+    virtual bool gatherConjunctions(ISteppedConjunctionCollector &collector)
     { 
         return input->gatherConjunctions(collector);
     }
-    void resetEOF() 
+    virtual void resetEOF()
     { 
-        input->resetEOF(); 
+        inputStream->resetEOF();
     }
-    bool isGrouped() { return input->isGrouped(); }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return input->isGrouped(); }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 // steppable
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
-        CSlaveActivity::setInput(index, inputActivity, inputOutIdx);
-        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+        CSlaveActivity::setInputStream(index, input, consumerOrdered);
+        CThorSteppable::setInputStream(index, input, consumerOrdered);
     }
     virtual IInputSteppingMeta *querySteppingMeta() { return CThorSteppable::inputStepping; }
 };

thorlcr/activities/when/thwhenslave.cpp

@@ -32,8 +32,6 @@ protected:
     CSlaveActivity *activity;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
-
     CDependencyExecutorSlaveActivity(CSlaveActivity *_activity) : activity(_activity)
     {
         global = !activity->queryContainer().queryOwner().queryOwner() || activity->queryContainer().queryOwner().isGlobal();
@@ -68,18 +66,17 @@ public:
 };
 
 
-class CWhenSlaveActivity : public CSlaveActivity, public CDependencyExecutorSlaveActivity, public CThorDataLink
+class CWhenSlaveActivity : public CSlaveActivity, public CDependencyExecutorSlaveActivity
 {
-protected:
-    Owned<IThorDataLink> input;
+    typedef CSlaveActivity PARENT;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CDependencyExecutorSlaveActivity);
 
-    CWhenSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CDependencyExecutorSlaveActivity(this), CThorDataLink(this)
+    CWhenSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CDependencyExecutorSlaveActivity(this)
     {
     }
-    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         CDependencyExecutorSlaveActivity::init(data, slaveData);
         appendOutputLinked(this);
@@ -88,25 +85,22 @@ public:
     {
         CDependencyExecutorSlaveActivity::preStart(parentExtractSz, parentExtract);
     }
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        input.set(inputs.item(0));
-        startInput(input);
-        dataLinkStart();
+        PARENT::start();
     }
-    virtual void stop()
+    virtual void stop() override
     {
-        stopInput(input);
+        PARENT::stop();
         if (!executeDependencies(abortSoon ? WhenFailureId : WhenSuccessId))
             abortSoon = true;
-        dataLinkStop();
     }
-    virtual bool isGrouped() { return input->isGrouped(); }
+    virtual bool isGrouped() const override { return input->isGrouped(); }
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
-        OwnedConstThorRow row(input->nextRow());
+        OwnedConstThorRow row(inputStream->nextRow());
         if (!row)
             return NULL;
         dataLinkIncrement();
@@ -118,11 +112,11 @@ public:
         if (global)
             barrier->cancel();
     }
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.fastThrough = false;
-        calcMetaInfoSize(info,inputs.item(0));
+        calcMetaInfoSize(info, queryInput(0));
     }
 };
 

thorlcr/activities/wuidread/thwuidreadslave.cpp

@@ -25,8 +25,10 @@
 
 #include "thwuidreadslave.ipp"
 
-class CWuidReadSlaveActivity : public CSlaveActivity, public CThorDataLink
+class CWuidReadSlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+
     Owned<ISerialStream> replyStream;
     CThorStreamDeserializerSource rowSource;
     IHThorWorkunitReadArg *helper;
@@ -36,25 +38,23 @@ class CWuidReadSlaveActivity : public CSlaveActivity, public CThorDataLink
     CMessageBuffer masterReplyMsg;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CWuidReadSlaveActivity(CGraphElementBase *_container) 
-        : CSlaveActivity(_container), CThorDataLink(this)
+        : CSlaveActivity(_container)
     {
         replyTag = queryMPServer().createReplyTag();
         replyStream.setown(createMemoryBufferSerialStream(masterReplyMsg));
         rowSource.setStream(replyStream);
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         appendOutputLinked(this);
         helper = (IHThorWorkunitReadArg *)queryHelper();
         grouped = helper->queryOutputMeta()->isGrouped();
     } 
-    void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
-        dataLinkStart();
+        PARENT::start();
 
         eogPending = false;
         if (container.queryLocal() || firstNode())
@@ -71,8 +71,6 @@ public:
             masterReplyMsg.swapWith(reqMsg);
         }
     }
-    void stop() { dataLinkStop(); }
-
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
@@ -92,8 +90,8 @@ public:
         dataLinkIncrement();
         return rowBuilder.finalizeRowClear(sz);
     }
-    bool isGrouped() { return grouped; }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return grouped; }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.isSource = true;

thorlcr/activities/wuidwrite/thwuidwriteslave.cpp

@@ -33,20 +33,13 @@
 class CWorkUnitWriteSlaveBase : public ProcessSlaveActivity
 {
 protected:
-    Owned<IThorDataLink> input;
     bool grouped;
 
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
     CWorkUnitWriteSlaveBase(CGraphElementBase *container) : ProcessSlaveActivity(container)
     {
         grouped = false;
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
-    {
-        mpTag = container.queryJobChannel().deserializeMPTag(data);
-    }
     void processBlock(unsigned &numGot, MemoryBuffer &mb)
     {
         Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input);
@@ -55,12 +48,12 @@ public:
         do
         {
             if (abortSoon) break;
-            OwnedConstThorRow row = input->nextRow();
+            OwnedConstThorRow row = inputStream->nextRow();
             if (grouped && !first)
                 mb.append(NULL == row.get());
             if (!row)
             {
-                row.setown(input->nextRow());
+                row.setown(inputStream->nextRow());
                 if (!row)
                     break;
             }
@@ -70,28 +63,37 @@ public:
             serializer->serialize(mbs,(const byte *)row.get());
         } while (mb.length() < PIPE_BUFFER_SIZE); // NB: allows at least 1
     }
-    void endProcess()
+
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
+    {
+        mpTag = container.queryJobChannel().deserializeMPTag(data);
+    }
+    virtual void endProcess() override
     {
         if (processed & THORDATALINK_STARTED)
         {
-            stopInput(input);
+            stop();
             processed |= THORDATALINK_STOPPED;
         }
-        input.clear();
     }
 };
 
 class CWorkUnitWriteGlobalSlaveBaseActivity : public CWorkUnitWriteSlaveBase
 {
+    typedef CWorkUnitWriteSlaveBase PARENT;
+
 public:
     CWorkUnitWriteGlobalSlaveBaseActivity(CGraphElementBase *container) : CWorkUnitWriteSlaveBase(container)
     {
     }
-    void process()
+    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
     {
-        input.setown(createDataLinkSmartBuffer(this, inputs.item(0), WORKUNITWRITE_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
-        startInput(input);
-
+        PARENT::setInputStream(index, _input, consumerOrdered);
+        setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), WORKUNITWRITE_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
+    }
+    virtual void process() override
+    {
+        start();
         processed = THORDATALINK_STARTED;
 
         ActPrintLog("WORKUNITWRITE: processing first block");
@@ -123,9 +125,9 @@ public:
             queryJobChannel().queryJobComm().send(msgMb, 0, mpTag);
         } while (!abortSoon && numGot);
     }
-    void abort()
+    virtual void abort() override
     {
-        CWorkUnitWriteSlaveBase::abort();
+        PARENT::abort();
         cancelReceiveMsg(0, mpTag);
     }
 };
@@ -141,10 +143,9 @@ public:
         grouped = 0 != (POFgrouped & helper->getFlags());
         replyTag = queryMPServer().createReplyTag();
     }
-    void process()
+    virtual void process() override
     {
-        input.set(inputs.item(0));
-        startInput(input);
+        start();
         processed = THORDATALINK_STARTED;
 
         ActPrintLog("WORKUNITWRITELOCAL: processing first block");
@@ -198,7 +199,7 @@ public:
             }
         } while (!abortSoon && numGot);
     }
-    void abort()
+    virtual void abort() override
     {
         CWorkUnitWriteSlaveBase::abort();
         cancelReceiveMsg(0, replyTag);

thorlcr/activities/xmlparse/thxmlparseslave.cpp

@@ -22,8 +22,10 @@
 #include "thactivityutil.ipp"
 #include "eclrtl.hpp"
 
-class CXmlParseSlaveActivity : public CSlaveActivity, public CThorDataLink, implements IXMLSelect
+class CXmlParseSlaveActivity : public CSlaveActivity, implements IXMLSelect
 {
+    typedef CSlaveActivity PARENT;
+
     IHThorXmlParseArg *helper;
     bool eogNext;
     bool anyThisGroup;
@@ -31,16 +33,14 @@ class CXmlParseSlaveActivity : public CSlaveActivity, public CThorDataLink, impl
     char *searchStr;
     Owned<IXMLParse> xmlParser;
     OwnedConstThorRow nxt;
-    IThorDataLink *input;
     Owned<IEngineRowAllocator> allocator;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
 
-    CXmlParseSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
+    CXmlParseSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         searchStr = NULL;
-        input = NULL;
     }
 
 // IXMLSelect
@@ -64,19 +64,12 @@ public:
             rtlFree(searchStr);
     }
 // IThorDataLink methods
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
+        PARENT::start();
         anyThisGroup = false;
         eogNext = false;
-        input = inputs.item(0);
-        startInput(input);
-        dataLinkStart();
-    }
-    virtual void stop()
-    {
-        stopInput(inputs.item(0));
-        dataLinkStop();
     }
     CATCH_NEXTROW()
     {
@@ -127,9 +120,9 @@ public:
                         }
                     }
                 }
-                nxt.setown(input->nextRow());
+                nxt.setown(inputStream->nextRow());
                 if (!nxt && !anyThisGroup)
-                    nxt.setown(input->nextRow());
+                    nxt.setown(inputStream->nextRow());
                 if (!nxt)
                     break;
                 unsigned len;
@@ -160,8 +153,8 @@ public:
         anyThisGroup = false;
         return NULL;
     }
-    virtual bool isGrouped() { return inputs.item(0)->isGrouped(); }
-    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    virtual bool isGrouped() const override { return queryInput(0)->isGrouped(); }
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override
     {
         initMetaInfo(info);
         info.fastThrough = true; // ish

thorlcr/activities/xmlread/thxmlreadslave.cpp

@@ -32,8 +32,10 @@
 #include "thorxmlread.hpp"
 #include "thdiskbaseslave.ipp"
 
-class CXmlReadSlaveActivity : public CDiskReadSlaveActivityBase, public CThorDataLink
+class CXmlReadSlaveActivity : public CDiskReadSlaveActivityBase
 {
+    typedef CDiskReadSlaveActivityBase PARENT;
+
     IHThorXmlReadArg *helper;
     IRowStream *out;
     rowcount_t limit;
@@ -193,9 +195,7 @@ class CXmlReadSlaveActivity : public CDiskReadSlaveActivityBase, public CThorDat
         }
     };
 public:
-    IMPLEMENT_IINTERFACE_USING(CSlaveActivity);
-
-    CXmlReadSlaveActivity(CGraphElementBase *_container) : CDiskReadSlaveActivityBase(_container), CThorDataLink(this)
+    CXmlReadSlaveActivity(CGraphElementBase *_container) : CDiskReadSlaveActivityBase(_container)
     {
         out = NULL;
         helper = (IHThorXmlReadArg *)queryHelper();
@@ -209,7 +209,7 @@ public:
     {
         ::Release(out);
     }
-    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
     {
         CDiskReadSlaveActivityBase::init(data, slaveData);
         partHandler.setown(new CXmlPartHandler(*this,queryRowAllocator()));
@@ -226,21 +226,20 @@ public:
     }
     
 // IThorDataLink
-    virtual void start()
+    virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         CDiskReadSlaveActivityBase::start();
         out = createSequentialPartHandler(partHandler, partDescs, false);
-        dataLinkStart();
     }
-    virtual void stop()
+    virtual void stop() override
     {
         if (out)
         {
             out->Release();
             out = NULL;
         }
-        dataLinkStop();
+        PARENT::stop();
     }
 
     CATCH_NEXTROW()
@@ -261,7 +260,7 @@ public:
         return row.getClear();
     }
     
-    virtual bool isGrouped() { return false; }
+    virtual bool isGrouped() const override { return false; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)
     {
         if (!gotMeta)
@@ -269,7 +268,7 @@ public:
             gotMeta = true;
             initMetaInfo(cachedMetaInfo);
             cachedMetaInfo.isSource = true;
-            getPartsMetaInfo(cachedMetaInfo, *this, partDescs.ordinality(), partDescs.getArray(), partHandler);
+            getPartsMetaInfo(cachedMetaInfo, partDescs.ordinality(), partDescs.getArray(), partHandler);
             cachedMetaInfo.unknownRowsOutput = true; // at least I don't think we know
         }
         info = cachedMetaInfo;

thorlcr/activities/xmlwrite/thxmlwriteslave.cpp

@@ -72,7 +72,7 @@ public:
         writer->outputBeginArray(rowTag); //need this to format rows, even if not outputting it below
         while(!abortSoon)
         {
-            OwnedConstThorRow row = input->ungroupedNextRow();
+            OwnedConstThorRow row = inputStream->ungroupedNextRow();
             if (!row)
                 break;
             writer->clear().outputBeginNested(rowTag, false);

thorlcr/graph/thgraph.cpp

@@ -2496,7 +2496,7 @@ CJobBase::CJobBase(ILoadedDllEntry *_querySo, const char *_graphName) : querySo(
     Owned<IConstWUGraph> graph = localWU->getGraph(graphName);
     graphXGMML.setown(graph->getXGMMLTree(false));
     if (!graphXGMML)
-    	throwUnexpected();
+        throwUnexpected();
 }
 
 void CJobBase::init()

thorlcr/graph/thgraph.hpp

@@ -867,7 +867,7 @@ public:
     virtual IThorAllocator *getThorAllocator(unsigned channel);
 
     virtual void abort(IException *e);
-    virtual void debugRequest(CMessageBuffer &msg, const char *request) const { }
+    virtual void debugRequest(MemoryBuffer &msg, const char *request) const { }
 
 //
     virtual void addCreatedFile(const char *file) { assertex(false); }

thorlcr/graph/thgraphslave.cpp

@@ -113,7 +113,7 @@ public:
 
 // 
 
-CSlaveActivity::CSlaveActivity(CGraphElementBase *_container) : CActivityBase(_container)
+CSlaveActivity::CSlaveActivity(CGraphElementBase *_container) : CActivityBase(_container), CEdgeProgress(this)
 {
     data = NULL;
 }
@@ -126,6 +126,13 @@ CSlaveActivity::~CSlaveActivity()
     ActPrintLog("DESTROYED");
 }
 
+void CSlaveActivity::setOutputStream(unsigned index, IEngineRowStream *stream)
+{
+    while (outputStreams.ordinality()<=index)
+        outputStreams.append(nullptr);
+    outputStreams.replace(stream, index);
+}
+
 void CSlaveActivity::setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
 {
     CActivityBase::setInput(index, inputActivity, inputOutIdx);
@@ -141,20 +148,77 @@ void CSlaveActivity::setInput(unsigned index, CActivityBase *inputActivity, unsi
         outLink.set(((CSlaveActivity *)inputActivity)->queryOutput(inputOutIdx));
     assertex(outLink);
 
-    while (inputs.ordinality()<=index) inputs.append(NULL);
+    while (inputs.ordinality()<=index)
+        inputs.append(* new CThorInput());
+    CThorInput &newInput = inputs.item(index);
+    newInput.set(outLink, inputOutIdx);
+    if (!input)
+    {
+        input = outLink;
+        inputSourceIdx = inputOutIdx;
+    }
+}
 
-    inputs.replace(outLink.getClear(), index);
+void CSlaveActivity::connectInputStreams(bool consumerOrdered)
+{
+    ForEachItemIn(index, inputs)
+    {
+        CThorInput &_input = inputs.item(index);
+        if (_input.itdl)
+            setInputStream(index, _input, consumerOrdered);
+    }
 }
 
-void CSlaveActivity::appendOutput(IThorDataLink *itdl)
+void CSlaveActivity::setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered)
 {
-    if (queryJob().getOptBool("TRACEROWS"))
+    if (input) // will be none if source act.
     {
-        const unsigned numTraceRows = queryJob().getOptInt("numTraceRows", 10);
-        outputs.append(new CTracingThorDataLink(itdl, queryHelper(), numTraceRows));
+        Owned<IStrandJunction> junction;
+        IEngineRowStream *_inputStream = connectSingleStream(*this, _input.itdl, _input.sourceIdx, junction, _input.itdl->isInputOrdered(consumerOrdered));
+        if (queryJob().getOptBool("TRACEROWS"))
+        {
+            const unsigned numTraceRows = queryJob().getOptInt("numTraceRows", 10);
+            CTracingStream *tracingStream = new CTracingStream(_input.itdl, _inputStream, _input.itdl->queryFromActivity()->queryHelper(), numTraceRows);
+            _input.tracingStream.setown(tracingStream);
+            _inputStream = tracingStream;
+        }
+        _input.stream.set(_inputStream);
+        _input.junction.setown(junction.getClear());
+        if (0 == index)
+            inputStream = _inputStream;
+        _input.itdl->setOutputStream(_input.sourceIdx, LINK(_inputStream)); // used by debug request only at moment.
     }
-    else
-        outputs.append(itdl);
+}
+
+void CSlaveActivity::setLookAhead(unsigned index, IStartableEngineRowStream *lookAhead)
+{
+    CThorInput &_input = inputs.item(index);
+    _input.lookAhead.setown(lookAhead);
+    _input.stream.set(lookAhead);
+    if (0 == index)
+        inputStream = lookAhead;
+}
+
+IStrandJunction *CSlaveActivity::getOutputStreams(CActivityBase &ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
+{
+    // Default non-stranded implementation, expects activity to have 1 output.
+    assertex(!idx);
+    // By default, activities are assumed NOT to support streams
+    bool inputOrdered = isInputOrdered(consumerOrdered);
+    connectInputStreams(inputOrdered);
+    // Return a single stream
+    // Default activity impl. adds single output as stream
+    streams.append(this);
+    return nullptr;
+}
+
+void CSlaveActivity::appendOutput(IThorDataLink *itdl)
+{
+    IThorDataLinkExt *itdlExt = QUERYINTERFACE(itdl, IThorDataLinkExt);
+    dbgassertex(itdlExt);
+    unsigned outputNum = outputs.ordinality();
+    itdlExt->setOutputIdx(outputNum);
+    outputs.append(itdl);
 }
 
 void CSlaveActivity::appendOutputLinked(IThorDataLink *itdl)
@@ -163,42 +227,88 @@ void CSlaveActivity::appendOutputLinked(IThorDataLink *itdl)
     appendOutput(itdl);
 }
 
-IThorDataLink *CSlaveActivity::queryOutput(unsigned index)
+IThorDataLink *CSlaveActivity::queryOutput(unsigned index) const
 {
-    if (index>=outputs.ordinality()) return NULL;
+    if (index>=outputs.ordinality()) return nullptr;
     return outputs.item(index);
 }
 
-IThorDataLink *CSlaveActivity::queryInput(unsigned index)
+IThorDataLink *CSlaveActivity::queryInput(unsigned index) const
 {
-    if (index>=inputs.ordinality()) return NULL;
-    return inputs.item(index);
+    if (index>=inputs.ordinality()) return nullptr;
+    return inputs.item(index).itdl;
 }
 
-void CSlaveActivity::startInput(IThorDataLink *itdl, const char *extra)
+IEngineRowStream *CSlaveActivity::queryInputStream(unsigned index) const
+{
+    if (index>=inputs.ordinality()) return nullptr;
+    return inputs.item(index).stream;
+}
+
+IEngineRowStream *CSlaveActivity::queryOutputStream(unsigned index) const
+{
+    if (index>=outputStreams.ordinality()) return nullptr;
+    return outputStreams.item(index);
+}
+
+void CSlaveActivity::start()
+{
+    if (inputs.ordinality()>1)
+        throwUnexpected();
+    if (input)
+        startInput(0);
+    dataLinkStart();
+}
+
+void CSlaveActivity::startAllInputs()
+{
+    ForEachItemIn(i, inputs)
+    {
+        startInput(i);
+    }
+}
+
+void CSlaveActivity::startInput(unsigned index, const char *extra)
 {
     StringBuffer s("Starting input");
     if (extra)
         s.append(" ").append(extra);
     ActPrintLog("%s", s.str());
 
+    CThorInput &_input = inputs.item(index);
 #ifdef TRACE_STARTSTOP_EXCEPTIONS
     try
     {
-        itdl->start();
+#endif
+        _input.itdl->start();
+        startJunction(_input.junction);
+        if (_input.lookAhead)
+            _input.lookAhead->start();
+        _input.stopped = false;
+        if (0 == index)
+            inputStopped = false;
+#ifdef TRACE_STARTSTOP_EXCEPTIONS
     }
     catch(IException *e)
     {
         ActPrintLog(e, "%s", s.str());
         throw;
     }
-#else
-    itdl->start();
 #endif
 }
 
-void CSlaveActivity::stopInput(IRowStream *itdl, const char *extra)
+void CSlaveActivity::stop()
 {
+    if (input)
+        stopInput(0);
+    dataLinkStop();
+}
+
+void CSlaveActivity::stopInput(unsigned index, const char *extra)
+{
+    CThorInput &_input = inputs.item(index);
+    if (_input.stopped)
+        return;
     StringBuffer s("Stopping input for");
     if (extra)
         s.append(" ").append(extra);
@@ -207,18 +317,41 @@ void CSlaveActivity::stopInput(IRowStream *itdl, const char *extra)
 #ifdef TRACE_STARTSTOP_EXCEPTIONS
     try
     {
-        itdl->stop();
+#endif
+        if (_input.stream)
+            _input.stream->stop();
+        _input.stopped = true;
+        if (0 == index)
+            inputStopped = true;
+
+#ifdef TRACE_STARTSTOP_EXCEPTIONS
     }
     catch(IException * e)
     {
         ActPrintLog(e, "%s", s.str());
         throw;
     }
-#else
-    itdl->stop();
 #endif
 }
 
+void CSlaveActivity::stopAllInputs()
+{
+    ForEachItemIn(i, inputs)
+    {
+        stopInput(i);
+    }
+}
+
+void CSlaveActivity::reset()
+{
+    CActivityBase::reset();
+    ForEachItemIn(i, inputs)
+        resetJunction(inputs.item(i).junction);
+    input = nullptr;
+    inputStream = nullptr;
+    inputStopped = true;
+}
+
 void CSlaveActivity::abort()
 {
     CActivityBase::abort();
@@ -275,7 +408,7 @@ unsigned __int64 CSlaveActivity::queryLocalCycles() const
     unsigned __int64 inputCycles = 0;
     if (1 == inputs.ordinality())
     {
-        IThorDataLink *input = inputs.item(0);
+        IThorDataLink *input = queryInput(0);
         inputCycles += input->queryTotalCycles();
     }
     else
@@ -286,7 +419,7 @@ unsigned __int64 CSlaveActivity::queryLocalCycles() const
             case TAKchildcase:
                 if (inputs.ordinality() && (((unsigned)-1) != container.whichBranch))
                 {
-                    IThorDataLink *input = inputs.item(container.whichBranch);
+                    IThorDataLink *input = queryInput(container.whichBranch);
                     if (input)
                         inputCycles += input->queryTotalCycles();
                 }
@@ -294,7 +427,7 @@ unsigned __int64 CSlaveActivity::queryLocalCycles() const
             default:
                 ForEachItemIn(i, inputs)
                 {
-                    IThorDataLink *input = inputs.item(i);
+                    IThorDataLink *input = queryInput(i);
                     inputCycles += input->queryTotalCycles();
                 }
                 break;
@@ -306,30 +439,252 @@ unsigned __int64 CSlaveActivity::queryLocalCycles() const
     return _totalCycles-inputCycles;
 }
 
-unsigned __int64 CSlaveActivity::queryTotalCycles() const
+void CSlaveActivity::serializeStats(MemoryBuffer &mb)
 {
-    return totalCycles.totalCycles;
+    CriticalBlock b(crit);
+    mb.append((unsigned __int64)cycle_to_nanosec(queryLocalCycles()));
+    ForEachItemIn(i, outputs)
+        outputs.item(i)->dataLinkSerialize(mb);
 }
 
-unsigned __int64 CSlaveActivity::queryEndCycles() const
+void CSlaveActivity::debugRequest(unsigned edgeIdx, MemoryBuffer &msg)
 {
-    return totalCycles.endCycles;
+    IEngineRowStream *outputStream = queryOutputStream(edgeIdx);
+    IThorDebug *debug = QUERYINTERFACE(outputStream, IThorDebug); // should probably use an extended IEngineRowStream, or store in separate array instead
+    if (debug) debug->debugRequest(msg);
 }
 
-void CSlaveActivity::serializeStats(MemoryBuffer &mb)
+bool CSlaveActivity::isGrouped() const
 {
-    CriticalBlock b(crit);
-    mb.append((unsigned __int64)cycle_to_nanosec(queryLocalCycles()));
-    ForEachItemIn(i, outputs)
-        outputs.item(i)->dataLinkSerialize(mb);
+    if (!input) return false; // should possible be an error if query and not set
+    return input->isGrouped();
 }
 
-void CSlaveActivity::debugRequest(unsigned edgeIdx, CMessageBuffer &msg)
+IOutputMetaData *CSlaveActivity::queryOutputMeta() const
 {
-    IThorDataLink *link = queryOutput(edgeIdx);
-    if (link) link->debugRequest(msg);
+    return queryHelper()->queryOutputMeta();
 }
-///
+
+void CSlaveActivity::dataLinkSerialize(MemoryBuffer &mb) const
+{
+    CEdgeProgress::dataLinkSerialize(mb);
+}
+
+void CSlaveActivity::debugRequest(MemoryBuffer &msg)
+{
+}
+
+
+/// CThorStrandProcessor
+
+CThorStrandProcessor::CThorStrandProcessor(CThorStrandedActivity &_parent, IEngineRowStream *_inputStream, unsigned _outputId)
+  : parent(_parent), inputStream(_inputStream), outputId(_outputId), timeActivities(_parent.queryTimeActivities())
+{
+    rowsProcessed = 0;
+    baseHelper.set(parent.queryHelper());
+}
+
+void CThorStrandProcessor::processAndThrowOwnedException(IException *_e)
+{
+    IThorException *e = QUERYINTERFACE(_e, IThorException);
+    if (e)
+    {
+        if (!e->queryActivityId())
+            setExceptionActivityInfo(parent.queryContainer(), e);
+    }
+    else
+    {
+        e = MakeActivityException(&parent, _e);
+        _e->Release();
+    }
+    throw e;
+}
+
+void CThorStrandProcessor::stop()
+{
+    if (!stopped)
+    {
+        if (inputStream)
+            inputStream->stop();
+        parent.strandedStop();
+    }
+    stopped = true;
+}
+
+
+/// CThorStrandedActivity
+
+void CThorStrandedActivity::onStartStrands()
+{
+    active = strands.ordinality();
+    ForEachItemIn(idx, strands)
+        strands.item(idx).start();
+}
+
+void CThorStrandedActivity::strandedStop()
+{
+    // Called from the strands... which should ensure that stop is not called more than once per strand
+    //The first strand to call
+    if (active)
+        --active;
+    if (!active)
+        stop();
+}
+
+//This function is pure (But also implemented out of line) to force the derived classes to implement it.
+//After calling the base class start method, and initialising any values from the helper they must call onStartStrands(),
+//this must also happen before any rows are read from the strands (e.g., by a source junction)
+//    virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused) = 0;
+
+//For some reason gcc doesn't let you specify a function as pure virtual and define it at the same time.
+void CThorStrandedActivity::start()
+{
+    CSlaveActivity::start();
+    startJunction(splitter);
+    onStartStrands();
+}
+
+void CThorStrandedActivity::reset()
+{
+    assertex(active==0);
+    ForEachItemIn(idx, strands)
+        strands.item(idx).reset();
+    strands.kill();
+    resetJunction(splitter);
+    CSlaveActivity::reset();
+    resetJunction(sourceJunction);
+}
+
+IStrandJunction *CThorStrandedActivity::getOutputStreams(CActivityBase &ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
+{
+    assertex(idx == 0);
+    assertex(strands.empty());
+
+    // JCSMORE these may be wrong if this is a source activity
+    bool inputOrdered = input ? input->isInputOrdered(consumerOrdered) : isInputOrdered(consumerOrdered);
+    //Note, numStrands == 1 is an explicit request to disable threading
+    if (consumerOptions && (consumerOptions->numStrands != 1) && (strandOptions.numStrands != 1))
+    {
+        //Check to see if the consumer's settings should override
+        if (strandOptions.numStrands == 0)
+        {
+            strandOptions.numStrands = consumerOptions->numStrands;
+            strandOptions.blockSize = consumerOptions->blockSize;
+        }
+        else if (consumerOptions->numStrands > strandOptions.numStrands)
+        {
+            strandOptions.numStrands = consumerOptions->numStrands;
+        }
+    }
+
+    Owned <IStrandJunction> recombiner;
+    if (input)
+    {
+        if (strandOptions.numStrands == 1)
+        {
+            // 1 means explicitly requested single-strand.
+            Owned<IStrandJunction> junction;
+            IEngineRowStream *instream = connectSingleStream(ctx, input, inputSourceIdx, junction, inputOrdered);
+            inputs.item(idx).junction.setown(junction.getClear());
+            strands.append(*createStrandProcessor(instream));
+        }
+        else
+        {
+            PointerArrayOf<IEngineRowStream> instreams;
+            recombiner.setown(input->getOutputStreams(ctx, inputSourceIdx, instreams, &strandOptions, inputOrdered, orderedCallbacks));
+            if ((instreams.length() == 1) && (strandOptions.numStrands != 0))  // 0 means did not specify - we should use the strands that our upstream provides
+            {
+                assertex(recombiner == nullptr);
+                // Create a splitter to split the input into n... and a recombiner if need to preserve sorting
+                if (inputOrdered)
+                {
+                    branch.setown(createStrandBranch(*ctx.queryRowManager(), strandOptions.numStrands, strandOptions.blockSize, true, input->queryOutputMeta()->isGrouped(), false, orderedCallbacks));
+                    splitter.set(branch->queryInputJunction());
+                    recombiner.set(branch->queryOutputJunction());
+                }
+                else
+                {
+                    splitter.setown(createStrandJunction(*ctx.queryRowManager(), 1, strandOptions.numStrands, strandOptions.blockSize, false));
+                }
+                splitter->setInput(0, instreams.item(0));
+                for (unsigned strandNo = 0; strandNo < strandOptions.numStrands; strandNo++)
+                    strands.append(*createStrandProcessor(splitter->queryOutput(strandNo)));
+            }
+            else
+            {
+                // Ignore my hint and just use the width already split into...
+                ForEachItemIn(strandNo, instreams)
+                    strands.append(*createStrandProcessor(instreams.item(strandNo)));
+            }
+        }
+    }
+    else
+    {
+        unsigned numStrands = strandOptions.numStrands ? strandOptions.numStrands : 1;
+        for (unsigned i=0; i < numStrands; i++)
+            strands.append(*createStrandSourceProcessor(inputOrdered));
+
+        if (inputOrdered && (numStrands > 1))
+        {
+            if (consumerOptions)
+            {
+                //If the output activities are also stranded then need to create a version of the branch
+                bool isGrouped = queryOutputMeta()->isGrouped();
+                branch.setown(createStrandBranch(*ctx.queryRowManager(), strandOptions.numStrands, strandOptions.blockSize, true, isGrouped, true, orderedCallbacks));
+                sourceJunction.set(branch->queryInputJunction());
+                recombiner.set(branch->queryOutputJunction());
+                assertex((orderedCallbacks && !recombiner) || (!orderedCallbacks && recombiner));
+
+                //This is different from the branch above.  The first "junction" has the source activity as the input, and the outputs as the result of the activity
+                for (unsigned strandNo = 0; strandNo < strandOptions.numStrands; strandNo++)
+                {
+                    sourceJunction->setInput(strandNo, &strands.item(strandNo));
+                    streams.append(sourceJunction->queryOutput(strandNo));
+                }
+#ifdef TRACE_STRANDS
+                if (traceLevel > 2)
+                    DBGLOG("Executing activity %u with %u strands", activityId, strands.ordinality());
+#endif
+                return recombiner.getClear();
+            }
+            else
+                recombiner.setown(createStrandJunction(*ctx.queryRowManager(), numStrands, 1, strandOptions.blockSize, inputOrdered));
+        }
+    }
+    ForEachItemIn(i, strands)
+        streams.append(&strands.item(i));
+#ifdef TRACE_STRANDS
+    if (traceLevel > 2)
+        DBGLOG("Executing activity %u with %u strands", activityId, strands.ordinality());
+#endif
+
+    return recombiner.getClear();
+}
+
+unsigned __int64 CThorStrandedActivity::queryTotalCycles() const
+{
+    unsigned __int64 total = 0;;
+    ForEachItemIn(i, strands)
+    {
+        CThorStrandProcessor &strand = strands.item(i);
+        total += strand.queryTotalCycles();
+    }
+    return total;
+}
+
+void CThorStrandedActivity::dataLinkSerialize(MemoryBuffer &mb) const
+{
+    rowcount_t totalCount = getCount();
+    ForEachItemIn(i, strands)
+    {
+        CThorStrandProcessor &strand = strands.item(i);
+        totalCount += strand.getCount();
+    }
+    mb.append(totalCount);
+}
+
+
+
 
 // CSlaveGraph
 
@@ -574,6 +929,13 @@ void CSlaveGraph::connect()
     Owned<IThorActivityIterator> iter = getConnectedIterator();
     ForEach(*iter)
         iter->query().doconnect();
+    iter.setown(getSinkIterator());
+    ForEach(*iter)
+    {
+        CGraphElementBase &container = iter->query();
+        CSlaveActivity *sinkAct = (CSlaveActivity *)container.queryActivity();
+        sinkAct->connectInputStreams(true);
+    }
 }
 
 void CSlaveGraph::executeSubGraph(size32_t parentExtractSz, const byte *parentExtract)
@@ -1197,7 +1559,7 @@ bool CJobSlave::getWorkUnitValueBool(const char *prop, bool defVal) const
     return workUnitInfo->queryPropTree("Debug")->getPropBool(propName.toLowerCase().str(), defVal);
 }
 
-void CJobSlave::debugRequest(CMessageBuffer &msg, const char *request) const
+void CJobSlave::debugRequest(MemoryBuffer &msg, const char *request) const
 {
     if (watchdog) watchdog->debugRequest(msg, request);
 }
@@ -1597,3 +1959,42 @@ IThorFileCache *createFileCache(unsigned limit)
 {
     return new CFileCache(limit);
 }
+
+/*
+ * strand stuff
+ */
+
+IEngineRowStream *connectSingleStream(CActivityBase &activity, IThorDataLink *input, unsigned idx, Owned<IStrandJunction> &junction, bool consumerOrdered)
+{
+    if (input)
+    {
+        PointerArrayOf<IEngineRowStream> instreams;
+        junction.setown(input->getOutputStreams(activity, idx, instreams, nullptr, consumerOrdered, nullptr));
+        if (instreams.length() != 1)
+        {
+            assertex(instreams.length());
+            if (!junction)
+                junction.setown(createStrandJunction(*activity.queryRowManager(), instreams.length(), 1, activity.getOptInt("strandBlockSize"), false));
+            ForEachItemIn(stream, instreams)
+            {
+                junction->setInput(stream, instreams.item(stream));
+            }
+            return junction->queryOutput(0);
+        }
+        else
+            return instreams.item(0);
+    }
+    else
+        return nullptr;
+}
+
+IEngineRowStream *connectSingleStream(CActivityBase &activity, IThorDataLink *input, unsigned idx, bool consumerOrdered)
+{
+    Owned<IStrandJunction> junction;
+    IEngineRowStream * result = connectSingleStream(activity, input, idx, junction, consumerOrdered);
+    assertex(!junction);
+    return result;
+}
+
+
+

thorlcr/graph/thgraphslave.hpp

@@ -35,19 +35,119 @@
 #include "thgraph.hpp"
 #include "jdebug.hpp"
 #include "traceslave.hpp"
+#include "thorstrand.hpp"
 
-class CSlaveActivity;
+interface IStartableEngineRowStream : extends IEngineRowStream
+{
+    virtual void start() = 0;
+};
+
+class COutputTiming
+{
+public:
+    ActivityTimeAccumulator totalCycles;
+
+    COutputTiming() { }
+
+    void resetTiming() { totalCycles.reset(); }
+    ActivityTimeAccumulator &getTotalCyclesRef() { return totalCycles; }
+    unsigned __int64 queryTotalCycles() const { return totalCycles.totalCycles; }
+    unsigned __int64 queryEndCycles() const { return totalCycles.endCycles; }
+};
+
+class CEdgeProgress
+{
+    CActivityBase &owner;
+    rowcount_t count = 0, icount = 0;
+    unsigned outputId = 0;
+public:
+    explicit CEdgeProgress(CActivityBase *_owner) : owner(*_owner) { }
+
+    inline void dataLinkStart()
+    {
+#ifdef _TESTING
+        owner.ActPrintLog("ITDL starting for output %d", outputId);
+#endif
+#ifdef _TESTING
+        assertex(!hasStarted() || hasStopped());      // ITDL started twice
+#endif
+        icount = 0;
+        rowcount_t prevCount = count & THORDATALINK_COUNT_MASK;
+        count = prevCount | THORDATALINK_STARTED;
+    }
+
+    inline void dataLinkStop()
+    {
+#ifdef _TESTING
+        assertex(hasStarted());        // ITDL stopped without being started
+#endif
+        count |= THORDATALINK_STOPPED;
+#ifdef _TESTING
+        owner.ActPrintLog("ITDL output %d stopped, count was %" RCPF "d", outputId, getDataLinkCount());
+#endif
+    }
+    inline void dataLinkIncrement() { dataLinkIncrement(1); }
+    inline void dataLinkIncrement(rowcount_t v)
+    {
+#ifdef _TESTING
+        assertex(hasStarted());
+#ifdef OUTPUT_RECORDSIZE
+        if (count==THORDATALINK_STARTED)
+        {
+            size32_t rsz = parent.queryRowMetaData(this)->getMinRecordSize();
+            parent.ActPrintLog("Record size %s= %d", parent.queryRowMetaData(this)->isVariableSize()?"(min) ":"",rsz);
+        }
+#endif
+#endif
+        icount += v;
+        count += v;
+    }
+    inline bool hasStarted() const { return (count & THORDATALINK_STARTED) ? true : false; }
+    inline bool hasStopped() const { return (count & THORDATALINK_STOPPED) ? true : false; }
+    inline void dataLinkSerialize(MemoryBuffer &mb) const { mb.append(count); }
+    inline rowcount_t getDataLinkGlobalCount() { return (count & THORDATALINK_COUNT_MASK); }
+    inline rowcount_t getDataLinkCount() const { return icount; }
+    inline rowcount_t getCount() const { return count; }
+};
+
+class CThorInput : public CSimpleInterfaceOf<IInterface>
+{
+public:
+    unsigned sourceIdx = 0;
+    Linked<IThorDataLink> itdl;
+    Linked<IStartableEngineRowStream> lookAhead;
+    Linked<IThorDebug> tracingStream;
+    Linked<IEngineRowStream> stream;
+    Linked<IStrandJunction> junction;
+    bool stopped = true;
+
+    explicit CThorInput() { }
+    void set(IThorDataLink *_itdl, unsigned idx) { itdl.set(_itdl); sourceIdx = idx; }
+};
+typedef IArrayOf<CThorInput> CThorInputArray;
 
 class CSlaveGraphElement;
-class graphslave_decl CSlaveActivity : public CActivityBase
+class graphslave_decl CSlaveActivity : public CActivityBase, public CEdgeProgress, public COutputTiming, implements IThorDataLinkExt, implements IEngineRowStream, implements IThorSlaveActivity
 {
     mutable MemoryBuffer *data;
     mutable CriticalSection crit;
 
 protected:
-    IPointerArrayOf<IThorDataLink> inputs, outputs;
-    ActivityTimeAccumulator totalCycles;
+    CThorInputArray inputs;
+    IPointerArrayOf<IThorDataLink> outputs;
+    IPointerArrayOf<IEngineRowStream> outputStreams;
+    IThorDataLink *input = nullptr;
+    bool inputStopped = true;
+    unsigned inputSourceIdx = 0;
+    IEngineRowStream *inputStream = nullptr;
     MemoryBuffer startCtx;
+    bool optStableInput = true; // is the input forced to ordered?
+    bool optUnstableInput = false;  // is the input forced to unordered?
+    bool optUnordered = false; // is the output specified as unordered?
+    unsigned outputIdx = 0; // for IThorDataLinkExt
+
+protected:
+    unsigned __int64 queryLocalCycles() const;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -56,28 +156,168 @@ public:
     ~CSlaveActivity();
     virtual void clearConnections();
     virtual void releaseIOs();
-    virtual void init(MemoryBuffer &in, MemoryBuffer &out) { }
-    virtual void processDone(MemoryBuffer &mb) { };
-    virtual void abort();
     virtual MemoryBuffer &queryInitializationData(unsigned slave) const;
     virtual MemoryBuffer &getInitializationData(unsigned slave, MemoryBuffer &mb) const;
-
-    IThorDataLink *queryOutput(unsigned index);
-    IThorDataLink *queryInput(unsigned index);
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx);
+    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx) override;
+    virtual void connectInputStreams(bool consumerOrdered);
+
+    void setLookAhead(unsigned index, IStartableEngineRowStream *lookAhead);
+    IThorDataLink *queryOutput(unsigned index) const;
+    IThorDataLink *queryInput(unsigned index) const;
+    IEngineRowStream *queryInputStream(unsigned index) const;
+    IEngineRowStream *queryOutputStream(unsigned index) const;
+    unsigned queryInputOutputIndex(unsigned inputIndex) const { return inputs.item(inputIndex).sourceIdx; }
+    unsigned queryNumInputs() const { return inputs.ordinality(); }
     void appendOutput(IThorDataLink *itdl);
     void appendOutputLinked(IThorDataLink *itdl);
-    void startInput(IThorDataLink *itdl, const char *extra=NULL);
-    void stopInput(IRowStream *itdl, const char *extra=NULL);
-
-    ActivityTimeAccumulator &getTotalCyclesRef() { return totalCycles; }
-    unsigned __int64 queryLocalCycles() const;
-    virtual unsigned __int64 queryTotalCycles() const; // some acts. may calculate accumulated total from inputs (e.g. splitter)
-    virtual unsigned __int64 queryEndCycles() const;
+    void startInput(unsigned index, const char *extra=NULL);
+    void startAllInputs();
+    void stopInput(unsigned index, const char *extra=NULL);
+    void stopAllInputs();
     virtual void serializeStats(MemoryBuffer &mb);
-    void debugRequest(unsigned edgeIdx, CMessageBuffer &msg);
+    void debugRequest(unsigned edgeIdx, MemoryBuffer &msg);
+
+// IThorDataLink
+    virtual CSlaveActivity *queryFromActivity() override { return this; }
+    virtual IStrandJunction *getOutputStreams(CActivityBase &_ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override;
+    virtual void setOutputStream(unsigned index, IEngineRowStream *stream) override;
+    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) override { }
+    virtual bool isGrouped() const override;
+    virtual IOutputMetaData * queryOutputMeta() const;
+    virtual unsigned queryOutputIdx() const override { return outputIdx; }
+    virtual void dataLinkSerialize(MemoryBuffer &mb) const override;
+    virtual bool isInputOrdered(bool consumerOrdered) const override
+    {
+        if (optStableInput)
+            return true;
+        if (optUnstableInput)
+            return false;
+        if (optUnordered)
+            return false;
+        return consumerOrdered;
+    }
+    virtual unsigned __int64 queryTotalCycles() const { return COutputTiming::queryTotalCycles(); }
+    virtual unsigned __int64 queryEndCycles() const { return COutputTiming::queryEndCycles(); }
+    virtual void debugRequest(MemoryBuffer &msg) override;
+
+// IThorDataLink
+    virtual void start() override;
+// IThorDataLinkExt
+    virtual void setOutputIdx(unsigned idx) override { outputIdx = idx; }
+
+// IEngineRowStream
+    virtual const void *nextRow() override { throwUnexpected(); }
+    virtual void stop() override;
+    virtual void resetEOF() override { throwUnexpected(); }
+
+// IThorSlaveActivity
+    virtual void init(MemoryBuffer &in, MemoryBuffer &out) override { }
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override;
+    virtual void processDone(MemoryBuffer &mb) override { };
+    virtual void reset() override;
+    virtual void abort() override;
+};
+
+
+IEngineRowStream *connectSingleStream(CActivityBase &activity, IThorDataLink *input, unsigned idx, Owned<IStrandJunction> &junction, bool consumerOrdered);
+IEngineRowStream *connectSingleStream(CActivityBase &activity, IThorDataLink *input, unsigned idx, bool consumerOrdered);
+
+
+#define STRAND_CATCH_NEXTROWX_CATCH \
+        catch (IException *_e) \
+        { \
+            parent->processAndThrowOwnedException(_e); \
+        }
+
+#define STRAND_CATCH_NEXTROW() \
+    virtual const void *nextRow() override \
+    { \
+        try \
+        { \
+            return nextRowNoCatch(); \
+        } \
+        CATCH_NEXTROWX_CATCH \
+    } \
+    inline const void *nextRowNoCatch() __attribute__((always_inline))
+
+
+class CThorStrandedActivity;
+class CThorStrandProcessor : public CInterfaceOf<IEngineRowStream>, public COutputTiming
+{
+protected:
+    CThorStrandedActivity &parent;
+    IEngineRowStream *inputStream;
+    unsigned numProcessedLastGroup = 0;
+    const bool timeActivities;
+    bool stopped = false;
+    unsigned outputId; // if activity had >1 , this identifies (for tracing purposes) which output this strand belongs to.
+    Linked<IHThorArg> baseHelper;
+    rowcount_t rowsProcessed;
+
+protected:
+    inline IHThorArg *queryHelper() const { return baseHelper; }
+
+public:
+    explicit CThorStrandProcessor(CThorStrandedActivity &_parent, IEngineRowStream *_inputStream, unsigned _outputId);
+    __declspec(noreturn) void processAndThrowOwnedException(IException *_e) __attribute__((noreturn));
+    rowcount_t getCount() const { return rowsProcessed; }
+    virtual void start()
+    {
+        rowsProcessed = 0;
+        numProcessedLastGroup = 0;
+        resetTiming();
+    }
+    virtual void reset()
+    {
+        rowsProcessed = 0;
+        stopped = false;
+    }
+
+// IRowStream
+    virtual void stop() override;
+// IEngineRowStream
+    virtual void resetEOF() override
+    {
+        inputStream->resetEOF();
+    }
+};
+
+class CThorStrandedActivity : public CSlaveActivity
+{
+protected:
+    CThorStrandOptions strandOptions;
+    IArrayOf<CThorStrandProcessor> strands;
+    Owned<IStrandBranch> branch;
+    Owned<IStrandJunction> splitter;
+    Owned<IStrandJunction> sourceJunction; // A junction applied to the output of a source activity
+    std::atomic<unsigned> active;
+protected:
+    void onStartStrands();
+public:
+    CThorStrandedActivity(CGraphElementBase *container)
+        : CSlaveActivity(container), strandOptions(*container), active(0)
+    {
+    }
+
+    void strandedStop();
+
+    virtual void start() override;
+    virtual void reset() override;
+    virtual CThorStrandProcessor *createStrandProcessor(IEngineRowStream *instream) = 0;
+
+    //MORE: Possibly this class should be split into two for sinks and non sinks...
+    virtual CThorStrandProcessor *createStrandSourceProcessor(bool inputOrdered) = 0;
+
+    inline unsigned numStrands() const { return strands.ordinality(); }
+
+// IThorDataLink
+    virtual IStrandJunction *getOutputStreams(CActivityBase &_ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override;
+    virtual unsigned __int64 queryTotalCycles() const override;
+    virtual void dataLinkSerialize(MemoryBuffer &mb) const override;
 };
 
+
+
 class graphslave_decl CSlaveGraphElement : public CGraphElementBase
 {
 public:
@@ -161,7 +401,7 @@ public:
     virtual StringBuffer &getWorkUnitValue(const char *prop, StringBuffer &str) const;
     virtual bool getWorkUnitValueBool(const char *prop, bool defVal) const;
     virtual IThorAllocator *getThorAllocator(unsigned channel);
-    virtual void debugRequest(CMessageBuffer &msg, const char *request) const;
+    virtual void debugRequest(MemoryBuffer &msg, const char *request) const;
 
 // IExceptionHandler
     virtual bool fireException(IException *e)

thorlcr/master/thmastermain.cpp

@@ -640,7 +640,7 @@ int main( int argc, char *argv[]  )
 #ifndef __64BIT__
             if (maxMem > 2048)
             {
-            	// 32 bit OS doesn't handle whole physically installed RAM
+                // 32 bit OS doesn't handle whole physically installed RAM
                 maxMem = 2048;
             }
 #ifdef __ARM_ARCH_7A__

thorlcr/slave/slave.hpp

@@ -39,20 +39,10 @@
 #include "roxiestream.hpp"
 
 
-/* ---- To implement IThorDataLink you need ----
-    virtual const void *nextRow() = 0;
-    virtual void stop();
-    virtual void start();
-    virtual bool isGrouped();
-    virtual bool getMetaInfo(ThorDataLinkMetaInfo &info);
-*/
-
-
 struct ThorDataLinkMetaInfo
 {
     __int64     totalRowsMin;           // set to 0 if not known
     __int64     totalRowsMax;           // set to -1 if not known
-    rowcount_t  rowsOutput;             // rows already output (supported by all data links)
     offset_t    spilled;                // amount "spilled" to disk (approx) (offset_t)-1 for not known
 
     bool        isSource;
@@ -72,23 +62,67 @@ struct ThorDataLinkMetaInfo
 #pragma warning (push)
 #pragma warning( disable : 4275 )
 #endif
-class CActivityBase;
 
-interface IThorDataLink : extends IEngineRowStream
+#define MAX_SENSIBLE_STRANDS 1024 // Architecture dependent...
+class CThorStrandOptions
 {
-    virtual void start() = 0;
-    virtual bool isGrouped() = 0;
-    virtual IInputSteppingMeta *querySteppingMeta() { return NULL; }
-    virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) { return false; }
-    virtual void resetEOF() { }
+    // Typically set from hints, common to many stranded activities
+public:
+    explicit CThorStrandOptions(CGraphElementBase &container)
+    {
+        //PARALLEL(1) can be used to explicitly disable parallel processing.
+        numStrands = container.queryXGMML().getPropInt("att[@name='parallel']/@value", 0);
+        if ((numStrands == NotFound) || (numStrands > MAX_SENSIBLE_STRANDS))
+            numStrands = getAffinityCpus();
+        if (0 == numStrands)
+            numStrands = container.queryJob().getOptInt("forceNumStrands");
+        blockSize = container.queryJob().getOptInt("strandBlockSize");
+    }
+public:
+    unsigned numStrands = 0; // if 1 it forces single-stranded operations.  (Useful for testing.)
+    unsigned blockSize = 0;
+};
+
 
-// information routines 
+interface IStrandJunction;
+interface IOrderedCallbackCollection;
+class CSlaveActivity;
+interface IThorDataLink : extends IInterface
+{
+    virtual void start() = 0; // prepares input
+    virtual CSlaveActivity *queryFromActivity() = 0; // activity that has this as an output
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info) = 0;
-    virtual CActivityBase *queryFromActivity() = 0; // activity that has this as an output
-    virtual void dataLinkSerialize(MemoryBuffer &mb)=0;
-    virtual unsigned __int64 queryTotalCycles() const=0;
-    virtual unsigned __int64 queryEndCycles() const=0;
+    virtual bool isGrouped() const { return false; }
+    virtual IOutputMetaData * queryOutputMeta() const = 0;
+    virtual unsigned queryOutputIdx() const = 0;
+    virtual bool isInputOrdered(bool consumerOrdered) const = 0;
+    virtual IStrandJunction *getOutputStreams(CActivityBase &_ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) = 0;
+    virtual void setOutputStream(unsigned index, IEngineRowStream *stream) = 0;
+// progress methods
+    virtual void dataLinkSerialize(MemoryBuffer &mb) const = 0;
+// timing methods
+    virtual unsigned __int64 queryTotalCycles() const = 0;
+    virtual unsigned __int64 queryEndCycles() const = 0;
+// debugging methods
     virtual void debugRequest(MemoryBuffer &mb) = 0;
+// Stepping methods
+    virtual IInputSteppingMeta *querySteppingMeta() { return NULL; }
+    virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) { return false; }
+};
+
+// helper interface. Used by maintainer of output links
+interface IThorDataLinkExt : extends IThorDataLink
+{
+    virtual void setOutputIdx(unsigned idx) = 0;
+};
+
+class CThorInput;
+interface IThorSlaveActivity
+{
+    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) = 0;
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) = 0;
+    virtual void processDone(MemoryBuffer &mb) = 0;
+    virtual void reset() = 0;
 };
 #ifdef _MSC_VER
 #pragma warning (pop)

thorlcr/slave/slave.ipp

@@ -64,22 +64,18 @@ protected:
     IRangeCompare *stepCompare;
 public:
     IInputSteppingMeta *inputStepping;
-    
+
     CThorSteppable(CSlaveActivity *_activity) { stepCompare = NULL; inputStepping = NULL; }
-    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered)
     {
-        if (0 == index)
+        if (0 == index && input.itdl)
         {
-            IThorDataLink *input = inputActivity ? ((CSlaveActivity *)inputActivity)->queryInput(inputOutIdx) : NULL;
-            if (input)
-            {
-                inputStepping = input->querySteppingMeta();
-                if (inputStepping)
-                    stepCompare = inputStepping->queryCompare();
-            }
+            inputStepping = input.itdl->querySteppingMeta();
+            if (inputStepping)
+                stepCompare = inputStepping->queryCompare();
         }
     }
-    virtual IInputSteppingMeta *querySteppingMeta() { return inputStepping; }    
+    virtual IInputSteppingMeta *querySteppingMeta() { return inputStepping; }
 };
 
 
@@ -94,25 +90,32 @@ interface IThorNWayInput
 
 class CThorNarySlaveActivity : public CSlaveActivity
 {
+    typedef CSlaveActivity PARENT;
+    
 protected:
     PointerArrayOf<IThorDataLink> expandedInputs;
+    Owned<IStrandJunction> *expandedJunctions = nullptr;
+    IPointerArrayOf<IEngineRowStream> expandedStreams;
 
 public:
     CThorNarySlaveActivity(CGraphElementBase *container) : CSlaveActivity(container)
     {
     }
-    void start()
+    ~CThorNarySlaveActivity()
+    {
+        delete [] expandedJunctions;
+    }
+    virtual void start() override
     {
         ForEachItemIn(i, inputs)
         {
-            IThorDataLink *cur = inputs.item(i);
+            IThorDataLink *cur = queryInput(i);
             CActivityBase *activity = cur->queryFromActivity();
             IThorNWayInput *nWayInput = dynamic_cast<IThorNWayInput *>(cur);
             if (nWayInput)
             {
                 unsigned numRealInputs = nWayInput->numConcreteOutputs();
-                unsigned i = 0;
-                for (; i < numRealInputs; i++)
+                for (unsigned i=0; i < numRealInputs; i++)
                 {
                     IThorDataLink *curReal = nWayInput->queryConcreteInput(i);
                     expandedInputs.append(curReal);
@@ -123,29 +126,42 @@ public:
         }
         ForEachItemIn(ei, expandedInputs)
             expandedInputs.item(ei)->start();
+        expandedJunctions = new Owned<IStrandJunction> [expandedInputs.length()];
+        ForEachItemIn(idx, expandedInputs)
+        {
+            expandedStreams.append(connectSingleStream(*this, expandedInputs.item(idx), 0, expandedJunctions[idx], true));  // MORE - is the index 0 right?
+            startJunction(expandedJunctions[idx]);
+        }
+        dataLinkStart();
     }
     void stop()
     {
-        ForEachItemIn(ei, expandedInputs)
-            expandedInputs.item(ei)->stop();
+        ForEachItemIn(ei, expandedStreams)
+            expandedStreams.item(ei)->stop();
+        ForEachItemIn(idx, expandedInputs)
+            resetJunction(expandedJunctions[idx]);
         expandedInputs.kill();
+        expandedStreams.kill();
+        delete [] expandedJunctions;
+        expandedJunctions = nullptr;
     }
 };
 
 
-class CThorSteppedInput : public CSimpleInterface, implements ISteppedInput
+class CThorSteppedInput : public CSimpleInterfaceOf<ISteppedInput>
 {
 protected:
+    IEngineRowStream *inputStream;
     IThorDataLink *input;
 
     virtual const void *nextInputRow()
     {
-        return input->ungroupedNextRow();
+        return inputStream->ungroupedNextRow();
     }
     virtual const void *nextInputRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
     {
         assertex(wasCompleteMatch);
-        return input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
+        return inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
     }
     virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) { return input->gatherConjunctions(collector); }
     virtual IInputSteppingMeta *queryInputSteppingMeta()
@@ -154,12 +170,12 @@ protected:
     }
     virtual void resetEOF()
     {
-        input->resetEOF(); 
+        inputStream->resetEOF();
     }
 public:
-    IMPLEMENT_IINTERFACE_USING(CSimpleInterface)
-
-    CThorSteppedInput(IThorDataLink *_input) : input(_input) { }
+    CThorSteppedInput(IThorDataLink *_input, IEngineRowStream *_inputStream) : input(_input), inputStream(_inputStream)
+    {
+    }
 };
 
 

thorlcr/slave/slwatchdog.cpp

@@ -137,7 +137,7 @@ public:
             activeGraphs.zap(graph);
         }
     }
-    virtual void debugRequest(CMessageBuffer &msg, const char *request) const
+    virtual void debugRequest(MemoryBuffer &msg, const char *request) const
     {
         Owned<IPTree> req = createPTreeFromXMLString(request);
 
@@ -161,7 +161,7 @@ public:
             if (element)
             {
                 CSlaveActivity *activity = (CSlaveActivity*) element->queryActivity();
-                if (activity) activity->debugRequest(edgeIdx,msg);
+                if (activity) activity->debugRequest(edgeIdx, msg);
             }
         }
     }

thorlcr/slave/slwatchdog.hpp

@@ -27,7 +27,7 @@ interface ISlaveWatchdog : extends IInterface
     virtual void startGraph(CGraphBase &graph) = 0;
     virtual void stopGraph(CGraphBase &graph, MemoryBuffer *mb=NULL) = 0;
     virtual void stop() = 0;
-    virtual void debugRequest(CMessageBuffer &msg, const char *request) const = 0;
+    virtual void debugRequest(MemoryBuffer &msg, const char *request) const = 0;
 };
 
 ISlaveWatchdog *createProgressHandler(bool udp=false);

thorlcr/slave/thslavemain.cpp

@@ -71,6 +71,9 @@ USE_JLIB_ALLOC_HOOK;
 static SocketEndpoint slfEp;
 static unsigned mySlaveNum;
 
+static const unsigned defaultStrandBlockSize = 512;
+static const unsigned defaultForceNumStrands = 0;
+
 static char **cmdArgs;
 void mergeCmdParams(IPropertyTree *props)
 {
@@ -123,6 +126,23 @@ static bool RegisterSelf(SocketEndpoint &masterEp)
         unsigned slaveBasePort = globals->getPropInt("@slaveport", DEFAULT_THORSLAVEPORT);
         setClusterGroup(masterNode, rawGroup, slavesPerNode, channelsPerSlave, slaveBasePort, localThorPortInc);
 
+        unsigned numStrands, blockSize;
+        if (globals->hasProp("Debug/@forceNumStrands"))
+            numStrands = globals->getPropInt("Debug/@forceNumStrands");
+        else
+        {
+            numStrands = defaultForceNumStrands;
+            globals->setPropInt("Debug/@forceNumStrands", defaultForceNumStrands);
+        }
+        if (globals->hasProp("Debug/@strandBlockSize"))
+            blockSize = globals->getPropInt("Debug/@strandBlockSize");
+        else
+        {
+            blockSize = defaultStrandBlockSize;
+            globals->setPropInt("Debug/@strandBlockSize", defaultStrandBlockSize);
+        }
+        PROGLOG("Strand defaults: numStrands=%u, blockSize=%u", numStrands, blockSize);
+
         const char *_masterBuildTag = globals->queryProp("@masterBuildTag");
         const char *masterBuildTag = _masterBuildTag?_masterBuildTag:"no build tag";
         PROGLOG("Master build: %s", masterBuildTag);

thorlcr/slave/traceslave.hpp

@@ -18,10 +18,26 @@
 #ifndef TRACESLAVE_HPP
 #define TRACESLAVE_HPP
 
-#include "thmem.hpp"
-#include "jstats.h"
+#include <eclhelper.hpp>
+#include <jatomic.hpp>
+#include <jbuff.hpp>
+#include <jdebug.hpp>
+#include <jiface.hpp>
+#include <jlog.hpp>
+#include <jscm.hpp>
+#include <jstats.h>
+#include <jstring.hpp>
+#include <platform.h>
+#include <stddef.h>
+#include <thmem.hpp>
+#include <thorxmlwrite.hpp>
+
+interface IThorDebug : extends IInterface
+{
+    virtual void debugRequest(MemoryBuffer &mb) = 0;
+};
 
-class CTracingThorDataLink : implements CInterfaceOf<IThorDataLink>
+class CTracingStream : public CSimpleInterfaceOf<IEngineRowStream>, implements IThorDebug
 {
 private:
     class CThorRowHistory
@@ -140,49 +156,18 @@ private:
     CTraceQueue buffers[2];
     CTraceQueue rowBufferLogCache;
     atomic_t rowBufInUse;
-    Owned<IThorDataLink> thorDataLink;
+    IThorDataLink *thorDataLink;
+    IEngineRowStream *inputStream;
     IHThorArg *helper;
 
     inline void enqueueRowForTrace(const void *row)
     {
-        buffers[atomic_read(&rowBufInUse)].enqueue(row,thorDataLink->queryEndCycles());
+        buffers[atomic_read(&rowBufInUse)].enqueue(row, thorDataLink->queryEndCycles());
     }
 public:
-    virtual void start()
-    {
-        thorDataLink->start();
-    }
-    virtual bool isGrouped() { return thorDataLink->isGrouped(); }
-    virtual const void *nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
-    {
-        const void *row = thorDataLink->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
-        enqueueRowForTrace(row);
-        return row;
-    }    // can only be called on stepping fields.
-    virtual IInputSteppingMeta *querySteppingMeta() { return thorDataLink->querySteppingMeta(); }
-    virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) { return thorDataLink->gatherConjunctions(collector); }
-    virtual void resetEOF() { thorDataLink->resetEOF(); }
-
-    virtual void getMetaInfo(ThorDataLinkMetaInfo &info) { thorDataLink->getMetaInfo(info); }
-    virtual CActivityBase *queryFromActivity() {return thorDataLink->queryFromActivity(); } // activity that has this as an output
-    virtual void dataLinkSerialize(MemoryBuffer &mb) {thorDataLink->dataLinkSerialize(mb); }
-    virtual unsigned __int64 queryTotalCycles() const { return thorDataLink->queryTotalCycles(); }
-    virtual unsigned __int64 queryEndCycles() const { return thorDataLink->queryEndCycles(); }
-
-    virtual const void *nextRow()
-    {
-        const void *row = thorDataLink->nextRow();
-        enqueueRowForTrace(row);
-        return row;
-    }
-    virtual void stop()
-    {
-        thorDataLink->stop();
-    }
-
-    inline const void *ungroupedNextRow() { return thorDataLink->ungroupedNextRow(); }
+    IMPLEMENT_IINTERFACE_USING(CSimpleInterfaceOf<IEngineRowStream>);
 
-    virtual void debugRequest(MemoryBuffer &mb)
+    virtual void debugRequest(MemoryBuffer &mb) override
     {
         // NOTE - cannot be called by more than one thread
         buffers[1].init(traceQueueSize+1);
@@ -200,8 +185,21 @@ public:
         rowBufferLogCache.dump(mb, helper);
     }
 
-    CTracingThorDataLink(IThorDataLink *_input, IHThorArg *_helper, unsigned _traceQueueSize)
-        : thorDataLink(_input), helper(_helper), traceQueueSize(_traceQueueSize)
+// IEngineRowStream
+    virtual void resetEOF() override { inputStream->resetEOF(); }
+    virtual const void *nextRow() override
+    {
+        const void *row = inputStream->nextRow();
+        enqueueRowForTrace(row);
+        return row;
+    }
+    virtual void stop() override
+    {
+        inputStream->stop();
+    }
+
+    CTracingStream(IThorDataLink *_thorDataLink, IEngineRowStream *_inputStream, IHThorArg *_helper, unsigned _traceQueueSize)
+        : thorDataLink(_thorDataLink), inputStream(_inputStream), helper(_helper), traceQueueSize(_traceQueueSize)
     {
         atomic_set(&rowBufInUse, 0);
         buffers[0].init(traceQueueSize+1);