HPCC-9335 Improvements to single stepping nested merge joins

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/thorhelper/thorstep.cpp

@@ -884,7 +884,7 @@ CMergeJoinProcessor::CMergeJoinProcessor(IHThorNWayMergeJoinArg & _arg) : helper
 
     assertex(helper.numOrderFields() == mergeSteppingMeta->getNumFields());
     bool hasPostfilter = false;
-    thisSteppingMeta.init(mergeSteppingMeta->getNumFields(), mergeSteppingMeta->queryFields(), stepCompare, mergeSteppingMeta->queryDistance(), hasPostfilter);
+    thisSteppingMeta.init(mergeSteppingMeta->getNumFields(), mergeSteppingMeta->queryFields(), stepCompare, mergeSteppingMeta->queryDistance(), hasPostfilter, SSFhaspriority|SSFisjoin);
 }
 
 CMergeJoinProcessor::~CMergeJoinProcessor()
@@ -898,6 +898,13 @@ void CMergeJoinProcessor::addInput(ISteppedInput * _input)
     IInputSteppingMeta * _meta = _input->queryInputSteppingMeta();
     verifySteppingCompatible(_meta, mergeSteppingMeta);
     rawInputs.append(*LINK(_input));
+    if (!_meta->hasPriority())
+        thisSteppingMeta.removePriority();
+    else
+        thisSteppingMeta.intersectPriority(_meta->getPriority());
+
+    if (_meta->isDistributed())
+        thisSteppingMeta.setDistributed();
 }
 
 void CMergeJoinProcessor::afterProcessing()
@@ -1190,6 +1197,22 @@ void CMergeJoinProcessor::setCandidateRow(const void * row, bool inputsMayBeEmpt
     outputProcessor->beforeProcessCandidates(restrictionRow, inputsMayBeEmpty, matched);
 }
 
+void CMergeJoinProcessor::connectRemotePriorityInputs()
+{
+    CIArrayOf<OrderedInput> orderedInputs;
+
+    ForEachItemIn(i, inputs)
+    {
+        CSteppedInputLookahead & cur = inputs.item(i);
+        if (!cur.hasPriority() || !cur.readsRowsRemotely())
+            return;
+
+        orderedInputs.append(*new OrderedInput(cur, i, false));
+    }
+    orderedInputs.sort(compareInitialInputOrder);
+    associateRemoteInputs(orderedInputs, orderedInputs.ordinality());
+    combineConjunctions = false;
+}
 
 //---------------------------------------------------------------------------
 
@@ -1200,6 +1223,7 @@ CAndMergeJoinProcessor::CAndMergeJoinProcessor(IHThorNWayMergeJoinArg & _arg) :
 void CAndMergeJoinProcessor::beforeProcessing(IEngineRowAllocator * _inputAllocator, IEngineRowAllocator * _outputAllocator)
 {
     CMergeJoinProcessor::beforeProcessing(_inputAllocator, _outputAllocator);
+    connectRemotePriorityInputs();
     if (flags & IHThorNWayMergeJoinArg::MJFtransform)
         createEqualityJoinProcessor();
     else

common/thorhelper/thorstep.hpp

@@ -28,8 +28,10 @@ interface IInputSteppingMeta : public ISteppingMeta
 //??virtual ISteppingMeta * querySteppingMeta() const = 0
     virtual bool hasPostFilter() = 0;                           // make this const?
     virtual bool isDistributed() const = 0;
-    virtual unsigned getSteppedFlags() = 0;
+    virtual unsigned getSteppedFlags() const = 0;
     virtual double getPriority() = 0;
+
+    inline bool hasPriority() const { return (getSteppedFlags() & SSFhaspriority) != 0; }
 };
 
 

common/thorhelper/thorstep.ipp

@@ -65,10 +65,10 @@ public:
     void intersect(IInputSteppingMeta * inputMeta);
     void init(ISteppingMeta * meta, bool _postFiltered)
     {
-        init(meta->getNumFields(), meta->queryFields(), meta->queryCompare(), meta->queryDistance(), _postFiltered);
+        init(meta->getNumFields(), meta->queryFields(), meta->queryCompare(), meta->queryDistance(), _postFiltered, 0);
     }
 
-    void init(unsigned _numFields, const CFieldOffsetSize * _fields, IRangeCompare * _rangeCompare, IDistanceCalculator * _distance, bool _postFiltered)
+    void init(unsigned _numFields, const CFieldOffsetSize * _fields, IRangeCompare * _rangeCompare, IDistanceCalculator * _distance, bool _postFiltered, unsigned flags)
     {
         hadStepExtra = false;
         numFields = _numFields;
@@ -76,7 +76,15 @@ public:
         rangeCompare = _rangeCompare;
         distance = _distance;
         postFiltered = _postFiltered;
+        stepFlags = flags;
+        priority = 1.0e+300;
     }
+    void intersectPriority(double value)
+    {
+        if (priority > value)
+            priority = value;
+    }
+    void removePriority() { stepFlags &= ~SSFhaspriority; }
     void setDistributed() { distributed = true; }
     void setExtra(IHThorSteppedSourceExtra * extra)
     {
@@ -111,7 +119,7 @@ public:
         return postFiltered;
     }
 
-    virtual unsigned getSteppedFlags() { return stepFlags; }
+    virtual unsigned getSteppedFlags() const { return stepFlags; }
     virtual double getPriority() { return priority; }
 
 protected:
@@ -600,6 +608,7 @@ protected:
         if (hasCandidates())
             finishCandidates();
     }
+    void connectRemotePriorityInputs();
     bool createConjunctionOptimizer();
     void setCandidateRow(const void * row, bool inputsMayBeEmpty, const bool * matched);
 

common/thorhelper/thorstep2.cpp

@@ -56,7 +56,7 @@ So the latest approach is as follows:
    occasionally skips a long way (e.g. state) it will get corrected/used. 
 
   */
-static int compareInitialInputOrder(CInterface * * _left, CInterface * * _right)
+int compareInitialInputOrder(CInterface * * _left, CInterface * * _right)
 {
     OrderedInput * left = static_cast<OrderedInput *>(*_left);
     OrderedInput * right = static_cast<OrderedInput *>(*_right);
@@ -298,6 +298,38 @@ void CSteppedConjunctionOptimizer::afterProcessing()
 
 }
 
+void associateRemoteInputs(CIArrayOf<OrderedInput> & orderedInputs, unsigned numPriorityInputs)
+{
+    //If we know for sure the primary input, then tag it as worth reading ahead - otherwise it will be dynamically set later.
+    if (numPriorityInputs > 0)
+    {
+        OrderedInput & input0 = orderedInputs.item(0);
+        //Only read ahead etc. if this is a real index - not if it is a join.
+        if (!input0.isJoin())
+        {
+            input0.setReadAhead(true);
+            input0.setAlwaysReadExact();
+        }
+    }
+
+    //Work out the last input of known priority which is read remotely.
+    unsigned maxPriorityRemote = numPriorityInputs;
+    while ((maxPriorityRemote >= 2) && !orderedInputs.item(maxPriorityRemote-1).readsRowsRemotely())
+        maxPriorityRemote--;
+
+    //If the second ordered input is known to be read remotely, then we want to send multiple seek requests at the same time.
+    //MORE: Maybe we should consider doing this to all other inputs if only one priority input is known.
+    if (maxPriorityRemote >= 2)
+    {
+        for (unsigned i=1; i < maxPriorityRemote; i++)
+        {
+            IMultipleStepSeekInfo * seekInfo = orderedInputs.item(i-1).createMutipleReadWrapper();
+            orderedInputs.item(i).createMultipleSeekWrapper(seekInfo);
+        }
+    }
+}
+
+
 void CSteppedConjunctionOptimizer::beforeProcessing()
 {
     //NB: This function is only called once, after we have decided it is worth processing.
@@ -324,30 +356,10 @@ void CSteppedConjunctionOptimizer::beforeProcessing()
     }
 
     maxOptimizeInput = numPriorityInputs + numOptimizeInputs;
-    //If we know for sure the primary input, then tag it as worth reading ahead - otherwise it will be dynamically set later.
-    if (numPriorityInputs > 0)
-    {
-        orderedInputs.item(0).setReadAhead(true);
-        orderedInputs.item(0).setAlwaysReadExact();
-    }
-
-    //Work out the last input of known priority whis is read remotely.
-    unsigned maxPriorityRemote = numPriorityInputs;
-    while ((maxPriorityRemote >= 2) && !orderedInputs.item(maxPriorityRemote-1).readsRowsRemotely())
-        maxPriorityRemote--;
 
-    //If the second ordered input is known to be read remotely, then we want to send multiple seek requests at the same time.
-    //MORE: Maybe we should consider doing this to all other inputs if only one priority input is known.
-    if (maxPriorityRemote >= 2)
-    {
-        for (unsigned i=1; i < maxPriorityRemote; i++)
-        {
-            IMultipleStepSeekInfo * seekInfo = orderedInputs.item(i-1).createMutipleReadWrapper();
-            orderedInputs.item(i).createMultipleSeekWrapper(seekInfo);
-        }
-    }
+    associateRemoteInputs(orderedInputs, numPriorityInputs);
     
-    //MORE: If some inputs have known priroity, and other remote inputs don't, then we could consider 
+    //MORE: If some inputs have known priority, and other remote inputs don't, then we could consider
     //      connecting the unknown inputs to the last known inputs.
     ForEachItemIn(i4, joins)
         joins.item(i4).markRestrictedJoin(numEqualFields);

common/thorhelper/thorstep2.ipp

@@ -106,7 +106,8 @@ public:
 
     inline bool canOptimizeOrder() { return optimizeOrder; }
     inline unsigned getNumSamples() { return numSamples; }
-    inline bool hasPriority() { return (stepFlags & SSFhaspriority) != 0; }
+    inline bool hasPriority() const { return (stepFlags & SSFhaspriority) != 0; }
+    inline bool isJoin() const { return (stepFlags & SSFisjoin) != 0; }
     inline double getPriority() { return priority; }
     inline bool readsRowsRemotely() { return input->readsRowsRemotely(); }
 
@@ -205,4 +206,7 @@ private:
     unsigned nextToMatch() const;
 };
 
+void associateRemoteInputs(CIArrayOf<OrderedInput> & orderedInputs, unsigned numPriorityInputs);
+int compareInitialInputOrder(CInterface * * _left, CInterface * * _right);
+
 #endif

roxie/ccd/ccdactivities.cpp

@@ -3252,16 +3252,24 @@ public:
             assertex(numSeeks); // Given that we put the first seek in here to there should always be at least one!
             steppingRow = smartStepInfoValue; // the first of them...
             stepExtra.set(flags, NULL);
-#ifdef _DEBUG
-            logctx.CTXLOG("%d seek rows provided", numSeeks);
-            for (unsigned i = 0; i < numSeeks; i++)
+            if (logctx.queryTraceLevel() > 10)
             {
-                StringBuffer b;
-                for (unsigned j = 0; j < steppingLength; j++)
-                    b.appendf("%02x ", steppingRow[i*steppingLength + j]);
-                logctx.CTXLOG("Seek row %d: %s", i+1, b.str());
+                logctx.CTXLOG("%d seek rows provided. mismatch(%d) readahead(%d) onlyfirst(%d)", numSeeks,
+                             (int)stepExtra.returnMismatches(), (int)stepExtra.readAheadManyResults(), (int)stepExtra.onlyReturnFirstSeekMatch());
+
+                for (unsigned i = 0; i < numSeeks; i++)
+                {
+                    StringBuffer b;
+                    for (unsigned j = 0; j < steppingLength; j++)
+                        b.appendf("%02x ", steppingRow[i*steppingLength + j]);
+                    logctx.CTXLOG("Seek row %d: %s", i+1, b.str());
+                }
             }
-#endif
+        }
+        else
+        {
+            if (logctx.queryTraceLevel() > 10)
+                logctx.CTXLOG("0 seek rows provided.");
         }
     }
 

rtl/include/eclhelper.hpp

@@ -2225,6 +2225,7 @@ enum
     SSFalwaysfilter     = 0x0001,
     SSFhaspriority      = 0x0002,
     SSFhasprefetch      = 0x0004,
+    SSFisjoin           = 0x0008,
 };
 
 interface IHThorSteppedSourceExtra : extends IInterface

testing/ecl/key/stepping7.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><src>1</src><doc>5952</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>5978</doc><cnt>16</cnt></Row>
+ <Row><src>1</src><doc>26929</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27753</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>34964</doc><cnt>4</cnt></Row>
+</Dataset>

testing/ecl/key/stepping7a.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><src>1</src><doc>5952</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>5978</doc><cnt>16</cnt></Row>
+ <Row><src>1</src><doc>26929</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27753</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>34964</doc><cnt>4</cnt></Row>
+</Dataset>

testing/ecl/key/stepping7b.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><src>1</src><doc>5952</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>5978</doc><cnt>16</cnt></Row>
+ <Row><src>1</src><doc>26929</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27753</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>34964</doc><cnt>4</cnt></Row>
+</Dataset>

testing/ecl/key/stepping7c.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><src>1</src><doc>5952</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>5978</doc><cnt>16</cnt></Row>
+ <Row><src>1</src><doc>26929</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27753</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>34964</doc><cnt>4</cnt></Row>
+</Dataset>

testing/ecl/key/stepping7d.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><src>1</src><doc>5952</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>5978</doc><cnt>16</cnt></Row>
+ <Row><src>1</src><doc>26929</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27753</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>34964</doc><cnt>4</cnt></Row>
+</Dataset>

testing/ecl/key/stepping7e.xml

@@ -0,0 +1,201 @@
+<Dataset name='Result 1'>
+ <Row><src>1</src><doc>2385</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>3147</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>3154</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>3497</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>3499</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>4092</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>4398</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>4848</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>4920</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>4980</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>5952</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>5957</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>5972</doc><cnt>10</cnt></Row>
+ <Row><src>1</src><doc>5978</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>8148</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>8573</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>8703</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>8862</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>8906</doc><cnt>18</cnt></Row>
+ <Row><src>1</src><doc>8907</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>8916</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>8930</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>8948</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>9608</doc><cnt>10</cnt></Row>
+ <Row><src>1</src><doc>10232</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>10238</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>11173</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>11242</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>11488</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>11696</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>11733</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>11753</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>11899</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>12129</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>12130</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>12133</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>12144</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>12149</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>12304</doc><cnt>10</cnt></Row>
+ <Row><src>1</src><doc>12315</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>12323</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>12332</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>12335</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>12348</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>12371</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>12379</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>12986</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>13008</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>13124</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>13269</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>13288</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>13294</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>15501</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>15540</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>15873</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>17470</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>18425</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>18436</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>19617</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>20444</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>20450</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>20840</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>20927</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>20994</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>21695</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>21724</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>21826</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>21933</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>22309</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>22791</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>23045</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>23088</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>23147</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>23202</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>23204</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>23215</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>24183</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>24189</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>24191</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>24211</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>24389</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>24442</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>24614</doc><cnt>18</cnt></Row>
+ <Row><src>1</src><doc>24618</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>24627</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>24630</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>24631</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>25157</doc><cnt>18</cnt></Row>
+ <Row><src>1</src><doc>25762</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>25932</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>25962</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>25978</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>26165</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>26441</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>26665</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>26731</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>26741</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>26929</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>26933</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>26963</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>26997</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>27151</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>27179</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27202</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27260</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>27336</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>27426</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>27534</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>27709</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27751</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>27752</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>27753</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>27765</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>34964</doc><cnt>3</cnt></Row>
+ <Row><src>2</src><doc>6</doc><cnt>31</cnt></Row>
+ <Row><src>2</src><doc>33</doc><cnt>1415</cnt></Row>
+ <Row><src>2</src><doc>46</doc><cnt>1384</cnt></Row>
+ <Row><src>2</src><doc>48</doc><cnt>1002</cnt></Row>
+ <Row><src>2</src><doc>66</doc><cnt>29</cnt></Row>
+ <Row><src>2</src><doc>84</doc><cnt>57</cnt></Row>
+ <Row><src>2</src><doc>92</doc><cnt>142</cnt></Row>
+ <Row><src>2</src><doc>108</doc><cnt>206</cnt></Row>
+ <Row><src>2</src><doc>110</doc><cnt>1611</cnt></Row>
+ <Row><src>2</src><doc>111</doc><cnt>1959</cnt></Row>
+ <Row><src>2</src><doc>112</doc><cnt>72</cnt></Row>
+ <Row><src>2</src><doc>153</doc><cnt>109</cnt></Row>
+ <Row><src>2</src><doc>156</doc><cnt>159</cnt></Row>
+ <Row><src>2</src><doc>170</doc><cnt>224</cnt></Row>
+ <Row><src>2</src><doc>194</doc><cnt>460</cnt></Row>
+ <Row><src>2</src><doc>206</doc><cnt>27</cnt></Row>
+ <Row><src>2</src><doc>252</doc><cnt>660</cnt></Row>
+ <Row><src>2</src><doc>256</doc><cnt>33</cnt></Row>
+ <Row><src>2</src><doc>272</doc><cnt>119</cnt></Row>
+ <Row><src>2</src><doc>275</doc><cnt>1318</cnt></Row>
+ <Row><src>2</src><doc>282</doc><cnt>17</cnt></Row>
+ <Row><src>2</src><doc>293</doc><cnt>143</cnt></Row>
+ <Row><src>2</src><doc>300</doc><cnt>2713</cnt></Row>
+ <Row><src>2</src><doc>302</doc><cnt>4691</cnt></Row>
+ <Row><src>2</src><doc>310</doc><cnt>479</cnt></Row>
+ <Row><src>2</src><doc>313</doc><cnt>122</cnt></Row>
+ <Row><src>2</src><doc>317</doc><cnt>43</cnt></Row>
+ <Row><src>2</src><doc>323</doc><cnt>188</cnt></Row>
+ <Row><src>2</src><doc>331</doc><cnt>160</cnt></Row>
+ <Row><src>2</src><doc>340</doc><cnt>195</cnt></Row>
+ <Row><src>2</src><doc>357</doc><cnt>137</cnt></Row>
+ <Row><src>2</src><doc>360</doc><cnt>35</cnt></Row>
+ <Row><src>2</src><doc>364</doc><cnt>45</cnt></Row>
+ <Row><src>2</src><doc>372</doc><cnt>520</cnt></Row>
+ <Row><src>2</src><doc>376</doc><cnt>565</cnt></Row>
+ <Row><src>2</src><doc>377</doc><cnt>66</cnt></Row>
+ <Row><src>2</src><doc>378</doc><cnt>340</cnt></Row>
+ <Row><src>2</src><doc>380</doc><cnt>66</cnt></Row>
+ <Row><src>2</src><doc>387</doc><cnt>206</cnt></Row>
+ <Row><src>2</src><doc>388</doc><cnt>153</cnt></Row>
+ <Row><src>2</src><doc>392</doc><cnt>225</cnt></Row>
+ <Row><src>2</src><doc>395</doc><cnt>46</cnt></Row>
+ <Row><src>2</src><doc>396</doc><cnt>1014</cnt></Row>
+ <Row><src>2</src><doc>402</doc><cnt>145</cnt></Row>
+ <Row><src>2</src><doc>417</doc><cnt>183</cnt></Row>
+ <Row><src>2</src><doc>418</doc><cnt>141</cnt></Row>
+ <Row><src>2</src><doc>440</doc><cnt>21</cnt></Row>
+ <Row><src>2</src><doc>441</doc><cnt>176</cnt></Row>
+ <Row><src>2</src><doc>453</doc><cnt>42</cnt></Row>
+ <Row><src>2</src><doc>487</doc><cnt>759</cnt></Row>
+ <Row><src>2</src><doc>489</doc><cnt>19</cnt></Row>
+ <Row><src>2</src><doc>493</doc><cnt>42</cnt></Row>
+ <Row><src>2</src><doc>496</doc><cnt>633</cnt></Row>
+ <Row><src>2</src><doc>498</doc><cnt>1616</cnt></Row>
+ <Row><src>2</src><doc>501</doc><cnt>1661</cnt></Row>
+ <Row><src>2</src><doc>503</doc><cnt>332</cnt></Row>
+ <Row><src>2</src><doc>505</doc><cnt>282</cnt></Row>
+ <Row><src>2</src><doc>517</doc><cnt>313</cnt></Row>
+ <Row><src>2</src><doc>518</doc><cnt>704</cnt></Row>
+ <Row><src>2</src><doc>527</doc><cnt>407</cnt></Row>
+ <Row><src>2</src><doc>537</doc><cnt>8</cnt></Row>
+ <Row><src>2</src><doc>540</doc><cnt>95</cnt></Row>
+ <Row><src>2</src><doc>548</doc><cnt>516</cnt></Row>
+ <Row><src>2</src><doc>558</doc><cnt>181</cnt></Row>
+ <Row><src>2</src><doc>596</doc><cnt>264</cnt></Row>
+ <Row><src>2</src><doc>604</doc><cnt>555</cnt></Row>
+ <Row><src>2</src><doc>615</doc><cnt>52</cnt></Row>
+ <Row><src>2</src><doc>618</doc><cnt>40</cnt></Row>
+ <Row><src>2</src><doc>622</doc><cnt>1220</cnt></Row>
+ <Row><src>2</src><doc>629</doc><cnt>45</cnt></Row>
+ <Row><src>2</src><doc>641</doc><cnt>1800</cnt></Row>
+ <Row><src>2</src><doc>652</doc><cnt>72</cnt></Row>
+ <Row><src>2</src><doc>654</doc><cnt>268</cnt></Row>
+ <Row><src>2</src><doc>657</doc><cnt>1208</cnt></Row>
+ <Row><src>2</src><doc>664</doc><cnt>259</cnt></Row>
+ <Row><src>2</src><doc>665</doc><cnt>526</cnt></Row>
+ <Row><src>2</src><doc>669</doc><cnt>201</cnt></Row>
+ <Row><src>2</src><doc>674</doc><cnt>636</cnt></Row>
+ <Row><src>2</src><doc>692</doc><cnt>42</cnt></Row>
+ <Row><src>2</src><doc>706</doc><cnt>38</cnt></Row>
+ <Row><src>2</src><doc>713</doc><cnt>310</cnt></Row>
+ <Row><src>2</src><doc>720</doc><cnt>111</cnt></Row>
+ <Row><src>2</src><doc>723</doc><cnt>222</cnt></Row>
+ <Row><src>2</src><doc>725</doc><cnt>855</cnt></Row>
+ <Row><src>2</src><doc>731</doc><cnt>341</cnt></Row>
+</Dataset>

testing/ecl/key/stepping7f.xml

@@ -0,0 +1,201 @@
+<Dataset name='Result 1'>
+ <Row><src>1</src><doc>2385</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>3147</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>3154</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>3497</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>3499</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>4092</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>4398</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>4848</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>4920</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>4980</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>5952</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>5957</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>5972</doc><cnt>10</cnt></Row>
+ <Row><src>1</src><doc>5978</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>8148</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>8573</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>8703</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>8862</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>8906</doc><cnt>18</cnt></Row>
+ <Row><src>1</src><doc>8907</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>8916</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>8930</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>8948</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>9608</doc><cnt>10</cnt></Row>
+ <Row><src>1</src><doc>10232</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>10238</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>11173</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>11242</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>11488</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>11696</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>11733</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>11753</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>11899</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>12129</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>12130</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>12133</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>12144</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>12149</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>12304</doc><cnt>10</cnt></Row>
+ <Row><src>1</src><doc>12315</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>12323</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>12332</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>12335</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>12348</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>12371</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>12379</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>12986</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>13008</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>13124</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>13269</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>13288</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>13294</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>15501</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>15540</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>15873</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>17470</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>18425</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>18436</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>19617</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>20444</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>20450</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>20840</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>20927</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>20994</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>21695</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>21724</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>21826</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>21933</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>22309</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>22791</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>23045</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>23088</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>23147</doc><cnt>13</cnt></Row>
+ <Row><src>1</src><doc>23202</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>23204</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>23215</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>24183</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>24189</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>24191</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>24211</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>24389</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>24442</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>24614</doc><cnt>18</cnt></Row>
+ <Row><src>1</src><doc>24618</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>24627</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>24630</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>24631</doc><cnt>11</cnt></Row>
+ <Row><src>1</src><doc>25157</doc><cnt>18</cnt></Row>
+ <Row><src>1</src><doc>25762</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>25932</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>25962</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>25978</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>26165</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>26441</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>26665</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>26731</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>26741</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>26929</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>26933</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>26963</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>26997</doc><cnt>9</cnt></Row>
+ <Row><src>1</src><doc>27151</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>27179</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27202</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27260</doc><cnt>5</cnt></Row>
+ <Row><src>1</src><doc>27336</doc><cnt>7</cnt></Row>
+ <Row><src>1</src><doc>27426</doc><cnt>6</cnt></Row>
+ <Row><src>1</src><doc>27534</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>27709</doc><cnt>3</cnt></Row>
+ <Row><src>1</src><doc>27751</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>27752</doc><cnt>4</cnt></Row>
+ <Row><src>1</src><doc>27753</doc><cnt>8</cnt></Row>
+ <Row><src>1</src><doc>27765</doc><cnt>2</cnt></Row>
+ <Row><src>1</src><doc>34964</doc><cnt>3</cnt></Row>
+ <Row><src>2</src><doc>6</doc><cnt>31</cnt></Row>
+ <Row><src>2</src><doc>33</doc><cnt>1415</cnt></Row>
+ <Row><src>2</src><doc>46</doc><cnt>1384</cnt></Row>
+ <Row><src>2</src><doc>48</doc><cnt>1002</cnt></Row>
+ <Row><src>2</src><doc>66</doc><cnt>29</cnt></Row>
+ <Row><src>2</src><doc>84</doc><cnt>57</cnt></Row>
+ <Row><src>2</src><doc>92</doc><cnt>142</cnt></Row>
+ <Row><src>2</src><doc>108</doc><cnt>206</cnt></Row>
+ <Row><src>2</src><doc>110</doc><cnt>1611</cnt></Row>
+ <Row><src>2</src><doc>111</doc><cnt>1959</cnt></Row>
+ <Row><src>2</src><doc>112</doc><cnt>72</cnt></Row>
+ <Row><src>2</src><doc>153</doc><cnt>109</cnt></Row>
+ <Row><src>2</src><doc>156</doc><cnt>159</cnt></Row>
+ <Row><src>2</src><doc>170</doc><cnt>224</cnt></Row>
+ <Row><src>2</src><doc>194</doc><cnt>460</cnt></Row>
+ <Row><src>2</src><doc>206</doc><cnt>27</cnt></Row>
+ <Row><src>2</src><doc>252</doc><cnt>660</cnt></Row>
+ <Row><src>2</src><doc>256</doc><cnt>33</cnt></Row>
+ <Row><src>2</src><doc>272</doc><cnt>119</cnt></Row>
+ <Row><src>2</src><doc>275</doc><cnt>1318</cnt></Row>
+ <Row><src>2</src><doc>282</doc><cnt>17</cnt></Row>
+ <Row><src>2</src><doc>293</doc><cnt>143</cnt></Row>
+ <Row><src>2</src><doc>300</doc><cnt>2713</cnt></Row>
+ <Row><src>2</src><doc>302</doc><cnt>4691</cnt></Row>
+ <Row><src>2</src><doc>310</doc><cnt>479</cnt></Row>
+ <Row><src>2</src><doc>313</doc><cnt>122</cnt></Row>
+ <Row><src>2</src><doc>317</doc><cnt>43</cnt></Row>
+ <Row><src>2</src><doc>323</doc><cnt>188</cnt></Row>
+ <Row><src>2</src><doc>331</doc><cnt>160</cnt></Row>
+ <Row><src>2</src><doc>340</doc><cnt>195</cnt></Row>
+ <Row><src>2</src><doc>357</doc><cnt>137</cnt></Row>
+ <Row><src>2</src><doc>360</doc><cnt>35</cnt></Row>
+ <Row><src>2</src><doc>364</doc><cnt>45</cnt></Row>
+ <Row><src>2</src><doc>372</doc><cnt>520</cnt></Row>
+ <Row><src>2</src><doc>376</doc><cnt>565</cnt></Row>
+ <Row><src>2</src><doc>377</doc><cnt>66</cnt></Row>
+ <Row><src>2</src><doc>378</doc><cnt>340</cnt></Row>
+ <Row><src>2</src><doc>380</doc><cnt>66</cnt></Row>
+ <Row><src>2</src><doc>387</doc><cnt>206</cnt></Row>
+ <Row><src>2</src><doc>388</doc><cnt>153</cnt></Row>
+ <Row><src>2</src><doc>392</doc><cnt>225</cnt></Row>
+ <Row><src>2</src><doc>395</doc><cnt>46</cnt></Row>
+ <Row><src>2</src><doc>396</doc><cnt>1014</cnt></Row>
+ <Row><src>2</src><doc>402</doc><cnt>145</cnt></Row>
+ <Row><src>2</src><doc>417</doc><cnt>183</cnt></Row>
+ <Row><src>2</src><doc>418</doc><cnt>141</cnt></Row>
+ <Row><src>2</src><doc>440</doc><cnt>21</cnt></Row>
+ <Row><src>2</src><doc>441</doc><cnt>176</cnt></Row>
+ <Row><src>2</src><doc>453</doc><cnt>42</cnt></Row>
+ <Row><src>2</src><doc>487</doc><cnt>759</cnt></Row>
+ <Row><src>2</src><doc>489</doc><cnt>19</cnt></Row>
+ <Row><src>2</src><doc>493</doc><cnt>42</cnt></Row>
+ <Row><src>2</src><doc>496</doc><cnt>633</cnt></Row>
+ <Row><src>2</src><doc>498</doc><cnt>1616</cnt></Row>
+ <Row><src>2</src><doc>501</doc><cnt>1661</cnt></Row>
+ <Row><src>2</src><doc>503</doc><cnt>332</cnt></Row>
+ <Row><src>2</src><doc>505</doc><cnt>282</cnt></Row>
+ <Row><src>2</src><doc>517</doc><cnt>313</cnt></Row>
+ <Row><src>2</src><doc>518</doc><cnt>704</cnt></Row>
+ <Row><src>2</src><doc>527</doc><cnt>407</cnt></Row>
+ <Row><src>2</src><doc>537</doc><cnt>8</cnt></Row>
+ <Row><src>2</src><doc>540</doc><cnt>95</cnt></Row>
+ <Row><src>2</src><doc>548</doc><cnt>516</cnt></Row>
+ <Row><src>2</src><doc>558</doc><cnt>181</cnt></Row>
+ <Row><src>2</src><doc>596</doc><cnt>264</cnt></Row>
+ <Row><src>2</src><doc>604</doc><cnt>555</cnt></Row>
+ <Row><src>2</src><doc>615</doc><cnt>52</cnt></Row>
+ <Row><src>2</src><doc>618</doc><cnt>40</cnt></Row>
+ <Row><src>2</src><doc>622</doc><cnt>1220</cnt></Row>
+ <Row><src>2</src><doc>629</doc><cnt>45</cnt></Row>
+ <Row><src>2</src><doc>641</doc><cnt>1800</cnt></Row>
+ <Row><src>2</src><doc>652</doc><cnt>72</cnt></Row>
+ <Row><src>2</src><doc>654</doc><cnt>268</cnt></Row>
+ <Row><src>2</src><doc>657</doc><cnt>1208</cnt></Row>
+ <Row><src>2</src><doc>664</doc><cnt>259</cnt></Row>
+ <Row><src>2</src><doc>665</doc><cnt>526</cnt></Row>
+ <Row><src>2</src><doc>669</doc><cnt>201</cnt></Row>
+ <Row><src>2</src><doc>674</doc><cnt>636</cnt></Row>
+ <Row><src>2</src><doc>692</doc><cnt>42</cnt></Row>
+ <Row><src>2</src><doc>706</doc><cnt>38</cnt></Row>
+ <Row><src>2</src><doc>713</doc><cnt>310</cnt></Row>
+ <Row><src>2</src><doc>720</doc><cnt>111</cnt></Row>
+ <Row><src>2</src><doc>723</doc><cnt>222</cnt></Row>
+ <Row><src>2</src><doc>725</doc><cnt>855</cnt></Row>
+ <Row><src>2</src><doc>731</doc><cnt>341</cnt></Row>
+</Dataset>

testing/ecl/key/stepping7h.xml

@@ -0,0 +1,9 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>28817</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>209</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>71</Result_3></Row>
+</Dataset>

testing/ecl/setup/setuptext.ecl

@@ -481,7 +481,7 @@ boolean splitBibleBook(string s) := function
         'THE ACTS OF|' +
         'THE ([A-Z]+ )?EPISTLE|' +
         'THE APOCALYPSE OF' +
-        ')', s);
+        ')|([0-9]+:[0-9]+)', s);
 END;
 
 boolean splitEncyclopedia(string s) := REGEXFIND('^A[A-Z]+ ', s);               // poor, but good enough for our purposes.

testing/ecl/stepping7.ecl

@@ -0,0 +1,32 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//UseIndexes
+//varskip payload
+//varskip varload
+//varskip trans
+//varskip dynamic
+
+// A single merge join, priorities in the correct order
+
+i1 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc);
+i2 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc);
+i3 := STEPPED(TS_searchIndex(kind=1 AND word='jericho'), doc);
+
+j := MERGEJOIN([i1, i2, i3], STEPPED(LEFT.doc =RIGHT.doc ), SORTED(doc), INTERNAL(0x40000000));
+output(TABLE(j, {src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));

testing/ecl/stepping7a.ecl

@@ -0,0 +1,33 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//UseIndexes
+//varskip payload
+//varskip varload
+//varskip trans
+//varskip dynamic
+
+//Multi level smart stepping, with priorities in the correct order
+
+i3 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc, PRIORITY(3),HINT(maxseeklookahead(50)));
+i2 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+i1 := STEPPED(TS_searchIndex(kind=1 AND word='jericho'), doc, PRIORITY(1),HINT(maxseeklookahead(50)));
+
+j1 := MERGEJOIN([i1, i2], STEPPED(LEFT.doc =RIGHT.doc ), SORTED(doc));
+j2 := MERGEJOIN([j1, i3], STEPPED(LEFT.doc =RIGHT.doc ), SORTED(doc));
+output(TABLE(j2, {src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));

testing/ecl/stepping7b.ecl

@@ -0,0 +1,33 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//UseIndexes
+//varskip payload
+//varskip varload
+//varskip trans
+//varskip dynamic
+
+//Multi level smart stepping, with priorities in the wrong order - to simulate large numbers of rows being buffered for the first term.
+
+i1 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc, PRIORITY(1),HINT(maxseeklookahead(50)));
+i2 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+i3 := STEPPED(TS_searchIndex(kind=1 AND word='jericho'), doc, PRIORITY(3),HINT(maxseeklookahead(50)));
+
+j1 := MERGEJOIN([i1, i2], STEPPED(LEFT.doc =RIGHT.doc ), SORTED(doc));
+j2 := MERGEJOIN([j1, i3], STEPPED(LEFT.doc =RIGHT.doc ), SORTED(doc));
+output(TABLE(j2, {src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));

testing/ecl/stepping7c.ecl

@@ -0,0 +1,32 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//UseIndexes
+//varskip payload
+//varskip varload
+//varskip trans
+//varskip dynamic
+
+// A single merge join, priorities in the correct order
+
+i1 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc, PRIORITY(3),HINT(maxseeklookahead(50)));
+i2 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+i3 := STEPPED(TS_searchIndex(kind=1 AND word='jericho'), doc, PRIORITY(1),HINT(maxseeklookahead(50)));
+
+j := MERGEJOIN([i1, i2, i3], STEPPED(LEFT.doc =RIGHT.doc ), SORTED(doc));
+output(TABLE(j, {src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));

testing/ecl/stepping7d.ecl

@@ -0,0 +1,32 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//UseIndexes
+//varskip payload
+//varskip varload
+//varskip trans
+//varskip dynamic
+
+// A single merge join, priorities in the wrong order
+
+i1 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc, PRIORITY(1),HINT(maxseeklookahead(50)));
+i2 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+i3 := STEPPED(TS_searchIndex(kind=1 AND word='jericho'), doc, PRIORITY(3),HINT(maxseeklookahead(50)));
+
+j := MERGEJOIN([i1, i2, i3], STEPPED(LEFT.doc =RIGHT.doc ), SORTED(doc));
+output(TABLE(j, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));

testing/ecl/stepping7e.ecl

@@ -0,0 +1,31 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//UseIndexes
+//varskip payload
+//varskip varload
+//varskip trans
+//varskip dynamic
+
+//Multi level smart stepping, with priorities in the correct order
+
+i2 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc, PRIORITY(3),HINT(maxseeklookahead(50)));
+i1 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+
+j1 := MERGEJOIN([i1, i2], STEPPED(LEFT.doc =RIGHT.doc ), SORTED(doc));
+output(TABLE(j1, {src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));

testing/ecl/stepping7f.ecl

@@ -0,0 +1,31 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//UseIndexes
+//varskip payload
+//varskip varload
+//varskip trans
+//varskip dynamic
+
+//Multi level smart stepping, with priorities in the wrong order
+
+i1 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+i2 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc, PRIORITY(3),HINT(maxseeklookahead(50)));
+
+j1 := MERGEJOIN([i1, i2], STEPPED(LEFT.doc =RIGHT.doc ), SORTED(doc));
+output(TABLE(j1, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));

testing/ecl/stepping7g.ecl

@@ -0,0 +1,33 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//UseIndexes
+//varskip payload
+//varskip varload
+//varskip trans
+//varskip dynamic
+
+//Multi level smart stepping, with priorities in the correct order
+
+i1 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc, PRIORITY(3),HINT(maxseeklookahead(50)));
+i2 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+i3 := STEPPED(TS_searchIndex(kind=1 AND word='jerico'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+
+output(TABLE(i1, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));
+output(TABLE(i2, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));
+output(TABLE(i3, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc));

testing/ecl/stepping7h.ecl

@@ -0,0 +1,33 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//UseIndexes
+//varskip payload
+//varskip varload
+//varskip trans
+//varskip dynamic
+
+//Multi level smart stepping, with priorities in the correct order
+
+i1 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc, PRIORITY(3),HINT(maxseeklookahead(50)));
+i2 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+i3 := STEPPED(TS_searchIndex(kind=1 AND word='jericho'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
+
+output(COUNT(TABLE(i1, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc)));
+output(COUNT(TABLE(i2, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc)));
+output(COUNT(TABLE(i3, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc)));