Merge pull request #12190 from ghalliday/issue21517

HPCC-21517 Ensure CHOOSEN is not evaluated remote if has local filter

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hthor/hthor.cpp

@@ -8651,7 +8651,7 @@ void CHThorDiskReadActivity::ready()
     if (helper.getFlags() & TDRlimitskips)
         limit = (unsigned __int64) -1;
     stopAfter = helper.getChooseNLimit();
-    if (!helper.transformMayFilter())
+    if (!helper.transformMayFilter() && !helper.hasMatchFilter())
         remoteLimit = stopAfter;
 }
 

testing/regress/ecl/key/remoteread.xml

@@ -9,25 +9,39 @@
 <Dataset name='Result 5'>
 </Dataset>
 <Dataset name='Result 6'>
+</Dataset>
+<Dataset name='Result 7'>
  <Row><fname>John      </fname><lname>Smith     </lname><age>42</age><logicalfile>regress::single::remoteread</logicalfile><fpos>0</fpos></Row>
  <Row><fname>Bob       </fname><lname>Brown     </lname><age>29</age><logicalfile>regress::single::remoteread</logicalfile><fpos>28</fpos></Row>
  <Row><fname>Samuel    </fname><lname>Jackson   </lname><age>58</age><logicalfile>regress::single::remoteread</logicalfile><fpos>56</fpos></Row>
 </Dataset>
-<Dataset name='Result 7'>
+<Dataset name='Result 8'>
  <Row><fname>John      </fname><lname>Smith     </lname><age>42</age><logicalfile>regress::single::remoteread_comp</logicalfile><fpos>0</fpos></Row>
  <Row><fname>Bob       </fname><lname>Brown     </lname><age>29</age><logicalfile>regress::single::remoteread_comp</logicalfile><fpos>28</fpos></Row>
  <Row><fname>Samuel    </fname><lname>Jackson   </lname><age>58</age><logicalfile>regress::single::remoteread_comp</logicalfile><fpos>56</fpos></Row>
 </Dataset>
-<Dataset name='Result 8'>
+<Dataset name='Result 9'>
  <Row><fname>Bob       </fname><lname>Brown     </lname><age>29</age></Row>
  <Row><fname>John      </fname><lname>Smith     </lname><age>42</age></Row>
  <Row><fname>Samuel    </fname><lname>Jackson   </lname><age>58</age></Row>
 </Dataset>
-<Dataset name='Result 9'>
+<Dataset name='Result 10'>
  <Row><lname>Smith     </lname><fname>John      </fname><fpos>0</fpos><logicalfile>regress::single::remoteread</logicalfile></Row>
  <Row><lname>Brown     </lname><fname>Bob       </fname><fpos>28</fpos><logicalfile>regress::single::remoteread</logicalfile></Row>
  <Row><lname>Jackson   </lname><fname>Samuel    </fname><fpos>56</fpos><logicalfile>regress::single::remoteread</logicalfile></Row>
 </Dataset>
-<Dataset name='Result 10'>
- <Row><Result_10>500043499986</Result_10></Row>
+<Dataset name='Result 11'>
+ <Row><fname>John12    </fname><lname>Smith12   </lname><age>54</age></Row>
+ <Row><fname>Bob25     </fname><lname>Brown25   </lname><age>54</age></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><fname>John12    </fname><lname>Smith12   </lname><age>54</age></Row>
+ <Row><fname>Bob25     </fname><lname>Brown25   </lname><age>54</age></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><fname>Bob25     </fname><lname>Brown25   </lname><age>54</age></Row>
+ <Row><fname>John12    </fname><lname>Smith12   </lname><age>54</age></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>500043499986</Result_14></Row>
 </Dataset>

testing/regress/ecl/remoteread.ecl

@@ -37,11 +37,17 @@ IMPORT STD;
 import $.setup;
 prefix := setup.Files(false, false).QueryFilePrefix;
 
+//MORE: If executed in parallel the prefix could cause problems - but outputing the filenames causes mismatches
+rrPrefix := IF(optRemoteRead, 'R', 'L');
+cmPrefix := IF(optCompression!='', optCompression, 'X');
+//xxprefix := filePrefix + rrPrefix + cmPrefix;
+
 fname := prefix + 'remoteread';
 fname_comp := prefix + 'remoteread_comp';
 fname_index := prefix + 'remoteread_index';
 fname_large := prefix + 'remoteread_large';
 fname_large_out := prefix + 'remoteread_largeout';
+fname_large_index := prefix + 'remoteread_large_index';
 
 rec := RECORD
  string10 fname;
@@ -83,6 +89,9 @@ largeoutrec := RECORD // shuffles order
 END;
 
 
+dsLarge := DATASET(fname_large, rec, FLAT);
+idxLarge := INDEX(dsLarge,,fname_large_index);
+
 
 SEQUENTIAL(
  OUTPUT(inds, , fname, OVERWRITE);
@@ -90,18 +99,22 @@ SEQUENTIAL(
  BUILDINDEX(i, OVERWRITE);
  OUTPUT(largeds, , fname_large, OVERWRITE);
  OUTPUT(largeds, , fname_large, OVERWRITE);
-
+ BUILD(idxLarge);
  OUTPUT(ds);
  OUTPUT(ds_comp);
  OUTPUT(i);
  OUTPUT(ds_trans);
- SUM(NOFOLD(DATASET(fname_large, rec, FLAT)), age);
+ output(choosen(dsLarge(age=54), 5));
+ output(choosen(dsLarge(keyed(age=54)), 5));
+ output(choosen(idxLarge(age=54), 5));
+ SUM(NOFOLD(dsLarge), age);
 
  // Clean-up
  FileServices.DeleteLogicalFile(fname),
  FileServices.DeleteLogicalFile(fname_comp),
  FileServices.DeleteLogicalFile(fname_index),
  FileServices.DeleteLogicalFile(fname_large),
+ FileServices.DeleteLogicalFile(fname_large_index),
 );
 
 

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -495,7 +495,7 @@ public:
         virtual void getMetaInfo(ThorDataLinkMetaInfo &info, IPartDescriptor *partDesc) const override
         {
             CDiskRecordPartHandler::getMetaInfo(info, partDesc);
-            if (activity.helper->transformMayFilter() || (TDRkeyed & activity.helper->getFlags()))
+            if (activity.helper->transformMayFilter() || activity.hasMatchFilter || (TDRkeyed & activity.helper->getFlags()))
             {
                 info.totalRowsMin = 0; // all bets off! 
                 info.unknownRowsOutput = info.canReduceNumRows = true;
@@ -611,7 +611,7 @@ public:
         else
             limit = (rowcount_t)helper->getRowLimit();
         stopAfter = (rowcount_t)helper->getChooseNLimit();
-        if (!helper->transformMayFilter())
+        if (!helper->transformMayFilter() && !hasMatchFilter)
         {
             remoteLimit = stopAfter;
             if (limit && (limit < remoteLimit))

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -542,7 +542,7 @@ class CIndexReadSlaveActivity : public CIndexReadSlaveBase
     virtual void prepareManager(IKeyManager *manager) override
     {
         PARENT::prepareManager(manager);
-        if (choosenLimit && !helper->transformMayFilter())
+        if (choosenLimit && !helper->transformMayFilter() && !helper->hasMatchFilter())
             manager->setChooseNLimit(choosenLimit);
     }
     const void *nextKeyGE(const void *seek, unsigned numFields)
@@ -740,11 +740,14 @@ public:
             steppingMeta.setExtra(steppedExtra);
 
         // NB: setup remoteLimit before base start() call which if parts remote, will use remoteLimit
-        if (choosenLimit)
+        if (!helper->transformMayFilter() && !helper->hasMatchFilter())
         {
-            remoteLimit = choosenLimit;
-            if (!helper->transformMayFilter() && (RCMAX != keyedLimit) && (keyedLimit+1 < remoteLimit))
-                remoteLimit = keyedLimit+1; // 1 more to ensure triggered when received back.
+            if (choosenLimit)
+            {
+                remoteLimit = choosenLimit;
+                if ((RCMAX != keyedLimit) && (keyedLimit+1 < remoteLimit))
+                    remoteLimit = keyedLimit+1; // 1 more to ensure triggered when received back.
+            }
         }
 
         if ((keyedLimit != RCMAX && (keyedLimitSkips || (helper->getFlags() & TIRcountkeyedlimit) != 0)))