HPCC-21882 Report error for filtered limit within compound index read

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hqlcpp/hqlcerrors.hpp

@@ -224,6 +224,7 @@
 #define HQLERR_AttributeXMustBeConstant         4212
 #define HQLERR_CannotInterpretRecord            4213
 #define HQLERR_NoVirtualAndAlien                4214
+#define HQLERR_CannotFilterLimitInsideActivity  4215
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -531,6 +532,7 @@
 #define HQLERR_AttributeXMustBeConstant_Text    "Attribute %s must be set to a constant value"
 #define HQLERR_CannotInterpretRecord_Text       "This dataset contains deprecated record formats and virtual fields.  Remove the alien data types, or temporarily add __OPTION__(LEGACY) to the table definition"
 #define HQLERR_NoVirtualAndAlien_Text           "GenericDiskReads: VIRTUAL fields are not currently supported with ALIEN types"
+#define HQLERR_CannotFilterLimitInsideActivity_Text "Cannot filter a non keyed limit within a compound activity"
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlsource.cpp

@@ -941,6 +941,10 @@ void SourceBuilder::analyse(IHqlExpression * expr)
         break;
     case no_filter:
         {
+            //LIMIT(ds(filter1))(filter2) cannot be implemented as a compound activity - because it is impossible to count the rows being
+            //filtered by filter1.
+            if (limitExpr)
+                throwError(HQLERR_CannotFilterLimitInsideActivity);
             OwnedHqlExpr unkeyedFilter;
             HqlExprArray conds;
             unwindFilterConditions(conds, expr);

ecl/regress/issue21882.ecl

@@ -0,0 +1,56 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//version optRemoteRead=false
+//version optRemoteRead=true
+
+#onwarning (4523, ignore);
+
+optRemoteRead := #IFDEFINED(root.optRemoteRead, false);
+#option('forceRemoteRead', optRemoteRead);
+
+IMPORT STD;
+//import $.setup;
+prefix := '~test::';//setup.Files(false, false).QueryFilePrefix;
+
+
+addrRec := RECORD
+ string addr;
+END;
+
+rec := RECORD
+ string15 fname;
+ string15 lname;
+ unsigned age;
+ DATASET(addrRec) addrs;
+END;
+
+thefname := prefix + 'indexlimits_file';
+thefname_index := prefix + 'indexlimits_index';
+inds := DATASET([{ 'Aaron', 'Jones', 100, [{'somewhere'}]}, {'Adam', 'Smith', 90, [{'somewhere'}]}, {'Bob', 'Brown', 80, [{'somewhere'}]}, {'Brian', 'Brown', 70, [{'somewhere'}]}, {'Charles', 'Dance', 60, [{'somewhere'}]}, {'Charles', 'Gould', 50, [{'somewhere'}]},  {'David', 'Brokenshire', 40, [{'somewhere'}]}, {'Edward', 'Green', 30, [{'somewhere'}]}, {'Egbert', 'Sillyname', 20, [{'somewhere'}]}, {'Freddy', 'Peters', 10, [{'somewhere'}]} ], rec, DISTRIBUTED);
+i := INDEX(inds, { fname }, { inds }, thefname_index);
+
+indexReadComboHitRowLimit := LIMIT(LIMIT(i(age >= 50), 1, SKIP)(fname = 'Charles'), 2, SKIP, KEYED); // keyed limit should not be triggered, but row limit should
+//indexReadComboHitRowLimit := LIMIT(i(age >= 50), 1, SKIP)(fname = 'Charles'); // keyed limit should not be triggered, but row limit should
+//indexReadComboHitRowLimit := LIMIT(LIMIT(i(age >= 50), 2, SKIP, KEYED), 1, SKIP)(fname = 'Charles'); // keyed limit should not be triggered, but row limit should
+
+SEQUENTIAL(
+ OUTPUT(inds, , thefname, OVERWRITE);
+ BUILDINDEX(i, SORTED, OVERWRITE);
+
+ OUTPUT(indexReadComboHitRowLimit, NAMED('indexReadComboHitRowLimit'));
+);

testing/regress/ecl/indexlimits.ecl

@@ -54,7 +54,7 @@ indexReadNoHitRowLimit := LIMIT(i(age < 30), 2, SKIP); // Skip should not be tri
 // NB: Index keyed limit tests
 indexReadHitKeyedLimit := LIMIT(i(fname = 'Charles'), 1, SKIP, KEYED); // could hit limit locally.
 indexReadHitKeyedOnFail := LIMIT(i(fname = 'Charles'), 1, KEYED, ONFAIL(TRANSFORM(RECORDOF(i), SELF.fname := 'KEYED LIMIT EXCEEDED'; SELF.lname := 'KEYED LIMIT EXCEEDED'; SELF.age := 0; SELF.addrs := [])));
-indexReadComboHitRowLimit := LIMIT(LIMIT(i(age >= 50), 1, SKIP)(fname = 'Charles'), 2, SKIP, KEYED); // keyed limit should not be triggered, but row limit should
+indexReadComboHitRowLimit := LIMIT(LIMIT(i(fname = 'Charles'), 2, SKIP, KEYED)(age >= 50), 1, SKIP); // keyed limit should not be triggered, but row limit should
 indexReadNoHitKeyedLimit := LIMIT(i(fname = 'Charles'), 2, SKIP, KEYED); // Skip should not be triggered
 
 // NB: Index read row limit tests on normalized index