Merge branch 'candidate-7.8.x' into candidate-7.10.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

configuration/configmgr/configmgrlib/SchemaValue.cpp

@@ -15,6 +15,7 @@
     limitations under the License.
 ############################################################################## */
 
+#include <stdexcept>
 #include "SchemaValue.hpp"
 #include "EnvironmentValue.hpp"
 #include "EnvironmentNode.hpp"

ecl/hql/hqlfilter.cpp

@@ -439,8 +439,8 @@ IHqlExpression * castToFieldAndBack(IHqlExpression * left, IHqlExpression * righ
 
 //---------------------------------------------------------------------------------------------------------------------
 
-FilterExtractor::FilterExtractor(IErrorReceiver & _errorReceiver, IHqlExpression * _tableExpr, int _numKeyableFields, bool _isDiskRead, bool forceValueSets)
-    : errorReceiver(_errorReceiver), createValueSets(forceValueSets)
+FilterExtractor::FilterExtractor(IErrorReceiver & _errorReceiver, IHqlExpression * _tableExpr, int _numKeyableFields, bool _isDiskRead, bool forceValueSets, bool _allKeyedFiltersOptional)
+    : errorReceiver(_errorReceiver), createValueSets(forceValueSets), allKeyedFiltersOptional(_allKeyedFiltersOptional)
 {
     tableExpr = _tableExpr;
 
@@ -1549,7 +1549,7 @@ bool FilterExtractor::extractFilters(KeyConditionInfo & matches, IHqlExpression
             KeyFailureInfo reason;
             reason.merge(failReason);
             failReason.clear();
-            bool extend = expr->hasAttribute(extendAtom);
+            bool extend = getBoolAttribute(expr, extendAtom, allKeyedFiltersOptional);
             if (!extractFilters(matches, l, extend ? KeyedExtend : KeyedYes))
             {
                 if (!extend)

ecl/hql/hqlfilter.hpp

@@ -105,7 +105,7 @@ public:
 class HQL_API FilterExtractor
 {
 public:
-    FilterExtractor(IErrorReceiver & _errorReceiver, IHqlExpression * _tableExpr, int _numKeyableFields, bool isDiskRead, bool forceValueSets);
+    FilterExtractor(IErrorReceiver & _errorReceiver, IHqlExpression * _tableExpr, int _numKeyableFields, bool isDiskRead, bool forceValueSets, bool _allKeyedFiltersOptional);
 
     void appendFilter(IHqlExpression * expr)                { keyed.appendPostFilter(expr); }
     void extractFilters(IHqlExpression * filter, SharedHqlExpr & extraFilter);
@@ -182,6 +182,7 @@ protected:
     bool keyedExplicitly;
     bool allowDynamicFormatChange;
     const bool createValueSets;
+    const bool allKeyedFiltersOptional;
 };
 
 extern HQL_API IHqlExpression * getExplicitlyPromotedCompare(IHqlExpression * filter);

ecl/hql/hqlgram.y

@@ -1771,7 +1771,7 @@ optPersistOpts
     ;
 
 persistOpts
-    : persistOpt
+: persistOpt
     | persistOpts ',' persistOpt
                         {
                             $$.setExpr(createComma($1.getExpr(), $3.getExpr()), $1);
@@ -6859,11 +6859,11 @@ primexpr1
                             IHqlExpression * e = $3.getExpr();
                             $$.setExpr(createValue(no_assertkeyed, e->getType(), e));
                         }
-    | KEYED '(' expression ',' OPT ')'
+    | KEYED '(' expression ',' OPT optConstBoolArg ')'
                         {
                             parser->normalizeExpression($3);
                             IHqlExpression * e = $3.getExpr();
-                            $$.setExpr(createValue(no_assertkeyed, e->getType(), e, createAttribute(extendAtom)));
+                            $$.setExpr(createValue(no_assertkeyed, e->getType(), e, createExprAttribute(extendAtom, $6.getExpr())));
                         }
     | STEPPED '(' expression ')'
                         {

ecl/hql/hqlutil.cpp

@@ -10285,7 +10285,7 @@ static IFieldFilter * createIfBlockFilter(IRtlFieldTypeDeserializer &deserialize
     OwnedHqlExpr mappedCondition = replaceSelector(cond, querySelfReference(), dummyDataset);
     Owned <IErrorReceiver> errorReceiver = createThrowingErrorReceiver();
 
-    FilterExtractor extractor(*errorReceiver, dummyDataset, rowRecord->numChildren(), true, true);
+    FilterExtractor extractor(*errorReceiver, dummyDataset, rowRecord->numChildren(), true, true, false);
     OwnedHqlExpr extraFilter;
     extractor.extractFilters(mappedCondition, extraFilter);
 

ecl/hqlcpp/hqlcfilter.cpp

@@ -76,7 +76,7 @@ void BuildFilterState::popSetName()
 //---------------------------------------------------------------------------------------------------------------------
 
 CppFilterExtractor::CppFilterExtractor(IHqlExpression * _tableExpr, HqlCppTranslator & _translator, int _numKeyableFields, bool _isDiskRead, bool forceValueSets)
-    : FilterExtractor(_translator, _tableExpr, _numKeyableFields, _isDiskRead, _isDiskRead || _translator.queryOptions().createValueSets || forceValueSets), translator(_translator)
+    : FilterExtractor(_translator, _tableExpr, _numKeyableFields, _isDiskRead, _isDiskRead || _translator.queryOptions().createValueSets || forceValueSets, _translator.queryOptions().allKeyedFiltersOptional), translator(_translator)
 {
     if (createValueSets)
     {

ecl/hqlcpp/hqlcpp.cpp

@@ -1856,6 +1856,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.checkDuplicateThreshold, "checkDuplicateThreshold", 0), // If non zero, create a warning if duplicates > this percentage increase
         DebugOption(options.checkDuplicateMinActivities, "checkDuplicateMinActivities", 100),
         DebugOption(options.diskReadsAreSimple, "diskReadsAreSimple", false), // Not yet enabled - needs filters to default to generating keyed info first
+        DebugOption(options.allKeyedFiltersOptional, "allKeyedFiltersOptional", false),
         DebugOption(options.genericDiskReads, "genericDiskReads", false),
         DebugOption(options.generateActivityFormats, "generateActivityFormats", false),
         DebugOption(options.generateDiskFormats, "generateDiskFormats", false),

ecl/hqlcpp/hqlcpp.ipp

@@ -831,6 +831,7 @@ struct HqlCppOptions
     bool                forceAllProjectedDiskSerialized;
     bool                newIndexReadMapping;
     bool                diskReadsAreSimple;
+    bool                allKeyedFiltersOptional;
     bool                genericDiskReads;
     bool                generateActivityFormats;
     bool                generateDiskFormats;

ecl/regress/indexkeyeda.ecl

@@ -0,0 +1,32 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('allKeyedFiltersOptional', false);
+i := index({ string30 surname, string20 forename, unsigned age } , {}, 'myDummyKey');
+
+filtered1  := i(WILD(surname) AND KEYED(forename = 'Gavin'));
+filtered2  := i(WILD(surname) AND KEYED(forename = 'Gavin', OPT));
+filtered3  := i(WILD(surname) AND KEYED(forename = 'Gavin', OPT(TRUE)));
+filtered4  := i(WILD(surname) AND KEYED(forename = 'Gavin', OPT(FALSE)));
+
+sequential(
+    output(filtered1);
+    output(filtered2);
+    output(filtered3);
+    output(filtered4);
+    output('done')
+);

ecl/regress/indexkeyedb.ecl

@@ -0,0 +1,32 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('allKeyedFiltersOptional', true);
+i := index({ string30 surname, string20 forename, unsigned age } , {}, 'myDummyKey');
+
+filtered1  := i(WILD(surname) AND KEYED(forename = 'Gavin'));
+filtered2  := i(WILD(surname) AND KEYED(forename = 'Gavin', OPT));
+filtered3  := i(WILD(surname) AND KEYED(forename = 'Gavin', OPT(TRUE)));
+filtered4  := i(WILD(surname) AND KEYED(forename = 'Gavin', OPT(FALSE)));
+
+sequential(
+    output(filtered1);
+    output(filtered2);
+    output(filtered3);
+    output(filtered4);
+    output('done')
+);

rtl/eclrtl/rtlkey.hpp

@@ -113,7 +113,7 @@ interface IBlobCreator
 
 enum FFoption : byte
 {
-    FFkeyed = 1,
+    FFkeyed = 0,
     FFopt = 1,
     FFpost = 2,
 };

testing/regress/ecl/key/keyedlimit.xml

@@ -0,0 +1,109 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>non-limited</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><kind>1</kind><word>am                  </word><doc>281474976710658</doc><segment>0</segment><wpos>30</wpos><wip>1</wip><flags>1</flags><original>am                  </original><dpos>148</dpos></Row>
+ <Row><kind>1</kind><word>am                  </word><doc>1125899906842631</doc><segment>0</segment><wpos>10</wpos><wip>1</wip><flags>1</flags><original>am                  </original><dpos>40</dpos></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>13</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710658</doc><segment>0</segment><wpos>19</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>98</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710658</doc><segment>0</segment><wpos>24</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>121</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710659</doc><segment>0</segment><wpos>9</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>48</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710659</doc><segment>0</segment><wpos>21</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>102</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710660</doc><segment>0</segment><wpos>21</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>112</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710661</doc><segment>0</segment><wpos>22</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>100</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710661</doc><segment>0</segment><wpos>27</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>122</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131971</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>13</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>5</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>14</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>7</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>24</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842625</doc><segment>0</segment><wpos>6</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>19</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842626</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>10</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842630</doc><segment>0</segment><wpos>8</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>38</dpos></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>8</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710658</doc><segment>0</segment><wpos>19</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>98</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710659</doc><segment>0</segment><wpos>9</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>48</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131971</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>13</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>5</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>14</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>7</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>24</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842625</doc><segment>0</segment><wpos>6</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>19</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842626</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>10</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842630</doc><segment>0</segment><wpos>8</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>38</dpos></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710658</doc><segment>0</segment><wpos>19</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>98</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710659</doc><segment>0</segment><wpos>9</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>48</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131971</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>13</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>5</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>14</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>7</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>24</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842625</doc><segment>0</segment><wpos>6</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>19</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842626</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>10</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842630</doc><segment>0</segment><wpos>8</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>38</dpos></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>limited</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><kind>1</kind><word>am                  </word><doc>281474976710658</doc><segment>0</segment><wpos>30</wpos><wip>1</wip><flags>1</flags><original>am                  </original><dpos>148</dpos></Row>
+ <Row><kind>1</kind><word>am                  </word><doc>1125899906842631</doc><segment>0</segment><wpos>10</wpos><wip>1</wip><flags>1</flags><original>am                  </original><dpos>40</dpos></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710658</doc><segment>0</segment><wpos>19</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>98</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710659</doc><segment>0</segment><wpos>9</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>48</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131971</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>13</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>5</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>14</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>7</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>24</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842625</doc><segment>0</segment><wpos>6</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>19</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842626</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>10</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842630</doc><segment>0</segment><wpos>8</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>38</dpos></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710658</doc><segment>0</segment><wpos>19</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>98</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>281474976710659</doc><segment>0</segment><wpos>9</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>48</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131971</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>13</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>5</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>14</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>7</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>24</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842625</doc><segment>0</segment><wpos>6</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>19</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842626</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>10</dpos></Row>
+ <Row><kind>1</kind><word>and                 </word><doc>1125899906842630</doc><segment>0</segment><wpos>8</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>38</dpos></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><kind>0</kind><word>Failed to match     </word><doc>0</doc><segment>0</segment><wpos>0</wpos><wip>0</wip><flags>0</flags><original>                    </original><dpos>0</dpos></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>keyed</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><search>am</search><kind>1</kind><word>am                  </word><doc>1125899906842631</doc><segment>0</segment><wpos>10</wpos><wip>1</wip><flags>1</flags><original>am                  </original><dpos>40</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>281474976710658</doc><segment>0</segment><wpos>19</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>98</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>281474976710659</doc><segment>0</segment><wpos>9</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>48</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>844424930131971</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>13</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>5</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>14</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>7</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>24</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>1125899906842625</doc><segment>0</segment><wpos>6</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>19</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>1125899906842626</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>10</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>1125899906842630</doc><segment>0</segment><wpos>8</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>38</dpos></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>keyed limited</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><search>am</search><kind>1</kind><word>am                  </word><doc>1125899906842631</doc><segment>0</segment><wpos>10</wpos><wip>1</wip><flags>1</flags><original>am                  </original><dpos>40</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>281474976710658</doc><segment>0</segment><wpos>19</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>98</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>281474976710659</doc><segment>0</segment><wpos>9</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>48</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>844424930131971</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>13</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>5</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>14</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>844424930131972</doc><segment>0</segment><wpos>7</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>24</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>1125899906842625</doc><segment>0</segment><wpos>6</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>19</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>1125899906842626</doc><segment>0</segment><wpos>4</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>10</dpos></Row>
+ <Row><search>and</search><kind>1</kind><word>and                 </word><doc>1125899906842630</doc><segment>0</segment><wpos>8</wpos><wip>1</wip><flags>1</flags><original>and                 </original><dpos>38</dpos></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>done</Result_17></Row>
+</Dataset>

testing/regress/ecl/keyedlimit.ecl

@@ -0,0 +1,83 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#onwarning (4522, ignore);
+#onwarning (4523, ignore);
+
+#option ('allKeyedFiltersOptional', false);
+
+//version multiPart=false,createValueSets=false
+//version multiPart=true,createValueSets=false
+
+//version multiPart=false,createValueSets=true
+//version multiPart=true,createValueSets=true
+//version multiPart=false,useLocal=true,createValueSets=true
+
+import ^ as root;
+multiPart := #IFDEFINED(root.multiPart, false);
+useLocal := #IFDEFINED(root.useLocal, false);
+createValueSets := #IFDEFINED(root.createValueSets, true);
+
+//--- end of version configuration ---
+#option ('createValueSets', createValueSets);
+
+import $.Setup;
+Files := Setup.Files(multiPart, useLocal);
+wordIndex := Files.getWordIndex();
+
+searchWords := DATASET(['am', 'and'], { string search});
+resultRecord := RECORD
+    recordof(searchWords);
+    recordof(wordIndex);
+END;
+
+wordIndex mkFailureIR(string search) := TRANSFORM
+    SELF.word := 'Failed to match';
+    SELF := [];
+END;
+
+resultRecord mkFailureKJ(string search) := TRANSFORM
+    SELF.search := search;
+    SELF.word := 'Failed to match';
+    SELF := [];
+END;
+
+SEQUENTIAL(
+    output('non-limited');
+    output(wordIndex(KEYED(kind =1) AND KEYED(word = 'am') AND WILD(doc) AND KEYED(segment = 0)));
+    count(wordIndex(KEYED(kind =1) AND KEYED(word = 'and') AND WILD(doc) AND KEYED(segment = 0)));
+    output(wordIndex(KEYED(kind =1) AND KEYED(word = 'and') AND WILD(doc) AND KEYED(segment = 0)));
+    count(wordIndex(KEYED(kind =1) AND KEYED(word = 'and') AND WILD(doc) AND KEYED(segment = 0) AND KEYED(wpos < 20)));
+    output(wordIndex(KEYED(kind =1) AND KEYED(word = 'and') AND WILD(doc) AND KEYED(segment = 0) AND KEYED(wpos < 20)));
+    output(wordIndex(KEYED(kind =1) AND KEYED(word = 'and') AND KEYED(doc=0 or 0 = 0) AND KEYED(segment = 0, OPT) AND KEYED(wpos < 20, OPT)));
+    output('limited');
+    output(limit(wordIndex(KEYED(kind =1) AND KEYED(word = 'am') AND WILD(doc) AND KEYED(segment = 0)), 3, keyed, onfail(mkFailureIR('am'))));
+    //Filter on wpos should ensure there are < 9
+    output(limit(wordIndex(KEYED(kind =1) AND KEYED(word = 'and') AND WILD(doc) AND KEYED(segment = 0) AND KEYED(wpos < 20)), 9, keyed, onfail(mkFailureIR('and'))));
+    output(limit(wordIndex(KEYED(kind =1) AND KEYED(word = 'and') AND WILD(doc) AND KEYED(segment = 0, OPT(FALSE)) AND KEYED(wpos < 20, OPT(FALSE))), 9, keyed, onfail(mkFailureIR('and'))));
+    //Filter on wpos is opt, so limit will be triggered
+    output(limit(wordIndex(KEYED(kind =1) AND KEYED(word = 'and') AND KEYED(doc=0 or 0 = 0) AND KEYED(segment = 0, OPT) AND KEYED(wpos < 20, OPT)), 9, keyed, onfail(mkFailureIR('and'))));
+    output('keyed');
+    output(join(searchWords, wordIndex,
+            KEYED(RIGHT.kind = 1) AND KEYED(RIGHT.word = LEFT.search) AND WILD(RIGHT.doc) AND KEYED(RIGHT.segment = 0) AND KEYED(RIGHT.wpos < 20)
+            ));
+    output('keyed limited');
+    output(join(searchWords, wordIndex,
+            KEYED(RIGHT.kind = 1) AND KEYED(RIGHT.word = LEFT.search) AND WILD(RIGHT.doc) AND KEYED(RIGHT.segment = 0) AND KEYED(RIGHT.wpos < 20),
+            LIMIT(9, COUNT), ONFAIL(mkFailureKJ(LEFT.search))));
+    output('done')
+);