HPCC-21957 Support KEYED ORed and conditional substring filters

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlfilter.cpp

@@ -1190,8 +1190,7 @@ bool FilterExtractor::matchSubstringFilter(KeyConditionInfo & matches, node_oper
         newTest.setown(createBalanced(combineOp, boolType, compares));
     }
 
-    KeyCondition * entry = new KeyCondition(selector, newTest, keyedKind);
-    entry->subrange.set(left->queryChild(1));
+    KeyCondition * entry = new KeyCondition(selector, newTest, keyedKind, left->queryChild(1));
     matches.appendCondition(*entry);
     if (guard)
         matches.appendPreFilter(guard);
@@ -1224,8 +1223,7 @@ bool FilterExtractor::extractSimpleCompareFilter(KeyConditionInfo & matches, IHq
         if (newOp != no_none)
         {
             OwnedHqlExpr newFilter = createValue(newOp, expr->getType(), LINK(l), LINK(r));
-            result.setown(new KeyCondition(matchedSelector, newFilter, keyedKind));
-            result->subrange.set(querySubStringRange(l));
+            result.setown(new KeyCondition(matchedSelector, newFilter, keyedKind, querySubStringRange(l)));
         }
     }
     else
@@ -1245,8 +1243,7 @@ bool FilterExtractor::extractSimpleCompareFilter(KeyConditionInfo & matches, IHq
             if (newOp != no_none)
             {
                 OwnedHqlExpr newFilter = createValue(newOp, expr->getType(), LINK(r), LINK(l));
-                result.setown(new KeyCondition(matchedSelector, newFilter, keyedKind));
-                result->subrange.set(querySubStringRange(r));
+                result.setown(new KeyCondition(matchedSelector, newFilter, keyedKind, querySubStringRange(r)));
             }
         }
     }
@@ -1316,7 +1313,7 @@ bool FilterExtractor::extractOrFilter(KeyConditionInfo & matches, IHqlExpression
                 }
                 else
                 {
-                    if ((firstBranch->selector != cur.selector) || multipleSelectors)
+                    if ((firstBranch->selector != cur.selector) || multipleSelectors || (createValueSets && (firstBranch->subrange != cur.subrange)))
                         validOrFilter = false;
                 }
             }
@@ -1344,7 +1341,7 @@ bool FilterExtractor::extractOrFilter(KeyConditionInfo & matches, IHqlExpression
             {
                 KeyCondition & cur = branch.conditions.item(i2);
                 OwnedHqlExpr filter = extendCondition(no_or, invariant, cur.expr);
-                matches.conditions.append(*new KeyCondition(cur.selector, filter, keyedKind));
+                matches.conditions.append(*new KeyCondition(cur.selector, filter, keyedKind, cur.subrange));
             }
         }
         else
@@ -1358,7 +1355,7 @@ bool FilterExtractor::extractOrFilter(KeyConditionInfo & matches, IHqlExpression
                 extendOrCondition(combinedCondition, conjunction);
             }
 
-            matches.conditions.append(*new KeyCondition(firstBranch->selector, combinedCondition, keyedKind));
+            matches.conditions.append(*new KeyCondition(firstBranch->selector, combinedCondition, keyedKind, firstBranch->subrange));
         }
         return true;
     }
@@ -1493,7 +1490,7 @@ bool FilterExtractor::extractBoolFieldFilter(KeyConditionInfo & matches, IHqlExp
         if (isKeySelect(selector) && okToKey(selector, keyedKind))
         {
             OwnedHqlExpr newFilter = createValue(no_eq, makeBoolType(), LINK(selector), createConstant(compareValue));
-            matches.appendCondition(*new KeyCondition(selector, newFilter, keyedKind));
+            matches.appendCondition(*new KeyCondition(selector, newFilter, keyedKind, nullptr));
             return true;
         }
     }

ecl/hql/hqlfilter.hpp

@@ -22,8 +22,8 @@ struct HQL_API KeyCondition : public CInterface
 {
 public:
     KeyCondition()          { keyedKind = KeyedNo; isWild = false; generated = false; wasKeyed = false; }
-    KeyCondition(IHqlExpression * _selector, IHqlExpression * _expr, KeyedKind _keyedKind)
-                            { selector.set(_selector); expr.set(_expr); keyedKind = _keyedKind; isWild = false; generated = false; wasKeyed = isKeyed(); }
+    KeyCondition(IHqlExpression * _selector, IHqlExpression * _expr, KeyedKind _keyedKind, IHqlExpression * _subrange)
+                            { selector.set(_selector); subrange.set(_subrange); expr.set(_expr); keyedKind = _keyedKind; isWild = false; generated = false; wasKeyed = isKeyed(); }
 
     bool isKeyed()          { return (keyedKind != KeyedNo); }
 

testing/regress/ecl/indexsubstring.ecl

@@ -55,6 +55,7 @@ BuildIndexOp := BUILDINDEX(INDX_Postcode, OVERWRITE);
 SET OF STRING4 PartialPostcode:= ['KT19','KT40','KT3 ','KT20 1AEEE','KT50','KT60 3DE'];
 SET OF STRING4 PartialPostcodeStored:= ['KT19','KT40','KT3 ','KT20 1AEEE', 'KT50','KT60 3DE']:stored('PartialPostcode');
 SET OF STRING PartialPostcodeStored2:= ['KT19','KT40','KT3 ','KT20 1AEEE', 'KT50','KT60 3DE']:stored('PartialPostcode2');
+SET OF STRING8 PartialPostcodeStored8:= ['KT19','KT40','KT3 ','KT50']:stored('PartialPostcode8');
 
 partialmatch1 := INDX_Postcode( KEYED(postcode[1..4] IN PartialPostcode) );
 partialmatch2 := INDX_Postcode( KEYED(postcode[1..4] IN PartialPostcodeStored) );
@@ -69,6 +70,9 @@ partialmatch5 := INDX_Postcode( postcode[1..four] IN PartialPostcode );
 partialmatch6 := INDX_Postcode( postcode[1..four] IN PartialPostcodeStored );
 partialmatch7 := INDX_Postcode( postcode[..four] IN PartialPostcodeStored );
 partialmatch8 := INDX_Postcode( postcode[..four] IN PartialPostcodestored2 );
+partialmatch9 := INDX_Postcode( KEYED(postcode[..4] IN PartialPostcodestored8) );
+partialmatch10 := INDX_Postcode( KEYED(IF(four=4, postcode[..four] IN PartialPostcodestored8,true)) );
+partialmatch11 := INDX_Postcode( KEYED(postcode[..four] IN IF(four=4, PartialPostcodestored8, all)) );
 
 SEQUENTIAL(
   outputraw,
@@ -83,4 +87,7 @@ SEQUENTIAL(
   OUTPUT(partialmatch6),  // match KT19*, KT40*, KT3, KT20*, KT50* and KT60*
   OUTPUT(partialmatch7),  // match KT19*, KT40*, KT3, KT20*, KT50* and KT60*
   OUTPUT(partialmatch8),  // match KT19*, KT40*, KT3 and KT50*
+  OUTPUT(partialmatch9),  // match KT19*, KT40*, KT3 and KT50*
+  OUTPUT(partialmatch10),  // match KT19*, KT40*, KT3 and KT50*
+  OUTPUT(partialmatch11),  // match KT19*, KT40*, KT3 and KT50*
 );

testing/regress/ecl/key/indexsubstring.xml

@@ -187,3 +187,51 @@
  <Row><postcode>KT50 4FG</postcode><__filepos>368</__filepos></Row>
  <Row><postcode>KT50 5HI</postcode><__filepos>376</__filepos></Row>
 </Dataset>
+<Dataset name='Result 13'>
+ <Row><postcode>KT19 1AA</postcode><__filepos>0</__filepos></Row>
+ <Row><postcode>KT19 1AB</postcode><__filepos>8</__filepos></Row>
+ <Row><postcode>KT19 1AC</postcode><__filepos>16</__filepos></Row>
+ <Row><postcode>KT19 1AD</postcode><__filepos>24</__filepos></Row>
+ <Row><postcode>KT3     </postcode><__filepos>416</__filepos></Row>
+ <Row><postcode>KT40 1AC</postcode><__filepos>256</__filepos></Row>
+ <Row><postcode>KT40 1AD</postcode><__filepos>264</__filepos></Row>
+ <Row><postcode>KT40 1AE</postcode><__filepos>272</__filepos></Row>
+ <Row><postcode>KT40 1AF</postcode><__filepos>280</__filepos></Row>
+ <Row><postcode>KT50    </postcode><__filepos>432</__filepos></Row>
+ <Row><postcode>KT50 2AB</postcode><__filepos>352</__filepos></Row>
+ <Row><postcode>KT50 3DE</postcode><__filepos>360</__filepos></Row>
+ <Row><postcode>KT50 4FG</postcode><__filepos>368</__filepos></Row>
+ <Row><postcode>KT50 5HI</postcode><__filepos>376</__filepos></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><postcode>KT19 1AA</postcode><__filepos>0</__filepos></Row>
+ <Row><postcode>KT19 1AB</postcode><__filepos>8</__filepos></Row>
+ <Row><postcode>KT19 1AC</postcode><__filepos>16</__filepos></Row>
+ <Row><postcode>KT19 1AD</postcode><__filepos>24</__filepos></Row>
+ <Row><postcode>KT3     </postcode><__filepos>416</__filepos></Row>
+ <Row><postcode>KT40 1AC</postcode><__filepos>256</__filepos></Row>
+ <Row><postcode>KT40 1AD</postcode><__filepos>264</__filepos></Row>
+ <Row><postcode>KT40 1AE</postcode><__filepos>272</__filepos></Row>
+ <Row><postcode>KT40 1AF</postcode><__filepos>280</__filepos></Row>
+ <Row><postcode>KT50    </postcode><__filepos>432</__filepos></Row>
+ <Row><postcode>KT50 2AB</postcode><__filepos>352</__filepos></Row>
+ <Row><postcode>KT50 3DE</postcode><__filepos>360</__filepos></Row>
+ <Row><postcode>KT50 4FG</postcode><__filepos>368</__filepos></Row>
+ <Row><postcode>KT50 5HI</postcode><__filepos>376</__filepos></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><postcode>KT19 1AA</postcode><__filepos>0</__filepos></Row>
+ <Row><postcode>KT19 1AB</postcode><__filepos>8</__filepos></Row>
+ <Row><postcode>KT19 1AC</postcode><__filepos>16</__filepos></Row>
+ <Row><postcode>KT19 1AD</postcode><__filepos>24</__filepos></Row>
+ <Row><postcode>KT3     </postcode><__filepos>416</__filepos></Row>
+ <Row><postcode>KT40 1AC</postcode><__filepos>256</__filepos></Row>
+ <Row><postcode>KT40 1AD</postcode><__filepos>264</__filepos></Row>
+ <Row><postcode>KT40 1AE</postcode><__filepos>272</__filepos></Row>
+ <Row><postcode>KT40 1AF</postcode><__filepos>280</__filepos></Row>
+ <Row><postcode>KT50    </postcode><__filepos>432</__filepos></Row>
+ <Row><postcode>KT50 2AB</postcode><__filepos>352</__filepos></Row>
+ <Row><postcode>KT50 3DE</postcode><__filepos>360</__filepos></Row>
+ <Row><postcode>KT50 4FG</postcode><__filepos>368</__filepos></Row>
+ <Row><postcode>KT50 5HI</postcode><__filepos>376</__filepos></Row>
+</Dataset>