Merge pull request #11250 from ghalliday/issue18860

HPCC-18860 Report error for full KEYED JOIN with no fileposition in the key

Reviewed-By: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlttcpp.cpp

@@ -14059,6 +14059,23 @@ void SemanticErrorChecker::checkJoin(IHqlExpression * join)
             }
         }
     }
+    IHqlExpression * keyed = join->queryAttribute(keyedAtom);
+    if (keyed)
+    {
+        //Ignore ,KEYED, only check ,KEYED(index)
+        IHqlExpression * index = keyed->queryChild(0);
+        if (index)
+        {
+            if (!getBoolAttribute(index, filepositionAtom, true))
+                reportError(ERR_BAD_JOINFLAG, "Cannot peform a full KEYED join with an index that has no fileposition");
+            else
+            {
+                IHqlExpression * lastField = queryLastField(index->queryRecord());
+                if (lastField && lastField->hasAttribute(_implicitFpos_Atom))
+                    reportError(ERR_BAD_JOINFLAG, "Cannot peform a full KEYED join with an index that has no fileposition");
+            }
+        }
+    }
 }
 
 void SemanticErrorChecker::checkChoosen(IHqlExpression * expr)

ecl/regress/indexreadv2.ecl

@@ -41,7 +41,7 @@ childRecord         myChild{blob};
 parentDataset := DATASET('test',parentRecord,FLAT);
 
 
-i := index(parentDataset, { id, address1, address2, address3 }, { parentDataset }, 'superkey');
+i := index(parentDataset, { id, address1, address2, address3 }, { parentDataset; unsigned dummyFpos := 0 }, 'superkey');
 
 
 string searchSurname := 'Hawthorn' : stored('searchName');

ecl/regress/issue18860.ecl

@@ -0,0 +1,29 @@
+rhsRec := RECORD
+ unsigned4 key;
+ string10 f1;
+ string10 f2;
+END;
+
+lhsRec := RECORD
+ unsigned4 lhsKey;
+END;
+
+rhs := DATASET([{1, 'a', 'b'}, {1, 'a2', 'b2'}, {1, 'a3', 'b3'}, {2, 'c', 'd'}, {3, 'e', 'f'}], rhsRec);
+rhsDs := DATASET('~rhsDs', rhsRec, FLAT);
+
+lhs := DATASET([{1}], lhsRec);
+
+i := INDEX(rhsDs, {key} , {f1}, '~rhsDs.idx');
+
+rhsRec doJoinTrans(lhsRec l, rhsRec r) := TRANSFORM
+ SELF.key := l.lhsKey;
+ SELF := r;
+END;
+
+j8 := JOIN(lhs, rhsDs, LEFT.lhsKey=RIGHT.key AND RIGHT.f2 != 'b2', doJoinTrans(LEFT, RIGHT), KEYED(i), KEEP(2));
+
+SEQUENTIAL(
+ OUTPUT(rhs, , '~rhsDs', OVERWRITE);
+ BUILD(i, OVERWRITE);
+ OUTPUT(j8);
+);

ecl/regress/issue18860b.ecl

@@ -0,0 +1,30 @@
+rhsRec := RECORD
+ unsigned4 key;
+ string10 f1;
+ string10 f2;
+ unsigned extra;
+END;
+
+lhsRec := RECORD
+ unsigned4 lhsKey;
+END;
+
+rhs := DATASET([{1, 'a', 'b', 1}, {1, 'a2', 'b2', 2}, {1, 'a3', 'b3', 3}, {2, 'c', 'd', 3}, {3, 'e', 'f', 5}], rhsRec);
+rhsDs := DATASET('~rhsDs', rhsRec, FLAT);
+
+lhs := DATASET([{1}], lhsRec);
+
+i := INDEX(rhsDs, {key} , { rhsDs} , '~rhsDs.idx', FILEPOSITION(FALSE));
+
+rhsRec doJoinTrans(lhsRec l, rhsRec r) := TRANSFORM
+ SELF.key := l.lhsKey;
+ SELF := r;
+END;
+
+j8 := JOIN(lhs, rhsDs, LEFT.lhsKey=RIGHT.key AND RIGHT.f2 != 'b2', doJoinTrans(LEFT, RIGHT), KEYED(i), KEEP(2));
+
+SEQUENTIAL(
+ OUTPUT(rhs, , '~rhsDs', OVERWRITE);
+ BUILD(i, OVERWRITE);
+ OUTPUT(j8);
+);

ecl/regress/optflag.ecl

@@ -33,7 +33,7 @@ count(p2);
 output(p2(f='NOT'));
 count(p2(f='NOT'));
 
-i := INDEX(d,{f},{},'regress::nor::this', OPT);
+i := INDEX(d,{f},{ unsigned dummyFpos := 0 },'regress::nor::this', OPT);
 output(i);
 count(i);
 output(i(f='NOT'));

testing/regress/ecl/dfsj.ecl

@@ -67,7 +67,8 @@ OUTPUT(JOIN(Files.DG_FlatFileEvens, Files.DG_FlatFile, KEYED(LEFT.DG_Firstname =
 OUTPUT(JOIN(Files.DG_FlatFileEvens, Files.DG_FlatFile, KEYED(LEFT.DG_Firstname = RIGHT.DG_firstname),KEYED(no_trans_needed),KEEP(1)));
 
 // Test removing some fields
-OUTPUT(JOIN(Files.DG_FlatFileEvens, Files.DG_FlatFile, KEYED(LEFT.DG_Firstname = RIGHT.DG_firstname),KEYED(slimmed),KEEP(1)));
+slimmed2 := INDEX(Files.DG_FlatFile, { DG_firstname, DG_lastname, unsigned dummyFpos := 0  }, DG_IndexName, LOOKUP(TRUE));
+OUTPUT(JOIN(Files.DG_FlatFileEvens, Files.DG_FlatFile, KEYED(LEFT.DG_Firstname = RIGHT.DG_firstname),KEYED(slimmed2),KEEP(1)));
 
 // Test adding/changing some fields
 OUTPUT(JOIN(Files.DG_FlatFileEvens, Files.DG_FlatFile, KEYED(LEFT.DG_Firstname = RIGHT.DG_firstname),KEYED(added),KEEP(1)));

testing/regress/ecl/dynamicoptflag.ecl

@@ -39,7 +39,7 @@ count(p2);
 output(p2(f='NOT'));
 count(p2(f='NOT'));
 
-i := INDEX(d,{f},{},DYNAMIC(prefix + 'nor_this'), OPT);
+i := INDEX(d,{f},{ unsigned dummyFpos := 0 },DYNAMIC(prefix + 'nor_this'), OPT);
 output(i);
 count(i);
 output(i(f='NOT'));

testing/regress/ecl/optflag.ecl

@@ -38,7 +38,7 @@ count(p2);
 output(p2(f='NOT'));
 count(p2(f='NOT'));
 
-i := INDEX(d,{f},{},prefix + 'nor_this', OPT);
+i := INDEX(d,{f},{ unsigned dummyFpos := 0 },prefix + 'nor_this', OPT);
 output(i);
 count(i);
 output(i(f='NOT'));