HPCC-13122 Fix problems with keydiff in regression suite

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hqlcpp/hqlckey.cpp

@@ -1668,6 +1668,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyPatch(BuildCtx & ctx, IHqlE
 
     //virtual int getSequence() = 0;
     doBuildSequenceFunc(instance->classctx, querySequence(expr), false);
+    HqlExprArray xmlnsAttrs;
+    Owned<IWUResult> result = createDatasetResultSchema(querySequence(expr), NULL, original->queryRecord(), xmlnsAttrs, false, true);
 
     buildExpiryHelper(instance->createctx, expr->queryAttribute(expireAtom));
 

testing/regress/ecl/key/keydiff.xml

@@ -1,2 +1 @@
-<Dataset name=''>
-</Dataset>
+

testing/regress/ecl/key/keydiff1.xml

@@ -6,9 +6,7 @@
 </Dataset>
 <Dataset name='Result 4'>
 </Dataset>
-<Dataset name=''>
-</Dataset>
-<Dataset name=''>
+<Dataset name='Result 6'>
 </Dataset>
 <Dataset name='Result 7'>
  <Row><Result_7>0</Result_7></Row>

testing/regress/ecl/key/setup_fetch.xml

@@ -10,3 +10,5 @@
 </Dataset>
 <Dataset name='Result 6'>
 </Dataset>
+<Dataset name='Result 7'>
+</Dataset>

testing/regress/ecl/keydiff.ecl

@@ -22,7 +22,7 @@
 
 //class=file
 //class=index
-//version multiPart=false
+//noversion multiPart=false   // This should be supported but hthor adds meta information into the index that key diff doesn't support
 //version multiPart=true
 //version multiPart=true,useLocal=true
 
@@ -39,6 +39,6 @@ import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);
 
 
-KEYDIFF(Files.DG_FetchIndex1, Files.DG_FetchIndex2, Files.DG_FetchIndexDiffName, OVERWRITE);
+KEYDIFF(Files.DG_KeyDiffIndex1, Files.DG_KeyDiffIndex2, Files.DG_FetchIndexDiffName, OVERWRITE);
 
 

testing/regress/ecl/setup/files.ecl

@@ -58,9 +58,10 @@ EXPORT DG_FetchFileName     := '~REGRESS::' + filePrefix + '::C.DG_FetchFile';
 EXPORT DG_FetchFilePreloadName := '~REGRESS::' + filePrefix + '::C.DG_FetchFilePreload';
 EXPORT DG_FetchFilePreloadIndexedName := '~REGRESS::' + filePrefix + '::C.DG_FetchFilePreloadIndexed';
 EXPORT DG_FetchIndex1Name   := '~REGRESS::' + indexPrefix + '::DG_FetchIndex1';
-EXPORT DG_FetchIndex2Name   := '~REGRESS::' + indexPrefix + '::DG_FetchIndex2';
 EXPORT DG_FetchTransIndexName   := '~REGRESS::' + indexPrefix + '::DG_FetchTransIndex';
 EXPORT DG_FetchIndexDiffName:= '~REGRESS::' + indexPrefix + '::DG_FetchIndexDiff';
+EXPORT DG_KeyDiffIndex1Name   := '~REGRESS::' + indexPrefix + '::DG_KeyDiffIndex1';
+EXPORT DG_KeyDiffIndex2Name   := '~REGRESS::' + indexPrefix + '::DG_KeyDiffIndex2';
 
 EXPORT DG_DsFilename        := '~REGRESS::' + filePrefix + '::SerialLibraryDs';
 EXPORT DG_DictFilename      := '~REGRESS::' + filePrefix + '::SerialLibraryDict';
@@ -84,9 +85,10 @@ EXPORT DG_FetchFilePreloadIndexed := PRELOAD(DATASET(DG_FetchFilePreloadIndexedN
 
 
 EXPORT DG_FetchIndex1 := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname, __filepos},DG_FetchIndex1Name);
-//This version of the index is only used for KEYDIFF
 
-EXPORT DG_FetchIndex2 := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname, __filepos},DG_FetchIndex2Name);
+//These versions of the index are only used for KEYDIFF
+EXPORT DG_KeyDiffIndex1 := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield := fname, __filepos},DG_KeyDiffIndex1Name);
+EXPORT DG_KeyDiffIndex2 := INDEX(DG_KeyDiffIndex1, DG_KeyDiffIndex2Name);
 
 //This version is used for testing reading from a file requiring translation 
 EXPORT DG_FetchTransIndex := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname, __filepos},DG_FetchTransIndexName);

testing/regress/ecl/setup/setup_fetch.ecl

@@ -58,7 +58,13 @@ sortedFile := SORT(Files.DG_FETCHFILE, Lname,Fname,state ,__filepos, LOCAL);
 BUILDINDEX(sortedFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname+lname, __filepos},Files.DG_FetchIndex1Name, OVERWRITE, SORTED);
 
 //This is only used to perform a keydiff.
-BUILDINDEX(sortedFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname+lname, __filepos},Files.DG_FetchIndex2Name, OVERWRITE, SORTED);
+Files.DG_KeyDiffIndex1 createDiffRow(sortedFile l) := TRANSFORM
+    SELF.tfn := TRIM(l.Fname);
+    SELF.blobfield := l.fname+l.lname;
+    SELF := l;
+END;
+BUILDINDEX(Files.DG_KeyDiffIndex1, PROJECT(sortedFile, createDiffRow(LEFT)),OVERWRITE);
+BUILDINDEX(Files.DG_KeyDiffIndex2, PROJECT(sortedFile(lname != 'Doe'),createDiffRow(LEFT)), OVERWRITE);
 
 //A version of the index with LName/FName transposed and x moved to the front.
 BUILDINDEX(sortedFile,{Fname,Lname},{STRING100 blobfield {blob}:= fname+lname, STRING tfn := TRIM(Fname), state, __filepos},Files.DG_FetchTransIndexName, OVERWRITE);
@@ -68,15 +74,18 @@ fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchFilePrel
 
 fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchIndex1Name, '', '', 'view', '1:1', false);
 fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchIndex1Name, '__fileposition__', '__filepos', 'link', '1:1', true);
-fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchIndex2Name, '', '', 'view', '1:1', false);
-fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchIndex2Name, '__fileposition__', '__filepos', 'link', '1:1', true);
+fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_KeyDiffIndex1Name, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_KeyDiffIndex1Name, '__fileposition__', '__filepos', 'link', '1:1', true);
+fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_KeyDiffIndex2Name, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_KeyDiffIndex2Name, '__fileposition__', '__filepos', 'link', '1:1', true);
 
 //Optionally Create local versions of the indexes.
 LocalFiles := $.Files(createMultiPart, TRUE);
 IF (createMultiPart,
     PARALLEL(
         BUILDINDEX(sortedFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname+lname, __filepos},LocalFiles.DG_FetchIndex1Name, OVERWRITE, SORTED, NOROOT);
-        BUILDINDEX(sortedFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname+lname, __filepos},LocalFiles.DG_FetchIndex2Name, OVERWRITE, SORTED, NOROOT);
+        BUILDINDEX(LocalFiles.DG_KeyDiffIndex1, PROJECT(sortedFile, createDiffRow(LEFT)),OVERWRITE,NOROOT);
+        BUILDINDEX(LocalFiles.DG_KeyDiffIndex2, PROJECT(sortedFile(lname != 'Doe'),createDiffRow(LEFT)), OVERWRITE,NOROOT);
         BUILDINDEX(sortedFile,{Fname,Lname},{STRING100 blobfield {blob}:= fname+lname, STRING tfn := TRIM(Fname), state, __filepos},LocalFiles.DG_FetchTransIndexName, OVERWRITE, NOROOT);
    )
 );

testing/regress/ecl/setup/thor/setup_fetch.xml

@@ -16,3 +16,7 @@
 </Dataset>
 <Dataset name='Result 9'>
 </Dataset>
+<Dataset name='Result 10'>
+</Dataset>
+<Dataset name='Result 11'>
+</Dataset>

thorlcr/activities/keydiff/thkeydiff.cpp

@@ -159,7 +159,7 @@ public:
     {
         Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
         Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
-        r->setResultStatus(ResultStatusCalculated);
+        //Do not mark the result as calculated - because the patch file isn't a valid result
         r->setResultLogicalName(outputName);
         r.clear();
         wu.clear();