HPCC-12103 Remove virtual attributes from crc of output files

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hqlcpp/hqlckey.cpp

@@ -1453,7 +1453,10 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedJoinOrDenormalize(BuildCt
     buildFormatCrcFunction(instance->classctx, "getIndexFormatCrc", info.queryRawKey(), info.queryRawKey(), 1);
     if (info.isFullJoin())
     {
-        buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", info.queryRawRhs(), NULL, 0);
+        //Remove virtual attributes from the record, so the crc will be compatible with the disk read record
+        //can occur with a (highly unusual) full keyed join to a persist file... (see indexread14.ecl)
+        OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(info.queryRawRhs()->queryRecord());
+        buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", noVirtualRecord, NULL, 0);
         buildEncryptHelper(instance->startctx, info.queryFile()->queryAttribute(encryptAtom), "getFileEncryptKey");
     }
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10531,7 +10531,9 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
         if (expr->hasAttribute(resultAtom))
             result->setResultRowLimit(-1);
 
-        buildFormatCrcFunction(instance->classctx, "getFormatCrc", dataset, NULL, 0);
+        //Remove virtual attributes from the record, so the crc will be compatible with the disk read record
+        OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(dataset->queryRecord());
+        buildFormatCrcFunction(instance->classctx, "getFormatCrc", noVirtualRecord, NULL, 0);
 
         bool grouped = isGrouped(dataset);
         bool ignoreGrouped = !expr->hasAttribute(groupedAtom);

ecl/hqlcpp/hqlsource.cpp

@@ -573,7 +573,7 @@ IHqlExpression * HqlCppTranslator::convertToPhysicalIndex(IHqlExpression * table
 IHqlExpression * convertToPhysicalTable(IHqlExpression * tableExpr, bool ensureSerialized)
 {
     VirtualFieldsInfo fieldInfo;
-    fieldInfo.gatherVirtualFields(tableExpr->queryRecord(), false, ensureSerialized);
+    fieldInfo.gatherVirtualFields(tableExpr->queryRecord(), tableExpr->hasAttribute(_noVirtual_Atom), ensureSerialized);
     if (fieldInfo.hasVirtualsOrDeserialize())
         return createTableWithoutVirtuals(fieldInfo, tableExpr);
     return LINK(tableExpr);
@@ -2667,7 +2667,11 @@ void DiskReadBuilderBase::buildMembers(IHqlExpression * expr)
     }
 
     if (includeFormatCrc)
-        translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", physicalRecord, NULL, 0);
+    {
+        //Spill files can still have virtual attributes in their physical records => remove them.
+        OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(physicalRecord);
+        translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", noVirtualRecord, NULL, 0);
+    }
 
     buildLimits(instance->startctx, expr, instance->activityId); 
     buildKeyedLimitHelper(expr);

testing/regress/ecl/issue12103.ecl

@@ -0,0 +1,42 @@
+
+//Record just containing the ids.
+r1 := RECORD
+    unsigned id;
+END;
+
+//Record containing ids with positions automatically added to it
+r2 := RECORD(r1)
+    unsigned8 filepos{virtual(fileposition)};
+END;
+
+//Record with real fileposition fields 
+r3 := RECORD(r1)
+    unsigned8 filepos;
+END;
+
+//Generate a datafile containing ids.
+ds := DATASET(10, TRANSFORM(r1, SELF.id := COUNTER));
+output(ds,,'REGRESS::temp::oraw',OVERWRITE);
+
+//Read it using a record which will associate file positions with each record.
+dsraw := DATASET('REGRESS::temp::oraw',r2, THOR);
+
+//Write as-is (will now have physical file position fields)
+output(dsraw, ,'REGRESS::temp::o1',OVERWRITE);
+
+//Project out the file posistion fields before writing
+output(dsraw, { id } ,'REGRESS::temp::o2',OVERWRITE);
+
+//Project it to a record that has no virtuals
+output(PROJECT(dsraw, TRANSFORM(r3, SELF := LEFT)), ,'REGRESS::temp::o3',OVERWRITE);
+
+//Now check each form can be read correctly...
+d1 := DATASET('REGRESS::temp::o1',r3, THOR);
+output(count(d1));
+
+d2 := DATASET('REGRESS::temp::o2',r1, THOR);
+output(count(d2));
+
+d3 := DATASET('REGRESS::temp::o3',r3, THOR);
+output(count(d3));
+

testing/regress/ecl/key/issue12103.xml

@@ -0,0 +1,17 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>10</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>10</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>10</Result_7></Row>
+</Dataset>