Merge pull request #14187 from richardkchapman/translate-override

HPCC-24647 Improve record translation handling/warnings

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/thorhelper/thorcommon.cpp

@@ -2086,7 +2086,12 @@ static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<con
     {
         IOutputMetaData * sourceFormat = expectedFormat;
         unsigned sourceCrc = expectedCrc;
-        if (mode != RecordTranslationMode::AlwaysECL)
+        if (mode == RecordTranslationMode::AlwaysECL)
+        {
+            if (publishedCrc && expectedCrc && (publishedCrc != expectedCrc))
+                DBGLOG("Overriding stored record layout reading file %s", tracing);
+        }
+        else
         {
             if (publishedFormat)
             {

common/thorhelper/thorread.cpp

@@ -113,7 +113,12 @@ void DiskReadMapping::ensureTranslators() const
     const char * filename = ""; // not known at this point
     IOutputMetaData * sourceMeta = expectedMeta;
     unsigned sourceCrc = expectedCrc;
-    if (mode != RecordTranslationMode::AlwaysECL)
+    if (mode == RecordTranslationMode::AlwaysECL)
+    {
+        if (actualCrc && expectedCrc && (actualCrc != expectedCrc))
+            DBGLOG("Overriding stored record layout reading file %s", filename);
+    }
+    else
     {
         if (actualCrc && actualMeta)
         {

common/workunit/pkgimpl.hpp

@@ -124,7 +124,7 @@ protected:
         const char *val = queryEnv("control:enableFieldTranslation");
         if (!val) val = queryEnv("enableFieldTranslation"); // Backward compatibility
         if (val)
-            return getTranslationMode(val);
+            return getTranslationMode(val, false);
         else
             return getSysFieldTranslationEnabled();
     }

ecl/eclagent/eclagent.cpp

@@ -705,7 +705,7 @@ RecordTranslationMode EclAgent::getLayoutTranslationMode() const
         wu->getDebugValue("layoutTranslation", val);
     else
         agentTopology->getProp("@fieldTranslationEnabled", val.s);
-    return getTranslationMode(val.str());
+    return getTranslationMode(val.str(), false);
 }
 
 IConstWUResult *EclAgent::getResult(const char *name, unsigned sequence)

ecl/hthor/hthor.cpp

@@ -8133,7 +8133,7 @@ CHThorDiskReadBaseActivity::CHThorDiskReadBaseActivity(IAgentContext &_agent, un
     {
         const char *recordTranslationModeHintText = _node->queryProp("hint[@name='layouttranslation']/@value");
         if (recordTranslationModeHintText)
-            recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText);
+            recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText, true);
         isCodeSigned = isActivityCodeSigned(*_node);
     }
 }
@@ -10542,7 +10542,7 @@ CHThorNewDiskReadBaseActivity::CHThorNewDiskReadBaseActivity(IAgentContext &_age
     {
         const char *recordTranslationModeHintText = _node->queryProp("hint[@name='layouttranslation']/@value");
         if (recordTranslationModeHintText)
-            recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText);
+            recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText, true);
         isCodeSigned = isActivityCodeSigned(*_node);
     }
 

ecl/hthor/hthorkey.cpp

@@ -373,7 +373,7 @@ CHThorIndexReadActivityBase::CHThorIndexReadActivityBase(IAgentContext &_agent,
     {
         const char *recordTranslationModeHintText = _node->queryProp("hint[@name='layouttranslation']/@value");
         if (recordTranslationModeHintText)
-            recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText);
+            recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText, true);
     }
 }
 
@@ -732,6 +732,7 @@ const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDist
     switch (getLayoutTranslationMode())
     {
     case RecordTranslationMode::AlwaysECL:
+        verifyFormatCrc(helper.getDiskFormatCrc(), f, (superIterator ? superName.str() : NULL) , true, false);
         break;
     case RecordTranslationMode::None:
         verifyFormatCrc(helper.getDiskFormatCrc(), f, (superIterator ? superName.str() : NULL) , true, true);
@@ -2291,7 +2292,7 @@ public:
         {
             const char *recordTranslationModeHintText = _node->queryProp("hint[@name='layouttranslation']/@value");
             if (recordTranslationModeHintText)
-                recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText);
+                recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText, true);
         }
     }
 
@@ -3484,7 +3485,7 @@ public:
         {
             const char *recordTranslationModeHintText = _node->queryProp("hint[@name='layouttranslation']/@value");
             if (recordTranslationModeHintText)
-                recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText);
+                recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText, true);
             isCodeSigned = isActivityCodeSigned(*_node);
         }
     }
@@ -4149,6 +4150,7 @@ protected:
     {
         if(getLayoutTranslationMode() == RecordTranslationMode::AlwaysECL)
         {
+            verifyFormatCrc(helper.getIndexFormatCrc(), f, super ? super->queryLogicalName() : NULL, true, false);  // Traces if mismatch
             return NULL;
         }
 

roxie/ccd/ccdactivities.cpp

@@ -114,7 +114,7 @@ CActivityFactory::CActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFac
     }
     const char *recordTranslationModeHintText = _graphNode.queryProp("hint[@name='layouttranslation']/@value");
     if (recordTranslationModeHintText)
-        recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText);
+        recordTranslationModeHint = getTranslationMode(recordTranslationModeHintText, true);
 }
 
 void CActivityFactory::addChildQuery(unsigned id, ActivityArray *childQuery) 

roxie/ccd/ccdfile.cpp

@@ -2728,7 +2728,12 @@ public:
                 {
                     int thisFormatCrc = 0;
                     bool actualUnknown = true;
-                    if (mode != RecordTranslationMode::AlwaysECL)
+                    if (mode == RecordTranslationMode::AlwaysECL)
+                    {
+                        if (formatCrcs.item(idx) && expectedFormatCrc && (formatCrcs.item(idx) != expectedFormatCrc))
+                            DBGLOG("Overriding stored record layout reading file %s", subname);
+                    }
+                    else
                     {
                         thisFormatCrc = formatCrcs.item(idx);
                         if (diskTypeInfo.item(idx))

roxie/ccd/ccdmain.cpp

@@ -1005,7 +1005,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         fieldTranslationEnabled = RecordTranslationMode::Payload;
         const char *val = topology->queryProp("@fieldTranslationEnabled");
         if (val)
-            fieldTranslationEnabled = getTranslationMode(val);
+            fieldTranslationEnabled = getTranslationMode(val, false);
 
         pretendAllOpt = topology->getPropBool("@ignoreMissingFiles", false);
         memoryStatsInterval = topology->getPropInt("@memoryStatsInterval", 60);

roxie/ccd/ccdquery.cpp

@@ -442,7 +442,7 @@ void QueryOptions::updateFromWorkUnit(RecordTranslationMode &value, IConstWorkUn
     SCMStringBuffer val;
     wu.getDebugValue(name, val);
     if (val.length())
-        value = getTranslationMode(val.str());
+        value = getTranslationMode(val.str(), false);
 }
 
 void QueryOptions::setFromContext(const IPropertyTree *ctx)

roxie/ccd/ccdstate.cpp

@@ -2277,7 +2277,7 @@ private:
             {
                 const char *val = control->queryProp("@val");
                 if (val)
-                    fieldTranslationEnabled = getTranslationMode(val);
+                    fieldTranslationEnabled = getTranslationMode(val, false);
                 else
                     fieldTranslationEnabled = RecordTranslationMode::Payload;
                 val = getTranslationModeText(fieldTranslationEnabled);

rtl/eclrtl/rtldynfield.cpp

@@ -34,14 +34,22 @@
 
 //---------------------------------------------------------------------------------------------------------------------
 
-extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val)
+extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool isLocal)
 {
     if (isEmptyString(val) || strToBool(val) || strieq(val, "payload"))
         return RecordTranslationMode::Payload;
     else if (strieq(val, "alwaysDisk") || strieq(val, "disk"))
+    {
+        if (!isLocal)
+            throw makeStringException(0, "alwaysDisk translation mode can only be set via HINT");
         return RecordTranslationMode::AlwaysDisk;
+    }
     else if (strieq(val, "alwaysECL") || strieq(val, "ecl"))
+    {
+        if (!isLocal)
+            throw makeStringException(0, "alwaysECL translation mode can only be set via HINT");
         return RecordTranslationMode::AlwaysECL;
+    }
     else
         return RecordTranslationMode::None;
 }

rtl/eclrtl/rtldynfield.hpp

@@ -111,9 +111,9 @@ enum class RecordTranslationMode : byte
     AlwaysDisk = 3, // Always translate - even if wouldn't normally (e.g. csv/xml source read as binary), or crcs happen to match
     AlwaysECL = 4,  // Ignore the published format - can make sense to force no translation e.g. when field names have changed
     Unspecified = 5
-};  // AlwaysDisk and AlwaysECL are for testing purposes only
+};  // AlwaysDisk and AlwaysECL are for testing purposes only, and can only be set per file (not globally)
 
-extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *modeStr);
+extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *modeStr, bool isLocal);
 extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val);
 
 interface IDynamicRowIterator;

testing/regress/ecl/serial3b.ecl

@@ -19,8 +19,6 @@
 
 import Setup.SerialTest;
 
-#option ('layoutTranslation', 'ecl');
-
-inDs := DATASET(SerialTest.DictFilename, SerialTest.LibraryDsRec, THOR);
+inDs := DATASET(SerialTest.DictFilename, SerialTest.LibraryDsRec, THOR, HINT(layoutTranslation, ecl));
 
 output(TABLE(inDs, { owner, cnt := COUNT(books) }));

testing/regress/ecl/serial3c.ecl

@@ -19,7 +19,5 @@
 
 import Setup.SerialTest;
 
-#option ('layoutTranslation', 'ecl');
-
-inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDictRec, THOR);
+inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDictRec, THOR, hint(layoutTranslation, ecl));
 output(TABLE(inDs, { owner, cnt := COUNT(books) }));

testing/regress/ecl/setup/files.ecl

@@ -22,7 +22,7 @@ import ^ as root;
 import std.str;
 
 //define constants
-EXPORT files(boolean multiPart, boolean useLocal, boolean useTranslation = false) := module
+EXPORT files(boolean multiPart, boolean useLocal, boolean useTranslation = false, string extraPrefix = '') := module
 
 EXPORT DG_MaxField          := 3;    // maximum number of fields to use building the data records
 EXPORT DG_MaxChildren       := 3;    //maximum (1 to n) number of child recs
@@ -42,27 +42,19 @@ SHARED STRING EmptyString := '' : STORED('dummy');
 SHARED STRING EmptyString := '';
 #end
 
-SHARED forceLayoutTranslation := #IFDEFINED(root.forceLayoutTranslation, 0);
-
 SHARED STRING _filePrefix := '~regress::' + 
         MAP(multiPart => 'multi', 'single') + 
-        IF(forceLayoutTranslation > 0, '_' + (STRING) forceLayoutTranslation, '') + 
+        extraPrefix +  
         '::' + EmptyString;
 
 //Yuk cannot use MAP because that creates a string6        
 SHARED STRING _indexPrefix := '~regress::' + 
-        IF(multiPart AND useLocal, 'local', IF(multiPart, 'multi', 'single')) + 
-        IF(forceLayoutTranslation > 0, '_' + (STRING) forceLayoutTranslation, '') + 
+        IF(multiPart AND useLocal, 'local', IF(multiPart, 'multi', 'single')) +
+        extraPrefix +
         '::' + EmptyString;
 
-#IF (forceLayoutTranslation != 0)
-  SHARED setLayout := #option('layoutTranslation', CASE(forceLayoutTranslation,1=>v'alwaysECL',2=>v'alwaysDisk',v''));
-  EXPORT filePrefix := WHEN(#IFDEFINED(root.filePrefix, _filePrefix), setLayout);
-  EXPORT indexPrefix := WHEN(#IFDEFINED(root.filePrefix, _indexPrefix), setLayout);
-#else      
-  EXPORT filePrefix := #IFDEFINED(root.filePrefix, _filePrefix);
-  EXPORT indexPrefix := #IFDEFINED(root.filePrefix, _indexPrefix);
-#end
+EXPORT filePrefix := #IFDEFINED(root.filePrefix, _filePrefix);
+EXPORT indexPrefix := #IFDEFINED(root.filePrefix, _indexPrefix);
 
 EXPORT QueryFilePrefixId := __TARGET_PLATFORM__ + '::' + Str.ToLowerCase(WORKUNIT) + '::';
 EXPORT QueryFilePrefix := filePrefix + QueryFilePrefixId;

testing/regress/ecl/toxml.ecl

@@ -21,7 +21,20 @@
 //version forceLayoutTranslation=2
 
 import $.setup;
-prefix := setup.Files(false, false).QueryFilePrefix;
+import ^ as root;
+
+forceLayoutTranslation := #IFDEFINED(root.forceLayoutTranslation, 0);
+prefix := setup.Files(false, false, false, IF(forceLayoutTranslation > 0, '_' + (STRING) forceLayoutTranslation, '')).QueryFilePrefix;
+
+LOADXML('');
+#DECLARE(translationMode);
+#IF (forceLayoutTranslation = 1)
+#SET(translationmode, 'alwaysECL');
+#ELSIF (forceLayoutTranslation = 2)
+#SET(translationmode, 'alwaysDisk');
+#ELSE
+#SET(translationmode, 'on');
+#END
 
 phoneRecord := 
             RECORD
@@ -62,7 +75,7 @@ string2         endmarker := '$$';
 namesTable := dataset([
         {U'Hälliday','Gavin','09876',123987,true,'07967',123987, 'n/a','n/a',true,'gavin@edata.com',[{U'εν αρχη ην ο λογος', 'john'}, {U'To kill a mocking bird','Lee'},{U'Zen and the art of motorcycle maintainence','Pirsig'}], ALL},
         {U'Halliday','Abigäil','09876',123987,false,'','',false,[{U'The cat in the hat','Suess'},{U'Wolly the sheep',''}], ['Red','Yellow']}
-        ], personRecord);
+        ], personRecord, HINT(layoutTranslation, %translationmode%));
 
 output(U'<row>'+(utf8)U'εν αρχη ην ο λογος'+U'</row>');
 output(U'<row>'+U8'εν αρχη ην ο λογος'+U'</row>');
@@ -73,7 +86,7 @@ setup1 := output(namesTable,,prefix+'toxml1.xml',expire(1),overwrite,xml(heading
 setup2 := output(namesTable,,prefix+'toxml2.flat',expire(1),overwrite);
 
 //Read from xml and then write out again - should be possible to compare the results easily
-inf := dataset(prefix+'toxml1.xml', personRecord, xml('/MyDataset/Row'));
+inf := dataset(prefix+'toxml1.xml', personRecord, xml('/MyDataset/Row'), HINT(layoutTranslation, %translationmode%));
 setup3 := output(inf,,prefix+'toxml3.xml',expire(1),overwrite,xml);
 
 setup4 := output(inf,{string xml{maxlength(1024)} := (string)toxml(row(inf))},prefix+'toxml4.utf8',expire(1),overwrite,csv(unicode));
@@ -91,7 +104,7 @@ END;
 z := parse(p, p.text, createRowFromXml(), xml('/row'));
 output(z);
 
-infraw := dataset(prefix+'toxml2.flat', personRecord, thor);
+infraw := dataset(prefix+'toxml2.flat', personRecord, thor, HINT(layoutTranslation, %translationmode%));
 setup7 := output(infraw,{string xml{maxlength(1024)} := (string)toxml(row(infraw))},prefix+'toxml7.utf8',expire(1),overwrite,csv(unicode));
 
 //Now read each of the xml files and output to the work unit as csv so we can check the content!
@@ -103,10 +116,10 @@ setup4;
 setup5;
 setup6;
 setup7;
-output(dataset(prefix+'toxml1.xml', { unicode line{maxlength(4096)} }, csv(unicode))(line not in ['<MyDataset>','</MyDataset>']));
-output(dataset(prefix+'toxml3.xml', { unicode line{maxlength(4096)} }, csv(unicode))(line not in ['<Dataset>','</Dataset>']));
-output(dataset(prefix+'toxml4.utf8', { unicode line{maxlength(4096)} }, csv(unicode)));
-output(dataset(prefix+'toxml5.utf8', { unicode line{maxlength(4096)} }, csv(unicode)));
-output(dataset(prefix+'toxml6.utf8', { unicode line{maxlength(4096)} }, csv(unicode)));
-output(dataset(prefix+'toxml7.utf8', { unicode line{maxlength(4096)} }, csv(unicode)));
+output(dataset(prefix+'toxml1.xml', { unicode line{maxlength(4096)} }, csv(unicode), HINT(layoutTranslation, %translationmode%))(line not in ['<MyDataset>','</MyDataset>']));
+output(dataset(prefix+'toxml3.xml', { unicode line{maxlength(4096)} }, csv(unicode), HINT(layoutTranslation, %translationmode%))(line not in ['<Dataset>','</Dataset>']));
+output(dataset(prefix+'toxml4.utf8', { unicode line{maxlength(4096)} }, csv(unicode), HINT(layoutTranslation, %translationmode%)));
+output(dataset(prefix+'toxml5.utf8', { unicode line{maxlength(4096)} }, csv(unicode), HINT(layoutTranslation, %translationmode%)));
+output(dataset(prefix+'toxml6.utf8', { unicode line{maxlength(4096)} }, csv(unicode), HINT(layoutTranslation, %translationmode%)));
+output(dataset(prefix+'toxml7.utf8', { unicode line{maxlength(4096)} }, csv(unicode), HINT(layoutTranslation, %translationmode%)));
 );

thorlcr/slave/slavmain.cpp

@@ -350,7 +350,11 @@ class CKJService : public CSimpleInterfaceOf<IKJService>, implements IThreaded,
                 if (RecordTranslationMode::AlwaysDisk == translationMode)
                     translator.setown(createRecordTranslator(projectedFormat->queryRecordAccessor(true), publishedFormat->queryRecordAccessor(true)));
                 else if (RecordTranslationMode::AlwaysECL == translationMode)
+                {
+                    if (publishedFormatCrc && publishedFormatCrc != expectedFormatCrc)
+                        DBGLOG("Overriding stored record layout reading file %s", tracing);
                     translator.setown(createRecordTranslator(projectedFormat->queryRecordAccessor(true), expectedFormat->queryRecordAccessor(true)));
+                }
                 else if (publishedFormatCrc && publishedFormatCrc != expectedFormatCrc)
                 {
                     if (!projectedFormat)

thorlcr/thorutil/thormisc.cpp

@@ -1468,11 +1468,15 @@ IThorException *checkAndCreateOOMContextException(CActivityBase *activity, IExce
 
 RecordTranslationMode getTranslationMode(CActivityBase &activity)
 {
+    bool local = true;
     StringBuffer val;
     activity.getOpt("layoutTranslation", val);
     if (!val.length())
+    {
         globals->getProp("@fieldTranslationEnabled", val);
-    return getTranslationMode(val);
+        local = false;
+    }
+    return getTranslationMode(val, local);
 }
 
 void getLayoutTranslations(IConstPointerArrayOf<ITranslator> &translators, const char *fname, IArrayOf<IPartDescriptor> &partDescriptors, RecordTranslationMode translationMode, unsigned expectedFormatCrc, IOutputMetaData *expectedFormat, unsigned projectedFormatCrc, IOutputMetaData *projectedFormat)