HPCC-19590 Avoid creating translator if projectedCrc == sourceCrc

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/thorhelper/thorcommon.cpp

@@ -2033,7 +2033,7 @@ extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &pr
     return nullptr;
 }
 
-static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> *keyedTranslator, const char *tracing, IOutputMetaData *expectedFormat, IOutputMetaData *publishedFormat, IOutputMetaData *projectedFormat, RecordTranslationMode mode, unsigned expectedCrc, bool skipFileFormatCrcCheck, unsigned publishedCrc)
+static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> *keyedTranslator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck)
 {
     if (expectedCrc)
     {
@@ -2051,9 +2051,8 @@ static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<con
             }
         }
 
-        //MORE: HPCC-19590 Could avoid creating translator if (projectedCrc != sourceCrc).
-        //However that would allow false negatives if the format crcs happened to match, leading to crashes on untranslated files.
-        if (projectedFormat != sourceFormat)
+        //This has a very low possibility of format crcs accidentally matching, which could lead to a crashes on an untranslated files.
+        if ((projectedFormat != sourceFormat) && (projectedCrc != sourceCrc))
         {
             translator.setown(createRecordTranslator(projectedFormat->queryRecordAccessor(true), sourceFormat->queryRecordAccessor(true)));
 
@@ -2080,21 +2079,21 @@ static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<con
     return nullptr != translator.get();
 }
 
-bool getTranslators(Owned<const IDynamicTransform> &translator, const char *tracing, IOutputMetaData *expectedFormat, IOutputMetaData *publishedFormat, IOutputMetaData *projectedFormat, RecordTranslationMode mode, unsigned expectedCrc, bool skipFileFormatCrcCheck, unsigned publishedCrc)
+bool getTranslators(Owned<const IDynamicTransform> &translator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck)
 {
-    return getTranslators(translator, nullptr, tracing, expectedFormat, publishedFormat, projectedFormat, mode, expectedCrc, skipFileFormatCrcCheck, publishedCrc);
+    return getTranslators(translator, nullptr, tracing, expectedCrc, expectedFormat, publishedCrc, publishedFormat, projectedCrc, projectedFormat, mode, skipFileFormatCrcCheck);
 }
 
-bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> &keyedTranslator, const char *tracing, IOutputMetaData *expectedFormat, IOutputMetaData *publishedFormat, IOutputMetaData *projectedFormat, RecordTranslationMode mode, unsigned expectedCrc, bool skipFileFormatCrcCheck, unsigned publishedCrc)
+bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> &keyedTranslator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck)
 {
-    return getTranslators(translator, &keyedTranslator, tracing, expectedFormat, publishedFormat, projectedFormat, mode, expectedCrc, skipFileFormatCrcCheck, publishedCrc);
+    return getTranslators(translator, &keyedTranslator, tracing, expectedCrc, expectedFormat, publishedCrc, publishedFormat, projectedCrc, projectedFormat, mode, skipFileFormatCrcCheck);
 }
 
-ITranslator *getTranslators(const char *tracing, IOutputMetaData *expectedFormat, IOutputMetaData *publishedFormat, IOutputMetaData *projectedFormat, RecordTranslationMode mode, unsigned expectedCrc, bool skipFileFormatCrcCheck, unsigned publishedCrc)
+ITranslator *getTranslators(const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck)
 {
     Owned<const IDynamicTransform> translator;
     Owned<const IKeyTranslator> keyedTranslator;
-    if (getTranslators(translator, &keyedTranslator, tracing, expectedFormat, publishedFormat, projectedFormat, mode, expectedCrc, skipFileFormatCrcCheck, publishedCrc))
+    if (getTranslators(translator, &keyedTranslator, tracing, expectedCrc, expectedFormat, publishedCrc, publishedFormat, projectedCrc, projectedFormat, mode, skipFileFormatCrcCheck))
     {
         if (!publishedFormat)
             publishedFormat = expectedFormat;

common/thorhelper/thorcommon.hpp

@@ -638,10 +638,10 @@ extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &pr
  * providing translation mode and crc's allow translation. Returns true if translator created.
  * NB: translator and keyedTranslator are expected to be empty before calling.
 */
-extern THORHELPER_API bool getTranslators(Owned<const IDynamicTransform> &translator, const char *tracing, IOutputMetaData *expectedFormat, IOutputMetaData *publishedFormat, IOutputMetaData *projectedFormat, RecordTranslationMode mode, unsigned expectedCrc, bool skipFileFormatCrcCheck, unsigned publishedCrc);
+extern THORHELPER_API bool getTranslators(Owned<const IDynamicTransform> &translator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck);
 // Same as above, but will also return a key field translator in 2nd parameter. Returns true if translator created.
-extern THORHELPER_API bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> &keyedTranslator, const char *tracing, IOutputMetaData *expectedFormat, IOutputMetaData *publishedFormat, IOutputMetaData *projectedFormat, RecordTranslationMode mode, unsigned expectedCrc, bool skipFileFormatCrcCheck, unsigned publishedCrc);
+extern THORHELPER_API bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> &keyedTranslator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck);
 // Returns a ITranslator that gives access to a dynamic translator, keyed translator and the format used
-extern THORHELPER_API ITranslator *getTranslators(const char *tracing, IOutputMetaData *expectedFormat, IOutputMetaData *publishedFormat, IOutputMetaData *projectedFormat, RecordTranslationMode mode, unsigned expectedCrc, bool skipFileFormatCrcCheck, unsigned publishedCrc);
+extern THORHELPER_API ITranslator *getTranslators(const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck);
 
 #endif // THORHELPER_HPP

thorlcr/activities/diskread/thdiskread.cpp

@@ -67,7 +67,7 @@ public:
             }
         }
         if (0 == (TDRnocrccheck & helper->getFlags()))
-            checkFormatCrc(this, file, helper->getDiskFormatCrc(), helper->getProjectedFormatCrc(), helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), false);
+            checkFormatCrc(this, file, helper->getDiskFormatCrc(), helper->queryDiskRecordSize(), helper->getProjectedFormatCrc(), helper->queryProjectedDiskRecordSize(), false);
     }
 };
 

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -67,14 +67,15 @@ protected:
     // return a ITranslator based on published format in part and expected/format
     ITranslator *getTranslators(IPartDescriptor &partDesc)
     {
-        unsigned expectedFormatCrc = helper->getDiskFormatCrc();
+        unsigned projectedFormatCrc = helper->getProjectedFormatCrc();
         IOutputMetaData *projectedFormat = helper->queryProjectedDiskRecordSize();
         IPropertyTree const &props = partDesc.queryOwner().queryProperties();
         Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
         unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
         RecordTranslationMode translationMode = getTranslationMode(*this);
+        unsigned expectedFormatCrc = helper->getDiskFormatCrc();
         IOutputMetaData *expectedFormat = helper->queryDiskRecordSize();
-        return ::getTranslators("rowstream", expectedFormat, publishedFormat, projectedFormat, translationMode, expectedFormatCrc, false, publishedFormatCrc);
+        return ::getTranslators("rowstream", expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
     }
 public:
     CDiskReadSlaveActivityRecord(CGraphElementBase *_container, IHThorArg *_helper=NULL) 

thorlcr/activities/fetch/thfetchslave.cpp

@@ -335,9 +335,10 @@ public:
         if (files)
         {
             unsigned expectedFormatCrc = fetchBaseHelper->getDiskFormatCrc();
+            unsigned projectedFormatCrc = fetchBaseHelper->getProjectedFormatCrc();
             IOutputMetaData *projectedFormat = fetchBaseHelper->queryProjectedDiskRecordSize();
             RecordTranslationMode translationMode = getTranslationMode(*this);
-            getLayoutTranslations(translators, fetchBaseHelper->getFileName(), parts, translationMode, fetchBaseHelper->queryDiskRecordSize(), projectedFormat, expectedFormatCrc);
+            getLayoutTranslations(translators, fetchBaseHelper->getFileName(), parts, translationMode, expectedFormatCrc, fetchBaseHelper->queryDiskRecordSize(), projectedFormatCrc, projectedFormat);
             ForEachItemIn(p, parts)
             {
                 const ITranslator *translator = translators.item(p);

thorlcr/activities/hashdistrib/thhashdistrib.cpp

@@ -134,7 +134,7 @@ public:
         if (!isFileKey(file))
             throw MakeActivityException(this, 0, "Attempting to read flat file as an index: %s", indexFileName.get());
 
-        checkFormatCrc(this, file, helper->getFormatCrc(), helper->getFormatCrc(), nullptr, nullptr, true);
+        checkFormatCrc(this, file, helper->getFormatCrc(), nullptr, helper->getFormatCrc(), nullptr, true);
         Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
         Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
         if (!tlkDesc->queryProperties().hasProp("@kind") || 0 != stricmp("topLevelKey", tlkDesc->queryProperties().queryProp("@kind")))

thorlcr/activities/indexread/thindexread.cpp

@@ -223,7 +223,7 @@ public:
                     nofilter = true;
             }
             //MORE: Change index getFormatCrc once we support projected rows for indexes.
-            checkFormatCrc(this, index, indexBaseHelper->getDiskFormatCrc(), indexBaseHelper->getProjectedFormatCrc(), indexBaseHelper->queryProjectedDiskRecordSize(), indexBaseHelper->queryDiskRecordSize(), true);
+            checkFormatCrc(this, index, indexBaseHelper->getDiskFormatCrc(), indexBaseHelper->queryDiskRecordSize(), indexBaseHelper->getProjectedFormatCrc(), indexBaseHelper->queryProjectedDiskRecordSize(), true);
             if ((container.queryLocalOrGrouped() || indexBaseHelper->canMatchAny()) && index->numParts())
             {
                 fileDesc.setown(getConfiguredFileDescriptor(*index));

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -107,15 +107,16 @@ protected:
     // return a ITranslator based on published format in part and expected/format
     ITranslator *getTranslators(IPartDescriptor &partDesc)
     {
-        unsigned expectedFormatCrc = helper->getDiskFormatCrc();
+        unsigned projectedFormatCrc = helper->getProjectedFormatCrc();
         IOutputMetaData *projectedFormat = helper->queryProjectedDiskRecordSize();
         IPropertyTree const &props = partDesc.queryOwner().queryProperties();
         Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
         unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
         RecordTranslationMode translationMode = getTranslationMode(*this);
+        unsigned expectedFormatCrc = helper->getDiskFormatCrc();
         IOutputMetaData *expectedFormat = helper->queryDiskRecordSize();
 
-        Owned<ITranslator> ret = ::getTranslators("rowstream", expectedFormat, publishedFormat, projectedFormat, translationMode, expectedFormatCrc, false, publishedFormatCrc);
+        Owned<ITranslator> ret = ::getTranslators("rowstream", expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
         if (!ret)
             return nullptr;
         if (!ret->queryTranslator().canTranslate())
@@ -145,16 +146,16 @@ public:
                     return nullptr;
             }
             RecordTranslationMode translationMode = getTranslationMode(*this);
-            IOutputMetaData *projectedFormat = helper->queryProjectedDiskRecordSize();
             unsigned expectedFormatCrc = helper->getDiskFormatCrc();
+            IOutputMetaData *expectedFormat = helper->queryDiskRecordSize();
+            unsigned projectedFormatCrc = helper->getProjectedFormatCrc();
+            IOutputMetaData *projectedFormat = helper->queryProjectedDiskRecordSize();
 
             unsigned p = partNum;
             while (p<partDescs.ordinality()) // will process all parts if localMerge
             {
                 IPartDescriptor &part = partDescs.item(p++);
 
-                IOutputMetaData *projectedFormat = helper->queryProjectedDiskRecordSize();
-                IOutputMetaData *expectedFormat = helper->queryDiskRecordSize();
                 Owned<ITranslator> translator = getTranslators(part);
                 IOutputMetaData *actualFormat = translator ? &translator->queryActualFormat() : expectedFormat;
                 bool canSerializeTypeInfo = actualFormat->queryTypeInfo()->canSerialize() && projectedFormat->queryTypeInfo()->canSerialize();
@@ -237,7 +238,7 @@ public:
                     if (!keyIndexSet)
                     {
                         keyIndexSet.setown(createKeyIndexSet());
-                        Owned<const ITranslator> translator = getLayoutTranslation(helper->getFileName(), part, translationMode, helper->queryDiskRecordSize(), projectedFormat, expectedFormatCrc);
+                        Owned<const ITranslator> translator = getLayoutTranslation(helper->getFileName(), part, translationMode, expectedFormatCrc, expectedFormat, projectedFormatCrc, projectedFormat);
                         translators.append(translator.getClear());
                     }
                     keyIndexSet->addIndex(keyIndex.getClear());
@@ -246,7 +247,7 @@ public:
                 }
                 else
                 {
-                    Owned<const ITranslator> translator = getLayoutTranslation(helper->getFileName(), part, translationMode, helper->queryDiskRecordSize(), projectedFormat, expectedFormatCrc);
+                    Owned<const ITranslator> translator = getLayoutTranslation(helper->getFileName(), part, translationMode, expectedFormatCrc, expectedFormat, projectedFormatCrc, projectedFormat);
                     if (translator)
                         klManager->setLayoutTranslator(&translator->queryTranslator());
                     translators.append(translator.getClear());

thorlcr/activities/indexwrite/thindexwrite.cpp

@@ -145,7 +145,7 @@ public:
             assertex(!isLocal);
             buildTlk = false;
             Owned<IDistributedFile> _f = queryThorFileManager().lookup(container.queryJob(), diName);
-            checkFormatCrc(this, _f, helper->getFormatCrc(), helper->getFormatCrc(), nullptr, nullptr, true);
+            checkFormatCrc(this, _f, helper->getFormatCrc(), nullptr, helper->getFormatCrc(), nullptr, true);
             IDistributedFile *f = _f->querySuperFile();
             if (!f) f = _f;
             Owned<IDistributedFilePart> existingTlk = f->getPart(f->numParts()-1);

thorlcr/activities/keyedjoin/thkeyedjoin-legacy.cpp

@@ -106,7 +106,7 @@ public:
             if (container.queryLocalData() && !localKey)
                 throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local");
 
-            checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), helper->getIndexFormatCrc(), helper->queryProjectedIndexRecordSize(), helper->queryIndexRecordSize(), true);
+            checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), helper->queryIndexRecordSize(), helper->getProjectedIndexFormatCrc(), helper->queryProjectedIndexRecordSize(), true);
             Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor();
             IDistributedSuperFile *superIndex = indexFile->querySuperFile();
             unsigned superIndexWidth = 0;

thorlcr/activities/keyedjoin/thkeyedjoin.cpp

@@ -338,7 +338,7 @@ public:
                     remoteKeyedFetch = false;
                 }
                 //MORE: Change to getIndexProjectedFormatCrc once we support projected rows for indexes?
-                checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), helper->getIndexFormatCrc(), helper->queryProjectedIndexRecordSize(), helper->queryIndexRecordSize(), true);
+                checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), helper->queryIndexRecordSize(), helper->getProjectedIndexFormatCrc(), helper->queryProjectedIndexRecordSize(), true);
                 indexFileDesc.setown(indexFile->getFileDescriptor());
 
                 unsigned superIndexWidth = 0;

thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp

@@ -727,9 +727,10 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
                 aborted = false;
                 threaded.start();
                 unsigned expectedFormatCrc = owner.helper->getDiskFormatCrc();
+                unsigned projectedFormatCrc = owner.helper->getProjectedFormatCrc();
                 IOutputMetaData *projectedFormat = owner.helper->queryProjectedDiskRecordSize();
                 RecordTranslationMode translationMode = getTranslationMode(owner);
-                getLayoutTranslations(translators, owner.helper->getFileName(), owner.dataParts, translationMode, owner.helper->queryDiskRecordSize(), projectedFormat, expectedFormatCrc);
+                getLayoutTranslations(translators, owner.helper->getFileName(), owner.dataParts, translationMode, expectedFormatCrc, owner.helper->queryDiskRecordSize(), projectedFormatCrc, projectedFormat);
                 ForEachItemIn(p, owner.dataParts)
                 {
                     const ITranslator *translator = translators.item(p);
@@ -1258,6 +1259,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
             reset();
             owner.getKeyIndexes(partKeyIndexes);
             RecordTranslationMode translationMode = getTranslationMode(owner);
+            unsigned projectedFormatCrc = owner.helper->getProjectedIndexFormatCrc();
             IOutputMetaData *projectedFormat = owner.helper->queryProjectedIndexRecordSize();
             unsigned expectedFormatCrc = owner.helper->getIndexFormatCrc();
             if (owner.localKey && (partKeyIndexes.ordinality() > 1))
@@ -1266,7 +1268,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
                 ForEachItemIn(i, partKeyIndexes)
                     partKeySet->addIndex(LINK(&partKeyIndexes.item(i)));
                 partManager.setown(createKeyMerger(owner.helper->queryIndexRecordSize()->queryRecordAccessor(true), partKeySet, 0, nullptr, owner.helper->hasNewSegmentMonitors()));
-                Owned<const ITranslator> translator = getLayoutTranslation(owner.helper->getFileName(), owner.indexParts.item(0), translationMode, owner.helper->queryIndexRecordSize(), projectedFormat, expectedFormatCrc);
+                Owned<const ITranslator> translator = getLayoutTranslation(owner.helper->getFileName(), owner.indexParts.item(0), translationMode, expectedFormatCrc, owner.helper->queryIndexRecordSize(), projectedFormatCrc, projectedFormat);
                 if (translator)
                     partManager->setLayoutTranslator(&translator->queryTranslator());
                 translators.append(translator.getClear());
@@ -1274,7 +1276,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
             else
             {
                 partManager.setown(createLocalKeyManager(owner.helper->queryIndexRecordSize()->queryRecordAccessor(true), nullptr, nullptr, owner.helper->hasNewSegmentMonitors()));
-                getLayoutTranslations(translators, owner.helper->getFileName(), owner.indexParts, translationMode, owner.helper->queryIndexRecordSize(), projectedFormat, expectedFormatCrc);
+                getLayoutTranslations(translators, owner.helper->getFileName(), owner.indexParts, translationMode, expectedFormatCrc, owner.helper->queryIndexRecordSize(), projectedFormatCrc, projectedFormat);
             }
         }
         ~CKeyLocalLookup()

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -744,11 +744,12 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
             unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
             Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
             unsigned expectedFormatCrc = helper->getIndexFormatCrc();
+            unsigned projectedFormatCrc = helper->getProjectedIndexFormatCrc();
             IOutputMetaData *projectedFormat = helper->queryProjectedIndexRecordSize();
 
             RecordTranslationMode translationMode = getTranslationMode(activity);
             const char *fname = helper->getIndexFileName();
-            translator.setown(getTranslators(fname, helper->queryIndexRecordSize(), publishedFormat, projectedFormat, translationMode, expectedFormatCrc, false, publishedFormatCrc));
+            translator.setown(getTranslators(fname, expectedFormatCrc, helper->queryIndexRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false));
             if (translator)
                 keyManager.setLayoutTranslator(&translator->queryTranslator());
         }
@@ -992,12 +993,13 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
                 IPropertyTree &props = part.queryOwner().queryProperties();
                 unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
                 Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
-                unsigned expectedFormatCrc = helper->getIndexFormatCrc();
+                unsigned projectedFormatCrc = helper->getProjectedIndexFormatCrc();
                 IOutputMetaData *projectedFormat = helper->queryProjectedIndexRecordSize();
+                unsigned expectedFormatCrc = helper->getIndexFormatCrc();
 
                 RecordTranslationMode translationMode = getTranslationMode(activity);
 
-                Owned<const ITranslator> translator = getTranslators(fname, helper->queryIndexRecordSize(), publishedFormat, projectedFormat, translationMode, expectedFormatCrc, false, publishedFormatCrc);
+                Owned<const ITranslator> translator = getTranslators(fname, expectedFormatCrc, helper->queryIndexRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
                 if (translator)
                 {
                     if (!publishedFormat->queryTypeInfo()->canSerialize() || !projectedFormat->queryTypeInfo()->canSerialize())
@@ -1271,12 +1273,13 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
                 IPropertyTree &props = part.queryOwner().queryProperties();
                 unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
                 Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
-                unsigned expectedFormatCrc = helper->getDiskFormatCrc();
+                unsigned projectedFormatCrc = helper->getProjectedFormatCrc();
                 IOutputMetaData *projectedFormat = helper->queryProjectedDiskRecordSize();
+                unsigned expectedFormatCrc = helper->getDiskFormatCrc();
 
                 RecordTranslationMode translationMode = getTranslationMode(activity);
 
-                Owned<const ITranslator> translator = getTranslators(fname, helper->queryDiskRecordSize(), publishedFormat, projectedFormat, translationMode, expectedFormatCrc, false, publishedFormatCrc);
+                Owned<const ITranslator> translator = getTranslators(fname, expectedFormatCrc, helper->queryDiskRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
                 if (translator)
                 {
                     if (!publishedFormat->queryTypeInfo()->canSerialize() || !projectedFormat->queryTypeInfo()->canSerialize())
@@ -1620,16 +1623,17 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
 
             partIO.stream = createFileSerialStream(partIO.iFileIO, 0, (offset_t)-1, 0);
 
+            unsigned expectedFormatCrc = helper->getDiskFormatCrc();
             IOutputMetaData *expectedFormat = helper->queryDiskRecordSize();
             // NB: potentially translation per part could be different if dealing with superkeys
             IPropertyTree &props = part.queryOwner().queryProperties();
             unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
             Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
-            unsigned expectedFormatCrc = helper->getDiskFormatCrc();
+            unsigned projectedFormatCrc = helper->getProjectedFormatCrc();
             IOutputMetaData *projectedFormat = helper->queryProjectedDiskRecordSize();
             RecordTranslationMode translationMode = getTranslationMode(*this);
             const char *fname = helper->getFileName();
-            partIO.translator = getTranslators(fname, expectedFormat, publishedFormat, projectedFormat, translationMode, expectedFormatCrc, false, publishedFormatCrc);
+            partIO.translator = getTranslators(fname, expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
             if (partIO.translator)
             {
                 partIO.prefetcher = partIO.translator->queryActualFormat().createDiskPrefetcher();

thorlcr/master/thactivitymaster.cpp

@@ -611,8 +611,8 @@ void checkSuperFileOwnership(IDistributedFile &file)
     }
 }
 
-void checkFormatCrc(CActivityBase *activity, IDistributedFile *file, unsigned expectedFormatCrc, unsigned projectedFormatCrc,
-                               IOutputMetaData *projected, IOutputMetaData *expected, bool index)
+void checkFormatCrc(CActivityBase *activity, IDistributedFile *file, unsigned expectedFormatCrc, IOutputMetaData *expected,
+                               unsigned projectedFormatCrc, IOutputMetaData *projected, bool index)
 {
     IDistributedFile *f = file;
     IDistributedSuperFile *super = f->querySuperFile();
@@ -644,7 +644,7 @@ void checkFormatCrc(CActivityBase *activity, IDistributedFile *file, unsigned ex
                 const char *subname = f->queryLogicalName();
                 translator.clear();
                 keyedTranslator.clear();
-                getTranslators(translator, keyedTranslator, subname, expected, actualFormat, projected, mode, expectedFormatCrc, false, dfsCrc);
+                getTranslators(translator, keyedTranslator, subname, expectedFormatCrc, expected, dfsCrc, actualFormat, projectedFormatCrc, projected, mode, false);
             }
         }
         prevFormatCrc = dfsCrc;

thorlcr/master/thactivitymaster.ipp

@@ -34,7 +34,6 @@
 WUFileKind getDiskOutputKind(unsigned flags);
 
 void updateActivityResult(IConstWorkUnit &workunit, unsigned helperFlags, unsigned sequence, const char *logicalFilename, unsigned __int64 recordCount);
-void checkFormatCrc(CActivityBase *activity, IDistributedFile *file, unsigned expectedFormatCrc, unsigned projectedFormatCrc,
-                               IOutputMetaData *projected, IOutputMetaData *expected, bool index);
+void checkFormatCrc(CActivityBase *activity, IDistributedFile *file, unsigned expectedFormatCrc, IOutputMetaData *expected, unsigned projectedFormatCrc, IOutputMetaData *projected, bool index);
 
 #endif

thorlcr/thorutil/thormisc.cpp

@@ -1431,7 +1431,7 @@ RecordTranslationMode getTranslationMode(CActivityBase &activity)
     return getTranslationMode(val);
 }
 
-void getLayoutTranslations(IConstPointerArrayOf<ITranslator> &translators, const char *fname, IArrayOf<IPartDescriptor> &partDescriptors, RecordTranslationMode translationMode, IOutputMetaData *expectedFormat, IOutputMetaData *projectedFormat, unsigned expectedFormatCrc)
+void getLayoutTranslations(IConstPointerArrayOf<ITranslator> &translators, const char *fname, IArrayOf<IPartDescriptor> &partDescriptors, RecordTranslationMode translationMode, unsigned expectedFormatCrc, IOutputMetaData *expectedFormat, unsigned projectedFormatCrc, IOutputMetaData *projectedFormat)
 {
     if (0 == partDescriptors.ordinality())
         return;
@@ -1452,7 +1452,7 @@ void getLayoutTranslations(IConstPointerArrayOf<ITranslator> &translators, const
         if (!translatorContainer)
         {
             Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
-            translatorContainer.setown(getTranslators(fname, expectedFormat, publishedFormat, projectedFormat, translationMode, expectedFormatCrc, false, publishedFormatCrc));
+            translatorContainer.setown(getTranslators(fname, expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false));
             if (translatorContainer)
                 translatorTable.replace(*new CITranslatorMapping(*translatorContainer.getLink(), publishedFormatCrc));
         }
@@ -1460,12 +1460,12 @@ void getLayoutTranslations(IConstPointerArrayOf<ITranslator> &translators, const
     }
 }
 
-const ITranslator *getLayoutTranslation(const char *fname, IPartDescriptor &partDesc, RecordTranslationMode translationMode, IOutputMetaData *expectedFormat, IOutputMetaData *projectedFormat, unsigned expectedFormatCrc)
+const ITranslator *getLayoutTranslation(const char *fname, IPartDescriptor &partDesc, RecordTranslationMode translationMode, unsigned expectedFormatCrc, IOutputMetaData *expectedFormat, unsigned projectedFormatCrc, IOutputMetaData *projectedFormat)
 {
     IPropertyTree const &props = partDesc.queryOwner().queryProperties();
     Owned<IOutputMetaData> actualFormat = getDaliLayoutInfo(props);
     unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
-    return getTranslators(fname, expectedFormat, actualFormat, projectedFormat, translationMode, expectedFormatCrc, false, publishedFormatCrc);
+    return getTranslators(fname, expectedFormatCrc, expectedFormat, publishedFormatCrc, actualFormat, projectedFormatCrc, projectedFormat, translationMode, false);
 }
 
 bool isRemoteReadCandidate(const CActivityBase &activity, const RemoteFilename &rfn, StringBuffer &localPath)

thorlcr/thorutil/thormisc.hpp

@@ -521,8 +521,8 @@ extern graph_decl bool isOOMException(IException *e);
 extern graph_decl IThorException *checkAndCreateOOMContextException(CActivityBase *activity, IException *e, const char *msg, rowcount_t numRows, IOutputMetaData *meta, const void *row);
 
 extern graph_decl RecordTranslationMode getTranslationMode(CActivityBase &activity);
-extern graph_decl void getLayoutTranslations(IConstPointerArrayOf<ITranslator> &translators, const char *fname, IArrayOf<IPartDescriptor> &partDescriptors, RecordTranslationMode translationMode, IOutputMetaData *expectedFormat, IOutputMetaData *projectedFormat, unsigned expectedFormatCrc);
-extern graph_decl const ITranslator *getLayoutTranslation(const char *fname, IPartDescriptor &partDesc, RecordTranslationMode translationMode, IOutputMetaData *expectedFormat, IOutputMetaData *projectedFormat, unsigned expectedFormatCrc);
+extern graph_decl void getLayoutTranslations(IConstPointerArrayOf<ITranslator> &translators, const char *fname, IArrayOf<IPartDescriptor> &partDescriptors, RecordTranslationMode translationMode, unsigned expectedFormatCrc, IOutputMetaData *expectedFormat, unsigned projectedFormatCrc, IOutputMetaData *projectedFormat);
+extern graph_decl const ITranslator *getLayoutTranslation(const char *fname, IPartDescriptor &partDesc, RecordTranslationMode translationMode, unsigned expectedFormatCrc, IOutputMetaData *expectedFormat, unsigned projectedFormatCrc, IOutputMetaData *projectedFormat);
 extern graph_decl bool isRemoteReadCandidate(const CActivityBase &activity, const RemoteFilename &rfn, StringBuffer &localPath);
 
 #endif