HPCC-18469 Calculate the minimal set of fields to pass to the disk read transform

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/thorhelper/thorcommon.cpp

@@ -2033,22 +2033,22 @@ extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &pr
     return nullptr;
 }
 
-static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> *keyedTranslator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck)
+static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> *keyedTranslator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode)
 {
     if (expectedCrc)
     {
         IOutputMetaData * sourceFormat = expectedFormat;
         unsigned sourceCrc = expectedCrc;
-        if (!skipFileFormatCrcCheck)
+        if (mode != RecordTranslationMode::AlwaysECL)
         {
-            if (mode != RecordTranslationMode::AlwaysECL)
+            if (publishedFormat)
             {
-                if (publishedFormat)
-                {
-                    sourceFormat = publishedFormat;
-                    sourceCrc = publishedCrc;
-                }
+                sourceFormat = publishedFormat;
+                sourceCrc = publishedCrc;
             }
+
+            if (publishedCrc && expectedCrc && (publishedCrc != expectedCrc) && (RecordTranslationMode::None == mode))
+                throwTranslationError(publishedFormat->queryRecordAccessor(true), expectedFormat->queryRecordAccessor(true), tracing);
         }
 
         //This has a very low possibility of format crcs accidentally matching, which could lead to a crashes on an untranslated files.
@@ -2061,10 +2061,6 @@ static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<con
 
             if (translator->needsTranslate())
             {
-                if ((RecordTranslationMode::None == mode) && translator->needsNonVirtualTranslate())
-                {
-                    throw MakeStringException(0, "Translatable record layout mismatch detected for file %s, but translation disabled", tracing);
-                }
                 if (keyedTranslator && (sourceFormat != expectedFormat))
                 {
                     Owned<const IKeyTranslator> _keyedTranslator = createKeyTranslator(sourceFormat->queryRecordAccessor(true), expectedFormat->queryRecordAccessor(true));
@@ -2079,21 +2075,21 @@ static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<con
     return nullptr != translator.get();
 }
 
-bool getTranslators(Owned<const IDynamicTransform> &translator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck)
+bool getTranslators(Owned<const IDynamicTransform> &translator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode)
 {
-    return getTranslators(translator, nullptr, tracing, expectedCrc, expectedFormat, publishedCrc, publishedFormat, projectedCrc, projectedFormat, mode, skipFileFormatCrcCheck);
+    return getTranslators(translator, nullptr, tracing, expectedCrc, expectedFormat, publishedCrc, publishedFormat, projectedCrc, projectedFormat, mode);
 }
 
-bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> &keyedTranslator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck)
+bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> &keyedTranslator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode)
 {
-    return getTranslators(translator, &keyedTranslator, tracing, expectedCrc, expectedFormat, publishedCrc, publishedFormat, projectedCrc, projectedFormat, mode, skipFileFormatCrcCheck);
+    return getTranslators(translator, &keyedTranslator, tracing, expectedCrc, expectedFormat, publishedCrc, publishedFormat, projectedCrc, projectedFormat, mode);
 }
 
-ITranslator *getTranslators(const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck)
+ITranslator *getTranslators(const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode)
 {
     Owned<const IDynamicTransform> translator;
     Owned<const IKeyTranslator> keyedTranslator;
-    if (getTranslators(translator, &keyedTranslator, tracing, expectedCrc, expectedFormat, publishedCrc, publishedFormat, projectedCrc, projectedFormat, mode, skipFileFormatCrcCheck))
+    if (getTranslators(translator, &keyedTranslator, tracing, expectedCrc, expectedFormat, publishedCrc, publishedFormat, projectedCrc, projectedFormat, mode))
     {
         if (!publishedFormat)
             publishedFormat = expectedFormat;

common/thorhelper/thorcommon.hpp

@@ -638,10 +638,10 @@ extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &pr
  * providing translation mode and crc's allow translation. Returns true if translator created.
  * NB: translator and keyedTranslator are expected to be empty before calling.
 */
-extern THORHELPER_API bool getTranslators(Owned<const IDynamicTransform> &translator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck);
+extern THORHELPER_API bool getTranslators(Owned<const IDynamicTransform> &translator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode);
 // Same as above, but will also return a key field translator in 2nd parameter. Returns true if translator created.
-extern THORHELPER_API bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> &keyedTranslator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck);
+extern THORHELPER_API bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<const IKeyTranslator> &keyedTranslator, const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode);
 // Returns a ITranslator that gives access to a dynamic translator, keyed translator and the format used
-extern THORHELPER_API ITranslator *getTranslators(const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode, bool skipFileFormatCrcCheck);
+extern THORHELPER_API ITranslator *getTranslators(const char *tracing, unsigned expectedCrc, IOutputMetaData *expectedFormat, unsigned publishedCrc, IOutputMetaData *publishedFormat, unsigned projectedCrc, IOutputMetaData *projectedFormat, RecordTranslationMode mode);
 
 #endif // THORHELPER_HPP

common/thorhelper/thorfile.cpp

@@ -78,7 +78,7 @@ public:
     }
     virtual unsigned getFlags() override
     {
-        return TDRnocrccheck;
+        return 0;
     }
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * src) override
     {

ecl/eclagent/agentctx.hpp

@@ -113,7 +113,7 @@ struct IAgentContext : extends IGlobalCodeContext
 
     virtual IGroup *getHThorGroup(StringBuffer &grpnameout) = 0;
 
-    virtual RecordTranslationMode rltEnabled() const = 0;
+    virtual RecordTranslationMode getLayoutTranslationMode() const = 0;
     virtual unsigned __int64 queryStopAfter() = 0;
     
     virtual const char *queryWuid() = 0;

ecl/eclagent/eclagent.cpp

@@ -701,14 +701,16 @@ void EclAgent::abort()
         activeGraph->abort();
 }
 
-RecordTranslationMode EclAgent::rltEnabled() const
+RecordTranslationMode EclAgent::getLayoutTranslationMode() const
 {
     IConstWorkUnit *wu = queryWorkUnit();
     SCMStringBuffer val;
-    if(wu->hasDebugValue("layoutTranslationEnabled"))
-        wu->getDebugValue("layoutTranslationEnabled", val);
+    if(wu->hasDebugValue("layoutTranslation"))
+        wu->getDebugValue("layoutTranslation", val);
     else
-        wu->getDebugValue("hthorLayoutTranslationEnabled", val);
+    {
+        // more should read from the configuration?
+    }
     return getTranslationMode(val.str());
 }
 

ecl/eclagent/eclagent.ipp

@@ -233,9 +233,9 @@ public:
 
     virtual void updateWULogfile()                  { return ctx->updateWULogfile(); }
 
-    virtual RecordTranslationMode rltEnabled() const override
+    virtual RecordTranslationMode getLayoutTranslationMode() const override
     {
-        return ctx->rltEnabled();
+        return ctx->getLayoutTranslationMode();
     }
 
 protected:
@@ -507,7 +507,7 @@ public:
     virtual IEngineContext *queryEngineContext() { return this; }
     virtual char *getDaliServers();
 
-    virtual RecordTranslationMode rltEnabled() const override;
+    virtual RecordTranslationMode getLayoutTranslationMode() const override;
     unsigned __int64 queryStopAfter() { return stopAfter; }
 
     virtual ISectionTimer * registerTimer(unsigned activityId, const char * name)

ecl/hqlcpp/CMakeLists.txt

@@ -87,6 +87,7 @@ set (    SRCS
          hqlgraph.ipp
          hqlhoist.hpp
          hqlhtcpp.ipp
+         hqliproj.hpp
          hqliproj.ipp
          hqliter.ipp
          hqllib.ipp

ecl/hqlcpp/hqlhtcpp.cpp

@@ -57,6 +57,7 @@
 #include "hqlusage.hpp"
 #include "hqlhoist.hpp"
 #include "hqlcppds.hpp"
+#include "hqliproj.hpp"
 
 //The following are include to ensure they call compile...
 #include "eclhelper.hpp"

ecl/hqlcpp/hqliproj.cpp

@@ -1264,8 +1264,8 @@ void ComplexImplicitProjectInfo::setMatchingOutput(ComplexImplicitProjectInfo *
 //-----------------------------------------------------------------------------------------------
 
 static HqlTransformerInfo implicitProjectTransformerInfo("ImplicitProjectTransformer");
-ImplicitProjectTransformer::ImplicitProjectTransformer(HqlCppTranslator & _translator, bool _optimizeSpills)
-: NewHqlTransformer(implicitProjectTransformerInfo), translator(_translator)
+ImplicitProjectTransformer::ImplicitProjectTransformer(HqlCppTranslator & _translator, bool _optimizeSpills, bool _calculatingMinimumFields)
+: NewHqlTransformer(implicitProjectTransformerInfo), translator(_translator), calculatingMinimumFields(_calculatingMinimumFields)
 {
     const HqlCppOptions & transOptions = translator.queryOptions();
     targetClusterType = translator.getTargetClusterType();
@@ -2046,6 +2046,8 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
             return AnyTypeActivity;
         return NonActivity;
     case no_table:
+        if (calculatingMinimumFields)
+            return CompoundableActivity;
         switch (expr->queryChild(2)->getOperator())
         {
         case no_thor:
@@ -2395,7 +2397,7 @@ void ImplicitProjectTransformer::calculateFieldsUsed(IHqlExpression * expr)
             {
             case no_newusertable:
             case no_hqlproject:
-                if (extra->okToOptimize())
+                if (extra->okToOptimize() && !calculatingMinimumFields)
                     extra->inputs.item(0).stopOptimizeCompound(false);
                 break;
             case no_newaggregate:
@@ -3250,6 +3252,41 @@ IHqlExpression * ImplicitProjectTransformer::process(IHqlExpression * expr)
 }
 
 
+IHqlExpression * ImplicitProjectTransformer::getMinimumInputRecord(IHqlExpression * expr)
+{
+    analyse(expr, 0);           // gather a list of activities, and link them together.
+
+    //Indicate all the fields are required in the output project, and then walk the inputs calculating which fields are required.
+    ComplexImplicitProjectInfo * exprExtra = queryBodyComplexExtra(expr);
+    if (expr->isDataset())
+        exprExtra->addAllOutputs();
+    percolateFields();
+
+    //Search for the first compoundable activity since preserve meta and other oddities may mean it isn't the root table.
+    IHqlExpression * root = queryRoot(expr);
+    assertex(root);
+    IHqlExpression * ds = expr;
+    for (;;)
+    {
+        ComplexImplicitProjectInfo * nextExtra = queryBodyComplexExtra(ds);
+        if (nextExtra->kind == CompoundableActivity)
+            break;
+        ds = ds->queryChild(0);
+    }
+
+    //Create a record that contains all the fields that were required to calculate the expression
+    ComplexImplicitProjectInfo * dsExtra = queryBodyComplexExtra(ds);
+    UsedFieldSet & resultFields = dsExtra->outputFields;
+    resultFields.calcFinalRecord(false, false, false);
+    IHqlExpression * result = resultFields.queryFinalRecord();
+
+    //Avoid creating a reduced record if the output record contains any weird type options.
+    if (!isSensibleRecord(result))
+        result = expr->queryChild(0)->queryRecord();
+
+    return LINK(result);
+}
+
 void ImplicitProjectTransformer::traceActivities()
 {
     ForEachItemIn(i, activities)
@@ -3347,13 +3384,13 @@ IHqlExpression * insertImplicitProjects(HqlCppTranslator & translator, IHqlExpre
 #if defined(POST_COMMON_ANNOTATION)
     HqlExprArray ret;
     {
-        ImplicitProjectTransformer transformer(translator, optimizeSpills);
+        ImplicitProjectTransformer transformer(translator, optimizeSpills, false);
         ret.append(*transformer.process(expr));
     }
     normalizeAnnotations(translator, ret);
     return createActionList(ret);
 #else
-    ImplicitProjectTransformer transformer(translator, optimizeSpills);
+    ImplicitProjectTransformer transformer(translator, optimizeSpills, false);
     return transformer.process(expr);
 #endif
 }
@@ -3390,3 +3427,10 @@ void insertImplicitProjects(HqlCppTranslator & translator, HqlExprArray & exprs)
     would need matchesSelector(list, selector) which worked recursively.  
 
   */
+
+
+IHqlExpression * getMinimumInputRecord(HqlCppTranslator &  translator, IHqlExpression * expr)
+{
+    ImplicitProjectTransformer transformer(translator, false, true);
+    return transformer.getMinimumInputRecord(expr);
+}

ecl/hqlcpp/hqliproj.hpp

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+#ifndef __HQLIPROJ_HPP_
+#define __HQLIPROJ_HPP_
+
+IHqlExpression * insertImplicitProjects(HqlCppTranslator & translator, IHqlExpression * expr, bool optimizeSpills);
+void insertImplicitProjects(HqlCppTranslator & translator, HqlExprArray & exprs);
+
+/* calculate the minimum set of fields that are required as input to this activity */
+IHqlExpression * getMinimumInputRecord(HqlCppTranslator & translator, IHqlExpression * expr);
+
+#endif

ecl/hqlcpp/hqliproj.ipp

@@ -345,8 +345,9 @@ class ImplicitProjectTransformer : public NewHqlTransformer
     typedef NewHqlTransformer Parent;
 
 public:
-    ImplicitProjectTransformer(HqlCppTranslator & _translator, bool _optimizeSpills);
+    ImplicitProjectTransformer(HqlCppTranslator & _translator, bool _optimizeSpills, bool _calculatingMinimumFields);
 
+    IHqlExpression * getMinimumInputRecord(IHqlExpression * expr);
     IHqlExpression * process(IHqlExpression * expr);
 
     virtual void analyseExpr(IHqlExpression * expr);
@@ -396,6 +397,7 @@ protected:
     ClusterType targetClusterType;
     ImplicitProjectOptions options;
     bool allowActivity;
+    bool calculatingMinimumFields;
 };
 
 

ecl/hqlcpp/hqlsource.cpp

@@ -50,6 +50,7 @@
 #include "hqliter.ipp"
 #include "thorcommon.hpp"
 #include "hqlinline.hpp"
+#include "hqliproj.hpp"
 
 //#define FLATTEN_DATASETS
 //#define HACK_TO_IGNORE_TABLE
@@ -834,6 +835,8 @@ public:
     HqlExprAttr     physicalRecord;
     LinkedHqlExpr   expectedRecord;
     LinkedHqlExpr   projectedRecord;
+    LinkedHqlExpr   tableSelector;
+    LinkedHqlExpr   projectedSelector;
     HqlExprAttr     steppedExpr;
     Linked<BuildCtx> globaliterctx;
     HqlExprCopyArray parentCursors;
@@ -1269,7 +1272,7 @@ void SourceBuilder::buildTransformBody(BuildCtx & transformCtx, IHqlExpression *
         {
             if (newDiskReadMapping)
             {
-                translator.bindTableCursor(transformCtx, tableExpr, "left");
+                translator.bindTableCursor(transformCtx, projectedSelector, "left");
             }
             else
             {
@@ -1517,9 +1520,13 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
     case no_hqlproject:
         {
             IHqlExpression * dataset = expr->queryChild(0);
+            IHqlExpression * datasetSelector = dataset->queryNormalizedSelector();
             IHqlExpression * selSeq = querySelSeq(expr);
             OwnedHqlExpr leftSelect = createSelector(no_left, dataset, querySelSeq(expr));
 
+            if ((projectedSelector != tableSelector) && (expr == firstTransformer))
+                datasetSelector = projectedSelector;
+
             //Following is a bit nasty....
             //Converting the no_hqlproject to a no_newusertable means that some of the expressions
             //are commoned up with expressions calculated by a previous filter, reducing the code.
@@ -1527,7 +1534,7 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
             //e.g. project(i(x), transform(exists(i....))) - see jholt25.xhql
             //And unfortunately it fails silently.
             //So we use queryReplaceSelector which fails if an ambiguity is introduced by the replacement
-            OwnedHqlExpr newSelect = ensureActiveRow(dataset->queryNormalizedSelector());
+            OwnedHqlExpr newSelect = ensureActiveRow(datasetSelector);
             OwnedHqlExpr transform = queryNewReplaceSelector(expr->queryChild(1), leftSelect, newSelect);
 
             BuildCtx subctx(ctx);       // buildTargetCursor adds group if necessary
@@ -1537,7 +1544,7 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
             if (!transform)
             {
                 //The replace introduced an ambiguity => need to use the unmapped expression.
-                BoundRow * prevCursor = translator.resolveSelectorDataset(ctx, dataset->queryNormalizedSelector());
+                BoundRow * prevCursor = translator.resolveSelectorDataset(ctx, datasetSelector);
                 transform.set(expr->queryChild(1));
 
                 translator.rebindTableCursor(subctx, dataset, prevCursor, no_left, selSeq);
@@ -1563,7 +1570,13 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
             Linked<BoundRow> tempRow;
             Linked<BoundRow> rowBuilder;
             buildTargetCursor(tempRow, rowBuilder, subctx, expr);
-            IHqlExpression * transform = expr->queryChild(2);
+            LinkedHqlExpr transform = expr->queryChild(2);
+            if (tableExpr && (expr == firstTransformer))
+            {
+                if (tableSelector != projectedSelector)
+                    transform.setown(newReplaceSelector(transform, tableSelector, projectedSelector));
+            }
+
             if (returnIfFilterFails)
             {
                 translator.associateSkipReturnMarker(subctx, failedFilterValue, NULL);      // failedFilterValue already handles clearing the result
@@ -1782,6 +1795,8 @@ void SourceBuilder::buildMatchFilter(BuildCtx & ctx, IHqlExpression * expr)
                 if (translator.options.spotCSE)
                     test.setown(spotScalarCSE(test, ds, translator.queryOptions().spotCseInIfDatasetConditions));
 
+                if (tableSelector != projectedSelector)
+                    test.setown(newReplaceSelector(test, tableSelector, projectedSelector));
                 translator.buildFilteredReturn(ctx, test, queryBoolExpr(false));
             }
         }
@@ -2715,7 +2730,10 @@ void SourceBuilder::buildCanMatch(IHqlExpression * expr)
         MemberFunction func(translator, instance->startctx);
         func.start("virtual bool canMatch(const void * _left) override");
         func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
-        translator.bindTableCursor(func.ctx, tableExpr, "left");
+        if (newDiskReadMapping)
+            translator.bindTableCursor(func.ctx, projectedSelector, "left");
+        else
+            translator.bindTableCursor(func.ctx, tableExpr, "left");
 
         //This will have no ill effect for disk read, and is used for blob lookup
         translator.associateBlobHelper(func.ctx, tableExpr, "fpp");
@@ -2802,6 +2820,8 @@ void SourceBuilder::gatherVirtualFields(bool ignoreVirtuals, bool ensureSerializ
         expectedRecord.set(physicalRecord);
         projectedRecord.set(physicalRecord);
     }
+    tableSelector.set(tableExpr->queryNormalizedSelector());
+    projectedSelector.set(tableSelector);
 }
 
 
@@ -3110,11 +3130,36 @@ public:
         extractCanMatch = (modeOp == no_thor) || (modeOp == no_flat);
     }
 
-    virtual void buildTransform(IHqlExpression * expr);
-    virtual void buildMembers(IHqlExpression * expr);
+protected:
+    virtual void buildTransform(IHqlExpression * expr) override;
+    virtual void buildMembers(IHqlExpression * expr) override;
+    virtual void analyseGraph(IHqlExpression * expr) override;
 };
 
 
+void DiskReadBuilder::analyseGraph(IHqlExpression * expr)
+{
+    DiskReadBuilderBase::analyseGraph(expr);
+    if (newDiskReadMapping && extractCanMatch && firstTransformer)
+    {
+        //Calculate the minimum set of fields required by any post-filters and projects.
+        projectedRecord.setown(getMinimumInputRecord(translator, firstTransformer));
+        if (projectedRecord != firstTransformer->queryChild(0)->queryRecord())
+        {
+            OwnedHqlExpr selSeq = createUniqueSelectorSequence();
+            projectedSelector.setown(createSelector(no_left, projectedRecord, selSeq));
+
+            //Check if projecting the input fields to the minimum now means the transform can be removed.
+            if ((firstTransformer == lastTransformer) && (projectedRecord == firstTransformer->queryRecord()))
+            {
+                if (isSimpleProject(firstTransformer))
+                    needToCallTransform = false;
+            }
+        }
+    }
+}
+
+
 void DiskReadBuilder::buildMembers(IHqlExpression * expr)
 {
     if (modeOp == no_csv)

ecl/hqlcpp/hqlttcpp.cpp

@@ -41,6 +41,7 @@
 #include "hqlerror.hpp"
 #include "hqlalias.hpp"
 #include "hqlir.hpp"
+#include "hqliproj.hpp"
 
 #define TraceExprPrintLog(x, expr) TOSTRLOG(MCdebugInfo(300), unknownJob, x, (expr)->toString);
 //Following are for code that currently cause problems, but are probably a good idea

ecl/hqlcpp/hqlttcpp.ipp

@@ -1298,8 +1298,6 @@ void migrateExprToNaturalLevel(WorkflowItem & curWorkflow, IWorkUnit * wu, HqlCp
 void removeTrivialGraphs(WorkflowItem & curWorkflow);
 void extractWorkflow(HqlCppTranslator & translator, HqlExprArray & exprs, WorkflowArray & out);
 void optimizeActivities(unsigned wfid, HqlExprArray & exprs, bool optimizeCountCompare, bool optimizeNonEmpty);
-IHqlExpression * insertImplicitProjects(HqlCppTranslator & translator, IHqlExpression * expr, bool optimizeSpills);
-void insertImplicitProjects(HqlCppTranslator & translator, HqlExprArray & exprs);
 
 bool reportSemanticErrors(IHqlExpression * expr, IErrorReceiver & errors);
 

ecl/hthor/hthor.cpp

@@ -560,7 +560,7 @@ void CHThorDiskWriteActivity::open()
     unsigned rwFlags = rw_autoflush;
     if (grouped)
         rwFlags |= rw_grouped;
-    if (!(helper.getFlags() & TDRnocrccheck))
+    if (true) // MORE: Should this be controlled by an activity hint/flag?
         rwFlags |= rw_crc;
     IExtRowWriter * writer = createRowWriter(diskout, rowIf, rwFlags);
     outSeq.setown(writer);
@@ -8102,7 +8102,7 @@ void CHThorDiskReadBaseActivity::resolve()
                 }
                 if((helper.getFlags() & (TDXtemporary | TDXjobtemp)) == 0)
                     agent.logFileAccess(dFile, "HThor", "READ");
-                if(agent.rltEnabled()==RecordTranslationMode::None && !agent.queryWorkUnit()->getDebugValueBool("skipFileFormatCrcCheck", false) && !(helper.getFlags() & TDRnocrccheck))
+                if(agent.getLayoutTranslationMode()==RecordTranslationMode::None)
                     verifyRecordFormatCrc();
             }
         }
@@ -8212,7 +8212,7 @@ bool CHThorDiskReadBaseActivity::openNext()
     localOffset = 0;
     saveOpenExc.clear();
     actualFilter.clear();
-    unsigned diskCrc = helper.getDiskFormatCrc();
+    unsigned expectedCrc = helper.getDiskFormatCrc();
     unsigned projectedCrc = helper.getProjectedFormatCrc();
 
     if (dfsParts||ldFile)
@@ -8321,21 +8321,20 @@ bool CHThorDiskReadBaseActivity::openNext()
                 closepart();
             }
 
+            //Check if the file requires translation, but translation is disabled
+            if (actualCrc && expectedCrc && (actualCrc != expectedCrc) && (agent.getLayoutTranslationMode()==RecordTranslationMode::None))
+            {
+                IOutputMetaData * expectedDiskMeta = helper.queryDiskRecordSize();
+                throwTranslationError(actualDiskMeta->queryRecordAccessor(true), expectedDiskMeta->queryRecordAccessor(true), logicalFileName.str());
+            }
+
+            //The projected format will often not match the expected format, and if it differs it must be translated
             if (projectedCrc && actualCrc != projectedCrc)
                 translator.setown(createRecordTranslator(projectedDiskMeta->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true)));
+
             if (translator && translator->needsTranslate())
             {
-                if (translator->canTranslate())
-                {
-                    if (translator->needsNonVirtualTranslate() && agent.rltEnabled()==RecordTranslationMode::None)
-                    {
-#ifdef _DEBUG
-                        translator->describe();
-#endif
-                        throw MakeStringException(0, "Translatable key layout mismatch reading file %s but translation disabled", logicalFileName.str());
-                    }
-                }
-                else
+                if (!translator->canTranslate())
                     throw MakeStringException(0, "Untranslatable key layout mismatch reading file %s", logicalFileName.str());
             }
             else
@@ -8389,6 +8388,23 @@ bool CHThorDiskReadBaseActivity::openNext()
             else
                 saveOpenExc.setown(E);
         }
+
+        //A spill file - actual will always equal expected, so keyed translator will never be used.
+        keyedTranslator.clear();
+
+        //The projected format will often not match the expected format, and if it differs it must be translated
+        translator.clear();
+        if (projectedCrc != expectedCrc)
+            translator.setown(createRecordTranslator(projectedDiskMeta->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true)));
+
+        if (translator && translator->needsTranslate())
+        {
+            assertex(translator->canTranslate());
+        }
+        else
+        {
+            translator.clear();
+        }
         partNum++;
         if (checkOpenedFile(file.str(), NULL))
         {

ecl/hthor/hthor.ipp

@@ -21,8 +21,7 @@
    hthorDiskWriteSizeLimit
    hthorSpillThreshold
    outputLimit (for pipe)
-   skipFileFormatCrcCheck
-   layoutTranslationEnabled and hthorLayoutTranslationEnabled (former takes priority)
+   layoutTranslation
    hthorMemoryLimit
  */
 
@@ -171,7 +170,7 @@ static bool verifyFormatCrc(unsigned helperCrc, IDistributedFile * df, char cons
 
 static bool verifyFormatCrcSuper(unsigned helperCrc, IDistributedFile * df, bool isIndex, bool fail)
 {
-    if(!df) return true;
+    if(!df || (helperCrc == 0)) return true;
     IDistributedSuperFile * super = df->querySuperFile();
     if(super)
     {

ecl/hthor/hthorkey.cpp

@@ -674,10 +674,10 @@ void CHThorIndexReadActivityBase::getLayoutTranslators()
 
 const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDistributedFile * f)
 {
-    if(agent.queryWorkUnit()->getDebugValueBool("skipFileFormatCrcCheck", false))
+    if(agent.getLayoutTranslationMode() == RecordTranslationMode::AlwaysECL)
         return NULL;
 
-    if(agent.rltEnabled() == RecordTranslationMode::None)
+    if(agent.getLayoutTranslationMode() == RecordTranslationMode::None)
     {
         verifyFormatCrc(helper.getDiskFormatCrc(), f, (superIterator ? superName.str() : NULL) , true, true);
         return NULL;
@@ -2442,10 +2442,9 @@ protected:
     {
         actualDiskMeta.set(helper.queryDiskRecordSize());
         translator.clear();
-        if (agent.rltEnabled()==RecordTranslationMode::None)
+        if (agent.getLayoutTranslationMode()==RecordTranslationMode::None)
         {
-            if(!agent.queryWorkUnit()->getDebugValueBool("skipFileFormatCrcCheck", false))
-                ::verifyFormatCrcSuper(helper.getDiskFormatCrc(), f, false, true);
+            ::verifyFormatCrcSuper(helper.getDiskFormatCrc(), f, false, true);
         }
         else
         {
@@ -2459,7 +2458,7 @@ protected:
                     translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true)));
                     if (translator->canTranslate())
                     {
-                        if (agent.rltEnabled()==RecordTranslationMode::None)
+                        if (agent.getLayoutTranslationMode()==RecordTranslationMode::None)
                             throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
 #ifdef _DEBUG
                         translator->describe();
@@ -4058,12 +4057,12 @@ public:
 protected:
     virtual const IDynamicTransform * getLayoutTranslator(IDistributedFile * f) override
     {
-        if(agent.queryWorkUnit()->getDebugValueBool("skipFileFormatCrcCheck", false))
+        if(agent.getLayoutTranslationMode() == RecordTranslationMode::AlwaysECL)
         {
             return NULL;
         }
 
-        if(agent.rltEnabled() == RecordTranslationMode::None)
+        if(agent.getLayoutTranslationMode() == RecordTranslationMode::None)
         {
             verifyFormatCrc(helper.getIndexFormatCrc(), f, super ? super->queryLogicalName() : NULL, true, true);
             return NULL;
@@ -4111,10 +4110,9 @@ protected:
     {
         actualDiskMeta.set(helper.queryDiskRecordSize());
         translator.clear();
-        if (agent.rltEnabled()==RecordTranslationMode::None)
+        if (agent.getLayoutTranslationMode()==RecordTranslationMode::None)
         {
-            if(!agent.queryWorkUnit()->getDebugValueBool("skipFileFormatCrcCheck", false))
-                ::verifyFormatCrcSuper(helper.getDiskFormatCrc(), f, false, true);
+            ::verifyFormatCrcSuper(helper.getDiskFormatCrc(), f, false, true);
         }
         else
         {
@@ -4128,7 +4126,7 @@ protected:
                     translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true)));
                     if (translator->canTranslate())
                     {
-                        if (agent.rltEnabled()==RecordTranslationMode::None)
+                        if (agent.getLayoutTranslationMode()==RecordTranslationMode::None)
                             throw MakeStringException(0, "Translatable file layout mismatch reading file %s but translation disabled", f->queryLogicalName());
                     }
                     else

ecl/regress/serial3b.ecl

@@ -17,7 +17,7 @@
 
 //Test reading a file containg a child and grand-child *dictionary* with a dataset definition
 
-#option ('skipFileFormatCrcCheck', true);
+#option ('layoutTranslation', 'ecl');
 #option ('pickBestEngine', false);
 
 IMPORT SerialTest;

ecl/regress/serial3c.ecl

@@ -17,7 +17,6 @@
 
 //Test reading a file containg a child and grand-child *dataset* with a dictionary definition
 
-#option ('skipFileFormatCrcCheck', true);
 #option ('pickBestEngine', false);
 
 IMPORT SerialTest;

roxie/ccd/ccdactivities.cpp

@@ -901,7 +901,7 @@ public:
         unsigned channel = packet->queryHeader().channel;
         unsigned projectedCrc = helper->getProjectedFormatCrc();
         unsigned expectedCrc = helper->getDiskFormatCrc();
-        translators.setown(varFileInfo->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), basefactory->skipFileFormatCrcCheck(), basefactory->getEnableFieldTranslation(), false));
+        translators.setown(varFileInfo->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation(), false));
         manager.setown(varFileInfo->getIndexManager(isOpt, channel, translators->queryActualLayout(0), false));
     }
 
@@ -999,7 +999,7 @@ public:
                 unsigned channel = queryFactory.queryChannel();
                 unsigned projectedFormatCrc = helper->getProjectedFormatCrc();
                 unsigned expectedFormatCrc = helper->getDiskFormatCrc();
-                translators.setown(datafile->getTranslators(projectedFormatCrc, helper->queryProjectedDiskRecordSize(), expectedFormatCrc, helper->queryDiskRecordSize(), skipFileFormatCrcCheck(), getEnableFieldTranslation(), false));
+                translators.setown(datafile->getTranslators(projectedFormatCrc, helper->queryProjectedDiskRecordSize(), expectedFormatCrc, helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                 manager.setown(datafile->getIndexManager(isOpt, channel, translators->queryActualLayout(0), _graphNode.getPropBool("att[@name=\"preload\"]/@value", false)));
                 Owned<IPropertyTreeIterator> memKeyInfo = queryFactory.queryPackage().getInMemoryIndexInfo(_graphNode);
                 Owned<IPropertyTree> memKeyHint;
@@ -2415,7 +2415,7 @@ public:
             datafile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
             if (datafile)
             {
-                translators.setown(datafile->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, skipFileFormatCrcCheck(), queryFactory.queryOptions().enableFieldTranslation, true));
+                translators.setown(datafile->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, queryFactory.queryOptions().enableFieldTranslation, true));
                 keyArray.setown(datafile->getKeyArray(isOpt, queryFactory.queryChannel()));
             }
         }
@@ -2479,7 +2479,7 @@ protected:
     virtual void setVariableFileInfo()
     {
         const CRoxieKeyedActivityFactory &aFactory = *static_cast<const CRoxieKeyedActivityFactory *>(basefactory);
-        translators.setown(varFileInfo->getTranslators(aFactory.projectedCrc, aFactory.projectedMeta, aFactory.expectedCrc, aFactory.expectedMeta, false, allowFieldTranslation, true));
+        translators.setown(varFileInfo->getTranslators(aFactory.projectedCrc, aFactory.projectedMeta, aFactory.expectedCrc, aFactory.expectedMeta, allowFieldTranslation, true));
         keyArray.setown(varFileInfo->getKeyArray(isOpt, packet->queryHeader().channel));
     }
 
@@ -3591,7 +3591,7 @@ public:
             {
                 unsigned expectedCrc = helper->getDiskFormatCrc();
                 unsigned projectedCrc = helper->getProjectedFormatCrc();
-                translators.setown(datafile->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), skipFileFormatCrcCheck(), getEnableFieldTranslation(), false));
+                translators.setown(datafile->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                 fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
             }
         }
@@ -3648,7 +3648,7 @@ public:
     {
         unsigned expectedCrc = helper->getDiskFormatCrc();
         unsigned projectedCrc = helper->getProjectedFormatCrc();
-        translators.setown(varFileInfo->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), basefactory->skipFileFormatCrcCheck(), basefactory->getEnableFieldTranslation(), false));
+        translators.setown(varFileInfo->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation(), false));
         files.setown(varFileInfo->getIFileIOArray(isOpt, packet->queryHeader().channel));
     }
 
@@ -3986,7 +3986,7 @@ public:
             datafile.setown(_queryFactory.queryPackage().lookupFileName(indexFileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
             if (datafile)
             {
-                translators.setown(datafile->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, skipFileFormatCrcCheck(), queryFactory.queryOptions().enableFieldTranslation, true));
+                translators.setown(datafile->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, queryFactory.queryOptions().enableFieldTranslation, true));
                 keyArray.setown(datafile->getKeyArray(isOpt, queryFactory.queryChannel()));
             }
         }
@@ -4326,7 +4326,7 @@ public:
             {
                 unsigned expectedCrc = helper->getDiskFormatCrc();
                 unsigned projectedCrc = helper->getProjectedFormatCrc();
-                translators.setown(datafile->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), skipFileFormatCrcCheck(), getEnableFieldTranslation(), false));
+                translators.setown(datafile->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                 files.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
             }
         }
@@ -4393,7 +4393,7 @@ public:
     {
         unsigned expectedCrc = helper->getDiskFormatCrc();
         unsigned projectedCrc = helper->getProjectedFormatCrc();
-        translators.setown(varFileInfo->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), basefactory->skipFileFormatCrcCheck(), basefactory->getEnableFieldTranslation(), false));
+        translators.setown(varFileInfo->getTranslators(projectedCrc, helper->queryProjectedDiskRecordSize(), expectedCrc, helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation(), false));
         files.setown(varFileInfo->getIFileIOArray(isOpt, packet->queryHeader().channel));
     }
 

roxie/ccd/ccdfile.cpp

@@ -2076,7 +2076,7 @@ public:
         else
             mb.append(false);
     }
-    virtual ITranslatorSet *getTranslators(int projectedFormatCrc, IOutputMetaData *projected, int expectedFormatCrc, IOutputMetaData *expected, bool skipFileFormatCrcCheck, RecordTranslationMode mode, bool isIndex) const override
+    virtual ITranslatorSet *getTranslators(int projectedFormatCrc, IOutputMetaData *projected, int expectedFormatCrc, IOutputMetaData *expected, RecordTranslationMode mode, bool isIndex) const override
     {
         // NOTE - projected and expected and anything fetched from them such as type info may reside in dynamically loaded (and unloaded)
         // query DLLs - this means it is not safe to include them in any sort of cache that might outlive the current query.
@@ -2093,22 +2093,25 @@ public:
                 const char *subname = subNames.item(idx);
                 int thisFormatCrc = 0;
                 bool actualUnknown = true;
-                if (!skipFileFormatCrcCheck)
+                if (mode != RecordTranslationMode::AlwaysECL)
                 {
-                    if (mode != RecordTranslationMode::AlwaysECL)
-                    {
-                        if (diskTypeInfo.item(idx))
-                            actual = diskTypeInfo.item(idx);
-                        else
-                            actualUnknown = false;
-                        thisFormatCrc = formatCrcs.item(idx);
-                    }
+                    if (diskTypeInfo.item(idx))
+                        actual = diskTypeInfo.item(idx);
+                    else
+                        actualUnknown = false;
+                    thisFormatCrc = formatCrcs.item(idx);
                 }
+
                 assertex(actual);
                 if ((thisFormatCrc != prevFormatCrc) || (idx == 0))  // Check if same translation as last subfile
                 {
                     translator.clear();
                     keyedTranslator.clear();
+
+                    //Check if the file requires translation, but translation is disabled
+                    if (thisFormatCrc && expectedFormatCrc && (thisFormatCrc != expectedFormatCrc) && (mode == RecordTranslationMode::None))
+                        throwTranslationError(actual->queryRecordAccessor(true), expected->queryRecordAccessor(true), subname);
+
                     if (thisFormatCrc == expectedFormatCrc && projectedFormatCrc == expectedFormatCrc && (actualUnknown || alwaysTrustFormatCrcs))
                     {
                         if (traceLevel > 5)
@@ -2123,8 +2126,6 @@ public:
                             throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname);
                         else if (translator->needsTranslate())
                         {
-                            if (mode == RecordTranslationMode::None && translator->needsNonVirtualTranslate())
-                                throw MakeStringException(ROXIE_MISMATCH, "Translatable record layout mismatch detected for file %s, but translation disabled", subname);
                             if (isIndex && translator->keyedTranslated())
                                 throw MakeStringException(ROXIE_MISMATCH, "Record layout mismatch detected in keyed fields for file %s", subname);
                             keyedTranslator.setown(createKeyTranslator(actual->queryRecordAccessor(true), expected->queryRecordAccessor(true)));

roxie/ccd/ccdfile.hpp

@@ -90,7 +90,7 @@ interface IResolvedFile : extends ISimpleSuperFileEnquiry
 {
     virtual void serializePartial(MemoryBuffer &mb, unsigned channel, bool localInfoOnly) const = 0;
 
-    virtual ITranslatorSet *getTranslators(int projectedFormatCrc, IOutputMetaData *projected, int expectedFormatCrc, IOutputMetaData *expected, bool skipFileFormatCrcCheck, RecordTranslationMode mode, bool isIndex) const = 0;
+    virtual ITranslatorSet *getTranslators(int projectedFormatCrc, IOutputMetaData *projected, int expectedFormatCrc, IOutputMetaData *expected, RecordTranslationMode mode, bool isIndex) const = 0;
     virtual IFileIOArray *getIFileIOArray(bool isOpt, unsigned channel) const = 0;
     virtual IKeyArray *getKeyArray(bool isOpt, unsigned channel) const = 0;
     virtual IFilePartMap *getFileMap() const = 0;

roxie/ccd/ccdquery.cpp

@@ -295,7 +295,6 @@ QueryOptions::QueryOptions()
     checkingHeap = defaultCheckingHeap;
     disableLocalOptimizations = defaultDisableLocalOptimizations;
     enableFieldTranslation = fieldTranslationEnabled;
-    skipFileFormatCrcCheck = false;
     stripWhitespaceFromStoredDataset = ((ptr_ignoreWhiteSpace & defaultXmlReadFlags) != 0);
     timeActivities = defaultTimeActivities;
     traceEnabled = defaultTraceEnabled;
@@ -326,7 +325,6 @@ QueryOptions::QueryOptions(const QueryOptions &other)
     checkingHeap = other.checkingHeap;
     disableLocalOptimizations = other.disableLocalOptimizations;
     enableFieldTranslation = other.enableFieldTranslation;
-    skipFileFormatCrcCheck = other.skipFileFormatCrcCheck;
     stripWhitespaceFromStoredDataset = other.stripWhitespaceFromStoredDataset;
     timeActivities = other.timeActivities;
     traceEnabled = other.traceEnabled;
@@ -366,8 +364,7 @@ void QueryOptions::setFromWorkUnit(IConstWorkUnit &wu, const IPropertyTree *stat
 
     updateFromWorkUnit(checkingHeap, wu, "checkingHeap");
     updateFromWorkUnit(disableLocalOptimizations, wu, "disableLocalOptimizations");
-    updateFromWorkUnit(enableFieldTranslation, wu, "layoutTranslationEnabled");  // Name is different for compatibility reasons
-    updateFromWorkUnit(skipFileFormatCrcCheck, wu, "skipFileFormatCrcCheck");
+    updateFromWorkUnit(enableFieldTranslation, wu, "layoutTranslation");  // Name is different for compatibility reasons
     updateFromWorkUnit(stripWhitespaceFromStoredDataset, wu, "stripWhitespaceFromStoredDataset");
     updateFromWorkUnit(timeActivities, wu, "timeActivities");
     updateFromWorkUnit(traceEnabled, wu, "traceEnabled");
@@ -426,7 +423,6 @@ void QueryOptions::setFromContext(const IPropertyTree *ctx)
         updateFromContext(checkingHeap, ctx, "@checkingHeap", "_CheckingHeap");
         // Note: disableLocalOptimizations is not permitted at context level (too late)
         // Note: enableFieldTranslation is not permitted at context level (generally too late anyway)
-        // Note: skipFileFormatCrcCheck is not permitted at context level (generally too late anyway)
         updateFromContext(stripWhitespaceFromStoredDataset, ctx, "_StripWhitespaceFromStoredDataset", "@stripWhitespaceFromStoredDataset");
         updateFromContext(timeActivities, ctx, "@timeActivities", "_TimeActivities");
         updateFromContext(traceEnabled, ctx, "@traceEnabled", "_TraceEnabled");

roxie/ccd/ccdquery.hpp

@@ -100,7 +100,6 @@ public:
     bool checkingHeap;
     bool disableLocalOptimizations;
     RecordTranslationMode enableFieldTranslation;
-    bool skipFileFormatCrcCheck;
     bool stripWhitespaceFromStoredDataset;
     bool timeActivities;
     bool allSortsMaySpill;
@@ -269,20 +268,6 @@ public:
         // Default is no additional information
     }
 
-    unsigned getFormatCrc(unsigned helperCrc) const
-    {
-        if (queryFactory.queryOptions().skipFileFormatCrcCheck)
-            return 0;
-        else
-            return helperCrc;
-    }
-
-    unsigned skipFileFormatCrcCheck() const
-    {
-        return queryFactory.queryOptions().skipFileFormatCrcCheck;
-    }
-
-
     RecordTranslationMode getEnableFieldTranslation() const
     {
         return queryFactory.queryOptions().enableFieldTranslation;

roxie/ccd/ccdserver.cpp

@@ -1595,10 +1595,6 @@ public:
     }
 
 protected:
-    bool skipFileFormatCrcCheck() const
-    {
-        return factory->queryQueryFactory().queryOptions().skipFileFormatCrcCheck;
-    }
     RecordTranslationMode getEnableFieldTranslation() const
     {
         return factory->queryQueryFactory().queryOptions().enableFieldTranslation;
@@ -11656,7 +11652,7 @@ public:
         diskout.setown(createBufferedIOStream(io));
         if (extend)
             diskout->seek(0, IFSend);
-        tallycrc = !(helper.getFlags() & TDRnocrccheck) && !blockcompressed;
+        tallycrc = !blockcompressed; // MORE: Should this be controlled by an activity hint/flag?
         Owned<IRowInterfaces> rowIf = createRowInterfaces(input->queryOutputMeta(), activityId, factory->getHeapFlags(), ctx->queryCodeContext());
         rowSerializer.set(rowIf->queryRowSerializer());
         unsigned rwFlags = rw_autoflush;
@@ -21766,7 +21762,7 @@ public:
                     unsigned channel = isLocal ? factory->queryQueryFactory().queryChannel() : 0;
                     unsigned expectedFormatCrc = helper.getDiskFormatCrc();
                     unsigned projectedFormatCrc = helper.getProjectedFormatCrc();
-                    translators.setown(varFileInfo->getTranslators(projectedFormatCrc, helper.queryProjectedDiskRecordSize(), expectedFormatCrc, helper.queryDiskRecordSize(), skipFileFormatCrcCheck(), getEnableFieldTranslation(), false));
+                    translators.setown(varFileInfo->getTranslators(projectedFormatCrc, helper.queryProjectedDiskRecordSize(), expectedFormatCrc, helper.queryDiskRecordSize(), getEnableFieldTranslation(), false));
                     manager.setown(varFileInfo->getIndexManager(isOpt, channel, translators->queryActualLayout(0), false));
                 }
                 assertex(manager != NULL);
@@ -22666,7 +22662,7 @@ public:
                     unsigned channel = isLocal ? queryFactory.queryChannel() : 0;
                     unsigned expectedFormatCrc = helper->getDiskFormatCrc();
                     unsigned projectedFormatCrc = helper->getProjectedFormatCrc();
-                    translators.setown(datafile->getTranslators(projectedFormatCrc, helper->queryProjectedDiskRecordSize(), expectedFormatCrc, helper->queryDiskRecordSize(), skipFileFormatCrcCheck(), getEnableFieldTranslation(), false));
+                    translators.setown(datafile->getTranslators(projectedFormatCrc, helper->queryProjectedDiskRecordSize(), expectedFormatCrc, helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                     manager.setown(datafile->getIndexManager(isOpt, channel, translators->queryActualLayout(0), _graphNode.getPropBool("att[@name=\"preload\"]/@value", false)));
                     const IPropertyTree *options = datafile->queryProperties();
                     if (options)
@@ -22793,7 +22789,7 @@ public:
                 unsigned expectedCrc = indexHelper->getDiskFormatCrc();
                 IOutputMetaData *projectedMeta = indexHelper->queryProjectedDiskRecordSize();
                 IOutputMetaData *expectedMeta = indexHelper->queryDiskRecordSize();
-                translators.setown(indexfile->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, skipFileFormatCrcCheck(), getEnableFieldTranslation(), true));
+                translators.setown(indexfile->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, getEnableFieldTranslation(), true));
                 keySet.setown(indexfile->getKeyArray(isOpt, isLocal ? queryFactory.queryChannel() : 0));
             }
         }
@@ -22884,7 +22880,7 @@ protected:
             unsigned expectedCrc = indexHelper.getDiskFormatCrc();
             IOutputMetaData *projectedMeta = indexHelper.queryProjectedDiskRecordSize();
             IOutputMetaData *expectedMeta = indexHelper.queryDiskRecordSize();
-            translators.setown(varFileInfo->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, skipFileFormatCrcCheck(), getEnableFieldTranslation(), true));
+            translators.setown(varFileInfo->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, getEnableFieldTranslation(), true));
             keySet.setown(varFileInfo->getKeyArray(isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
         }
         variableInfoPending = false;
@@ -23543,7 +23539,7 @@ class CRoxieServerSimpleIndexReadActivity : public CRoxieServerActivity, impleme
         unsigned expectedCrc = indexHelper.getDiskFormatCrc();
         IOutputMetaData *projectedMeta = indexHelper.queryProjectedDiskRecordSize();
         IOutputMetaData *expectedMeta = indexHelper.queryDiskRecordSize();
-        translators.setown(varFileInfo->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, skipFileFormatCrcCheck(), getEnableFieldTranslation(), true));
+        translators.setown(varFileInfo->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, getEnableFieldTranslation(), true));
         keySet.setown(varFileInfo->getKeyArray(isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
         initKeySet();
         variableInfoPending = false;
@@ -25460,7 +25456,7 @@ public:
             {
                 unsigned expectedCrc = helper.getIndexFormatCrc();
                 unsigned projectedCrc = helper.getProjectedIndexFormatCrc();
-                translators.setown(varFileInfo->getTranslators(projectedCrc, helper.queryProjectedIndexRecordSize(), expectedCrc, helper.queryIndexRecordSize(), skipFileFormatCrcCheck(), getEnableFieldTranslation(), true));
+                translators.setown(varFileInfo->getTranslators(projectedCrc, helper.queryProjectedIndexRecordSize(), expectedCrc, helper.queryIndexRecordSize(), getEnableFieldTranslation(), true));
                 keySet.setown(varFileInfo->getKeyArray(false, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
             }
         }
@@ -26314,7 +26310,7 @@ public:
             {
                 unsigned expectedCrc = helper.getIndexFormatCrc();
                 unsigned projectedCrc = helper.getProjectedIndexFormatCrc();
-                translators.setown(varFileInfo->getTranslators(projectedCrc, helper.queryProjectedIndexRecordSize(), expectedCrc, helper.queryIndexRecordSize(), skipFileFormatCrcCheck(), getEnableFieldTranslation(), true));
+                translators.setown(varFileInfo->getTranslators(projectedCrc, helper.queryProjectedIndexRecordSize(), expectedCrc, helper.queryIndexRecordSize(), getEnableFieldTranslation(), true));
                 keySet.setown(varFileInfo->getKeyArray(false, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
             }
         }
@@ -26555,7 +26551,7 @@ public:
             {
                 unsigned expectedCrc = helper->getIndexFormatCrc();
                 unsigned projectedCrc = helper->getProjectedIndexFormatCrc();
-                keyTranslators.setown(indexfile->getTranslators(projectedCrc, helper->queryProjectedIndexRecordSize(), expectedCrc, helper->queryIndexRecordSize(), skipFileFormatCrcCheck(), getEnableFieldTranslation(), true));
+                keyTranslators.setown(indexfile->getTranslators(projectedCrc, helper->queryProjectedIndexRecordSize(), expectedCrc, helper->queryIndexRecordSize(), getEnableFieldTranslation(), true));
                 keySet.setown(indexfile->getKeyArray(isOpt, isLocal ? queryFactory.queryChannel() : 0));
             }
         }
@@ -26581,7 +26577,7 @@ public:
                 {
                     unsigned expectedCrc = helper->getDiskFormatCrc();
                     unsigned projectedCrc = helper->getProjectedFormatCrc();
-                    translators.setown(datafile->getTranslators(expectedCrc, helper->queryProjectedDiskRecordSize(), projectedCrc, helper->queryDiskRecordSize(), skipFileFormatCrcCheck(), getEnableFieldTranslation(), false));
+                    translators.setown(datafile->getTranslators(expectedCrc, helper->queryProjectedDiskRecordSize(), projectedCrc, helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                     files.setown(datafile->getIFileIOArray(isFetchOpt, queryFactory.queryChannel()));
                 }
                 else

rtl/eclrtl/rtldynfield.cpp

@@ -38,9 +38,9 @@ extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val)
 {
     if (!val || strToBool(val) || strieq(val, "payload"))
         return RecordTranslationMode::Payload;
-    else if (strieq(val, "alwaysDisk"))
+    else if (strieq(val, "alwaysDisk") || strieq(val, "disk"))
         return RecordTranslationMode::AlwaysDisk;
-    else if (strieq(val, "alwaysECL"))
+    else if (strieq(val, "alwaysECL") || strieq(val, "ecl"))
         return RecordTranslationMode::AlwaysECL;
     else
         return RecordTranslationMode::None;
@@ -1554,6 +1554,17 @@ extern ECLRTL_API const IDynamicTransform *createRecordTranslator(const RtlRecor
     return new GeneralRecordTranslator(_destRecInfo, _srcRecInfo);
 }
 
+extern ECLRTL_API void throwTranslationError(const RtlRecord & destRecInfo, const RtlRecord & srcRecInfo, const char * filename)
+{
+    Owned<const IDynamicTransform> translator = createRecordTranslator(destRecInfo, srcRecInfo);
+#ifdef _DEBUG
+    translator->describe();
+#endif
+    if (!translator->canTranslate())
+        throw MakeStringException(0, "Untranslatable record layout mismatch detected for: %s", filename);
+    throw MakeStringException(0, "Translatable key layout mismatch reading file %s but translation disabled", filename);
+}
+
 class TranslatedRowStream : public CInterfaceOf<IRowStream>
 {
 public:

rtl/eclrtl/rtldynfield.hpp

@@ -149,6 +149,8 @@ interface IKeyTranslator : public IInterface
 };
 
 extern ECLRTL_API const IDynamicTransform *createRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo);
+extern ECLRTL_API void throwTranslationError(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo, const char * filename);
+
 extern ECLRTL_API const IKeyTranslator *createKeyTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo);
 
 extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(IThorIndexCallback *callback);

rtl/eclrtl/rtlrecord.cpp

@@ -309,6 +309,10 @@ RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : f
             break;
         }
     }
+
+    //Zero length records cause problems (allocation, and indicating if skipped) => force the length to 1 byte so it is possible to tell
+    if (numFields == 0)
+        fixedOffsets[0] = 1;
 }
 
 

rtl/include/eclhelper.hpp

@@ -1095,7 +1095,7 @@ enum
     TDRcountkeyedlimit  = 0x00008000,
     TDRkeyedlimitskips  = 0x00010000,
     TDRlimitskips       = 0x00020000,
-    TDRnocrccheck       = 0x00040000,
+    //unused              0x00040000,
     TDRaggregateexists  = 0x00080000,       // only aggregate is exists()
     TDRgroupmonitors    = 0x00100000,       // are segement monitors created for all group by conditions.
     TDRlimitcreates     = 0x00200000,

testing/regress/ecl/bug12130.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4522, ignore);
 #onwarning (5402, ignore);
 

testing/regress/ecl/bug5236.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4522, ignore);
 #onwarning (5402, ignore);
 

testing/regress/ecl/canmatch.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4523, ignore);
 #onwarning (5402, ignore);
 

testing/regress/ecl/childindex.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4515, ignore);
 #onwarning (4523, ignore);
 #onwarning (5402, ignore);

testing/regress/ecl/countindex.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4515, ignore);
 #onwarning (4523, ignore);
 #onwarning (5402, ignore);

testing/regress/ecl/diskread.ecl

@@ -26,7 +26,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);

testing/regress/ecl/diskread2.ecl

@@ -26,7 +26,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);

testing/regress/ecl/fetch.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4523, ignore);
 #onwarning (5402, ignore);
 

testing/regress/ecl/fullkeyed.ecl

@@ -34,7 +34,7 @@ forceRemoteKeyedFetch := #IFDEFINED(root.forceRemoteKeyedLookup, false);
 
 #option('forceRemoteKeyedLookup', forceRemoteKeyedLookup);
 #option('forceRemoteKeyedFetch', forceRemoteKeyedFetch);
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4522, ignore);
 
 import $.setup;

testing/regress/ecl/indexagg.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (5402, ignore);
 
 import $.setup;

testing/regress/ecl/indexagg_choosen.ecl

@@ -27,7 +27,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (5402, ignore);
 
 import $.setup;

testing/regress/ecl/indexread.ecl

@@ -28,7 +28,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (5402, ignore);
 
 import $.setup;

testing/regress/ecl/indexread2.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4522, ignore);
 #onwarning (4523, ignore);
 #onwarning (5402, ignore);

testing/regress/ecl/indexread3.ecl

@@ -30,7 +30,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4523, ignore);
 #onwarning (4527, ignore);
 #onwarning (4528, ignore);

testing/regress/ecl/indexread4.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (5402, ignore);
 
 import $.setup;

testing/regress/ecl/indexread6.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4523, ignore);
 #onwarning (5402, ignore);
 

testing/regress/ecl/join.ecl

@@ -26,7 +26,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);

testing/regress/ecl/keydiff.ecl

@@ -33,7 +33,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);

testing/regress/ecl/keyed_denormalize.ecl

@@ -37,7 +37,7 @@ forceRemoteKeyedFetch := #IFDEFINED(root.forceRemoteKeyedLookup, false);
 
 #option('forceRemoteKeyedLookup', forceRemoteKeyedLookup);
 #option('forceRemoteKeyedFetch', forceRemoteKeyedFetch);
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4522, ignore);
 #onwarning (4523, ignore);
 #onwarning (5402, ignore);

testing/regress/ecl/keyed_fetch.ecl

@@ -35,7 +35,7 @@ forceRemoteKeyedFetch := #IFDEFINED(root.forceRemoteKeyedLookup, false);
 
 #option('forceRemoteKeyedLookup', forceRemoteKeyedLookup);
 #option('forceRemoteKeyedFetch', forceRemoteKeyedFetch);
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4515, ignore);
 #onwarning (4523, ignore);
 

testing/regress/ecl/keyed_join.ecl

@@ -37,7 +37,7 @@ forceRemoteKeyedFetch := #IFDEFINED(root.forceRemoteKeyedLookup, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4522, ignore);
 #onwarning (5402, ignore);
 

testing/regress/ecl/keyed_join2.ecl

@@ -36,7 +36,7 @@ forceRemoteKeyedFetch := #IFDEFINED(root.forceRemoteKeyedLookup, false);
 
 #option('forceRemoteKeyedLookup', forceRemoteKeyedLookup);
 #option('forceRemoteKeyedFetch', forceRemoteKeyedFetch);
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4522, ignore);
 #onwarning (5402, ignore);
 

testing/regress/ecl/layouttrans.ecl

@@ -26,7 +26,7 @@ version := #IFDEFINED(root.version, 1);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', true);
+#option ('layoutTranslation', true);
 #onwarning (4515, ignore);
 #onwarning (4522, ignore);
 #onwarning (4523, ignore);

testing/regress/ecl/layouttrans_disabled.ecl

@@ -33,7 +33,7 @@ import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);
 
 // this would use RLT, but we have not enabled it, so it should fail
-#option ('layoutTranslationEnabled', false);
+#option ('layoutTranslation', false);
 
 DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Fname,Lname,__filepos},Files.DG_FetchIndex1Name);
 

testing/regress/ecl/partition.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (4523, ignore);
 #onwarning (5402, ignore);
 

testing/regress/ecl/prefetch.ecl

@@ -25,7 +25,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 import setup.TS;

testing/regress/ecl/prefetch2.ecl

@@ -25,7 +25,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 import setup.TS;

testing/regress/ecl/remoteread.ecl

@@ -18,7 +18,7 @@
 //version optRemoteRead=false
 //version optRemoteRead=true
 
-#option('layoutTranslationEnabled', true);
+#option('layoutTranslation', true);
 
 import ^ as root;
 optRemoteRead := #IFDEFINED(root.optRemoteRead, false);

testing/regress/ecl/serial3b.ecl

@@ -19,7 +19,7 @@
 
 import Setup.SerialTest;
 
-#option ('skipFileFormatCrcCheck', true);
+#option ('layoutTranslation', 'ecl');
 
 inDs := DATASET(SerialTest.DictFilename, SerialTest.LibraryDsRec, THOR);
 

testing/regress/ecl/serial3c.ecl

@@ -19,7 +19,7 @@
 
 import Setup.SerialTest;
 
-#option ('skipFileFormatCrcCheck', true);
+#option ('layoutTranslation', 'ecl');
 
 inDs := DATASET(SerialTest.DsFilename, SerialTest.LibraryDictRec, THOR);
 output(TABLE(inDs, { owner, cnt := COUNT(books) }));

testing/regress/ecl/setup/files.ecl

@@ -55,7 +55,7 @@ SHARED STRING _indexPrefix := '~regress::' +
         '::' + EmptyString;
 
 #IF (forceLayoutTranslation != 0)
-  SHARED setLayout := #option('layoutTranslationEnabled', CASE(forceLayoutTranslation,1=>v'alwaysECL',2=>v'alwaysDisk',v''));
+  SHARED setLayout := #option('layoutTranslation', CASE(forceLayoutTranslation,1=>v'alwaysECL',2=>v'alwaysDisk',v''));
   EXPORT filePrefix := WHEN(#IFDEFINED(root.filePrefix, _filePrefix), setLayout);
   EXPORT indexPrefix := WHEN(#IFDEFINED(root.filePrefix, _indexPrefix), setLayout);
 #else      

testing/regress/ecl/spillsort.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 //--- end of version configuration ---
 
 #option ('hthorSpillThreshold',3000);
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);

testing/regress/ecl/stepskip.ecl

@@ -28,7 +28,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);

testing/regress/ecl/tablecount.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (5402, ignore);
 
 import $.setup;

testing/regress/ecl/tablecount2.ecl

@@ -29,7 +29,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 #onwarning (5402, ignore);
 
 import $.setup;

testing/regress/ecl/translate-diskgrouped.ecl

@@ -27,7 +27,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 sq := setup.sq(multiPart);

testing/regress/ecl/translatedisk.ecl

@@ -25,7 +25,7 @@ multiPart := #IFDEFINED(root.multiPart, false);
 
 #onwarning(2036, ignore);
 #onwarning(4522, ignore);
-#option ('layoutTranslationEnabled', true);
+#option ('layoutTranslation', true);
 import $.Setup;
 
 boolean useLocal := false;

testing/regress/ecl/xmlfetch.ecl

@@ -26,7 +26,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
-#option ('layoutTranslationEnabled', useTranslation);
+#option ('layoutTranslation', useTranslation);
 
 import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);

thorlcr/activities/diskread/thdiskread.cpp

@@ -66,8 +66,7 @@ public:
                 }
             }
         }
-        if (0 == (TDRnocrccheck & helper->getFlags()))
-            checkFormatCrc(this, file, helper->getDiskFormatCrc(), helper->queryDiskRecordSize(), helper->getProjectedFormatCrc(), helper->queryProjectedDiskRecordSize(), false);
+        checkFormatCrc(this, file, helper->getDiskFormatCrc(), helper->queryDiskRecordSize(), helper->getProjectedFormatCrc(), helper->queryProjectedDiskRecordSize(), false);
     }
 };
 

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -76,7 +76,7 @@ protected:
         RecordTranslationMode translationMode = getTranslationMode(*this);
         unsigned expectedFormatCrc = helper->getDiskFormatCrc();
         IOutputMetaData *expectedFormat = helper->queryDiskRecordSize();
-        return ::getTranslators("rowstream", expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
+        return ::getTranslators("rowstream", expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode);
     }
 public:
     CDiskReadSlaveActivityRecord(CGraphElementBase *_container, IHThorArg *_helper=NULL) 

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -116,7 +116,7 @@ protected:
         unsigned expectedFormatCrc = helper->getDiskFormatCrc();
         IOutputMetaData *expectedFormat = helper->queryDiskRecordSize();
 
-        Owned<ITranslator> ret = ::getTranslators("rowstream", expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
+        Owned<ITranslator> ret = ::getTranslators("rowstream", expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode);
         if (!ret)
             return nullptr;
         if (!ret->queryTranslator().canTranslate())

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -749,7 +749,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
 
             RecordTranslationMode translationMode = getTranslationMode(activity);
             const char *fname = helper->getIndexFileName();
-            translator.setown(getTranslators(fname, expectedFormatCrc, helper->queryIndexRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false));
+            translator.setown(getTranslators(fname, expectedFormatCrc, helper->queryIndexRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode));
             if (translator)
                 keyManager.setLayoutTranslator(&translator->queryTranslator());
         }
@@ -999,7 +999,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
 
                 RecordTranslationMode translationMode = getTranslationMode(activity);
 
-                Owned<const ITranslator> translator = getTranslators(fname, expectedFormatCrc, helper->queryIndexRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
+                Owned<const ITranslator> translator = getTranslators(fname, expectedFormatCrc, helper->queryIndexRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode);
                 if (translator)
                 {
                     if (!publishedFormat->queryTypeInfo()->canSerialize() || !projectedFormat->queryTypeInfo()->canSerialize())
@@ -1279,7 +1279,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
 
                 RecordTranslationMode translationMode = getTranslationMode(activity);
 
-                Owned<const ITranslator> translator = getTranslators(fname, expectedFormatCrc, helper->queryDiskRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
+                Owned<const ITranslator> translator = getTranslators(fname, expectedFormatCrc, helper->queryDiskRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode);
                 if (translator)
                 {
                     if (!publishedFormat->queryTypeInfo()->canSerialize() || !projectedFormat->queryTypeInfo()->canSerialize())
@@ -1633,7 +1633,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
             IOutputMetaData *projectedFormat = helper->queryProjectedDiskRecordSize();
             RecordTranslationMode translationMode = getTranslationMode(*this);
             const char *fname = helper->getFileName();
-            partIO.translator = getTranslators(fname, expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false);
+            partIO.translator = getTranslators(fname, expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode);
             if (partIO.translator)
             {
                 partIO.prefetcher = partIO.translator->queryActualFormat().createDiskPrefetcher();

thorlcr/master/thactivitymaster.cpp

@@ -644,7 +644,7 @@ void checkFormatCrc(CActivityBase *activity, IDistributedFile *file, unsigned ex
                 const char *subname = f->queryLogicalName();
                 translator.clear();
                 keyedTranslator.clear();
-                getTranslators(translator, keyedTranslator, subname, expectedFormatCrc, expected, dfsCrc, actualFormat, projectedFormatCrc, projected, mode, false);
+                getTranslators(translator, keyedTranslator, subname, expectedFormatCrc, expected, dfsCrc, actualFormat, projectedFormatCrc, projected, mode);
             }
         }
         prevFormatCrc = dfsCrc;

thorlcr/slave/slavmain.cpp

@@ -333,7 +333,12 @@ class CKJService : public CSimpleInterfaceOf<IKJService>, implements IThreaded,
         const IDynamicTransform *queryTranslator(const char *tracing)
         {
             if (RecordTranslationMode::None == translationMode)
+            {
+                //Check if the file requires translation, but translation is disabled
+                if (publishedFormatCrc && expectedFormatCrc && (publishedFormatCrc != expectedFormatCrc))
+                    throwTranslationError(publishedFormat->queryRecordAccessor(true), expectedFormat->queryRecordAccessor(true), tracing);
                 return nullptr;
+            }
             else if (!translator)
             {
                 if (RecordTranslationMode::AlwaysDisk == translationMode)
@@ -347,11 +352,6 @@ class CKJService : public CSimpleInterfaceOf<IKJService>, implements IThreaded,
                     translator.setown(createRecordTranslator(projectedFormat->queryRecordAccessor(true), publishedFormat->queryRecordAccessor(true)));
                     if (!translator->canTranslate())
                         throw MakeStringException(0, "Untranslatable record layout mismatch detected for: %s", tracing);
-                    if (translator->needsTranslate())
-                    {
-                        if (RecordTranslationMode::None == translationMode)
-                            throw MakeStringException(0, "Translatable record layout mismatch detected for file: %s, but translation disabled", tracing);
-                    }
                 }
                 dbgassertex(translator->canTranslate());
             }

thorlcr/thorutil/thormisc.cpp

@@ -1427,7 +1427,7 @@ IThorException *checkAndCreateOOMContextException(CActivityBase *activity, IExce
 RecordTranslationMode getTranslationMode(CActivityBase &activity)
 {
     StringBuffer val;
-    activity.getOpt("layoutTranslationEnabled", val);
+    activity.getOpt("layoutTranslation", val);
     return getTranslationMode(val);
 }
 
@@ -1452,7 +1452,7 @@ void getLayoutTranslations(IConstPointerArrayOf<ITranslator> &translators, const
         if (!translatorContainer)
         {
             Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
-            translatorContainer.setown(getTranslators(fname, expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode, false));
+            translatorContainer.setown(getTranslators(fname, expectedFormatCrc, expectedFormat, publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode));
             if (translatorContainer)
                 translatorTable.replace(*new CITranslatorMapping(*translatorContainer.getLink(), publishedFormatCrc));
         }
@@ -1465,7 +1465,7 @@ const ITranslator *getLayoutTranslation(const char *fname, IPartDescriptor &part
     IPropertyTree const &props = partDesc.queryOwner().queryProperties();
     Owned<IOutputMetaData> actualFormat = getDaliLayoutInfo(props);
     unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
-    return getTranslators(fname, expectedFormatCrc, expectedFormat, publishedFormatCrc, actualFormat, projectedFormatCrc, projectedFormat, translationMode, false);
+    return getTranslators(fname, expectedFormatCrc, expectedFormat, publishedFormatCrc, actualFormat, projectedFormatCrc, projectedFormat, translationMode);
 }
 
 bool isRemoteReadCandidate(const CActivityBase &activity, const RemoteFilename &rfn, StringBuffer &localPath)