Merge pull request #10690 from richardkchapman/disk-translate

HPCC-18466 Roxie to support record translation on disk files

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/remote/sockfile.cpp

@@ -3477,13 +3477,14 @@ static IOutputMetaData *getTypeInfoOutputMetaData(IPropertyTree &actNode, const
 {
     IPropertyTree *inputJson = actNode.queryPropTree(typePropName);
     if (inputJson)
-        return createTypeInfoOutputMetaData(*inputJson);
+        return createTypeInfoOutputMetaData(*inputJson, nullptr);
     else
     {
         StringBuffer binTypePropName(typePropName);
         MemoryBuffer mb;
         actNode.getPropBin(binTypePropName.append("Bin").str(), mb);
-        return createTypeInfoOutputMetaData(mb);
+        bool grouped = actNode.getPropBool(binTypePropName.append("_grouped").str(), false);
+        return createTypeInfoOutputMetaData(mb, grouped, nullptr);
     }
 }
 

common/thorhelper/layouttrans.hpp

@@ -62,7 +62,7 @@ public:
         byte const * * * ptrs;
     };
 
-    typedef enum { NoTranslation = 0, TranslateAll = 1, TranslatePayload = 2 } Mode;
+    typedef enum { NoTranslation = 0, TranslateAll = 1, TranslatePayload = 2, TranslateAlwaysDisk = 3, TranslateAlwaysECL = 4 } Mode;
 
     virtual bool querySuccess() const = 0;
     virtual Failure const & queryFailure() const = 0;

ecl/hql/hqlexpr.cpp

@@ -14067,11 +14067,11 @@ void exportJsonType(StringBuffer &ret, IHqlExpression *table)
     dumpTypeInfo(ret, typeInfo);
 }
 
-void exportBinaryType(MemoryBuffer &ret, IHqlExpression *table)
+bool exportBinaryType(MemoryBuffer &ret, IHqlExpression *table)
 {
     Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
     const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
-    dumpTypeInfo(ret, typeInfo);
+    return dumpTypeInfo(ret, typeInfo);
 }
 
 const RtlTypeInfo *queryRtlType(IRtlFieldTypeDeserializer &deserializer, IHqlExpression *table)

ecl/hql/hqlexpr.hpp

@@ -1834,7 +1834,7 @@ interface IRtlFieldTypeDeserializer;
 
 extern HQL_API void exportData(IPropertyTree *data, IHqlExpression *table, bool flatten=false);
 extern HQL_API void exportJsonType(StringBuffer &ret, IHqlExpression *table);
-extern HQL_API void exportBinaryType(MemoryBuffer &ret, IHqlExpression *table);
+extern HQL_API bool exportBinaryType(MemoryBuffer &ret, IHqlExpression *table);
 extern HQL_API const RtlTypeInfo *queryRtlType(IRtlFieldTypeDeserializer &deserializer, IHqlExpression *table);
 
 extern HQL_API void clearCacheCounts();

ecl/hthor/hthor.cpp

@@ -748,6 +748,12 @@ void CHThorDiskWriteActivity::setFormat(IFileDescriptor * desc)
     const char *recordECL = helper.queryRecordECL();
     if (recordECL && *recordECL)
         desc->queryProperties().setProp("ECL", recordECL);
+    if (helper.queryDiskRecordSize()->queryTypeInfo())
+    {
+        MemoryBuffer out;
+        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
+        desc->queryProperties().setPropBin("_rtlType", out.length(), out.toByteArray());
+    }
     desc->queryProperties().setProp("@kind", "flat");
 }
 
@@ -885,6 +891,12 @@ void CHThorCsvWriteActivity::setFormat(IFileDescriptor * desc)
     const char *recordECL = helper.queryRecordECL();
     if (recordECL && *recordECL)
         desc->queryProperties().setProp("ECL", recordECL);
+    if (helper.queryDiskRecordSize()->queryTypeInfo())
+    {
+        MemoryBuffer out;
+        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
+        desc->queryProperties().setPropBin("_rtlType", out.length(), out.toByteArray());
+    }
 }
 
 //=====================================================================================================
@@ -973,6 +985,12 @@ void CHThorXmlWriteActivity::setFormat(IFileDescriptor * desc)
     const char *recordECL = helper.queryRecordECL();
     if (recordECL && *recordECL)
         desc->queryProperties().setProp("ECL", recordECL);
+    if (helper.queryDiskRecordSize()->queryTypeInfo())
+    {
+        MemoryBuffer out;
+        dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
+        desc->queryProperties().setPropBin("_rtlType", out.length(), out.toByteArray());
+    }
 }
 
 //=====================================================================================================

roxie/ccd/CMakeLists.txt

@@ -70,6 +70,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/roxie/udplib
          ${HPCC_SOURCE_DIR}/roxie/roxie
          ${HPCC_SOURCE_DIR}/common/environment
+         ${HPCC_SOURCE_DIR}/ecl/hql
          ${HPCC_SOURCE_DIR}/ecl/hthor
          ${HPCC_SOURCE_DIR}/ecl/schedulectrl
          ${HPCC_SOURCE_DIR}/rtl/nbcd
@@ -102,7 +103,8 @@ install ( TARGETS ccd RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_
 
 target_link_libraries ( ccd 
          jlib
-         nbcd 
+         nbcd
+         hql
          roxiemem 
          udplib 
          remote 

roxie/ccd/ccd.hpp

@@ -435,6 +435,7 @@ extern unsigned defaultForceNumStrands;
 extern unsigned defaultHeapFlags;
 
 extern bool defaultCheckingHeap;
+extern bool defaultDisableLocalOptimizations;
 
 extern unsigned slaveQueryReleaseDelaySeconds;
 extern unsigned coresPerQuery;

roxie/ccd/ccdactivities.cpp

@@ -33,6 +33,7 @@
 #include "eclrtl_imp.hpp"
 #include "rtlread_imp.hpp"
 #include "rtlcommon.hpp"
+#include "rtldynfield.hpp"
 
 #include "jhtree.hpp"
 #include "jlog.hpp"
@@ -157,11 +158,6 @@ public:
         return ret.appendf("%p", this);
     }
 
-    IRecordLayoutTranslator::Mode getEnableFieldTranslation() const
-    {
-        return queryFactory.queryOptions().enableFieldTranslation;
-    }
-
     const char *queryQueryName() const
     {
         return queryFactory.queryQueryName();
@@ -852,16 +848,18 @@ protected:
     CachedOutputMetaData diskSize;
     Owned<IInMemoryIndexCursor> cursor;
     Linked<IInMemoryIndexManager> manager;
+    Linked<ITranslatorSet> translators;
     Owned<IInMemoryFileProcessor> processor;
-    Owned<IFileIOArray> varFiles;
     CriticalSection pcrit;
 
 public:
     CRoxieDiskReadBaseActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
-        IInMemoryIndexManager *_manager, 
+        IInMemoryIndexManager *_manager,
+        ITranslatorSet *_translators,
         unsigned _parallelPartNo, unsigned _numParallel, bool _forceUnkeyed)
         : CRoxieSlaveActivity(_logctx, _packet, _hFactory, _aFactory),
         manager(_manager),
+        translators(_translators),
         parallelPartNo(_parallelPartNo),
         numParallel(_numParallel),
         forceUnkeyed(_forceUnkeyed)
@@ -869,15 +867,8 @@ public:
         helper = (IHThorDiskReadBaseArg *) basehelper;
         variableFileName = allFilesDynamic || basefactory->queryQueryFactory().isDynamic() || ((helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
         isOpt = (helper->getFlags() & TDRoptional) != 0;
-        Linked<IOutputMetaData> diskMeta(helper->queryDiskRecordSize()->querySerializedDiskMeta());
-        if (diskMeta->isGrouped())
-        {
-            diskMeta.setown(new CSuffixedOutputMeta(+1, diskMeta.getClear()));
-            isGrouped = true;
-        }
-        else
-            isGrouped = false;
-        diskSize.set(diskMeta);
+        diskSize.set(helper->queryProjectedDiskRecordSize()->querySerializedDiskMeta());
+        isGrouped = diskSize.isGrouped();
         processed = 0;
         readPos = 0;
         isKeyed = false;
@@ -888,7 +879,7 @@ public:
             resentInfo.read(usedKey);
             if (usedKey)
             {
-                cursor.setown(manager->createCursor(diskMeta->queryRecordAccessor(true)));
+                cursor.setown(manager->createCursor(diskSize.queryRecordAccessor(true)));
                 cursor->deserializeCursorPos(resentInfo);
                 isKeyed = true;
             }
@@ -913,25 +904,10 @@ public:
 
     virtual void setVariableFileInfo()
     {
-        const IPropertyTree *options =  varFileInfo->queryProperties();
-        if (options)
-        {
-            bool isFileGrouped = options->getPropBool("@grouped");
-            if (isFileGrouped && !isGrouped)
-            {
-                // We are prepared to read contents of a grouped persist ungrouped... But not vice versa
-                WARNLOG("Published group information for file %s does not match coded information - assuming grouped", queryDynamicFileName());
-                Owned<IOutputMetaData> diskMeta(new CSuffixedOutputMeta(+1, LINK(diskSize.queryOriginal())));
-                diskSize.set(diskMeta);
-                isGrouped = true;
-            }
-            size32_t dfsSize = options->getPropInt("@recordSize");
-            if (dfsSize && diskSize.isFixedSize() && dfsSize != diskSize.getFixedSize())
-                throw MakeStringException(ROXIE_LAYOUT_MISMATCH, "Published record size %d for file %s (%s) does not match coded record size %d", dfsSize, queryDynamicFileName(), isGrouped ? "grouped" : "ungrouped", diskSize.getFixedSize());
-        }
         unsigned channel = packet->queryHeader().channel;
-        varFiles.setown(varFileInfo->getIFileIOArray(isOpt, channel)); // MORE could combine 
-        manager.setown(varFileInfo->getIndexManager(isOpt, channel, varFiles, diskSize, false, 0));
+        unsigned formatCrc = basefactory->getFormatCrc(helper->getFormatCrc());
+        translators.setown(varFileInfo->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation())); // MORE - FormatCRC may be wrong here. Needs to be crc of projected not expected
+        manager.setown(varFileInfo->getIndexManager(isOpt, channel, nullptr, false, 0));
     }
 
     inline bool queryKeyed() const
@@ -1030,9 +1006,8 @@ public:
 class CRoxieDiskBaseActivityFactory : public CSlaveActivityFactory
 {
 protected:
-    Owned<IFileIOArray> fileArray;
+    Owned<ITranslatorSet> translators;
     Owned<IInMemoryIndexManager> manager;
-
 public:
     CRoxieDiskBaseActivityFactory(IPropertyTree &_graphNode, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory)
         : CSlaveActivityFactory(_graphNode, _subgraphId, _queryFactory, _helperFactory)
@@ -1047,8 +1022,9 @@ public:
             if (datafile)
             {
                 unsigned channel = queryFactory.queryChannel();
-                fileArray.setown(datafile->getIFileIOArray(isOpt, channel));
-                manager.setown(datafile->getIndexManager(isOpt, channel, fileArray, helper->queryDiskRecordSize(), _graphNode.getPropBool("att[@name=\"preload\"]/@value", false), _graphNode.getPropInt("att[@name=\"_preloadSize\"]/@value", 0)));
+                unsigned formatCrc = getFormatCrc(helper->getFormatCrc());
+                translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
+                manager.setown(datafile->getIndexManager(isOpt, channel, translators->queryActualLayout(0), _graphNode.getPropBool("att[@name=\"preload\"]/@value", false), _graphNode.getPropInt("att[@name=\"_preloadSize\"]/@value", 0)));
                 Owned<IPropertyTreeIterator> memKeyInfo = queryFactory.queryPackage().getInMemoryIndexInfo(_graphNode);
                 if (memKeyInfo)
                 {
@@ -1089,8 +1065,8 @@ protected:
 
 public:
     CRoxieDiskReadActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
-        IInMemoryIndexManager *_manager)
-        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, 0, 1, false)
+        IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, 0, 1, false)
     {
         onCreate();
         helper = (IHThorDiskReadArg *) basehelper;
@@ -1106,7 +1082,9 @@ public:
         {
             CriticalBlock p(pcrit);
             processor.setown(isKeyed ? createKeyedRecordProcessor(cursor, *this, resent) : 
-                                       createUnkeyedRecordProcessor(cursor, *this, diskSize.isVariableSize(), isGrouped, manager->createReader(readPos, parallelPartNo, numParallel)));
+                                       createUnkeyedRecordProcessor(cursor, *this, diskSize.isVariableSize(), isGrouped,
+                                               manager->createReader(readPos, parallelPartNo, numParallel, translators,
+                                                                     queryContext->queryCodeContext(), basefactory->queryId())));
         }
         unsigned __int64 rowLimit = helper->getRowLimit();
         unsigned __int64 stopAfter = helper->getChooseNLimit();
@@ -1134,8 +1112,8 @@ protected:
 
 public:
     CRoxieCsvReadActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
-                          IInMemoryIndexManager *_manager, const IResolvedFile *_datafile, size32_t _maxRowSize)
-        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, 0, 1, true), datafile(_datafile), maxRowSize(_maxRowSize)
+                          IInMemoryIndexManager *_manager, ITranslatorSet *_translators, const IResolvedFile *_datafile, size32_t _maxRowSize)
+        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, 0, 1, true), datafile(_datafile), maxRowSize(_maxRowSize)
     {
         onCreate();
         helper = (IHThorCsvReadArg *) basehelper;
@@ -1152,7 +1130,7 @@ public:
             CriticalBlock p(pcrit);
             processor.setown(
                     createCsvRecordProcessor(*this,
-                                             manager->createReader(readPos, parallelPartNo, numParallel),
+                                             manager->createReader(readPos, parallelPartNo, numParallel, translators, queryContext->queryCodeContext(), basefactory->queryId()),
                                              packet->queryHeader().channel==1 && !resent,
                                              varFileInfo ? varFileInfo.get() : datafile, maxRowSize));
         }
@@ -1179,8 +1157,8 @@ protected:
 
 public:
     CRoxieXmlReadActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
-        IInMemoryIndexManager *_manager)
-        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, 0, 1, true)
+        IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, 0, 1, true)
     {
         onCreate();
         helper = (IHThorXmlReadArg *) basehelper;
@@ -1195,7 +1173,7 @@ public:
     {
         {
             CriticalBlock p(pcrit);
-            processor.setown(createXmlRecordProcessor(*this, manager->createReader(readPos, parallelPartNo, numParallel))); 
+            processor.setown(createXmlRecordProcessor(*this, manager->createReader(readPos, parallelPartNo, numParallel, translators, queryContext->queryCodeContext(), basefactory->queryId())));
         }
         unsigned __int64 rowLimit = helper->getRowLimit();
         unsigned __int64 stopAfter = helper->getChooseNLimit();
@@ -1220,7 +1198,7 @@ public:
 
     virtual IRoxieSlaveActivity *createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
     {
-        return new CRoxieDiskReadActivity(logctx, packet, helperFactory, this, manager);
+        return new CRoxieDiskReadActivity(logctx, packet, helperFactory, this, manager, translators);
     }
 
     virtual StringBuffer &toString(StringBuffer &s) const
@@ -1246,7 +1224,7 @@ public:
 
     virtual IRoxieSlaveActivity *createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
     {
-        return new CRoxieCsvReadActivity(logctx, packet, helperFactory, this, manager, datafile, maxRowSize);
+        return new CRoxieCsvReadActivity(logctx, packet, helperFactory, this, manager, translators, datafile, maxRowSize);
     }
 
     virtual StringBuffer &toString(StringBuffer &s) const
@@ -1266,7 +1244,7 @@ public:
 
     virtual IRoxieSlaveActivity *createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
     {
-        return new CRoxieXmlReadActivity(logctx, packet, helperFactory, this, manager);
+        return new CRoxieXmlReadActivity(logctx, packet, helperFactory, this, manager, translators);
     }
 
     virtual StringBuffer &toString(StringBuffer &s) const
@@ -1475,9 +1453,8 @@ class UnkeyedVariableRecordProcessor : public UnkeyedRecordProcessor
 {
 public:
     UnkeyedVariableRecordProcessor(IInMemoryIndexCursor *_cursor, CRoxieDiskReadActivity &_owner, bool _isGrouped, IDirectReader *_reader)
-      : UnkeyedRecordProcessor(_cursor, _owner, _reader), isGrouped(_isGrouped), deserializeSource(_reader)
+      : UnkeyedRecordProcessor(_cursor, _owner, _reader), isGrouped(_isGrouped)
     {
-        prefetcher.setown(owner.diskSize.queryOriginal()->createDiskPrefetcher(owner.queryContext->queryCodeContext(), owner.basefactory->queryId()));
     }
 
     virtual void doQuery(IMessagePacker *output, unsigned processed, unsigned __int64 rowLimit, unsigned __int64 stopAfter)
@@ -1485,25 +1462,17 @@ public:
         unsigned totalSizeSent = 0;
         helper->setCallback(reader->queryThorDiskCallback());
         unsigned lastGroupProcessed = processed;
-        while (!aborted && !deserializeSource.eos())
+        while (!aborted && !reader->eos())
         {
             // This loop is the inner loop for memory diskreads - so keep it efficient!
-            prefetcher->readAhead(deserializeSource);
-            const byte *nextRec = deserializeSource.queryRow();
+            const byte *nextRec = reader->nextRow();
             size32_t transformedSize;
             if (cursor && cursor->isFiltered(nextRec))
                 transformedSize = 0;
             else
                 transformedSize = owner.doTransform(output, nextRec);
-            bool eog;
-            if (isGrouped)
-            {
-                size32_t sizeRead = deserializeSource.queryRowSize();
-                eog = nextRec[sizeRead-1];
-            }
-            else
-                eog = false;
-            deserializeSource.finishedRow();
+            bool eog = isGrouped && reader->eog();
+            reader->finishedRow();
             if (transformedSize)
             {
                 processed++;
@@ -1522,7 +1491,7 @@ public:
                     si.append(siLen);
                     si.append(processed);
                     si.append(false);  // not using a key
-                    offset_t readPos = deserializeSource.tell();
+                    offset_t readPos = reader->tell();
                     si.append(readPos);
                     siLen = si.length() - sizeof(siLen);
                     si.writeDirect(0, sizeof(siLen), &siLen);
@@ -1538,7 +1507,7 @@ public:
                 si.append(siLen);
                 si.append(processed);
                 si.append(false);  // not using a key
-                offset_t readPos = deserializeSource.tell();
+                offset_t readPos = reader->tell();
                 si.append(readPos);
                 siLen = si.length() - sizeof(siLen);
                 si.writeDirect(0, sizeof(siLen), &siLen);
@@ -1549,14 +1518,12 @@ public:
     }
 
 protected:
-    CThorContiguousRowBuffer deserializeSource;
-    Owned<ISourceRowPrefetcher> prefetcher;
     bool isGrouped;
 };
 
 IInMemoryFileProcessor *createUnkeyedRecordProcessor(IInMemoryIndexCursor *cursor, CRoxieDiskReadActivity &owner, bool variableDisk, bool isGrouped, IDirectReader *_reader)
 {
-    if (variableDisk || isGrouped)
+    if (variableDisk || isGrouped || _reader->isTranslating())
         return new UnkeyedVariableRecordProcessor(cursor, owner, isGrouped, _reader);
     else
         return new UnkeyedRecordProcessor(cursor, owner, _reader);
@@ -1795,8 +1762,8 @@ protected:
 
 public:
     CRoxieDiskNormalizeActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
-        IInMemoryIndexManager *_manager)
-        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, 0, 1, false)
+        IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, 0, 1, false)
     {
         onCreate();
         helper = (IHThorDiskNormalizeArg *) basehelper;
@@ -1813,7 +1780,7 @@ public:
             CriticalBlock p(pcrit);
             processor.setown(isKeyed ? 
                 createKeyedNormalizeRecordProcessor(cursor, *this, resent) : 
-                createUnkeyedNormalizeRecordProcessor(cursor, *this, manager->createReader(readPos, parallelPartNo, numParallel)));
+                createUnkeyedNormalizeRecordProcessor(cursor, *this, manager->createReader(readPos, parallelPartNo, numParallel, translators, queryContext->queryCodeContext(), basefactory->queryId())));
         }
         unsigned __int64 rowLimit = helper->getRowLimit();
         unsigned __int64 stopAfter = helper->getChooseNLimit();
@@ -1839,7 +1806,7 @@ public:
 
     virtual IRoxieSlaveActivity *createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
     {
-        return new CRoxieDiskNormalizeActivity(logctx, packet, helperFactory, this, manager);
+        return new CRoxieDiskNormalizeActivity(logctx, packet, helperFactory, this, manager, translators);
     }
 
     virtual StringBuffer &toString(StringBuffer &s) const
@@ -2035,8 +2002,8 @@ protected:
 
 public:
     CRoxieDiskCountActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
-        IInMemoryIndexManager *_manager)
-        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, 0, 1, false)
+        IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, 0, 1, false)
     {
         onCreate();
         helper = (IHThorDiskCountArg *) basehelper;
@@ -2053,7 +2020,7 @@ public:
             CriticalBlock p(pcrit);
             processor.setown(isKeyed ? 
                 createKeyedCountRecordProcessor(cursor, *this) : 
-                createUnkeyedCountRecordProcessor(cursor, *this, diskSize.isVariableSize(), manager->createReader(readPos, parallelPartNo, numParallel)));
+                createUnkeyedCountRecordProcessor(cursor, *this, diskSize.isVariableSize(), manager->createReader(readPos, parallelPartNo, numParallel, translators, queryContext->queryCodeContext(), basefactory->queryId())));
         }
         unsigned __int64 stopAfter = helper->getChooseNLimit();
         processor->doQuery(output, processed, (unsigned __int64) -1, stopAfter);
@@ -2070,7 +2037,7 @@ public:
 
     virtual IRoxieSlaveActivity *createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
     {
-        return new CRoxieDiskCountActivity(logctx, packet, helperFactory, this, manager);
+        return new CRoxieDiskCountActivity(logctx, packet, helperFactory, this, manager, translators);
     }
 
     virtual StringBuffer &toString(StringBuffer &s) const
@@ -2302,8 +2269,9 @@ protected:
 public:
     CRoxieDiskAggregateActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
         IInMemoryIndexManager *_manager,
+        ITranslatorSet *_translators,
         unsigned _parallelPartNo, unsigned _numParallel, bool _forceUnkeyed)
-        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _parallelPartNo, _numParallel, _forceUnkeyed)
+        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, _parallelPartNo, _numParallel, _forceUnkeyed)
     {
         onCreate();
         helper = (IHThorDiskAggregateArg *) basehelper;
@@ -2324,7 +2292,7 @@ public:
         {
             CriticalBlock p(pcrit);
             processor.setown(isKeyed ? createKeyedAggregateRecordProcessor(cursor, *this) : 
-                                       createUnkeyedAggregateRecordProcessor(cursor, *this, diskSize.isVariableSize(), manager->createReader(readPos, parallelPartNo, numParallel))); 
+                                       createUnkeyedAggregateRecordProcessor(cursor, *this, diskSize.isVariableSize(), manager->createReader(readPos, parallelPartNo, numParallel, translators, queryContext->queryCodeContext(), basefactory->queryId())));
         }
         processor->doQuery(output, 0, 0, 0);
     }
@@ -2434,7 +2402,7 @@ protected:
     OwnedConstRoxieRow finalRow;
 public:
     CParallelRoxieDiskAggregateActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
-        IInMemoryIndexManager *_manager, unsigned _numParallel) :
+        IInMemoryIndexManager *_manager, ITranslatorSet *_translators, unsigned _numParallel) :
         CParallelRoxieActivity(_logctx, _packet, _hFactory, _aFactory, _numParallel)
     {
         helper = (IHThorDiskAggregateArg *) basehelper;
@@ -2444,7 +2412,7 @@ public:
             // MORE - avoiding serializing to dummy would be more efficient...
             deserializer.setown(meta.createDiskDeserializer(queryContext->queryCodeContext(), basefactory->queryId()));
         }
-        CRoxieDiskAggregateActivity *part0 = new CRoxieDiskAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, 0, numParallel, false);
+        CRoxieDiskAggregateActivity *part0 = new CRoxieDiskAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, 0, numParallel, false);
         parts.append(*part0);
         if (part0->queryKeyed())
         {
@@ -2454,7 +2422,7 @@ public:
         else
         {
             for (unsigned i = 1; i < numParallel; i++)
-                parts.append(*new CRoxieDiskAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, i, numParallel, true));
+                parts.append(*new CRoxieDiskAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, i, numParallel, true));
         }
     }
 
@@ -2560,9 +2528,9 @@ public:
     virtual IRoxieSlaveActivity *createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
     {
         if (parallelAggregate > 1)
-            return new CParallelRoxieDiskAggregateActivity(logctx, packet, helperFactory, this, manager, parallelAggregate);
+            return new CParallelRoxieDiskAggregateActivity(logctx, packet, helperFactory, this, manager, translators, parallelAggregate);
         else
-            return new CRoxieDiskAggregateActivity(logctx, packet, helperFactory, this, manager, 0, 1, false);
+            return new CRoxieDiskAggregateActivity(logctx, packet, helperFactory, this, manager, translators, 0, 1, false);
     }
 
     virtual StringBuffer &toString(StringBuffer &s) const
@@ -2779,8 +2747,9 @@ protected:
 public:
     CRoxieDiskGroupAggregateActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
         IInMemoryIndexManager *_manager,
+        ITranslatorSet *_translators,
         unsigned partNo, unsigned numParts, bool _forceUnkeyed)
-        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, partNo, numParts, _forceUnkeyed),
+        : CRoxieDiskReadBaseActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, partNo, numParts, _forceUnkeyed),
           helper((IHThorDiskGroupAggregateArg *) basehelper),
           results(*helper, *helper)
     {
@@ -2804,7 +2773,7 @@ public:
             CriticalBlock p(pcrit);
             processor.setown(isKeyed ? 
                 createKeyedGroupAggregateRecordProcessor(cursor, results, *helper) : 
-                createUnkeyedGroupAggregateRecordProcessor(cursor, results, *helper, manager->createReader(readPos, parallelPartNo, numParallel), 
+                createUnkeyedGroupAggregateRecordProcessor(cursor, results, *helper, manager->createReader(readPos, parallelPartNo, numParallel, translators, queryContext->queryCodeContext(), basefactory->queryId()),
                                                            queryContext->queryCodeContext(), basefactory->queryId()));
         }
         processor->doQuery(output, 0, 0, 0);
@@ -2824,7 +2793,7 @@ protected:
 
 public:
     CParallelRoxieDiskGroupAggregateActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CSlaveActivityFactory *_aFactory,
-        IInMemoryIndexManager *_manager, unsigned _numParallel) :
+        IInMemoryIndexManager *_manager, ITranslatorSet *_translators, unsigned _numParallel) :
         CParallelRoxieActivity(_logctx, _packet, _hFactory, _aFactory, _numParallel),
         helper((IHThorDiskGroupAggregateArg *) basehelper),
         resultAggregator(*helper, *helper)
@@ -2836,7 +2805,7 @@ public:
             // MORE - avoiding serializing to dummy would be more efficient...
             deserializer.setown(meta.createDiskDeserializer(queryContext->queryCodeContext(), basefactory->queryId()));
         }
-        CRoxieDiskGroupAggregateActivity *part0 = new CRoxieDiskGroupAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, 0, numParallel, false);
+        CRoxieDiskGroupAggregateActivity *part0 = new CRoxieDiskGroupAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, 0, numParallel, false);
         parts.append(*part0);
         if (part0->queryKeyed())
         {
@@ -2846,7 +2815,7 @@ public:
         else
         {
             for (unsigned i = 1; i < numParallel; i++)
-                parts.append(*new CRoxieDiskGroupAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, i, numParallel, true));
+                parts.append(*new CRoxieDiskGroupAggregateActivity(_logctx, _packet, _hFactory, _aFactory, _manager, _translators, i, numParallel, true));
         }
     }
 
@@ -2931,9 +2900,9 @@ public:
     virtual IRoxieSlaveActivity *createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
     {
         if (parallelAggregate > 1)
-            return new CParallelRoxieDiskGroupAggregateActivity(logctx, packet, helperFactory, this, manager, parallelAggregate);
+            return new CParallelRoxieDiskGroupAggregateActivity(logctx, packet, helperFactory, this, manager, translators, parallelAggregate);
         else
-            return new CRoxieDiskGroupAggregateActivity(logctx, packet, helperFactory, this, manager, 0, 1, false);
+            return new CRoxieDiskGroupAggregateActivity(logctx, packet, helperFactory, this, manager, translators, 0, 1, false);
     }
 
     virtual StringBuffer &toString(StringBuffer &s) const
@@ -4279,6 +4248,7 @@ ISlaveActivityFactory *createRoxieIndexGroupAggregateActivityFactory(IPropertyTr
 class CRoxieFetchActivityFactory : public CSlaveActivityFactory
 {
 public:
+    Owned<ITranslatorSet> translators; // MORE - use them!
     Owned<IFileIOArray> fileArray;
 
     CRoxieFetchActivityFactory(IPropertyTree &_graphNode, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory)
@@ -4292,7 +4262,11 @@ public:
             OwnedRoxieString fname(helper->getFileName());
             datafile.setown(_queryFactory.queryPackage().lookupFileName(fname, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
             if (datafile)
+            {
+                unsigned formatCrc = getFormatCrc(helper->getDiskFormatCrc());
+                translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
                 fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
+            }
         }
     }
 
@@ -4302,11 +4276,6 @@ public:
     {
         return CSlaveActivityFactory::toString(s.append("FETCH "));
     }
-
-    inline IFileIO *getFilePart(unsigned partNo, offset_t &_base) const
-    {
-        return fileArray->getFilePart(partNo, _base);
-    }
 };
 
 class CRoxieFetchActivityBase : public CRoxieSlaveActivity
@@ -4316,18 +4285,22 @@ protected:
     const CRoxieFetchActivityFactory *factory;
     Owned<IFileIO> rawFile;
     Owned<ISerialStream> rawStream;
-    CThorStreamDeserializerSource deserializeSource;
     offset_t base;
     char *inputData;
     char *inputLimit;
-    Owned<IFileIOArray> varFiles;
+    Linked<ITranslatorSet> translators;
+    Linked<IFileIOArray> files;
+    const IDynamicTransform *translator = nullptr;
     bool needsRHS;
 
     virtual size32_t doFetch(ARowBuilder & rowBuilder, offset_t pos, offset_t rawpos, void *inputData) = 0;
 
 public:
-    CRoxieFetchActivityBase(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CRoxieFetchActivityFactory *_aFactory)
-        : CRoxieSlaveActivity(_logctx, _packet, _hFactory, _aFactory), factory(_aFactory)
+    CRoxieFetchActivityBase(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet,  HelperFactory *_hFactory,
+                            const CRoxieFetchActivityFactory *_aFactory,
+                            ITranslatorSet *_translators,
+                            IFileIOArray *_files)
+        : CRoxieSlaveActivity(_logctx, _packet, _hFactory, _aFactory), factory(_aFactory), translators(_translators), files(_files)
     {
         helper = (IHThorFetchBaseArg *) basehelper;
         base = 0;
@@ -4346,7 +4319,9 @@ public:
 
     virtual void setVariableFileInfo()
     {
-        varFiles.setown(varFileInfo->getIFileIOArray(isOpt, packet->queryHeader().channel));
+        unsigned formatCrc = basefactory->getFormatCrc(helper->getDiskFormatCrc());
+        translators.setown(varFileInfo->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation()));
+        files.setown(varFileInfo->getIFileIOArray(isOpt, packet->queryHeader().channel));
     }
 
     virtual IMessagePacker *process();
@@ -4418,31 +4393,55 @@ IMessagePacker *CRoxieFetchActivityBase::process()
 class CRoxieFetchActivity : public CRoxieFetchActivityBase
 {
     Owned<IEngineRowAllocator> diskAllocator;
-    Owned<IOutputRowDeserializer> rowDeserializer;
+    CThorContiguousRowBuffer prefetchSource;
+    Owned<ISourceRowPrefetcher> rowPrefetcher;
+
 public:
-    CRoxieFetchActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CRoxieFetchActivityFactory *_aFactory)
-        : CRoxieFetchActivityBase(_logctx, _packet, _hFactory, _aFactory)
+    CRoxieFetchActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory,
+                        const CRoxieFetchActivityFactory *_aFactory,
+                        ITranslatorSet *_translators,
+                        IFileIOArray *_files)
+        : CRoxieFetchActivityBase(_logctx, _packet, _hFactory, _aFactory, _translators, _files)
     {
-        IOutputMetaData *diskMeta = helper->queryDiskRecordSize();
+        // If we ever supported superfiles this would need to move to setPartNo, and pass proper subfile idx in
+        rowPrefetcher.setown(translators->getPrefetcher(0, false, queryContext->queryCodeContext(), basefactory->queryId()));
+
+        IOutputMetaData *diskMeta = helper->queryProjectedDiskRecordSize();
         diskAllocator.setown(getRowAllocator(diskMeta, basefactory->queryId()));
-        rowDeserializer.setown(diskMeta->createDiskDeserializer(queryContext->queryCodeContext(), basefactory->queryId()));
     }
 
     virtual size32_t doFetch(ARowBuilder & rowBuilder, offset_t pos, offset_t rawpos, void *inputData)
     {
-        RtlDynamicRowBuilder diskRowBuilder(diskAllocator);
-        deserializeSource.reset(pos);
-        unsigned sizeRead = rowDeserializer->deserialize(diskRowBuilder.ensureRow(), deserializeSource);
-        OwnedConstRoxieRow rawBuffer = diskRowBuilder.finalizeRowClear(sizeRead);
-        //  note the swapped parameters - left and right map to input and raw differently for JOIN vs FETCH
-        IHThorFetchArg *h = (IHThorFetchArg *) helper;
-        return h->transform(rowBuilder, rawBuffer, inputData, rawpos);
+        prefetchSource.reset(pos);
+        rowPrefetcher->readAhead(prefetchSource);
+        const byte *diskRow = prefetchSource.queryRow();
+        if (translator)
+        {
+            MemoryBuffer buf;
+            MemoryBufferBuilder aBuilder(buf, 0);
+            translator->translate(aBuilder, diskRow);
+            //  note the swapped parameters - left and right map to input and raw differently for JOIN vs FETCH
+            IHThorFetchArg *h = (IHThorFetchArg *) helper;
+            return h->transform(rowBuilder, buf.toByteArray(), inputData, rawpos);
+        }
+        else
+        {
+            //  note the swapped parameters - left and right map to input and raw differently for JOIN vs FETCH
+            IHThorFetchArg *h = (IHThorFetchArg *) helper;
+            return h->transform(rowBuilder, diskRow, inputData, rawpos);
+        }
+    }
+
+    virtual void setPartNo(bool filechanged)
+    {
+        CRoxieFetchActivityBase::setPartNo(filechanged);
+        prefetchSource.setStream(rawStream);
     }
 };
 
 IRoxieSlaveActivity *CRoxieFetchActivityFactory::createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
 {
-    return new CRoxieFetchActivity(logctx, packet, helperFactory, this);
+    return new CRoxieFetchActivity(logctx, packet, helperFactory, this, translators, fileArray);
 }
 
 //------------------------------------------------------------------------------------
@@ -4451,10 +4450,15 @@ class CRoxieCSVFetchActivity : public CRoxieFetchActivityBase
 {
     CSVSplitter csvSplitter;
     size32_t maxRowSize;
+    CThorStreamDeserializerSource deserializeSource;
 
 public:
-    CRoxieCSVFetchActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CRoxieFetchActivityFactory *_aFactory, unsigned _maxColumns, size32_t _maxRowSize)
-        : CRoxieFetchActivityBase(_logctx, _packet, _hFactory, _aFactory), maxRowSize(_maxRowSize)
+    CRoxieCSVFetchActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory,
+                           const CRoxieFetchActivityFactory *_aFactory,
+                           ITranslatorSet *_translators,
+                           IFileIOArray *_files,
+                           unsigned _maxColumns, size32_t _maxRowSize)
+        : CRoxieFetchActivityBase(_logctx, _packet, _hFactory, _aFactory, _translators, _files), maxRowSize(_maxRowSize)
     {
         const char * quotes = NULL;
         const char * separators = NULL;
@@ -4477,6 +4481,8 @@ public:
         IHThorCsvFetchArg *h = (IHThorCsvFetchArg *) helper;
         ICsvParameters *csvInfo = h->queryCsvParameters();
         csvSplitter.init(_maxColumns, csvInfo, quotes, separators, terminators, escapes);
+        if (translator)
+            UNIMPLEMENTED;  // It's not obvious how we would support this.
     }
 
     virtual size32_t doFetch(ARowBuilder & rowBuilder, offset_t pos, offset_t rawpos, void *inputData)
@@ -4499,6 +4505,12 @@ public:
         }
         return h->transform(rowBuilder, csvSplitter.queryLengths(), (const char * *)csvSplitter.queryData(), inputData, rawpos);
     }
+
+    virtual void setPartNo(bool filechanged)
+    {
+        CRoxieFetchActivityBase::setPartNo(filechanged);
+        deserializeSource.setStream(rawStream);
+    }
 };
 
 class CRoxieXMLFetchActivity : public CRoxieFetchActivityBase, implements IXMLSelect
@@ -4507,14 +4519,21 @@ class CRoxieXMLFetchActivity : public CRoxieFetchActivityBase, implements IXMLSe
     Owned<IColumnProvider> lastMatch;
     Owned<IFileIOStream> rawStreamX;
     unsigned streamBufferSize;
+    CThorStreamDeserializerSource deserializeSource;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CRoxieFetchActivityBase)
 
-    CRoxieXMLFetchActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CRoxieFetchActivityFactory *_aFactory, unsigned _streamBufferSize)
-        : CRoxieFetchActivityBase(_logctx, _packet, _hFactory, _aFactory),
+    CRoxieXMLFetchActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory,
+                           const CRoxieFetchActivityFactory *_aFactory,
+                           ITranslatorSet *_translators,
+                           IFileIOArray *_files,
+                           unsigned _streamBufferSize)
+        : CRoxieFetchActivityBase(_logctx, _packet, _hFactory, _aFactory, _translators, _files),
           streamBufferSize(_streamBufferSize)
     {
+        if (translator)
+            UNIMPLEMENTED;
     }
 
     virtual size32_t doFetch(ARowBuilder & rowBuilder, offset_t pos, offset_t rawpos, void *inputData)
@@ -4523,8 +4542,8 @@ public:
         try
         {
             while(!lastMatch)
-            if(!parser->next())
-                throw MakeStringException(ROXIE_RECORD_FETCH_ERROR, "XML parse error at position %" I64F "d", pos);
+                if(!parser->next())
+                    throw MakeStringException(ROXIE_RECORD_FETCH_ERROR, "XML parse error at position %" I64F "d", pos);
             IHThorXmlFetchArg *h = (IHThorXmlFetchArg *) helper;
             unsigned thisSize = h->transform(rowBuilder, lastMatch, inputData, rawpos);
             lastMatch.clear();
@@ -4546,6 +4565,7 @@ public:
     virtual void setPartNo(bool filechanged)
     {
         CRoxieFetchActivityBase::setPartNo(filechanged);
+        deserializeSource.setStream(rawStream);
         rawStreamX.setown(createBufferedIOStream(rawFile, streamBufferSize));
         parser.setown((factory->getKind()==TAKjsonfetch) ? createJSONParse(*rawStreamX, "/", *this) : createXMLParse(*rawStreamX, "/", *this));
     }
@@ -4554,10 +4574,10 @@ public:
 
 void CRoxieFetchActivityBase::setPartNo(bool filechanged)
 {
-    rawFile.setown(variableFileName ? varFiles->getFilePart(lastPartNo.partNo, base) : factory->getFilePart(lastPartNo.partNo, base)); // MORE - superfiles
+    rawFile.setown(files->getFilePart(lastPartNo.partNo, base)); // MORE - superfiles
+    translator = translators->queryTranslator(0);                // MORE - superfiles
     assertex(rawFile != NULL);
     rawStream.setown(createFileSerialStream(rawFile, 0, -1, 0));
-    deserializeSource.setStream(rawStream);
 }
 
 class CRoxieCSVFetchActivityFactory : public CRoxieFetchActivityFactory
@@ -4581,7 +4601,7 @@ public:
 
     virtual IRoxieSlaveActivity *createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
     {
-        return new CRoxieCSVFetchActivity(logctx, packet, helperFactory, this, maxColumns, maxRowSize);
+        return new CRoxieCSVFetchActivity(logctx, packet, helperFactory, this, translators, fileArray, maxColumns, maxRowSize);
     }
 };
 
@@ -4595,7 +4615,7 @@ public:
 
     virtual IRoxieSlaveActivity *createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
     {
-        return new CRoxieXMLFetchActivity(logctx, packet, helperFactory, this, 4096);
+        return new CRoxieXMLFetchActivity(logctx, packet, helperFactory, this, translators, fileArray, 4096);
     }
 };
 
@@ -4962,7 +4982,8 @@ ISlaveActivityFactory *createRoxieKeyedJoinIndexActivityFactory(IPropertyTree &_
 class CRoxieKeyedJoinFetchActivityFactory : public CSlaveActivityFactory
 {
 public:
-    Owned<IFileIOArray> fileArray;
+    Owned<ITranslatorSet> translators;
+    Owned<IFileIOArray> files;
 
     CRoxieKeyedJoinFetchActivityFactory(IPropertyTree &_graphNode, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory)
         : CSlaveActivityFactory(_graphNode, _subgraphId, _queryFactory, _helperFactory)
@@ -4976,7 +4997,11 @@ public:
             OwnedRoxieString fileName(helper->getFileName());
             datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
             if (datafile)
-                fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
+            {
+                unsigned formatCrc = getFormatCrc(helper->getDiskFormatCrc());
+                translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
+                files.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
+            }
         }
 
     }
@@ -4987,35 +5012,36 @@ public:
     {
         return CSlaveActivityFactory::toString(s.append("KEYEDJOIN FETCH "));
     }
-
-    IFileIO *getFilePart(unsigned partNo, offset_t &_base) const
-    {
-        return fileArray->getFilePart(partNo, _base);
-    }
 };
 
 class CRoxieKeyedJoinFetchActivity : public CRoxieSlaveActivity
 {
     IHThorKeyedJoinArg *helper;
     Owned<IFileIO> rawFile;
-    const CRoxieKeyedJoinFetchActivityFactory *factory;
     offset_t base;
     const char *inputLimit;
     const char *inputData;
-    Owned<IFileIOArray> varFiles;
+    Linked<ITranslatorSet> translators;
+    Linked<IFileIOArray> files;
     Owned<ISerialStream> rawStream;
-    CThorStreamDeserializerSource deserializeSource;
+    CThorContiguousRowBuffer prefetchSource;
+    Owned<ISourceRowPrefetcher> prefetcher;
+    const IDynamicTransform *translator = nullptr;
 
     virtual void setPartNo(bool filechanged)
     {
-        rawFile.setown(variableFileName ? varFiles->getFilePart(lastPartNo.partNo, base) : factory->getFilePart(lastPartNo.partNo, base)); // MORE - superfiles
+        rawFile.setown(files->getFilePart(lastPartNo.partNo, base)); // MORE - superfiles
+        translator = translators->queryTranslator(0);                // MORE - superfiles
         rawStream.setown(createFileSerialStream(rawFile, 0, -1, 0));
-        deserializeSource.setStream(rawStream);
+        prefetcher.setown(translators->getPrefetcher(0, false, queryContext->queryCodeContext(), basefactory->queryId()));
+        prefetchSource.setStream(rawStream);
     }
 
 public:
-    CRoxieKeyedJoinFetchActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CRoxieKeyedJoinFetchActivityFactory *_aFactory)
-        : factory(_aFactory), 
+    CRoxieKeyedJoinFetchActivity(SlaveContextLogger &_logctx, IRoxieQueryPacket *_packet, HelperFactory *_hFactory, const CRoxieKeyedJoinFetchActivityFactory *_aFactory,
+                                 IFileIOArray *_files, ITranslatorSet *_translators)
+        : files(_files),
+          translators(_translators),
           CRoxieSlaveActivity(_logctx, _packet, _hFactory, _aFactory)
     {
         // MORE - no continuation row support?
@@ -5038,7 +5064,9 @@ public:
 
     virtual void setVariableFileInfo()
     {
-        varFiles.setown(varFileInfo->getIFileIOArray(isOpt, packet->queryHeader().channel));
+        unsigned formatCrc = basefactory->getFormatCrc(helper->getDiskFormatCrc());
+        translators.setown(varFileInfo->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation()));
+        files.setown(varFileInfo->getIFileIOArray(isOpt, packet->queryHeader().channel));
     }
 
     virtual IMessagePacker *process();
@@ -5058,9 +5086,6 @@ IMessagePacker *CRoxieKeyedJoinFetchActivity::process()
     unsigned skipped = 0;
     unsigned __int64 rowLimit = helper->getRowLimit();
     unsigned totalSizeSent = 0;
-    Owned<IOutputRowDeserializer> rowDeserializer = helper->queryDiskRecordSize()->createDiskDeserializer(queryContext->queryCodeContext(), basefactory->queryId());
-    Owned<IEngineRowAllocator> diskAllocator = getRowAllocator(helper->queryDiskRecordSize(), basefactory->queryId());
-    RtlDynamicRowBuilder diskRowBuilder(diskAllocator);
 
     CachedOutputMetaData joinFieldsMeta(helper->queryJoinFieldsRecordSize());
     Owned<IEngineRowAllocator> joinFieldsAllocator = getRowAllocator(joinFieldsMeta, basefactory->queryId());
@@ -5080,9 +5105,16 @@ IMessagePacker *CRoxieKeyedJoinFetchActivity::process()
         else
             pos = rp-base;
 
-        deserializeSource.reset(pos);
-        unsigned sizeRead = rowDeserializer->deserialize(diskRowBuilder.ensureRow(), deserializeSource);
-        OwnedConstRoxieRow rawBuffer = diskRowBuilder.finalizeRowClear(sizeRead);
+        prefetchSource.reset(pos);
+        prefetcher->readAhead(prefetchSource);
+        const byte *rawRHS = prefetchSource.queryRow();
+        MemoryBuffer buf;
+        if (translator)
+        {
+            MemoryBufferBuilder aBuilder(buf, 0);
+            translator->translate(aBuilder, rawRHS);
+            rawRHS = (const byte *) buf.toByteArray();
+        }
 
         const KeyedJoinHeader *headerPtr = (KeyedJoinHeader *) inputData;
         inputData = &headerPtr->rhsdata[0];
@@ -5091,9 +5123,9 @@ IMessagePacker *CRoxieKeyedJoinFetchActivity::process()
             memcpy(&inputSize, inputData, sizeof(inputSize));
             inputData += sizeof(inputSize);
         }
-        if (helper->fetchMatch(inputData, rawBuffer))
+        if (helper->fetchMatch(inputData, rawRHS))
         {
-            unsigned thisSize = helper->extractJoinFields(jfRowBuilder, rawBuffer, (IBlobProvider*)NULL);
+            unsigned thisSize = helper->extractJoinFields(jfRowBuilder, rawRHS, (IBlobProvider*)NULL);
             jfRowBuilder.writeToOutput(thisSize, headerPtr->fpos, headerPtr->thisGroup, headerPtr->partNo);
             totalSizeSent += KEYEDJOIN_RECORD_SIZE(thisSize);
             processed++;
@@ -5131,7 +5163,7 @@ IMessagePacker *CRoxieKeyedJoinFetchActivity::process()
 
 IRoxieSlaveActivity *CRoxieKeyedJoinFetchActivityFactory::createActivity(SlaveContextLogger &logctx, IRoxieQueryPacket *packet) const
 {
-    return new CRoxieKeyedJoinFetchActivity(logctx, packet, helperFactory, this);
+    return new CRoxieKeyedJoinFetchActivity(logctx, packet, helperFactory, this, files, translators);
 }
 
 ISlaveActivityFactory *createRoxieKeyedJoinFetchActivityFactory(IPropertyTree &_graphNode, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory)
@@ -5315,7 +5347,9 @@ public:
                     bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isOpt']/@value");
                     datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
                     if (datafile)
+                    {
                         fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
+                    }
                 }
             }
         }

roxie/ccd/ccdfile.cpp

@@ -41,6 +41,9 @@
 #include <sys/syscall.h>
 #include "ioprio.h"
 #endif
+#include "eclhelper_dyn.hpp"
+#include "rtldynfield.hpp"
+#include "hqlexpr.hpp"
 
 atomic_t numFilesOpen[2];
 
@@ -1539,9 +1542,16 @@ extern IFilePartMap *createFilePartMap(const char *fileName, IFileDescriptor &fd
 
 class CFileIOArray : implements IFileIOArray, public CInterface
 {
-    unsigned __int64 totalSize;
     mutable CriticalSection crit;
-    mutable StringAttr id;
+    mutable unsigned __int64 totalSize = (unsigned __int64) -1;  // Calculated on demand, and cached
+    mutable StringAttr id;               // Calculated on demand, and cached
+    IPointerArrayOf<IFileIO> files;
+    UnsignedArray subfiles;
+    StringArray filenames;
+    Int64Array bases;
+    int actualCrc = 0;
+    unsigned valid = 0;
+    bool multipleFormatsSeen = false;
 
     void _getId() const
     {
@@ -1568,19 +1578,10 @@ class CFileIOArray : implements IFileIOArray, public CInterface
 
 public:
     IMPLEMENT_IINTERFACE;
-    CFileIOArray()
-    {
-        valid = 0;
-        totalSize = (unsigned __int64) -1;
-    }
 
     virtual bool IsShared() const { return CInterface::IsShared(); };
-    IPointerArrayOf<IFileIO> files;
-    StringArray filenames;
-    Int64Array bases;
-    unsigned valid;
 
-    virtual IFileIO *getFilePart(unsigned partNo, offset_t &base)
+    virtual IFileIO *getFilePart(unsigned partNo, offset_t &base) const override
     {
         if (!files.isItem(partNo))
         {
@@ -1590,8 +1591,6 @@ public:
         IFileIO *file = files.item(partNo);
         if (!file)
         {
-//          DBGLOG("getFilePart requested nonBonded part %d", partNo);
-//          throw MakeStringException(ROXIE_FILE_FAIL, "getFilePart requested nonBonded part %d", partNo);
             base = 0;
             return NULL;
         }
@@ -1599,7 +1598,7 @@ public:
         return LINK(file);
     }
 
-    virtual const char *queryLogicalFilename(unsigned partNo)
+    virtual const char *queryLogicalFilename(unsigned partNo) const override
     {
         if (!filenames.isItem(partNo))
         {
@@ -1609,26 +1608,45 @@ public:
         return filenames.item(partNo);
     }
 
-    void addFile(IFileIO *f, offset_t base, const char *filename)
+    void addFile(IFileIO *f, offset_t base, unsigned subfile, const char *filename, int _actualCrc)
     {
         if (f)
             valid++;
         files.append(f);
         bases.append(base);
-        filenames.append(filename ? filename : "");  // Hack!
+        if (_actualCrc)
+        {
+            if (actualCrc && actualCrc != _actualCrc)
+                multipleFormatsSeen = true;
+            else
+                actualCrc = _actualCrc;
+        }
+        // MORE - lots of duplication in subfiles and filenames arrays
+        subfiles.append(subfile);
+        filenames.append(filename ? filename : "");
     }
 
-    virtual unsigned length()
+    virtual unsigned length() const override
     {
         return files.length();
     }
 
-    virtual unsigned numValid()
+    virtual unsigned numValid() const override
     {
         return valid;
     }
 
-    virtual bool isValid(unsigned partNo)
+    virtual int queryActualFormatCrc() const override
+    {
+        return actualCrc;
+    }
+
+    virtual bool allFormatsMatch() const override
+    {
+        return !multipleFormatsSeen;
+    }
+
+    virtual bool isValid(unsigned partNo) const override
     {
         if (!files.isItem(partNo))
             return false;
@@ -1638,7 +1656,7 @@ public:
         return true;
     }
 
-    virtual unsigned __int64 size()
+    virtual unsigned __int64 size() const override
     {
         CriticalBlock b(crit);
         if (totalSize == (unsigned __int64) -1)
@@ -1654,13 +1672,75 @@ public:
         return totalSize;
     }
 
-    virtual StringBuffer &getId(StringBuffer &ret) const
+    virtual StringBuffer &getId(StringBuffer &ret) const override
     {
         CriticalBlock b(crit);
         if (!id)
             _getId();
         return ret.append(id);
     }
+
+    virtual unsigned getSubFile(unsigned partNo) const override
+    {
+        return subfiles.item(partNo);
+    }
+};
+
+class CTranslatorSet : implements CInterfaceOf<ITranslatorSet>
+{
+    IConstPointerArrayOf<IDynamicTransform> transformers;
+    IPointerArrayOf<IOutputMetaData> actualLayouts;
+    int targetFormatCrc = 0;
+    bool anyTranslators = false;
+public:
+    CTranslatorSet(int _targetFormatCrc) : targetFormatCrc(_targetFormatCrc) {}
+
+    void addTranslator(const IDynamicTransform *translator, IOutputMetaData *actualLayout)
+    {
+        assertex(actualLayout);
+        if (translator)
+            anyTranslators = true;
+        transformers.append(translator);
+        actualLayouts.append(actualLayout);
+    }
+
+    virtual int queryTargetFormatCrc() const override
+    {
+        return targetFormatCrc;
+    }
+
+    virtual const IDynamicTransform *queryTranslator(unsigned subFile) const override
+    {
+        // We need to have translated partnos to subfiles before calling this!
+        // Note: while the required projected format will be the same for all parts, the
+        // actual layout - and thus the required translation - may not be, for example if
+        // we have a superfile with mismatching formats.
+        if (anyTranslators && transformers.isItem(subFile))
+            return transformers.item(subFile);
+        return nullptr;
+    }
+
+    virtual ISourceRowPrefetcher *getPrefetcher(unsigned subFile, bool addGroupFlag, ICodeContext *ctx, unsigned actId) const override
+    {
+        IOutputMetaData *actualLayout = actualLayouts.item(subFile);
+        assertex(actualLayout);
+        if (addGroupFlag && actualLayout->isGrouped())
+            return new CSuffixedRowPrefetcher(1, actualLayout->createDiskPrefetcher(ctx, actId));
+        else
+            return actualLayout->createDiskPrefetcher(ctx, actId);
+    }
+
+    virtual IOutputMetaData *queryActualLayout(unsigned subFile) const override
+    {
+        IOutputMetaData *actualLayout = actualLayouts.item(subFile);
+        assertex(actualLayout);
+        return actualLayout;
+    }
+
+    virtual bool isTranslating() const override
+    {
+        return anyTranslators;
+    }
 };
 
 template <class X> class PerChannelCacheOf
@@ -1668,6 +1748,7 @@ template <class X> class PerChannelCacheOf
     IPointerArrayOf<X> cache;
     IntArray channels;
 public:
+    // NOTE - typically only a couple of entries (but see PerFormatCacheOf below
     void set(X *value, unsigned channel)
     {
         cache.append(value);
@@ -1685,6 +1766,12 @@ public:
     }
 };
 
+template <class X> class PerFormatCacheOf : public PerChannelCacheOf<X>
+{
+    // Identical for now, but characteristics are different so implementations may diverge.
+    // For example, this one may want to be a hash table, and there may be many more entries
+};
+
 CRoxieFileCache * fileCache;
 
 class CResolvedFile : implements IResolvedFileCreator, implements ISDSSubscription, public CInterface
@@ -1703,7 +1790,9 @@ protected:
     StringArray subNames;
     IPointerArrayOf<IFileDescriptor> subFiles; // note - on slaves, the file descriptors may have incomplete info. On originating server is always complete
     IPointerArrayOf<IFileDescriptor> remoteSubFiles; // note - on slaves, the file descriptors may have incomplete info. On originating server is always complete
-    IPointerArrayOf<IDefRecordMeta> diskMeta;
+    IPointerArrayOf<IDefRecordMeta> diskMeta;  // Old info used by Pete's format translator
+    IntArray formatCrcs;
+    IPointerArrayOf<IOutputMetaData> diskTypeInfo;  // New info using RtlTypeInfo structures
     IArrayOf<IDistributedFile> subDFiles;  // To make sure subfiles get locked too
     IArrayOf<IResolvedFile> subRFiles;  // To make sure subfiles get locked too
 
@@ -1726,6 +1815,39 @@ protected:
         else
             diskMeta.append(NULL);
 
+        // NOTE - grouping is not included in the formatCRC, nor is the trailing byte that indicates grouping
+        // included in the rtlTypeInfo.
+
+        bool isGrouped = props.getPropBool("@grouped", false);
+        int formatCrc = props.getPropInt("@formatCrc", 0);
+        // If formatCrc and grouping are same as previous, reuse previous typeInfo
+        Owned<IOutputMetaData> actualFormat;
+        unsigned prevIdx = formatCrcs.length()-1;
+        if (formatCrcs.length() && formatCrc == formatCrcs.item(prevIdx) &&
+            diskTypeInfo.item(prevIdx) && isGrouped==diskTypeInfo.item(prevIdx)->isGrouped())
+            actualFormat.set(diskTypeInfo.item(prevIdx));
+        else if (props.hasProp("_rtlType"))
+        {
+            MemoryBuffer layoutBin;
+            props.getPropBin("_rtlType", layoutBin);
+            actualFormat.setown(createTypeInfoOutputMetaData(layoutBin, isGrouped, nullptr));
+        }
+        else if (props.hasProp("ECL"))
+        {
+            StringBuffer layoutECL;
+            props.getProp("ECL", layoutECL);
+            MultiErrorReceiver errs;
+            Owned<IHqlExpression> expr = parseQuery(layoutECL.str(), &errs);
+            if (errs.errCount() == 0)
+            {
+                MemoryBuffer layoutBin;
+                if (exportBinaryType(layoutBin, expr))
+                    actualFormat.setown(createTypeInfoOutputMetaData(layoutBin, isGrouped, nullptr));
+            }
+        }
+        diskTypeInfo.append(actualFormat.getClear());
+        formatCrcs.append(formatCrc);
+
         unsigned numParts = fdesc->numParts();
         offset_t base = 0;
         for (unsigned i = 0; i < numParts; i++)
@@ -1890,7 +2012,7 @@ public:
         }
         return numParts;
     }
-    virtual void serializeFDesc(MemoryBuffer &mb, IFileDescriptor *fdesc, unsigned channel, bool isLocal) const
+    bool serializeFDesc(MemoryBuffer &mb, IFileDescriptor *fdesc, unsigned channel, bool isLocal) const
     {
         // Find all the partno's that go to this channel
         unsigned numParts = fdesc->numParts();
@@ -1906,8 +2028,9 @@ public:
             }
         }
         fdesc->serializeParts(mb, partNos);
+        return partNos.length();
     }
-    virtual void serializePartial(MemoryBuffer &mb, unsigned channel, bool isLocal) const
+    virtual void serializePartial(MemoryBuffer &mb, unsigned channel, bool isLocal) const override
     {
         if (traceLevel > 6)
             DBGLOG("Serializing file information for dynamic file %s, channel %d, local %d", lfn.get(), channel, isLocal);
@@ -1922,15 +2045,30 @@ public:
         {
             mb.append(subNames.item(idx));
             IFileDescriptor *fdesc = subFiles.item(idx);
-            serializeFDesc(mb, fdesc, channel, isLocal);
+            bool anyparts = serializeFDesc(mb, fdesc, channel, isLocal);
             IFileDescriptor *remoteFDesc = remoteSubFiles.item(idx);
             if (remoteFDesc)
             {
                 mb.append(true);
-                serializeFDesc(mb, remoteFDesc, channel, isLocal);
+                anyparts |= serializeFDesc(mb, remoteFDesc, channel, isLocal);
             }
             else
                 mb.append(false);
+            mb.append(formatCrcs.item(idx));
+            IOutputMetaData *diskType = diskTypeInfo.item(idx);
+            if (anyparts && diskType)
+            {
+                if (idx && formatCrcs.item(idx)==formatCrcs.item(idx-1))
+                    mb.append((byte) 3);  // indicating same format as previous
+                else
+                {
+                    mb.append((byte) (diskType->isGrouped() ? 2 : 1));
+                    verifyex(dumpTypeInfo(mb, diskType->queryTypeInfo()));  // Must be serializable, as we deserialized it...
+                }
+            }
+            else
+                mb.append((byte) 0);
+
             if (fileType == ROXIE_KEY) // for now we only support translation on index files
             {
                 IDefRecordMeta *meta = diskMeta.item(idx);
@@ -1951,6 +2089,56 @@ public:
         else
             mb.append(false);
     }
+    virtual ITranslatorSet *getTranslators(int formatCrc, IOutputMetaData *projected, IOutputMetaData *expected, IRecordLayoutTranslator::Mode mode) const override
+    {
+        // NOTE - projected and expected and anything fetched from them such as type info may reside in dynamically loaded (and unloaded)
+        // query DLLs - this means it is not safe to include them in any sort of cache that might outlive the current query.
+        Owned<CTranslatorSet> result = new CTranslatorSet(formatCrc);
+        Owned<const IDynamicTransform> translator;
+        int prevFormatCrc = 0;
+        ForEachItemIn(idx, subFiles)
+        {
+            IOutputMetaData *actual = expected;
+            if (formatCrc)
+            {
+                const char *subname = subNames.item(idx);
+                int thisFormatCrc = formatCrcs.item(idx);
+                if (mode == IRecordLayoutTranslator::TranslateAlwaysDisk && diskTypeInfo.item(idx))
+                {
+                    actual = diskTypeInfo.item(idx);
+                    translator.setown(createRecordTranslator(projected->queryRecordAccessor(true), actual->queryRecordAccessor(true)));
+                    if (!translator->canTranslate())
+                        throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname);
+                }
+                else if (mode == IRecordLayoutTranslator::TranslateAlwaysECL)
+                {
+                    translator.setown(createRecordTranslator(projected->queryRecordAccessor(true), expected->queryRecordAccessor(true)));
+                    if (!translator->canTranslate())
+                        throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname);
+                }
+                else if (!thisFormatCrc || thisFormatCrc==formatCrc)
+                    translator.clear();
+                else
+                {
+                    actual = diskTypeInfo.item(idx);
+                    if (thisFormatCrc != prevFormatCrc)  // Check if same translation as last subfile
+                    {
+                        translator.clear();
+                        if (projected && actual)
+                        {
+                            translator.setown(createRecordTranslator(projected->queryRecordAccessor(true), actual->queryRecordAccessor(true)));
+                            // translator->describe();
+                        }
+                        if (!translator || !translator->canTranslate())
+                            throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname);
+                    }
+                }
+                prevFormatCrc = thisFormatCrc;
+            }
+            result->addTranslator(LINK(translator), LINK(actual));
+        }
+        return result.getClear();
+    }
     virtual IFileIOArray *getIFileIOArray(bool isOpt, unsigned channel) const
     {
         CriticalBlock b(lock);
@@ -1964,13 +2152,14 @@ public:
     }
     IFileIOArray *createIFileIOArray(bool isOpt, unsigned channel) const
     {
-        Owned<CFileIOArray> f = new CFileIOArray();
-        f->addFile(NULL, 0, NULL);
+        Owned<CFileIOArray> f = new CFileIOArray;
+        f->addFile(nullptr, 0, 0, nullptr, 0);
         ForEachItemIn(idx, subFiles)
         {
             IFileDescriptor *fdesc = subFiles.item(idx);
             IFileDescriptor *remoteFDesc = remoteSubFiles.item(idx);
             const char *subname = subNames.item(idx);
+            int thisFormatCrc = formatCrcs.item(idx);
             if (fdesc)
             {
                 unsigned numParts = fdesc->numParts();
@@ -1985,7 +2174,7 @@ public:
                             IPartDescriptor *remotePDesc = queryMatchingRemotePart(pdesc, remoteFDesc, i-1);
                             Owned<ILazyFileIO> file = createPhysicalFile(subNames.item(idx), pdesc, remotePDesc, ROXIE_FILE, numParts, cached != NULL, channel);
                             IPropertyTree &partProps = pdesc->queryProperties();
-                            f->addFile(file.getClear(), partProps.getPropInt64("@offset"), subname);
+                            f->addFile(file.getClear(), partProps.getPropInt64("@offset"), idx, subname, thisFormatCrc);
                         }
                         catch (IException *E)
                         {
@@ -1996,11 +2185,11 @@ public:
                             if (!isOpt)
                                 throw;
                             E->Release();
-                            f->addFile(NULL, 0, NULL);
+                            f->addFile(nullptr, 0, idx, nullptr, 0);
                         }
                     }
                     else
-                        f->addFile(NULL, 0, NULL);
+                        f->addFile(nullptr, 0, idx, nullptr, 0);
                 }
             }
         }
@@ -2140,7 +2329,7 @@ public:
         return ret.getClear();
     }
 
-    virtual IInMemoryIndexManager *getIndexManager(bool isOpt, unsigned channel, IFileIOArray *files, IRecordSize *recs, bool preload, int numKeys) const 
+    virtual IInMemoryIndexManager *getIndexManager(bool isOpt, unsigned channel, IOutputMetaData *preloadLayout, bool preload, int numKeys) const
     {
         // MORE - I don't know that it makes sense to pass isOpt in to these calls
         // Failures to resolve will not be cached, only successes.
@@ -2151,7 +2340,8 @@ public:
         if (!ret)
         {
             ret = createInMemoryIndexManager(isOpt, lfn);
-            ret->load(files, recs, preload, numKeys);   // note - files (passed in) are channel specific
+            Owned<IFileIOArray> files = getIFileIOArray(isOpt, channel);
+            ret->load(files, preloadLayout, preload, numKeys);   // note - files (passed in) are also channel specific
             indexMap.set(ret, channel);
         }
         return LINK(ret);
@@ -2358,6 +2548,29 @@ public:
                         deserializeFilePart(serverData, remoteSubFiles, fileNo, true);
                     else
                         remoteSubFiles.append(NULL);
+                    unsigned formatCrc;
+                    serverData.read(formatCrc);
+                    formatCrcs.append(formatCrc);
+                    byte diskTypeInfoPresent;
+                    serverData.read(diskTypeInfoPresent);
+                    switch (diskTypeInfoPresent)
+                    {
+                    case 0:
+                        diskTypeInfo.append(NULL);
+                        break;
+                    case 1:
+                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, false, nullptr));
+                        break;
+                    case 2:
+                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, true, nullptr));
+                        break;
+                    case 3:
+                        assertex(fileNo > 0);
+                        diskTypeInfo.append(LINK(diskTypeInfo.item(fileNo-1)));
+                        break;
+                    default:
+                        throwUnexpected();
+                    }
                     if (fileType==ROXIE_KEY)
                     {
                         bool diskMetaPresent;

roxie/ccd/ccdfile.hpp

@@ -84,18 +84,20 @@ interface IKeyArray;
 interface IDefRecordMeta;
 interface IFilePartMap;
 class TranslatorArray;
-interface IInMemoryIndexManager ;
+interface ITranslatorSet;
+interface IInMemoryIndexManager;
 interface IResolvedFileCache;
 
 interface IResolvedFile : extends ISimpleSuperFileEnquiry
 {
     virtual void serializePartial(MemoryBuffer &mb, unsigned channel, bool localInfoOnly) const = 0;
 
+    virtual ITranslatorSet *getTranslators(int formatCrc, IOutputMetaData *projected, IOutputMetaData *expected, IRecordLayoutTranslator::Mode mode) const = 0;
     virtual IFileIOArray *getIFileIOArray(bool isOpt, unsigned channel) const = 0;
     virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, IRecordLayoutTranslator::Mode allowFieldTranslation) const = 0;
     virtual IFilePartMap *getFileMap() const = 0;
     virtual unsigned getNumParts() const = 0;
-    virtual IInMemoryIndexManager *getIndexManager(bool isOpt, unsigned channel, IFileIOArray *files, IRecordSize *recs, bool preload, int numKeys) const = 0;
+    virtual IInMemoryIndexManager *getIndexManager(bool isOpt, unsigned channel, IOutputMetaData *disklayout, bool preload, int numKeys) const = 0;
     virtual offset_t getFileSize() const = 0;
 
     virtual const CDateTime &queryTimeStamp() const = 0;

roxie/ccd/ccdkey.cpp

@@ -29,6 +29,7 @@
 #include "ccdstate.hpp"
 #include "rtlrecord.hpp"
 #include "rtlkey.hpp"
+#include "rtldynfield.hpp"
 
 #ifdef _DEBUG
 //#define _LIMIT_FILECOUNT 5    // Useful for debugging queries when not enough memory to load data...
@@ -38,7 +39,7 @@
 #define BASED_POINTERS
 #endif
 
-class PtrToOffsetMapper 
+class PtrToOffsetMapper
 {
     struct FragmentInfo
     {
@@ -151,13 +152,13 @@ public:
         return makeLocalFposOffset(frag.partNo, ptr - frag.base);
     }
 
-    offset_t makeFilePositionLocal(offset_t pos)
+    offset_t makeFilePositionLocal(offset_t pos) const
     {
         FragmentInfo &frag = findBase(pos);
         return makeLocalFposOffset(frag.partNo, pos - frag.baseOffset);
     }
 
-    void splitPos(offset_t &offset, offset_t &size, unsigned partNo, unsigned numParts)
+    void splitPos(offset_t &offset, offset_t &size, unsigned partNo, unsigned numParts) const
     {
         assert(numParts > 0);
         assert(partNo < numParts);
@@ -549,7 +550,50 @@ typedef IArrayOf<InMemoryIndex> InMemoryIndexSet;
  *   giving entire file, and uses getRecordSize(). Will avoid a one or more virtual peek() calls per row.
  *====================================================================================================*/
 
-class InMemoryDirectReader : implements IDirectReader, implements IThorDiskCallback, implements ISimpleReadStream, public CInterface
+class CDirectReaderBase : public CInterface, implements IDirectReader, implements IThorDiskCallback, implements ISimpleReadStream
+{
+protected:
+    MemoryBuffer buf;  // Used if translating to hold on to current row;
+    CThorContiguousRowBuffer deserializeSource;
+    Owned<ISourceRowPrefetcher> prefetcher;
+    Linked<const ITranslatorSet> translators;
+    const IDynamicTransform *translator = nullptr;
+
+public:
+    CDirectReaderBase(const ITranslatorSet *_translators) : translators(_translators) {}
+    virtual const byte *nextRow() override
+    {
+        if (prefetcher)
+            prefetcher->readAhead(deserializeSource);
+        if (translator)
+        {
+            MemoryBufferBuilder aBuilder(buf, 0);
+            translator->translate(aBuilder, deserializeSource.queryRow());
+            return reinterpret_cast<const byte *>(buf.toByteArray());
+        }
+        else
+            return deserializeSource.queryRow();
+    }
+    virtual bool eog() const override
+    {
+        size32_t sizeRead = deserializeSource.queryRowSize();
+        return (bool) deserializeSource.queryRow()[sizeRead-1];
+    }
+
+    virtual void finishedRow() override
+    {
+        buf.setLength(0);
+        deserializeSource.finishedRow();
+    }
+
+    virtual bool isTranslating() const override
+    {
+        // Note that this returns true if I will translate for ANY file part, not just whether I am translating for current file part
+        return translators && translators->isTranslating();
+    }
+};
+
+class InMemoryDirectReader : public CDirectReaderBase
 {
     // MORE - might be able to use some of the jlib IStream implementations. But I don't want any indirections...
 public:
@@ -557,11 +601,17 @@ public:
     memsize_t pos;
     const char *start;
     offset_t memsize;
-    PtrToOffsetMapper &baseMap;
+    const PtrToOffsetMapper &baseMap;
 
-    InMemoryDirectReader(offset_t _readPos, const char *_start, memsize_t _memsize, PtrToOffsetMapper &_baseMap, unsigned _partNo, unsigned _numParts)
-        : baseMap(_baseMap)
+    InMemoryDirectReader(offset_t _readPos, const char *_start, memsize_t _memsize,
+                         const PtrToOffsetMapper &_baseMap, unsigned _partNo, unsigned _numParts,
+                         const ITranslatorSet *_translators,
+                         ICodeContext *ctx, unsigned id)
+        : CDirectReaderBase(_translators), baseMap(_baseMap)
     {
+        translator = translators->queryTranslator(0);  // Any one would do
+        prefetcher.setown(translators->getPrefetcher(0, false, ctx, id));
+        deserializeSource.setStream(this);
         if (_numParts == 1)
         {
             memsize = _memsize;
@@ -579,7 +629,7 @@ public:
 
     // Interface ISerialStream
 
-    virtual const void * peek(size32_t wanted,size32_t &got)
+    virtual const void * peek(size32_t wanted,size32_t &got) override
     {
         offset_t remaining = memsize - pos;
         if (remaining)
@@ -596,7 +646,7 @@ public:
             return NULL;
         }
     }
-    virtual void get(size32_t len, void * ptr)
+    virtual void get(size32_t len, void * ptr) override
     {
         offset_t remaining = memsize - pos;
         if (len > remaining)
@@ -604,20 +654,20 @@ public:
         memcpy(ptr, start+pos, len);
         pos += len;
     }
-    virtual bool eos()
+    virtual bool eos() override
     {
         return pos == memsize;
     }
-    virtual void skip(size32_t sz)
+    virtual void skip(size32_t sz) override
     {
         assertex(pos + sz <= memsize);
         pos += sz;
     }
-    virtual offset_t tell()
+    virtual offset_t tell() override
     {
         return pos;
     }
-    virtual void reset(offset_t _offset, offset_t _flen)
+    virtual void reset(offset_t _offset, offset_t _flen) override
     {
         assertex(_offset <= memsize);
         pos = (memsize_t) _offset;
@@ -625,7 +675,7 @@ public:
 
     // Interface ISimpleReadStream
 
-    virtual size32_t read(size32_t max_len, void * data)
+    virtual size32_t read(size32_t max_len, void * data) override
     {
         size32_t got;
         const void *ptr = peek(max_len, got);
@@ -638,47 +688,56 @@ public:
 
     // Interface IDirectReader
 
-    virtual ISimpleReadStream *querySimpleStream()
+    virtual ISimpleReadStream *querySimpleStream() override
     {
         return this;
     }
 
-    virtual IThorDiskCallback *queryThorDiskCallback()
+    virtual IThorDiskCallback *queryThorDiskCallback() override
     {
         return this;
     }
 
-    virtual unsigned queryFilePart() const
+    virtual unsigned queryFilePart() const override
     {
         throwUnexpected(); // only supported for disk files
     }
 
     // Interface IThorDiskCallback 
 
-    virtual unsigned __int64 getFilePosition(const void *_ptr)
+    virtual unsigned __int64 getFilePosition(const void *_ptr) override
     {
+        if (translator)
+        {
+            assertex(_ptr==buf.toByteArray());
+            _ptr = deserializeSource.queryRow();
+        }
         return baseMap.ptrToFilePosition(_ptr);
     }
 
-    virtual unsigned __int64 getLocalFilePosition(const void *_ptr)
+    virtual unsigned __int64 getLocalFilePosition(const void *_ptr) override
     {
+        if (translator)
+        {
+            assertex(_ptr==buf.toByteArray());
+            _ptr = deserializeSource.queryRow();
+        }
         return baseMap.ptrToLocalFilePosition(_ptr);
     }
 
-    virtual unsigned __int64 makeFilePositionLocal(offset_t pos)
+    virtual unsigned __int64 makeFilePositionLocal(offset_t pos) override
     {
         return baseMap.makeFilePositionLocal(pos);
     }
 
-    virtual const char * queryLogicalFilename(const void * row) 
+    virtual const char * queryLogicalFilename(const void * row) override
     { 
         UNIMPLEMENTED;
     }
 };
 
-class BufferedDirectReader : implements IDirectReader, implements IThorDiskCallback, implements ISimpleReadStream, public CInterface
+class BufferedDirectReader : public CDirectReaderBase
 {
-    // MORE - could combine some code with in memory version.
 public:
     IMPLEMENT_IINTERFACE;
 
@@ -687,10 +746,14 @@ public:
     offset_t completedStreamsSize;
     Owned<IFileIO> thisPart;
     Owned<ISerialStream> curStream;
+    ICodeContext *ctx;
+    unsigned activityId;
     unsigned thisPartIdx;
 
-    BufferedDirectReader(offset_t _startPos, IFileIOArray *_f, unsigned _partNo, unsigned _numParts) : f(_f)
+    BufferedDirectReader(offset_t _startPos, IFileIOArray *_f, unsigned _partNo, unsigned _numParts, const ITranslatorSet *_translators, ICodeContext *_ctx, unsigned _id)
+    : CDirectReaderBase(_translators), f(_f), ctx(_ctx), activityId(_id)
     {
+        deserializeSource.setStream(this);
         thisFileStartPos = 0;
         completedStreamsSize = 0;
 
@@ -714,6 +777,9 @@ public:
         if (thisPart)
         {       
             curStream.setown(createFileSerialStream(thisPart, _startPos));
+            unsigned subFileIdx = f->getSubFile(thisPartIdx);
+            prefetcher.setown(translators->getPrefetcher(subFileIdx, true, ctx, activityId));
+            translator = translators->queryTranslator(subFileIdx);
         }
         else
         {
@@ -728,6 +794,7 @@ public:
         unsigned maxParts = f->length();
         thisPart.clear();
         curStream.clear();
+        prefetcher.clear();
         while (!thisPart && ++thisPartIdx < maxParts)
         {
             thisPart.setown(f->getFilePart(thisPartIdx, thisFileStartPos ));
@@ -735,10 +802,13 @@ public:
         if (thisPart)
         {
             curStream.setown(createFileSerialStream(thisPart));
+            unsigned subFileIdx = f->getSubFile(thisPartIdx);
+            prefetcher.setown(translators->getPrefetcher(subFileIdx, true, ctx, activityId));
+            translator = translators->queryTranslator(subFileIdx);
         }
     }
 
-    virtual const void * peek(size32_t wanted,size32_t &got)
+    virtual const void * peek(size32_t wanted,size32_t &got) override
     {
         if (curStream)
         {
@@ -760,14 +830,14 @@ public:
         }
     }
 
-    virtual void get(size32_t len, void * ptr)
+    virtual void get(size32_t len, void * ptr) override
     {
         if (curStream)
             curStream->get(len, ptr);
         else
             throw MakeStringException(-1, "BufferedDirectReader::get: requested %u bytes at eof", len);
     }
-    virtual bool eos() 
+    virtual bool eos() override
     {
         for (;;)
         {
@@ -779,14 +849,14 @@ public:
                 return false;
         }
     }
-    virtual void skip(size32_t len)
+    virtual void skip(size32_t len) override
     {
         if (curStream)
             curStream->skip(len);
         else
             throw MakeStringException(-1, "BufferedDirectReader::skip: tried to skip %u bytes at eof", len);
     }
-    virtual offset_t tell()
+    virtual offset_t tell() override
     {
         // Note that tell() means the position with this stream, not the file position within the overall logical file.
         if (curStream)
@@ -794,14 +864,14 @@ public:
         else
             return completedStreamsSize;
     }
-    virtual void reset(offset_t _offset,offset_t _flen)
+    virtual void reset(offset_t _offset,offset_t _flen) override
     {
         throwUnexpected(); // Not designed to be reset
     }
 
     // Interface ISimpleReadStream
 
-    virtual size32_t read(size32_t max_len, void * data)
+    virtual size32_t read(size32_t max_len, void * data) override
     {
         size32_t got;
         const void *ptr = peek(max_len, got);
@@ -816,50 +886,59 @@ public:
 
     // Interface IDirectReader
 
-    virtual ISimpleReadStream *querySimpleStream()
+    virtual ISimpleReadStream *querySimpleStream() override
     {
         return this;
     }
 
-    virtual IThorDiskCallback *queryThorDiskCallback()
+    virtual IThorDiskCallback *queryThorDiskCallback() override
     {
         return this;
     }
 
-    virtual unsigned queryFilePart() const
+    virtual unsigned queryFilePart() const override
     {
         return thisPartIdx;
     }
 
     // Interface IThorDiskCallback
 
-    virtual unsigned __int64 getFilePosition(const void *_ptr)
+    virtual unsigned __int64 getFilePosition(const void *_ptr) override
     {
         // MORE - could do with being faster than this!
         assertex(curStream != NULL);
-        size32_t dummy;
-        return thisFileStartPos + curStream->tell() + ((const char *)_ptr - (const char *)curStream->peek(1, dummy));
+        unsigned __int64 pos = curStream->tell();
+        if (_ptr != buf.toByteArray())
+        {
+            size32_t dummy;
+            pos +=  (const char *)_ptr - (const char *)curStream->peek(1, dummy);
+        }
+        return pos + thisFileStartPos;
     }
 
-    virtual unsigned __int64 getLocalFilePosition(const void *_ptr)
+    virtual unsigned __int64 getLocalFilePosition(const void *_ptr) override
     {
         // MORE - could do with being faster than this!
         assertex(curStream != NULL);
-        size32_t dummy;
-        return makeLocalFposOffset(thisPartIdx-1, curStream->tell() + (const char *)_ptr - (const char *)curStream->peek(1, dummy));
+        unsigned __int64 pos = curStream->tell();
+        if (_ptr != buf.toByteArray())
+        {
+            size32_t dummy;
+            pos +=  (const char *)_ptr - (const char *)curStream->peek(1, dummy);
+        }
+        return makeLocalFposOffset(thisPartIdx-1, pos);
     }
 
-    virtual unsigned __int64 makeFilePositionLocal(offset_t pos)
+    virtual unsigned __int64 makeFilePositionLocal(offset_t pos) override
     {
         assertex(pos >= thisFileStartPos);
         return makeLocalFposOffset(thisPartIdx-1, pos - thisFileStartPos);
     }
 
-    virtual const char * queryLogicalFilename(const void * row) 
+    virtual const char * queryLogicalFilename(const void * row) override
     { 
         return f->queryLogicalFilename(thisPartIdx);
     }
-
 };
 
 class InMemoryIndexManager : implements IInMemoryIndexManager, public CInterface
@@ -872,7 +951,7 @@ class InMemoryIndexManager : implements IInMemoryIndexManager, public CInterface
     unsigned *hits;
     unsigned trackLimit;
     unsigned numTracked;
-    CriticalSection trackedCrit;
+    mutable CriticalSection trackedCrit;
     CriticalSection activeCrit;
     CriticalSection pendingCrit;
     CriticalSection loadCrit;
@@ -970,7 +1049,7 @@ class InMemoryIndexManager : implements IInMemoryIndexManager, public CInterface
         }
     }
 
-    void getTrackedInfo(const char *id, StringBuffer &xml)
+    virtual void getTrackedInfo(const char *id, StringBuffer &xml) const override
     {
         CriticalBlock cb(trackedCrit);
         xml.appendf("<File id='%s' numKeys='%d' fileName='%s'>", id, numKeys, fileName.get());
@@ -1105,7 +1184,7 @@ class InMemoryIndexManager : implements IInMemoryIndexManager, public CInterface
 
 public:
     IMPLEMENT_IINTERFACE;
-    virtual bool IsShared() const { return CInterface::IsShared(); }
+    virtual bool IsShared() const override { return CInterface::IsShared(); }
 
     InMemoryIndexManager(bool _isOpt, const char *_fileName) : fileName(_fileName)
     {
@@ -1143,7 +1222,7 @@ public:
     }
 
     void deserializeCursorPos(MemoryBuffer &mb, InMemoryIndexCursor *cursor);
-    virtual IInMemoryIndexCursor *createCursor(const RtlRecord &recInfo);
+    virtual IInMemoryIndexCursor *createCursor(const RtlRecord &recInfo) override;
     bool selectKey(InMemoryIndexCursor *cursor);
 
     inline const char *queryFileName() const
@@ -1174,12 +1253,12 @@ public:
         return false;
     }
 
-    virtual IDirectReader *createReader(offset_t readPos, unsigned partNo, unsigned numParts)
+    virtual IDirectReader *createReader(offset_t readPos, unsigned partNo, unsigned numParts, const ITranslatorSet *translators, ICodeContext *ctx, unsigned id) const override
     {
         if (loadedIntoMemory)
-            return new InMemoryDirectReader(readPos, fileStart, fileEnd-fileStart, baseMap, partNo, numParts);
+            return new InMemoryDirectReader(readPos, fileStart, fileEnd-fileStart, baseMap, partNo, numParts, translators, ctx, id);
         else
-            return new BufferedDirectReader(readPos, files, partNo, numParts);
+            return new BufferedDirectReader(readPos, files, partNo, numParts, translators, ctx, id);
     }
 
     StringBuffer &queryId(StringBuffer &ret)
@@ -1189,7 +1268,7 @@ public:
         return ret;
     }
 
-    virtual void load(IFileIOArray *_files, IRecordSize *recordSize, bool preload, int _numKeys)
+    virtual void load(IFileIOArray *_files, IOutputMetaData *preloadLayout, bool preload, int _numKeys) override
     {
         // MORE - if numKeys is greater than previously then we may need to take action here....
         CriticalBlock b(loadCrit);
@@ -1222,6 +1301,12 @@ public:
         }
         if (preload && !loadedIntoMemory) // loaded but NOT originally seen as preload, lets try to generate keys...
         {
+            if (!files->allFormatsMatch())
+            {
+                IException *E = makeStringException(ROXIE_MEMORY_ERROR, "Cannot load superfile with mismatching formats into memory");
+                EXCLOG(MCoperatorError, E);
+                throw E;
+            }
             if ((size_t)totalSize != totalSize)
             {
                 IException *E = makeStringException(ROXIE_MEMORY_ERROR, "Preload file is larger than maximum object size");
@@ -1258,7 +1343,7 @@ public:
                 }
             }
             if (_numKeys > 0)
-                processInMemoryKeys(recordSize, _numKeys);
+                processInMemoryKeys(preloadLayout, _numKeys);
             loadedIntoMemory = true;
         }
         else if (_numKeys > numKeys)  // already in memory, but more in memory keys are requested, so let's try to create them
@@ -1381,7 +1466,7 @@ public:
             numKeys = n;
     }
 
-    virtual void setKeyInfo(IPropertyTree &indexInfo)
+    virtual void setKeyInfo(IPropertyTree &indexInfo) override
     {
         StringBuffer x;
         toXML(&indexInfo, x);
@@ -1518,7 +1603,7 @@ class InMemoryIndexCursor : implements IInMemoryIndexCursor, public CInterface
     unsigned keySize;
     bool canMatch;
     bool postFiltering;
-    PtrToOffsetMapper &baseMap;
+    const PtrToOffsetMapper &baseMap;
     const RtlRecord &recInfo;
     RtlDynRow rowinfo;
     unsigned numSegFieldsRequired = 0;

roxie/ccd/ccdkey.hpp

@@ -20,8 +20,11 @@
 #include "rtlkey.hpp"
 #include "eclhelper.hpp"
 #include "jfile.hpp"
+#include "rtlcommon.hpp"
 
 interface IFileIOArray;
+interface ITranslatorSet;
+
 typedef IArrayOf<IKeySegmentMonitor> SegMonitorArray;
 
 interface IDirectReader : public ISerialStream
@@ -30,6 +33,10 @@ interface IDirectReader : public ISerialStream
     virtual ISimpleReadStream *querySimpleStream() = 0;
     virtual unsigned queryFilePart() const = 0;
     virtual unsigned __int64 makeFilePositionLocal(offset_t pos) = 0;
+    virtual const byte *nextRow() = 0;
+    virtual bool eog() const = 0;
+    virtual void finishedRow() = 0;
+    virtual bool isTranslating() const = 0;
 };
 
 interface IInMemoryIndexCursor : public IThorDiskCallback, public IIndexReadContext
@@ -44,11 +51,11 @@ interface IInMemoryIndexCursor : public IThorDiskCallback, public IIndexReadCont
 
 interface IInMemoryIndexManager : extends IInterface
 {
-    virtual void load(IFileIOArray *, IRecordSize *, bool preload, int numKeys) = 0;
+    virtual void load(IFileIOArray *, IOutputMetaData *preloadLayout, bool preload, int numKeys) = 0;
     virtual bool IsShared() const = 0;
     virtual IInMemoryIndexCursor *createCursor(const RtlRecord &recInfo) = 0;
-    virtual IDirectReader *createReader(offset_t readPos, unsigned partNo, unsigned numParts) = 0;
-    virtual void getTrackedInfo(const char *id, StringBuffer &xml) = 0;
+    virtual IDirectReader *createReader(offset_t readPos, unsigned partNo, unsigned numParts, const ITranslatorSet *translators, ICodeContext *ctx, unsigned id) const = 0;
+    virtual void getTrackedInfo(const char *id, StringBuffer &xml) const = 0;
     virtual void setKeyInfo(IPropertyTree &indexInfo) = 0;
 };
 

roxie/ccd/ccdmain.cpp

@@ -141,6 +141,7 @@ unsigned defaultFullKeyedJoinPreload = 0;
 unsigned defaultKeyedJoinPreload = 0;
 unsigned dafilesrvLookupTimeout = 10000;
 bool defaultCheckingHeap = false;
+bool defaultDisableLocalOptimizations = false;
 unsigned defaultStrandBlockSize = 512;
 unsigned defaultForceNumStrands = 0;
 unsigned defaultHeapFlags = roxiemem::RHFnofragment;
@@ -547,6 +548,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             topology->setProp("@traceLevel", globals->queryProp("--traceLevel"));
             topology->setPropInt("@allFilesDynamic", globals->getPropInt("--allFilesDynamic", 1));
             topology->setProp("@memTraceLevel", globals->queryProp("--memTraceLevel"));
+            topology->setProp("@disableLocalOptimizations", globals->queryProp("--disableLocalOptimizations"));
         }
         if (topology->hasProp("PreferredCluster"))
         {
@@ -826,6 +828,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         defaultStrandBlockSize = topology->getPropInt("@defaultStrandBlockSize", 512);
         defaultForceNumStrands = topology->getPropInt("@defaultForceNumStrands", 0);
         defaultCheckingHeap = topology->getPropBool("@checkingHeap", false);  // NOTE - not in configmgr - too dangerous!
+        defaultDisableLocalOptimizations = topology->getPropBool("@disableLocalOptimizations", false);  // NOTE - not in configmgr - too dangerous!
 
         slaveQueryReleaseDelaySeconds = topology->getPropInt("@slaveQueryReleaseDelaySeconds", 60);
         coresPerQuery = topology->getPropInt("@coresPerQuery", 0);
@@ -835,7 +838,11 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         const char *val = topology->queryProp("@fieldTranslationEnabled");
         if (val)
         {
-            if (strieq(val, "payload"))
+            if (strieq(val, "alwaysDisk"))
+                fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAlwaysDisk;
+            else if (strieq(val, "alwaysECL"))
+                fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAlwaysECL;
+            else if (strieq(val, "payload"))
                 fieldTranslationEnabled = IRecordLayoutTranslator::TranslatePayload;
             else if (strToBool(val))
                 fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAll;

roxie/ccd/ccdquery.cpp

@@ -294,7 +294,7 @@ QueryOptions::QueryOptions()
     heapFlags = defaultHeapFlags;
 
     checkingHeap = defaultCheckingHeap;
-    disableLocalOptimizations = false;  // No global default for this
+    disableLocalOptimizations = defaultDisableLocalOptimizations;
     enableFieldTranslation = fieldTranslationEnabled;
     skipFileFormatCrcCheck = false;
     stripWhitespaceFromStoredDataset = ((ptr_ignoreWhiteSpace & defaultXmlReadFlags) != 0);
@@ -403,7 +403,11 @@ void QueryOptions::updateFromWorkUnit(IRecordLayoutTranslator::Mode &value, ICon
     wu.getDebugValue(name, val);
     if (val.length())
     {
-        if (strieq(val.str(), "payload"))
+        if (strieq(val.str(), "alwaysDisk"))
+            value = IRecordLayoutTranslator::TranslateAlwaysDisk;
+        else if (strieq(val.str(), "alwaysECL"))
+            value = IRecordLayoutTranslator::TranslateAlwaysECL;
+        else if (strieq(val.str(), "payload"))
             value = IRecordLayoutTranslator::TranslatePayload;
         else if (strToBool(val.str()))
             value = IRecordLayoutTranslator::TranslateAll;
@@ -434,7 +438,7 @@ void QueryOptions::setFromContext(const IPropertyTree *ctx)
         updateFromContext(checkingHeap, ctx, "@checkingHeap", "_CheckingHeap");
         // Note: disableLocalOptimizations is not permitted at context level (too late)
         // Note: enableFieldTranslation is not permitted at context level (generally too late anyway)
-        updateFromContext(skipFileFormatCrcCheck, ctx, "_SkipFileFormatCrcCheck", "@skipFileFormatCrcCheck");
+        // Note: skipFileFormatCrcCheck is not permitted at context level (generally too late anyway)
         updateFromContext(stripWhitespaceFromStoredDataset, ctx, "_StripWhitespaceFromStoredDataset", "@stripWhitespaceFromStoredDataset");
         updateFromContext(timeActivities, ctx, "@timeActivities", "_TimeActivities");
         updateFromContext(traceEnabled, ctx, "@traceEnabled", "_TraceEnabled");

roxie/ccd/ccdquery.hpp

@@ -287,6 +287,18 @@ public:
         // Default is no additional information
     }
 
+    unsigned getFormatCrc(unsigned helperCrc) const
+    {
+        if (queryFactory.queryOptions().skipFileFormatCrcCheck)
+            return 0;
+        else
+            return helperCrc;
+    }
+
+    IRecordLayoutTranslator::Mode getEnableFieldTranslation() const
+    {
+        return queryFactory.queryOptions().enableFieldTranslation;
+    }
 };
 
 extern void addXrefFileInfo(IPropertyTree &reply, const IResolvedFile *dataFile);

roxie/ccd/ccdserver.cpp

@@ -568,11 +568,6 @@ public:
         return NULL; // Activities that wish to support server-side caching will need to do better....
     }
 
-    virtual bool getEnableFieldTranslation() const
-    {
-        throwUnexpected(); // only implemented by index-related subclasses
-    }
-
     virtual IDefRecordMeta *queryActivityMeta() const
     {
         throwUnexpected(); // only implemented by index-related subclasses
@@ -1533,6 +1528,19 @@ public:
     {
         return (factory != NULL) && factory->isSink();
     }
+
+protected:
+    unsigned getFormatCrc(unsigned helperCrc) const
+    {
+        if (factory->queryQueryFactory().queryOptions().skipFileFormatCrcCheck)
+            return 0;
+        else
+            return helperCrc;
+    }
+    IRecordLayoutTranslator::Mode getEnableFieldTranslation() const
+    {
+        return factory->queryQueryFactory().queryOptions().enableFieldTranslation;
+    }
 };
 
 //=====================================================================================================
@@ -11681,6 +11689,13 @@ public:
         const char *recordECL = helper.queryRecordECL();
         if (recordECL && *recordECL)
             fileProps.setProp("ECL", recordECL);
+        if (helper.queryDiskRecordSize()->queryTypeInfo())
+        {
+            MemoryBuffer out;
+            if (dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo()))
+                fileProps.setPropBin("_rtlType", out.length(), out.toByteArray());
+        }
+
         fileProps.setProp("@kind", "flat"); // default, derivitives may override
     }
 
@@ -12022,8 +12037,8 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
         if (helper.queryDiskRecordSize()->queryTypeInfo())
         {
             MemoryBuffer out;
-            dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
-            metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
+            if (dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo()))
+                metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
         }
     }
 
@@ -12232,8 +12247,8 @@ public:
         if (helper.queryDiskRecordSize()->queryTypeInfo())
         {
             MemoryBuffer out;
-            dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo());
-            properties.setPropBin("_rtlType", out.length(), out.toByteArray());
+            if (dumpTypeInfo(out, helper.queryDiskRecordSize()->queryTypeInfo()))
+                properties.setPropBin("_rtlType", out.length(), out.toByteArray());
         }
     }
 
@@ -21421,10 +21436,9 @@ protected:
     unsigned __int64 rowLimit;
     unsigned __int64 stopAfter;
     Linked<IInMemoryIndexManager> manager;
+    Linked<ITranslatorSet> translators;
     Owned<IInMemoryIndexCursor> cursor;
     Owned<IDirectReader> reader;
-    CThorContiguousRowBuffer deserializeSource;
-    Owned<ISourceRowPrefetcher> prefetcher;
     bool eof;
     bool isKeyed;
     bool variableFileName;
@@ -21436,7 +21450,6 @@ protected:
     bool isGrouped = false;
     CachedOutputMetaData diskSize;
     Owned<const IResolvedFile> varFileInfo;
-    Owned<IFileIOArray> varFiles;
 
     inline bool useRemote()
     {
@@ -21446,16 +21459,17 @@ protected:
 public:
     IMPLEMENT_IINTERFACE_USING(CRoxieServerActivity)
 
-    CRoxieServerDiskReadBaseActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager)
+    CRoxieServerDiskReadBaseActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
+                                     unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
         : CRoxieServerActivity(_ctx, _factory, _probeManager),
           helper((IHThorDiskReadBaseArg &)basehelper),
           numParts(_numParts),
           remoteId(_remoteId),
           manager(_manager),
+          translators(_translators),
           isLocal(_isLocal),
           sorted(_sorted),
-          maySkip(_maySkip),
-          deserializeSource(NULL)
+          maySkip(_maySkip)
     {
         forceRemote = factory->queryQueryFactory().queryOptions().disableLocalOptimizations;
         if ((forceRemote || numParts != 1) && !isLocal)  // NOTE : when numParts == 0 (variable case) we create, even though we may not use
@@ -21465,13 +21479,8 @@ public:
         rowLimit = (unsigned __int64) -1;
         isKeyed = false;
         stopAfter = I64C(0x7FFFFFFFFFFFFFFF);
-        Linked<IOutputMetaData> diskMeta(helper.queryDiskRecordSize()->querySerializedDiskMeta());
-        if (diskMeta->isGrouped())
-        {
-            diskMeta.setown(new CSuffixedOutputMeta(+1, diskMeta.getClear()));
-            isGrouped = true;
-        }
-        diskSize.set(diskMeta);
+        diskSize.set(helper.queryDiskRecordSize()->querySerializedDiskMeta());
+        isGrouped = diskSize.isGrouped();
         variableFileName = allFilesDynamic || factory->queryQueryFactory().isDynamic() || ((helper.getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
         isOpt = (helper.getFlags() & TDRoptional) != 0;
     }
@@ -21508,26 +21517,7 @@ public:
                 varFileInfo.setown(resolveLFN(fileName, isOpt));
                 numParts = 0;
                 if (varFileInfo)
-                {
                     numParts = varFileInfo->getNumParts();
-                    const IPropertyTree *options =  varFileInfo->queryProperties();
-                    if (options)
-                    {
-                        isGrouped = options->getPropBool("@grouped");
-                        if (isGrouped && !diskSize.isGrouped())
-                        {
-                            // We are prepared to read contents of a grouped persist ungrouped... But not vice versa
-                            VStringBuffer msg("Published group information for file %s does not match coded information - assuming grouped", fileName.get());
-                            WARNLOG("%s", msg.str());
-                            ctx->queryCodeContext()->addWuException(msg.str(), ROXIE_LAYOUT_MISMATCH, SeverityError, "roxie");
-                            Owned<IOutputMetaData> diskMeta(new CSuffixedOutputMeta(+1, LINK(diskSize.queryOriginal())));
-                            diskSize.set(diskMeta);
-                        }
-                        size32_t dfsSize = options->getPropInt("@recordSize");
-                        if (dfsSize && diskSize.isFixedSize() && dfsSize != diskSize.getFixedSize())
-                            throw MakeStringException(ROXIE_LAYOUT_MISMATCH, "Published record size %d for file %s (%s) does not match coded record size %d", dfsSize, fileName.get(), isGrouped ? "grouped" : "ungrouped", diskSize.getFixedSize());
-                    }
-                }
             }
             if (!numParts)
             {
@@ -21548,8 +21538,9 @@ public:
                 if (variableFileName)
                 {
                     unsigned channel = isLocal ? factory->queryQueryFactory().queryChannel() : 0;
-                    varFiles.setown(varFileInfo->getIFileIOArray(isOpt, channel));
-                    manager.setown(varFileInfo->getIndexManager(isOpt, channel, varFiles, diskSize, false, 0));
+                    unsigned formatCrc = getFormatCrc(helper.getFormatCrc());
+                    translators.setown(varFileInfo->getTranslators(formatCrc, helper.queryProjectedDiskRecordSize(), helper.queryDiskRecordSize(), getEnableFieldTranslation()));
+                    manager.setown(varFileInfo->getIndexManager(isOpt, channel, nullptr, false, 0));
                 }
                 assertex(manager != NULL);
                 helper.createSegmentMonitors(this);
@@ -21560,9 +21551,7 @@ public:
                 }
                 if (!isKeyed)
                 {
-                    reader.setown(manager->createReader(0, 0, 1));
-                    deserializeSource.setStream(reader);
-                    prefetcher.setown(diskSize.queryOriginal()->createDiskPrefetcher(ctx->queryCodeContext(), activityId));
+                    reader.setown(manager->createReader(0, 0, 1, translators, ctx->queryCodeContext(), activityId));
                 }
                 helper.setCallback(reader ? reader->queryThorDiskCallback() : cursor);
             }
@@ -21618,7 +21607,7 @@ public:
         eof = false;
         if (cursor)
             cursor->reset();
-        deserializeSource.clearStream();
+        reader.clear();
         CRoxieServerActivity::reset();
     }
 
@@ -21687,8 +21676,8 @@ class CRoxieServerDiskReadActivity : public CRoxieServerDiskReadBaseActivity
     unsigned lastGroupProcessed;
 
 public:
-    CRoxieServerDiskReadActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager)
-        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, _maySkip, _manager)
+    CRoxieServerDiskReadActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, _maySkip, _manager, _translators)
     {
         compoundHelper = (IHThorDiskReadArg *)&helper;
         readHelper = (IHThorDiskReadArg *)&helper;
@@ -21832,26 +21821,18 @@ public:
             assertex(reader != NULL);
             for (;;)
             {
-                if (deserializeSource.eos())
+                if (reader->eos())
                 {
                     eof = true;
                     return NULL;
                 }
-                prefetcher->readAhead(deserializeSource);
-                const byte *nextRec = deserializeSource.queryRow();
+                const byte *nextRec = reader->nextRow();
                 if (cursor && cursor->isFiltered(nextRec))
                     transformedSize = 0;
                 else
                     transformedSize = readHelper->transform(rowBuilder, nextRec);
-                bool eog;
-                if (isGrouped)
-                {
-                    size32_t sizeRead = diskSize.getRecordSize(nextRec);
-                    eog = nextRec[sizeRead-1];
-                }
-                else
-                    eog = false;
-                deserializeSource.finishedRow();
+                bool eog = isGrouped && reader->eog();
+                reader->finishedRow();
                 if (transformedSize)
                 {
                     if (isGrouped)
@@ -21881,8 +21862,9 @@ class CRoxieServerXmlReadActivity : public CRoxieServerDiskReadBaseActivity, imp
     unsigned __int64 fileoffset;
 public:
     IMPLEMENT_IINTERFACE_USING(CRoxieServerDiskReadBaseActivity)
-    CRoxieServerXmlReadActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager)
-        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, _maySkip, _manager)
+    CRoxieServerXmlReadActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
+                                unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, _maySkip, _manager, _translators)
     {
         compoundHelper = NULL;
         readHelper = (IHThorXmlReadArg *)&helper;
@@ -22001,9 +21983,9 @@ class CRoxieServerCsvReadActivity : public CRoxieServerDiskReadBaseActivity
     size32_t maxRowSize;
 public:
     CRoxieServerCsvReadActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
-                                unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager,
+                                unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager, ITranslatorSet *_translators,
                                 const char *_quotes, const char *_separators, const char *_terminators, const char *_escapes, size32_t _maxRowSize)
-        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, _maySkip, _manager),
+        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, _maySkip, _manager, _translators),
           quotes(_quotes), separators(_separators), terminators(_terminators), escapes(_escapes), maxRowSize(_maxRowSize)
     {
         compoundHelper = NULL;
@@ -22138,8 +22120,8 @@ class CRoxieServerDiskNormalizeActivity : public CRoxieServerDiskReadBaseActivit
     bool firstPending;
 
 public:
-    CRoxieServerDiskNormalizeActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool _sorted, IInMemoryIndexManager *_manager)
-        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, false, _manager)
+    CRoxieServerDiskNormalizeActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool _sorted, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, false, _manager, _translators)
     {
         compoundHelper = (IHThorDiskNormalizeArg *)&helper;
         normalizeHelper = (IHThorDiskNormalizeArg *)&helper;
@@ -22193,19 +22175,18 @@ public:
             {
                 while (firstPending)
                 {
-                    if (deserializeSource.eos())
+                    if (reader->eos())
                     {
                         eof = true;
                         return NULL;
                     }
-                    prefetcher->readAhead(deserializeSource);
-                    const byte *nextRec = deserializeSource.queryRow();
+                    const byte *nextRec = reader->nextRow();
                     if (!cursor || !cursor->isFiltered(nextRec))
                     {
                         if (normalizeHelper->first(nextRec))
                             firstPending = false;
                     }
-                    deserializeSource.finishedRow();
+                    reader->finishedRow();
                 }
                 transformedSize = normalizeHelper->transform(rowBuilder);
                 firstPending = !normalizeHelper->next();
@@ -22236,8 +22217,8 @@ protected:
     bool done;
 
 public:
-    CRoxieServerDiskAggregateBaseActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager)
-        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, false, false, _manager),
+    CRoxieServerDiskAggregateBaseActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, false, false, _manager, _translators),
           done(false)
     {
     }
@@ -22278,8 +22259,8 @@ class CRoxieServerDiskCountActivity : public CRoxieServerDiskAggregateBaseActivi
     }
 
 public:
-    CRoxieServerDiskCountActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager)
-        : CRoxieServerDiskAggregateBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _manager),
+    CRoxieServerDiskCountActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieServerDiskAggregateBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _manager, _translators),
           countHelper((IHThorDiskCountArg &)basehelper)
     {
         choosenLimit = 0;
@@ -22351,15 +22332,14 @@ public:
                 else
                 {
                     assertex(reader != NULL);
-                    while (!deserializeSource.eos())
+                    while (!reader->eos())
                     {
-                        prefetcher->readAhead(deserializeSource);
-                        const byte *nextRec = deserializeSource.queryRow();
+                        const byte *nextRec = reader->nextRow();
                         if (!cursor || !cursor->isFiltered(nextRec))
                         {
                             totalCount += countHelper.numValid(nextRec);
                         }
-                        deserializeSource.finishedRow();
+                        reader->finishedRow();
                         if (totalCount > rowLimit)
                         {
                             totalCount = getSkippedCount();
@@ -22393,8 +22373,8 @@ class CRoxieServerDiskAggregateActivity : public CRoxieServerDiskAggregateBaseAc
 
 public:
     CRoxieServerDiskAggregateActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
-        unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager)
-        : CRoxieServerDiskAggregateBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _manager),
+        unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieServerDiskAggregateBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _manager, _translators),
           aggregateHelper((IHThorDiskAggregateArg &)basehelper)
     {
     }
@@ -22444,15 +22424,14 @@ public:
                 else
                 {
                     assertex(reader != NULL);
-                    while (!deserializeSource.eos())
+                    while (!reader->eos())
                     {
-                        prefetcher->readAhead(deserializeSource);
-                        const byte *nextRec = deserializeSource.queryRow();
+                        const byte *nextRec = reader->nextRow();
                         if (!cursor || !cursor->isFiltered(nextRec))
                         {
                             aggregateHelper.processRow(rowBuilder, nextRec);
                         }
-                        deserializeSource.finishedRow();
+                        reader->finishedRow();
                     }
                 }
             }
@@ -22479,8 +22458,8 @@ class CRoxieServerDiskGroupAggregateActivity : public CRoxieServerDiskAggregateB
     bool gathered;
 
 public:
-    CRoxieServerDiskGroupAggregateActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager)
-        : CRoxieServerDiskAggregateBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _manager),
+    CRoxieServerDiskGroupAggregateActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
+        : CRoxieServerDiskAggregateBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _manager, _translators),
           aggregateHelper((IHThorDiskGroupAggregateArg &)basehelper),
           resultAggregator(aggregateHelper, aggregateHelper),
           gathered(false)
@@ -22519,7 +22498,7 @@ public:
             {
                 Owned<IInMemoryFileProcessor> processor = isKeyed ?
                     createKeyedGroupAggregateRecordProcessor(cursor, resultAggregator, aggregateHelper) :
-                    createUnkeyedGroupAggregateRecordProcessor(cursor, resultAggregator, aggregateHelper, manager->createReader(0, 0, 1),
+                    createUnkeyedGroupAggregateRecordProcessor(cursor, resultAggregator, aggregateHelper, manager->createReader(0, 0, 1, translators, ctx->queryCodeContext(), activityId),
                                                                ctx->queryCodeContext(), activityId);
                 processor->doQuery(NULL, 0, 0, 0);
             }
@@ -22556,7 +22535,7 @@ public:
     bool sorted;
     bool maySkip;
     bool variableFileName;
-    Owned<IFileIOArray> files;
+    Owned<ITranslatorSet> translators;
     Owned<IInMemoryIndexManager> manager;
     Owned<const IResolvedFile> datafile;
     const char *quotes;
@@ -22579,34 +22558,15 @@ public:
             bool isOpt = (helper->getFlags() & TDRoptional) != 0;
             OwnedRoxieString fileName(helper->getFileName());
             datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
-            Linked<IOutputMetaData> diskMeta(helper->queryDiskRecordSize()->querySerializedDiskMeta());
-            if (diskMeta->isGrouped())
-                diskMeta.setown(new CSuffixedOutputMeta(+1, diskMeta.getClear()));
-            if (datafile)
-            {
-                const IPropertyTree *options =  datafile->queryProperties();
-                if (options)
-                {
-                    bool isGrouped = options->getPropBool("@grouped");
-                    if (isGrouped && !diskMeta->isGrouped())
-                    {
-                        // We are prepared to read contents of a grouped persist ungrouped... But not vice versa
-                        WARNLOG("Published group information for file %s does not match coded information - assuming grouped", fileName.get());
-                        diskMeta.setown(new CSuffixedOutputMeta(+1, diskMeta.getClear()));
-                    }
-                    size32_t dfsSize = options->getPropInt("@recordSize");
-                    if (dfsSize && diskMeta->isFixedSize() && dfsSize != diskMeta->getFixedSize())
-                        throw MakeStringException(ROXIE_LAYOUT_MISMATCH, "Published record size %d for file %s (%s) does not match coded record size %d", dfsSize, fileName.get(), isGrouped ? "grouped" : "ungrouped", diskMeta->getFixedSize());
-                }
-            }
             bool isSimple = (datafile && datafile->getNumParts()==1 && !_queryFactory.queryOptions().disableLocalOptimizations);
             if (isLocal || isSimple)
             {
                 if (datafile)
                 {
                     unsigned channel = isLocal ? queryFactory.queryChannel() : 0;
-                    files.setown(datafile->getIFileIOArray(isOpt, channel));
-                    manager.setown(datafile->getIndexManager(isOpt, channel, files, diskMeta, _graphNode.getPropBool("att[@name=\"preload\"]/@value", false), _graphNode.getPropInt("att[@name=\"_preloadSize\"]/@value", 0)));
+                    unsigned formatCrc = getFormatCrc(helper->getFormatCrc());
+                    translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
+                    manager.setown(datafile->getIndexManager(isOpt, channel, translators->queryActualLayout(0), _graphNode.getPropBool("att[@name=\"preload\"]/@value", false), _graphNode.getPropInt("att[@name=\"_preloadSize\"]/@value", 0)));
                     const IPropertyTree *options = datafile->queryProperties();
                     if (options)
                     {
@@ -22641,21 +22601,21 @@ public:
         switch (kind)
         {
         case TAKcsvread:
-            return new CRoxieServerCsvReadActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager,
+            return new CRoxieServerCsvReadActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager, translators,
                                                    quotes, separators, terminators, escapes, maxCsvRowSize);
         case TAKxmlread:
         case TAKjsonread:
-            return new CRoxieServerXmlReadActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager);
+            return new CRoxieServerXmlReadActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager, translators);
         case TAKdiskread:
-            return new CRoxieServerDiskReadActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager);
+            return new CRoxieServerDiskReadActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager, translators);
         case TAKdisknormalize:
-            return new CRoxieServerDiskNormalizeActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, manager);
+            return new CRoxieServerDiskNormalizeActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, manager, translators);
         case TAKdiskcount:
-            return new CRoxieServerDiskCountActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, manager);
+            return new CRoxieServerDiskCountActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, manager, translators);
         case TAKdiskaggregate:
-            return new CRoxieServerDiskAggregateActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, manager);
+            return new CRoxieServerDiskAggregateActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, manager, translators);
         case TAKdiskgroupaggregate:
-            return new CRoxieServerDiskGroupAggregateActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, manager);
+            return new CRoxieServerDiskGroupAggregateActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, manager, translators);
         }
         throwUnexpected();
     }
@@ -22726,7 +22686,7 @@ protected:
         if (varFileInfo)
         {
             translators.setown(new TranslatorArray) ;
-            keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().queryOptions().enableFieldTranslation));
+            keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0, getEnableFieldTranslation()));
         }
         variableInfoPending = false;
     }
@@ -23374,7 +23334,7 @@ class CRoxieServerSimpleIndexReadActivity : public CRoxieServerActivity, impleme
         OwnedRoxieString indexName(indexHelper.getFileName());
         varFileInfo.setown(resolveLFN(indexName, isOpt));
         translators.setown(new TranslatorArray) ;
-        keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().queryOptions().enableFieldTranslation));
+        keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0, getEnableFieldTranslation()));
         initKeySet();
         variableInfoPending = false;
     }
@@ -23738,7 +23698,6 @@ public:
     bool maySkip;
     bool sorted;
     bool variableFileName;
-    IRecordLayoutTranslator::Mode enableFieldTranslation;
     unsigned maxSeekLookahead;
     Owned<const IResolvedFile> indexfile;
 
@@ -23761,7 +23720,6 @@ public:
         }
         else
             assertex(indexLayoutSize==0);
-        enableFieldTranslation = queryFactory.queryOptions().enableFieldTranslation;
         translatorArray.setown(new TranslatorArray);
         variableFileName = allFilesDynamic || _queryFactory.isDynamic() || ((flags & (TIRvarfilename|TIRdynamicfilename)) != 0);
         if (!variableFileName)
@@ -23770,7 +23728,7 @@ public:
             OwnedRoxieString indexName(indexHelper->getFileName());
             indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
             if (indexfile)
-                keySet.setown(indexfile->getKeyArray(activityMeta, translatorArray, isOpt, isLocal ? queryFactory.queryChannel() : 0, enableFieldTranslation));
+                keySet.setown(indexfile->getKeyArray(activityMeta, translatorArray, isOpt, isLocal ? queryFactory.queryChannel() : 0, getEnableFieldTranslation()));
         }
         isSimple = isLocal;
         maySkip = (flags & (TIRkeyedlimitskips|TIRlimitskips|TIRlimitcreates|TIRkeyedlimitcreates)) != 0;
@@ -23818,11 +23776,6 @@ public:
         return cache;
     }
 
-    virtual bool getEnableFieldTranslation() const
-    {
-        return enableFieldTranslation; 
-    }
-
     virtual IDefRecordMeta *queryActivityMeta() const
     {
         return activityMeta;
@@ -25394,7 +25347,7 @@ public:
             if (varFileInfo)
             {
                 translators.setown(new TranslatorArray);
-                keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, false, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().queryOptions().enableFieldTranslation)); // MORE - isLocal?
+                keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, false, isLocal ? factory->queryQueryFactory().queryChannel() : 0, getEnableFieldTranslation()));
             }
         }
         puller.start(parentExtractSize, parentExtract, paused, ctx->queryOptions().fullKeyedJoinPreload, false, ctx);
@@ -26245,7 +26198,7 @@ public:
             if (varFileInfo)
             {
                 translators.setown(new TranslatorArray);
-                keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, false, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().queryOptions().enableFieldTranslation));
+                keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, false, isLocal ? factory->queryQueryFactory().queryChannel() : 0, getEnableFieldTranslation()));
             }
         }
         puller.start(parentExtractSize, parentExtract, paused, ctx->queryOptions().keyedJoinPreload, isSimple, ctx);
@@ -26450,11 +26403,11 @@ class CRoxieServerKeyedJoinActivityFactory : public CRoxieServerMultiInputFactor
     RemoteActivityId tailId;
     IOutputMetaData *indexReadMeta;
     Owned<IFilePartMap> map;
+    Owned<ITranslatorSet> translators;
     Owned<IFileIOArray> files;
     unsigned joinFlags;
     bool isHalfKeyed;
     bool isLocal;
-    IRecordLayoutTranslator::Mode enableFieldTranslation;
     bool variableFetchFileName;
     bool variableIndexFileName;
     bool isSimple;
@@ -26473,7 +26426,6 @@ public:
         MemoryBuffer m;
         m.setBuffer(indexLayoutSize, indexLayoutMeta.getdata());
         activityMeta.setown(deserializeRecordMeta(m, true));
-        enableFieldTranslation = queryFactory.queryOptions().enableFieldTranslation;
         translatorArray.setown(new TranslatorArray);
         joinFlags = helper->getJoinFlags();
         variableIndexFileName = allFilesDynamic || _queryFactory.isDynamic() || ((joinFlags & (JFvarindexfilename|JFdynamicindexfilename|JFindexfromactivity)) != 0);
@@ -26484,7 +26436,7 @@ public:
             OwnedRoxieString indexFileName(helper->getIndexFileName());
             indexfile.setown(queryFactory.queryPackage().lookupFileName(indexFileName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
             if (indexfile)
-                keySet.setown(indexfile->getKeyArray(activityMeta, translatorArray, isOpt, isLocal ? queryFactory.queryChannel() : 0, enableFieldTranslation));
+                keySet.setown(indexfile->getKeyArray(activityMeta, translatorArray, isOpt, isLocal ? queryFactory.queryChannel() : 0, getEnableFieldTranslation()));
         }
         if (keySet && keySet->length()==1 && !isSimple)
         {
@@ -26505,18 +26457,17 @@ public:
             if (datafile)
             {
                 if (isLocal)
+                {
+                    unsigned formatCrc = getFormatCrc(helper->getDiskFormatCrc());
+                    translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
                     files.setown(datafile->getIFileIOArray(isFetchOpt, queryFactory.queryChannel()));
+                }
                 else
                     map.setown(datafile->getFileMap());
             }
         }
     }
 
-    virtual bool getEnableFieldTranslation() const
-    {
-        return enableFieldTranslation;
-    }
-
     virtual IDefRecordMeta *queryActivityMeta() const
     {
         return activityMeta;

roxie/ccd/ccdstate.cpp

@@ -2212,7 +2212,11 @@ private:
                 const char *val = control->queryProp("@val");
                 if (val)
                 {
-                    if (strieq(val, "payload"))
+                    if (strieq(val, "alwaysDisk"))
+                        fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAlwaysDisk;
+                    else if (strieq(val, "alwaysECL"))
+                        fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAlwaysECL;
+                    else if (strieq(val, "payload"))
                         fieldTranslationEnabled = IRecordLayoutTranslator::TranslatePayload;
                     else if (!val || strToBool(val))
                         fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAll;
@@ -2553,6 +2557,10 @@ private:
                 }
                 reply.appendf("<State hash='%" I64F "u' topologyHash='%" I64F "u'/>", shash, thash);
             }
+            else if (stricmp(queryName, "control:resetcache")==0)
+            {
+                releaseSlaveDynamicFileCache();
+            }
             else if (stricmp(queryName, "control:resetindexmetrics")==0)
             {
                 resetIndexMetrics();

roxie/ccd/ccdstate.hpp

@@ -84,16 +84,30 @@ interface ISlaveDynamicFileCache : extends IInterface
 extern ISlaveDynamicFileCache *querySlaveDynamicFileCache();
 extern void releaseSlaveDynamicFileCache();
 
+interface IDynamicTransform;
+
 interface IFileIOArray : extends IInterface
 {
     virtual bool IsShared() const = 0;
-    virtual IFileIO *getFilePart(unsigned partNo, offset_t &base) = 0;
-    virtual unsigned length() = 0;
-    virtual unsigned numValid() = 0;
-    virtual bool isValid(unsigned partNo) = 0;
-    virtual unsigned __int64 size() = 0;
+    virtual IFileIO *getFilePart(unsigned partNo, offset_t &base) const = 0;
+    virtual unsigned length() const = 0;
+    virtual unsigned numValid() const = 0;
+    virtual bool isValid(unsigned partNo) const = 0;
+    virtual unsigned __int64 size() const = 0;
     virtual StringBuffer &getId(StringBuffer &) const = 0;
-    virtual const char *queryLogicalFilename(unsigned partNo) = 0;
+    virtual const char *queryLogicalFilename(unsigned partNo) const = 0;
+    virtual int queryActualFormatCrc() const = 0;    // Actual format on disk
+    virtual bool allFormatsMatch() const = 0;  // i.e. not a superfile with mixed formats
+    virtual unsigned getSubFile(unsigned partNo) const = 0;
+};
+
+interface ITranslatorSet : extends IInterface
+{
+    virtual const IDynamicTransform *queryTranslator(unsigned subFile) const = 0;
+    virtual ISourceRowPrefetcher *getPrefetcher(unsigned subFile, bool addGroupedFlag, ICodeContext *ctx, unsigned actId) const = 0;
+    virtual IOutputMetaData *queryActualLayout(unsigned subFile) const = 0;
+    virtual int queryTargetFormatCrc() const = 0;
+    virtual bool isTranslating() const = 0;
 };
 
 interface IRoxieQuerySetManagerSet : extends IInterface

rtl/eclrtl/eclhelper_dyn.cpp

@@ -40,20 +40,25 @@
 class CDeserializedOutputMetaData : public COutputMetaData
 {
 public:
-    CDeserializedOutputMetaData(MemoryBuffer &binInfo, IThorIndexCallback *callback);
+    CDeserializedOutputMetaData(MemoryBuffer &binInfo, bool isGrouped, IThorIndexCallback *callback);
     CDeserializedOutputMetaData(IPropertyTree &jsonInfo, IThorIndexCallback *callback);
     CDeserializedOutputMetaData(const char *json, IThorIndexCallback *callback);
 
     virtual const RtlTypeInfo * queryTypeInfo() const override { return typeInfo; }
+    virtual unsigned getMetaFlags() override { return flags; }
 protected:
     Owned<IRtlFieldTypeDeserializer> deserializer;
     const RtlTypeInfo *typeInfo = nullptr;
+    unsigned flags = MDFhasserialize|MDFhasxml;
+
 };
 
-CDeserializedOutputMetaData::CDeserializedOutputMetaData(MemoryBuffer &binInfo, IThorIndexCallback *callback)
+CDeserializedOutputMetaData::CDeserializedOutputMetaData(MemoryBuffer &binInfo, bool isGrouped, IThorIndexCallback *callback)
 {
     deserializer.setown(createRtlFieldTypeDeserializer(callback));
     typeInfo = deserializer->deserialize(binInfo);
+    if (isGrouped)
+        flags |= MDFgrouped;
 }
 
 CDeserializedOutputMetaData::CDeserializedOutputMetaData(IPropertyTree &jsonInfo, IThorIndexCallback *callback)
@@ -68,9 +73,9 @@ CDeserializedOutputMetaData::CDeserializedOutputMetaData(const char *json, IThor
     typeInfo = deserializer->deserialize(json);
 }
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &binInfo, IThorIndexCallback *callback)
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &binInfo, bool isGrouped, IThorIndexCallback *callback)
 {
-    return new CDeserializedOutputMetaData(binInfo, callback);
+    return new CDeserializedOutputMetaData(binInfo, isGrouped, callback);
 }
 
 extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonInfo, IThorIndexCallback *callback)
@@ -350,7 +355,9 @@ static IOutputMetaData *loadTypeInfo(IPropertyTree &xgmml, const char *key)
     MemoryBuffer binInfo;
     xgmml.getPropBin(xpath.setf("att[@name='%s_binary']/value", key), binInfo);
     assertex(binInfo.length());
-    return new CDeserializedOutputMetaData(binInfo, nullptr);
+    bool grouped = xgmml.getPropBool(xpath.setf("att[@name='%s_binary']/value", key), false);
+
+    return new CDeserializedOutputMetaData(binInfo, grouped, nullptr);
 }
 
 extern ECLRTL_API IHThorDiskReadArg *createDiskReadArg(IPropertyTree &xgmml)

rtl/eclrtl/eclhelper_dyn.hpp

@@ -22,9 +22,9 @@
 #include "eclrtl.hpp"
 #include "eclhelper.hpp"
 
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &mb, IThorIndexCallback *callback=nullptr);
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonTree, IThorIndexCallback *callback=nullptr);
-extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json, IThorIndexCallback *callback=nullptr);
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(MemoryBuffer &mb, bool isGroupedPersist, IThorIndexCallback *callback);
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(IPropertyTree &jsonTree, IThorIndexCallback *callback);
+extern ECLRTL_API IOutputMetaData *createTypeInfoOutputMetaData(const char *json, IThorIndexCallback *callback);
 
 interface IDynamicIndexReadArg
 {

rtl/eclrtl/rtlcommon.hpp

@@ -12,9 +12,10 @@
 class ECLRTL_API CThorContiguousRowBuffer : implements IRowPrefetcherSource
 {
 public:
+    CThorContiguousRowBuffer() {};
     CThorContiguousRowBuffer(ISerialStream * _in);
 
-    inline void setStream(ISerialStream *_in) { in.set(_in); maxOffset = 0; readOffset = 0; }
+    inline void setStream(ISerialStream *_in) { in = _in; maxOffset = 0; readOffset = 0; }
 
     virtual const byte * peek(size32_t maxSize) override;
     virtual offset_t beginNested() override;
@@ -50,13 +51,13 @@ public:
 
     inline void clearStream()
     {
-        in.clear();
+        in = nullptr;
         maxOffset = 0;
         readOffset = 0;
     }
 
-    inline const byte * queryRow() { return buffer; }
-    inline size32_t queryRowSize() { return readOffset; }
+    inline const byte * queryRow() const { return buffer; }
+    inline size32_t queryRowSize() const { return readOffset; }
     inline void finishedRow()
     {
         if (readOffset)
@@ -65,6 +66,14 @@ public:
         readOffset = 0;
     }
 
+    inline void reset(offset_t offset, offset_t flen = (offset_t)-1)
+    {
+        in->reset(offset, flen);
+        buffer = nullptr;
+        maxOffset = 0;
+        readOffset = 0;
+    }
+
 
 protected:
     size32_t sizePackedInt();
@@ -91,10 +100,10 @@ private:
     }
 
 protected:
-    Linked<ISerialStream> in;
-    const byte * buffer;
-    size32_t maxOffset;
-    size32_t readOffset;
+    ISerialStream* in = nullptr;
+    const byte * buffer = nullptr;
+    size32_t maxOffset = 0;
+    size32_t readOffset = 0;
     UnsignedArray childStartOffsets;
 };
 

rtl/eclrtl/rtldynfield.cpp

@@ -82,6 +82,10 @@ const RtlTypeInfo *FieldTypeInfoStruct::createRtlTypeInfo(IThorIndexCallback *_c
         assert(childType);
         ret = new RtlDatasetTypeInfo(fieldType, length, childType);
         break;
+    case type_dictionary:
+        assert(childType);
+        ret = new RtlDictionaryTypeInfo(fieldType, length, childType);
+        break;
     case type_set:
         assert(childType);
         ret = new RtlSetTypeInfo(fieldType, length, childType);
@@ -779,9 +783,18 @@ extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInf
     return CRtlFieldTypeSerializer::serialize(ret, t);
 }
 
-extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
+extern ECLRTL_API bool dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
 {
-    return CRtlFieldTypeBinSerializer::serialize(ret, t);
+    try
+    {
+        CRtlFieldTypeBinSerializer::serialize(ret, t);
+        return true;
+    }
+    catch (IException *E)
+    {
+        EXCLOG(E);
+        return false;
+    }
 }
 
 extern ECLRTL_API void serializeRecordType(size32_t & __lenResult, void * & __result, IOutputMetaData &  metaVal)
@@ -937,6 +950,7 @@ private:
         unsigned numOffsets = sourceRecInfo.getNumVarFields() + 1;
         size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
         byte * destConditions = (byte *)alloca(destRecInfo.getNumIfBlocks() * sizeof(byte));
+        memset(destConditions, 2, destRecInfo.getNumIfBlocks() * sizeof(byte));
         RtlRow sourceRow(sourceRecInfo, sourceRec, numOffsets, variableOffsets);
         size32_t estimate = destRecInfo.getFixedSize();
         if (!estimate)
@@ -1380,8 +1394,9 @@ private:
             for (unsigned idx = 0; idx < sourceRecInfo.getNumFields(); idx++)
             {
                 const RtlFieldInfo *field = sourceRecInfo.queryField(idx);
+                const char *name = sourceRecInfo.queryName(idx);
                 const RtlTypeInfo *type = field->type;
-                if (destRecInfo.getFieldNum(field->name) == (unsigned) -1)
+                if (destRecInfo.getFieldNum(name) == (unsigned) -1)
                 {
                     // unmatched field
                     if (type->isFixedSize())

rtl/eclrtl/rtldynfield.hpp

@@ -115,7 +115,7 @@ extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer(ITho
 
 extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t);
 
-extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t);
+extern ECLRTL_API bool dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t);
 
 /**
  * Serialize metadata of supplied record to JSON, and return it to ECL caller as a string. Used for testing serializer.

rtl/eclrtl/rtlrecord.cpp

@@ -112,9 +112,10 @@ static unsigned countFields(const RtlFieldInfo * const * fields, bool & contains
 class IfBlockInfo
 {
 public:
-    IfBlockInfo(const RtlFieldInfo &_field, const IfBlockInfo *_parent, unsigned _idx)
-    : field(_field), parent(_parent), idx(_idx)
-    {}
+    IfBlockInfo(const RtlFieldInfo &_field, const IfBlockInfo *_parent, unsigned _idx, unsigned _startIdx)
+    : field(_field), parent(_parent), idx(_idx), startIdx(_startIdx)
+    {
+    }
 
     bool excluded(const byte *row, byte *conditions) const
     {
@@ -130,10 +131,12 @@ public:
         }
         return conditions[idx]==0;
     }
+    inline unsigned queryStartField() const { return startIdx; }
 private:
     const RtlFieldInfo &field;
     const IfBlockInfo *parent = nullptr;  // for nested ifblocks
     unsigned idx;
+    unsigned startIdx; // For ifblocks inside child records
 };
 
 class RtlCondFieldStrInfo : public RtlFieldStrInfo
@@ -149,7 +152,7 @@ public:
     const IfBlockInfo &ifblock;
 };
 
-static unsigned expandNestedRows(unsigned idx, const char *prefix, const RtlFieldInfo * const * fields, const RtlFieldInfo * * target, const char * *names, const IfBlockInfo *inIfBlock, ConstPointerArrayOf<IfBlockInfo> &ifblocks)
+static unsigned expandNestedRows(unsigned idx, unsigned startIdx, const char *prefix, const RtlFieldInfo * const * fields, const RtlFieldInfo * * target, const char * *names, const IfBlockInfo *inIfBlock, ConstPointerArrayOf<IfBlockInfo> &ifblocks)
 {
     for (;*fields;fields++)
     {
@@ -161,7 +164,7 @@ static unsigned expandNestedRows(unsigned idx, const char *prefix, const RtlFiel
             const IfBlockInfo *nestIfBlock = inIfBlock;
             if (isIfBlock)
             {
-                nestIfBlock = new IfBlockInfo(*cur, inIfBlock, ifblocks.ordinality());
+                nestIfBlock = new IfBlockInfo(*cur, inIfBlock, ifblocks.ordinality(), startIdx);
                 ifblocks.append(nestIfBlock);
             }
             const RtlFieldInfo * const * nested = type->queryFields();
@@ -170,7 +173,7 @@ static unsigned expandNestedRows(unsigned idx, const char *prefix, const RtlFiel
                 StringBuffer newPrefix(prefix);
                 if (cur->name && *cur->name)
                     newPrefix.append(cur->name).append('.');
-                idx = expandNestedRows(idx, newPrefix.str(), nested, target, names, nestIfBlock, ifblocks);
+                idx = expandNestedRows(idx, isIfBlock ? startIdx : idx, newPrefix.str(), nested, target, names, nestIfBlock, ifblocks);
             }
         }
         else
@@ -237,7 +240,7 @@ RtlRecord::RtlRecord(const RtlFieldInfo * const *_fields, bool expandFields) : f
             const RtlFieldInfo * * allocated  = new const RtlFieldInfo * [numFields+1];
             names = new const char *[numFields];
             fields = allocated;
-            unsigned idx = expandNestedRows(0, nullptr, originalFields, allocated, names, nullptr, _ifblocks);
+            unsigned idx = expandNestedRows(0, 0, nullptr, originalFields, allocated, names, nullptr, _ifblocks);
             ifblocks = _ifblocks.detach();
             assertex(idx == numFields);
             allocated[idx] = nullptr;
@@ -376,7 +379,11 @@ void RtlRecord::calcRowOffsets(size_t * variableOffsets, const void * _row, unsi
             if (field->flags & RFTMinifblock)
             {
                 const RtlCondFieldStrInfo *condfield = static_cast<const RtlCondFieldStrInfo *>(field);
-                if (condfield->ifblock.excluded(row, conditions))
+                unsigned startField = condfield->ifblock.queryStartField();
+                const byte *childRow = row;
+                if (startField)
+                    childRow += getOffset(variableOffsets, startField);
+                if (condfield->ifblock.excluded(childRow, conditions))
                 {
                     variableOffsets[i+1] = offset; // (meaning size ends up as zero);
                     continue;
@@ -433,7 +440,7 @@ size32_t RtlRecord::deserialize(ARowBuilder & rowBuilder, IRowDeserializerSource
         size32_t fixedSize = fixedOffsets[fieldIndex];
         byte * self = rowBuilder.ensureCapacity(offset + fixedSize, ""); // Why not field->name?
         in.read(fixedSize, self + offset);
-        if (field->omitable() && static_cast<const RtlCondFieldStrInfo *>(field)->ifblock.excluded(self, conditionValues))
+        if (excluded(field, self, conditionValues))
             continue;
         offset = queryType(fieldIndex)->deserialize(rowBuilder, in, offset);
     }
@@ -514,9 +521,16 @@ const RtlFieldInfo *RtlRecord::queryOriginalField(unsigned idx) const
 }
 
 
-bool RtlRecord::excluded(const RtlFieldInfo *field, const byte *row, byte *conditionValues)
+bool RtlRecord::excluded(const RtlFieldInfo *field, const byte *row, byte *conditionValues) const
 {
-    return (field->omitable() && static_cast<const RtlCondFieldStrInfo *>(field)->ifblock.excluded(row, conditionValues));
+    if (!field->omitable())
+        return false;
+    const RtlCondFieldStrInfo *condfield = static_cast<const RtlCondFieldStrInfo *>(field);
+    unsigned startField = condfield->ifblock.queryStartField();
+    const byte *childRow = row;
+    if (startField)
+        childRow += calculateOffset(row, startField);
+    return condfield->ifblock.excluded(childRow, conditionValues);
 }
 
 size_t RtlRecord::getFixedOffset(unsigned field) const
@@ -527,36 +541,57 @@ size_t RtlRecord::getFixedOffset(unsigned field) const
 
 size32_t RtlRecord::getRecordSize(const void *_row) const
 {
-    size32_t size = getFixedSize();
-    if (!size)
+    if (numIfBlocks)
     {
-        const byte * row = static_cast<const byte *>(_row);
-        size32_t varoffset = 0;
-        for (unsigned i = 0; i < numVarFields; i++)
+        unsigned numOffsets = getNumVarFields() + 1;
+        size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
+        RtlRow sourceRow(*this, _row, numOffsets, variableOffsets);
+        return sourceRow.getOffset(numFields+1);
+    }
+    else
+    {
+        size32_t size = getFixedSize();
+        if (!size)
         {
-            unsigned fieldIndex = variableFieldIds[i];
-            size32_t offset = fixedOffsets[fieldIndex] + varoffset;
-            size32_t fieldSize = queryType(fieldIndex)->size(row + offset, row);
-            varoffset = offset + fieldSize;
+            const byte * row = static_cast<const byte *>(_row);
+            size32_t varoffset = 0;
+            for (unsigned i = 0; i < numVarFields; i++)
+            {
+                unsigned fieldIndex = variableFieldIds[i];
+                size32_t offset = fixedOffsets[fieldIndex] + varoffset;
+                size32_t fieldSize = queryType(fieldIndex)->size(row + offset, row);
+                varoffset = offset + fieldSize;
+            }
+            size = fixedOffsets[numFields] + varoffset;
         }
-        size = fixedOffsets[numFields] + varoffset;
+        return size;
     }
-    return size;
 }
 
 size32_t RtlRecord::calculateOffset(const void *_row, unsigned field) const
 {
-    const byte * row = static_cast<const byte *>(_row);
-    size32_t varoffset = 0;
-    unsigned varFields = whichVariableOffset[field];
-    for (unsigned i = 0; i < varFields; i++)
+    if (numIfBlocks)
     {
-        unsigned fieldIndex = variableFieldIds[i];
-        size32_t offset = fixedOffsets[fieldIndex] + varoffset;
-        size32_t fieldSize = queryType(fieldIndex)->size(row + offset, row);
-        varoffset = offset + fieldSize;
+        unsigned numOffsets = getNumVarFields() + 1;
+        size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
+        RtlRow sourceRow(*this, nullptr, numOffsets, variableOffsets);
+        sourceRow.setRow(_row, field);
+        return sourceRow.getOffset(field);
+    }
+    else
+    {
+        const byte * row = static_cast<const byte *>(_row);
+        size32_t varoffset = 0;
+        unsigned varFields = whichVariableOffset[field];
+        for (unsigned i = 0; i < varFields; i++)
+        {
+            unsigned fieldIndex = variableFieldIds[i];
+            size32_t offset = fixedOffsets[fieldIndex] + varoffset;
+            size32_t fieldSize = queryType(fieldIndex)->size(row + offset, row);
+            varoffset = offset + fieldSize;
+        }
+        return fixedOffsets[field] + varoffset;
     }
-    return fixedOffsets[field] + varoffset;
 }
 
 //---------------------------------------------------------------------------------------------------------------------
@@ -631,7 +666,23 @@ void RtlRow::setRow(const void * _row)
 {
     row = (const byte *)_row;
     if (_row)
+    {
         info.calcRowOffsets(variableOffsets, _row);
+#if defined(_DEBUG) && defined(TRACE_ROWOFFSETS)
+        for (unsigned i = 0; i < info.getNumFields(); i++)
+        {
+            printf("Field %d (%s) offset %d", i, info.queryName(i), (int) getOffset(i));
+            if (getSize(i))
+            {
+                unsigned bufflen;
+                rtlDataAttr buff;
+                getString(bufflen, buff.refstr(), i);
+                printf(" value %.*s", bufflen, buff.getstr());
+            }
+            printf("\n");
+        }
+#endif
+    }
 }
 
 void RtlRow::setRow(const void * _row, unsigned _numFields)

rtl/eclrtl/rtlrecord.hpp

@@ -230,14 +230,14 @@ public:
     inline unsigned getNumFields() const { return numFields; }
     unsigned getNumKeyedFields() const;
     inline unsigned getNumVarFields() const { return numVarFields; }
-    inline unsigned getNumIfBlocks() const { return numIfBlocks > 0; }
+    inline unsigned getNumIfBlocks() const { return numIfBlocks; }
     inline const RtlFieldInfo * queryField(unsigned field) const { return fields[field]; }
     const RtlFieldInfo * queryOriginalField(unsigned field) const;
     inline const RtlTypeInfo * queryType(unsigned field) const { return fields[field]->type; }
     const char * queryName(unsigned field) const;
     unsigned getFieldNum(const char *fieldName) const;
     const RtlRecord *queryNested(unsigned field) const;
-    static bool excluded(const RtlFieldInfo *field, const byte *row, byte *conditions);
+    bool excluded(const RtlFieldInfo *field, const byte *row, byte *conditions) const;
 protected:
     size_t * fixedOffsets;         // fixed portion of the field offsets + 1 extra
     unsigned * whichVariableOffset;// which variable offset should be added to the fixed

system/jlib/jarray.hpp

@@ -417,6 +417,11 @@ class OwnedPointerArrayOf : public ArrayOf<INTERFACE *, INTERFACE *, OwnedPointe
 {
 };
 
+template <typename INTERFACE>
+class OwnedConstPointerArrayOf : public ArrayOf<const INTERFACE *, const INTERFACE *, OwnedPointerArrayMapper<const INTERFACE> >
+{
+};
+
 //--------------------------------------------------------------------------------------------------------------------
 
 template <typename ITEM>

system/jlib/jlib.hpp

@@ -44,6 +44,7 @@
 class jlib_decl ICopyArray : public CopyReferenceArrayOf<IInterface> {};
 class jlib_decl IArray : public OwnedReferenceArrayOf<IInterface> {};
 class jlib_decl IPointerArray : public OwnedPointerArrayOf<IInterface> {};
+class jlib_decl IConstPointerArray : public OwnedConstPointerArrayOf<IInterface> {};
 
 class jlib_decl CICopyArray : public CopyReferenceArrayOf<CInterface> {};
 class jlib_decl CIArray : public OwnedReferenceArrayOf<CInterface> {};
@@ -190,6 +191,25 @@ public:
     inline bool zap(TYPE * obj, bool nodel=false) { return IPointerArray::zap(obj, nodel); }
 };
 
+template <class BTYPE>
+class IConstPointerArrayOf : public IConstPointerArray
+{
+    typedef const BTYPE TYPE;
+public:
+    inline TYPE * item(aindex_t pos) const        { return (TYPE *)IConstPointerArray::item(pos); }
+    inline TYPE * popGet()                        { return (TYPE *)IConstPointerArray::popGet(); }
+    inline TYPE * tos(void) const                 { return (TYPE *)IConstPointerArray::tos(); }
+    inline TYPE * tos(aindex_t num) const         { return (TYPE *)IConstPointerArray::tos(num); }
+    inline TYPE **getArray(aindex_t pos = 0)      { return (TYPE **)IConstPointerArray::getArray(pos); }
+    inline TYPE **detach()                        { return (TYPE **)IConstPointerArray::detach(); }
+    inline void append(TYPE * obj)                { IConstPointerArray::append(obj); }
+    inline void appendUniq(TYPE * obj)            { IConstPointerArray::appendUniq(obj); }
+    inline void add(TYPE * obj, aindex_t pos)     { IConstPointerArray::add(obj, pos); }
+    inline aindex_t find(TYPE * obj) const        { return IConstPointerArray::find(obj); }
+    inline void replace(TYPE * obj, aindex_t pos, bool nodel=false) { IConstPointerArray::replace(obj, pos, nodel); }
+    inline bool zap(TYPE * obj, bool nodel=false) { return IConstPointerArray::zap(obj, nodel); }
+};
+
 template <class TYPE>
 class PointerArrayOf : public PointerArray
 {

system/jlib/jthread.cpp

@@ -2379,7 +2379,9 @@ IWorkQueueThread *createWorkQueueThread(unsigned persisttime)
 
 unsigned threadLogID()  // for use in logging
 {
-#ifndef _WIN32
+#if defined(__APPLE__)
+     return pthread_mach_thread_np(pthread_self());
+#elif !defined(_WIN32)
 #ifdef SYS_gettid
     return (unsigned) (memsize_t) syscall(SYS_gettid);
 #endif

testing/regress/ecl/setup/files.ecl

@@ -50,9 +50,15 @@ SHARED STRING _indexPrefix := '~regress::'+ IF(multiPart AND useLocal, 'local',
                              IF(multiPart, 'multi',
                              'single')) + '::' + EmptyString;
 
-EXPORT filePrefix := #IFDEFINED(root.filePrefix, _filePrefix);
-EXPORT indexPrefix := #IFDEFINED(root.filePrefix, _indexPrefix);
-        
+SHARED forceLayoutTranslation := #IFDEFINED(root.forceLayoutTranslation, 0);
+#IF (forceLayoutTranslation != 0)
+  SHARED setLayout := #option('layoutTranslationEnabled', CASE(forceLayoutTranslation,1=>v'alwaysECL',2=>v'alwaysDisk',v''));
+  EXPORT filePrefix := WHEN(#IFDEFINED(root.filePrefix, _filePrefix), setLayout);
+  EXPORT indexPrefix := WHEN(#IFDEFINED(root.filePrefix, _indexPrefix), setLayout);
+#else      
+  EXPORT filePrefix := #IFDEFINED(root.filePrefix, _filePrefix);
+  EXPORT indexPrefix := #IFDEFINED(root.filePrefix, _indexPrefix);
+#end
 EXPORT DG_FileOut              := filePrefix + 'DG_';
 EXPORT DG_IndexOut             := indexPrefix + 'DG_';
 EXPORT DG_ParentFileOut        := filePrefix + 'DG_Parent.d00';

testing/regress/ecl/toxml.ecl

@@ -16,6 +16,10 @@
 ############################################################################## */
 
 //UseStandardFiles
+//version forceLayoutTranslation=0
+//version forceLayoutTranslation=1
+// it would be nice to also add version forceLayoutTranslation=2, but can't until ifblock condition metadata serialization is done properly
+
 import $.setup;
 prefix := setup.Files(false, false).FilePrefix;
 

testing/regress/ecl/translateFields2.ecl

@@ -0,0 +1,76 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+// Test various record translations using ifblocks
+
+phonerecord := RECORD
+  string5 areacode;
+  string7 number;
+END;
+
+contactrecord := RECORD
+  phonerecord phone;
+  boolean hasemail;
+  IFBLOCK(SELF.hasemail)
+    string email;
+    boolean hasSecondEmail;
+    IFBLOCK(SELF.hasSecondEmail)
+      string secondEmail;
+    END;
+  END;
+END;
+
+secretaryRecord := RECORD
+  boolean hasSecretary;
+  IFBLOCK(Self.hasSecretary)
+    contactRecord secretaryContact;
+  END;
+END;
+
+mainrec := RECORD
+  string surname;
+  string forename;
+  phonerecord homephone;
+  boolean hasmobile;
+  IFBLOCK(SELF.hasmobile)
+    phonerecord mobilephone;
+  END;
+  contactrecord contact;
+  secretaryRecord secretary;
+  string2 endmarker;
+END;
+
+outrec := RECORD
+  mainrec;
+  string1 endmarker2 { default('$')};
+END;
+
+d := dataset([
+  {'Hallidayaaaaa','Richard','01526','1234567',false,'01943','7654321',false,false,'$$'},
+  {'Halliday','Richard','01526','1234567',false,'01943','7654321',true,'me@home',false,false,'$$'},
+  {'Holliday','Richard','01526','1234567',false,'01943','7654321',true,'me@home',true,'me@work',false,'$$'},
+  {'Hallidayaaaaa','Richard','01526','1234567',false,'01943','7654321',false,true,'123','456',false,'$$'},
+  {'Halliday','Richard','01526','1234567',false,'01943','7654321',false,true,'123','456',true,'secretary@office.com',false,'$$'},
+  {'Holliday','Richard','01526','1234567',false,'01943','7654321',true,'me@home',true,'me@work',true,'123','456',true,'secretary@office.com',true,'secretary@home','$$'}
+  ], mainrec);
+d;
+
+s := SERVICE
+   streamed dataset(outrec) stransform(streamed dataset input) : eclrtl,pure,library='eclrtl',entrypoint='transformRecord',passParameterMeta(true);
+END;
+
+s.stransform(d);

thorlcr/activities/indexwrite/thindexwrite.cpp

@@ -181,8 +181,8 @@ public:
         if (helper->queryDiskRecordSize()->queryTypeInfo())
         {
             MemoryBuffer out;
-            dumpTypeInfo(out, helper->queryDiskRecordSize()->queryTypeInfo());
-            props.setPropBin("_rtlType", out.length(), out.toByteArray());
+            if (dumpTypeInfo(out, helper->queryDiskRecordSize()->queryTypeInfo()))
+                props.setPropBin("_rtlType", out.length(), out.toByteArray());
         }
         mpTag = container.queryJob().allocateMPTag();
         mpTag2 = container.queryJob().allocateMPTag();

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -207,8 +207,8 @@ public:
         if (helper->queryDiskRecordSize()->queryTypeInfo())
         {
             MemoryBuffer out;
-            dumpTypeInfo(out, helper->queryDiskRecordSize()->queryTypeInfo());
-            metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
+            if (dumpTypeInfo(out, helper->queryDiskRecordSize()->queryTypeInfo()))
+                metadata->setPropBin("_rtlType", out.length(), out.toByteArray());
         }
     }
 

thorlcr/activities/thdiskbase.cpp

@@ -28,6 +28,7 @@
 
 #include "eclhelper.hpp" // tmp for IHThorArg interface
 #include "thdiskbase.ipp"
+#include "rtldynfield.hpp"
 
 CDiskReadMasterBase::CDiskReadMasterBase(CMasterGraphElement *info) : CMasterActivity(info), diskStats(info->queryJob(), diskReadRemoteStatistics)
 {
@@ -190,6 +191,13 @@ void CWriteMasterBase::init()
         const char *rececl= diskHelperBase->queryRecordECL();
         if (rececl&&*rececl)
             props.setProp("ECL", rececl);
+        if (diskHelperBase->queryDiskRecordSize()->queryTypeInfo())
+        {
+            MemoryBuffer out;
+            if (dumpTypeInfo(out, diskHelperBase->queryDiskRecordSize()->queryTypeInfo()))
+                props.setPropBin("_rtlType", out.length(), out.toByteArray());
+        }
+
         bool blockCompressed=false;
         void *ekey;
         size32_t ekeylen;

tools/dumpkey/dumpkey.cpp

@@ -188,7 +188,7 @@ int main(int argc, const char **argv)
                 {
                     MemoryBuffer layoutBin;
                     metadata->getPropBin("_rtlType", layoutBin);
-                    diskmeta.setown(createTypeInfoOutputMetaData(layoutBin, &callback));
+                    diskmeta.setown(createTypeInfoOutputMetaData(layoutBin, false, &callback));
                 }
                 if (diskmeta)
                 {