HPCC-17271 Add options to layout translation

Add an additional option "payload" in addition to the "true" or
"false" options for field translation.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/layouttrans.cpp

@@ -501,7 +501,7 @@ IRecordLayoutTranslator::RowTransformContext::~RowTransformContext()
     delete [] ptrs;
 }
 
-CRecordLayoutTranslator::CRecordLayoutTranslator(IDefRecordMeta const * _diskMeta, IDefRecordMeta const * _activityMeta) : diskMeta(const_cast<IDefRecordMeta *>(_diskMeta)), activityMeta(const_cast<IDefRecordMeta *>(_activityMeta)), activityKeySizes(NULL)
+CRecordLayoutTranslator::CRecordLayoutTranslator(IDefRecordMeta const * _diskMeta, IDefRecordMeta const * _activityMeta, IRecordLayoutTranslator::Mode _mode) : diskMeta(const_cast<IDefRecordMeta *>(_diskMeta)), activityMeta(const_cast<IDefRecordMeta *>(_activityMeta)), activityKeySizes(NULL)
 {
     numKeyedDisk = diskMeta->numKeyedFields();
     numKeyedActivity = activityMeta->numKeyedFields();
@@ -516,6 +516,8 @@ CRecordLayoutTranslator::CRecordLayoutTranslator(IDefRecordMeta const * _diskMet
         topMappingLevel.calculateMappings(diskMeta->queryRecord(), numKeyedDisk, activityMeta->queryRecord(), numKeyedActivity);
         calculateActivityKeySizes();
         calculateKeysTransformed();
+        if (keysTransformed && _mode != TranslateAll)
+            throw makeFailure(IRecordLayoutTranslator::Failure::KeyedDisallowed)->append("Translation of key fields would be required");
         transformer.build(numTransformers, mappings);
     }
     catch(Failure * f)
@@ -543,7 +545,7 @@ void CRecordLayoutTranslator::calculateActivityKeySizes()
 
 void CRecordLayoutTranslator::calculateKeysTransformed()
 {
-    keysTransformed = true;;
+    keysTransformed = true;
     if(numKeyedActivity != numKeyedDisk)
         return;
     for(unsigned diskFieldNum=0; diskFieldNum<numKeyedDisk; ++diskFieldNum)
@@ -651,9 +653,9 @@ void ExpandedSegmentMonitorList::setMergeBarrier(unsigned offset)
     // MORE - It's possible that I need to do something here??
 }
 
-IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta)
+IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta, IRecordLayoutTranslator::Mode _mode)
 {
-    Owned<IRecordLayoutTranslator> layoutTrans = new CRecordLayoutTranslator(diskMeta, activityMeta);
+    Owned<IRecordLayoutTranslator> layoutTrans = new CRecordLayoutTranslator(diskMeta, activityMeta, _mode);
     if(!layoutTrans->querySuccess())
     {
         StringBuffer cause;
@@ -663,7 +665,7 @@ IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * di
     return layoutTrans.getClear();
 };
 
-extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(size32_t diskMetaSize, const void *diskMetaData, size32_t activityMetaSize, const void *activityMetaData)
+extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(size32_t diskMetaSize, const void *diskMetaData, size32_t activityMetaSize, const void *activityMetaData, IRecordLayoutTranslator::Mode _mode)
 {
     MemoryBuffer activityMetaSerialized;
     activityMetaSerialized.setBuffer(activityMetaSize, (void *) activityMetaData, false);
@@ -673,7 +675,7 @@ extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(siz
     diskMetaSerialized.setBuffer(diskMetaSize, (void *) diskMetaData, false);
     Owned<IDefRecordMeta> diskMeta = deserializeRecordMeta(diskMetaSerialized, true);
 
-    return createRecordLayoutTranslator(diskMeta, activityMeta);
+    return createRecordLayoutTranslator(diskMeta, activityMeta, _mode);
 }
 
 #ifdef DEBUG_HELPERS_REQUIRED
@@ -720,21 +722,21 @@ StringBuffer & CRecordLayoutTranslator::getMappingsAsString(StringBuffer & out)
 
 #endif
 
-CacheKey::CacheKey(size32_t _s1, void const * _d1, size32_t _s2, void const * _d2)
-    : s1(_s1), d1(static_cast<byte const *>(_d1)), s2(_s2), d2(static_cast<byte const *>(_d2))
+CacheKey::CacheKey(IRecordLayoutTranslator::Mode _mode, size32_t _s1, void const * _d1, size32_t _s2, void const * _d2)
+    : mode(_mode), s1(_s1), d1(static_cast<byte const *>(_d1)), s2(_s2), d2(static_cast<byte const *>(_d2))
 {
-    hashval = hashc(d1, s1, 0);
+    hashval = hashc(d1, s1, (unsigned) mode);
     hashval = hashc(d2, s2, hashval);
 }
 
-CacheValue::CacheValue(size32_t s1, void const * d1, size32_t s2, void const * d2, IRecordLayoutTranslator * _trans)
-    : b1(s1, d1), b2(s2, d2), key((size32_t)b1.length(), b1.get(), (size32_t)b2.length(), b2.get()), trans(_trans)
+CacheValue::CacheValue(IRecordLayoutTranslator::Mode _mode, size32_t s1, void const * d1, size32_t s2, void const * d2, IRecordLayoutTranslator * _trans)
+    : b1(s1, d1), b2(s2, d2), key(_mode, (size32_t)b1.length(), b1.get(), (size32_t)b2.length(), b2.get()), trans(_trans)
 {
 }
 
-IRecordLayoutTranslator * CRecordLayoutTranslatorCache::get(size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta)
+IRecordLayoutTranslator * CRecordLayoutTranslatorCache::get(IRecordLayoutTranslator::Mode mode, size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta)
 {
-    CacheKey key(diskMetaSize, diskMetaData, activityMetaSize, activityMetaData);
+    CacheKey key(mode, diskMetaSize, diskMetaData, activityMetaSize, activityMetaData);
     CacheValue * value = find(&key);
     if(!value)
     {
@@ -751,9 +753,9 @@ IRecordLayoutTranslator * CRecordLayoutTranslatorCache::get(size32_t diskMetaSiz
         diskMetaSerialized.setBuffer(diskMetaSize, (void *) diskMetaData, false);
         Owned<IDefRecordMeta> diskMeta = deserializeRecordMeta(diskMetaSerialized, true);
 
-        Owned<IRecordLayoutTranslator> trans = createRecordLayoutTranslator(diskMeta, activityMeta);
+        Owned<IRecordLayoutTranslator> trans = createRecordLayoutTranslator(diskMeta, activityMeta, mode);
 
-        value = new CacheValue(diskMetaSize, diskMetaData, activityMetaSize, activityMetaData, trans.getLink());
+        value = new CacheValue(mode, diskMetaSize, diskMetaData, activityMetaSize, activityMetaData, trans.getLink());
         addNew(value);
     }
     return value->getTranslator();
@@ -868,11 +870,11 @@ public:
         Owned<IRecordLayoutTranslatorCache> cache = createRecordLayoutTranslatorCache();
         CPPUNIT_ASSERT(cache.get() != 0);
         CPPUNIT_ASSERT(cache->count() == 0);
-        Owned<IRecordLayoutTranslator> t1 = cache->get(buff[0].length(), buff[0].bufferBase(), buff[1].length(), buff[1].bufferBase(), NULL);
+        Owned<IRecordLayoutTranslator> t1 = cache->get(IRecordLayoutTranslator::TranslateAll, buff[0].length(), buff[0].bufferBase(), buff[1].length(), buff[1].bufferBase(), NULL);
         CPPUNIT_ASSERT(cache->count() == 1);
-        Owned<IRecordLayoutTranslator> t2 = cache->get(buff[0].length(), buff[0].bufferBase(), buff[1].length(), buff[1].bufferBase(), NULL);
+        Owned<IRecordLayoutTranslator> t2 = cache->get(IRecordLayoutTranslator::TranslateAll, buff[0].length(), buff[0].bufferBase(), buff[1].length(), buff[1].bufferBase(), NULL);
         CPPUNIT_ASSERT(cache->count() == 1);
-        Owned<IRecordLayoutTranslator> t3 = cache->get(buff[0].length(), buff[0].bufferBase(), buff[2].length(), buff[2].bufferBase(), NULL);
+        Owned<IRecordLayoutTranslator> t3 = cache->get(IRecordLayoutTranslator::TranslateAll, buff[0].length(), buff[0].bufferBase(), buff[2].length(), buff[2].bufferBase(), NULL);
         CPPUNIT_ASSERT(cache->count() == 2);
         CPPUNIT_ASSERT(t1.get() == t2.get());
         CPPUNIT_ASSERT(t1.get() != t3.get());
@@ -1114,14 +1116,14 @@ private:
 
     void doTranslate(unsigned disk, unsigned activity)
     {
-        Owned<IRecordLayoutTranslator> trans = new CRecordLayoutTranslator(&meta.item(disk), &meta.item(activity));
+        Owned<IRecordLayoutTranslator> trans = new CRecordLayoutTranslator(&meta.item(disk), &meta.item(activity), IRecordLayoutTranslator::TranslateAll);
         CPPUNIT_ASSERT(trans.get() != NULL);
         CPPUNIT_ASSERT(trans->querySuccess());
     }
     
     void doTranslateFail(unsigned disk, unsigned activity, unsigned code)
     {
-        Owned<IRecordLayoutTranslator> trans = new CRecordLayoutTranslator(&meta.item(disk), &meta.item(activity));
+        Owned<IRecordLayoutTranslator> trans = new CRecordLayoutTranslator(&meta.item(disk), &meta.item(activity), IRecordLayoutTranslator::TranslateAll);
         CPPUNIT_ASSERT(trans.get() != 0);
         CPPUNIT_ASSERT(!trans->querySuccess());
         CPPUNIT_ASSERT(trans->queryFailure().queryCode() == code);

common/thorhelper/layouttrans.hpp

@@ -20,7 +20,6 @@
 
 #include "deffield.hpp"
 #include "rtlkey.hpp"
-#include "jhtree.hpp"
 #include "thorhelper.hpp"
 
 #ifdef _DEBUG
@@ -33,7 +32,7 @@ public:
     class THORHELPER_API Failure : public CInterface
     {
     public:
-        typedef enum { BadStructure = 1, MissingDiskField, UnkeyedDiskField, UntranslatableField, UnsupportedFilter } Code;
+        typedef enum { BadStructure = 1, MissingDiskField, UnkeyedDiskField, UntranslatableField, UnsupportedFilter, KeyedDisallowed } Code;
         
         virtual Code queryCode() const = 0;
         virtual StringBuffer & getDetail(StringBuffer & out) const = 0;
@@ -66,6 +65,8 @@ public:
         offset_t fposIn;
     };
 
+    typedef enum { NoTranslation = 0, TranslateAll = 1, TranslatePayload = 2 } Mode;
+
     virtual bool querySuccess() const = 0;
     virtual Failure const & queryFailure() const = 0;
     virtual void checkSizes(char const * filename, size32_t activitySize, size32_t diskSize) const = 0;
@@ -79,13 +80,13 @@ public:
 #endif
 };
 
-extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta);
-extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(size32_t diskMetaSize, const void *diskMetaData, size32_t activityMetaSize, const void *activityMetaData);
+extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta, IRecordLayoutTranslator::Mode _mode);
+extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(size32_t diskMetaSize, const void *diskMetaData, size32_t activityMetaSize, const void *activityMetaData, IRecordLayoutTranslator::Mode _mode);
 
 class THORHELPER_API IRecordLayoutTranslatorCache : public IInterface
 {
 public:
-    virtual IRecordLayoutTranslator * get(size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta = NULL) = 0; //if activityMeta is NULL, it is calculated by deserializing from buffer --- but option to supply so caller can deserialize once and use many
+    virtual IRecordLayoutTranslator * get(IRecordLayoutTranslator::Mode mode, size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta = NULL) = 0; //if activityMeta is NULL, it is calculated by deserializing from buffer --- but option to supply so caller can deserialize once and use many
     virtual unsigned count() const = 0;
 };
 

common/thorhelper/layouttrans.ipp

@@ -191,7 +191,7 @@ private:
 class CRecordLayoutTranslator : public IRecordLayoutTranslator, public CInterface
 {
 public:
-    CRecordLayoutTranslator(IDefRecordMeta const * _diskMeta, IDefRecordMeta const * _activityMeta);
+    CRecordLayoutTranslator(IDefRecordMeta const * _diskMeta, IDefRecordMeta const * _activityMeta, IRecordLayoutTranslator::Mode _mode);
     ~CRecordLayoutTranslator() { delete [] activityKeySizes; }
     IMPLEMENT_IINTERFACE;
     virtual bool querySuccess() const { return !failure; }
@@ -229,10 +229,11 @@ private:
 class CacheKey
 {
 public:
-    CacheKey(size32_t _s1, void const * _d1, size32_t _s2, void const * _d2);
+    CacheKey(IRecordLayoutTranslator::Mode mode, size32_t _s1, void const * _d1, size32_t _s2, void const * _d2);
     unsigned getHash() const { return hashval; }
-    bool operator==(CacheKey const & other) const { return((s1 == other.s1) && (s2 == other.s2) && (memcmp(d1, other.d1, s1) == 0) && (memcmp(d2, other.d2, s2) == 0)); }
+    bool operator==(CacheKey const & other) const { return((mode == other.mode) && (s1 == other.s1) && (s2 == other.s2) && (memcmp(d1, other.d1, s1) == 0) && (memcmp(d2, other.d2, s2) == 0)); }
 private:
+    IRecordLayoutTranslator::Mode mode;
     size32_t s1;
     byte const * d1;
     size32_t s2;
@@ -243,7 +244,7 @@ private:
 class CacheValue
 {
 public:
-    CacheValue(size32_t s1, void const * d1, size32_t s2, void const * d2, IRecordLayoutTranslator * _trans);
+    CacheValue(IRecordLayoutTranslator::Mode _mode, size32_t s1, void const * d1, size32_t s2, void const * d2, IRecordLayoutTranslator * _trans);
     CacheKey const & queryKey() const { return key; }
     IRecordLayoutTranslator * getTranslator() const { return trans.getLink(); }
 private:
@@ -260,7 +261,7 @@ class CRecordLayoutTranslatorCache : public CacheTable, public IRecordLayoutTran
 public:
     IMPLEMENT_IINTERFACE;
     virtual ~CRecordLayoutTranslatorCache() { _releaseAll(); }
-    virtual IRecordLayoutTranslator * get(size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta = NULL);
+    virtual IRecordLayoutTranslator * get(IRecordLayoutTranslator::Mode mode, size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta = NULL);
     virtual unsigned count() const { return CacheTable::count(); }
 
 private:

common/workunit/package.h

@@ -21,6 +21,7 @@
 #include "errorlist.h"
 #include "dadfs.hpp"
 #include "workunit.hpp"
+#include "layouttrans.hpp"
 
 // error codes
 #define PACKAGE_TARGET_NOT_FOUND      PACKAGE_ERROR_START
@@ -37,7 +38,7 @@ interface IHpccPackage : extends IInterface
     virtual const char *locateSuperFile(const char *superFileName) const = 0;
 
     virtual const char *queryEnv(const char *varname) const = 0;
-    virtual bool getEnableFieldTranslation() const = 0;
+    virtual IRecordLayoutTranslator::Mode getEnableFieldTranslation() const = 0;
     virtual bool isCompulsory() const = 0;
     virtual bool isPreload() const = 0;
     virtual const IPropertyTree *queryTree() const = 0;

common/workunit/pkgimpl.hpp

@@ -118,13 +118,20 @@ protected:
         return (node) ? node->getPropBool("@resolveLocally", false) : true;  // default is false for explicit package files, but true for the default empty package
     }
 
-    virtual bool getSysFieldTranslationEnabled() const {return false;}
-    virtual bool getEnableFieldTranslation() const
+    virtual IRecordLayoutTranslator::Mode getSysFieldTranslationEnabled() const { return IRecordLayoutTranslator::NoTranslation; }
+    virtual IRecordLayoutTranslator::Mode getEnableFieldTranslation() const
     {
         const char *val = queryEnv("control:enableFieldTranslation");
         if (!val) val = queryEnv("enableFieldTranslation"); // Backward compatibility
         if (val)
-            return strToBool(val);
+        {
+            if (strieq(val, "payload"))
+                return IRecordLayoutTranslator::TranslatePayload;
+            else if (strToBool(val))
+                return IRecordLayoutTranslator::TranslateAll;
+            else
+                return IRecordLayoutTranslator::NoTranslation;
+        }
         else
             return getSysFieldTranslationEnabled();
     }

ecl/eclagent/eclagent.cpp

@@ -40,6 +40,7 @@
 #include "workunit.hpp"
 #include "eventqueue.hpp"
 #include "schedulectrl.hpp"
+#include "jhtree.hpp"
 
 #include "mpbase.hpp"
 #include "daclient.hpp"

ecl/hthor/hthorkey.cpp

@@ -90,7 +90,7 @@ bool rltEnabled(IConstWorkUnit const * wu)
         return wu->getDebugValueBool("hthorLayoutTranslationEnabled", false);
 }
 
-IRecordLayoutTranslator * getRecordLayoutTranslator(IDefRecordMeta const * activityMeta, size32_t activityMetaSize, void const * activityMetaBuff, IDistributedFile * df, IRecordLayoutTranslatorCache * cache)
+IRecordLayoutTranslator * getRecordLayoutTranslator(IDefRecordMeta const * activityMeta, size32_t activityMetaSize, void const * activityMetaBuff, IDistributedFile * df, IRecordLayoutTranslatorCache * cache, IRecordLayoutTranslator::Mode mode)
 {
     IPropertyTree const & props = df->queryAttributes();
     MemoryBuffer diskMetaBuff;
@@ -107,9 +107,9 @@ IRecordLayoutTranslator * getRecordLayoutTranslator(IDefRecordMeta const * activ
     try
     {
         if(cache)
-            return cache->get(diskMetaBuff.length(), diskMetaBuff.bufferBase(), activityMetaSize, activityMetaBuff, activityMeta);
+            return cache->get(mode, diskMetaBuff.length(), diskMetaBuff.bufferBase(), activityMetaSize, activityMetaBuff, activityMeta);
         else
-            return createRecordLayoutTranslator(diskMetaBuff.length(), diskMetaBuff.bufferBase(), activityMetaSize, activityMetaBuff);
+            return createRecordLayoutTranslator(diskMetaBuff.length(), diskMetaBuff.bufferBase(), activityMetaSize, activityMetaBuff, mode);
     }
     catch (IException *E)
     {
@@ -742,7 +742,7 @@ IRecordLayoutTranslator * CHThorIndexReadActivityBase::getLayoutTranslator(IDist
         activityRecordMeta.setown(deserializeRecordMeta(buff, true));
     }
 
-    return getRecordLayoutTranslator(activityRecordMeta, activityRecordMetaSize, activityRecordMetaBuff, f, agent.queryRecordLayoutTranslatorCache());
+    return getRecordLayoutTranslator(activityRecordMeta, activityRecordMetaSize, activityRecordMetaBuff, f, agent.queryRecordLayoutTranslatorCache(), IRecordLayoutTranslator::TranslateAll);
 }
 
 void CHThorIndexReadActivityBase::verifyIndex(IKeyIndex * idx)
@@ -4062,7 +4062,7 @@ protected:
             activityRecordMeta.setown(deserializeRecordMeta(buff, true));
         }
 
-        return getRecordLayoutTranslator(activityRecordMeta, activityRecordMetaSize, activityRecordMetaBuff, f, agent.queryRecordLayoutTranslatorCache());
+        return getRecordLayoutTranslator(activityRecordMeta, activityRecordMetaSize, activityRecordMetaBuff, f, agent.queryRecordLayoutTranslatorCache(), IRecordLayoutTranslator::TranslateAll);
     }
 
     virtual void verifyIndex(IDistributedFile * f, IKeyIndex * idx, IRecordLayoutTranslator * trans)

esp/services/ws_dfu/CMakeLists.txt

@@ -49,6 +49,7 @@ include_directories (
          ./../../../rtl/eclrtl 
          ./../../../rtl/nbcd
          ./../../../common/environment 
+         ./../../../common/thorhelper 
          ./../../services 
          ./../../../dali/ft 
          ./../common 

esp/services/ws_packageprocess/CMakeLists.txt

@@ -38,6 +38,9 @@ include_directories (
          ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
          ${HPCC_SOURCE_DIR}/dali/base
          ${HPCC_SOURCE_DIR}/common/environment
+         ${HPCC_SOURCE_DIR}/common/deftype
+         ${HPCC_SOURCE_DIR}/common/thorhelper
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl
          ${HPCC_SOURCE_DIR}/dali/dfu
          ${HPCC_SOURCE_DIR}/common/remote
          ${HPCC_SOURCE_DIR}/common/workunit

esp/services/ws_workunits/CMakeLists.txt

@@ -54,6 +54,7 @@ include_directories (
          ./../../../common/remote
          ./../../../system/jlib
          ./../../../common/environment
+         ./../../../rtl/eclrtl
          ./../../../roxie/roxie
          ./../../services
          ./../common

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -578,12 +578,19 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="fieldTranslationEnabled" type="xs:boolean" use="optional" default="false">
+    <xs:attribute name="fieldTranslationEnabled" use="optional" default="false">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Enables translation (where possible) of mismatched index formats on-the-fly</tooltip>
+          <tooltip>Enables translation (where possible) of mismatched index layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only</tooltip>
         </xs:appinfo>
       </xs:annotation>
+      <xs:simpleType>
+        <xs:restriction base="xs:string">
+          <xs:enumeration value="false"/>
+          <xs:enumeration value="true"/>
+          <xs:enumeration value="payload"/>
+        </xs:restriction>
+      </xs:simpleType>
     </xs:attribute>
     <xs:attribute name="highTimeout" type="xs:nonNegativeInteger" use="optional" default="2000">
       <xs:annotation>

roxie/ccd/ccd.hpp

@@ -31,6 +31,7 @@
 #include "roxiedebug.ipp"
 #include "eclrtl.hpp"
 #include "workunit.hpp"
+#include "layouttrans.hpp"
 
 #ifdef CCD_EXPORTS
 #define CCD_API DECL_EXPORT
@@ -417,7 +418,7 @@ struct PartNoType
 extern unsigned statsExpiryTime;
 extern time_t startupTime;
 extern unsigned miscDebugTraceLevel;
-extern bool fieldTranslationEnabled;
+extern IRecordLayoutTranslator::Mode fieldTranslationEnabled;
 
 extern unsigned defaultParallelJoinPreload;
 extern unsigned defaultConcatPreload;

roxie/ccd/ccdactivities.cpp

@@ -156,7 +156,7 @@ public:
         return ret.appendf("%p", this);
     }
 
-    bool getEnableFieldTranslation() const
+    IRecordLayoutTranslator::Mode getEnableFieldTranslation() const
     {
         return queryFactory.queryOptions().enableFieldTranslation;
     }
@@ -308,7 +308,7 @@ protected:
     bool resent;
     bool isOpt;
     bool variableFileName;
-    bool allowFieldTranslation;
+    IRecordLayoutTranslator::Mode allowFieldTranslation;
     Owned<const IResolvedFile> varFileInfo;
 
     virtual void setPartNo(bool filechanged) = 0;

roxie/ccd/ccdfile.cpp

@@ -1962,7 +1962,7 @@ public:
         }
         return f.getClear();
     }
-    virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, bool allowFieldTranslation) const
+    virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, IRecordLayoutTranslator::Mode allowFieldTranslation) const override
     {
         unsigned maxParts = 0;
         ForEachItemIn(subFile, subFiles)
@@ -1981,8 +1981,8 @@ public:
             if (translators)
             {
                 if (fdesc && thisDiskMeta && activityMeta && !thisDiskMeta->equals(activityMeta))
-                    if (allowFieldTranslation)
-                        translators->append(createRecordLayoutTranslator(lfn, thisDiskMeta, activityMeta));
+                    if (allowFieldTranslation != IRecordLayoutTranslator::NoTranslation)
+                        translators->append(createRecordLayoutTranslator(lfn, thisDiskMeta, activityMeta, allowFieldTranslation));
                     else
                     {
                         DBGLOG("Key layout mismatch: %s", lfn.get());

roxie/ccd/ccdfile.hpp

@@ -21,6 +21,7 @@
 #include "eclhelper.hpp"
 #include "ccddali.hpp"
 #include "dautils.hpp"
+#include "layouttrans.hpp"
 
 enum RoxieFileStatus { FileSizeMismatch, FileDateMismatch, FileCRCMismatch, FileIsValid, FileNotFound };
 enum RoxieFileType { ROXIE_KEY, ROXIE_FILE, ROXIE_PATCH, ROXIE_BASEINDEX };
@@ -91,7 +92,7 @@ interface IResolvedFile : extends ISimpleSuperFileEnquiry
     virtual void serializePartial(MemoryBuffer &mb, unsigned channel, bool localInfoOnly) const = 0;
 
     virtual IFileIOArray *getIFileIOArray(bool isOpt, unsigned channel) const = 0;
-    virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, bool allowFieldTranslation) const = 0;
+    virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, IRecordLayoutTranslator::Mode allowFieldTranslation) const = 0;
     virtual IFilePartMap *getFileMap() const = 0;
     virtual unsigned getNumParts() const = 0;
     virtual IInMemoryIndexManager *getIndexManager(bool isOpt, unsigned channel, IFileIOArray *files, IRecordSize *recs, bool preload, int numKeys) const = 0;

roxie/ccd/ccdmain.cpp

@@ -78,7 +78,7 @@ unsigned defaultTraceLimit = 10;
 unsigned watchActivityId = 0;
 unsigned testSlaveFailure = 0;
 unsigned restarts = 0;
-bool fieldTranslationEnabled = false;
+IRecordLayoutTranslator::Mode fieldTranslationEnabled = IRecordLayoutTranslator::NoTranslation;
 bool mergeSlaveStatistics = true;
 PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
 bool runOnce = false;
@@ -798,7 +798,15 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         coresPerQuery = topology->getPropInt("@coresPerQuery", 0);
 
         diskReadBufferSize = topology->getPropInt("@diskReadBufferSize", 0x10000);
-        fieldTranslationEnabled = topology->getPropBool("@fieldTranslationEnabled", false);
+        fieldTranslationEnabled = IRecordLayoutTranslator::NoTranslation;
+        const char *val = topology->queryProp("@fieldTranslationEnabled");
+        if (val)
+        {
+            if (strieq(val, "payload"))
+                fieldTranslationEnabled = IRecordLayoutTranslator::TranslatePayload;
+            else if (strToBool(val))
+                fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAll;
+        }
 
         pretendAllOpt = topology->getPropBool("@ignoreMissingFiles", false);
         memoryStatsInterval = topology->getPropInt("@memoryStatsInterval", 60);

roxie/ccd/ccdquery.cpp

@@ -397,6 +397,21 @@ void QueryOptions::updateFromWorkUnit(bool &value, IConstWorkUnit &wu, const cha
     value = wu.getDebugValueBool(name, value);
 }
 
+void QueryOptions::updateFromWorkUnit(IRecordLayoutTranslator::Mode &value, IConstWorkUnit &wu, const char *name)
+{
+    SCMStringBuffer val;
+    wu.getDebugValue(name, val);
+    if (val.length())
+    {
+        if (strieq(val.str(), "payload"))
+            value = IRecordLayoutTranslator::TranslatePayload;
+        else if (strToBool(val.str()))
+            value = IRecordLayoutTranslator::TranslateAll;
+        else
+            value = IRecordLayoutTranslator::NoTranslation;
+    }
+}
+
 void QueryOptions::setFromContext(const IPropertyTree *ctx)
 {
     if (ctx)
@@ -1971,11 +1986,11 @@ extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned
     return createSlaveQueryFactory(wu->queryWuid(), dll.getClear(), queryRootRoxiePackage(), channelNo, NULL, true, false);  // MORE - if use a constant for id might cache better?
 }
 
-IRecordLayoutTranslator * createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta)
+IRecordLayoutTranslator * createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta, IRecordLayoutTranslator::Mode mode)
 {
     try
     {
-        return ::createRecordLayoutTranslator(diskMeta, activityMeta);
+        return ::createRecordLayoutTranslator(diskMeta, activityMeta, mode);
     }
     catch (IException *E)
     {

roxie/ccd/ccdquery.hpp

@@ -116,7 +116,7 @@ public:
 
     bool checkingHeap;
     bool disableLocalOptimizations;
-    bool enableFieldTranslation;
+    IRecordLayoutTranslator::Mode enableFieldTranslation;
     bool skipFileFormatCrcCheck;
     bool stripWhitespaceFromStoredDataset;
     bool timeActivities;
@@ -130,6 +130,7 @@ private:
     static void updateFromWorkUnit(int &value, IConstWorkUnit &wu, const char *name);
     static void updateFromWorkUnit(unsigned &value, IConstWorkUnit &wu, const char *name);
     static void updateFromWorkUnit(bool &value, IConstWorkUnit &wu, const char *name);
+    static void updateFromWorkUnit(IRecordLayoutTranslator::Mode &value, IConstWorkUnit &wu, const char *name);
     static void updateFromContextM(memsize_t &val, const IPropertyTree *ctx, const char *name, const char *name2 = NULL); // Needs different name to ensure works in 32-bit where memsize_t and unsigned are same type
     static void updateFromContext(int &val, const IPropertyTree *ctx, const char *name, const char *name2 = NULL);
     static void updateFromContext(unsigned &val, const IPropertyTree *ctx, const char *name, const char *name2 = NULL);
@@ -299,7 +300,7 @@ extern const IQueryDll *createQueryDll(const char *dllName);
 extern const IQueryDll *createExeQueryDll(const char *exeName);
 extern const IQueryDll *createWuQueryDll(IConstWorkUnit *wu);
 
-extern IRecordLayoutTranslator *createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta);
+extern IRecordLayoutTranslator *createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta, IRecordLayoutTranslator::Mode _mode);
 extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo, bool isDynamic, bool forceRetry);
 extern IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned _channelNo, const IPropertyTree *stateInfo, bool isDynamic, bool forceRetry);
 extern IQueryFactory *getQueryFactory(hash64_t hashvalue, unsigned channel);

roxie/ccd/ccdserver.cpp

@@ -23600,7 +23600,7 @@ public:
     bool maySkip;
     bool sorted;
     bool variableFileName;
-    bool enableFieldTranslation;
+    IRecordLayoutTranslator::Mode enableFieldTranslation;
     unsigned maxSeekLookahead;
     Owned<const IResolvedFile> indexfile;
 
@@ -24722,7 +24722,7 @@ public:
                     bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isIndexOpt']/@value");
                     indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
                     if (indexfile)
-                        keySet.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, isLocal ? queryFactory.queryChannel() : 0, false));
+                        keySet.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, isLocal ? queryFactory.queryChannel() : 0, IRecordLayoutTranslator::NoTranslation));
                 }
                 if (fileName && !allFilesDynamic && !queryFactory.isDynamic())
                 {
@@ -26289,7 +26289,7 @@ class CRoxieServerKeyedJoinActivityFactory : public CRoxieServerMultiInputFactor
     unsigned joinFlags;
     bool isHalfKeyed;
     bool isLocal;
-    bool enableFieldTranslation;
+    IRecordLayoutTranslator::Mode enableFieldTranslation;
     bool variableFetchFileName;
     bool variableIndexFileName;
     bool isSimple;

roxie/ccd/ccdstate.cpp

@@ -431,7 +431,7 @@ protected:
     virtual aindex_t getBaseCount() const = 0;
     virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
 
-    virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
+    virtual IRecordLayoutTranslator::Mode getSysFieldTranslationEnabled() const { return fieldTranslationEnabled; } //roxie configured value
 
     // Use local package file only to resolve subfile into physical file info
     IResolvedFile *resolveLFNusingPackage(const char *fileName) const
@@ -598,7 +598,7 @@ protected:
     void doPreload(unsigned channel, const IResolvedFile *resolved)
     {
         if (resolved->isKey())
-            keyArrays.append(*resolved->getKeyArray(NULL, NULL, false, channel, false));
+            keyArrays.append(*resolved->getKeyArray(NULL, NULL, false, channel, IRecordLayoutTranslator::NoTranslation));
         else
             fileArrays.append(*resolved->getIFileIOArray(false, channel));
     }
@@ -739,7 +739,7 @@ public:
     {
         return CPackageNode::queryEnv(varname);
     }
-    virtual bool getEnableFieldTranslation() const
+    virtual IRecordLayoutTranslator::Mode getEnableFieldTranslation() const override
     {
         return CPackageNode::getEnableFieldTranslation();
     }
@@ -2188,7 +2188,16 @@ private:
         case 'F':
             if (stricmp(queryName, "control:fieldTranslationEnabled")==0)
             {
-                fieldTranslationEnabled = control->getPropBool("@val", true);
+                const char *val = control->queryProp("@val");
+                if (val)
+                {
+                    if (strieq(val, "payload"))
+                        fieldTranslationEnabled = IRecordLayoutTranslator::TranslatePayload;
+                    else if (!val || strToBool(val))
+                        fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAll;
+                    else
+                        fieldTranslationEnabled = IRecordLayoutTranslator::NoTranslation;
+                }
                 topology->setPropInt("@fieldTranslationEnabled", fieldTranslationEnabled);
             }
             else if (stricmp(queryName, "control:flushJHtreeCacheOnOOM")==0)

roxie/roxie/CMakeLists.txt

@@ -43,6 +43,7 @@ include_directories (
          ./../../roxie/roxie 
          ./../../roxie/roxiemem 
          ./../../common/thorhelper
+         ./../../common/deftype
          ./../../common/workunit
     )