HPCC-20126 Allow index translation when merging index parts

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

roxie/ccd/ccdfile.cpp

@@ -1693,6 +1693,8 @@ class CTranslatorSet : implements CInterfaceOf<ITranslatorSet>
     const RtlRecord &targetLayout;
     int targetFormatCrc = 0;
     bool anyTranslators = false;
+    bool anyKeyedTranslators = false;
+    bool translatorsMatch = true;
 public:
     CTranslatorSet(const RtlRecord &_targetLayout, int _targetFormatCrc)
     : targetLayout(_targetLayout), targetFormatCrc(_targetFormatCrc)
@@ -1703,6 +1705,10 @@ public:
         assertex(actualLayout);
         if (translator || keyTranslator)
             anyTranslators = true;
+        if (translator && translator->keyedTranslated())
+            anyKeyedTranslators = true;
+        if (transformers.ordinality() && (translator != transformers.item(0)))
+            translatorsMatch = false;
         transformers.append(translator);
         keyTranslators.append(keyTranslator);
         actualLayouts.append(actualLayout);
@@ -1754,6 +1760,16 @@ public:
     {
         return anyTranslators;
     }
+
+    virtual bool isTranslatingKeyed() const override
+    {
+        return anyKeyedTranslators;
+    }
+
+    virtual bool hasConsistentTranslation() const override
+    {
+        return translatorsMatch;
+    }
 };
 
 template <class X> class PerChannelCacheOf

roxie/ccd/ccdkey.cpp

@@ -1329,7 +1329,9 @@ protected:
         virtual IOutputMetaData *queryActualLayout(unsigned subFile) const override { throwUnexpected(); }
         virtual int queryTargetFormatCrc() const override { throwUnexpected(); }
         virtual const RtlRecord &queryTargetFormat() const override { throwUnexpected(); }
-        virtual bool isTranslating() const { return false; }
+        virtual bool isTranslating() const override { return false; }
+        virtual bool isTranslatingKeyed() const override { return false; }
+        virtual bool hasConsistentTranslation() const override { return true; }
     } dummyTranslator;
 
     void test1()

roxie/ccd/ccdserver.cpp

@@ -23015,6 +23015,7 @@ public:
     {
         try
         {
+            bool noLocal = factory->queryQueryFactory().queryOptions().disableLocalOptimizations || seekGEOffset > 0;
             if (indexHelper.canMatchAny())
             {
                 if (variableInfoPending)
@@ -23025,7 +23026,6 @@ public:
                     // MORE - this recreates the segmonitors per part but not per fileno (which is a little backwards).
                     // With soft layout support may need to recreate per fileno too (i.e. different keys in a superkey have different layout) but never per partno
                     // However order is probably better to iterate fileno's inside partnos 
-                    // MORE - also not properly supporting STEPPED I fear.
                     // A superkey that mixes single and multipart or tlk and roroot keys might be hard
                     for (unsigned partNo = 0; partNo < keySet->length(); partNo++)
                     {
@@ -23057,6 +23057,7 @@ public:
                             if (seekGEOffset && !thisKey->isTopLevelKey())
                             {
                                 tlk.setown(createSingleKeyMerger(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, seekGEOffset, this, indexHelper.hasNewSegmentMonitors()));
+                                tlk->setLayoutTranslator(translators->queryTranslator(fileNo));
                             }
                             else
                             {
@@ -23105,7 +23106,13 @@ public:
                                     }
                                     else
                                     {
-                                        if (processSingleKey(thisKey, translators->queryTranslator(fileNo)))
+                                        // Single - part key. We process locally unless smart-stepping is involved, in which case we need to ensure that
+                                        // we do the proper two-stage merge.
+                                        // Unfortunately processSingleKey will not use the merger we carefully set up before,
+                                        // so we disable the local processing in that case
+                                        if (noLocal)
+                                            remote.getMem(1, fileNo, 0);  // 1 because it's a single part key
+                                        else if (processSingleKey(thisKey, translators->queryTranslator(fileNo)))
                                             break;
                                     }
                                 }
@@ -23587,9 +23594,12 @@ class CRoxieServerSimpleIndexReadActivity : public CRoxieServerActivity, impleme
 
     void initKeySet()
     {
-        if ((keySet->length() > 1 || rawMeta != NULL) && translators->isTranslating())
+        if (keySet->length() > 1 || rawMeta != NULL)
         {
-            throw MakeStringException(ROXIE_UNIMPLEMENTED_ERROR, "Layout translation is not available when merging key parts or smart-stepping, as it may change record order");
+            if (translators->isTranslatingKeyed())
+                throw MakeStringException(ROXIE_UNIMPLEMENTED_ERROR, "Keyed Layout translation is not available when merging key parts or smart-stepping, as it may change record order");
+            if (!translators->hasConsistentTranslation())
+                throw MakeStringException(ROXIE_UNIMPLEMENTED_ERROR, "Mixture of layout translation is not available when merging key parts or smart-stepping");
         }
         keyIndexSet.setown(createKeyIndexSet());
         for (unsigned part = 0; part < keySet->length(); part++)
@@ -23782,7 +23792,7 @@ public:
             if (numParts > 1 || seekGEOffset)
             {
                 tlk.setown(createKeyMerger(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet, seekGEOffset, this, indexHelper.hasNewSegmentMonitors()));
-                // note that we don't set up translator because we don't support it. If that ever changes...
+                tlk->setLayoutTranslator(translators->queryTranslator(0));
             }
             else
             {

roxie/ccd/ccdstate.hpp

@@ -109,6 +109,8 @@ interface ITranslatorSet : extends IInterface
     virtual int queryTargetFormatCrc() const = 0;
     virtual const RtlRecord &queryTargetFormat() const = 0;
     virtual bool isTranslating() const = 0;
+    virtual bool isTranslatingKeyed() const = 0;
+    virtual bool hasConsistentTranslation() const = 0;
 };
 
 interface IRoxieQuerySetManagerSet : extends IInterface

system/jhtree/jhtree.cpp

@@ -2440,7 +2440,6 @@ class CKeyMerger : public CKeyLevelManager
     unsigned *mergeheap;
     unsigned numkeys;
     unsigned activekeys;
-    unsigned compareSize = 0;
     IArrayOf<IKeyCursor> cursorArray;
     UnsignedArray mergeHeapArray;
     UnsignedArray keyNoArray;
@@ -2451,17 +2450,22 @@ class CKeyMerger : public CKeyLevelManager
 
     bool resetPending;
 
-//  #define BuffCompare(a, b) memcmp(buffers[mergeheap[a]]+sortFieldOffset, buffers[mergeheap[b]]+sortFieldOffset, keySize-sortFieldOffset) // NOTE - compare whole key not just keyed part.
     inline int BuffCompare(unsigned a, unsigned b)
     {
         const byte *c1 = cursors[mergeheap[a]]->queryKeyBuffer();
         const byte *c2 = cursors[mergeheap[b]]->queryKeyBuffer();
-        //Backwards compatibility - do not compare the fileposition field, even if it would be significant.
-        //int ret = memcmp(c1+sortFieldOffset, c2+sortFieldOffset, keySize-sortFieldOffset); // NOTE - compare whole key not just keyed part.
-        int ret = memcmp(c1+sortFieldOffset, c2+sortFieldOffset, compareSize-sortFieldOffset); // NOTE - compare whole key not just keyed part.
-        // MORE - this is wrong! variable-size fields will not compare correctly.
-        if (!ret && sortFieldOffset)
-            ret = memcmp(c1, c2, sortFieldOffset);
+
+        //Only compare the keyed portion, and if equal tie-break on lower input numbers having priority
+        //In the future this should use the comparison functions from the type info
+        int ret = memcmp(c1+sortFieldOffset, c2+sortFieldOffset, keyedSize-sortFieldOffset);
+        if (!ret)
+        {
+            if (sortFieldOffset)
+                ret = memcmp(c1, c2, sortFieldOffset);
+            //If they are equal, earlier inputs have priority
+            if (!ret)
+                ret = a - b;
+        }
         return ret;
     }
 
@@ -2639,11 +2643,13 @@ public:
 
     virtual void setLayoutTranslator(const IDynamicTransform * trans) override
     { 
-        if (trans)
+        if (trans && trans->keyedTranslated())
             throw MakeStringException(0, "Layout translation not supported when merging key parts, as it may change sort order"); 
+
         // It MIGHT be possible to support translation still if all keyCursors have the same translation
         // would have to translate AFTER the merge, but that's ok
         // HOWEVER the result won't be guaranteed to be in sorted order afterwards so is there any point?
+        CKeyLevelManager::setLayoutTranslator(trans);
     }
 
     virtual void setKey(IKeyIndexBase *_keyset)
@@ -2654,7 +2660,8 @@ public:
             IKeyIndex *ki = _keyset->queryPart(0);
             keyedSize = ki->keyedSize();
             numkeys = _keyset->numParts();
-            compareSize = ki->keySize() - (ki->hasSpecialFileposition() ? sizeof(offset_t) : 0);  // MORE - feels wrong
+            if (sortFieldOffset > keyedSize)
+                throw MakeStringException(0, "Index sort order can only include keyed fields");
         }
         else
             numkeys = 0;

testing/regress/ecl/key/setup.xml

@@ -32,3 +32,5 @@
 </Dataset>
 <Dataset name='Result 17'>
 </Dataset>
+<Dataset name='Result 18'>
+</Dataset>

testing/regress/ecl/key/setupwordindex.xml

@@ -2,3 +2,7 @@
 </Dataset>
 <Dataset name='Result 2'>
 </Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+</Dataset>

testing/regress/ecl/key/stepping10.xml

@@ -0,0 +1,34 @@
+<Dataset name='Result 1'>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>0</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>1</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>200</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>201</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>100</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>101</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>300</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>301</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>26</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>25</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>225</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>226</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>126</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>125</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>325</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>326</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>51</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>50</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>251</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>250</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>150</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>151</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>351</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>350</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>75</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>76</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>275</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>276</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>176</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>175</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>375</filepos></Row>
+ <Row><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>376</filepos></Row>
+</Dataset>

testing/regress/ecl/key/stepping5.xml

@@ -0,0 +1,18 @@
+<Dataset name='Result 1'>
+ <Row><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange></Row>
+ <Row><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange></Row>
+ <Row><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange></Row>
+ <Row><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange></Row>
+ <Row><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange></Row>
+ <Row><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange></Row>
+ <Row><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange></Row>
+ <Row><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange></Row>
+ <Row><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange></Row>
+ <Row><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange></Row>
+ <Row><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange></Row>
+ <Row><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange></Row>
+</Dataset>

testing/regress/ecl/setup/files.ecl

@@ -152,7 +152,12 @@ EXPORT DG_indexFile      := INDEX(DG_NormalIndexFile, indexName);
 indexName := IF(useTranslation, __nameof__(DG_TransIndexFileEvens), __nameof__(DG_NormalIndexFileEvens));
 EXPORT DG_indexFileEvens := INDEX(DG_NormalIndexFileEvens, indexName);
 
-EXPORT DG_KeyedIndexFile      := INDEX(DG_FlatFile, { DG_firstname, DG_lastname, DG_Prange}, { filepos }, DG_IndexOut+'KEYED_INDEX');
+
+EXPORT DG_KeyedIndexFile      := INDEX(DG_FlatFile, { DG_firstname, DG_lastname, DG_Prange}, { filepos }, DG_IndexOut+'KEYED_INDEX', fileposition(false));
+EXPORT DG_KeyedIndexFileDelta := INDEX(DG_FlatFile, { DG_firstname, DG_lastname, DG_Prange}, { UNSIGNED8 filepos := filepos + 1 }, DG_IndexOut+'KEYED_INDEX_DELTA', fileposition(false));
+//Combine the two files above with an implicit superkey
+EXPORT DG_DupKeyedIndexFile   := INDEX(DG_FlatFile, { DG_firstname, DG_lastname, DG_Prange}, { filepos }, '{' + DG_IndexOut+'KEYED_INDEX' + ',' + DG_IndexOut+'KEYED_INDEX_DELTA' + '}', fileposition(false));
+EXPORT DG_DupKeyedIndexSuperFileName   := DG_IndexOut+'KEYED_INDEX_DUP';
 
 EXPORT DG_CSVFile   := DATASET(DG_FileOut+'CSV',DG_OutRec,CSV);
 EXPORT DG_XMLFile   := DATASET(DG_FileOut+'XML',DG_OutRec,XML);
@@ -195,7 +200,7 @@ EXPORT SET OF STRING3 DG_MONTHS := ['JAN','FEB','MAR','APR','MAY','JUN','JUL','A
 
 //----------------------------- Text search definitions ----------------------------------
 
-EXPORT NameWordIndex() := indexPrefix + 'wordIndex' + IF(useLocal, '_Local', '');
+EXPORT NameWordIndex() := indexPrefix + 'wordIndex' + IF(useLocal, '_Local', '') + IF(useTranslation, '_Trans', '') ;
 EXPORT NameSearchIndex := indexPrefix + 'searchIndex';
 EXPORT getWordIndex() := INDEX(TS.textSearchIndex, NameWordIndex());
 EXPORT getSearchIndex() := INDEX(TS.textSearchIndex, NameSearchIndex);

testing/regress/ecl/setup/setup.ecl

@@ -83,6 +83,9 @@ EvensFilter := DG_ParentRecs.DG_firstname in [Files.DG_Fnames[2],Files.DG_Fnames
                                               Files.DG_Fnames[10],Files.DG_Fnames[12],Files.DG_Fnames[14],Files.DG_Fnames[16]];
 
 SEQUENTIAL(
+    ORDERED(
+        FileServices.DeleteSuperFile(Files.DG_DupKeyedIndexSuperFileName)
+    ),
     PARALLEL(output(DG_ParentRecs,,Files.DG_FileOut+'FLAT',overwrite),
              output(GROUP(SORT(DG_ParentRecs, DG_FirstName),DG_Firstname),,Files.DG_FileOut+'GROUPED',__GROUPED__,overwrite),
              output(DG_ParentRecs(EvensFilter),,Files.DG_FileOut+'FLAT_EVENS',overwrite)),
@@ -90,7 +93,16 @@ SEQUENTIAL(
              buildindex(Files.DG_NormalIndexFileEvens,overwrite,SET('_nodeSize', 512)),
              buildindex(Files.DG_TransIndexFile,overwrite),
              buildindex(Files.DG_TransIndexFileEvens,overwrite),
-             buildindex(Files.DG_KeyedIndexFile,overwrite))
+             buildindex(Files.DG_KeyedIndexFile,overwrite),
+             buildindex(Files.DG_KeyedIndexFileDelta,overwrite),
+             ),
+    ORDERED(
+        FileServices.CreateSuperFile(Files.DG_DupKeyedIndexSuperFileName),
+        FileServices.StartSuperFileTransaction(),
+        FileServices.AddSuperFile(Files.DG_DupKeyedIndexSuperFileName,__nameof__(Files.DG_KeyedIndexFile)),
+        FileServices.AddSuperFile(Files.DG_DupKeyedIndexSuperFileName,__nameof__(Files.DG_KeyedIndexFileDelta)),
+        FileServices.FinishSuperFileTransaction(),
+    )
     );
 
     fileServices.AddFileRelationship( __nameof__(Files.DG_FlatFile), __nameof__(Files.DG_NormalIndexFile), '', '', 'view', '1:1', false);

testing/regress/ecl/setup/setuptext.ecl

@@ -427,16 +427,24 @@ wordsAndAliases := merge(orderedWords, orderedAliases, sorted(doc, segment, wpos
 
 normalizedInversion := normalizeWordFormat(wordsAndAliases);
 
-doCreateSimpleIndex(boolean useLocal) := FUNCTION
-    Files := $.Files(createMultiPart, useLocal);
+doCreateSimpleIndex(boolean useLocal, boolean useTranslation) := FUNCTION
+    Files := $.Files(createMultiPart, useLocal, useTranslation);
     distributedWords := DISTRIBUTE(normalizedInversion, IF(doc > 6, 0, 1));
     
     RETURN sequential(
-        IF(useLocal,
-            BUILD(distributedWords, { kind, word, doc, segment, wpos, wip }, { flags, original, dpos }, Files.NameWordIndex(), 
-                    OVERWRITE, NOROOT, COMPRESSED(row)),
-            BUILD(normalizedInversion, { kind, word, doc, segment, wpos, wip }, { flags, original, dpos }, Files.NameWordIndex(), 
-                    OVERWRITE, COMPRESSED(row))
+        IF(useTranslation,
+            IF(useLocal,
+                BUILD(distributedWords, { kind, word, doc, segment, wpos, wip }, { unsigned extra := dpos; flags, original, dpos }, Files.NameWordIndex(),
+                        OVERWRITE, NOROOT, COMPRESSED(row)),
+                BUILD(normalizedInversion, { kind, word, doc, segment, wpos, wip }, { unsigned extra := dpos, flags, original, dpos }, Files.NameWordIndex(),
+                        OVERWRITE, COMPRESSED(row))
+            ),
+            IF(useLocal,
+                BUILD(distributedWords, { kind, word, doc, segment, wpos, wip }, { flags, original, dpos }, Files.NameWordIndex(),
+                        OVERWRITE, NOROOT, COMPRESSED(row)),
+                BUILD(normalizedInversion, { kind, word, doc, segment, wpos, wip }, { flags, original, dpos }, Files.NameWordIndex(),
+                        OVERWRITE, COMPRESSED(row))
+            )
         );
         //Add a column mapping, testing done L->R, multiple transforms, and that parameters work
         fileServices.setColumnMapping(Files.NameWordIndex(), 'word{set(stringlib.StringToLowerCase,stringlib.StringFilterOut(\'AEIOU$?.:;,()\'))}')
@@ -517,7 +525,8 @@ END;
     exports := MODULE
         EXPORT createSearchIndex() := doCreateSearchIndex();
     
-        EXPORT createSimpleIndex(boolean useLocal) := doCreateSimpleIndex(useLocal);
+        EXPORT createSimpleIndex(boolean useLocal, boolean useTranslation = FALSE) :=
+                    doCreateSimpleIndex(useLocal, useTranslation);
     END;
     
     RETURN exports;

testing/regress/ecl/setup/setupwordindex.ecl

@@ -21,3 +21,5 @@ import $.SetupText;
 
 SetupText.createSimpleIndex(FALSE);
 SetupText.createSimpleIndex(TRUE);
+SetupText.createSimpleIndex(FALSE, TRUE);
+SetupText.createSimpleIndex(TRUE, TRUE);

testing/regress/ecl/setup/thor/setup.xml

@@ -46,3 +46,5 @@
 </Dataset>
 <Dataset name='Result 24'>
 </Dataset>
+<Dataset name='Result 25'>
+</Dataset>

testing/regress/ecl/stepping10.ecl

@@ -0,0 +1,50 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=file
+//class=index
+//version multiPart=false,useExplicitSuper=false
+//version multiPart=true,useExplicitSuper=false
+//version multiPart=false,useExplicitSuper=true
+//version multiPart=true,useExplicitSuper=true
+
+//nothor
+//Stepped global joins unsupported, see issue HPCC-8148
+//skip type==thorlcr TBD
+
+import ^ as root;
+multiPart := #IFDEFINED(root.multiPart, false);
+useLocal := #IFDEFINED(root.useLocal, false);
+useTranslation := #IFDEFINED(root.useTranslation, false);
+useExplicitSuper := #IFDEFINED(root.useExplicitSuper, false);
+
+//--- end of version configuration ---
+
+#option ('layoutTranslation', useTranslation);
+#onwarning (4126, ignore);
+
+import $.setup;
+Files := setup.Files(multiPart, useLocal, useTranslation);
+
+#if (useExplicitSuper)
+idx := INDEX(Files.DG_DupKeyedIndexFile, Files.DG_DupKeyedIndexSuperFileName);
+#else
+idx := Files.DG_DupKeyedIndexFile;
+#end
+
+// should be equivalent to OUTPUT(SORT(Files.DG_IndexFile(DG_firstname = 'DAVID'), DG_Prange, DG_firstname, DG_lastname));
+OUTPUT(STEPPED(idx(KEYED(DG_firstname = 'DAVID')), DG_Prange));

testing/regress/ecl/stepping2.ecl

@@ -17,18 +17,25 @@
 
 //version multiPart=false
 //version multiPart=true
+//version multiPart=false,useTranslation=true
+//version multiPart=true,useTranslation=true
 
 import ^ as root;
 multiPart := #IFDEFINED(root.multiPart, false);
+useTranslation := #IFDEFINED(root.useTranslation, true);
+
 
 //--- end of version configuration ---
 
+#option ('layoutTranslation', useTranslation);
+#onwarning (5402, ignore);
+
 //nothor
 //Stepped Thor support
 
 import $.Setup;
 import $.Setup.TS;
-wordIndex := Setup.Files(multiPart, false).getWordIndex();
+wordIndex := Setup.Files(multiPart, false, useTranslation).getWordIndex();
 
 OUTPUT(SORTED(STEPPED(WordIndex(keyed(kind = TS.kindType.TextEntry and word in ['boy', 'sheep'])), doc, segment, wpos), doc, segment, wpos, assert)) : independent;
 OUTPUT(SORTED(STEPPED(WordIndex(keyed(kind = TS.kindType.TextEntry and word in ['b%%%', 'sheep'])), doc, segment, wpos), doc, segment, wpos, assert)) : independent;

testing/regress/ecl/stepping3.ecl

@@ -27,6 +27,7 @@ useTranslation := #IFDEFINED(root.useTranslation, false);
 
 //--- end of version configuration ---
 
+#option ('layoutTranslation', useTranslation);
 #onwarning (4126, ignore);
 
 import $.setup;

testing/regress/ecl/stepping5.ecl

@@ -0,0 +1,44 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=file
+//class=index
+//version multiPart=false
+//version multiPart=true
+//version multiPart=true,useTranslation=true
+
+import ^ as root;
+multiPart := #IFDEFINED(root.multiPart, true);
+useLocal := #IFDEFINED(root.useLocal, false);
+useTranslation := #IFDEFINED(root.useTranslation, false);
+
+//--- end of version configuration ---
+
+#option ('layoutTranslation', useTranslation);
+#onwarning (4126, ignore);
+
+import $.setup;
+Files := setup.Files(multiPart, useLocal, useTranslation);
+
+//nothor
+//nohthor
+
+ indexRead := STEPPED(Files.DG_indexFile(KEYED(DG_firstname = 'DAVID')), DG_lastname);
+
+ distrib := ALLNODES(LOCAL(indexRead));
+ nf := NOFOLD(distrib);
+ output(nf,{dg_lastname, dg_prange});