Merge pull request #10775 from richardkchapman/translate-keyed-field

HPCC-18900 Apply keyed filters on translated disk reads correctly

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

roxie/ccd/ccdactivities.cpp

@@ -880,7 +880,7 @@ public:
                 reader.setown(manager->selectKey(indexSig, postFilter, translators));
         }
         else if (!forceUnkeyed && !isGrouped)
-            reader.setown(manager->selectKey(postFilter, translators));
+            reader.setown(manager->selectKey(postFilter, translators, logctx));
         if (!reader)
             reader.setown(manager->createReader(postFilter, isGrouped, readPos, parallelPartNo, numParallel, translators));
     }
@@ -894,7 +894,7 @@ public:
     {
         unsigned channel = packet->queryHeader().channel;
         unsigned formatCrc = basefactory->getFormatCrc(helper->getFormatCrc());
-        translators.setown(varFileInfo->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation())); // MORE - FormatCRC may be wrong here. Needs to be crc of projected not expected
+        translators.setown(varFileInfo->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation(), false)); // MORE - FormatCRC may be wrong here. Needs to be crc of projected not expected
         manager.setown(varFileInfo->getIndexManager(isOpt, channel, translators->queryActualLayout(0), false));
     }
 
@@ -1001,9 +1001,15 @@ public:
             {
                 unsigned channel = queryFactory.queryChannel();
                 unsigned formatCrc = getFormatCrc(helper->getFormatCrc());
-                translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
+                translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                 manager.setown(datafile->getIndexManager(isOpt, channel, translators->queryActualLayout(0), _graphNode.getPropBool("att[@name=\"preload\"]/@value", false)));
                 Owned<IPropertyTreeIterator> memKeyInfo = queryFactory.queryPackage().getInMemoryIndexInfo(_graphNode);
+                Owned<IPropertyTree> memKeyHint;
+                if (!memKeyInfo && _graphNode.hasProp("hint[@name=\"key\"]"))
+                {
+                    memKeyHint.setown(createPTreeFromXMLString(_graphNode.queryProp("hint[@name=\"key\"]/@value")));
+                    memKeyInfo.setown(memKeyHint->getElements("MemIndex"));
+                }
                 if (memKeyInfo)
                 {
                     // There's a small potential flaw that I can end up with indexes from a query that was unloaded active on some nodes but not others
@@ -2395,7 +2401,7 @@ public:
             datafile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
             if (datafile)
             {
-                translators.setown(datafile->getTranslators(formatCrc, projectedMeta, expectedMeta, queryFactory.queryOptions().enableFieldTranslation));
+                translators.setown(datafile->getTranslators(formatCrc, projectedMeta, expectedMeta, queryFactory.queryOptions().enableFieldTranslation, true));
                 keyArray.setown(datafile->getKeyArray(isOpt, queryFactory.queryChannel()));
             }
         }
@@ -2458,7 +2464,7 @@ protected:
     virtual void setVariableFileInfo()
     {
         const CRoxieKeyedActivityFactory &aFactory = *static_cast<const CRoxieKeyedActivityFactory *>(basefactory);
-        translators.setown(varFileInfo->getTranslators(aFactory.formatCrc, aFactory.projectedMeta, aFactory.expectedMeta, allowFieldTranslation));
+        translators.setown(varFileInfo->getTranslators(aFactory.formatCrc, aFactory.projectedMeta, aFactory.expectedMeta, allowFieldTranslation, true));
         keyArray.setown(varFileInfo->getKeyArray(isOpt, packet->queryHeader().channel));
     }
 
@@ -3557,7 +3563,7 @@ public:
             if (datafile)
             {
                 unsigned formatCrc = getFormatCrc(helper->getDiskFormatCrc());
-                translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
+                translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                 fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
             }
         }
@@ -3613,7 +3619,7 @@ public:
     virtual void setVariableFileInfo()
     {
         unsigned formatCrc = basefactory->getFormatCrc(helper->getDiskFormatCrc());
-        translators.setown(varFileInfo->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation()));
+        translators.setown(varFileInfo->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation(), false));
         files.setown(varFileInfo->getIFileIOArray(isOpt, packet->queryHeader().channel));
     }
 
@@ -3949,7 +3955,7 @@ public:
             datafile.setown(_queryFactory.queryPackage().lookupFileName(indexFileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true));
             if (datafile)
             {
-                translators.setown(datafile->getTranslators(formatCrc, projectedMeta, expectedMeta, queryFactory.queryOptions().enableFieldTranslation));
+                translators.setown(datafile->getTranslators(formatCrc, projectedMeta, expectedMeta, queryFactory.queryOptions().enableFieldTranslation, true));
                 keyArray.setown(datafile->getKeyArray(isOpt, queryFactory.queryChannel()));
             }
         }
@@ -4290,7 +4296,7 @@ public:
             if (datafile)
             {
                 unsigned formatCrc = getFormatCrc(helper->getDiskFormatCrc());
-                translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
+                translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                 files.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
             }
         }
@@ -4356,7 +4362,7 @@ public:
     virtual void setVariableFileInfo()
     {
         unsigned formatCrc = basefactory->getFormatCrc(helper->getDiskFormatCrc());
-        translators.setown(varFileInfo->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation()));
+        translators.setown(varFileInfo->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), basefactory->getEnableFieldTranslation(), false));
         files.setown(varFileInfo->getIFileIOArray(isOpt, packet->queryHeader().channel));
     }
 

roxie/ccd/ccdfile.cpp

@@ -1702,7 +1702,7 @@ public:
     void addTranslator(const IDynamicTransform *translator, const IKeyTranslator *keyTranslator, IOutputMetaData *actualLayout)
     {
         assertex(actualLayout);
-        if (translator)
+        if (translator || keyTranslator)
             anyTranslators = true;
         transformers.append(translator);
         keyTranslators.append(keyTranslator);
@@ -1730,6 +1730,13 @@ public:
         return nullptr;
     }
 
+    virtual const IKeyTranslator *queryKeyTranslator(unsigned subFile) const override
+    {
+        if (anyTranslators && keyTranslators.isItem(subFile))
+            return keyTranslators.item(subFile);
+        return nullptr;
+    }
+
     virtual ISourceRowPrefetcher *getPrefetcher(unsigned subFile) const override
     {
         IOutputMetaData *actualLayout = actualLayouts.item(subFile);
@@ -2076,7 +2083,7 @@ public:
         else
             mb.append(false);
     }
-    virtual ITranslatorSet *getTranslators(int formatCrc, IOutputMetaData *projected, IOutputMetaData *expected, RecordTranslationMode mode) const override
+    virtual ITranslatorSet *getTranslators(int formatCrc, IOutputMetaData *projected, IOutputMetaData *expected, RecordTranslationMode mode, bool isIndex) const override
     {
         // NOTE - projected and expected and anything fetched from them such as type info may reside in dynamically loaded (and unloaded)
         // query DLLs - this means it is not safe to include them in any sort of cache that might outlive the current query.
@@ -2127,7 +2134,10 @@ public:
                         {
                             if (mode == RecordTranslationMode::None)
                                 throw MakeStringException(ROXIE_MISMATCH, "Translatable record layout mismatch detected for file %s, but translation disabled", subname);
-                            keyedTranslator.setown(createKeyTranslator(actual->queryRecordAccessor(true), expected->queryRecordAccessor(true)));
+                            keyedTranslator.setown(createKeyTranslator(actual->queryRecordAccessor(true), expected->queryRecordAccessor(true))); // NOTE - index cases should check (elsewhere) that this is empty.
+                            if (isIndex && keyedTranslator->needsTranslate())
+                                throw MakeStringException(ROXIE_MISMATCH, "Record layout mismatch detected in keyed fields for file %s", subname);
+
                         }
                     }
                 }

roxie/ccd/ccdfile.hpp

@@ -90,7 +90,7 @@ interface IResolvedFile : extends ISimpleSuperFileEnquiry
 {
     virtual void serializePartial(MemoryBuffer &mb, unsigned channel, bool localInfoOnly) const = 0;
 
-    virtual ITranslatorSet *getTranslators(int formatCrc, IOutputMetaData *projected, IOutputMetaData *expected, RecordTranslationMode mode) const = 0;
+    virtual ITranslatorSet *getTranslators(int formatCrc, IOutputMetaData *projected, IOutputMetaData *expected, RecordTranslationMode mode, bool isIndex) const = 0;
     virtual IFileIOArray *getIFileIOArray(bool isOpt, unsigned channel) const = 0;
     virtual IKeyArray *getKeyArray(bool isOpt, unsigned channel) const = 0;
     virtual IFilePartMap *getFileMap() const = 0;

roxie/ccd/ccdkey.cpp

@@ -222,8 +222,19 @@ public:
         ForEach(*fields)
         {
             IPropertyTree &field = fields->query();
-            unsigned fieldNum = field.getPropInt("@fieldNum", -1);
-            assertex(fieldNum != (unsigned) -1);
+            const char *fieldname = field.queryProp("@name");
+            if (!fieldname || !*fieldname)
+                throw MakeStringException(0, "Invalid MemIndex specification - missing field name");
+            unsigned fieldNum = recInfo.getFieldNum(fieldname);
+            if (fieldNum == (unsigned) -1)
+            {
+                StringBuffer s;
+                for (unsigned idx = 0; idx <  recInfo.getNumFields(); idx++)
+                    s.append(',').append(recInfo.queryName(idx));
+                if (!s.length())
+                    s.append(",<no fields found>");
+                throw MakeStringException(0, "Invalid MemIndex specification - field name %s not found (fields are %s)", fieldname, s.str()+1);
+            }
             append(fieldNum);
         }
     }
@@ -282,7 +293,7 @@ public:
         {
             if (idx)
                 ret.append('.');
-            ret.appendf("%u", sortFields.item(idx));
+            ret.appendf("%s.%d", recInfo.queryName(sortFields.item(idx)),sortFields.item(idx));
         }
         return ret;
     }
@@ -435,30 +446,37 @@ protected:
     Owned<ISourceRowPrefetcher> prefetcher;
     Linked<const ITranslatorSet> translators;
     const IDynamicTransform *translator = nullptr;
+    const RtlRecord *actual= nullptr;
     const RowFilter &postFilter;
-    RtlDynRow row;
     bool grouped = false;
     bool eogPending = false;
     bool anyThisGroup = false;
 
-    void _nextRow()
+    const byte * _nextRow(bool &eogPending)
     {
         prefetcher->readAhead(deserializeSource);
         if (grouped)
             deserializeSource.read(1, &eogPending);
-        if (translator)
+
+        unsigned numOffsets = actual->getNumVarFields() + 1;
+        size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
+        RtlRow row(*actual, nullptr, numOffsets, variableOffsets);
+        row.setRow(deserializeSource.queryRow(), 0);
+        if (!postFilter.matches(row))
+            return nullptr;
+        else if (translator)
         {
             buf.setLength(0);
             MemoryBufferBuilder aBuilder(buf, 0);
-            translator->translate(aBuilder, deserializeSource.queryRow());
-            row.setRow(reinterpret_cast<const byte *>(buf.toByteArray()), 0);
+            translator->translate(aBuilder, row);
+            return reinterpret_cast<const byte *>(buf.toByteArray());
         }
         else
-            row.setRow(deserializeSource.queryRow(), 0);
+            return row.queryRow();
     }
 public:
     CDirectReaderBase(const ITranslatorSet *_translators, const RowFilter &_postFilter, bool _grouped)
-    : translators(_translators), postFilter(_postFilter), row(_translators->queryTargetFormat()), grouped(_grouped)
+    : translators(_translators), postFilter(_postFilter), grouped(_grouped)
     {}
     virtual bool isKeyed() const override { return false; }
 
@@ -487,11 +505,11 @@ public:
                     return nullptr;
                 }
             }
-            _nextRow();
-            if (postFilter.matches(row))  // MORE - could filter before translation.
+            const byte *row = _nextRow(eogPending);
+            if (row)
             {
                 anyThisGroup = true;
-                return row.queryRow();
+                return row;
             }
             else
                 finishedRow();
@@ -524,6 +542,7 @@ public:
     {
         translator = translators->queryTranslator(0);  // Any one would do
         prefetcher.setown(translators->getPrefetcher(0));
+        actual = &translators->queryActualLayout(0)->queryRecordAccessor(true);
         deserializeSource.setStream(this);
         if (_numParts == 1)
         {
@@ -683,6 +702,7 @@ public:
             unsigned subFileIdx = f->getSubFile(thisPartIdx);
             prefetcher.setown(translators->getPrefetcher(subFileIdx));
             translator = translators->queryTranslator(subFileIdx);
+            actual = &translators->queryActualLayout(subFileIdx)->queryRecordAccessor(true);
         }
         else
         {
@@ -708,6 +728,7 @@ public:
             unsigned subFileIdx = f->getSubFile(thisPartIdx);
             prefetcher.setown(translators->getPrefetcher(subFileIdx));
             translator = translators->queryTranslator(subFileIdx);
+            actual = &translators->queryActualLayout(subFileIdx)->queryRecordAccessor(true);
         }
     }
 
@@ -907,7 +928,7 @@ public:
     }
 
     virtual IDirectReader *selectKey(const char *sig, ScoredRowFilter &postFilters, const ITranslatorSet *translators) const override;
-    virtual IDirectReader *selectKey(ScoredRowFilter &filter, const ITranslatorSet *translators) const override;
+    virtual IDirectReader *selectKey(ScoredRowFilter &filter, const ITranslatorSet *translators, IRoxieContextLogger &logctx) const override;
 
     InMemoryIndex &findIndex(const char *indexSig) const
     {
@@ -1087,11 +1108,12 @@ public:
         {
             unsigned fieldIdx = index->sortFields.item(idx);
             const IFieldFilter *match = postFilter.extractFilter(fieldIdx);
-            if (!match)
-                break;
-            indexedFields.addFilter(*match);
+            if (match)
+                indexedFields.addFilter(*match);
+            else
+                break;  // MORE - this means we never use wilds in in-memory indexes - always leave to postfilter. Is that right?
         }
-        keySearcher.setown(new KeySearcher(_recInfo, indexedFields, this));
+        keySearcher.setown(new KeySearcher(_recInfo, index->sortFields, indexedFields, this));
         translator = translators->queryTranslator(0);  // Any one would do - we require all to match
 #ifdef BASED_POINTERS
         base = index->base;
@@ -1192,11 +1214,21 @@ public:
     }
     virtual unsigned __int64 getFilePosition(const void *_ptr)
     {
+        if (translator)
+        {
+            assertex(_ptr==buf.toByteArray());
+            _ptr = keySearcher->queryRow().queryRow();
+        }
         return baseMap.ptrToFilePosition(_ptr);
     }
 
     virtual unsigned __int64 getLocalFilePosition(const void *_ptr)
     {
+        if (translator)
+        {
+            assertex(_ptr==buf.toByteArray());
+            _ptr = keySearcher->queryRow().queryRow();
+        }
         return baseMap.ptrToLocalFilePosition(_ptr);
     }
 
@@ -1218,8 +1250,11 @@ IDirectReader *InMemoryIndexManager::selectKey(const char *indexSig, ScoredRowFi
     return new InMemoryIndexCursor(this, &thisIndex, baseMap, postFilters, recInfo, translators);
 }
 
-IDirectReader *InMemoryIndexManager::selectKey(ScoredRowFilter &filter, const ITranslatorSet *translators) const
+IDirectReader *InMemoryIndexManager::selectKey(ScoredRowFilter &filter, const ITranslatorSet *translators, IRoxieContextLogger &logctx) const
 {
+    const IKeyTranslator *keyTranslator = translators->queryKeyTranslator(0);  // any part would do - in-memory requires all actuals to have same layout
+    if (keyTranslator)
+        keyTranslator->translate(filter);
     if (!inMemoryKeysEnabled)
         return nullptr;
     unsigned best = 0;
@@ -1244,7 +1279,14 @@ IDirectReader *InMemoryIndexManager::selectKey(ScoredRowFilter &filter, const IT
         }
     }
     if (bestIndex)
+    {
+        if (logctx.queryTraceLevel() > 5)
+        {
+            StringBuffer ret;
+            logctx.CTXLOG("Using key %s", bestIndex->toString(ret).str());
+        }
         return new InMemoryIndexCursor(this, bestIndex, baseMap, filter, recInfo, translators);
+    }
     else
         return nullptr;
 }
@@ -1270,6 +1312,7 @@ protected:
     class CDummyTranslatorSet : implements CInterfaceOf<ITranslatorSet>
     {
         virtual const IDynamicTransform *queryTranslator(unsigned subFile) const override { return nullptr; }
+        virtual const IKeyTranslator *queryKeyTranslator(unsigned subFile) const override { return nullptr; }
         virtual ISourceRowPrefetcher *getPrefetcher(unsigned subFile) const override { throwUnexpected(); }
         virtual IOutputMetaData *queryActualLayout(unsigned subFile) const override { throwUnexpected(); }
         virtual int queryTargetFormatCrc() const override { throwUnexpected(); }
@@ -1279,6 +1322,7 @@ protected:
 
     void test1()
     {
+        StringContextLogger logctx("dummy");
         RtlIntTypeInfo ty1(type_int|type_unsigned, sizeof(unsigned));
         RtlFieldInfo f1("f1", nullptr, &ty1);
         const RtlFieldInfo * const fields [] = {&f1, nullptr};
@@ -1295,7 +1339,7 @@ protected:
         unsigned searchval = 1;
         ScoredRowFilter filter;
         filter.addFilter(*createFieldFilter(0, ty1, &searchval));
-        Owned<IDirectReader> d = indexes.selectKey(filter, &dummyTranslator);
+        Owned<IDirectReader> d = indexes.selectKey(filter, &dummyTranslator, logctx);
         ASSERT(d->nextRow()==(const byte *) &testarray[0]);
         ASSERT(d->nextRow()==nullptr);
         ASSERT(d->nextRow()==nullptr);
@@ -1303,7 +1347,7 @@ protected:
         searchval = 10;
         ScoredRowFilter filter1;
         filter1.addFilter(*createFieldFilter(0, ty1, &searchval));
-        d.setown(indexes.selectKey(filter1, &dummyTranslator));
+        d.setown(indexes.selectKey(filter1, &dummyTranslator, logctx));
         ASSERT(d->nextRow()==nullptr);
         ASSERT(d->nextRow()==nullptr);
         d.clear();
@@ -1318,7 +1362,7 @@ protected:
         unsigned searchval2 = 9;
         set->addRawRange(&searchval2, &searchval2);
         filter2.addFilter(*createFieldFilter(0, set));
-        d.setown(indexes.selectKey(filter2, &dummyTranslator));
+        d.setown(indexes.selectKey(filter2, &dummyTranslator, logctx));
         ASSERT(*(int *) d->nextRow()==2);
         ASSERT(*(int *) d->nextRow()==2);
         ASSERT(*(int *) d->nextRow()==2);

roxie/ccd/ccdkey.hpp

@@ -65,7 +65,7 @@ interface IInMemoryIndexManager : extends IInterface
 {
     virtual void load(IFileIOArray *, IOutputMetaData *preloadLayout, bool preload) = 0;
     virtual bool IsShared() const = 0;
-    virtual IDirectReader *selectKey(ScoredRowFilter &filter, const ITranslatorSet *translators) const = 0;
+    virtual IDirectReader *selectKey(ScoredRowFilter &filter, const ITranslatorSet *translators, IRoxieContextLogger &logctx) const = 0;
     virtual IDirectReader *selectKey(const char *sig, ScoredRowFilter &filter, const ITranslatorSet *translators) const = 0;
     virtual IDirectReader *createReader(const RowFilter &postFilter, bool _grouped, offset_t readPos, unsigned partNo, unsigned numParts, const ITranslatorSet *translators) const = 0;
     virtual void setKeyInfo(IPropertyTree &indexInfo) = 0;

roxie/ccd/ccdserver.cpp

@@ -21579,12 +21579,12 @@ public:
                 {
                     unsigned channel = isLocal ? factory->queryQueryFactory().queryChannel() : 0;
                     unsigned formatCrc = getFormatCrc(helper.getFormatCrc());
-                    translators.setown(varFileInfo->getTranslators(formatCrc, helper.queryProjectedDiskRecordSize(), helper.queryDiskRecordSize(), getEnableFieldTranslation()));
+                    translators.setown(varFileInfo->getTranslators(formatCrc, helper.queryProjectedDiskRecordSize(), helper.queryDiskRecordSize(), getEnableFieldTranslation(), false));
                     manager.setown(varFileInfo->getIndexManager(isOpt, channel, translators->queryActualLayout(0), false));
                 }
                 assertex(manager != NULL);
                 helper.createSegmentMonitors(this);
-                reader.setown(manager->selectKey(postFilter, translators));
+                reader.setown(manager->selectKey(postFilter, translators, *this));
                 if (!reader)
                     reader.setown(manager->createReader(postFilter, isGrouped, 0, 0, 1, translators));
                 assertex(reader);
@@ -22476,7 +22476,7 @@ public:
                 {
                     unsigned channel = isLocal ? queryFactory.queryChannel() : 0;
                     unsigned formatCrc = getFormatCrc(helper->getFormatCrc());
-                    translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
+                    translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                     manager.setown(datafile->getIndexManager(isOpt, channel, translators->queryActualLayout(0), _graphNode.getPropBool("att[@name=\"preload\"]/@value", false)));
                     const IPropertyTree *options = datafile->queryProperties();
                     if (options)
@@ -22601,7 +22601,7 @@ public:
                 unsigned formatCrc = getFormatCrc(indexHelper->getFormatCrc());
                 IOutputMetaData *projectedMeta = indexHelper->queryProjectedDiskRecordSize();
                 IOutputMetaData *expectedMeta = indexHelper->queryDiskRecordSize();
-                translators.setown(indexfile->getTranslators(formatCrc, projectedMeta, expectedMeta, getEnableFieldTranslation()));
+                translators.setown(indexfile->getTranslators(formatCrc, projectedMeta, expectedMeta, getEnableFieldTranslation(), true));
                 keySet.setown(indexfile->getKeyArray(isOpt, isLocal ? queryFactory.queryChannel() : 0));
             }
         }
@@ -22691,7 +22691,7 @@ protected:
             unsigned formatCrc = getFormatCrc(indexHelper.getFormatCrc());
             IOutputMetaData *projectedMeta = indexHelper.queryProjectedDiskRecordSize();
             IOutputMetaData *expectedMeta = indexHelper.queryDiskRecordSize();
-            translators.setown(varFileInfo->getTranslators(formatCrc, projectedMeta, expectedMeta, getEnableFieldTranslation()));
+            translators.setown(varFileInfo->getTranslators(formatCrc, projectedMeta, expectedMeta, getEnableFieldTranslation(), true));
             keySet.setown(varFileInfo->getKeyArray(isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
         }
         variableInfoPending = false;
@@ -23342,7 +23342,7 @@ class CRoxieServerSimpleIndexReadActivity : public CRoxieServerActivity, impleme
         unsigned formatCrc = getFormatCrc(indexHelper.getFormatCrc());
         IOutputMetaData *projectedMeta = indexHelper.queryProjectedDiskRecordSize();
         IOutputMetaData *expectedMeta = indexHelper.queryDiskRecordSize();
-        translators.setown(varFileInfo->getTranslators(formatCrc, projectedMeta, expectedMeta, getEnableFieldTranslation()));
+        translators.setown(varFileInfo->getTranslators(formatCrc, projectedMeta, expectedMeta, getEnableFieldTranslation(), true));
         keySet.setown(varFileInfo->getKeyArray(isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
         initKeySet();
         variableInfoPending = false;
@@ -25255,7 +25255,7 @@ public:
             if (varFileInfo)
             {
                 unsigned formatCrc = getFormatCrc(helper.getIndexFormatCrc());
-                translators.setown(varFileInfo->getTranslators(formatCrc, helper.queryProjectedIndexRecordSize(), helper.queryIndexRecordSize(), getEnableFieldTranslation()));
+                translators.setown(varFileInfo->getTranslators(formatCrc, helper.queryProjectedIndexRecordSize(), helper.queryIndexRecordSize(), getEnableFieldTranslation(), true));
                 keySet.setown(varFileInfo->getKeyArray(false, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
             }
         }
@@ -26110,7 +26110,7 @@ public:
             if (varFileInfo)
             {
                 unsigned formatCrc = getFormatCrc(helper.getIndexFormatCrc());
-                translators.setown(varFileInfo->getTranslators(formatCrc, helper.queryProjectedIndexRecordSize(), helper.queryIndexRecordSize(), getEnableFieldTranslation()));
+                translators.setown(varFileInfo->getTranslators(formatCrc, helper.queryProjectedIndexRecordSize(), helper.queryIndexRecordSize(), getEnableFieldTranslation(), true));
                 keySet.setown(varFileInfo->getKeyArray(false, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
             }
         }
@@ -26346,7 +26346,7 @@ public:
             if (indexfile)
             {
                 unsigned formatCrc = getFormatCrc(helper->getIndexFormatCrc());
-                keyTranslators.setown(indexfile->getTranslators(formatCrc, helper->queryProjectedIndexRecordSize(), helper->queryIndexRecordSize(), getEnableFieldTranslation()));
+                keyTranslators.setown(indexfile->getTranslators(formatCrc, helper->queryProjectedIndexRecordSize(), helper->queryIndexRecordSize(), getEnableFieldTranslation(), true));
                 keySet.setown(indexfile->getKeyArray(isOpt, isLocal ? queryFactory.queryChannel() : 0));
             }
         }
@@ -26371,7 +26371,7 @@ public:
                 if (isLocal)  // Not sure this works
                 {
                     unsigned formatCrc = getFormatCrc(helper->getDiskFormatCrc());
-                    translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation()));
+                    translators.setown(datafile->getTranslators(formatCrc, helper->queryProjectedDiskRecordSize(), helper->queryDiskRecordSize(), getEnableFieldTranslation(), false));
                     files.setown(datafile->getIFileIOArray(isFetchOpt, queryFactory.queryChannel()));
                 }
                 else

roxie/ccd/ccdstate.hpp

@@ -103,6 +103,7 @@ interface IFileIOArray : extends IInterface
 interface ITranslatorSet : extends IInterface
 {
     virtual const IDynamicTransform *queryTranslator(unsigned subFile) const = 0;
+    virtual const IKeyTranslator *queryKeyTranslator(unsigned subFile) const = 0;
     virtual ISourceRowPrefetcher *getPrefetcher(unsigned subFile) const = 0;
     virtual IOutputMetaData *queryActualLayout(unsigned subFile) const = 0;
     virtual int queryTargetFormatCrc() const = 0;

rtl/eclrtl/rtldynfield.cpp

@@ -1005,9 +1005,13 @@ public:
     }
     virtual size32_t translate(ARowBuilder &builder, const byte *sourceRec) const override
     {
-        dbgassertex(canTranslate());
         return doTranslate(builder, 0, sourceRec);
     }
+    virtual size32_t translate(ARowBuilder &builder, const RtlRow &sourceRow) const override
+    {
+        sourceRow.lazyCalcOffsets(-1);  // MORE - could save the max one we actually need...
+        return doTranslate(builder, 0, sourceRow);
+    }
     virtual bool canTranslate() const override
     {
         return (matchFlags & match_fail) == 0;
@@ -1047,9 +1051,14 @@ private:
     {
         unsigned numOffsets = sourceRecInfo.getNumVarFields() + 1;
         size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t));
+        RtlRow sourceRow(sourceRecInfo, sourceRec, numOffsets, variableOffsets);  // MORE - could save the max source offset we actually need, and only set up that many...
+        return doTranslate(builder, 0, sourceRow);
+    }
+    size32_t doTranslate(ARowBuilder &builder, size32_t offset, const RtlRow &sourceRow) const
+    {
+        dbgassertex(canTranslate());
         byte * destConditions = (byte *)alloca(destRecInfo.getNumIfBlocks() * sizeof(byte));
         memset(destConditions, 2, destRecInfo.getNumIfBlocks() * sizeof(byte));
-        RtlRow sourceRow(sourceRecInfo, sourceRec, numOffsets, variableOffsets);
         size32_t estimate = destRecInfo.getFixedSize();
         if (!estimate)
         {
@@ -1071,7 +1080,7 @@ private:
                 unsigned matchField = match.matchIdx;
                 const RtlTypeInfo *sourceType = sourceRecInfo.queryType(matchField);
                 size_t sourceOffset = sourceRow.getOffset(matchField);
-                const byte *source = sourceRec + sourceOffset;
+                const byte *source = sourceRow.queryRow() + sourceOffset;
                 size_t copySize = sourceRow.getSize(matchField);
                 if (copySize == 0 && (match.matchType & match_inifblock))  // Field is missing because of an ifblock - use default value
                 {
@@ -1587,19 +1596,22 @@ extern ECLRTL_API IRowStream * transformRecord(IEngineRowAllocator * resultAlloc
 class CKeyTranslator : public CInterfaceOf<IKeyTranslator>
 {
 public:
-    CKeyTranslator(const RtlRecord &destRecInfo, const RtlRecord &srcRecInfo)
+    CKeyTranslator(const RtlRecord &actual, const RtlRecord &expected)
     {
-        for (unsigned idx = 0; idx < srcRecInfo.getNumFields(); idx++)
+        translateNeeded = false;
+        for (unsigned expectedIdx = 0; expectedIdx < expected.getNumFields(); expectedIdx++)
         {
-            unsigned matchIdx = destRecInfo.getFieldNum(srcRecInfo.queryName(idx));
-            if (matchIdx != -1)
+            unsigned actualIdx = actual.getFieldNum(expected.queryName(expectedIdx));
+            if (actualIdx != -1)
             {
-                const RtlTypeInfo *srcType = srcRecInfo.queryType(idx);
-                const RtlTypeInfo *destType = destRecInfo.queryType(idx);
-                if (!destType->equivalent(srcType))
-                    matchIdx = (unsigned) -2;
+                const RtlTypeInfo *expectedType = expected.queryType(expectedIdx);
+                const RtlTypeInfo *actualType = actual.queryType(actualIdx);
+                if (!actualType->equivalent(expectedType))
+                    actualIdx = (unsigned) -2;
             }
-            map.append(matchIdx);
+            map.append(actualIdx);
+            if (actualIdx != expectedIdx)
+                translateNeeded = true;
         }
     }
     virtual void describe() const override
@@ -1618,30 +1630,38 @@ public:
     virtual bool translate(RowFilter &filters) const override
     {
         bool mapNeeded = false;
-        unsigned numFields = filters.numFilterFields();
-        for (unsigned idx = 0; idx < numFields; idx++)
+        if (translateNeeded)
         {
-            unsigned fieldNum = filters.queryFilter(idx).queryFieldIndex();
-            unsigned mappedFieldNum = map.isItem(fieldNum) ? map.item(fieldNum) : (unsigned) -1;
-            if (mappedFieldNum != fieldNum)
+            unsigned numFields = filters.numFilterFields();
+            for (unsigned idx = 0; idx < numFields; idx++)
             {
-                mapNeeded = true;
-                switch (mappedFieldNum)
+                unsigned fieldNum = filters.queryFilter(idx).queryFieldIndex();
+                unsigned mappedFieldNum = map.isItem(fieldNum) ? map.item(fieldNum) : (unsigned) -1;
+                if (mappedFieldNum != fieldNum)
                 {
-                case (unsigned) -1: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - no matching field", idx);
-                case (unsigned) -2: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - incompatible matching field type", idx);
-                default:
-                    filters.remapField(idx, mappedFieldNum);
-                    break;
+                    mapNeeded = true;
+                    switch (mappedFieldNum)
+                    {
+                    case (unsigned) -1: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - no matching field", idx);
+                    case (unsigned) -2: throw makeStringExceptionV(0, "Cannot translate keyed filter on field %u - incompatible matching field type", idx);
+                    default:
+                        filters.remapField(idx, mappedFieldNum);
+                        break;
+                    }
                 }
             }
+            if (mapNeeded)
+                filters.recalcFieldsRequired();
         }
-        if (mapNeeded)
-            filters.recalcFieldsRequired();
         return mapNeeded;
     }
+    virtual bool needsTranslate() const
+    {
+        return translateNeeded;
+    }
 protected:
     UnsignedArray map;
+    bool translateNeeded = false;
 };
 
 extern ECLRTL_API const IKeyTranslator *createKeyTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo)

rtl/eclrtl/rtldynfield.hpp

@@ -108,6 +108,7 @@ interface IDynamicTransform : public IInterface
 {
     virtual void describe() const = 0;
     virtual size32_t translate(ARowBuilder &builder, const byte *sourceRec) const = 0;
+    virtual size32_t translate(ARowBuilder &builder, const RtlRow &sourceRow) const = 0;
     virtual bool canTranslate() const = 0;
     virtual bool needsTranslate() const = 0;
 };
@@ -117,6 +118,7 @@ interface IKeyTranslator : public IInterface
 {
     virtual void describe() const = 0;
     virtual bool translate(RowFilter &filters) const = 0;
+    virtual bool needsTranslate() const = 0;
 };
 
 extern ECLRTL_API const IDynamicTransform *createRecordTranslator(const RtlRecord &_destRecInfo, const RtlRecord &_srcRecInfo);

rtl/eclrtl/rtlnewkey.cpp

@@ -1369,6 +1369,31 @@ void RowFilter::extractKeyFilter(const RtlRecord & record, IConstArrayOf<IFieldF
     }
 }
 
+void RowFilter::extractMemKeyFilter(const RtlRecord & record, const UnsignedArray &sortOrder, IConstArrayOf<IFieldFilter> & keyFilters) const
+{
+    if (!filters)
+        return;
+
+    // for in-memory index, we want filters in the same order as the sort fields, with wilds added
+    ForEachItemIn(idx, sortOrder)
+    {
+        unsigned sortField = sortOrder.item(idx);
+        bool needWild = true;
+        ForEachItemIn(fidx, filters)
+        {
+            const IFieldFilter &filter = filters.item(fidx);
+            if (filter.queryFieldIndex()==sortField)
+            {
+                keyFilters.append(OLINK(filter));
+                needWild = false;
+                break;
+            }
+        }
+        if (needWild)
+            keyFilters.append(*createWildFieldFilter(sortField, *record.queryType(sortField)));
+    }
+}
+
 const IFieldFilter *RowFilter::findFilter(unsigned fieldNum) const
 {
     ForEachItemIn(i, filters)
@@ -1420,7 +1445,7 @@ void RowFilter::remapField(unsigned filterIdx, unsigned newFieldNum)
 
 bool RowCursor::setRowForward(const byte * row)
 {
-    currentRow.setRow(row, numFilterFields());
+    currentRow.setRow(row, numFieldsRequired);
 
     unsigned field = 0;
     //Now check which of the fields matches, and update matchedRanges to indicate

rtl/eclrtl/rtlnewkey.hpp

@@ -35,6 +35,7 @@ public:
     bool matches(const RtlRow & row) const;
 
     void extractKeyFilter(const RtlRecord & record, IConstArrayOf<IFieldFilter> & keyFilters) const;
+    void extractMemKeyFilter(const RtlRecord & record, const UnsignedArray &sortOrder, IConstArrayOf<IFieldFilter> & keyFilters) const;
     unsigned numFilterFields() const { return filters.ordinality(); }
     const IFieldFilter & queryFilter(unsigned i) const { return filters.item(i); }
     const IFieldFilter *findFilter(unsigned fieldIdx) const;
@@ -62,6 +63,15 @@ public:
         filter.extractKeyFilter(record, filters);
         ForEachItemIn(i, filters)
             matchedRanges.append(0);
+        numFieldsRequired = filter.getNumFieldsRequired();
+    }
+
+    RowCursor(const RtlRecord & record, const UnsignedArray &sortOrder, RowFilter & filter) : currentRow(record, nullptr)
+    {
+        filter.extractMemKeyFilter(record, sortOrder, filters);
+        ForEachItemIn(i, filters)
+            matchedRanges.append(0);
+        numFieldsRequired = filter.getNumFieldsRequired();
     }
 
     void selectFirst()
@@ -146,8 +156,9 @@ protected:
     unsigned numMatched = 0;
     unsigned nextUnmatchedRange = 0;
     UnsignedArray matchedRanges;
+    unsigned numFieldsRequired = 0;
     bool eos = false;
-    IConstArrayOf<IFieldFilter> filters; // for an index must be in field order, and all values present - more thought required
+    IConstArrayOf<IFieldFilter> filters;
 };
 
 interface ISourceRowCursor
@@ -168,6 +179,10 @@ public:
     {
     }
 
+    KeySearcher(const RtlRecord & _info, const UnsignedArray &_sortOrder, RowFilter & _filter, ISourceRowCursor * _rows) : cursor(_info, _sortOrder, _filter), rows(_rows)
+    {
+    }
+
     void reset()
     {
         rows->reset();

rtl/eclrtl/rtlrecord.cpp

@@ -204,7 +204,7 @@ static unsigned expandNestedRows(unsigned idx, unsigned startIdx, const char *pr
 class FieldNameToFieldNumMap
 {
 public:
-    FieldNameToFieldNumMap(const RtlRecord &record)
+    FieldNameToFieldNumMap(const RtlRecord &record) : map(true)
     {
         unsigned numFields = record.getNumFields();
         for (unsigned idx = 0; idx < numFields;idx++)

rtl/eclrtl/rtlrecord.hpp

@@ -283,7 +283,7 @@ public:
     void setRow(const void * _row, unsigned _numFieldsUsed = (unsigned) -1);
     void lazyCalcOffsets(unsigned _numFieldsUsed) const;
 
-    const byte *queryRow() const
+    inline const byte *queryRow() const
     {
         return row;
     }

testing/regress/ecl/key/translatedisk.xml

@@ -1,73 +1,43 @@
 <Dataset name='Result 1'>
- <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>0</filepos></Row>
- <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>25</filepos></Row>
- <Row><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>50</filepos></Row>
- <Row><dg_parentid>3</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>75</filepos></Row>
- <Row><dg_parentid>4</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>100</filepos></Row>
- <Row><dg_parentid>5</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>125</filepos></Row>
- <Row><dg_parentid>6</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>150</filepos></Row>
- <Row><dg_parentid>7</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>175</filepos></Row>
- <Row><dg_parentid>8</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>200</filepos></Row>
- <Row><dg_parentid>9</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>225</filepos></Row>
- <Row><dg_parentid>10</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>250</filepos></Row>
- <Row><dg_parentid>11</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>275</filepos></Row>
- <Row><dg_parentid>12</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>300</filepos></Row>
- <Row><dg_parentid>13</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>325</filepos></Row>
- <Row><dg_parentid>14</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>350</filepos></Row>
- <Row><dg_parentid>15</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>375</filepos></Row>
- <Row><dg_parentid>16</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>400</filepos></Row>
- <Row><dg_parentid>17</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>425</filepos></Row>
- <Row><dg_parentid>18</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>450</filepos></Row>
- <Row><dg_parentid>19</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>475</filepos></Row>
- <Row><dg_parentid>20</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>500</filepos></Row>
- <Row><dg_parentid>21</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>525</filepos></Row>
- <Row><dg_parentid>22</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>550</filepos></Row>
- <Row><dg_parentid>23</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>575</filepos></Row>
- <Row><dg_parentid>24</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>600</filepos></Row>
- <Row><dg_parentid>25</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>625</filepos></Row>
- <Row><dg_parentid>26</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>650</filepos></Row>
- <Row><dg_parentid>27</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>675</filepos></Row>
- <Row><dg_parentid>28</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>700</filepos></Row>
- <Row><dg_parentid>29</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>725</filepos></Row>
- <Row><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>750</filepos></Row>
- <Row><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>775</filepos></Row>
- <Row><dg_parentid>32</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>800</filepos></Row>
- <Row><dg_parentid>33</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>825</filepos></Row>
- <Row><dg_parentid>34</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>850</filepos></Row>
- <Row><dg_parentid>35</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>875</filepos></Row>
- <Row><dg_parentid>36</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>900</filepos></Row>
- <Row><dg_parentid>37</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>925</filepos></Row>
- <Row><dg_parentid>38</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>950</filepos></Row>
- <Row><dg_parentid>39</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>975</filepos></Row>
- <Row><dg_parentid>40</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>1000</filepos></Row>
- <Row><dg_parentid>41</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>1025</filepos></Row>
- <Row><dg_parentid>42</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>1050</filepos></Row>
- <Row><dg_parentid>43</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>1075</filepos></Row>
- <Row><dg_parentid>44</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>1100</filepos></Row>
- <Row><dg_parentid>45</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>1125</filepos></Row>
- <Row><dg_parentid>46</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>1150</filepos></Row>
- <Row><dg_parentid>47</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>1175</filepos></Row>
- <Row><dg_parentid>48</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><filepos>1200</filepos></Row>
- <Row><dg_parentid>49</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><filepos>1225</filepos></Row>
- <Row><dg_parentid>50</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><filepos>1250</filepos></Row>
- <Row><dg_parentid>51</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><filepos>1275</filepos></Row>
- <Row><dg_parentid>52</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><filepos>1300</filepos></Row>
- <Row><dg_parentid>53</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><filepos>1325</filepos></Row>
- <Row><dg_parentid>54</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><filepos>1350</filepos></Row>
- <Row><dg_parentid>55</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><filepos>1375</filepos></Row>
- <Row><dg_parentid>56</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><filepos>1400</filepos></Row>
- <Row><dg_parentid>57</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><filepos>1425</filepos></Row>
- <Row><dg_parentid>58</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><filepos>1450</filepos></Row>
- <Row><dg_parentid>59</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><filepos>1475</filepos></Row>
- <Row><dg_parentid>60</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><filepos>1500</filepos></Row>
- <Row><dg_parentid>61</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><filepos>1525</filepos></Row>
- <Row><dg_parentid>62</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><filepos>1550</filepos></Row>
- <Row><dg_parentid>63</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><filepos>1575</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>4</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>1</dg_prange><filepos>100</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>5</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>2</dg_prange><filepos>125</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>6</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>3</dg_prange><filepos>150</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>7</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>4</dg_prange><filepos>175</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>20</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_prange>1</dg_prange><filepos>500</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>21</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_prange>2</dg_prange><filepos>525</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>22</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_prange>3</dg_prange><filepos>550</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>23</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_prange>4</dg_prange><filepos>575</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>36</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_prange>1</dg_prange><filepos>900</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>37</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_prange>2</dg_prange><filepos>925</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>38</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_prange>3</dg_prange><filepos>950</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>39</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_prange>4</dg_prange><filepos>975</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>52</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_prange>1</dg_prange><filepos>1300</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>53</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_prange>2</dg_prange><filepos>1325</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>54</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_prange>3</dg_prange><filepos>1350</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>55</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_prange>4</dg_prange><filepos>1375</filepos></Row>
 </Dataset>
 <Dataset name='Result 2'>
- <Row><Result_2>----</Result_2></Row>
+ <Row><dg_lastname>BAYLISS   </dg_lastname><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>1</dg_prange><filepos>0</filepos></Row>
+ <Row><dg_lastname>BAYLISS   </dg_lastname><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>2</dg_prange><filepos>25</filepos></Row>
+ <Row><dg_lastname>BAYLISS   </dg_lastname><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>3</dg_prange><filepos>50</filepos></Row>
+ <Row><dg_lastname>BAYLISS   </dg_lastname><dg_parentid>3</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>4</dg_prange><filepos>75</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>4</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>1</dg_prange><filepos>100</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>5</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>2</dg_prange><filepos>125</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>6</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>3</dg_prange><filepos>150</filepos></Row>
+ <Row><dg_lastname>DOLSON    </dg_lastname><dg_parentid>7</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>4</dg_prange><filepos>175</filepos></Row>
+ <Row><dg_lastname>BILLINGTON</dg_lastname><dg_parentid>8</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>1</dg_prange><filepos>200</filepos></Row>
+ <Row><dg_lastname>BILLINGTON</dg_lastname><dg_parentid>9</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>2</dg_prange><filepos>225</filepos></Row>
+ <Row><dg_lastname>BILLINGTON</dg_lastname><dg_parentid>10</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>3</dg_prange><filepos>250</filepos></Row>
+ <Row><dg_lastname>BILLINGTON</dg_lastname><dg_parentid>11</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>4</dg_prange><filepos>275</filepos></Row>
+ <Row><dg_lastname>SMITH     </dg_lastname><dg_parentid>12</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>1</dg_prange><filepos>300</filepos></Row>
+ <Row><dg_lastname>SMITH     </dg_lastname><dg_parentid>13</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>2</dg_prange><filepos>325</filepos></Row>
+ <Row><dg_lastname>SMITH     </dg_lastname><dg_parentid>14</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>3</dg_prange><filepos>350</filepos></Row>
+ <Row><dg_lastname>SMITH     </dg_lastname><dg_parentid>15</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_prange>4</dg_prange><filepos>375</filepos></Row>
 </Dataset>
 <Dataset name='Result 3'>
+ <Row><Result_3>----</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
  <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>new</newfield><filepos>0</filepos></Row>
  <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>new</newfield><filepos>25</filepos></Row>
  <Row><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>50</filepos></Row>
@@ -133,10 +103,10 @@
  <Row><dg_parentid>62</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>new</newfield><filepos>1550</filepos></Row>
  <Row><dg_parentid>63</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>new</newfield><filepos>1575</filepos></Row>
 </Dataset>
-<Dataset name='Result 4'>
- <Row><Result_4>----</Result_4></Row>
-</Dataset>
 <Dataset name='Result 5'>
+ <Row><Result_5>----</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
  <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>0</filepos></Row>
  <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>25</filepos></Row>
  <Row><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>50</filepos></Row>
@@ -202,7 +172,7 @@
  <Row><dg_parentid>62</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1550</filepos></Row>
  <Row><dg_parentid>63</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1575</filepos></Row>
 </Dataset>
-<Dataset name='Result 6'>
+<Dataset name='Result 7'>
  <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>NEW</newfield><sub></sub><filepos>0</filepos></Row>
  <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>NEW</newfield><sub></sub><filepos>25</filepos></Row>
  <Row><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>50</filepos></Row>
@@ -268,3 +238,69 @@
  <Row><dg_parentid>62</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1550</filepos></Row>
  <Row><dg_parentid>63</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>NEW</newfield><sub></sub><filepos>1575</filepos></Row>
 </Dataset>
+<Dataset name='Result 8'>
+ <Row><dg_parentid>0</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>1</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>2</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>3</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>4</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>5</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>6</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>7</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>8</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>9</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>10</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>11</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>12</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>13</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>14</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>15</dg_parentid><dg_firstname>DAVID     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>16</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>17</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>18</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>19</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>20</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>21</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>22</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>23</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>24</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>25</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>26</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>27</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>28</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>29</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>30</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>31</dg_parentid><dg_firstname>CLAIRE    </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>32</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>33</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>34</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>35</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>36</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>37</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>38</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>39</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>40</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>41</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>42</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>43</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>44</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>45</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>46</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>47</dg_parentid><dg_firstname>KELLY     </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>48</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>49</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>50</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>51</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BAYLISS   </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>52</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>53</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>54</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>55</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>DOLSON    </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>56</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>57</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>58</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>59</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>BILLINGTON</dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>60</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>1</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>61</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>2</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>62</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>3</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+ <Row><dg_parentid>63</dg_parentid><dg_firstname>KIMBERLY  </dg_firstname><dg_lastname>SMITH     </dg_lastname><dg_prange>4</dg_prange><newfield>Translated Full Keyed</newfield><filepos>0</filepos></Row>
+</Dataset>

testing/regress/ecl/translatedisk.ecl

@@ -31,9 +31,19 @@ import $.Setup;
 boolean useLocal := false;
 Files := Setup.Files(multiPart, useLocal);
 
-DG_FlatFile := PRELOAD(DATASET(Files.DG_FileOut+'FLAT',{Files.DG_OutRec, UNSIGNED8 filepos{virtual(fileposition)}},FLAT));
-output(DG_FlatFile);
+DG_FlatFile := PRELOAD(DATASET(Files.DG_FileOut+'FLAT',{Files.DG_OutRec.DG_LastName, Files.DG_OutRec, UNSIGNED8 filepos{virtual(fileposition)}},FLAT, HINT(key('''
+<Keys>
+ <MemIndex>
+  <FieldSet>
+   <Field name='DG_LastName'/>
+  </FieldSet>
+ </MemIndex>
+</Keys>
+'''))));
+output(DG_FlatFile(KEYED(DG_lastname = 'DOLSON')));  // Should use in-memory key
+output(DG_FlatFile(KEYED(DG_firstname = 'DAVID')));  // Should use in-memory, but no key
 OUTPUT('----');
+
 DG_FlatFile_add1 := PRELOAD(DATASET(Files.DG_FileOut+'FLAT',{Files.DG_OutRec,STRING newfield { default('new')}, UNSIGNED8 filepos{virtual(fileposition)}},FLAT));
 output(DG_FlatFile_add1);
 OUTPUT('----');
@@ -47,3 +57,10 @@ output(DG_FlatFile_add2);
 
 d := table(DG_FlatFile, {unsigned8 fpos := DG_FlatFile.filepos} );
 output(FETCH(DG_FlatFile_add2, d, right.fpos, TRANSFORM (RECORDOF(DG_FlatFile_add2), SELF.filepos := right.fpos; SELF := LEFT)));
+
+DG_FlatFile_add3 := DATASET(Files.DG_FileOut+'FLAT',{Files.DG_OutRec,STRING newfield { default('Translated Full Keyed')}, UNSIGNED8 filepos{virtual(fileposition)}},FLAT);
+fkj :=JOIN(Files.DG_FlatFile, DG_FlatFile_add3, left.DG_firstname = right.DG_firstname 
+         AND left.DG_lastname=right.DG_lastname 
+         AND left.DG_Prange=right.DG_Prange     
+      , TRANSFORM(RECORDOF(DG_FlatFile_add3), SELF := RIGHT), KEYED(Files.DG_IndexFile));
+output(fkj);