HPCC-18672 Access fileposition from an index row instead of special casing

The current indexes store a fileposition with each index entry, but stored
before the row.  Previously this value was accessed by adding extra
parameters to functions and functions to interfaces.

This commit moves the fileposition into the logical row by appending it.  This
means it can be treated in the same way as all the other fields in a key.
The code generator now retrieves the fileposition from the row, and the layout
mapping code is simplified to remove the special casing.

The commit ensures CRCs for index files stay identical to avoid backward
compatibility problems.

A future commit will modify the activity helpers and other code that can now be
simplified.

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/fileview2/fvidxsource.cpp

@@ -304,8 +304,7 @@ bool IndexDataSource::getNextRow(MemoryBuffer & out, bool extractRow)
                 }
                 else
                 {
-                    offset_t filepos;
-                    const byte * thisRow = manager->queryKeyBuffer(filepos);
+                    const byte * thisRow = manager->queryKeyBuffer();
                     unsigned thisSize = diskMeta->getRecordSize(thisRow);
                     void * temp = out.reserve(thisSize);
                     memcpy(temp, thisRow, thisSize);
@@ -451,7 +450,7 @@ void IndexDataSource::applyFilter()
         manager->reset();
         while (manager->lookup(false))
         {
-            offset_t node = manager->queryFpos();
+            offset_t node = extractFpos(manager);
             if (node)
                 matchingParts.append((unsigned)(node-1));
         }

common/remote/sockfile.cpp

@@ -2931,7 +2931,6 @@ class CRemoteKeyManager : public CSimpleInterfaceOf<IKeyManager>
     MemoryBuffer keyCursorMb;        // used for continuation
     unsigned __int64 totalGot = 0;
     size32_t currentSize = 0;
-    offset_t currentFpos = 0;
     const byte *currentRow = nullptr;
     bool first = true;
     unsigned __int64 chooseNLimit = 0;
@@ -3016,7 +3015,6 @@ public:
         rowDataRemaining = 0;
         keyCursorMb.clear();
         currentSize = 0;
-        currentFpos = 0;
         currentRow = nullptr;
         first = true;
         totalGot = 0;
@@ -3030,19 +3028,12 @@ public:
         }
         segs.reset();
     }
-    virtual const byte *queryKeyBuffer(offset_t & fpos) override
+    virtual const byte *queryKeyBuffer() override
     {
         if (!remoteSupport())
-            return directKM->queryKeyBuffer(fpos);;
-        fpos = currentFpos;
+            return directKM->queryKeyBuffer();
         return currentRow;
     }
-    virtual offset_t queryFpos() override
-    {
-        if (!remoteSupport())
-            return directKM->queryFpos();
-        return currentFpos;
-    }
     virtual unsigned queryRecordSize() override
     {
         if (!remoteSupport())
@@ -3069,10 +3060,9 @@ public:
         {
             if (rowDataRemaining)
             {
-                rowDataBuffer.read(currentFpos);
                 rowDataBuffer.read(currentSize);
                 currentRow = rowDataBuffer.readDirect(currentSize);
-                rowDataRemaining -= sizeof(currentFpos) + sizeof(currentSize) + currentSize;
+                rowDataRemaining -= sizeof(currentSize) + currentSize;
                 return true;
             }
             else
@@ -4785,9 +4775,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
         while (keyManager->lookup(true))
         {
             unsigned size = keyManager->queryRecordSize();
-            offset_t fpos;
-            const byte *result = keyManager->queryKeyBuffer(fpos);
-            reply.append(fpos);
+            const byte *result = keyManager->queryKeyBuffer();
             reply.append(size);
             reply.append(size, result);
             ++numRecs;

common/thorhelper/layouttrans.cpp

@@ -257,29 +257,9 @@ void RowTransformer::createRowRecord(FieldMapping const & mapping, CIArrayOf<Row
     switch(mapping.queryType())
     {
     case FieldMapping::Simple:
-        if(mapping.isDiskFieldFpos())
-            if(mapping.isActivityFieldFpos())
-            {
-                ensureItem(records, activityFieldNum).setVals(0, 0, diskSize, false).setFpos(true, true);
-                prevActivityField = false;
-            }
-            else
-            {
-                ensureItem(records, activityFieldNum).setVals(0, 0, diskSize, false).setFpos(false, true);
-                prevActivityField = false;
-            }
-        else
-            if(mapping.isActivityFieldFpos())
-            {
-                ensureItem(records, activityFieldNum).setVals(diskOffset, numVarFields, diskSize, false).setFpos(true, false);
-                prevActivityField = false;
-            }
-            else
-            {
-                ensureItem(records, activityFieldNum).setVals(diskOffset, numVarFields, diskSize, (prevActivityField && (activityFieldNum == (prevActivityFieldNum+1))));
-                prevActivityField = true;
-                prevActivityFieldNum = activityFieldNum;
-            }
+        ensureItem(records, activityFieldNum).setVals(diskOffset, numVarFields, diskSize, (prevActivityField && (activityFieldNum == (prevActivityFieldNum+1))));
+        prevActivityField = true;
+        prevActivityFieldNum = activityFieldNum;
         break;
 
     case FieldMapping::ChildDataset:
@@ -337,47 +317,13 @@ void RowTransformer::generateSimpleCopy(unsigned & seq, RowRecord const & record
         copy.setChildTransformer(new RowTransformer(seq, *childMappings));
 }
 
-void RowTransformer::generateCopyToFpos(RowRecord const & record)
-{
-    copyToFpos = true;
-    copyToFposRelOffset = record.queryRelOffset();
-    copyToFposRelBase = record.queryRelBase();
-    copyToFposSize = record.querySize();
-    assertex(copyToFposSize != UNKNOWN_LENGTH);
-    assertex(copyToFposSize <= sizeof(offset_t));
-}
-
-void RowTransformer::generateCopyFromFpos(RowRecord const & record)
-{
-    assertex(sequence == 0);
-    copies.append(*new FieldCopy(static_cast<unsigned>(-1), 0, 0));
-    size32_t size = record.querySize();
-    assertex(size != UNKNOWN_LENGTH);
-    assertex(size <= sizeof(offset_t));
-    copies.tos().addFixedSize(size);
-}
-
 void RowTransformer::generateCopies(unsigned & seq, CIArrayOf<RowRecord> const & records)
 {
     sequence = seq++;
     ForEachItemIn(fieldNum, records)
     {
         RowRecord const & record = records.item(fieldNum);
-        if(record.isToFpos())
-        {
-            assertex(sequence == 0);
-            if(record.isFromFpos())
-                keepFpos = true;
-            else
-                generateCopyToFpos(record);
-        }
-        else
-        {
-            if(record.isFromFpos())
-                generateCopyFromFpos(record);
-            else
-                generateSimpleCopy(seq, record);
-        }
+        generateSimpleCopy(seq, record);
     }
 }
 
@@ -406,25 +352,6 @@ void RowTransformer::transform(IRecordLayoutTranslator::RowTransformContext * ct
         copies.item(copyIdx).copy(ctx, out, outOffset);
 }
 
-void RowTransformer::getFposOut(IRecordLayoutTranslator::RowTransformContext const * ctx, offset_t & fpos) const
-{
-    if(copyToFpos)
-    {
-        fpos = 0;
-        const byte * in = ctx->queryPointer(0, copyToFposRelBase) + copyToFposRelOffset;
-        // integer field in row is big-endian
-#if __BYTE_ORDER == __BIG_ENDIAN
-        memcpy(reinterpret_cast<byte const *>(&fpos) + sizeof(offset_t) - copyToFposSize, in, copyToFposSize);
-#else
-        _cpyrevn(&fpos, in, copyToFposSize);
-#endif
-    }
-    else if(!keepFpos)
-    {
-        fpos = 0;
-    }
-}
-
 void RowTransformer::createRowTransformContext(IRecordLayoutTranslator::RowTransformContext * ctx) const
 {
     ctx->init(sequence, diskVarFieldRelOffsets.ordinality()+1);
@@ -438,19 +365,6 @@ void RowTransformer::createRowTransformContext(IRecordLayoutTranslator::RowTrans
 
 void FieldCopy::copy(IRecordLayoutTranslator::RowTransformContext * ctx, IMemoryBlock & out, size32_t & outOffset) const
 {
-    if(sequence == static_cast<unsigned>(-1))
-    {
-        byte * target = out.ensure(outOffset+fixedSize);
-        // integer field in row is big-endian
-#if __BYTE_ORDER == __BIG_ENDIAN
-        memcpy(target+outOffset, reinterpret_cast<byte const *>(ctx->queryFposIn()) + sizeof(offset_t) - fixedSize, fixedSize);
-#else
-        _cpyrevn(target+outOffset, ctx->queryFposIn(), fixedSize);
-#endif
-        outOffset += fixedSize;
-        return;
-    }
-
     size32_t diskFieldSize = fixedSize;
     ForEachItemIn(varIdx, varFields)
         diskFieldSize += ctx->querySize(sequence, varFields.item(varIdx));
@@ -617,13 +531,11 @@ IRecordLayoutTranslator::RowTransformContext * CRecordLayoutTranslator::getRowTr
     return ctx.getClear();
 }
 
-size32_t CRecordLayoutTranslator::transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out, offset_t & fpos) const
+size32_t CRecordLayoutTranslator::transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out) const
 {
     size32_t inOffset = 0;
     size32_t outOffset = 0;
-    ctx->setFposIn(fpos);
     transformer.transform(ctx, in, inSize, inOffset, out, outOffset);
-    transformer.getFposOut(ctx, fpos);
     return outOffset;
 }
 

common/thorhelper/layouttrans.hpp

@@ -53,16 +53,13 @@ public:
         ~RowTransformContext();
         void init(unsigned seq, unsigned num) { sizes[seq] = new unsigned[num]; ptrs[seq] = new byte const *[num]; }
         void set(unsigned seq, unsigned i, size32_t size, byte const * ptr) { sizes[seq][i] = size; ptrs[seq][i] = ptr; }
-        void setFposIn(offset_t fpos) { fposIn = fpos; }
 
         size32_t querySize(unsigned seq, unsigned i) const { return sizes[seq][i]; }
         byte const * queryPointer(unsigned seq, unsigned i) const { return ptrs[seq][i]; }
-        offset_t const * queryFposIn() const { return &fposIn; }
     private:
         unsigned num;
         size32_t * * sizes;
         byte const * * * ptrs;
-        offset_t fposIn;
     };
 
     typedef enum { NoTranslation = 0, TranslateAll = 1, TranslatePayload = 2 } Mode;
@@ -74,7 +71,7 @@ public:
     virtual SegmentMonitorContext * getSegmentMonitorContext() = 0;
     virtual void createDiskSegmentMonitors(SegmentMonitorContext const & in, IIndexReadContext & out) = 0;
     virtual RowTransformContext * getRowTransformContext() = 0;
-    virtual size32_t transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out, offset_t & fpos) const = 0;
+    virtual size32_t transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out) const = 0;
 #ifdef DEBUG_HELPERS_REQUIRED
     virtual StringBuffer & getMappingsAsString(StringBuffer & out) const = 0;
 #endif

common/thorhelper/layouttrans.ipp

@@ -57,12 +57,10 @@ public:
     char const * queryDiskFieldName() const { return str(diskRecord->queryChild(diskFieldNum)->queryName()); }
     unsigned queryDiskFieldNum() const { return diskFieldNum; }
     size32_t queryDiskFieldSize() const { return diskRecord->queryChild(diskFieldNum)->queryType()->getSize(); }
-    bool isDiskFieldFpos() const { return ((type != ChildDataset) && !diskFieldKeyed && (diskFieldNum == diskRecord->numChildren()-1) && diskRecord->queryChild(diskFieldNum)->queryType()->isInteger()); }
     bool isDiskFieldSet() const { return (diskRecord->queryChild(diskFieldNum)->queryType()->getTypeCode() == type_set); }
     char const * queryActivityFieldName() const { return str(activityRecord->queryChild(activityFieldNum)->queryName()); }
     unsigned queryActivityFieldNum() const { return activityFieldNum; }
     size32_t queryActivityFieldSize() const { return activityRecord->queryChild(activityFieldNum)->queryType()->getSize(); }
-    bool isActivityFieldFpos() const { return ((type == Simple) && !activityFieldKeyed && (activityFieldNum == activityRecord->numChildren()-1) && activityRecord->queryChild(activityFieldNum)->queryType()->isInteger()); }
     List const & queryChildMappings() const { return childMappings; }
 
 private:
@@ -121,23 +119,19 @@ public:
     RowTransformer(unsigned & seq, FieldMapping::List const & mappings) { build(seq, mappings); }
     void build(unsigned & seq, FieldMapping::List const & mappings);
     void transform(IRecordLayoutTranslator::RowTransformContext * ctx, byte const * in, size32_t inSize, size32_t & inOffset, IMemoryBlock & out, size32_t & outOffset) const;
-    void getFposOut(IRecordLayoutTranslator::RowTransformContext const * ctx, offset_t & fpos) const;
     void createRowTransformContext(IRecordLayoutTranslator::RowTransformContext * ctx) const;
 
 private:
     class RowRecord : public CInterface
     {
     public:
-        RowRecord() : relOffset(0), relBase(0), size(0), followOn(false), childMappings(NULL), toFpos(false), fromFpos(false) {}
+        RowRecord() : relOffset(0), relBase(0), size(0), followOn(false), childMappings(NULL) {}
         RowRecord & setVals(size32_t _relOffset, unsigned _relBase, size32_t _size, bool _followOn) { relOffset = _relOffset; relBase = _relBase; size = _size; followOn = _followOn; return *this; }
         RowRecord & setChildMappings(FieldMapping::List const * _mappings) { childMappings = _mappings; return *this; }
-        RowRecord & setFpos(bool to, bool from) { toFpos = to; fromFpos = from; return *this; }
         size32_t queryRelOffset() const { return relOffset; }
         unsigned queryRelBase() const { return relBase; }
         size32_t querySize() const { return size; }
         bool queryFollowOn() const { return followOn; }
-        bool isToFpos() const { return toFpos; }
-        bool isFromFpos() const { return fromFpos; }
         FieldMapping::List const * queryChildMappings() const { return childMappings; }
     private:
         size32_t relOffset;
@@ -145,16 +139,12 @@ private:
         size32_t size;
         bool followOn;
         FieldMapping::List const * childMappings;
-        bool toFpos;
-        bool fromFpos;
     };
 
     RowRecord & ensureItem(CIArrayOf<RowRecord> & arr, unsigned pos);
     void createRowRecord(FieldMapping const & mapping, CIArrayOf<RowRecord> & records, size32_t diskOffset, unsigned numVarFields, bool & prevActivityField, unsigned & prevActivityFieldNum);
     void analyseMappings(FieldMapping::List const & mappings, CIArrayOf<RowRecord> & records);
     void generateSimpleCopy(unsigned & seq, RowRecord const & record);
-    void generateCopyToFpos(RowRecord const & record);
-    void generateCopyFromFpos(RowRecord const & record);
     void generateCopies(unsigned & seq, CIArrayOf<RowRecord> const & records);
 
 private:
@@ -201,7 +191,7 @@ public:
     virtual SegmentMonitorContext * getSegmentMonitorContext() { return new ExpandedSegmentMonitorList(this); }
     virtual void createDiskSegmentMonitors(SegmentMonitorContext const & in, IIndexReadContext & out);
     virtual RowTransformContext * getRowTransformContext();
-    virtual size32_t transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out, offset_t & fpos) const;
+    virtual size32_t transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out) const;
 #ifdef DEBUG_HELPERS_REQUIRED
     virtual StringBuffer & getMappingsAsString(StringBuffer & out) const;
 #endif

ecl/hql/hqlutil.cpp

@@ -5139,6 +5139,14 @@ IHqlExpression * removeOperand(IHqlExpression * expr, IHqlExpression * operand)
     return expr->clone(args);
 }
 
+extern HQL_API IHqlExpression * removeChild(IHqlExpression * expr, unsigned child)
+{
+    HqlExprArray args;
+    unwindChildren(args, expr);
+    args.remove(child);
+    return expr->clone(args);
+}
+
 IHqlExpression * removeChildOp(IHqlExpression * expr, node_operator op)
 {
     HqlExprArray args;

ecl/hql/hqlutil.hpp

@@ -199,6 +199,7 @@ extern HQL_API IHqlExpression * removeLocalAttribute(IHqlExpression * expr);
 extern HQL_API IHqlExpression * removeAttribute(IHqlExpression * expr, IAtom * attr);
 extern HQL_API IHqlExpression * removeOperand(IHqlExpression * expr, IHqlExpression * operand);
 extern HQL_API IHqlExpression * removeChildOp(IHqlExpression * expr, node_operator op);
+extern HQL_API IHqlExpression * removeChild(IHqlExpression * expr, unsigned child);
 extern HQL_API IHqlExpression * appendAttribute(IHqlExpression * expr, IAtom * attr);
 extern HQL_API IHqlExpression * appendOwnedOperand(IHqlExpression * expr, IHqlExpression * ownedOperand);
 extern HQL_API IHqlExpression * replaceOwnedAttribute(IHqlExpression * expr, IHqlExpression * ownedAttribute);

ecl/hqlcpp/hqlckey.cpp

@@ -1477,13 +1477,13 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedJoinOrDenormalize(BuildCt
     //virtual unsigned getKeepLimit()
     doBuildJoinRowLimitHelper(*instance, rowlimit, info.queryKeyFilename(), implicitLimit);
 
-    buildFormatCrcFunction(instance->classctx, "getIndexFormatCrc", info.queryRawKey(), info.queryRawKey(), 1);
+    buildFormatCrcFunction(instance->classctx, "getIndexFormatCrc", true, info.queryRawKey(), info.queryRawKey(), 1);
     if (info.isFullJoin())
     {
         //Remove virtual attributes from the record, so the crc will be compatible with the disk read record
         //can occur with a (highly unusual) full keyed join to a persist file... (see indexread14.ecl)
         OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(info.queryRawRhs()->queryRecord());
-        buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", noVirtualRecord, NULL, 0);
+        buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", false, noVirtualRecord, NULL, 0);
         buildEncryptHelper(instance->startctx, info.queryFile()->queryAttribute(encryptAtom), "getFileEncryptKey");
     }
 
@@ -1597,7 +1597,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedDistribute(BuildCtx & ctx
 
     doCompareLeftRight(instance->nestedctx, "CompareRowKey", leftDs, rightDs, joinInfo.queryLeftReq(), normalizedRight);
 
-    buildFormatCrcFunction(instance->classctx, "getFormatCrc", info.queryRawKey(), info.queryRawKey(), 1);
+    buildFormatCrcFunction(instance->classctx, "getFormatCrc", true, info.queryRawKey(), info.queryRawKey(), 1);
     buildSerializedLayoutMember(instance->classctx, indexRecord, "getIndexLayout", numKeyedFields);
 
     OwnedHqlExpr matchExpr = info.getMatchExpr(true);

ecl/hqlcpp/hqlcpp.ipp

@@ -1816,7 +1816,7 @@ public:
     void endNestedClass(IHqlStmt * stmt);
 
     void buildEncryptHelper(BuildCtx & ctx, IHqlExpression * encryptAttr, const char * funcname = NULL);
-    void buildFormatCrcFunction(BuildCtx & ctx, const char * name, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta);
+    void buildFormatCrcFunction(BuildCtx & ctx, const char * name, bool removeFilepos, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta);
     void buildLimitHelpers(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * filename, unique_id_t id);
     void buildLimitHelpers(BuildCtx & ctx, IHqlExpression * rowLimit, IHqlExpression * failAction, bool isSkip, IHqlExpression * filename, unique_id_t id);
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9956,19 +9956,26 @@ void HqlCppTranslator::buildRecordEcl(BuildCtx & subctx, IHqlExpression * record
 }
 
 
-void HqlCppTranslator::buildFormatCrcFunction(BuildCtx & ctx, const char * name, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta)
+void HqlCppTranslator::buildFormatCrcFunction(BuildCtx & ctx, const char * name, bool removeFilepos, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta)
 {
     IHqlExpression * payload = expr ? expr->queryAttribute(_payload_Atom) : NULL;
     // MORE - do we need to keep this consistent - if so will have to trim out the originals and the filepos
     OwnedHqlExpr exprToCrc = getSerializedForm(dataset->queryRecord(), diskAtom);
 
-    unsigned payloadSize = 1;
+    unsigned payloadSize = getBoolAttribute(expr, filepositionAtom, true) ? 1 : 0;
     if (payload)
-        payloadSize = (unsigned)getIntValue(payload->queryChild(0)) + payloadDelta;
+        payloadSize = (unsigned)getIntValue(payload->queryChild(0));
 
     //FILEPOSITION(FALSE) means we have counted 1 too many in the payload
-    if (!getBoolAttribute(expr, filepositionAtom, true))
-        payloadSize--;
+    if (getBoolAttribute(expr, filepositionAtom, true) && removeFilepos)
+    {
+        assertex(payloadSize);
+        //Backward compatibility - remove the fileposition field from the CRC
+        HqlExprArray args;
+        unwindChildren(args, exprToCrc);
+        args.pop();
+        exprToCrc.setown(exprToCrc->clone(args));
+    }
 
     exprToCrc.setown(createComma(exprToCrc.getClear(), getSizetConstant(payloadSize)));
 
@@ -10372,7 +10379,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH
 
     OwnedHqlExpr rawRecord;
     doBuildIndexOutputTransform(instance->startctx, record, rawRecord, hasFileposition, expr->queryAttribute(maxLengthAtom));
-    buildFormatCrcFunction(instance->classctx, "getFormatCrc", rawRecord, expr, 0);
+    buildFormatCrcFunction(instance->classctx, "getFormatCrc", false, rawRecord, expr, 0);
 
     if (compressAttr && compressAttr->hasAttribute(rowAtom))
     {
@@ -10673,7 +10680,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
         if (!pipe)
         {
             OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(dataset->queryRecord());
-            buildFormatCrcFunction(instance->classctx, "getFormatCrc", noVirtualRecord, NULL, 0);
+            buildFormatCrcFunction(instance->classctx, "getFormatCrc", false, noVirtualRecord, NULL, 0);
         }
 
         bool grouped = isGrouped(dataset);

ecl/hqlcpp/hqlsource.cpp

@@ -408,23 +408,18 @@ static void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression
             {
                 OwnedHqlExpr target = createSelectExpr(LINK(self), LINK(cur));
                 OwnedHqlExpr newValue;
-                if (idx2  == fileposIndex)
-                    newValue.setown(getFilepos(diskDataset, false));
-                else
+                IHqlExpression * curPhysical = nextDiskField(diskRecord, diskIndex);
+                OwnedHqlExpr physicalSelect = createSelectExpr(LINK(diskDataset), LINK(curPhysical));
+                if (cur->isDatarow() && !cur->hasAttribute(blobAtom) && (!isInPayload() || (physicalSelect->queryType() != target->queryType())))
                 {
-                    IHqlExpression * curPhysical = nextDiskField(diskRecord, diskIndex);
-                    OwnedHqlExpr physicalSelect = createSelectExpr(LINK(diskDataset), LINK(curPhysical));
-                    if (cur->isDatarow() && !cur->hasAttribute(blobAtom) && (!isInPayload() || (physicalSelect->queryType() != target->queryType())))
-                    {
-                        HqlExprArray subassigns;
-                        OwnedHqlExpr childSelf = createSelector(no_self, cur, NULL);
-                        createPhysicalLogicalAssigns(subassigns, childSelf, curPhysical->queryRecord(), cur->queryRecord(), physicalSelect, false, NotFound);
-                        OwnedHqlExpr transform = createValue(no_transform, makeTransformType(cur->queryRecord()->getType()), subassigns);
-                        newValue.setown(createRow(no_createrow, transform.getClear()));
-                    }
-                    else
-                        newValue.setown(convertIndexPhysical2LogicalValue(cur, physicalSelect, allowTranslate));
+                    HqlExprArray subassigns;
+                    OwnedHqlExpr childSelf = createSelector(no_self, cur, NULL);
+                    createPhysicalLogicalAssigns(subassigns, childSelf, curPhysical->queryRecord(), cur->queryRecord(), physicalSelect, false, NotFound);
+                    OwnedHqlExpr transform = createValue(no_transform, makeTransformType(cur->queryRecord()->getType()), subassigns);
+                    newValue.setown(createRow(no_createrow, transform.getClear()));
                 }
+                else
+                    newValue.setown(convertIndexPhysical2LogicalValue(cur, physicalSelect, allowTranslate && (idx2 != fileposIndex)));
 
                 if (newValue)
                     assigns.append(*createAssign(target.getClear(), newValue.getClear()));
@@ -524,11 +519,20 @@ static IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IH
             }
         }
     }
-    if (hasInternalFileposition && createKeyedTypes)
+    if (hasInternalFileposition)
     {
-        IHqlExpression * cur = record->queryChild(record->numChildren()-1);
-        IHqlExpression *fposField = createField(cur->queryId(), makeFilePosType(cur->getType()), nullptr, extractFieldAttrs(cur));
-        physicalFields.append(*fposField);
+        if (createKeyedTypes)
+        {
+            IHqlExpression * cur = record->queryChild(record->numChildren()-1);
+            IHqlExpression *fposField = createField(cur->queryId(), makeFilePosType(cur->getType()), nullptr, extractFieldAttrs(cur));
+            physicalFields.append(*fposField);
+        }
+        else
+        {
+            IHqlExpression * cur = record->queryChild(record->numChildren()-1);
+            IHqlExpression *fposField = createField(cur->queryId(), makeSwapIntType(8, false), nullptr, extractFieldAttrs(cur));
+            physicalFields.append(*fposField);
+        }
     }
 
     return createRecord(physicalFields);
@@ -559,12 +563,11 @@ IHqlExpression * HqlCppTranslator::convertToPhysicalIndex(IHqlExpression * table
 
     unsigned payload = numPayloadFields(tableExpr);
     assertex(payload || !hasFileposition);
-    unsigned newPayload = hasFileposition ? payload-1 : payload;
     HqlExprArray args;
     unwindChildren(args, tableExpr);
     args.replace(*diskRecord, 1);
     removeAttribute(args, _payload_Atom);
-    args.append(*createAttribute(_payload_Atom, getSizetConstant(newPayload)));
+    args.append(*createAttribute(_payload_Atom, getSizetConstant(payload)));
     args.append(*createAttribute(_original_Atom, LINK(tableExpr)));
 
     //remove the preload attribute and replace with correct value
@@ -2380,9 +2383,17 @@ void SourceBuilder::buildGlobalGroupAggregateHelpers(IHqlExpression * expr)
 
     //virtual void processRows(void * self, size32_t srcLen, const void * src) = 0;
     {
+        OwnedHqlExpr newTableExpr = LINK(tableExpr);
+        if (isKey(tableExpr))
+        {
+            IHqlExpression * record = tableExpr->queryRecord();
+            OwnedHqlExpr newRecord = removeChild(record, record->numChildren()-1);
+            newTableExpr.setown(replaceChild(tableExpr, 1, newRecord));
+        }
+
         MemberFunction func(translator, instance->startctx, "virtual void processRows(size32_t srcLen, const void * _left, IHThorGroupAggregateCallback * callback) override");
         func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
-        OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(tableExpr->getType()));
+        OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(newTableExpr->getType()));
         OwnedHqlExpr len = createVariable("srcLen", LINK(sizetType));
         OwnedHqlExpr fullDs = createTranslated(ds, len);
 
@@ -2757,7 +2768,7 @@ void DiskReadBuilderBase::buildMembers(IHqlExpression * expr)
     {
         //Spill files can still have virtual attributes in their physical records => remove them.
         OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(physicalRecord);
-        translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", noVirtualRecord, NULL, 0);
+        translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", false, noVirtualRecord, NULL, 0);
     }
 
     buildLimits(instance->startctx, expr, instance->activityId); 
@@ -6281,8 +6292,6 @@ public:
     IndexReadBuilderBase(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
         : SourceBuilder(_translator, _tableExpr, _nameExpr), monitors(_tableExpr, _translator, -(int)numPayloadFields(_tableExpr), false)
     { 
-        fpos.setown(getFilepos(tableExpr, false));
-        lfpos.setown(getFilepos(tableExpr, true));
     }
 
     virtual void buildMembers(IHqlExpression * expr);
@@ -6338,7 +6347,7 @@ void IndexReadBuilderBase::buildMembers(IHqlExpression * expr)
 
     buildKeyedLimitHelper(expr);
 
-    translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", tableExpr, tableExpr, 1);
+    translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", true, tableExpr, tableExpr, 1);
     IHqlExpression * originalKey = queryAttributeChild(tableExpr, _original_Atom, 0);
     translator.buildSerializedLayoutMember(instance->classctx, originalKey->queryRecord(), "getIndexLayout", numKeyedFields(originalKey));
 
@@ -6644,9 +6653,13 @@ void IndexAggregateBuilder::buildMembers(IHqlExpression * expr)
     }
 
     {
+        IHqlExpression * record = tableExpr->queryRecord();
+        OwnedHqlExpr newRecord = removeChild(record, record->numChildren()-1);
+        OwnedHqlExpr newTableExpr = replaceChild(tableExpr, 1, newRecord);
+
         MemberFunction func(translator, instance->startctx, "virtual void processRows(ARowBuilder & crSelf, size32_t srcLen, const void * _left) override");
         func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
-        OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(tableExpr->getType()));
+        OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(newTableExpr->getType()));
         OwnedHqlExpr len = createVariable("srcLen", LINK(sizetType));
         OwnedHqlExpr fullDs = createTranslated(ds, len);
 
@@ -7248,7 +7261,7 @@ void FetchBuilder::buildMembers(IHqlExpression * expr)
     case no_json:
         break;
     default:
-        translator.buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", physicalRecord, NULL, 0);
+        translator.buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", false, physicalRecord, NULL, 0);
         break;
     }
 

ecl/hthor/hthorkey.cpp

@@ -91,6 +91,10 @@ bool rltEnabled(IConstWorkUnit const * wu)
 
 IRecordLayoutTranslator * getRecordLayoutTranslator(IDefRecordMeta const * activityMeta, size32_t activityMetaSize, void const * activityMetaBuff, IDistributedFile * df, IRecordLayoutTranslatorCache * cache, IRecordLayoutTranslator::Mode mode)
 {
+    //Disallow any keyed translation
+    if (mode == IRecordLayoutTranslator::TranslateAll)
+        mode = IRecordLayoutTranslator::TranslatePayload;
+
     IPropertyTree const & props = df->queryAttributes();
     MemoryBuffer diskMetaBuff;
     if(!props.getPropBin("_record_layout", diskMetaBuff))
@@ -140,7 +144,7 @@ public:
 //IThorIndexCallback
     virtual unsigned __int64 getFilePosition(const void * row)
     {
-        return filepos;
+        throwUnexpected();
     }
     virtual byte * lookupBlob(unsigned __int64 id) 
     { 
@@ -150,7 +154,6 @@ public:
 
 
 public:
-    offset_t & getFPosRef()                                 { return filepos; }
     void setManager(IKeyManager * _manager)
     {
         finishedRow();
@@ -165,7 +168,6 @@ public:
 
 protected:
     IKeyManager * keyManager;
-    offset_t filepos;
 };
 
 //-------------------------------------------------------------------------------------------------------------
@@ -307,7 +309,7 @@ protected:
     void getLayoutTranslators();
     IRecordLayoutTranslator * getLayoutTranslator(IDistributedFile * f);
     void verifyIndex(IKeyIndex * idx);
-    void initManager(IKeyManager *manager);
+    void initManager(IKeyManager *manager, bool isTlk);
     bool firstPart();
     virtual bool nextPart();
     virtual void initPart();
@@ -468,10 +470,10 @@ bool CHThorIndexReadActivityBase::doPreopenLimitFile(unsigned __int64 & count, u
             Owned<IKeyIndex> tlk = openKeyFile(df->queryPart(num));
             verifyIndex(tlk);
             Owned<IKeyManager> tlman = createLocalKeyManager(tlk, NULL);
-            initManager(tlman);
+            initManager(tlman, true);
             while(tlman->lookup(false) && (count<=limit))
             {
-                unsigned slavePart = (unsigned)tlman->queryFpos();
+                unsigned slavePart = (unsigned)extractFpos(tlman);
                 if (slavePart)
                 {
                     keyIndexCache->addIndex(doPreopenLimitPart(count, limit, slavePart-1));
@@ -504,7 +506,7 @@ IKeyIndex * CHThorIndexReadActivityBase::doPreopenLimitPart(unsigned __int64 & r
     if (limit != (unsigned) -1)
     {
         Owned<IKeyManager> kman = createLocalKeyManager(kidx, NULL);
-        initManager(kman);
+        initManager(kman, false);
         result += kman->checkCount(limit-result);
     }
     return kidx.getClear();
@@ -593,9 +595,9 @@ bool CHThorIndexReadActivityBase::nextPart()
 }
 
 
-void CHThorIndexReadActivityBase::initManager(IKeyManager *manager)
+void CHThorIndexReadActivityBase::initManager(IKeyManager *manager, bool isTlk)
 {
-    if(layoutTrans)
+    if(layoutTrans && !isTlk)
         manager->setLayoutTranslator(layoutTrans);
     helper.createSegmentMonitors(manager);
     manager->finishSegmentMonitors();
@@ -604,8 +606,9 @@ void CHThorIndexReadActivityBase::initManager(IKeyManager *manager)
 
 void CHThorIndexReadActivityBase::initPart()                                    
 { 
+    assertex(!keyIndex->isTopLevelKey());
     klManager.setown(createLocalKeyManager(keyIndex, NULL));
-    initManager(klManager);     
+    initManager(klManager, false);
     callback.setManager(klManager);
 }
 
@@ -635,7 +638,7 @@ bool CHThorIndexReadActivityBase::firstMultiPart()
         openTlk();
     verifyIndex(tlk);
     tlManager.setown(createLocalKeyManager(tlk, NULL));
-    initManager(tlManager);
+    initManager(tlManager, true);
     nextPartNumber = 0;
     return nextMultiPart();
 }
@@ -654,8 +657,9 @@ bool CHThorIndexReadActivityBase::nextMultiPart()
         {
             while (tlManager->lookup(false))
             {
-                if (tlManager->queryFpos())
-                    return setCurrentPart((unsigned)tlManager->queryFpos()-1);
+                offset_t node = extractFpos(tlManager);
+                if (node)
+                    return setCurrentPart((unsigned)node-1);
             }
         }
     }
@@ -866,7 +870,7 @@ bool CHThorIndexReadActivity::nextPart()
     {
         klManager.setown(createKeyMerger(keyIndexCache, seekGEOffset, NULL));
         keyIndexCache.clear();
-        initManager(klManager);
+        initManager(klManager, false);
         callback.setManager(klManager);
         return true;
     }
@@ -911,7 +915,7 @@ const void *CHThorIndexReadActivity::nextRow()
             keyedProcessed++;
             if ((keyedLimit != (unsigned __int64) -1) && keyedProcessed > keyedLimit)
                 helper.onKeyedLimitExceeded();
-            byte const * keyRow = klManager->queryKeyBuffer(callback.getFPosRef());
+            byte const * keyRow = klManager->queryKeyBuffer();
             if (needTransform)
             {
                 try
@@ -992,7 +996,7 @@ const void *CHThorIndexReadActivity::nextRowGE(const void * seek, unsigned numFi
 
         if (klManager->lookupSkip(rawSeek, seekGEOffset, seekSize))
         {
-            const byte * row = klManager->queryKeyBuffer(callback.getFPosRef());
+            const byte * row = klManager->queryKeyBuffer();
 #ifdef _DEBUG
             if (memcmp(row + seekGEOffset, rawSeek, seekSize) < 0)
                 assertex("smart seek failure");
@@ -1207,7 +1211,7 @@ const void *CHThorIndexNormalizeActivity::nextRow()
             }
 
             agent.reportProgress(NULL);
-            expanding = helper.first(klManager->queryKeyBuffer(callback.getFPosRef()));
+            expanding = helper.first(klManager->queryKeyBuffer());
             if (expanding)
             {
                 const void * ret = createNextRow();
@@ -1343,7 +1347,7 @@ void CHThorIndexAggregateActivity::gather()
         agent.reportProgress(NULL);
         try
         {
-            helper.processRow(outBuilder, klManager->queryKeyBuffer(callback.getFPosRef()));
+            helper.processRow(outBuilder, klManager->queryKeyBuffer());
         }
         catch(IException * e)
         {
@@ -1436,7 +1440,7 @@ const void *CHThorIndexCountActivity::nextRow()
                     agent.reportProgress(NULL);
                     if (!klManager->lookup(true))
                         break;
-                    totalCount += helper.numValid(klManager->queryKeyBuffer(callback.getFPosRef()));
+                    totalCount += helper.numValid(klManager->queryKeyBuffer());
                     callback.finishedRow();
                     if ((totalCount > choosenLimit))
                         break;
@@ -1554,7 +1558,7 @@ void CHThorIndexGroupAggregateActivity::gather()
         agent.reportProgress(NULL);
         try
         {
-            helper.processRow(klManager->queryKeyBuffer(callback.getFPosRef()), this);
+            helper.processRow(klManager->queryKeyBuffer(), this);
         }
         catch(IException * e)
         {
@@ -2702,20 +2706,18 @@ public:
             ReleaseRoxieRow(rows.item(idx));
     }
 
-    void addRightMatch(void * right, offset_t fpos);
+    void addRightMatch(void * right);
     offset_t addRightPending();
-    void setPendingRightMatch(offset_t seq, void * right, offset_t fpos);
+    void setPendingRightMatch(offset_t seq, void * right);
     void incRightMatchCount();
 
     unsigned count() const { return rows.ordinality(); }
     CJoinGroup * queryJoinGroup() const { return jg; }
     void * queryRow(unsigned idx) const { return rows.item(idx); }
-    offset_t queryOffset(unsigned idx) const { return offsets.item(idx); }
 
 private:
     CJoinGroup * jg;
     PointerArray rows;
-    Int64Array offsets;
 };
 
 interface IJoinProcessor
@@ -2738,7 +2740,6 @@ public:
     public:
         // Single threaded by now
         void const * queryRow() const { return owner.matchsets.item(ms).queryRow(idx); }
-        offset_t queryOffset() const { return owner.matchsets.item(ms).queryOffset(idx); }
         bool start()
         {
             idx = 0;
@@ -2873,12 +2874,11 @@ protected:
     unsigned candidates;
 };
 
-void MatchSet::addRightMatch(void * right, offset_t fpos)
+void MatchSet::addRightMatch(void * right)
 {
     assertex(!jg->complete());
     CriticalBlock b(jg->crit);
     rows.append(right);
-    offsets.append(fpos);
     jg->matchcount++;
 }
 
@@ -2888,16 +2888,14 @@ offset_t MatchSet::addRightPending()
     CriticalBlock b(jg->crit);
     offset_t seq = rows.ordinality();
     rows.append(NULL);
-    offsets.append(0);
     return seq;
 }
 
-void MatchSet::setPendingRightMatch(offset_t seq, void * right, offset_t fpos)
+void MatchSet::setPendingRightMatch(offset_t seq, void * right)
 {
     assertex(!jg->complete());
     CriticalBlock b(jg->crit);
     rows.replace(right, (aindex_t)seq);
-    offsets.replace(fpos, (aindex_t)seq);
     jg->matchcount++;
 }
 
@@ -3090,7 +3088,7 @@ public:
                 owner.readyManager(&manager, row);
                 while(manager.lookup(false))
                 {
-                    unsigned recptr = (unsigned)manager.queryFpos();
+                    unsigned recptr = (unsigned)extractFpos(&manager);
                     if (recptr)
                     {
                         jg->notePending();
@@ -3119,8 +3117,6 @@ public:
             trans.setown(owner.getLayoutTranslator(&f));
             owner.verifyIndex(&f, index, trans);
             Owned<IKeyManager> manager = createLocalKeyManager(index, NULL);
-            if(trans)
-                manager->setLayoutTranslator(trans);
             managers.append(*manager.getLink());
         }
         opened = true;
@@ -3161,7 +3157,7 @@ void KeyedLookupPartHandler::openPart()
     Owned<IKeyIndex> index = openKeyFile(*part);
     manager.setown(createLocalKeyManager(index, NULL));
     IRecordLayoutTranslator * trans = tlk->queryRecordLayoutTranslator();
-    if(trans)
+    if(trans && !index->isTopLevelKey())
         manager->setLayoutTranslator(trans);
 }
 
@@ -3554,7 +3550,7 @@ public:
                 RtlDynamicRowBuilder extractBuilder(queryRightRowAllocator()); 
                 size32_t size = helper.extractJoinFields(extractBuilder, row, fetch->pos, NULL);
                 void * ret = (void *) extractBuilder.finalizeRowClear(size);
-                fetch->ms->setPendingRightMatch(fetch->seq, ret, fetch->pos);
+                fetch->ms->setPendingRightMatch(fetch->seq, ret);
             }
         }
         fetch->ms->queryJoinGroup()->noteEnd();
@@ -3761,7 +3757,7 @@ public:
                                 RtlDynamicRowBuilder rowBuilder(rowAllocator);
                                 void const * row = jg->matches.queryRow();
                                 if(!row) continue;
-                                offset_t fpos = jg->matches.queryOffset();
+                                offset_t fpos = 0;
                                 size32_t transformedSize;
                                 transformedSize = helper.transform(rowBuilder, left, row, fpos, ++counter);
                                 if (transformedSize)
@@ -3800,7 +3796,7 @@ public:
                             void const * row = jg->matches.queryRow();
                             if(!row) continue;
                             ++count;
-                            offset_t fpos = jg->matches.queryOffset();
+                            offset_t fpos = 0;
                             size32_t transformedSize;
                             try
                             {
@@ -3953,15 +3949,16 @@ public:
             return true;
         }
         KLBlobProviderAdapter adapter(manager);
-        offset_t recptr;
-        byte const * rhs = manager->queryKeyBuffer(recptr);
-        if(indexReadMatch(jg->queryLeft(), rhs, recptr, &adapter))
+        byte const * rhs = manager->queryKeyBuffer();
+        size_t fposOffset = manager->queryRowSize() - sizeof(offset_t);
+        offset_t fpos = rtlReadBigUInt8(rhs + fposOffset);
+        if(indexReadMatch(jg->queryLeft(), rhs, fpos, &adapter))
         {
             if(needsDiskRead)
             {
                 jg->notePending();
                 offset_t seq = ms->addRightPending();
-                parts->addRow(ms, recptr, seq);
+                parts->addRow(ms, fpos, seq);
             }
             else
             {
@@ -3970,9 +3967,9 @@ public:
                 else
                 {
                     RtlDynamicRowBuilder rowBuilder(queryRightRowAllocator()); 
-                    size32_t size = helper.extractJoinFields(rowBuilder, rhs, recptr, &adapter);
+                    size32_t size = helper.extractJoinFields(rowBuilder, rhs, fpos, &adapter);
                     void * ret = (void *)rowBuilder.finalizeRowClear(size);
-                    ms->addRightMatch(ret, recptr);
+                    ms->addRightMatch(ret);
                 }
             }
         }

roxie/ccd/ccdactivities.cpp

@@ -3541,7 +3541,7 @@ public:
 
                         atomic_inc(&indexRecordsRead);
                         size32_t transformedSize;
-                        const byte * keyRow = tlk->queryKeyBuffer(callback.getFPosRef());
+                        const byte * keyRow = tlk->queryKeyBuffer();
                         int diff = 0;
                         if (steppingRow)
                         {
@@ -3801,7 +3801,7 @@ public:
                     }
 
                     atomic_inc(&indexRecordsRead);
-                    if (normalizeHelper->first(tlk->queryKeyBuffer(callback.getFPosRef())))
+                    if (normalizeHelper->first(tlk->queryKeyBuffer()))
                     {
                         do
                         {
@@ -3932,7 +3932,7 @@ public:
                     {
                         keyprocessed++;
                         atomic_inc(&indexRecordsRead);
-                        count += countHelper->numValid(tlk->queryKeyBuffer(callback.getFPosRef()));
+                        count += countHelper->numValid(tlk->queryKeyBuffer());
                         if (count > rowLimit)
                             limitExceeded(false);
                         else if (count > keyedLimit)
@@ -4052,7 +4052,7 @@ public:
                 {
                     keyprocessed++;
                     atomic_inc(&indexRecordsRead);
-                    aggregateHelper->processRow(rowBuilder, tlk->queryKeyBuffer(callback.getFPosRef()));
+                    aggregateHelper->processRow(rowBuilder, tlk->queryKeyBuffer());
                     callback.finishedRow();
                 }
                 callback.setManager(NULL);
@@ -4193,7 +4193,7 @@ public:
                     {
                         if (groupSegCount && !layoutTranslators->item(lastPartNo.fileNo))
                         {
-                            AggregateRowBuilder &rowBuilder = results.addRow(tlk->queryKeyBuffer(callback.getFPosRef()));
+                            AggregateRowBuilder &rowBuilder = results.addRow(tlk->queryKeyBuffer());
                             callback.finishedRow();
                             if (kind == TAKindexgroupcount)
                             {
@@ -4207,7 +4207,7 @@ public:
                         {
                             keyprocessed++;
                             atomic_inc(&indexRecordsRead);
-                            aggregateHelper->processRow(tlk->queryKeyBuffer(callback.getFPosRef()), this);
+                            aggregateHelper->processRow(tlk->queryKeyBuffer(), this);
                             callback.finishedRow();
                         }
                     }
@@ -4858,9 +4858,10 @@ IMessagePacker *CRoxieKeyedJoinIndexActivity::process()
                     candidateCount++;
                     atomic_inc(&indexRecordsRead);
                     KLBlobProviderAdapter adapter(tlk);
-                    offset_t recptr;
-                    const byte *indexRow = tlk->queryKeyBuffer(recptr);
-                    if (helper->indexReadMatch(inputRow, indexRow, recptr, &adapter))
+                    const byte *indexRow = tlk->queryKeyBuffer();
+                    size_t fposOffset = tlk->queryRowSize() - sizeof(offset_t);
+                    offset_t fpos = rtlReadBigUInt8(indexRow + fposOffset);
+                    if (helper->indexReadMatch(inputRow, indexRow, fpos, &adapter))
                     {
                         processed++;
                         if (keepLimit)
@@ -4885,7 +4886,7 @@ IMessagePacker *CRoxieKeyedJoinIndexActivity::process()
                         {
                             const void *self = output->getBuffer(KEYEDJOIN_RECORD_SIZE(0), true);
                             KeyedJoinHeader *rec = (KeyedJoinHeader *) self;
-                            rec->fpos = recptr;
+                            rec->fpos = fpos;
                             rec->thisGroup = jg;
                             rec->partNo = lastPartNo.partNo;
                             output->putBuffer(self, KEYEDJOIN_RECORD_SIZE(0), true);
@@ -4893,8 +4894,8 @@ IMessagePacker *CRoxieKeyedJoinIndexActivity::process()
                         else
                         {
                             KLBlobProviderAdapter adapter(tlk);
-                            totalSize = helper->extractJoinFields(rowBuilder, indexRow, recptr, &adapter);
-                            rowBuilder.writeToOutput(totalSize, recptr, jg, lastPartNo.partNo);
+                            totalSize = helper->extractJoinFields(rowBuilder, indexRow, fpos, &adapter);
+                            rowBuilder.writeToOutput(totalSize, fpos, jg, lastPartNo.partNo);
                         }
                         totalSizeSent += KEYEDJOIN_RECORD_SIZE(totalSize);
                         if (totalSizeSent > indexReadChunkSize && !continuationFailed)

roxie/ccd/ccdserver.cpp

@@ -22811,7 +22811,10 @@ public:
                             else
                             {
                                 tlk.setown(createLocalKeyManager(thisKey, this));
-                                tlk->setLayoutTranslator(translators->item(fileNo));
+                                if (!thisKey->isTopLevelKey())
+                                    tlk->setLayoutTranslator(translators->item(fileNo));
+                                else
+                                    tlk->setLayoutTranslator(nullptr);
                             }
                             createSegmentMonitors(tlk);
                             if (queryTraceLevel() > 3 || ctx->queryProbeManager())
@@ -22834,7 +22837,7 @@ public:
                                         {
                                             while (tlk->lookup(false))
                                             {
-                                                unsigned slavePart = (unsigned) tlk->queryFpos();
+                                                unsigned slavePart = (unsigned)extractFpos(tlk);
                                                 if (slavePart)
                                                 {
                                                     accepted++;
@@ -22861,7 +22864,10 @@ public:
                                 {
                                     thisKey = thisBase->queryPart(fileNo);
                                     tlk->setKey(thisKey);
-                                    tlk->setLayoutTranslator(translators->item(fileNo));
+                                    if (!thisKey->isTopLevelKey())
+                                        tlk->setLayoutTranslator(translators->item(fileNo));
+                                    else
+                                        tlk->setLayoutTranslator(nullptr);
                                     tlk->reset();
                                 }
                                 else
@@ -23073,7 +23079,8 @@ public:
                 tlk.setown(createKeyMerger(keySet, owner.seekGEOffset, &owner));
             else
                 tlk.setown(createLocalKeyManager(keySet->queryPart(0), &owner));
-            tlk->setLayoutTranslator(trans);
+            if (!key->isTopLevelKey())
+                tlk->setLayoutTranslator(trans);
             owner.indexHelper.createSegmentMonitors(tlk);
             tlk->finishSegmentMonitors();
             tlk->reset();
@@ -23091,7 +23098,7 @@ public:
                 }
                 size32_t transformedSize;
                 RtlDynamicRowBuilder rowBuilder(owner.rowAllocator);
-                byte const * keyRow = tlk->queryKeyBuffer(owner.callback.getFPosRef());
+                byte const * keyRow = tlk->queryKeyBuffer();
                 try
                 {
                     transformedSize = owner.readHelper.transform(rowBuilder, keyRow);
@@ -23598,7 +23605,7 @@ public:
                 break;
             }
 
-            byte const * keyRow = tlk->queryKeyBuffer(callback.getFPosRef());
+            byte const * keyRow = tlk->queryKeyBuffer();
 #ifdef _DEBUG
 //          StringBuffer recstr;
 //          unsigned size = (tlk->queryRecordSize()<80)  ? tlk->queryRecordSize() : 80;
@@ -23908,7 +23915,7 @@ public:
             {
                 try
                 {
-                    count += countHelper.numValid(tlk->queryKeyBuffer(callback.getFPosRef()));
+                    count += countHelper.numValid(tlk->queryKeyBuffer());
                     callback.finishedRow();
                 }
                 catch (IException *E)
@@ -24150,7 +24157,7 @@ public:
             }
             try
             {
-                aggregateHelper.processRow(rowBuilder, tlk->queryKeyBuffer(callback.getFPosRef()));
+                aggregateHelper.processRow(rowBuilder, tlk->queryKeyBuffer());
                 callback.finishedRow();
             }
             catch (IException *E)
@@ -24331,7 +24338,7 @@ public:
             {
                 if (groupSegCount && !trans)
                 {
-                    AggregateRowBuilder &rowBuilder = singleAggregator.addRow(tlk->queryKeyBuffer(callback.getFPosRef()));
+                    AggregateRowBuilder &rowBuilder = singleAggregator.addRow(tlk->queryKeyBuffer());
                     callback.finishedRow();
                     if (kind==TAKindexgroupcount)                   
                     {
@@ -24343,7 +24350,7 @@ public:
                 }
                 else
                 {
-                    aggregateHelper.processRow(tlk->queryKeyBuffer(callback.getFPosRef()), this);
+                    aggregateHelper.processRow(tlk->queryKeyBuffer(), this);
                     callback.finishedRow();
                 }
             }
@@ -24479,7 +24486,7 @@ public:
             }
             size32_t transformedSize;
     
-            if (readHelper.first(tlk->queryKeyBuffer(callback.getFPosRef())))
+            if (readHelper.first(tlk->queryKeyBuffer()))
             {
                 Owned<CRowArrayMessageResult> result = new CRowArrayMessageResult(ctx->queryRowManager(), meta.isVariableSize());
                 do
@@ -25444,7 +25451,10 @@ public:
                     try
                     {
                         tlk->setKey(thisKey);
-                        tlk->setLayoutTranslator(translators->item(fileNo));
+                        if (!thisKey->isTopLevelKey())
+                            tlk->setLayoutTranslator(translators->item(fileNo));
+                        else
+                            tlk->setLayoutTranslator(nullptr);
                         helper.createSegmentMonitors(tlk, extracted);
                         if (rootIndex)
                             rootIndex->mergeSegmentMonitors(tlk);
@@ -25458,7 +25468,7 @@ public:
                                 bool locallySorted = !thisKey->isFullySorted();
                                 while (locallySorted || tlk->lookup(false)) 
                                 {
-                                    unsigned slavePart = locallySorted ? 0 : (unsigned) tlk->queryFpos();
+                                    unsigned slavePart = locallySorted ? 0 : (unsigned)extractFpos(tlk);
                                     if (locallySorted || slavePart)
                                     {
                                         cvp *outputBuffer = (cvp *) remote.getMem(slavePart, fileNo, indexReadSize + sizeof(cvp) + (indexReadInputRecordVariable ? sizeof(unsigned) : 0));
@@ -25492,12 +25502,13 @@ public:
                                     candidateCount++;
                                     atomic_inc(&indexRecordsRead);
                                     KLBlobProviderAdapter adapter(tlk);
-                                    offset_t recptr;
-                                    const byte *indexRow = tlk->queryKeyBuffer(recptr);
-                                    if (helper.indexReadMatch(extracted, indexRow, recptr, &adapter))
+                                    const byte *indexRow = tlk->queryKeyBuffer();
+                                    size_t fposOffset = tlk->queryRowSize() - sizeof(offset_t);
+                                    offset_t fpos = rtlReadBigUInt8(indexRow + fposOffset);
+                                    if (helper.indexReadMatch(extracted, indexRow, fpos, &adapter))
                                     {
                                         KeyedJoinHeader *rhs = (KeyedJoinHeader *) ctx->queryRowManager().allocate(KEYEDJOIN_RECORD_SIZE(0), activityId);
-                                        rhs->fpos = recptr; 
+                                        rhs->fpos = fpos;
                                         rhs->thisGroup = jg; 
                                         rhs->partNo = partNo; 
                                         result->append(rhs);
@@ -25525,7 +25536,10 @@ public:
                             {
                                 thisKey = thisBase->queryPart(fileNo);
                                 tlk->setKey(thisKey);
-                                tlk->setLayoutTranslator(translators->item(fileNo));
+                                if (!thisKey->isTopLevelKey())
+                                    tlk->setLayoutTranslator(translators->item(fileNo));
+                                else
+                                    tlk->setLayoutTranslator(nullptr);
                                 tlk->reset();
                             }
                             else
@@ -26276,7 +26290,10 @@ public:
                     unsigned fileNo = 0;
                     IKeyIndex *thisKey = thisBase->queryPart(fileNo);
                     tlk->setKey(thisKey);
-                    tlk->setLayoutTranslator(translators->item(fileNo));
+                    if (thisKey && !thisKey->isTopLevelKey())
+                        tlk->setLayoutTranslator(translators->item(fileNo));
+                    else
+                        tlk->setLayoutTranslator(nullptr);
                     helper.createSegmentMonitors(tlk, extracted);
                     if (rootIndex)
                         rootIndex->mergeSegmentMonitors(tlk);
@@ -26292,7 +26309,7 @@ public:
                                 bool locallySorted = (!thisKey->isFullySorted());
                                 while (locallySorted || tlk->lookup(false))
                                 {
-                                    unsigned slavePart = locallySorted ? 0 : (unsigned) tlk->queryFpos();
+                                    unsigned slavePart = locallySorted ? 0 : (unsigned)extractFpos(tlk);
                                     if (locallySorted || slavePart)
                                     {
                                         cvp *outputBuffer = (cvp *) remote.getMem(slavePart, fileNo, indexReadRecordSize + sizeof(cvp) + (indexReadInputRecordVariable ? sizeof(unsigned) : 0));
@@ -26328,17 +26345,18 @@ public:
                                     candidateCount++;
                                     atomic_inc(&indexRecordsRead);
                                     KLBlobProviderAdapter adapter(tlk);
-                                    offset_t recptr;
-                                    const byte *indexRow = tlk->queryKeyBuffer(recptr);
-                                    if (helper.indexReadMatch(extracted, indexRow, recptr, &adapter))
+                                    const byte *indexRow = tlk->queryKeyBuffer();
+                                    size_t fposOffset = tlk->queryRowSize() - sizeof(offset_t);
+                                    offset_t fpos = rtlReadBigUInt8(indexRow + fposOffset);
+                                    if (helper.indexReadMatch(extracted, indexRow, fpos, &adapter))
                                     {
                                         RtlDynamicRowBuilder rb(joinFieldsAllocator, true); 
                                         CPrefixedRowBuilder pb(KEYEDJOIN_RECORD_SIZE(0), rb);
                                         accepted++;
                                         KLBlobProviderAdapter adapter(tlk);
-                                        size32_t joinFieldsSize = helper.extractJoinFields(pb, indexRow, recptr, &adapter);
+                                        size32_t joinFieldsSize = helper.extractJoinFields(pb, indexRow, fpos, &adapter);
                                         KeyedJoinHeader *rec = (KeyedJoinHeader *) rb.getUnfinalizedClear(); // lack of finalize ok as unserialized data here.
-                                        rec->fpos = recptr;
+                                        rec->fpos = fpos;
                                         rec->thisGroup = jg;
                                         rec->partNo = partNo;
                                         if (isSimple)
@@ -26373,7 +26391,10 @@ public:
                             {
                                 thisKey = thisBase->queryPart(fileNo);
                                 tlk->setKey(thisKey);
-                                tlk->setLayoutTranslator(translators->item(fileNo));
+                                if (thisKey && !thisKey->isTopLevelKey())
+                                    tlk->setLayoutTranslator(translators->item(fileNo));
+                                else
+                                    tlk->setLayoutTranslator(nullptr);
                                 tlk->reset();
                             }
                             else

roxie/ccd/ccdserver.hpp

@@ -280,13 +280,13 @@ extern void setStartRuid(unsigned restarts);
 class CIndexTransformCallback : implements IThorIndexCallback, public CInterface
 {
 public:
-    CIndexTransformCallback() { keyManager = NULL; cleanupRequired = false; filepos = 0; };
+    CIndexTransformCallback() { keyManager = NULL; cleanupRequired = false; };
     IMPLEMENT_IINTERFACE
 
 //IThorIndexCallback
     virtual unsigned __int64 getFilePosition(const void * row)
     {
-        return filepos;
+        throwUnexpected();
     }
     virtual byte * lookupBlob(unsigned __int64 id) 
     { 
@@ -297,7 +297,6 @@ public:
 
 
 public:
-    inline offset_t & getFPosRef()                              { return filepos; }
     inline void setManager(IKeyManager * _manager)
     {
         finishedRow();
@@ -314,7 +313,6 @@ public:
 
 protected:
     IKeyManager * keyManager;
-    offset_t filepos;
     bool cleanupRequired;
 };
 

rtl/eclrtl/eclrtl.hpp

@@ -470,6 +470,12 @@ ECLRTL_API void rtlWriteSwapInt7(void * data, unsigned __int64 value);
 ECLRTL_API void rtlWriteSwapInt8(void * data, unsigned __int64 value);
 ECLRTL_API void rtlWriteSwapInt(void * self, __int64 val, unsigned length);
 
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+inline unsigned __int64 rtlReadBigUInt8(const void * data) { return rtlReadSwapUInt8(data); }
+#else
+inline unsigned __int64 rtlReadBigUInt8(const void * data) { return rtlReadUInt8(data); }
+#endif
+
 ECLRTL_API short rtlRevInt2(const void * data);
 ECLRTL_API int rtlRevInt3(const void * data);
 ECLRTL_API int rtlRevInt4(const void * data);

system/jhtree/ctfile.cpp

@@ -777,7 +777,10 @@ bool CJHTreeNode::getValueAt(unsigned int index, char *dst) const
 
 size32_t CJHTreeNode::getSizeAt(unsigned int index) const
 {
-    return keyLen;
+    if (keyHdr->hasSpecialFileposition())
+        return keyLen + sizeof(offset_t);
+    else
+        return keyLen;
 }
 
 offset_t CJHTreeNode::getFPosAt(unsigned int index) const
@@ -951,7 +954,10 @@ size32_t CJHVarTreeNode::getSizeAt(unsigned int num) const
     const char * p = recArray[num];
     KEYRECSIZE_T reclen = ((KEYRECSIZE_T *) p)[-1];
     _WINREV(reclen);
-    return reclen;
+    if (keyHdr->hasSpecialFileposition())
+        return reclen + sizeof(offset_t);
+    else
+        return reclen;
 }
 
 offset_t CJHVarTreeNode::getFPosAt(unsigned int num) const

system/jhtree/jhtree.cpp

@@ -392,7 +392,6 @@ protected:
     SegMonitorList segs;
     IKeyCursor *keyCursor;
     char *keyBuffer;
-    offset_t lookupFpos;
     unsigned keySize;       // size of key record including payload
     unsigned keyedSize;     // size of non-payload part of key
     unsigned numsegs;
@@ -410,6 +409,7 @@ protected:
     bool transformSegs;
     IIndexReadContext * activitySegs;
     CMemoryBlock layoutTransBuff;
+    size32_t layoutSize = 0;
     Owned<IRecordLayoutTranslator::SegmentMonitorContext> layoutTransSegCtx;
     Owned<IRecordLayoutTranslator::RowTransformContext> layoutTransRowCtx;
 
@@ -433,9 +433,9 @@ protected:
         segs.endRange(segno, keyBuffer);
     }
 
-    byte const * doRowLayoutTransform(offset_t & fpos)
+    byte const * doRowLayoutTransform()
     {
-        layoutTrans->transformRow(layoutTransRowCtx, reinterpret_cast<byte const *>(keyBuffer), keySize, layoutTransBuff, fpos);
+        layoutSize = layoutTrans->transformRow(layoutTransRowCtx, reinterpret_cast<byte const *>(keyBuffer), keySize, layoutTransBuff);
         return layoutTransBuff.get();
     }
 
@@ -690,18 +690,20 @@ public:
         return activitySegs->item(idx);
     }
 
-    inline const byte *queryKeyBuffer(offset_t & fpos)
+    inline const byte *queryKeyBuffer()
     {
-        fpos = lookupFpos;
         if(layoutTrans)
-            return doRowLayoutTransform(fpos);
+            return doRowLayoutTransform();
         else
             return reinterpret_cast<byte const *>(keyBuffer);
     }
 
     inline size32_t queryRowSize()
     {
-        return keyCursor ? keyCursor->getSize() : 0;
+        if (layoutTrans)
+            return layoutSize;
+        else
+            return keyCursor ? keyCursor->getSize() : 0;
     }
 
     inline unsigned __int64 querySequence()
@@ -709,11 +711,6 @@ public:
         return keyCursor ? keyCursor->getSequence() : 0;
     }
 
-    inline offset_t queryFpos()
-    {
-        return lookupFpos;
-    }
-
     inline unsigned queryRecordSize() { return keySize; }
 
     bool _lookup(bool exact, unsigned lastSeg)
@@ -737,7 +734,6 @@ public:
             }
             if (ok)
             {
-                lookupFpos = keyCursor->getFPos();
                 unsigned i = 0;
                 matched = true;
                 if (segs.segMonitors.length())
@@ -2371,10 +2367,8 @@ class CKeyMerger : public CKeyLevelManager
     PointerArray bufferArray;
     PointerArray fixedArray;
     BoolArray matchedArray; 
-    UInt64Array fposArray;
     UnsignedArray mergeHeapArray;
     UnsignedArray keyNoArray;
-    offset_t *fposes;
 
     IKeyCursor **cursors;
     char **buffers;
@@ -2448,7 +2442,6 @@ class CKeyMerger : public CKeyLevelManager
         keyBuffer = buffers[key];
         keyCursor = cursors[key];
         matched = matcheds[key];
-        lookupFpos = fposes[key];
         for (unsigned segno = 0; segno < sortFromSeg; segno++)
         {
             IOverrideableKeySegmentMonitor *sm = QUERYINTERFACE(&segs.segMonitors.item(segno), IOverrideableKeySegmentMonitor);
@@ -2492,7 +2485,6 @@ public:
         cursorArray.kill();
         keyCursor = NULL; // cursorArray owns cursors
         matchedArray.kill();
-        fposArray.kill();
         mergeHeapArray.kill();
         bufferArray.kill();
         fixedArray.kill();
@@ -2500,7 +2492,6 @@ public:
 
         cursors = NULL;
         matcheds = NULL;
-        fposes = NULL;
         mergeheap = NULL;
         buffers = NULL; 
         fixeds = NULL;
@@ -2546,11 +2537,7 @@ public:
             unsigned compares = 0;
             for (;;)
             {
-                if (CKeyLevelManager::lookupSkip(seek, seekOffset, seeklen) )
-                {
-                    fposes[key] = lookupFpos;
-                }
-                else
+                if (!CKeyLevelManager::lookupSkip(seek, seekOffset, seeklen) )
                 {
                     activekeys--;
                     if (!activekeys)
@@ -2700,7 +2687,6 @@ public:
                     cursorArray.append(*keyCursor);
                     bufferArray.append(keyBuffer);
                     matchedArray.append(matched);
-                    fposArray.append(lookupFpos);
                     mergeHeapArray.append(activekeys++);
                     if (!sortFromSeg)
                     {
@@ -2755,7 +2741,6 @@ public:
             if (ctx)
                 ctx->noteStatistic(StNumIndexMerges, activekeys);
             cursors = cursorArray.getArray();
-            fposes = fposArray.getArray();
             matcheds = (bool *) matchedArray.getArray();  // For some reason BoolArray is typedef'd to CharArray on linux...
             buffers = (char **) bufferArray.getArray();
             mergeheap = mergeHeapArray.getArray();
@@ -2827,11 +2812,7 @@ public:
             if (!activekeys)
                 return false;
             unsigned key = mergeheap[0];
-            if (CKeyLevelManager::lookup(exact)) 
-            {
-                fposes[key] = lookupFpos;
-            }
-            else
+            if (!CKeyLevelManager::lookup(exact)) 
             {
                 activekeys--;
                 if (!activekeys)
@@ -2917,7 +2898,6 @@ public:
             mb.append(keyNoArray.item(key));
             cursors[key]->serializeCursorPos(mb);
             mb.append(matcheds[key]);
-            mb.append(fposes[key]);
         }
     }
 
@@ -2937,9 +2917,6 @@ public:
             keyCursor->deserializeCursorPos(mb, keyBuffer);
             mb.read(matched);
             matchedArray.append(matched);
-            offset_t fpos;
-            mb.read(fpos);
-            fposArray.append(fpos);
             bufferArray.append(keyBuffer);
             void *fixedValue = (char *) malloc(sortFieldOffset);
             memcpy(fixedValue, keyBuffer, sortFieldOffset); // If it's not at EOF then it must match
@@ -2947,7 +2924,6 @@ public:
             mergeHeapArray.append(i);
         }
         cursors = cursorArray.getArray();
-        fposes = fposArray.getArray();
         matcheds = (bool *) matchedArray.getArray();  // For some reason BoolArray is typedef'd to CharArray on linux...
         buffers = (char **) bufferArray.getArray();
         mergeheap = mergeHeapArray.getArray();
@@ -3067,23 +3043,23 @@ class IKeyManagerTest : public CppUnit::TestFixture
             tlk1->reset();
 
             offset_t fpos;
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003010", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005010", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006010", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003030", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005030", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006030", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003031", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005031", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006031", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003010", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005010", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006010", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003030", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005030", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006030", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003031", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005031", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006031", 10)==0);
             MemoryBuffer mb;
             tlk1->serializeCursorPos(mb);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003032", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005032", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006032", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003033", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005033", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006033", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003032", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005032", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006032", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003033", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005033", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006033", 10)==0);
             ASSERT(!tlk1->lookup(true)); 
             ASSERT(!tlk1->lookup(true)); 
 
@@ -3094,12 +3070,12 @@ class IKeyManagerTest : public CppUnit::TestFixture
             tlk2->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
             tlk2->finishSegmentMonitors();
             tlk2->reset(true);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000003032", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000005032", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000006032", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000003033", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000005033", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000006033", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000003032", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000005032", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000006032", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000003033", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000005033", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000006033", 10)==0);
             ASSERT(!tlk2->lookup(true)); 
             ASSERT(!tlk2->lookup(true)); 
 
@@ -3109,10 +3085,10 @@ class IKeyManagerTest : public CppUnit::TestFixture
             tlk3->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
             tlk3->finishSegmentMonitors();
             tlk3->reset(false);
-            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), "0000003010", 10)==0);
-            ASSERT(tlk3->lookupSkip("031", 7, 3)); ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), "0000003031", 10)==0);
-            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), "0000005031", 10)==0);
-            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), "0000006031", 10)==0);
+            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(), "0000003010", 10)==0);
+            ASSERT(tlk3->lookupSkip("031", 7, 3)); ASSERT(memcmp(tlk3->queryKeyBuffer(), "0000003031", 10)==0);
+            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(), "0000005031", 10)==0);
+            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(), "0000006031", 10)==0);
             ASSERT(!tlk3->lookupSkip("081", 7, 3)); 
             ASSERT(!tlk3->lookup(true)); 
 
@@ -3122,18 +3098,18 @@ class IKeyManagerTest : public CppUnit::TestFixture
             tlk4->append(createKeySegmentMonitor(false, sset3.getLink(), 7, 3));
             tlk4->finishSegmentMonitors();
             tlk4->reset(false);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000003000", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000005000", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000006000", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000003001", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000005001", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000006001", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000003002", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000005002", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000006002", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000003999", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000005999", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000006999", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000003000", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000005000", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000006000", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000003001", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000005001", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000006001", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000003002", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000005002", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000006002", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000003999", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000005999", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000006999", 10)==0);
             ASSERT(!tlk4->lookup(true)); 
             ASSERT(!tlk4->lookup(true)); 
 
@@ -3208,8 +3184,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
     void checkBlob(IKeyManager *key, unsigned size)
     {
         unsigned __int64 blobid;
-        offset_t fpos;
-        memcpy(&blobid, key->queryKeyBuffer(fpos)+10, sizeof(blobid));
+        memcpy(&blobid, key->queryKeyBuffer()+10, sizeof(blobid));
         ASSERT(blobid != 0);
         size32_t blobsize;
         const byte *blob = key->loadBlob(blobid, blobsize);
@@ -3261,10 +3236,9 @@ protected:
 
 /*          for (;;)
             {
-                offset_t fpos;
                 if (!tlk1a->lookup(true))
                     break;
-                DBGLOG("%.10s", tlk1a->queryKeyBuffer(fpos));
+                DBGLOG("%.10s", tlk1a->queryKeyBuffer());
             }
             tlk1a->reset();
 */
@@ -3346,42 +3320,42 @@ protected:
             {
                 offset_t fpos;
                 tlk1->reset();
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000001", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000002", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000003", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000005", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000006", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000007", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000009", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000010", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000001", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000002", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000003", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000005", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000006", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000007", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000009", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000010", 10)==0);
                 if (blobby)
                     checkBlob(tlk1, 10+100000);
 
                 tlk1a->reset();
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000001", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000010", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000011", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000021", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000030", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000031", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000041", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000050", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000001", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000010", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000011", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000021", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000030", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000031", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000041", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000050", 10)==0);
 
                 tlk2->reset();
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000004", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000008", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000012", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000016", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000020", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000024", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000028", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000032", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000036", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000040", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000044", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000048", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000048", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000052", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000004", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000008", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000012", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000016", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000020", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000024", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000028", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000032", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000036", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000040", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000044", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000048", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000048", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000052", 10)==0);
 
                 if (tlk3)
                 {
@@ -3390,11 +3364,11 @@ protected:
                     {
                         ASSERT(tlk3->lookup(true)); 
                         sprintf(buf, "%010d", i);
-                        ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), buf, 10)==0);
+                        ASSERT(memcmp(tlk3->queryKeyBuffer(), buf, 10)==0);
                         if (i==48 || i==49)
                         {
                             ASSERT(tlk3->lookup(true)); 
-                            ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), buf, 10)==0);
+                            ASSERT(memcmp(tlk3->queryKeyBuffer(), buf, 10)==0);
                         }
                     }
                     ASSERT(!tlk3->lookup(true)); 

system/jhtree/jhtree.hpp

@@ -43,7 +43,6 @@ interface jhtree_decl IKeyCursor : public IInterface
     virtual bool gtEqual(const char *src, char *dst, bool seekForward = false) = 0; // returns first record >= src
     virtual bool ltEqual(const char *src, char *dst, bool seekForward = false) = 0; // returns last record <= src
     virtual size32_t getSize() = 0;
-    virtual offset_t getFPos() = 0;
     virtual void serializeCursorPos(MemoryBuffer &mb) = 0;
     virtual void deserializeCursorPos(MemoryBuffer &mb, char *keyBuffer) = 0;
     virtual unsigned __int64 getSequence() = 0;
@@ -197,8 +196,7 @@ interface IKeyManager : public IInterface, extends IIndexReadContext
     virtual void reset(bool crappyHack = false) = 0;
     virtual void releaseSegmentMonitors() = 0;
 
-    virtual const byte *queryKeyBuffer(offset_t & fpos) = 0; //if using RLT: fpos is the translated value, so correct in a normal row
-    virtual offset_t queryFpos() = 0; //if using RLT: this is the untranslated fpos, so correct as the part number in TLK but not in a normal row
+    virtual const byte *queryKeyBuffer() = 0; //if using RLT: fpos is the translated value, so correct in a normal row
     virtual unsigned __int64 querySequence() = 0;
     virtual size32_t queryRowSize() = 0;     // Size of current row as returned by queryKeyBuffer()
     virtual unsigned queryRecordSize() = 0;  // Max size
@@ -228,6 +226,14 @@ interface IKeyManager : public IInterface, extends IIndexReadContext
     virtual bool lookupSkip(const void *seek, size32_t seekGEOffset, size32_t seeklen) = 0;
 };
 
+inline offset_t extractFpos(IKeyManager * manager)
+{
+    byte const * keyRow = manager->queryKeyBuffer();
+    size32_t rowSize = manager->queryRowSize();
+    size32_t offset = rowSize - sizeof(offset_t);
+    return rtlReadBigUInt8(keyRow + offset);
+}
+
 extern jhtree_decl IKeyManager *createLocalKeyManager(IKeyIndex * _key, IContextLogger *ctx);
 extern jhtree_decl IKeyManager *createKeyMerger(IKeyIndexSet * _key, unsigned sortFieldOffset, IContextLogger *ctx);
 extern jhtree_decl IKeyManager *createSingleKeyMerger(IKeyIndex * _onekey, unsigned sortFieldOffset, IContextLogger *ctx);

system/jhtree/keydiff.cpp

@@ -59,9 +59,8 @@ public:
     {
         if(keyCursor->next(row))
         {
-            if(isVar)
-                thisrowsize = keyCursor->getSize();
-            *fpos = keyCursor->getFPos();
+            thisrowsize = keyCursor->getSize() - sizeof(offset_t);
+            *fpos = rtlReadBigUInt8(row + thisrowsize);
             return true;
         }
         *fpos = 0;
@@ -285,7 +284,8 @@ public:
         {
             if(keyCursor->next(buff))
             {
-                offset_t fpos = keyCursor->getFPos();
+                size32_t offset = keyCursor->getSize() - sizeof(offset_t);
+                offset_t fpos = rtlReadBigUInt8(buff + offset);
                 crc.tally(rowsize, buff);
                 crc.tally(sizeof(fpos), &fpos);
             }

testing/regress/ecl/layouttrans.ecl

@@ -38,14 +38,14 @@ Files := setup.Files(multiPart, useLocal, false);
 //nothorlcr
 
 #IF (version=1)
-DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Fname,Lname,__filepos},Files.DG_FetchIndex1Name);
-DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Fname,Lname,__filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Lname,Fname,__filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Lname,Fname,__filepos},Files.DG_FetchIndex1Name);
 #ELIF (version=2)
-DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Fname,Lname},{state, STRING100 blobfield {blob}:= fname, STRING tfn := TRIM(Fname), __filepos},Files.DG_FetchIndex1Name);
-DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Fname,Lname},{ STRING100 blobfield {blob}:= fname, __filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Lname,Fname},{state, STRING100 blobfield {blob}:= fname, STRING tfn := TRIM(Fname), __filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Lname,Fname},{ STRING100 blobfield {blob}:= fname, __filepos},Files.DG_FetchIndex1Name);
 #ELSE
-DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Fname,Lname},{state ,__filepos},Files.DG_FetchIndex1Name);
-DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Fname,Lname},{__filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Lname,Fname},{state ,__filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Lname,Fname},{__filepos},Files.DG_FetchIndex1Name);
 #END
 
 ds := DATASET([{'Anderson'}, {'Doe'}], {STRING25 Lname});

testing/regress/ecl/setup/files.ecl

@@ -96,7 +96,7 @@ EXPORT DG_KeyDiffIndex1 := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(
 EXPORT DG_KeyDiffIndex2 := INDEX(DG_KeyDiffIndex1, DG_KeyDiffIndex2Name);
 
 //This version is used for testing reading from a file requiring translation 
-EXPORT DG_FetchTransIndex := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname, __filepos},DG_FetchTransIndexName);
+EXPORT DG_FetchTransIndex := INDEX(DG_FetchFile,{Lname,Fname},{state, STRING tfn := TRIM(Fname), STRING100 blobfield {blob}:= fname, __filepos},DG_FetchTransIndexName);
 
 indexName := IF(useTranslation, __nameof__(DG_FetchTransIndex), __nameof__(DG_FetchIndex1));
 EXPORT DG_FetchIndex := INDEX(DG_FetchIndex1,indexName);
@@ -134,8 +134,8 @@ EXPORT DG_FlatFileEvens := DATASET(DG_FileOut+'FLAT_EVENS',{DG_OutRec,UNSIGNED8
 EXPORT DG_NormalIndexFile      := INDEX(DG_FlatFile, { DG_firstname, DG_lastname }, { DG_Prange, filepos }, DG_IndexOut+'INDEX');
 EXPORT DG_NormalIndexFileEvens := INDEX(DG_FlatFileEvens, { DG_firstname; DG_lastname; }, { DG_Prange, filepos } ,DG_IndexOut+'INDEX_EVENS');
 
-EXPORT DG_TransIndexFile      := INDEX(DG_FlatFile, { DG_lastname, DG_firstname }, { DG_Prange, filepos }, DG_IndexOut+'TRANS_INDEX');
-EXPORT DG_TransIndexFileEvens := INDEX(DG_FlatFileEvens, { DG_lastname, DG_firstname }, { DG_Prange, filepos } ,DG_IndexOut+'TRANS_INDEX_EVENS');
+EXPORT DG_TransIndexFile      := INDEX(DG_FlatFile, { DG_firstname, DG_lastname }, { DG_ChildID := filepos, DG_Prange, filepos }, DG_IndexOut+'TRANS_INDEX');
+EXPORT DG_TransIndexFileEvens := INDEX(DG_FlatFileEvens, { DG_firstname, DG_lastname }, { DG_ChildID := filepos, DG_Prange, filepos } ,DG_IndexOut+'TRANS_INDEX_EVENS');
 
 indexName := IF(useTranslation, __nameof__(DG_TransIndexFile), __nameof__(DG_NormalIndexFile));
 EXPORT DG_indexFile      := INDEX(DG_NormalIndexFile, indexName);
@@ -156,7 +156,7 @@ END;
 EXPORT DG_VarFile   := DATASET(DG_FileOut+'VAR',DG_VarOutRecPlus,FLAT);
 
 EXPORT DG_NormalVarIndex  := INDEX(DG_VarFile, { DG_firstname; DG_lastname; __filepos } ,DG_IndexOut+'VARINDEX');
-EXPORT DG_TransVarIndex  := INDEX(DG_VarFile, { DG_lastname; DG_firstname; __filepos } ,DG_IndexOut+'TRANS_VARINDEX');
+EXPORT DG_TransVarIndex  := INDEX(DG_VarFile, { DG_firstname; DG_lastname; }, { DGextra := DG_lastname; __filepos } ,DG_IndexOut+'TRANS_VARINDEX');
 
 indexName := IF(useTranslation, __nameof__(DG_TransVarIndex), __nameof__(DG_NormalVarIndex));
 EXPORT DG_VarIndex  := INDEX(DG_NormalVarIndex, indexName);

testing/regress/ecl/setup/setup_fetch.ecl

@@ -67,7 +67,7 @@ BUILDINDEX(Files.DG_KeyDiffIndex1, PROJECT(sortedFile, createDiffRow(LEFT)),OVER
 BUILDINDEX(Files.DG_KeyDiffIndex2, PROJECT(sortedFile(lname != 'Doe'),createDiffRow(LEFT)), OVERWRITE);
 
 //A version of the index with LName/FName transposed and x moved to the front.
-BUILDINDEX(sortedFile,{Fname,Lname},{STRING100 blobfield {blob}:= fname+lname, STRING tfn := TRIM(Fname), state, __filepos},Files.DG_FetchTransIndexName, OVERWRITE);
+BUILDINDEX(Files.DG_FetchTransIndex, TABLE(sortedFile, {STRING tfn := TRIM(Fname), STRING100 blobfield := fname+lname, sortedFile}), OVERWRITE);
 
 fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchFilePreloadName, '', '', 'view', '1:1', false);
 fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchFilePreloadIndexedName, '', '', 'view', '1:1', false);

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -2389,7 +2389,7 @@ class IndexDistributeSlaveActivity : public HashDistributeSlaveBase
             tlkManager->reset();
             verifyex(tlkManager->lookup(false));
             tlkManager->releaseSegmentMonitors();
-            offset_t partNo = tlkManager->queryFpos();
+            offset_t partNo = extractFpos(tlkManager);
             if (partNo)
                 partNo--; // note that partNo==0 means lower than anything in the key - should be treated same as partNo==1 here
             return ((unsigned)partNo % numslaves);

thorlcr/activities/indexread/thindexread.cpp

@@ -167,8 +167,9 @@ protected:
                 tlk->reset();
                 while (tlk->lookup(false))
                 {
-                    if (tlk->queryFpos())
-                        performPartLookup.replace(true, (aindex_t)(super?super->numSubFiles(true)*(tlk->queryFpos()-1)+superSubIndex:tlk->queryFpos()-1));
+                    offset_t node = extractFpos(tlk);
+                    if (node)
+                        performPartLookup.replace(true, (aindex_t)(super?super->numSubFiles(true)*(node-1)+superSubIndex:node-1));
                 }
             }
             if (!super||!iter->next())

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -68,7 +68,6 @@ protected:
     {
     protected:
         IKeyManager *keyManager;
-        offset_t filepos;
     public:
         TransformCallback() { keyManager = NULL; };
         IMPLEMENT_IINTERFACE_USING(CSimpleInterface)
@@ -76,14 +75,13 @@ protected:
     //IThorIndexCallback
         virtual unsigned __int64 getFilePosition(const void *row)
         {
-            return filepos;
+            throwUnexpected();
         }
         virtual byte *lookupBlob(unsigned __int64 id) 
         { 
             size32_t dummy;
             return (byte *) keyManager->loadBlob(id, dummy); 
         }
-        offset_t & getFPosRef() { return filepos; }
         void setManager(IKeyManager *_keyManager)
         {
             finishedRow();
@@ -143,7 +141,7 @@ protected:
             if (currentManager->lookup(true))
             {
                 noteStats(currentManager->querySeeks(), currentManager->queryScans());
-                ret = (const void *)currentManager->queryKeyBuffer(callback.getFPosRef());
+                ret = (const void *)currentManager->queryKeyBuffer();
             }
             if (ret || keyMergerManager)
                 break;
@@ -358,7 +356,7 @@ class CIndexReadSlaveActivity : public CIndexReadSlaveBase
         if (!currentManager->lookupSkip(rawSeek, seekGEOffset, seekSize))
             return NULL;
         noteStats(currentManager->querySeeks(), currentManager->queryScans());
-        const byte *row = currentManager->queryKeyBuffer(callback.getFPosRef());
+        const byte *row = currentManager->queryKeyBuffer();
 #ifdef _DEBUG
         if (memcmp(row + seekGEOffset, rawSeek, seekSize) < 0)
             assertex("smart seek failure");
@@ -721,7 +719,7 @@ public:
                     {
                         ++progress;
                         noteStats(keyManager.querySeeks(), keyManager.queryScans());
-                        helper->processRow(keyManager.queryKeyBuffer(callback.getFPosRef()), this);
+                        helper->processRow(keyManager.queryKeyBuffer(), this);
                         callback.finishedRow();
                     }
                     clearManager();
@@ -839,7 +837,7 @@ public:
                         if (!l)
                             break;
                         ++progress;
-                        totalCount += helper->numValid(keyManager.queryKeyBuffer(callback.getFPosRef()));
+                        totalCount += helper->numValid(keyManager.queryKeyBuffer());
                         callback.finishedRow();
                         if ((totalCount > choosenLimit))
                             break;

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -1289,8 +1289,9 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
                             if (candidateCount > owner.atMost)
                                 break;
                             KLBlobProviderAdapter adapter(currentPartKeyManager);
-                            offset_t fpos;
-                            byte const * keyRow = currentPartKeyManager->queryKeyBuffer(fpos);
+                            byte const * keyRow = currentPartKeyManager->queryKeyBuffer();
+                            size_t fposOffset = currentPartKeyManager->queryRowSize() - sizeof(offset_t);
+                            offset_t fpos = rtlReadBigUInt8(keyRow + fposOffset);
                             if (owner.helper->indexReadMatch(indexReadFieldsRow.getSelf(), keyRow, fpos, &adapter))
                             {
                                 if (currentJG->rowsSeen() >= owner.keepLimit)
@@ -1349,9 +1350,10 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
                         for (;;)
                         {
                             if (!tlkManager->lookup(false)) break;
-                            if (tlkManager->queryFpos()) // don't bail out if part0 match, test again for 'real' tlk match.
+                            offset_t node = extractFpos(tlkManager);
+                            if (node) // don't bail out if part0 match, test again for 'real' tlk match.
                             {
-                                unsigned partNo = (unsigned)tlkManager->queryFpos();
+                                unsigned partNo = (unsigned)node;
                                 partNo = owner.superWidth ? owner.superWidth*nextTlk+(partNo-1) : partNo-1;
 
                                 currentPartKeyManager = &partKeyManagers.item(partNo);

tools/dumpkey/dumpkey.cpp

@@ -179,7 +179,7 @@ int main(int argc, const char **argv)
                     MyIndexCallback(IKeyManager *_manager) : manager(_manager) {}
                     virtual unsigned __int64 getFilePosition(const void * row)
                     {
-                        return manager->queryFpos();
+                        return 0;
                     }
                     virtual byte * lookupBlob(unsigned __int64 id)
                     {
@@ -257,8 +257,7 @@ int main(int argc, const char **argv)
                 manager->reset();
                 while (manager->lookup(true) && count--)
                 {
-                    offset_t pos;
-                    byte const * buffer = manager->queryKeyBuffer(pos);
+                    byte const * buffer = manager->queryKeyBuffer();
                     size32_t size = manager->queryRowSize();
                     unsigned __int64 seq = manager->querySequence();
                     if (optRaw)
@@ -269,7 +268,7 @@ int main(int argc, const char **argv)
                     {
                         for (unsigned i = 0; i < size; i++)
                             printf("%02x", ((unsigned char) buffer[i]) & 0xff);
-                        printf("  :%" I64F "u:%012" I64F "x\n", seq, pos);
+                        printf("  :%" I64F "u\n", seq);
                     }
                     else if (helper)
                     {
@@ -285,7 +284,7 @@ int main(int argc, const char **argv)
                             count++;  // Don't count this row as it was postfiltered
                     }
                     else
-                        printf("%.*s  :%" I64F "u:%012" I64F "x\n", size, buffer, seq, pos);
+                        printf("%.*s  :%" I64F "u\n", size, buffer, seq);
                 }
                 if (outRecType)
                     outRecType->doDelete();