Merge pull request #10663 from ghalliday/issue18672

HPCC-18672 Access fileposition from an index row instead of special casing

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/fileview2/fvidxsource.cpp

@@ -304,8 +304,7 @@ bool IndexDataSource::getNextRow(MemoryBuffer & out, bool extractRow)
                 }
                 else
                 {
-                    offset_t filepos;
-                    const byte * thisRow = manager->queryKeyBuffer(filepos);
+                    const byte * thisRow = manager->queryKeyBuffer();
                     unsigned thisSize = diskMeta->getRecordSize(thisRow);
                     void * temp = out.reserve(thisSize);
                     memcpy(temp, thisRow, thisSize);
@@ -451,7 +450,7 @@ void IndexDataSource::applyFilter()
         manager->reset();
         while (manager->lookup(false))
         {
-            offset_t node = manager->queryFpos();
+            offset_t node = extractFpos(manager);
             if (node)
                 matchingParts.append((unsigned)(node-1));
         }

common/remote/sockfile.cpp

@@ -2931,7 +2931,6 @@ class CRemoteKeyManager : public CSimpleInterfaceOf<IKeyManager>
     MemoryBuffer keyCursorMb;        // used for continuation
     unsigned __int64 totalGot = 0;
     size32_t currentSize = 0;
-    offset_t currentFpos = 0;
     const byte *currentRow = nullptr;
     bool first = true;
     unsigned __int64 chooseNLimit = 0;
@@ -3016,7 +3015,6 @@ public:
         rowDataRemaining = 0;
         keyCursorMb.clear();
         currentSize = 0;
-        currentFpos = 0;
         currentRow = nullptr;
         first = true;
         totalGot = 0;
@@ -3030,19 +3028,12 @@ public:
         }
         segs.reset();
     }
-    virtual const byte *queryKeyBuffer(offset_t & fpos) override
+    virtual const byte *queryKeyBuffer() override
     {
         if (!remoteSupport())
-            return directKM->queryKeyBuffer(fpos);;
-        fpos = currentFpos;
+            return directKM->queryKeyBuffer();
         return currentRow;
     }
-    virtual offset_t queryFpos() override
-    {
-        if (!remoteSupport())
-            return directKM->queryFpos();
-        return currentFpos;
-    }
     virtual unsigned queryRecordSize() override
     {
         if (!remoteSupport())
@@ -3069,10 +3060,9 @@ public:
         {
             if (rowDataRemaining)
             {
-                rowDataBuffer.read(currentFpos);
                 rowDataBuffer.read(currentSize);
                 currentRow = rowDataBuffer.readDirect(currentSize);
-                rowDataRemaining -= sizeof(currentFpos) + sizeof(currentSize) + currentSize;
+                rowDataRemaining -= sizeof(currentSize) + currentSize;
                 return true;
             }
             else
@@ -4785,9 +4775,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
         while (keyManager->lookup(true))
         {
             unsigned size = keyManager->queryRecordSize();
-            offset_t fpos;
-            const byte *result = keyManager->queryKeyBuffer(fpos);
-            reply.append(fpos);
+            const byte *result = keyManager->queryKeyBuffer();
             reply.append(size);
             reply.append(size, result);
             ++numRecs;

common/thorhelper/layouttrans.cpp

@@ -257,29 +257,9 @@ void RowTransformer::createRowRecord(FieldMapping const & mapping, CIArrayOf<Row
     switch(mapping.queryType())
     {
     case FieldMapping::Simple:
-        if(mapping.isDiskFieldFpos())
-            if(mapping.isActivityFieldFpos())
-            {
-                ensureItem(records, activityFieldNum).setVals(0, 0, diskSize, false).setFpos(true, true);
-                prevActivityField = false;
-            }
-            else
-            {
-                ensureItem(records, activityFieldNum).setVals(0, 0, diskSize, false).setFpos(false, true);
-                prevActivityField = false;
-            }
-        else
-            if(mapping.isActivityFieldFpos())
-            {
-                ensureItem(records, activityFieldNum).setVals(diskOffset, numVarFields, diskSize, false).setFpos(true, false);
-                prevActivityField = false;
-            }
-            else
-            {
-                ensureItem(records, activityFieldNum).setVals(diskOffset, numVarFields, diskSize, (prevActivityField && (activityFieldNum == (prevActivityFieldNum+1))));
-                prevActivityField = true;
-                prevActivityFieldNum = activityFieldNum;
-            }
+        ensureItem(records, activityFieldNum).setVals(diskOffset, numVarFields, diskSize, (prevActivityField && (activityFieldNum == (prevActivityFieldNum+1))));
+        prevActivityField = true;
+        prevActivityFieldNum = activityFieldNum;
         break;
 
     case FieldMapping::ChildDataset:
@@ -337,47 +317,13 @@ void RowTransformer::generateSimpleCopy(unsigned & seq, RowRecord const & record
         copy.setChildTransformer(new RowTransformer(seq, *childMappings));
 }
 
-void RowTransformer::generateCopyToFpos(RowRecord const & record)
-{
-    copyToFpos = true;
-    copyToFposRelOffset = record.queryRelOffset();
-    copyToFposRelBase = record.queryRelBase();
-    copyToFposSize = record.querySize();
-    assertex(copyToFposSize != UNKNOWN_LENGTH);
-    assertex(copyToFposSize <= sizeof(offset_t));
-}
-
-void RowTransformer::generateCopyFromFpos(RowRecord const & record)
-{
-    assertex(sequence == 0);
-    copies.append(*new FieldCopy(static_cast<unsigned>(-1), 0, 0));
-    size32_t size = record.querySize();
-    assertex(size != UNKNOWN_LENGTH);
-    assertex(size <= sizeof(offset_t));
-    copies.tos().addFixedSize(size);
-}
-
 void RowTransformer::generateCopies(unsigned & seq, CIArrayOf<RowRecord> const & records)
 {
     sequence = seq++;
     ForEachItemIn(fieldNum, records)
     {
         RowRecord const & record = records.item(fieldNum);
-        if(record.isToFpos())
-        {
-            assertex(sequence == 0);
-            if(record.isFromFpos())
-                keepFpos = true;
-            else
-                generateCopyToFpos(record);
-        }
-        else
-        {
-            if(record.isFromFpos())
-                generateCopyFromFpos(record);
-            else
-                generateSimpleCopy(seq, record);
-        }
+        generateSimpleCopy(seq, record);
     }
 }
 
@@ -406,25 +352,6 @@ void RowTransformer::transform(IRecordLayoutTranslator::RowTransformContext * ct
         copies.item(copyIdx).copy(ctx, out, outOffset);
 }
 
-void RowTransformer::getFposOut(IRecordLayoutTranslator::RowTransformContext const * ctx, offset_t & fpos) const
-{
-    if(copyToFpos)
-    {
-        fpos = 0;
-        const byte * in = ctx->queryPointer(0, copyToFposRelBase) + copyToFposRelOffset;
-        // integer field in row is big-endian
-#if __BYTE_ORDER == __BIG_ENDIAN
-        memcpy(reinterpret_cast<byte const *>(&fpos) + sizeof(offset_t) - copyToFposSize, in, copyToFposSize);
-#else
-        _cpyrevn(&fpos, in, copyToFposSize);
-#endif
-    }
-    else if(!keepFpos)
-    {
-        fpos = 0;
-    }
-}
-
 void RowTransformer::createRowTransformContext(IRecordLayoutTranslator::RowTransformContext * ctx) const
 {
     ctx->init(sequence, diskVarFieldRelOffsets.ordinality()+1);
@@ -438,19 +365,6 @@ void RowTransformer::createRowTransformContext(IRecordLayoutTranslator::RowTrans
 
 void FieldCopy::copy(IRecordLayoutTranslator::RowTransformContext * ctx, IMemoryBlock & out, size32_t & outOffset) const
 {
-    if(sequence == static_cast<unsigned>(-1))
-    {
-        byte * target = out.ensure(outOffset+fixedSize);
-        // integer field in row is big-endian
-#if __BYTE_ORDER == __BIG_ENDIAN
-        memcpy(target+outOffset, reinterpret_cast<byte const *>(ctx->queryFposIn()) + sizeof(offset_t) - fixedSize, fixedSize);
-#else
-        _cpyrevn(target+outOffset, ctx->queryFposIn(), fixedSize);
-#endif
-        outOffset += fixedSize;
-        return;
-    }
-
     size32_t diskFieldSize = fixedSize;
     ForEachItemIn(varIdx, varFields)
         diskFieldSize += ctx->querySize(sequence, varFields.item(varIdx));
@@ -617,13 +531,11 @@ IRecordLayoutTranslator::RowTransformContext * CRecordLayoutTranslator::getRowTr
     return ctx.getClear();
 }
 
-size32_t CRecordLayoutTranslator::transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out, offset_t & fpos) const
+size32_t CRecordLayoutTranslator::transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out) const
 {
     size32_t inOffset = 0;
     size32_t outOffset = 0;
-    ctx->setFposIn(fpos);
     transformer.transform(ctx, in, inSize, inOffset, out, outOffset);
-    transformer.getFposOut(ctx, fpos);
     return outOffset;
 }
 

common/thorhelper/layouttrans.hpp

@@ -53,16 +53,13 @@ public:
         ~RowTransformContext();
         void init(unsigned seq, unsigned num) { sizes[seq] = new unsigned[num]; ptrs[seq] = new byte const *[num]; }
         void set(unsigned seq, unsigned i, size32_t size, byte const * ptr) { sizes[seq][i] = size; ptrs[seq][i] = ptr; }
-        void setFposIn(offset_t fpos) { fposIn = fpos; }
 
         size32_t querySize(unsigned seq, unsigned i) const { return sizes[seq][i]; }
         byte const * queryPointer(unsigned seq, unsigned i) const { return ptrs[seq][i]; }
-        offset_t const * queryFposIn() const { return &fposIn; }
     private:
         unsigned num;
         size32_t * * sizes;
         byte const * * * ptrs;
-        offset_t fposIn;
     };
 
     typedef enum { NoTranslation = 0, TranslateAll = 1, TranslatePayload = 2 } Mode;
@@ -74,7 +71,7 @@ public:
     virtual SegmentMonitorContext * getSegmentMonitorContext() = 0;
     virtual void createDiskSegmentMonitors(SegmentMonitorContext const & in, IIndexReadContext & out) = 0;
     virtual RowTransformContext * getRowTransformContext() = 0;
-    virtual size32_t transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out, offset_t & fpos) const = 0;
+    virtual size32_t transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out) const = 0;
 #ifdef DEBUG_HELPERS_REQUIRED
     virtual StringBuffer & getMappingsAsString(StringBuffer & out) const = 0;
 #endif

common/thorhelper/layouttrans.ipp

@@ -57,12 +57,10 @@ public:
     char const * queryDiskFieldName() const { return str(diskRecord->queryChild(diskFieldNum)->queryName()); }
     unsigned queryDiskFieldNum() const { return diskFieldNum; }
     size32_t queryDiskFieldSize() const { return diskRecord->queryChild(diskFieldNum)->queryType()->getSize(); }
-    bool isDiskFieldFpos() const { return ((type != ChildDataset) && !diskFieldKeyed && (diskFieldNum == diskRecord->numChildren()-1) && diskRecord->queryChild(diskFieldNum)->queryType()->isInteger()); }
     bool isDiskFieldSet() const { return (diskRecord->queryChild(diskFieldNum)->queryType()->getTypeCode() == type_set); }
     char const * queryActivityFieldName() const { return str(activityRecord->queryChild(activityFieldNum)->queryName()); }
     unsigned queryActivityFieldNum() const { return activityFieldNum; }
     size32_t queryActivityFieldSize() const { return activityRecord->queryChild(activityFieldNum)->queryType()->getSize(); }
-    bool isActivityFieldFpos() const { return ((type == Simple) && !activityFieldKeyed && (activityFieldNum == activityRecord->numChildren()-1) && activityRecord->queryChild(activityFieldNum)->queryType()->isInteger()); }
     List const & queryChildMappings() const { return childMappings; }
 
 private:
@@ -121,23 +119,19 @@ public:
     RowTransformer(unsigned & seq, FieldMapping::List const & mappings) { build(seq, mappings); }
     void build(unsigned & seq, FieldMapping::List const & mappings);
     void transform(IRecordLayoutTranslator::RowTransformContext * ctx, byte const * in, size32_t inSize, size32_t & inOffset, IMemoryBlock & out, size32_t & outOffset) const;
-    void getFposOut(IRecordLayoutTranslator::RowTransformContext const * ctx, offset_t & fpos) const;
     void createRowTransformContext(IRecordLayoutTranslator::RowTransformContext * ctx) const;
 
 private:
     class RowRecord : public CInterface
     {
     public:
-        RowRecord() : relOffset(0), relBase(0), size(0), followOn(false), childMappings(NULL), toFpos(false), fromFpos(false) {}
+        RowRecord() : relOffset(0), relBase(0), size(0), followOn(false), childMappings(NULL) {}
         RowRecord & setVals(size32_t _relOffset, unsigned _relBase, size32_t _size, bool _followOn) { relOffset = _relOffset; relBase = _relBase; size = _size; followOn = _followOn; return *this; }
         RowRecord & setChildMappings(FieldMapping::List const * _mappings) { childMappings = _mappings; return *this; }
-        RowRecord & setFpos(bool to, bool from) { toFpos = to; fromFpos = from; return *this; }
         size32_t queryRelOffset() const { return relOffset; }
         unsigned queryRelBase() const { return relBase; }
         size32_t querySize() const { return size; }
         bool queryFollowOn() const { return followOn; }
-        bool isToFpos() const { return toFpos; }
-        bool isFromFpos() const { return fromFpos; }
         FieldMapping::List const * queryChildMappings() const { return childMappings; }
     private:
         size32_t relOffset;
@@ -145,16 +139,12 @@ private:
         size32_t size;
         bool followOn;
         FieldMapping::List const * childMappings;
-        bool toFpos;
-        bool fromFpos;
     };
 
     RowRecord & ensureItem(CIArrayOf<RowRecord> & arr, unsigned pos);
     void createRowRecord(FieldMapping const & mapping, CIArrayOf<RowRecord> & records, size32_t diskOffset, unsigned numVarFields, bool & prevActivityField, unsigned & prevActivityFieldNum);
     void analyseMappings(FieldMapping::List const & mappings, CIArrayOf<RowRecord> & records);
     void generateSimpleCopy(unsigned & seq, RowRecord const & record);
-    void generateCopyToFpos(RowRecord const & record);
-    void generateCopyFromFpos(RowRecord const & record);
     void generateCopies(unsigned & seq, CIArrayOf<RowRecord> const & records);
 
 private:
@@ -201,7 +191,7 @@ public:
     virtual SegmentMonitorContext * getSegmentMonitorContext() { return new ExpandedSegmentMonitorList(this); }
     virtual void createDiskSegmentMonitors(SegmentMonitorContext const & in, IIndexReadContext & out);
     virtual RowTransformContext * getRowTransformContext();
-    virtual size32_t transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out, offset_t & fpos) const;
+    virtual size32_t transformRow(RowTransformContext * ctx, byte const * in, size32_t inSize, IMemoryBlock & out) const;
 #ifdef DEBUG_HELPERS_REQUIRED
     virtual StringBuffer & getMappingsAsString(StringBuffer & out) const;
 #endif

ecl/hql/hqlutil.cpp

@@ -5139,6 +5139,14 @@ IHqlExpression * removeOperand(IHqlExpression * expr, IHqlExpression * operand)
     return expr->clone(args);
 }
 
+extern HQL_API IHqlExpression * removeChild(IHqlExpression * expr, unsigned child)
+{
+    HqlExprArray args;
+    unwindChildren(args, expr);
+    args.remove(child);
+    return expr->clone(args);
+}
+
 IHqlExpression * removeChildOp(IHqlExpression * expr, node_operator op)
 {
     HqlExprArray args;

ecl/hql/hqlutil.hpp

@@ -199,6 +199,7 @@ extern HQL_API IHqlExpression * removeLocalAttribute(IHqlExpression * expr);
 extern HQL_API IHqlExpression * removeAttribute(IHqlExpression * expr, IAtom * attr);
 extern HQL_API IHqlExpression * removeOperand(IHqlExpression * expr, IHqlExpression * operand);
 extern HQL_API IHqlExpression * removeChildOp(IHqlExpression * expr, node_operator op);
+extern HQL_API IHqlExpression * removeChild(IHqlExpression * expr, unsigned child);
 extern HQL_API IHqlExpression * appendAttribute(IHqlExpression * expr, IAtom * attr);
 extern HQL_API IHqlExpression * appendOwnedOperand(IHqlExpression * expr, IHqlExpression * ownedOperand);
 extern HQL_API IHqlExpression * replaceOwnedAttribute(IHqlExpression * expr, IHqlExpression * ownedAttribute);

ecl/hqlcpp/hqlckey.cpp

@@ -1477,13 +1477,13 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedJoinOrDenormalize(BuildCt
     //virtual unsigned getKeepLimit()
     doBuildJoinRowLimitHelper(*instance, rowlimit, info.queryKeyFilename(), implicitLimit);
 
-    buildFormatCrcFunction(instance->classctx, "getIndexFormatCrc", info.queryRawKey(), info.queryRawKey(), 1);
+    buildFormatCrcFunction(instance->classctx, "getIndexFormatCrc", true, info.queryRawKey(), info.queryRawKey(), 1);
     if (info.isFullJoin())
     {
         //Remove virtual attributes from the record, so the crc will be compatible with the disk read record
         //can occur with a (highly unusual) full keyed join to a persist file... (see indexread14.ecl)
         OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(info.queryRawRhs()->queryRecord());
-        buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", noVirtualRecord, NULL, 0);
+        buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", false, noVirtualRecord, NULL, 0);
         buildEncryptHelper(instance->startctx, info.queryFile()->queryAttribute(encryptAtom), "getFileEncryptKey");
     }
 
@@ -1597,7 +1597,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedDistribute(BuildCtx & ctx
 
     doCompareLeftRight(instance->nestedctx, "CompareRowKey", leftDs, rightDs, joinInfo.queryLeftReq(), normalizedRight);
 
-    buildFormatCrcFunction(instance->classctx, "getFormatCrc", info.queryRawKey(), info.queryRawKey(), 1);
+    buildFormatCrcFunction(instance->classctx, "getFormatCrc", true, info.queryRawKey(), info.queryRawKey(), 1);
     buildSerializedLayoutMember(instance->classctx, indexRecord, "getIndexLayout", numKeyedFields);
 
     OwnedHqlExpr matchExpr = info.getMatchExpr(true);

ecl/hqlcpp/hqlcpp.ipp

@@ -1816,7 +1816,7 @@ public:
     void endNestedClass(IHqlStmt * stmt);
 
     void buildEncryptHelper(BuildCtx & ctx, IHqlExpression * encryptAttr, const char * funcname = NULL);
-    void buildFormatCrcFunction(BuildCtx & ctx, const char * name, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta);
+    void buildFormatCrcFunction(BuildCtx & ctx, const char * name, bool removeFilepos, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta);
     void buildLimitHelpers(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * filename, unique_id_t id);
     void buildLimitHelpers(BuildCtx & ctx, IHqlExpression * rowLimit, IHqlExpression * failAction, bool isSkip, IHqlExpression * filename, unique_id_t id);
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9956,19 +9956,26 @@ void HqlCppTranslator::buildRecordEcl(BuildCtx & subctx, IHqlExpression * record
 }
 
 
-void HqlCppTranslator::buildFormatCrcFunction(BuildCtx & ctx, const char * name, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta)
+void HqlCppTranslator::buildFormatCrcFunction(BuildCtx & ctx, const char * name, bool removeFilepos, IHqlExpression * dataset, IHqlExpression * expr, unsigned payloadDelta)
 {
     IHqlExpression * payload = expr ? expr->queryAttribute(_payload_Atom) : NULL;
     // MORE - do we need to keep this consistent - if so will have to trim out the originals and the filepos
     OwnedHqlExpr exprToCrc = getSerializedForm(dataset->queryRecord(), diskAtom);
 
-    unsigned payloadSize = 1;
+    unsigned payloadSize = getBoolAttribute(expr, filepositionAtom, true) ? 1 : 0;
     if (payload)
-        payloadSize = (unsigned)getIntValue(payload->queryChild(0)) + payloadDelta;
+        payloadSize = (unsigned)getIntValue(payload->queryChild(0));
 
     //FILEPOSITION(FALSE) means we have counted 1 too many in the payload
-    if (!getBoolAttribute(expr, filepositionAtom, true))
-        payloadSize--;
+    if (getBoolAttribute(expr, filepositionAtom, true) && removeFilepos)
+    {
+        assertex(payloadSize);
+        //Backward compatibility - remove the fileposition field from the CRC
+        HqlExprArray args;
+        unwindChildren(args, exprToCrc);
+        args.pop();
+        exprToCrc.setown(exprToCrc->clone(args));
+    }
 
     exprToCrc.setown(createComma(exprToCrc.getClear(), getSizetConstant(payloadSize)));
 
@@ -10372,7 +10379,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH
 
     OwnedHqlExpr rawRecord;
     doBuildIndexOutputTransform(instance->startctx, record, rawRecord, hasFileposition, expr->queryAttribute(maxLengthAtom));
-    buildFormatCrcFunction(instance->classctx, "getFormatCrc", rawRecord, expr, 0);
+    buildFormatCrcFunction(instance->classctx, "getFormatCrc", false, rawRecord, expr, 0);
 
     if (compressAttr && compressAttr->hasAttribute(rowAtom))
     {
@@ -10673,7 +10680,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
         if (!pipe)
         {
             OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(dataset->queryRecord());
-            buildFormatCrcFunction(instance->classctx, "getFormatCrc", noVirtualRecord, NULL, 0);
+            buildFormatCrcFunction(instance->classctx, "getFormatCrc", false, noVirtualRecord, NULL, 0);
         }
 
         bool grouped = isGrouped(dataset);

ecl/hqlcpp/hqlsource.cpp

@@ -408,23 +408,18 @@ static void createPhysicalLogicalAssigns(HqlExprArray & assigns, IHqlExpression
             {
                 OwnedHqlExpr target = createSelectExpr(LINK(self), LINK(cur));
                 OwnedHqlExpr newValue;
-                if (idx2  == fileposIndex)
-                    newValue.setown(getFilepos(diskDataset, false));
-                else
+                IHqlExpression * curPhysical = nextDiskField(diskRecord, diskIndex);
+                OwnedHqlExpr physicalSelect = createSelectExpr(LINK(diskDataset), LINK(curPhysical));
+                if (cur->isDatarow() && !cur->hasAttribute(blobAtom) && (!isInPayload() || (physicalSelect->queryType() != target->queryType())))
                 {
-                    IHqlExpression * curPhysical = nextDiskField(diskRecord, diskIndex);
-                    OwnedHqlExpr physicalSelect = createSelectExpr(LINK(diskDataset), LINK(curPhysical));
-                    if (cur->isDatarow() && !cur->hasAttribute(blobAtom) && (!isInPayload() || (physicalSelect->queryType() != target->queryType())))
-                    {
-                        HqlExprArray subassigns;
-                        OwnedHqlExpr childSelf = createSelector(no_self, cur, NULL);
-                        createPhysicalLogicalAssigns(subassigns, childSelf, curPhysical->queryRecord(), cur->queryRecord(), physicalSelect, false, NotFound);
-                        OwnedHqlExpr transform = createValue(no_transform, makeTransformType(cur->queryRecord()->getType()), subassigns);
-                        newValue.setown(createRow(no_createrow, transform.getClear()));
-                    }
-                    else
-                        newValue.setown(convertIndexPhysical2LogicalValue(cur, physicalSelect, allowTranslate));
+                    HqlExprArray subassigns;
+                    OwnedHqlExpr childSelf = createSelector(no_self, cur, NULL);
+                    createPhysicalLogicalAssigns(subassigns, childSelf, curPhysical->queryRecord(), cur->queryRecord(), physicalSelect, false, NotFound);
+                    OwnedHqlExpr transform = createValue(no_transform, makeTransformType(cur->queryRecord()->getType()), subassigns);
+                    newValue.setown(createRow(no_createrow, transform.getClear()));
                 }
+                else
+                    newValue.setown(convertIndexPhysical2LogicalValue(cur, physicalSelect, allowTranslate && (idx2 != fileposIndex)));
 
                 if (newValue)
                     assigns.append(*createAssign(target.getClear(), newValue.getClear()));
@@ -524,11 +519,20 @@ static IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IH
             }
         }
     }
-    if (hasInternalFileposition && createKeyedTypes)
+    if (hasInternalFileposition)
     {
-        IHqlExpression * cur = record->queryChild(record->numChildren()-1);
-        IHqlExpression *fposField = createField(cur->queryId(), makeFilePosType(cur->getType()), nullptr, extractFieldAttrs(cur));
-        physicalFields.append(*fposField);
+        if (createKeyedTypes)
+        {
+            IHqlExpression * cur = record->queryChild(record->numChildren()-1);
+            IHqlExpression *fposField = createField(cur->queryId(), makeFilePosType(cur->getType()), nullptr, extractFieldAttrs(cur));
+            physicalFields.append(*fposField);
+        }
+        else
+        {
+            IHqlExpression * cur = record->queryChild(record->numChildren()-1);
+            IHqlExpression *fposField = createField(cur->queryId(), makeSwapIntType(8, false), nullptr, extractFieldAttrs(cur));
+            physicalFields.append(*fposField);
+        }
     }
 
     return createRecord(physicalFields);
@@ -559,12 +563,11 @@ IHqlExpression * HqlCppTranslator::convertToPhysicalIndex(IHqlExpression * table
 
     unsigned payload = numPayloadFields(tableExpr);
     assertex(payload || !hasFileposition);
-    unsigned newPayload = hasFileposition ? payload-1 : payload;
     HqlExprArray args;
     unwindChildren(args, tableExpr);
     args.replace(*diskRecord, 1);
     removeAttribute(args, _payload_Atom);
-    args.append(*createAttribute(_payload_Atom, getSizetConstant(newPayload)));
+    args.append(*createAttribute(_payload_Atom, getSizetConstant(payload)));
     args.append(*createAttribute(_original_Atom, LINK(tableExpr)));
 
     //remove the preload attribute and replace with correct value
@@ -2380,9 +2383,17 @@ void SourceBuilder::buildGlobalGroupAggregateHelpers(IHqlExpression * expr)
 
     //virtual void processRows(void * self, size32_t srcLen, const void * src) = 0;
     {
+        OwnedHqlExpr newTableExpr = LINK(tableExpr);
+        if (isKey(tableExpr))
+        {
+            IHqlExpression * record = tableExpr->queryRecord();
+            OwnedHqlExpr newRecord = removeChild(record, record->numChildren()-1);
+            newTableExpr.setown(replaceChild(tableExpr, 1, newRecord));
+        }
+
         MemberFunction func(translator, instance->startctx, "virtual void processRows(size32_t srcLen, const void * _left, IHThorGroupAggregateCallback * callback) override");
         func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
-        OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(tableExpr->getType()));
+        OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(newTableExpr->getType()));
         OwnedHqlExpr len = createVariable("srcLen", LINK(sizetType));
         OwnedHqlExpr fullDs = createTranslated(ds, len);
 
@@ -2757,7 +2768,7 @@ void DiskReadBuilderBase::buildMembers(IHqlExpression * expr)
     {
         //Spill files can still have virtual attributes in their physical records => remove them.
         OwnedHqlExpr noVirtualRecord = removeVirtualAttributes(physicalRecord);
-        translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", noVirtualRecord, NULL, 0);
+        translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", false, noVirtualRecord, NULL, 0);
     }
 
     buildLimits(instance->startctx, expr, instance->activityId); 
@@ -6281,8 +6292,6 @@ public:
     IndexReadBuilderBase(HqlCppTranslator & _translator, IHqlExpression *_tableExpr, IHqlExpression *_nameExpr)
         : SourceBuilder(_translator, _tableExpr, _nameExpr), monitors(_tableExpr, _translator, -(int)numPayloadFields(_tableExpr), false)
     { 
-        fpos.setown(getFilepos(tableExpr, false));
-        lfpos.setown(getFilepos(tableExpr, true));
     }
 
     virtual void buildMembers(IHqlExpression * expr);
@@ -6338,7 +6347,7 @@ void IndexReadBuilderBase::buildMembers(IHqlExpression * expr)
 
     buildKeyedLimitHelper(expr);
 
-    translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", tableExpr, tableExpr, 1);
+    translator.buildFormatCrcFunction(instance->classctx, "getFormatCrc", true, tableExpr, tableExpr, 1);
     IHqlExpression * originalKey = queryAttributeChild(tableExpr, _original_Atom, 0);
     translator.buildSerializedLayoutMember(instance->classctx, originalKey->queryRecord(), "getIndexLayout", numKeyedFields(originalKey));
 
@@ -6644,9 +6653,13 @@ void IndexAggregateBuilder::buildMembers(IHqlExpression * expr)
     }
 
     {
+        IHqlExpression * record = tableExpr->queryRecord();
+        OwnedHqlExpr newRecord = removeChild(record, record->numChildren()-1);
+        OwnedHqlExpr newTableExpr = replaceChild(tableExpr, 1, newRecord);
+
         MemberFunction func(translator, instance->startctx, "virtual void processRows(ARowBuilder & crSelf, size32_t srcLen, const void * _left) override");
         func.ctx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
-        OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(tableExpr->getType()));
+        OwnedHqlExpr ds = createVariable("left", makeReferenceModifier(newTableExpr->getType()));
         OwnedHqlExpr len = createVariable("srcLen", LINK(sizetType));
         OwnedHqlExpr fullDs = createTranslated(ds, len);
 
@@ -7248,7 +7261,7 @@ void FetchBuilder::buildMembers(IHqlExpression * expr)
     case no_json:
         break;
     default:
-        translator.buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", physicalRecord, NULL, 0);
+        translator.buildFormatCrcFunction(instance->classctx, "getDiskFormatCrc", false, physicalRecord, NULL, 0);
         break;
     }
 

ecl/hthor/hthorkey.cpp

@@ -91,6 +91,10 @@ bool rltEnabled(IConstWorkUnit const * wu)
 
 IRecordLayoutTranslator * getRecordLayoutTranslator(IDefRecordMeta const * activityMeta, size32_t activityMetaSize, void const * activityMetaBuff, IDistributedFile * df, IRecordLayoutTranslatorCache * cache, IRecordLayoutTranslator::Mode mode)
 {
+    //Disallow any keyed translation
+    if (mode == IRecordLayoutTranslator::TranslateAll)
+        mode = IRecordLayoutTranslator::TranslatePayload;
+
     IPropertyTree const & props = df->queryAttributes();
     MemoryBuffer diskMetaBuff;
     if(!props.getPropBin("_record_layout", diskMetaBuff))
@@ -140,7 +144,7 @@ public:
 //IThorIndexCallback
     virtual unsigned __int64 getFilePosition(const void * row)
     {
-        return filepos;
+        throwUnexpected();
     }
     virtual byte * lookupBlob(unsigned __int64 id) 
     { 
@@ -150,7 +154,6 @@ public:
 
 
 public:
-    offset_t & getFPosRef()                                 { return filepos; }
     void setManager(IKeyManager * _manager)
     {
         finishedRow();
@@ -165,7 +168,6 @@ public:
 
 protected:
     IKeyManager * keyManager;
-    offset_t filepos;
 };
 
 //-------------------------------------------------------------------------------------------------------------
@@ -307,7 +309,7 @@ protected:
     void getLayoutTranslators();
     IRecordLayoutTranslator * getLayoutTranslator(IDistributedFile * f);
     void verifyIndex(IKeyIndex * idx);
-    void initManager(IKeyManager *manager);
+    void initManager(IKeyManager *manager, bool isTlk);
     bool firstPart();
     virtual bool nextPart();
     virtual void initPart();
@@ -468,10 +470,10 @@ bool CHThorIndexReadActivityBase::doPreopenLimitFile(unsigned __int64 & count, u
             Owned<IKeyIndex> tlk = openKeyFile(df->queryPart(num));
             verifyIndex(tlk);
             Owned<IKeyManager> tlman = createLocalKeyManager(tlk, NULL);
-            initManager(tlman);
+            initManager(tlman, true);
             while(tlman->lookup(false) && (count<=limit))
             {
-                unsigned slavePart = (unsigned)tlman->queryFpos();
+                unsigned slavePart = (unsigned)extractFpos(tlman);
                 if (slavePart)
                 {
                     keyIndexCache->addIndex(doPreopenLimitPart(count, limit, slavePart-1));
@@ -504,7 +506,7 @@ IKeyIndex * CHThorIndexReadActivityBase::doPreopenLimitPart(unsigned __int64 & r
     if (limit != (unsigned) -1)
     {
         Owned<IKeyManager> kman = createLocalKeyManager(kidx, NULL);
-        initManager(kman);
+        initManager(kman, false);
         result += kman->checkCount(limit-result);
     }
     return kidx.getClear();
@@ -593,9 +595,9 @@ bool CHThorIndexReadActivityBase::nextPart()
 }
 
 
-void CHThorIndexReadActivityBase::initManager(IKeyManager *manager)
+void CHThorIndexReadActivityBase::initManager(IKeyManager *manager, bool isTlk)
 {
-    if(layoutTrans)
+    if(layoutTrans && !isTlk)
         manager->setLayoutTranslator(layoutTrans);
     helper.createSegmentMonitors(manager);
     manager->finishSegmentMonitors();
@@ -604,8 +606,9 @@ void CHThorIndexReadActivityBase::initManager(IKeyManager *manager)
 
 void CHThorIndexReadActivityBase::initPart()                                    
 { 
+    assertex(!keyIndex->isTopLevelKey());
     klManager.setown(createLocalKeyManager(keyIndex, NULL));
-    initManager(klManager);     
+    initManager(klManager, false);
     callback.setManager(klManager);
 }
 
@@ -635,7 +638,7 @@ bool CHThorIndexReadActivityBase::firstMultiPart()
         openTlk();
     verifyIndex(tlk);
     tlManager.setown(createLocalKeyManager(tlk, NULL));
-    initManager(tlManager);
+    initManager(tlManager, true);
     nextPartNumber = 0;
     return nextMultiPart();
 }
@@ -654,8 +657,9 @@ bool CHThorIndexReadActivityBase::nextMultiPart()
         {
             while (tlManager->lookup(false))
             {
-                if (tlManager->queryFpos())
-                    return setCurrentPart((unsigned)tlManager->queryFpos()-1);
+                offset_t node = extractFpos(tlManager);
+                if (node)
+                    return setCurrentPart((unsigned)node-1);
             }
         }
     }
@@ -866,7 +870,7 @@ bool CHThorIndexReadActivity::nextPart()
     {
         klManager.setown(createKeyMerger(keyIndexCache, seekGEOffset, NULL));
         keyIndexCache.clear();
-        initManager(klManager);
+        initManager(klManager, false);
         callback.setManager(klManager);
         return true;
     }
@@ -911,7 +915,7 @@ const void *CHThorIndexReadActivity::nextRow()
             keyedProcessed++;
             if ((keyedLimit != (unsigned __int64) -1) && keyedProcessed > keyedLimit)
                 helper.onKeyedLimitExceeded();
-            byte const * keyRow = klManager->queryKeyBuffer(callback.getFPosRef());
+            byte const * keyRow = klManager->queryKeyBuffer();
             if (needTransform)
             {
                 try
@@ -992,7 +996,7 @@ const void *CHThorIndexReadActivity::nextRowGE(const void * seek, unsigned numFi
 
         if (klManager->lookupSkip(rawSeek, seekGEOffset, seekSize))
         {
-            const byte * row = klManager->queryKeyBuffer(callback.getFPosRef());
+            const byte * row = klManager->queryKeyBuffer();
 #ifdef _DEBUG
             if (memcmp(row + seekGEOffset, rawSeek, seekSize) < 0)
                 assertex("smart seek failure");
@@ -1207,7 +1211,7 @@ const void *CHThorIndexNormalizeActivity::nextRow()
             }
 
             agent.reportProgress(NULL);
-            expanding = helper.first(klManager->queryKeyBuffer(callback.getFPosRef()));
+            expanding = helper.first(klManager->queryKeyBuffer());
             if (expanding)
             {
                 const void * ret = createNextRow();
@@ -1343,7 +1347,7 @@ void CHThorIndexAggregateActivity::gather()
         agent.reportProgress(NULL);
         try
         {
-            helper.processRow(outBuilder, klManager->queryKeyBuffer(callback.getFPosRef()));
+            helper.processRow(outBuilder, klManager->queryKeyBuffer());
         }
         catch(IException * e)
         {
@@ -1436,7 +1440,7 @@ const void *CHThorIndexCountActivity::nextRow()
                     agent.reportProgress(NULL);
                     if (!klManager->lookup(true))
                         break;
-                    totalCount += helper.numValid(klManager->queryKeyBuffer(callback.getFPosRef()));
+                    totalCount += helper.numValid(klManager->queryKeyBuffer());
                     callback.finishedRow();
                     if ((totalCount > choosenLimit))
                         break;
@@ -1554,7 +1558,7 @@ void CHThorIndexGroupAggregateActivity::gather()
         agent.reportProgress(NULL);
         try
         {
-            helper.processRow(klManager->queryKeyBuffer(callback.getFPosRef()), this);
+            helper.processRow(klManager->queryKeyBuffer(), this);
         }
         catch(IException * e)
         {
@@ -2702,20 +2706,18 @@ public:
             ReleaseRoxieRow(rows.item(idx));
     }
 
-    void addRightMatch(void * right, offset_t fpos);
+    void addRightMatch(void * right);
     offset_t addRightPending();
-    void setPendingRightMatch(offset_t seq, void * right, offset_t fpos);
+    void setPendingRightMatch(offset_t seq, void * right);
     void incRightMatchCount();
 
     unsigned count() const { return rows.ordinality(); }
     CJoinGroup * queryJoinGroup() const { return jg; }
     void * queryRow(unsigned idx) const { return rows.item(idx); }
-    offset_t queryOffset(unsigned idx) const { return offsets.item(idx); }
 
 private:
     CJoinGroup * jg;
     PointerArray rows;
-    Int64Array offsets;
 };
 
 interface IJoinProcessor
@@ -2738,7 +2740,6 @@ public:
     public:
         // Single threaded by now
         void const * queryRow() const { return owner.matchsets.item(ms).queryRow(idx); }
-        offset_t queryOffset() const { return owner.matchsets.item(ms).queryOffset(idx); }
         bool start()
         {
             idx = 0;
@@ -2873,12 +2874,11 @@ protected:
     unsigned candidates;
 };
 
-void MatchSet::addRightMatch(void * right, offset_t fpos)
+void MatchSet::addRightMatch(void * right)
 {
     assertex(!jg->complete());
     CriticalBlock b(jg->crit);
     rows.append(right);
-    offsets.append(fpos);
     jg->matchcount++;
 }
 
@@ -2888,16 +2888,14 @@ offset_t MatchSet::addRightPending()
     CriticalBlock b(jg->crit);
     offset_t seq = rows.ordinality();
     rows.append(NULL);
-    offsets.append(0);
     return seq;
 }
 
-void MatchSet::setPendingRightMatch(offset_t seq, void * right, offset_t fpos)
+void MatchSet::setPendingRightMatch(offset_t seq, void * right)
 {
     assertex(!jg->complete());
     CriticalBlock b(jg->crit);
     rows.replace(right, (aindex_t)seq);
-    offsets.replace(fpos, (aindex_t)seq);
     jg->matchcount++;
 }
 
@@ -3090,7 +3088,7 @@ public:
                 owner.readyManager(&manager, row);
                 while(manager.lookup(false))
                 {
-                    unsigned recptr = (unsigned)manager.queryFpos();
+                    unsigned recptr = (unsigned)extractFpos(&manager);
                     if (recptr)
                     {
                         jg->notePending();
@@ -3119,8 +3117,6 @@ public:
             trans.setown(owner.getLayoutTranslator(&f));
             owner.verifyIndex(&f, index, trans);
             Owned<IKeyManager> manager = createLocalKeyManager(index, NULL);
-            if(trans)
-                manager->setLayoutTranslator(trans);
             managers.append(*manager.getLink());
         }
         opened = true;
@@ -3161,7 +3157,7 @@ void KeyedLookupPartHandler::openPart()
     Owned<IKeyIndex> index = openKeyFile(*part);
     manager.setown(createLocalKeyManager(index, NULL));
     IRecordLayoutTranslator * trans = tlk->queryRecordLayoutTranslator();
-    if(trans)
+    if(trans && !index->isTopLevelKey())
         manager->setLayoutTranslator(trans);
 }
 
@@ -3554,7 +3550,7 @@ public:
                 RtlDynamicRowBuilder extractBuilder(queryRightRowAllocator()); 
                 size32_t size = helper.extractJoinFields(extractBuilder, row, fetch->pos, NULL);
                 void * ret = (void *) extractBuilder.finalizeRowClear(size);
-                fetch->ms->setPendingRightMatch(fetch->seq, ret, fetch->pos);
+                fetch->ms->setPendingRightMatch(fetch->seq, ret);
             }
         }
         fetch->ms->queryJoinGroup()->noteEnd();
@@ -3761,7 +3757,7 @@ public:
                                 RtlDynamicRowBuilder rowBuilder(rowAllocator);
                                 void const * row = jg->matches.queryRow();
                                 if(!row) continue;
-                                offset_t fpos = jg->matches.queryOffset();
+                                offset_t fpos = 0;
                                 size32_t transformedSize;
                                 transformedSize = helper.transform(rowBuilder, left, row, fpos, ++counter);
                                 if (transformedSize)
@@ -3800,7 +3796,7 @@ public:
                             void const * row = jg->matches.queryRow();
                             if(!row) continue;
                             ++count;
-                            offset_t fpos = jg->matches.queryOffset();
+                            offset_t fpos = 0;
                             size32_t transformedSize;
                             try
                             {
@@ -3953,15 +3949,16 @@ public:
             return true;
         }
         KLBlobProviderAdapter adapter(manager);
-        offset_t recptr;
-        byte const * rhs = manager->queryKeyBuffer(recptr);
-        if(indexReadMatch(jg->queryLeft(), rhs, recptr, &adapter))
+        byte const * rhs = manager->queryKeyBuffer();
+        size_t fposOffset = manager->queryRowSize() - sizeof(offset_t);
+        offset_t fpos = rtlReadBigUInt8(rhs + fposOffset);
+        if(indexReadMatch(jg->queryLeft(), rhs, fpos, &adapter))
         {
             if(needsDiskRead)
             {
                 jg->notePending();
                 offset_t seq = ms->addRightPending();
-                parts->addRow(ms, recptr, seq);
+                parts->addRow(ms, fpos, seq);
             }
             else
             {
@@ -3970,9 +3967,9 @@ public:
                 else
                 {
                     RtlDynamicRowBuilder rowBuilder(queryRightRowAllocator()); 
-                    size32_t size = helper.extractJoinFields(rowBuilder, rhs, recptr, &adapter);
+                    size32_t size = helper.extractJoinFields(rowBuilder, rhs, fpos, &adapter);
                     void * ret = (void *)rowBuilder.finalizeRowClear(size);
-                    ms->addRightMatch(ret, recptr);
+                    ms->addRightMatch(ret);
                 }
             }
         }

roxie/ccd/ccdactivities.cpp

@@ -3541,7 +3541,7 @@ public:
 
                         atomic_inc(&indexRecordsRead);
                         size32_t transformedSize;
-                        const byte * keyRow = tlk->queryKeyBuffer(callback.getFPosRef());
+                        const byte * keyRow = tlk->queryKeyBuffer();
                         int diff = 0;
                         if (steppingRow)
                         {
@@ -3801,7 +3801,7 @@ public:
                     }
 
                     atomic_inc(&indexRecordsRead);
-                    if (normalizeHelper->first(tlk->queryKeyBuffer(callback.getFPosRef())))
+                    if (normalizeHelper->first(tlk->queryKeyBuffer()))
                     {
                         do
                         {
@@ -3932,7 +3932,7 @@ public:
                     {
                         keyprocessed++;
                         atomic_inc(&indexRecordsRead);
-                        count += countHelper->numValid(tlk->queryKeyBuffer(callback.getFPosRef()));
+                        count += countHelper->numValid(tlk->queryKeyBuffer());
                         if (count > rowLimit)
                             limitExceeded(false);
                         else if (count > keyedLimit)
@@ -4052,7 +4052,7 @@ public:
                 {
                     keyprocessed++;
                     atomic_inc(&indexRecordsRead);
-                    aggregateHelper->processRow(rowBuilder, tlk->queryKeyBuffer(callback.getFPosRef()));
+                    aggregateHelper->processRow(rowBuilder, tlk->queryKeyBuffer());
                     callback.finishedRow();
                 }
                 callback.setManager(NULL);
@@ -4193,7 +4193,7 @@ public:
                     {
                         if (groupSegCount && !layoutTranslators->item(lastPartNo.fileNo))
                         {
-                            AggregateRowBuilder &rowBuilder = results.addRow(tlk->queryKeyBuffer(callback.getFPosRef()));
+                            AggregateRowBuilder &rowBuilder = results.addRow(tlk->queryKeyBuffer());
                             callback.finishedRow();
                             if (kind == TAKindexgroupcount)
                             {
@@ -4207,7 +4207,7 @@ public:
                         {
                             keyprocessed++;
                             atomic_inc(&indexRecordsRead);
-                            aggregateHelper->processRow(tlk->queryKeyBuffer(callback.getFPosRef()), this);
+                            aggregateHelper->processRow(tlk->queryKeyBuffer(), this);
                             callback.finishedRow();
                         }
                     }
@@ -4858,9 +4858,10 @@ IMessagePacker *CRoxieKeyedJoinIndexActivity::process()
                     candidateCount++;
                     atomic_inc(&indexRecordsRead);
                     KLBlobProviderAdapter adapter(tlk);
-                    offset_t recptr;
-                    const byte *indexRow = tlk->queryKeyBuffer(recptr);
-                    if (helper->indexReadMatch(inputRow, indexRow, recptr, &adapter))
+                    const byte *indexRow = tlk->queryKeyBuffer();
+                    size_t fposOffset = tlk->queryRowSize() - sizeof(offset_t);
+                    offset_t fpos = rtlReadBigUInt8(indexRow + fposOffset);
+                    if (helper->indexReadMatch(inputRow, indexRow, fpos, &adapter))
                     {
                         processed++;
                         if (keepLimit)
@@ -4885,7 +4886,7 @@ IMessagePacker *CRoxieKeyedJoinIndexActivity::process()
                         {
                             const void *self = output->getBuffer(KEYEDJOIN_RECORD_SIZE(0), true);
                             KeyedJoinHeader *rec = (KeyedJoinHeader *) self;
-                            rec->fpos = recptr;
+                            rec->fpos = fpos;
                             rec->thisGroup = jg;
                             rec->partNo = lastPartNo.partNo;
                             output->putBuffer(self, KEYEDJOIN_RECORD_SIZE(0), true);
@@ -4893,8 +4894,8 @@ IMessagePacker *CRoxieKeyedJoinIndexActivity::process()
                         else
                         {
                             KLBlobProviderAdapter adapter(tlk);
-                            totalSize = helper->extractJoinFields(rowBuilder, indexRow, recptr, &adapter);
-                            rowBuilder.writeToOutput(totalSize, recptr, jg, lastPartNo.partNo);
+                            totalSize = helper->extractJoinFields(rowBuilder, indexRow, fpos, &adapter);
+                            rowBuilder.writeToOutput(totalSize, fpos, jg, lastPartNo.partNo);
                         }
                         totalSizeSent += KEYEDJOIN_RECORD_SIZE(totalSize);
                         if (totalSizeSent > indexReadChunkSize && !continuationFailed)

roxie/ccd/ccdserver.cpp

@@ -22811,7 +22811,10 @@ public:
                             else
                             {
                                 tlk.setown(createLocalKeyManager(thisKey, this));
-                                tlk->setLayoutTranslator(translators->item(fileNo));
+                                if (!thisKey->isTopLevelKey())
+                                    tlk->setLayoutTranslator(translators->item(fileNo));
+                                else
+                                    tlk->setLayoutTranslator(nullptr);
                             }
                             createSegmentMonitors(tlk);
                             if (queryTraceLevel() > 3 || ctx->queryProbeManager())
@@ -22834,7 +22837,7 @@ public:
                                         {
                                             while (tlk->lookup(false))
                                             {
-                                                unsigned slavePart = (unsigned) tlk->queryFpos();
+                                                unsigned slavePart = (unsigned)extractFpos(tlk);
                                                 if (slavePart)
                                                 {
                                                     accepted++;
@@ -22861,7 +22864,10 @@ public:
                                 {
                                     thisKey = thisBase->queryPart(fileNo);
                                     tlk->setKey(thisKey);
-                                    tlk->setLayoutTranslator(translators->item(fileNo));
+                                    if (!thisKey->isTopLevelKey())
+                                        tlk->setLayoutTranslator(translators->item(fileNo));
+                                    else
+                                        tlk->setLayoutTranslator(nullptr);
                                     tlk->reset();
                                 }
                                 else
@@ -23073,7 +23079,8 @@ public:
                 tlk.setown(createKeyMerger(keySet, owner.seekGEOffset, &owner));
             else
                 tlk.setown(createLocalKeyManager(keySet->queryPart(0), &owner));
-            tlk->setLayoutTranslator(trans);
+            if (!key->isTopLevelKey())
+                tlk->setLayoutTranslator(trans);
             owner.indexHelper.createSegmentMonitors(tlk);
             tlk->finishSegmentMonitors();
             tlk->reset();
@@ -23091,7 +23098,7 @@ public:
                 }
                 size32_t transformedSize;
                 RtlDynamicRowBuilder rowBuilder(owner.rowAllocator);
-                byte const * keyRow = tlk->queryKeyBuffer(owner.callback.getFPosRef());
+                byte const * keyRow = tlk->queryKeyBuffer();
                 try
                 {
                     transformedSize = owner.readHelper.transform(rowBuilder, keyRow);
@@ -23598,7 +23605,7 @@ public:
                 break;
             }
 
-            byte const * keyRow = tlk->queryKeyBuffer(callback.getFPosRef());
+            byte const * keyRow = tlk->queryKeyBuffer();
 #ifdef _DEBUG
 //          StringBuffer recstr;
 //          unsigned size = (tlk->queryRecordSize()<80)  ? tlk->queryRecordSize() : 80;
@@ -23908,7 +23915,7 @@ public:
             {
                 try
                 {
-                    count += countHelper.numValid(tlk->queryKeyBuffer(callback.getFPosRef()));
+                    count += countHelper.numValid(tlk->queryKeyBuffer());
                     callback.finishedRow();
                 }
                 catch (IException *E)
@@ -24150,7 +24157,7 @@ public:
             }
             try
             {
-                aggregateHelper.processRow(rowBuilder, tlk->queryKeyBuffer(callback.getFPosRef()));
+                aggregateHelper.processRow(rowBuilder, tlk->queryKeyBuffer());
                 callback.finishedRow();
             }
             catch (IException *E)
@@ -24331,7 +24338,7 @@ public:
             {
                 if (groupSegCount && !trans)
                 {
-                    AggregateRowBuilder &rowBuilder = singleAggregator.addRow(tlk->queryKeyBuffer(callback.getFPosRef()));
+                    AggregateRowBuilder &rowBuilder = singleAggregator.addRow(tlk->queryKeyBuffer());
                     callback.finishedRow();
                     if (kind==TAKindexgroupcount)                   
                     {
@@ -24343,7 +24350,7 @@ public:
                 }
                 else
                 {
-                    aggregateHelper.processRow(tlk->queryKeyBuffer(callback.getFPosRef()), this);
+                    aggregateHelper.processRow(tlk->queryKeyBuffer(), this);
                     callback.finishedRow();
                 }
             }
@@ -24479,7 +24486,7 @@ public:
             }
             size32_t transformedSize;
     
-            if (readHelper.first(tlk->queryKeyBuffer(callback.getFPosRef())))
+            if (readHelper.first(tlk->queryKeyBuffer()))
             {
                 Owned<CRowArrayMessageResult> result = new CRowArrayMessageResult(ctx->queryRowManager(), meta.isVariableSize());
                 do
@@ -25444,7 +25451,10 @@ public:
                     try
                     {
                         tlk->setKey(thisKey);
-                        tlk->setLayoutTranslator(translators->item(fileNo));
+                        if (!thisKey->isTopLevelKey())
+                            tlk->setLayoutTranslator(translators->item(fileNo));
+                        else
+                            tlk->setLayoutTranslator(nullptr);
                         helper.createSegmentMonitors(tlk, extracted);
                         if (rootIndex)
                             rootIndex->mergeSegmentMonitors(tlk);
@@ -25458,7 +25468,7 @@ public:
                                 bool locallySorted = !thisKey->isFullySorted();
                                 while (locallySorted || tlk->lookup(false)) 
                                 {
-                                    unsigned slavePart = locallySorted ? 0 : (unsigned) tlk->queryFpos();
+                                    unsigned slavePart = locallySorted ? 0 : (unsigned)extractFpos(tlk);
                                     if (locallySorted || slavePart)
                                     {
                                         cvp *outputBuffer = (cvp *) remote.getMem(slavePart, fileNo, indexReadSize + sizeof(cvp) + (indexReadInputRecordVariable ? sizeof(unsigned) : 0));
@@ -25492,12 +25502,13 @@ public:
                                     candidateCount++;
                                     atomic_inc(&indexRecordsRead);
                                     KLBlobProviderAdapter adapter(tlk);
-                                    offset_t recptr;
-                                    const byte *indexRow = tlk->queryKeyBuffer(recptr);
-                                    if (helper.indexReadMatch(extracted, indexRow, recptr, &adapter))
+                                    const byte *indexRow = tlk->queryKeyBuffer();
+                                    size_t fposOffset = tlk->queryRowSize() - sizeof(offset_t);
+                                    offset_t fpos = rtlReadBigUInt8(indexRow + fposOffset);
+                                    if (helper.indexReadMatch(extracted, indexRow, fpos, &adapter))
                                     {
                                         KeyedJoinHeader *rhs = (KeyedJoinHeader *) ctx->queryRowManager().allocate(KEYEDJOIN_RECORD_SIZE(0), activityId);
-                                        rhs->fpos = recptr; 
+                                        rhs->fpos = fpos;
                                         rhs->thisGroup = jg; 
                                         rhs->partNo = partNo; 
                                         result->append(rhs);
@@ -25525,7 +25536,10 @@ public:
                             {
                                 thisKey = thisBase->queryPart(fileNo);
                                 tlk->setKey(thisKey);
-                                tlk->setLayoutTranslator(translators->item(fileNo));
+                                if (!thisKey->isTopLevelKey())
+                                    tlk->setLayoutTranslator(translators->item(fileNo));
+                                else
+                                    tlk->setLayoutTranslator(nullptr);
                                 tlk->reset();
                             }
                             else
@@ -26276,7 +26290,10 @@ public:
                     unsigned fileNo = 0;
                     IKeyIndex *thisKey = thisBase->queryPart(fileNo);
                     tlk->setKey(thisKey);
-                    tlk->setLayoutTranslator(translators->item(fileNo));
+                    if (thisKey && !thisKey->isTopLevelKey())
+                        tlk->setLayoutTranslator(translators->item(fileNo));
+                    else
+                        tlk->setLayoutTranslator(nullptr);
                     helper.createSegmentMonitors(tlk, extracted);
                     if (rootIndex)
                         rootIndex->mergeSegmentMonitors(tlk);
@@ -26292,7 +26309,7 @@ public:
                                 bool locallySorted = (!thisKey->isFullySorted());
                                 while (locallySorted || tlk->lookup(false))
                                 {
-                                    unsigned slavePart = locallySorted ? 0 : (unsigned) tlk->queryFpos();
+                                    unsigned slavePart = locallySorted ? 0 : (unsigned)extractFpos(tlk);
                                     if (locallySorted || slavePart)
                                     {
                                         cvp *outputBuffer = (cvp *) remote.getMem(slavePart, fileNo, indexReadRecordSize + sizeof(cvp) + (indexReadInputRecordVariable ? sizeof(unsigned) : 0));
@@ -26328,17 +26345,18 @@ public:
                                     candidateCount++;
                                     atomic_inc(&indexRecordsRead);
                                     KLBlobProviderAdapter adapter(tlk);
-                                    offset_t recptr;
-                                    const byte *indexRow = tlk->queryKeyBuffer(recptr);
-                                    if (helper.indexReadMatch(extracted, indexRow, recptr, &adapter))
+                                    const byte *indexRow = tlk->queryKeyBuffer();
+                                    size_t fposOffset = tlk->queryRowSize() - sizeof(offset_t);
+                                    offset_t fpos = rtlReadBigUInt8(indexRow + fposOffset);
+                                    if (helper.indexReadMatch(extracted, indexRow, fpos, &adapter))
                                     {
                                         RtlDynamicRowBuilder rb(joinFieldsAllocator, true); 
                                         CPrefixedRowBuilder pb(KEYEDJOIN_RECORD_SIZE(0), rb);
                                         accepted++;
                                         KLBlobProviderAdapter adapter(tlk);
-                                        size32_t joinFieldsSize = helper.extractJoinFields(pb, indexRow, recptr, &adapter);
+                                        size32_t joinFieldsSize = helper.extractJoinFields(pb, indexRow, fpos, &adapter);
                                         KeyedJoinHeader *rec = (KeyedJoinHeader *) rb.getUnfinalizedClear(); // lack of finalize ok as unserialized data here.
-                                        rec->fpos = recptr;
+                                        rec->fpos = fpos;
                                         rec->thisGroup = jg;
                                         rec->partNo = partNo;
                                         if (isSimple)
@@ -26373,7 +26391,10 @@ public:
                             {
                                 thisKey = thisBase->queryPart(fileNo);
                                 tlk->setKey(thisKey);
-                                tlk->setLayoutTranslator(translators->item(fileNo));
+                                if (thisKey && !thisKey->isTopLevelKey())
+                                    tlk->setLayoutTranslator(translators->item(fileNo));
+                                else
+                                    tlk->setLayoutTranslator(nullptr);
                                 tlk->reset();
                             }
                             else

roxie/ccd/ccdserver.hpp

@@ -280,13 +280,13 @@ extern void setStartRuid(unsigned restarts);
 class CIndexTransformCallback : implements IThorIndexCallback, public CInterface
 {
 public:
-    CIndexTransformCallback() { keyManager = NULL; cleanupRequired = false; filepos = 0; };
+    CIndexTransformCallback() { keyManager = NULL; cleanupRequired = false; };
     IMPLEMENT_IINTERFACE
 
 //IThorIndexCallback
     virtual unsigned __int64 getFilePosition(const void * row)
     {
-        return filepos;
+        throwUnexpected();
     }
     virtual byte * lookupBlob(unsigned __int64 id) 
     { 
@@ -297,7 +297,6 @@ public:
 
 
 public:
-    inline offset_t & getFPosRef()                              { return filepos; }
     inline void setManager(IKeyManager * _manager)
     {
         finishedRow();
@@ -314,7 +313,6 @@ public:
 
 protected:
     IKeyManager * keyManager;
-    offset_t filepos;
     bool cleanupRequired;
 };
 

rtl/eclrtl/eclrtl.hpp

@@ -470,6 +470,12 @@ ECLRTL_API void rtlWriteSwapInt7(void * data, unsigned __int64 value);
 ECLRTL_API void rtlWriteSwapInt8(void * data, unsigned __int64 value);
 ECLRTL_API void rtlWriteSwapInt(void * self, __int64 val, unsigned length);
 
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+inline unsigned __int64 rtlReadBigUInt8(const void * data) { return rtlReadSwapUInt8(data); }
+#else
+inline unsigned __int64 rtlReadBigUInt8(const void * data) { return rtlReadUInt8(data); }
+#endif
+
 ECLRTL_API short rtlRevInt2(const void * data);
 ECLRTL_API int rtlRevInt3(const void * data);
 ECLRTL_API int rtlRevInt4(const void * data);

system/jhtree/ctfile.cpp

@@ -777,7 +777,10 @@ bool CJHTreeNode::getValueAt(unsigned int index, char *dst) const
 
 size32_t CJHTreeNode::getSizeAt(unsigned int index) const
 {
-    return keyLen;
+    if (keyHdr->hasSpecialFileposition())
+        return keyLen + sizeof(offset_t);
+    else
+        return keyLen;
 }
 
 offset_t CJHTreeNode::getFPosAt(unsigned int index) const
@@ -951,7 +954,10 @@ size32_t CJHVarTreeNode::getSizeAt(unsigned int num) const
     const char * p = recArray[num];
     KEYRECSIZE_T reclen = ((KEYRECSIZE_T *) p)[-1];
     _WINREV(reclen);
-    return reclen;
+    if (keyHdr->hasSpecialFileposition())
+        return reclen + sizeof(offset_t);
+    else
+        return reclen;
 }
 
 offset_t CJHVarTreeNode::getFPosAt(unsigned int num) const

system/jhtree/jhtree.cpp

@@ -392,7 +392,6 @@ protected:
     SegMonitorList segs;
     IKeyCursor *keyCursor;
     char *keyBuffer;
-    offset_t lookupFpos;
     unsigned keySize;       // size of key record including payload
     unsigned keyedSize;     // size of non-payload part of key
     unsigned numsegs;
@@ -410,6 +409,7 @@ protected:
     bool transformSegs;
     IIndexReadContext * activitySegs;
     CMemoryBlock layoutTransBuff;
+    size32_t layoutSize = 0;
     Owned<IRecordLayoutTranslator::SegmentMonitorContext> layoutTransSegCtx;
     Owned<IRecordLayoutTranslator::RowTransformContext> layoutTransRowCtx;
 
@@ -433,9 +433,9 @@ protected:
         segs.endRange(segno, keyBuffer);
     }
 
-    byte const * doRowLayoutTransform(offset_t & fpos)
+    byte const * doRowLayoutTransform()
     {
-        layoutTrans->transformRow(layoutTransRowCtx, reinterpret_cast<byte const *>(keyBuffer), keySize, layoutTransBuff, fpos);
+        layoutSize = layoutTrans->transformRow(layoutTransRowCtx, reinterpret_cast<byte const *>(keyBuffer), keySize, layoutTransBuff);
         return layoutTransBuff.get();
     }
 
@@ -690,18 +690,20 @@ public:
         return activitySegs->item(idx);
     }
 
-    inline const byte *queryKeyBuffer(offset_t & fpos)
+    inline const byte *queryKeyBuffer()
     {
-        fpos = lookupFpos;
         if(layoutTrans)
-            return doRowLayoutTransform(fpos);
+            return doRowLayoutTransform();
         else
             return reinterpret_cast<byte const *>(keyBuffer);
     }
 
     inline size32_t queryRowSize()
     {
-        return keyCursor ? keyCursor->getSize() : 0;
+        if (layoutTrans)
+            return layoutSize;
+        else
+            return keyCursor ? keyCursor->getSize() : 0;
     }
 
     inline unsigned __int64 querySequence()
@@ -709,11 +711,6 @@ public:
         return keyCursor ? keyCursor->getSequence() : 0;
     }
 
-    inline offset_t queryFpos()
-    {
-        return lookupFpos;
-    }
-
     inline unsigned queryRecordSize() { return keySize; }
 
     bool _lookup(bool exact, unsigned lastSeg)
@@ -737,7 +734,6 @@ public:
             }
             if (ok)
             {
-                lookupFpos = keyCursor->getFPos();
                 unsigned i = 0;
                 matched = true;
                 if (segs.segMonitors.length())
@@ -2371,10 +2367,8 @@ class CKeyMerger : public CKeyLevelManager
     PointerArray bufferArray;
     PointerArray fixedArray;
     BoolArray matchedArray; 
-    UInt64Array fposArray;
     UnsignedArray mergeHeapArray;
     UnsignedArray keyNoArray;
-    offset_t *fposes;
 
     IKeyCursor **cursors;
     char **buffers;
@@ -2448,7 +2442,6 @@ class CKeyMerger : public CKeyLevelManager
         keyBuffer = buffers[key];
         keyCursor = cursors[key];
         matched = matcheds[key];
-        lookupFpos = fposes[key];
         for (unsigned segno = 0; segno < sortFromSeg; segno++)
         {
             IOverrideableKeySegmentMonitor *sm = QUERYINTERFACE(&segs.segMonitors.item(segno), IOverrideableKeySegmentMonitor);
@@ -2492,7 +2485,6 @@ public:
         cursorArray.kill();
         keyCursor = NULL; // cursorArray owns cursors
         matchedArray.kill();
-        fposArray.kill();
         mergeHeapArray.kill();
         bufferArray.kill();
         fixedArray.kill();
@@ -2500,7 +2492,6 @@ public:
 
         cursors = NULL;
         matcheds = NULL;
-        fposes = NULL;
         mergeheap = NULL;
         buffers = NULL; 
         fixeds = NULL;
@@ -2546,11 +2537,7 @@ public:
             unsigned compares = 0;
             for (;;)
             {
-                if (CKeyLevelManager::lookupSkip(seek, seekOffset, seeklen) )
-                {
-                    fposes[key] = lookupFpos;
-                }
-                else
+                if (!CKeyLevelManager::lookupSkip(seek, seekOffset, seeklen) )
                 {
                     activekeys--;
                     if (!activekeys)
@@ -2700,7 +2687,6 @@ public:
                     cursorArray.append(*keyCursor);
                     bufferArray.append(keyBuffer);
                     matchedArray.append(matched);
-                    fposArray.append(lookupFpos);
                     mergeHeapArray.append(activekeys++);
                     if (!sortFromSeg)
                     {
@@ -2755,7 +2741,6 @@ public:
             if (ctx)
                 ctx->noteStatistic(StNumIndexMerges, activekeys);
             cursors = cursorArray.getArray();
-            fposes = fposArray.getArray();
             matcheds = (bool *) matchedArray.getArray();  // For some reason BoolArray is typedef'd to CharArray on linux...
             buffers = (char **) bufferArray.getArray();
             mergeheap = mergeHeapArray.getArray();
@@ -2827,11 +2812,7 @@ public:
             if (!activekeys)
                 return false;
             unsigned key = mergeheap[0];
-            if (CKeyLevelManager::lookup(exact)) 
-            {
-                fposes[key] = lookupFpos;
-            }
-            else
+            if (!CKeyLevelManager::lookup(exact)) 
             {
                 activekeys--;
                 if (!activekeys)
@@ -2917,7 +2898,6 @@ public:
             mb.append(keyNoArray.item(key));
             cursors[key]->serializeCursorPos(mb);
             mb.append(matcheds[key]);
-            mb.append(fposes[key]);
         }
     }
 
@@ -2937,9 +2917,6 @@ public:
             keyCursor->deserializeCursorPos(mb, keyBuffer);
             mb.read(matched);
             matchedArray.append(matched);
-            offset_t fpos;
-            mb.read(fpos);
-            fposArray.append(fpos);
             bufferArray.append(keyBuffer);
             void *fixedValue = (char *) malloc(sortFieldOffset);
             memcpy(fixedValue, keyBuffer, sortFieldOffset); // If it's not at EOF then it must match
@@ -2947,7 +2924,6 @@ public:
             mergeHeapArray.append(i);
         }
         cursors = cursorArray.getArray();
-        fposes = fposArray.getArray();
         matcheds = (bool *) matchedArray.getArray();  // For some reason BoolArray is typedef'd to CharArray on linux...
         buffers = (char **) bufferArray.getArray();
         mergeheap = mergeHeapArray.getArray();
@@ -3067,23 +3043,23 @@ class IKeyManagerTest : public CppUnit::TestFixture
             tlk1->reset();
 
             offset_t fpos;
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003010", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005010", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006010", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003030", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005030", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006030", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003031", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005031", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006031", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003010", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005010", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006010", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003030", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005030", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006030", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003031", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005031", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006031", 10)==0);
             MemoryBuffer mb;
             tlk1->serializeCursorPos(mb);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003032", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005032", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006032", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000003033", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000005033", 10)==0);
-            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000006033", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003032", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005032", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006032", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000003033", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000005033", 10)==0);
+            ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000006033", 10)==0);
             ASSERT(!tlk1->lookup(true)); 
             ASSERT(!tlk1->lookup(true)); 
 
@@ -3094,12 +3070,12 @@ class IKeyManagerTest : public CppUnit::TestFixture
             tlk2->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
             tlk2->finishSegmentMonitors();
             tlk2->reset(true);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000003032", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000005032", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000006032", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000003033", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000005033", 10)==0);
-            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000006033", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000003032", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000005032", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000006032", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000003033", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000005033", 10)==0);
+            ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000006033", 10)==0);
             ASSERT(!tlk2->lookup(true)); 
             ASSERT(!tlk2->lookup(true)); 
 
@@ -3109,10 +3085,10 @@ class IKeyManagerTest : public CppUnit::TestFixture
             tlk3->append(createKeySegmentMonitor(false, sset2.getLink(), 7, 3));
             tlk3->finishSegmentMonitors();
             tlk3->reset(false);
-            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), "0000003010", 10)==0);
-            ASSERT(tlk3->lookupSkip("031", 7, 3)); ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), "0000003031", 10)==0);
-            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), "0000005031", 10)==0);
-            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), "0000006031", 10)==0);
+            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(), "0000003010", 10)==0);
+            ASSERT(tlk3->lookupSkip("031", 7, 3)); ASSERT(memcmp(tlk3->queryKeyBuffer(), "0000003031", 10)==0);
+            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(), "0000005031", 10)==0);
+            ASSERT(tlk3->lookup(true)); ASSERT(memcmp(tlk3->queryKeyBuffer(), "0000006031", 10)==0);
             ASSERT(!tlk3->lookupSkip("081", 7, 3)); 
             ASSERT(!tlk3->lookup(true)); 
 
@@ -3122,18 +3098,18 @@ class IKeyManagerTest : public CppUnit::TestFixture
             tlk4->append(createKeySegmentMonitor(false, sset3.getLink(), 7, 3));
             tlk4->finishSegmentMonitors();
             tlk4->reset(false);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000003000", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000005000", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000006000", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000003001", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000005001", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000006001", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000003002", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000005002", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000006002", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000003999", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000005999", 10)==0);
-            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(fpos), "0000006999", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000003000", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000005000", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000006000", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000003001", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000005001", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000006001", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000003002", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000005002", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000006002", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000003999", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000005999", 10)==0);
+            ASSERT(tlk4->lookup(true)); ASSERT(memcmp(tlk4->queryKeyBuffer(), "0000006999", 10)==0);
             ASSERT(!tlk4->lookup(true)); 
             ASSERT(!tlk4->lookup(true)); 
 
@@ -3208,8 +3184,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
     void checkBlob(IKeyManager *key, unsigned size)
     {
         unsigned __int64 blobid;
-        offset_t fpos;
-        memcpy(&blobid, key->queryKeyBuffer(fpos)+10, sizeof(blobid));
+        memcpy(&blobid, key->queryKeyBuffer()+10, sizeof(blobid));
         ASSERT(blobid != 0);
         size32_t blobsize;
         const byte *blob = key->loadBlob(blobid, blobsize);
@@ -3261,10 +3236,9 @@ protected:
 
 /*          for (;;)
             {
-                offset_t fpos;
                 if (!tlk1a->lookup(true))
                     break;
-                DBGLOG("%.10s", tlk1a->queryKeyBuffer(fpos));
+                DBGLOG("%.10s", tlk1a->queryKeyBuffer());
             }
             tlk1a->reset();
 */
@@ -3346,42 +3320,42 @@ protected:
             {
                 offset_t fpos;
                 tlk1->reset();
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000001", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000002", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000003", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000005", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000006", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000007", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000009", 10)==0);
-                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(fpos), "0000000010", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000001", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000002", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000003", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000005", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000006", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000007", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000009", 10)==0);
+                ASSERT(tlk1->lookup(true)); ASSERT(memcmp(tlk1->queryKeyBuffer(), "0000000010", 10)==0);
                 if (blobby)
                     checkBlob(tlk1, 10+100000);
 
                 tlk1a->reset();
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000001", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000010", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000011", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000021", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000030", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000031", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000041", 10)==0);
-                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(fpos), "0000000050", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000001", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000010", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000011", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000021", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000030", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000031", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000041", 10)==0);
+                ASSERT(tlk1a->lookup(true)); ASSERT(memcmp(tlk1a->queryKeyBuffer(), "0000000050", 10)==0);
 
                 tlk2->reset();
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000004", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000008", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000012", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000016", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000020", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000024", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000028", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000032", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000036", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000040", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000044", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000048", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000048", 10)==0);
-                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(fpos), "0000000052", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000004", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000008", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000012", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000016", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000020", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000024", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000028", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000032", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000036", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000040", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000044", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000048", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000048", 10)==0);
+                ASSERT(tlk2->lookup(true)); ASSERT(memcmp(tlk2->queryKeyBuffer(), "0000000052", 10)==0);
 
                 if (tlk3)
                 {
@@ -3390,11 +3364,11 @@ protected:
                     {
                         ASSERT(tlk3->lookup(true)); 
                         sprintf(buf, "%010d", i);
-                        ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), buf, 10)==0);
+                        ASSERT(memcmp(tlk3->queryKeyBuffer(), buf, 10)==0);
                         if (i==48 || i==49)
                         {
                             ASSERT(tlk3->lookup(true)); 
-                            ASSERT(memcmp(tlk3->queryKeyBuffer(fpos), buf, 10)==0);
+                            ASSERT(memcmp(tlk3->queryKeyBuffer(), buf, 10)==0);
                         }
                     }
                     ASSERT(!tlk3->lookup(true)); 

system/jhtree/jhtree.hpp

@@ -43,7 +43,6 @@ interface jhtree_decl IKeyCursor : public IInterface
     virtual bool gtEqual(const char *src, char *dst, bool seekForward = false) = 0; // returns first record >= src
     virtual bool ltEqual(const char *src, char *dst, bool seekForward = false) = 0; // returns last record <= src
     virtual size32_t getSize() = 0;
-    virtual offset_t getFPos() = 0;
     virtual void serializeCursorPos(MemoryBuffer &mb) = 0;
     virtual void deserializeCursorPos(MemoryBuffer &mb, char *keyBuffer) = 0;
     virtual unsigned __int64 getSequence() = 0;
@@ -197,8 +196,7 @@ interface IKeyManager : public IInterface, extends IIndexReadContext
     virtual void reset(bool crappyHack = false) = 0;
     virtual void releaseSegmentMonitors() = 0;
 
-    virtual const byte *queryKeyBuffer(offset_t & fpos) = 0; //if using RLT: fpos is the translated value, so correct in a normal row
-    virtual offset_t queryFpos() = 0; //if using RLT: this is the untranslated fpos, so correct as the part number in TLK but not in a normal row
+    virtual const byte *queryKeyBuffer() = 0; //if using RLT: fpos is the translated value, so correct in a normal row
     virtual unsigned __int64 querySequence() = 0;
     virtual size32_t queryRowSize() = 0;     // Size of current row as returned by queryKeyBuffer()
     virtual unsigned queryRecordSize() = 0;  // Max size
@@ -228,6 +226,14 @@ interface IKeyManager : public IInterface, extends IIndexReadContext
     virtual bool lookupSkip(const void *seek, size32_t seekGEOffset, size32_t seeklen) = 0;
 };
 
+inline offset_t extractFpos(IKeyManager * manager)
+{
+    byte const * keyRow = manager->queryKeyBuffer();
+    size32_t rowSize = manager->queryRowSize();
+    size32_t offset = rowSize - sizeof(offset_t);
+    return rtlReadBigUInt8(keyRow + offset);
+}
+
 extern jhtree_decl IKeyManager *createLocalKeyManager(IKeyIndex * _key, IContextLogger *ctx);
 extern jhtree_decl IKeyManager *createKeyMerger(IKeyIndexSet * _key, unsigned sortFieldOffset, IContextLogger *ctx);
 extern jhtree_decl IKeyManager *createSingleKeyMerger(IKeyIndex * _onekey, unsigned sortFieldOffset, IContextLogger *ctx);

system/jhtree/keydiff.cpp

@@ -59,9 +59,8 @@ public:
     {
         if(keyCursor->next(row))
         {
-            if(isVar)
-                thisrowsize = keyCursor->getSize();
-            *fpos = keyCursor->getFPos();
+            thisrowsize = keyCursor->getSize() - sizeof(offset_t);
+            *fpos = rtlReadBigUInt8(row + thisrowsize);
             return true;
         }
         *fpos = 0;
@@ -285,7 +284,8 @@ public:
         {
             if(keyCursor->next(buff))
             {
-                offset_t fpos = keyCursor->getFPos();
+                size32_t offset = keyCursor->getSize() - sizeof(offset_t);
+                offset_t fpos = rtlReadBigUInt8(buff + offset);
                 crc.tally(rowsize, buff);
                 crc.tally(sizeof(fpos), &fpos);
             }

testing/regress/ecl/layouttrans.ecl

@@ -38,14 +38,14 @@ Files := setup.Files(multiPart, useLocal, false);
 //nothorlcr
 
 #IF (version=1)
-DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Fname,Lname,__filepos},Files.DG_FetchIndex1Name);
-DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Fname,Lname,__filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Lname,Fname,__filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Lname,Fname,__filepos},Files.DG_FetchIndex1Name);
 #ELIF (version=2)
-DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Fname,Lname},{state, STRING100 blobfield {blob}:= fname, STRING tfn := TRIM(Fname), __filepos},Files.DG_FetchIndex1Name);
-DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Fname,Lname},{ STRING100 blobfield {blob}:= fname, __filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Lname,Fname},{state, STRING100 blobfield {blob}:= fname, STRING tfn := TRIM(Fname), __filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Lname,Fname},{ STRING100 blobfield {blob}:= fname, __filepos},Files.DG_FetchIndex1Name);
 #ELSE
-DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Fname,Lname},{state ,__filepos},Files.DG_FetchIndex1Name);
-DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Fname,Lname},{__filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt1 := INDEX(Files.DG_FetchFile,{Lname,Fname},{state ,__filepos},Files.DG_FetchIndex1Name);
+DG_FetchIndex1Alt2 := INDEX(Files.DG_FetchFile,{Lname,Fname},{__filepos},Files.DG_FetchIndex1Name);
 #END
 
 ds := DATASET([{'Anderson'}, {'Doe'}], {STRING25 Lname});

testing/regress/ecl/setup/files.ecl

@@ -96,7 +96,7 @@ EXPORT DG_KeyDiffIndex1 := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(
 EXPORT DG_KeyDiffIndex2 := INDEX(DG_KeyDiffIndex1, DG_KeyDiffIndex2Name);
 
 //This version is used for testing reading from a file requiring translation 
-EXPORT DG_FetchTransIndex := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname, __filepos},DG_FetchTransIndexName);
+EXPORT DG_FetchTransIndex := INDEX(DG_FetchFile,{Lname,Fname},{state, STRING tfn := TRIM(Fname), STRING100 blobfield {blob}:= fname, __filepos},DG_FetchTransIndexName);
 
 indexName := IF(useTranslation, __nameof__(DG_FetchTransIndex), __nameof__(DG_FetchIndex1));
 EXPORT DG_FetchIndex := INDEX(DG_FetchIndex1,indexName);
@@ -134,8 +134,8 @@ EXPORT DG_FlatFileEvens := DATASET(DG_FileOut+'FLAT_EVENS',{DG_OutRec,UNSIGNED8
 EXPORT DG_NormalIndexFile      := INDEX(DG_FlatFile, { DG_firstname, DG_lastname }, { DG_Prange, filepos }, DG_IndexOut+'INDEX');
 EXPORT DG_NormalIndexFileEvens := INDEX(DG_FlatFileEvens, { DG_firstname; DG_lastname; }, { DG_Prange, filepos } ,DG_IndexOut+'INDEX_EVENS');
 
-EXPORT DG_TransIndexFile      := INDEX(DG_FlatFile, { DG_lastname, DG_firstname }, { DG_Prange, filepos }, DG_IndexOut+'TRANS_INDEX');
-EXPORT DG_TransIndexFileEvens := INDEX(DG_FlatFileEvens, { DG_lastname, DG_firstname }, { DG_Prange, filepos } ,DG_IndexOut+'TRANS_INDEX_EVENS');
+EXPORT DG_TransIndexFile      := INDEX(DG_FlatFile, { DG_firstname, DG_lastname }, { DG_ChildID := filepos, DG_Prange, filepos }, DG_IndexOut+'TRANS_INDEX');
+EXPORT DG_TransIndexFileEvens := INDEX(DG_FlatFileEvens, { DG_firstname, DG_lastname }, { DG_ChildID := filepos, DG_Prange, filepos } ,DG_IndexOut+'TRANS_INDEX_EVENS');
 
 indexName := IF(useTranslation, __nameof__(DG_TransIndexFile), __nameof__(DG_NormalIndexFile));
 EXPORT DG_indexFile      := INDEX(DG_NormalIndexFile, indexName);
@@ -156,7 +156,7 @@ END;
 EXPORT DG_VarFile   := DATASET(DG_FileOut+'VAR',DG_VarOutRecPlus,FLAT);
 
 EXPORT DG_NormalVarIndex  := INDEX(DG_VarFile, { DG_firstname; DG_lastname; __filepos } ,DG_IndexOut+'VARINDEX');
-EXPORT DG_TransVarIndex  := INDEX(DG_VarFile, { DG_lastname; DG_firstname; __filepos } ,DG_IndexOut+'TRANS_VARINDEX');
+EXPORT DG_TransVarIndex  := INDEX(DG_VarFile, { DG_firstname; DG_lastname; }, { DGextra := DG_lastname; __filepos } ,DG_IndexOut+'TRANS_VARINDEX');
 
 indexName := IF(useTranslation, __nameof__(DG_TransVarIndex), __nameof__(DG_NormalVarIndex));
 EXPORT DG_VarIndex  := INDEX(DG_NormalVarIndex, indexName);

testing/regress/ecl/setup/setup_fetch.ecl

@@ -67,7 +67,7 @@ BUILDINDEX(Files.DG_KeyDiffIndex1, PROJECT(sortedFile, createDiffRow(LEFT)),OVER
 BUILDINDEX(Files.DG_KeyDiffIndex2, PROJECT(sortedFile(lname != 'Doe'),createDiffRow(LEFT)), OVERWRITE);
 
 //A version of the index with LName/FName transposed and x moved to the front.
-BUILDINDEX(sortedFile,{Fname,Lname},{STRING100 blobfield {blob}:= fname+lname, STRING tfn := TRIM(Fname), state, __filepos},Files.DG_FetchTransIndexName, OVERWRITE);
+BUILDINDEX(Files.DG_FetchTransIndex, TABLE(sortedFile, {STRING tfn := TRIM(Fname), STRING100 blobfield := fname+lname, sortedFile}), OVERWRITE);
 
 fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchFilePreloadName, '', '', 'view', '1:1', false);
 fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchFilePreloadIndexedName, '', '', 'view', '1:1', false);

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -2389,7 +2389,7 @@ class IndexDistributeSlaveActivity : public HashDistributeSlaveBase
             tlkManager->reset();
             verifyex(tlkManager->lookup(false));
             tlkManager->releaseSegmentMonitors();
-            offset_t partNo = tlkManager->queryFpos();
+            offset_t partNo = extractFpos(tlkManager);
             if (partNo)
                 partNo--; // note that partNo==0 means lower than anything in the key - should be treated same as partNo==1 here
             return ((unsigned)partNo % numslaves);

thorlcr/activities/indexread/thindexread.cpp

@@ -167,8 +167,9 @@ protected:
                 tlk->reset();
                 while (tlk->lookup(false))
                 {
-                    if (tlk->queryFpos())
-                        performPartLookup.replace(true, (aindex_t)(super?super->numSubFiles(true)*(tlk->queryFpos()-1)+superSubIndex:tlk->queryFpos()-1));
+                    offset_t node = extractFpos(tlk);
+                    if (node)
+                        performPartLookup.replace(true, (aindex_t)(super?super->numSubFiles(true)*(node-1)+superSubIndex:node-1));
                 }
             }
             if (!super||!iter->next())

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -68,7 +68,6 @@ protected:
     {
     protected:
         IKeyManager *keyManager;
-        offset_t filepos;
     public:
         TransformCallback() { keyManager = NULL; };
         IMPLEMENT_IINTERFACE_USING(CSimpleInterface)
@@ -76,14 +75,13 @@ protected:
     //IThorIndexCallback
         virtual unsigned __int64 getFilePosition(const void *row)
         {
-            return filepos;
+            throwUnexpected();
         }
         virtual byte *lookupBlob(unsigned __int64 id) 
         { 
             size32_t dummy;
             return (byte *) keyManager->loadBlob(id, dummy); 
         }
-        offset_t & getFPosRef() { return filepos; }
         void setManager(IKeyManager *_keyManager)
         {
             finishedRow();
@@ -143,7 +141,7 @@ protected:
             if (currentManager->lookup(true))
             {
                 noteStats(currentManager->querySeeks(), currentManager->queryScans());
-                ret = (const void *)currentManager->queryKeyBuffer(callback.getFPosRef());
+                ret = (const void *)currentManager->queryKeyBuffer();
             }
             if (ret || keyMergerManager)
                 break;
@@ -358,7 +356,7 @@ class CIndexReadSlaveActivity : public CIndexReadSlaveBase
         if (!currentManager->lookupSkip(rawSeek, seekGEOffset, seekSize))
             return NULL;
         noteStats(currentManager->querySeeks(), currentManager->queryScans());
-        const byte *row = currentManager->queryKeyBuffer(callback.getFPosRef());
+        const byte *row = currentManager->queryKeyBuffer();
 #ifdef _DEBUG
         if (memcmp(row + seekGEOffset, rawSeek, seekSize) < 0)
             assertex("smart seek failure");
@@ -721,7 +719,7 @@ public:
                     {
                         ++progress;
                         noteStats(keyManager.querySeeks(), keyManager.queryScans());
-                        helper->processRow(keyManager.queryKeyBuffer(callback.getFPosRef()), this);
+                        helper->processRow(keyManager.queryKeyBuffer(), this);
                         callback.finishedRow();
                     }
                     clearManager();
@@ -839,7 +837,7 @@ public:
                         if (!l)
                             break;
                         ++progress;
-                        totalCount += helper->numValid(keyManager.queryKeyBuffer(callback.getFPosRef()));
+                        totalCount += helper->numValid(keyManager.queryKeyBuffer());
                         callback.finishedRow();
                         if ((totalCount > choosenLimit))
                             break;

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -1289,8 +1289,9 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
                             if (candidateCount > owner.atMost)
                                 break;
                             KLBlobProviderAdapter adapter(currentPartKeyManager);
-                            offset_t fpos;
-                            byte const * keyRow = currentPartKeyManager->queryKeyBuffer(fpos);
+                            byte const * keyRow = currentPartKeyManager->queryKeyBuffer();
+                            size_t fposOffset = currentPartKeyManager->queryRowSize() - sizeof(offset_t);
+                            offset_t fpos = rtlReadBigUInt8(keyRow + fposOffset);
                             if (owner.helper->indexReadMatch(indexReadFieldsRow.getSelf(), keyRow, fpos, &adapter))
                             {
                                 if (currentJG->rowsSeen() >= owner.keepLimit)
@@ -1349,9 +1350,10 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem
                         for (;;)
                         {
                             if (!tlkManager->lookup(false)) break;
-                            if (tlkManager->queryFpos()) // don't bail out if part0 match, test again for 'real' tlk match.
+                            offset_t node = extractFpos(tlkManager);
+                            if (node) // don't bail out if part0 match, test again for 'real' tlk match.
                             {
-                                unsigned partNo = (unsigned)tlkManager->queryFpos();
+                                unsigned partNo = (unsigned)node;
                                 partNo = owner.superWidth ? owner.superWidth*nextTlk+(partNo-1) : partNo-1;
 
                                 currentPartKeyManager = &partKeyManagers.item(partNo);

tools/dumpkey/dumpkey.cpp

@@ -179,7 +179,7 @@ int main(int argc, const char **argv)
                     MyIndexCallback(IKeyManager *_manager) : manager(_manager) {}
                     virtual unsigned __int64 getFilePosition(const void * row)
                     {
-                        return manager->queryFpos();
+                        return 0;
                     }
                     virtual byte * lookupBlob(unsigned __int64 id)
                     {
@@ -257,8 +257,7 @@ int main(int argc, const char **argv)
                 manager->reset();
                 while (manager->lookup(true) && count--)
                 {
-                    offset_t pos;
-                    byte const * buffer = manager->queryKeyBuffer(pos);
+                    byte const * buffer = manager->queryKeyBuffer();
                     size32_t size = manager->queryRowSize();
                     unsigned __int64 seq = manager->querySequence();
                     if (optRaw)
@@ -269,7 +268,7 @@ int main(int argc, const char **argv)
                     {
                         for (unsigned i = 0; i < size; i++)
                             printf("%02x", ((unsigned char) buffer[i]) & 0xff);
-                        printf("  :%" I64F "u:%012" I64F "x\n", seq, pos);
+                        printf("  :%" I64F "u\n", seq);
                     }
                     else if (helper)
                     {
@@ -285,7 +284,7 @@ int main(int argc, const char **argv)
                             count++;  // Don't count this row as it was postfiltered
                     }
                     else
-                        printf("%.*s  :%" I64F "u:%012" I64F "x\n", size, buffer, seq, pos);
+                        printf("%.*s  :%" I64F "u\n", size, buffer, seq);
                 }
                 if (outRecType)
                     outRecType->doDelete();