HPCC-19563 COMPRESSED(ROW) issues

Loading a blob from a row-compressed index would crash.

The needCopy flag was not properly passed in meaning memory-mapped indexes
would still use the node buffers.

Better to use a derived class for such things as compareRowAt, getValueAt etc
to avoid conditional tests in the normal cases (which need to be fast).

Also contains a fix for HPCC-19564 Partition keys issue.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlckey.cpp

@@ -1429,7 +1429,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedJoinOrDenormalize(BuildCt
         flags.append("|JFdynamicindexfilename");
     if (boundIndexActivity)
         flags.append("|JFindexfromactivity");
-
+    if (options.createValueSets)
+        flags.append("|JFnewfilters");
     if (flags.length())
         doBuildUnsignedFunction(instance->classctx, "getJoinFlags", flags.str()+1);
 

roxie/ccd/ccdfile.cpp

@@ -1822,6 +1822,15 @@ protected:
 
         // NOTE - grouping is not included in the formatCRC, nor is the trailing byte that indicates grouping
         // included in the rtlTypeInfo.
+        const char *kind = props.queryProp("@kind");
+        if (kind)
+        {
+            RoxieFileType thisFileType = streq(kind, "key") ? ROXIE_KEY : ROXIE_FILE;
+            if (subFiles.length()==1)
+                fileType = thisFileType;
+            else
+                assertex(thisFileType==fileType);
+        }
 
         bool isGrouped = props.getPropBool("@grouped", false);
         int formatCrc = props.getPropInt("@formatCrc", 0);
@@ -2364,8 +2373,11 @@ public:
     {
         Owned<IFile> file = createIFile(localFileName);
         assertex(file->exists());
+
         offset_t size = file->size();
         Owned<IFileDescriptor> fdesc = createFileDescriptor();
+        if (isIndexFile(file))
+            fdesc->queryProperties().setProp("@kind", "key");
         Owned<IPropertyTree> pp = createPTree("Part", ipt_lowmem);
         pp->setPropInt64("@size",size);
         pp->setPropBool("@local", true);

system/jhtree/ctfile.cpp

@@ -29,7 +29,7 @@
 #include "ctfile.hpp"
 #include "jstats.h"
 
-inline void SwapBigEndian(KeyHdr &hdr)
+void SwapBigEndian(KeyHdr &hdr)
 {
     _WINREV(hdr.phyrec);
     _WINREV(hdr.delstk);
@@ -120,6 +120,31 @@ extern bool isCompressedIndex(const char *filename)
     return false;
 }
 
+extern jhtree_decl bool isIndexFile(IFile *file)
+{
+    try
+    {
+        offset_t size = file->size();
+        if (size <= sizeof(KeyHdr))
+            return false;
+        Owned<IFileIO> io = file->open(IFOread);
+        KeyHdr hdr;
+        if (io->read(0, sizeof(hdr), &hdr) != sizeof(hdr))
+            return false;
+        SwapBigEndian(hdr);
+        if (!hdr.root || !hdr.nodeSize || !hdr.root || size % hdr.nodeSize || hdr.root % hdr.nodeSize || hdr.root >= size)
+            return false;
+        return true;    // Reasonable heuristic...
+    }
+    catch (IException *E)
+    {
+        E->Release();
+    }
+    return false;
+}
+
+
+
 // CKeyHdr
 CKeyHdr::CKeyHdr()
 {
@@ -266,47 +291,6 @@ bool CWriteNode::add(offset_t pos, const void *indata, size32_t insize, unsigned
         keyPtr += sizeof(rsequence);
         hdr.keyBytes += sizeof(rsequence);
     }
-
-#if 0
-    // This test is no longer valid if we don't treat all fields as keyed
-    if (hdr.numKeys)
-    {
-        if (memcmp(indata, lastKeyValue, keyedSize) < 0)
-        {
-            // dump out the rows in question
-            StringBuffer hex;
-            unsigned i;
-            for (i = 0; i < insize; i++)
-            {
-                hex.appendf("%02x ", ((unsigned char *) indata)[i]);
-            }
-            DBGLOG("this: %s", hex.str());
-            hex.clear();
-            for (i = 0; i < insize; i++)
-            {
-                hex.appendf("%02x ", ((unsigned char *) lastKeyValue)[i]);
-            }
-            DBGLOG("last: %s", hex.str());
-            hex.clear();
-            for (i = 0; i < insize; i++)
-            {
-                unsigned char c = ((unsigned char *) indata)[i];
-                hex.appendf("%c", isprint(c) ? c : '.');
-            }
-            DBGLOG("this: %s", hex.str());
-            hex.clear();
-            for (i = 0; i < insize; i++)
-            {
-                unsigned char c = ((unsigned char *) lastKeyValue)[i];
-                hex.appendf("%c", isprint(c) ? c : '.');
-            }
-            DBGLOG("last: %s", hex.str());
-
-            throw MakeStringException(0, "Data written to key must be in sorted order");
-        }
-    }
-#endif
-
     if (isLeaf() && keyType & HTREE_COMPRESSED_KEY)
     {
         if (0 == hdr.numKeys)
@@ -546,11 +530,12 @@ void *CJHTreeNode::allocMem(size32_t len)
     return ret;
 }
 
-char *CJHTreeNode::expandKeys(void *src,unsigned keylength,size32_t &retsize, bool rowcompression)
+char *CJHTreeNode::expandKeys(void *src,unsigned keylength,size32_t &retsize)
 {
-    Owned<IExpander> exp = rowcompression?createRDiffExpander():createLZWExpander(true);
+    Owned<IExpander> exp = createLZWExpander(true);
     int len=exp->init(src);
-    if (len==0) {
+    if (len==0)
+    {
         retsize = 0;
         return NULL;
     }
@@ -560,18 +545,6 @@ char *CJHTreeNode::expandKeys(void *src,unsigned keylength,size32_t &retsize, bo
     return outkeys;
 }
 
-IRandRowExpander *CJHTreeNode::expandQuickKeys(void *src, bool needCopy)
-{
-    if (IRandRowExpander::isRand(src)) {
-        // we are going to use node
-        IRandRowExpander *rowexp=createRandRDiffExpander();
-        rowexp->init(src, needCopy);
-        return rowexp;
-    }
-    return NULL;
-}
-
-
 void CJHTreeNode::unpack(const void *node, bool needCopy)
 {
     memcpy(&hdr, node, sizeof(hdr));
@@ -617,18 +590,11 @@ void CJHTreeNode::unpack(const void *node, bool needCopy)
         {
             MTIME_SECTION(queryActiveTimer(), "Compressed node expand");
             expandedSize = keyHdr->getNodeSize();
-            bool quick = (keyType&HTREE_QUICK_COMPRESSED_KEY)==HTREE_QUICK_COMPRESSED_KEY;
-#ifndef _OLD_VERSION
+            bool quick = !isBlob() && (keyType&HTREE_QUICK_COMPRESSED_KEY)==HTREE_QUICK_COMPRESSED_KEY;
             keyBuf = NULL;
-            if (quick)
-                rowexp.setown(expandQuickKeys(keys, needCopy));
-            if (!quick||!rowexp.get())
-#endif
-            {
-                keyBuf = expandKeys(keys,keyLen,expandedSize,quick);
-            }
+            if (!quick)
+                keyBuf = expandKeys(keys,keyLen,expandedSize);
         }
-        assertex(keyBuf||rowexp.get());
     }
     else
     {
@@ -762,27 +728,9 @@ offset_t CJHTreeNode::nextNodeFpos() const
     return ll;
 }
 
-void CJHTreeNode::dump()
-{
-    for (unsigned int i=0; i<getNumKeys(); i++)
-    {
-        unsigned char *dst = (unsigned char *) alloca(keyLen+50);
-        getValueAt(i,(char *) dst);
-        offset_t pos = getFPosAt(i);
-
-        StringBuffer nodeval;
-        for (unsigned j = 0; j < keyLen; j++)
-            nodeval.appendf("%02x", dst[j] & 0xff);
-        DBGLOG("keyVal %d [%" I64F "d] = %s", i, pos, nodeval.str());
-    }
-    DBGLOG("==========");
-}
-
 int CJHTreeNode::compareValueAt(const char *src, unsigned int index) const
 {
-    if (rowexp.get()) 
-        return rowexp->cmpRow(src,index,sizeof(__int64),keyCompareLen);
-    return memcmp(src, keyBuf + index*keyRecLen + sizeof(__int64), keyCompareLen);
+    return memcmp(src, keyBuf + index*keyRecLen + (keyHdr->hasSpecialFileposition() ? sizeof(offset_t) : 0), keyCompareLen);
 }
 
 bool CJHTreeNode::getValueAt(unsigned int index, char *dst) const
@@ -792,37 +740,44 @@ bool CJHTreeNode::getValueAt(unsigned int index, char *dst) const
     {
         if (keyHdr->hasSpecialFileposition())
         {
-            //It would make sense to have the fileposition at the start of the row from he perspective of the
+            //It would make sense to have the fileposition at the start of the row from the perspective of the
             //internal representation, but that would complicate everything else which assumes the keyed
             //fields start at the beginning of the row.
-            if (rowexp.get())
-            {
-                rowexp->expandRow(dst,index,sizeof(offset_t),keyLen);
-                rowexp->expandRow(dst+keyLen,index,0,sizeof(offset_t));
-            }
-            else
-            {
-                const char * p = keyBuf + index*keyRecLen;
-                memcpy(dst, p + sizeof(offset_t), keyLen);
-                memcpy(dst+keyLen, p, sizeof(offset_t));
-            }
+            const char * p = keyBuf + index*keyRecLen;
+            memcpy(dst, p + sizeof(offset_t), keyLen);
+            memcpy(dst+keyLen, p, sizeof(offset_t));
         }
         else
         {
-            if (rowexp.get())
-            {
-                rowexp->expandRow(dst,index,0,keyLen);
-            }
-            else
-            {
-                const char * p = keyBuf + index*keyRecLen;
-                memcpy(dst, p, keyLen);
-            }
+            const char * p = keyBuf + index*keyRecLen;
+            memcpy(dst, p, keyLen);
         }
     }
     return true;
 }
 
+const char * CJHTreeNode::queryValueAt(unsigned int index, char *scratchBuffer) const
+{
+    if (index >= hdr.numKeys)
+        return nullptr;
+    else if (keyHdr->hasSpecialFileposition())
+    {
+        getValueAt(index, scratchBuffer);
+        return scratchBuffer;
+    }
+    else
+        return keyBuf + index*keyRecLen;
+}
+
+const char * CJHTreeNode::queryKeyAt(unsigned int index, char *scratchBuffer) const
+{
+    if (index >= hdr.numKeys)
+        return nullptr;
+    else
+        return keyBuf + index*keyRecLen + (keyHdr->hasSpecialFileposition() ? sizeof(offset_t) : 0);
+}
+
+
 size32_t CJHTreeNode::getSizeAt(unsigned int index) const
 {
     if (keyHdr->hasSpecialFileposition())
@@ -836,12 +791,8 @@ offset_t CJHTreeNode::getFPosAt(unsigned int index) const
     if (index >= hdr.numKeys) return 0;
 
     offset_t pos;
-    if (rowexp.get())
-        rowexp->expandRow(&pos,index,0,sizeof(pos));
-    else {
-        const char * p = keyBuf + index*keyRecLen;
-        memcpy( &pos, p, sizeof(__int64));
-    }
+    const char * p = keyBuf + index*keyRecLen;
+    memcpy( &pos, p, sizeof(__int64));
     _WINREV(pos);
     return pos;
 }
@@ -953,28 +904,9 @@ CJHVarTreeNode::~CJHVarTreeNode()
     delete [] recArray;
 }
 
-void CJHVarTreeNode::dump()
-{
-    for (unsigned int i=0; i<getNumKeys(); i++)
-    {
-        const void * p = recArray[i];
-        unsigned reclen = ((KEYRECSIZE_T *) p)[-1];
-        _WINREV(reclen);
-        unsigned char *dst = (unsigned char *) alloca(reclen);
-        getValueAt(i,(char *) dst);
-        offset_t pos = getFPosAt(i);
-
-        StringBuffer nodeval;
-        for (unsigned j = 0; j < reclen; j++)
-            nodeval.appendf("%02x", dst[j] & 0xff);
-        DBGLOG("keyVal %d [%" I64F "d] = %s", i, pos, nodeval.str());
-    }
-    DBGLOG("==========");
-}
-
 int CJHVarTreeNode::compareValueAt(const char *src, unsigned int index) const
 {
-    return memcmp(src, recArray[index] + sizeof(offset_t), keyCompareLen);
+    return memcmp(src, recArray[index] + (keyHdr->hasSpecialFileposition() ? sizeof(offset_t) : 0), keyCompareLen);
 }
 
 bool CJHVarTreeNode::getValueAt(unsigned int num, char *dst) const
@@ -997,6 +929,26 @@ bool CJHVarTreeNode::getValueAt(unsigned int num, char *dst) const
     return true;
 }
 
+const char *CJHVarTreeNode::queryValueAt(unsigned int index, char *scratchBuffer) const
+{
+    if (index >= hdr.numKeys)
+        return nullptr;
+    else if (keyHdr->hasSpecialFileposition())
+    {
+        getValueAt(index, scratchBuffer);
+        return scratchBuffer;
+    }
+    else
+        return recArray[index];
+}
+
+const char *CJHVarTreeNode::queryKeyAt(unsigned int index, char *scratchBuffer) const
+{
+    if (index >= hdr.numKeys)
+        return nullptr;
+    return recArray[index] + (keyHdr->hasSpecialFileposition() ? sizeof(offset_t) : 0);
+}
+
 size32_t CJHVarTreeNode::getSizeAt(unsigned int num) const
 {
     const char * p = recArray[num];
@@ -1019,6 +971,74 @@ offset_t CJHVarTreeNode::getFPosAt(unsigned int num) const
     return pos;
 }
 
+//=========================================================================================================
+
+void CJHRowCompressedNode::load(CKeyHdr *_keyHdr, const void *rawData, offset_t _fpos, bool needCopy)
+{
+    CJHTreeNode::load(_keyHdr, rawData, _fpos, needCopy);
+    assertex(hdr.leafFlag==1);
+    char *keys = ((char *) rawData) + sizeof(hdr)+sizeof(firstSequence);
+    assertex(IRandRowExpander::isRand(keys));
+    rowexp.setown(createRandRDiffExpander());
+    rowexp->init(keys, needCopy);
+}
+
+int CJHRowCompressedNode::compareValueAt(const char *src, unsigned int index) const
+{
+    return rowexp->cmpRow(src,index,keyHdr->hasSpecialFileposition() ? sizeof(offset_t) : 0,keyCompareLen);
+}
+
+bool CJHRowCompressedNode::getValueAt(unsigned int num, char *dst) const
+{
+    if (num >= hdr.numKeys) return false;
+    if (dst)
+    {
+        if (keyHdr->hasSpecialFileposition())
+        {
+            rowexp->expandRow(dst,num,sizeof(offset_t),keyLen);
+            rowexp->expandRow(dst+keyLen,num,0,sizeof(offset_t));
+        }
+        else
+            rowexp->expandRow(dst,num,0,keyLen);
+    }
+    return true;
+}
+
+const char *CJHRowCompressedNode::queryValueAt(unsigned int num, char *scratchBuffer) const
+{
+    if (num >= hdr.numKeys)
+        return nullptr;
+    else
+    {
+        getValueAt(num, scratchBuffer);
+        return scratchBuffer;
+    }
+}
+
+const char *CJHRowCompressedNode::queryKeyAt(unsigned int num, char *scratchBuffer) const
+{
+    if (num >= hdr.numKeys)
+        return nullptr;
+    unsigned keyedSize = keyHdr->getNodeKeyLength();
+    if (keyHdr->hasSpecialFileposition())
+        rowexp->expandRow(scratchBuffer,num,sizeof(offset_t),keyedSize);
+    else
+        rowexp->expandRow(scratchBuffer,num,0,keyedSize);
+    return scratchBuffer;
+}
+
+offset_t CJHRowCompressedNode::getFPosAt(unsigned int num) const
+{
+    if (num >= hdr.numKeys)
+        return 0;
+    else
+    {
+        offset_t pos;
+        rowexp->expandRow(&pos,num,0,sizeof(pos));
+        _WINREV(pos);
+        return pos;
+    }
+}
 
 //=========================================================================================================
 

system/jhtree/ctfile.hpp

@@ -153,6 +153,7 @@ public:
     inline static size32_t getSize() { return sizeof(KeyHdr); }
     inline unsigned getNodeSize() { return hdr.nodeSize; }
     inline bool hasSpecialFileposition() const { return true; }
+    inline bool isRowCompressed() const { return (hdr.ktype & HTREE_QUICK_COMPRESSED_KEY) == HTREE_QUICK_COMPRESSED_KEY; }
     __uint64 getPartitionFieldMask()
     {
         if (hdr.partitionFieldMask == (__uint64) -1)
@@ -205,11 +206,8 @@ protected:
     void unpack(const void *node, bool needCopy);
     unsigned __int64 firstSequence;
     size32_t expandedSize;
-    Owned<IRandRowExpander> rowexp;  // expander for rand rowdiff   
-
-    static char *expandKeys(void *src,unsigned keylength,size32_t &retsize, bool rowcompression);
-    static IRandRowExpander *expandQuickKeys(void *src, bool needCopy);
 
+    static char *expandKeys(void *src,unsigned keylength,size32_t &retsize);
     static void releaseMem(void *togo, size32_t size);
     static void *allocMem(size32_t size);
 
@@ -223,6 +221,8 @@ public:
     offset_t prevNodeFpos() const;
     offset_t nextNodeFpos() const ;
     virtual bool getValueAt(unsigned int num, char *key) const;
+    virtual const char *queryValueAt(unsigned int index, char *scratchBuffer) const;
+    virtual const char *queryKeyAt(unsigned int index, char *scratchBuffer) const;
     virtual size32_t getSizeAt(unsigned int num) const;
     virtual offset_t getFPosAt(unsigned int num) const;
     virtual int compareValueAt(const char *src, unsigned int index) const;
@@ -230,8 +230,6 @@ public:
     inline offset_t getRightSib() const { return hdr.rightSib; }
     inline offset_t getLeftSib() const { return hdr.leftSib; }
     unsigned __int64 getSequence(unsigned int num) const;
-
-    virtual void dump();
 };
 
 class CJHVarTreeNode : public CJHTreeNode 
@@ -243,10 +241,24 @@ public:
     ~CJHVarTreeNode();
     virtual void load(CKeyHdr *keyHdr, const void *rawData, offset_t pos, bool needCopy);
     virtual bool getValueAt(unsigned int num, char *key) const;
+    virtual const char *queryValueAt(unsigned int index, char *scratchBuffer) const;
+    virtual const char *queryKeyAt(unsigned int index, char *scratchBuffer) const;
     virtual size32_t getSizeAt(unsigned int num) const;
     virtual offset_t getFPosAt(unsigned int num) const;
     virtual int compareValueAt(const char *src, unsigned int index) const;
-    virtual void dump();
+};
+
+class CJHRowCompressedNode : public CJHTreeNode
+{
+    Owned<IRandRowExpander> rowexp;  // expander for rand rowdiff
+    static IRandRowExpander *expandQuickKeys(void *src, bool needCopy);
+public:
+    virtual void load(CKeyHdr *keyHdr, const void *rawData, offset_t pos, bool needCopy);
+    virtual bool getValueAt(unsigned int num, char *key) const;
+    virtual const char *queryValueAt(unsigned int index, char *scratchBuffer) const;
+    virtual const char *queryKeyAt(unsigned int index, char *scratchBuffer) const;
+    virtual offset_t getFPosAt(unsigned int num) const;
+    virtual int compareValueAt(const char *src, unsigned int index) const;
 };
 
 class CJHTreeBlobNode : public CJHTreeNode
@@ -255,6 +267,8 @@ public:
     CJHTreeBlobNode ();
     ~CJHTreeBlobNode ();
     virtual bool getValueAt(unsigned int num, char *key) const {throwUnexpected();}
+    virtual const char *queryValueAt(unsigned int index, char *scratchBuffer) const {throwUnexpected();}
+    virtual const char *queryKeyAt(unsigned int index, char *scratchBuffer) const {throwUnexpected();}
     virtual offset_t getFPosAt(unsigned int num) const {throwUnexpected();}
     virtual size32_t getSizeAt(unsigned int num) const {throwUnexpected();}
     virtual int compareValueAt(const char *src, unsigned int index) const {throwUnexpected();}
@@ -268,6 +282,8 @@ class CJHTreeMetadataNode : public CJHTreeNode
 {
 public:
     virtual bool getValueAt(unsigned int num, char *key) const {throwUnexpected();}
+    virtual const char *queryValueAt(unsigned int index, char *scratchBuffer) const {throwUnexpected();}
+    virtual const char *queryKeyAt(unsigned int index, char *scratchBuffer) const {throwUnexpected();}
     virtual offset_t getFPosAt(unsigned int num) const {throwUnexpected();}
     virtual size32_t getSizeAt(unsigned int num) const {throwUnexpected();}
     virtual int compareValueAt(const char *src, unsigned int index) const {throwUnexpected();}
@@ -279,6 +295,8 @@ class CJHTreeBloomTableNode : public CJHTreeNode
 {
 public:
     virtual bool getValueAt(unsigned int num, char *key) const {throwUnexpected();}
+    virtual const char *queryValueAt(unsigned int index, char *scratchBuffer) const {throwUnexpected();}
+    virtual const char *queryKeyAt(unsigned int index, char *scratchBuffer) const {throwUnexpected();}
     virtual offset_t getFPosAt(unsigned int num) const {throwUnexpected();}
     virtual size32_t getSizeAt(unsigned int num) const {throwUnexpected();}
     virtual int compareValueAt(const char *src, unsigned int index) const {throwUnexpected();}

system/jhtree/jhtree.cpp

@@ -224,7 +224,7 @@ unsigned SegMonitorList::_lastRealSeg() const
         if (!seg)
             return 0;
         seg--;
-        if (!segMonitors.item(seg).isWild())
+        if (!segMonitors.item(seg).isWild()) // MORE - why not just remove them? Stepping/overrides?
             return seg;
     }
 }
@@ -418,7 +418,7 @@ public:
         {
             hash64_t hash = HASH64_INIT;
             if (getBloomHash(partitionFieldMask, segs, hash))
-                return (hash % indexParts) + 1;
+                return (((unsigned) hash) % indexParts) + 1;  // NOTE - the Hash distribute function that distributes the index when building will truncate to 32-bits before taking modulus - so we must too!
         }
         return 0;
     }
@@ -1099,10 +1099,12 @@ CJHTreeNode *CKeyIndex::loadNode(char *nodeData, offset_t pos, bool needsCopy)
             ret.setown(new CJHTreeNode());
             break;
         case 1:
-        	if (keyHdr->isVariable())
-        		ret.setown(new CJHVarTreeNode());
-        	else
-        		ret.setown(new CJHTreeNode());
+            if (keyHdr->isVariable())
+                ret.setown(new CJHVarTreeNode());
+            else if (keyHdr->isRowCompressed())
+                ret.setown(new CJHRowCompressedNode());
+            else
+                ret.setown(new CJHTreeNode());
             break;
         case 2:
             ret.setown(new CJHTreeBlobNode());
@@ -1118,7 +1120,7 @@ CJHTreeNode *CKeyIndex::loadNode(char *nodeData, offset_t pos, bool needsCopy)
         }
         {
             MTIME_SECTION(queryActiveTimer(), "JHTREE load node");
-            ret->load(keyHdr, nodeData, pos, true);
+            ret->load(keyHdr, nodeData, pos, needsCopy);
         }
         return ret.getClear();
     }
@@ -1204,6 +1206,11 @@ bool CKeyIndex::hasSpecialFileposition() const
     return keyHdr->hasSpecialFileposition();
 }
 
+bool CKeyIndex::needsRowBuffer() const
+{
+    return keyHdr->hasSpecialFileposition() || keyHdr->isRowCompressed();
+}
+
 size32_t CKeyIndex::keySize()
 {
     size32_t fileposSize = keyHdr->hasSpecialFileposition() ? sizeof(offset_t) : 0;
@@ -1334,6 +1341,40 @@ IPropertyTree * CKeyIndex::getMetadata()
     return ret;
 }
 
+CJHTreeNode *CKeyIndex::locateFirstNode(KeyStatsCollector &stats)
+{
+    keySeeks++;
+    stats.seeks++;
+    CJHTreeNode * n = 0;
+    CJHTreeNode * p = LINK(rootNode);
+    while (p != 0)
+    {
+        n = p;
+        p = getNode(n->prevNodeFpos(), stats.ctx);
+        if (p != 0)
+            n->Release();
+    }
+    return n;
+}
+
+CJHTreeNode *CKeyIndex::locateLastNode(KeyStatsCollector &stats)
+{
+    keySeeks++;
+    stats.seeks++;
+    CJHTreeNode * n = 0;
+    CJHTreeNode * p = LINK(rootNode);
+
+    while (p != 0)
+    {
+        n = p;
+        p = getNode(n->nextNodeFpos(), stats.ctx);
+        if (p != 0)
+            n->Release();
+    }
+    return n;
+}
+
+
 void KeyStatsCollector::noteSeeks(unsigned lseeks, unsigned lscans, unsigned lwildseeks)
 {
     seeks += lseeks;
@@ -1392,49 +1433,23 @@ CKeyCursor::~CKeyCursor()
     free(keyBuffer);
 }
 
-void CKeyCursor::reset(unsigned sortFromSeg)
+void CKeyCursor::reset()
 {
     node.clear();
     matched = false;
     eof = key.bloomFilterReject(*segs);
     if (!eof)
-        setLow(sortFromSeg);
-}
-
-CJHTreeNode *CKeyCursor::locateFirstNode(KeyStatsCollector &stats)
-{
-    CJHTreeNode * n = 0;
-    CJHTreeNode * p = LINK(key.rootNode);
-    while (p != 0)
-    {
-        n = p;
-        p = key.getNode(n->prevNodeFpos(), stats.ctx);
-        if (p != 0)
-            n->Release();
-    }
-    return n;
-}
-
-CJHTreeNode *CKeyCursor::locateLastNode(KeyStatsCollector &stats)
-{
-    CJHTreeNode * n = 0;
-    CJHTreeNode * p = LINK(key.rootNode);
-
-    while (p != 0)
-    {
-        n = p;
-        p = key.getNode(n->nextNodeFpos(), stats.ctx);
-        if (p != 0)
-            n->Release();
-    }
-
-    return n;
+        setLow(0);
 }
 
 bool CKeyCursor::next(char *dst, KeyStatsCollector &stats)
 {
     if (!node)
-        return first(dst, stats);
+    {
+        node.setown(key.locateFirstNode(stats));
+        nodeKey = 0;
+        return node && node->getValueAt(nodeKey, dst);
+    }
     else
     {
         key.keyScans++;
@@ -1492,18 +1507,9 @@ unsigned __int64 CKeyCursor::getSequence()
     return node->getSequence(nodeKey);
 }
 
-bool CKeyCursor::first(char *dst, KeyStatsCollector &stats)
-{
-    key.keySeeks++;
-    node.setown(locateFirstNode(stats));
-    nodeKey = 0;
-    return node->getValueAt(nodeKey, dst);
-}
-
 bool CKeyCursor::last(char *dst, KeyStatsCollector &stats)
 {
-    key.keySeeks++;
-    node.setown(locateLastNode(stats));
+    node.setown(key.locateLastNode(stats));
     nodeKey = node->getNumKeys()-1;
     return node->getValueAt( nodeKey, dst );
 }
@@ -1821,7 +1827,6 @@ unsigned __int64 CKeyCursor::getCount(KeyStatsCollector &stats)
 
 unsigned __int64 CKeyCursor::checkCount(unsigned __int64 max, KeyStatsCollector &stats)
 {
-    setLow(0);
     reset();
     unsigned __int64 result = 0;
     unsigned lseeks = 0;
@@ -1962,13 +1967,13 @@ class CLazyKeyIndex : implements IKeyIndex, public CInterface
     StringAttr keyfile;
     unsigned crc; 
     Linked<IDelayedFile> delayedFile;
-    Owned<IFileIO> iFileIO;
-    Owned<IKeyIndex> realKey;
-    CriticalSection c;
+    mutable Owned<IFileIO> iFileIO;
+    mutable Owned<IKeyIndex> realKey;
+    mutable CriticalSection c;
     bool isTLK;
     bool preloadAllowed;
 
-    inline IKeyIndex &checkOpen()
+    inline IKeyIndex &checkOpen() const
     {
         CriticalBlock b(c);
         if (!realKey)
@@ -2020,7 +2025,8 @@ public:
     virtual IPropertyTree * getMetadata() { return checkOpen().getMetadata(); }
     virtual unsigned getNodeSize() { return checkOpen().getNodeSize(); }
     virtual const IFileIO *queryFileIO() const override { return iFileIO; } // NB: if not yet opened, will be null
-    virtual bool hasSpecialFileposition() const { return realKey ? realKey->hasSpecialFileposition() : false; }
+    virtual bool hasSpecialFileposition() const { return checkOpen().hasSpecialFileposition(); }
+    virtual bool needsRowBuffer() const { return checkOpen().needsRowBuffer(); }
 };
 
 extern jhtree_decl IKeyIndex *createKeyIndex(const char *keyfile, unsigned crc, IFileIO &iFileIO, bool isTLK, bool preloadAllowed)
@@ -2326,13 +2332,6 @@ class CKeyMerger : public CKeyLevelManager
 
     Linked<IKeyIndexBase> keyset;
 
-    void resetKey(unsigned i)
-    {
-        IKeyCursor *cursor = cursors[i];
-        if (cursor)
-            cursor->reset(sortFromSeg);
-    }
-
     void calculateSortSeg()
     {
         // Make sure that sortFromSeg is properly set

system/jhtree/jhtree.hpp

@@ -57,11 +57,7 @@ public:
 
 interface jhtree_decl IKeyCursor : public IInterface
 {
-    virtual bool next(char *dst, KeyStatsCollector &stats) = 0;
-    virtual bool first(char *dst, KeyStatsCollector &stats) = 0;
-    virtual bool last(char *dst, KeyStatsCollector &stats) = 0;
-    virtual bool gtEqual(const char *src, char *dst, KeyStatsCollector &stats) = 0; // returns first record >= src
-    virtual bool ltEqual(const char *src, KeyStatsCollector &stats) = 0; // returns last record <= src
+    virtual bool next(char *dst, KeyStatsCollector &stats) = 0; // MORE - remove
     virtual const char *queryName() const = 0;
     virtual size32_t getSize() = 0;  // Size of current row
     virtual size32_t getKeyedSize() const = 0;  // Size of keyed fields
@@ -69,7 +65,7 @@ interface jhtree_decl IKeyCursor : public IInterface
     virtual void deserializeCursorPos(MemoryBuffer &mb, KeyStatsCollector &stats) = 0;
     virtual unsigned __int64 getSequence() = 0;
     virtual const byte *loadBlob(unsigned __int64 blobid, size32_t &blobsize) = 0;
-    virtual void reset(unsigned sortFromSeg = 0) = 0;
+    virtual void reset() = 0;
     virtual bool lookup(bool exact, KeyStatsCollector &stats) = 0;
 
     virtual bool lookupSkip(const void *seek, size32_t seekOffset, size32_t seeklen, KeyStatsCollector &stats) = 0;
@@ -116,6 +112,7 @@ interface jhtree_decl IKeyIndex : public IKeyIndexBase
     virtual unsigned getNodeSize() = 0;
     virtual const IFileIO *queryFileIO() const = 0;
     virtual bool hasSpecialFileposition() const = 0;
+    virtual bool needsRowBuffer() const = 0;
 };
 
 interface IKeyArray : extends IInterface
@@ -293,6 +290,7 @@ public:
 };
 
 extern jhtree_decl bool isCompressedIndex(const char *filename);
+extern jhtree_decl bool isIndexFile(IFile *filename);
 
 #define JHTREE_KEY_NOT_SORTED JHTREE_ERROR_START
 

system/jhtree/jhtree.ipp

@@ -63,6 +63,8 @@ enum request { LTE, GTE };
 interface INodeLoader
 {
     virtual CJHTreeNode *loadNode(offset_t offset) = 0;
+    virtual CJHTreeNode *locateFirstNode(KeyStatsCollector &stats) = 0;
+    virtual CJHTreeNode *locateLastNode(KeyStatsCollector &stats) = 0;
 };
 
 class jhtree_decl CKeyIndex : implements IKeyIndex, implements INodeLoader, public CInterface
@@ -132,9 +134,12 @@ public:
 
     virtual unsigned getNodeSize() { return keyHdr->getNodeSize(); }
     virtual bool hasSpecialFileposition() const;
+    virtual bool needsRowBuffer() const;
  
  // INodeLoader impl.
     virtual CJHTreeNode *loadNode(offset_t offset) = 0;
+    CJHTreeNode *locateFirstNode(KeyStatsCollector &stats);
+    CJHTreeNode *locateLastNode(KeyStatsCollector &stats);
 };
 
 class jhtree_decl CMemKeyIndex : public CKeyIndex
@@ -183,10 +188,6 @@ public:
     ~CKeyCursor();
 
     virtual bool next(char *dst, KeyStatsCollector &stats) override;
-    virtual bool first(char *dst, KeyStatsCollector &stats) override;
-    virtual bool last(char *dst, KeyStatsCollector &stats) override;
-    virtual bool gtEqual(const char *src, char *dst, KeyStatsCollector &stats) override;
-    virtual bool ltEqual(const char *src, KeyStatsCollector &stats) override;
     virtual const char *queryName() const override;
     virtual size32_t getSize();
     virtual size32_t getKeyedSize() const;
@@ -195,7 +196,7 @@ public:
     virtual void deserializeCursorPos(MemoryBuffer &mb, KeyStatsCollector &stats);
     virtual unsigned __int64 getSequence(); 
     virtual const byte *loadBlob(unsigned __int64 blobid, size32_t &blobsize);
-    virtual void reset(unsigned sortFromSeg = 0);
+    virtual void reset();
     virtual bool lookup(bool exact, KeyStatsCollector &stats) override;
     virtual bool lookupSkip(const void *seek, size32_t seekOffset, size32_t seeklen, KeyStatsCollector &stats) override;
     virtual bool skipTo(const void *_seek, size32_t seekOffset, size32_t seeklen) override;
@@ -209,11 +210,11 @@ public:
 protected:
     CKeyCursor(const CKeyCursor &from);
 
+    bool last(char *dst, KeyStatsCollector &stats);
+    bool gtEqual(const char *src, char *dst, KeyStatsCollector &stats);
+    bool ltEqual(const char *src, KeyStatsCollector &stats);
     bool _lookup(bool exact, unsigned lastSeg, KeyStatsCollector &stats);
     void reportExcessiveSeeks(unsigned numSeeks, unsigned lastSeg, KeyStatsCollector &stats);
-    CJHTreeNode *locateFirstNode(KeyStatsCollector &stats);
-    CJHTreeNode *locateLastNode(KeyStatsCollector &stats);
-
 
     inline void setLow(unsigned segNo)
     {

testing/regress/ecl/setup/files.ecl

@@ -164,7 +164,7 @@ END;
 
 EXPORT DG_VarFile   := DATASET(DG_FileOut+'VAR',DG_VarOutRecPlus,FLAT);
 
-EXPORT DG_NormalVarIndex  := INDEX(DG_VarFile, { DG_firstname; DG_lastname; __filepos } ,DG_IndexOut+'VARINDEX');
+EXPORT DG_NormalVarIndex  := INDEX(DG_VarFile, { DG_firstname; DG_lastname; __filepos } ,DG_IndexOut+'VARINDEX'); // THIS IS NOT VARIABLE - stupid test!
 EXPORT DG_TransVarIndex  := INDEX(DG_VarFile, { DG_firstname; DG_lastname; }, { DGextra := DG_lastname; __filepos } ,DG_IndexOut+'TRANS_VARINDEX');
 
 indexName := IF(useTranslation, __nameof__(DG_TransVarIndex), __nameof__(DG_NormalVarIndex));