HPCC-23428 Avoid the need to backpatch indexes

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hthor/hthor.cpp

@@ -1129,7 +1129,18 @@ void CHThorIndexWriteActivity::execute()
         Owned<IPropertyTree> metadata;
         buildUserMetadata(metadata);
         buildLayoutMetadata(metadata);
-        unsigned nodeSize = metadata ? metadata->getPropInt("_nodeSize", NODESIZE) : NODESIZE;
+        unsigned nodeSize = NODESIZE;
+        if (metadata)
+        {
+            nodeSize = metadata->getPropInt("_nodeSize", NODESIZE);
+            if (metadata->getPropBool("_noSeek", false))
+                flags |= TRAILING_HEADER_ONLY;
+            if (metadata->getPropBool("_useTrailingHeader", true))
+                flags |= USE_TRAILING_HEADER;
+        }
+        else
+            flags |= USE_TRAILING_HEADER;
+
         size32_t keyMaxSize = helper.queryDiskRecordSize()->getRecordSize(NULL);
         if (hasTrailingFileposition(helper.queryDiskRecordSize()->queryTypeInfo()))
             keyMaxSize -= sizeof(offset_t);

roxie/ccd/ccdserver.cpp

@@ -12331,7 +12331,17 @@ public:
             Owned<IPropertyTree> metadata;
             buildUserMetadata(metadata);
             buildLayoutMetadata(metadata);
-            unsigned nodeSize = metadata ? metadata->getPropInt("_nodeSize", NODESIZE) : NODESIZE;
+            unsigned nodeSize = NODESIZE;
+            if (metadata)
+            {
+                nodeSize = metadata->getPropInt("_nodeSize", NODESIZE);
+                if (metadata->getPropBool("_noSeek", false))
+                    flags |= TRAILING_HEADER_ONLY;
+                if (metadata->getPropBool("_useTrailingHeader", true))
+                    flags |= USE_TRAILING_HEADER;
+            }
+            else
+                flags |= USE_TRAILING_HEADER;
             Owned<IKeyBuilder> builder = createKeyBuilder(out, flags, maxDiskRecordSize, nodeSize, helper.getKeyedSize(), 0, &helper, true, false);
             class BcWrapper : implements IBlobCreator
             {

system/jhtree/ctfile.cpp

@@ -169,20 +169,6 @@ void CKeyHdr::load(KeyHdr &_hdr)
         throw MakeKeyException(KeyExcpt_IncompatVersion, "This build is compatible with key versions <= %u. Key is version %u", KEYBUILD_VERSION, (unsigned) hdr.version);
 }
 
-void CKeyHdr::write(IWriteSeq *out, CRC32 *crc)
-{
-    unsigned nodeSize = hdr.nodeSize;
-    assertex(out->getRecordSize()==nodeSize);
-    MemoryAttr ma;
-    byte *buf = (byte *) ma.allocate(nodeSize); 
-    memcpy(buf, &hdr, sizeof(hdr));
-    memset(buf+sizeof(hdr), 0xff, nodeSize-sizeof(hdr));
-    SwapBigEndian(*(KeyHdr*) buf);
-    out->put(buf);
-    if (crc)
-        crc->tally(nodeSize, buf);
-}
-
 void CKeyHdr::write(IFileIOStream *out, CRC32 *crc)
 {
     unsigned nodeSize = hdr.nodeSize;
@@ -837,6 +823,12 @@ extern jhtree_decl void validateKeyFile(const char *filename, offset_t nodePos)
         throw MakeStringException(4, "Invalid key %s: failed to read key header", filename);
     CKeyHdr keyHdr;
     keyHdr.load(hdr);
+    if (keyHdr.getKeyType() & USE_TRAILING_HEADER)
+    {
+        if (io->read(size - keyHdr.getNodeSize(), sizeof(hdr), &hdr) != sizeof(hdr))
+            throw MakeStringException(4, "Invalid key %s: failed to read trailing key header", filename);
+        keyHdr.load(hdr);
+    }
 
     _WINREV(hdr.phyrec);
     _WINREV(hdr.root);

system/jhtree/ctfile.hpp

@@ -31,10 +31,10 @@
 #define HTREE_FPOS_OFFSET   0x01 // Obsolete, not supported
 #define HTREE_TOPLEVEL_KEY  0x02
 #define COL_PREFIX          0x04
-#define COL_SUFFIX          0x08 // Obsolete, not supported
+#define TRAILING_HEADER_ONLY  0x08 // Leading header not updated - use trailing one
 #define HTREE_VARSIZE       0x10
 #define HTREE_FULLSORT_KEY  0x20
-#define INDAR_TRAILING_SEG  0x80 // Obsolete, not supported
+#define USE_TRAILING_HEADER  0x80 // Real index header node located at end of file
 #define HTREE_COMPRESSED_KEY 0x40
 #define HTREE_QUICK_COMPRESSED_KEY 0x48
 #define KEYBUILD_VERSION 1 // unsigned short. NB: This should upped if a change would make existing keys incompatible with current build.
@@ -132,7 +132,6 @@ public:
     CKeyHdr();
 
     void load(KeyHdr &_hdr);
-    void write(IWriteSeq *, CRC32 *crc = NULL);
     void write(IFileIOStream *, CRC32 *crc = NULL);
 
     unsigned int getMaxKeyLength();

system/jhtree/jhtree.cpp

@@ -1016,6 +1016,11 @@ CMemKeyIndex::CMemKeyIndex(int _iD, IMemoryMappedFile *_io, const char *_name, b
     if (io->length() < sizeof(hdr))
         throw MakeStringException(0, "Failed to read key header: file too small, could not read %u bytes", (unsigned) sizeof(hdr));
     memcpy(&hdr, io->base(), sizeof(hdr));
+    if (hdr.ktype & USE_TRAILING_HEADER)
+    {
+        _WINREV(hdr.nodeSize);
+        memcpy(&hdr, (io->base()+io->length()) - hdr.nodeSize, sizeof(hdr));
+    }
     init(hdr, isTLK, false);
 }
 
@@ -1042,6 +1047,12 @@ CDiskKeyIndex::CDiskKeyIndex(int _iD, IFileIO *_io, const char *_name, bool isTL
     KeyHdr hdr;
     if (io->read(0, sizeof(hdr), &hdr) != sizeof(hdr))
         throw MakeStringException(0, "Failed to read key header: file too small, could not read %u bytes", (unsigned) sizeof(hdr));
+    if (hdr.ktype & USE_TRAILING_HEADER)
+    {
+        _WINREV(hdr.nodeSize);
+        if (!io->read(io->size() - hdr.nodeSize, sizeof(hdr), &hdr))
+            throw MakeStringException(4, "Invalid key %s: failed to read trailing key header", _name);
+    }
     init(hdr, isTLK, allowPreload);
 }
 
@@ -3030,7 +3041,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
 
     void testStepping()
     {
-        buildTestKeys(false);
+        buildTestKeys(false, true, false);
         {
             // We are going to treat as a 7-byte field then a 3-byte field, and request the datasorted by the 3-byte...
             Owned <IKeyIndex> index1 = createKeyIndex("keyfile1.$$$", 0, false, false);
@@ -3138,20 +3149,24 @@ class IKeyManagerTest : public CppUnit::TestFixture
         removeTestKeys();
     }
 
-    void buildTestKeys(bool variable)
+    void buildTestKeys(bool variable, bool useTrailingHeader, bool noSeek)
     {
-        buildTestKey("keyfile1.$$$", false, variable);
-        buildTestKey("keyfile2.$$$", true, variable);
+        buildTestKey("keyfile1.$$$", false, variable, useTrailingHeader, noSeek);
+        buildTestKey("keyfile2.$$$", true, variable, useTrailingHeader, noSeek);
     }
 
-    void buildTestKey(const char *filename, bool skip, bool variable)
+    void buildTestKey(const char *filename, bool skip, bool variable, bool useTrailingHeader, bool noSeek)
     {
         OwnedIFile file = createIFile(filename);
         OwnedIFileIO io = file->openShared(IFOcreate, IFSHfull);
         Owned<IFileIOStream> out = createIOStream(io);
         unsigned maxRecSize = variable ? 18 : 10;
         unsigned keyedSize = 10;
-        Owned<IKeyBuilder> builder = createKeyBuilder(out, COL_PREFIX | HTREE_FULLSORT_KEY | HTREE_COMPRESSED_KEY |  (variable ? HTREE_VARSIZE : 0), maxRecSize, NODESIZE, keyedSize, 0, nullptr, true, false);
+        Owned<IKeyBuilder> builder = createKeyBuilder(out, COL_PREFIX | HTREE_FULLSORT_KEY | HTREE_COMPRESSED_KEY |
+                (variable ? HTREE_VARSIZE : 0) |
+                (useTrailingHeader ? USE_TRAILING_HEADER : 0) |
+                (noSeek ? TRAILING_HEADER_ONLY : 0),
+                maxRecSize, NODESIZE, keyedSize, 0, nullptr, true, false);
 
         char keybuf[18];
         memset(keybuf, '0', 18);
@@ -3219,7 +3234,7 @@ class IKeyManagerTest : public CppUnit::TestFixture
         key->releaseBlobs();
     }
 protected:
-    void testKeys(bool variable)
+    void testKeys(bool variable, bool useTrailingHeader, bool noSeek)
     {
         const char *json = variable ?
                 "{ \"ty1\": { \"fieldType\": 4, \"length\": 10 }, "
@@ -3239,7 +3254,7 @@ protected:
                 "}";
         Owned<IOutputMetaData> meta = createTypeInfoOutputMetaData(json, false);
         const RtlRecord &recInfo = meta->queryRecordAccessor(true);
-        buildTestKeys(variable);
+        buildTestKeys(variable, useTrailingHeader, noSeek);
         {
             Owned <IKeyIndex> index1 = createKeyIndex("keyfile1.$$$", 0, false, false);
             Owned <IKeyManager> tlk1 = createLocalKeyManager(recInfo, index1, NULL, false, false);
@@ -3405,8 +3420,10 @@ protected:
     void testKeys()
     {
         ASSERT(sizeof(CKeyIdAndPos) == sizeof(unsigned __int64) + sizeof(offset_t));
-        testKeys(false);
-        testKeys(true);
+        for (bool var : { false, true })
+            for (bool trail : { false, true })
+                for (bool noseek : { false, true })
+                    testKeys(var, trail, noseek);
     }
 };
 

system/jhtree/keybuild.cpp

@@ -105,12 +105,14 @@ public:
         prevLeafNode = NULL;
 
         assertex(nodeSize >= CKeyHdr::getSize());
-        assertex(nodeSize <= 0xffff); // stored in a short in the header - we should fix that if/when we restructure header 
+        assertex(nodeSize <= 0xffff); // stored in a short in the header - we should fix that if/when we restructure header
+        if (flags & TRAILING_HEADER_ONLY)
+            flags |= USE_TRAILING_HEADER;
         KeyHdr *hdr = keyHdr->getHdrStruct();
         hdr->nodeSize = nodeSize;
         hdr->extsiz = 4096;
         hdr->length = keyValueSize; 
-        hdr->ktype = flags; 
+        hdr->ktype = flags;
         hdr->timeid = 0;
         hdr->clstyp = 1;  // IDX_CLOSE
         hdr->maxkbn = nodeSize-sizeof(NodeHdr);
@@ -136,6 +138,7 @@ public:
 
     CKeyBuilderBase(CKeyHdr * chdr)
     {
+        sequence = 0;
         levels = 0;
         records = 0;
         prevLeafNode = NULL;
@@ -201,8 +204,13 @@ protected:
         if (out)
         {
             out->flush();
-            out->seek(0, IFSbegin);
-            keyHdr->write(out, crc);
+            if (keyHdr->getKeyType() & USE_TRAILING_HEADER)
+                keyHdr->write(out, crc);  // write a copy at end too, for use on systems that can't seek
+            if (!(keyHdr->getKeyType() & TRAILING_HEADER_ONLY))
+            {
+                out->seek(0, IFSbegin);
+                keyHdr->write(out, crc);
+            }
         }
     }
 
@@ -641,6 +649,12 @@ extern jhtree_decl IKeyDesprayer * createKeyDesprayer(IFile * in, IFileIOStream
 
     Owned<CKeyHdr> hdr = new CKeyHdr;
     hdr->load(*(KeyHdr *)buffer.get());
+    if (hdr->getKeyType() & USE_TRAILING_HEADER)
+    {
+        if (io->read(in->size() - hdr->getNodeSize(), sizeof(KeyHdr), (void *)buffer.get()) != sizeof(KeyHdr))
+            throw MakeStringException(4, "Invalid key %s: failed to read trailing key header", in->queryFilename());
+        hdr->load(*(KeyHdr*)buffer.get());
+    }
     hdr->getHdrStruct()->nument = 0;
     return new CKeyDesprayer(hdr, out);
 }

system/jhtree/keydiff.cpp

@@ -413,7 +413,7 @@ public:
         if(!keyFileIO)
             throw MakeStringException(0, "Could not write index file %s", filename);
         keyStream.setown(createIOStream(keyFileIO));
-        unsigned flags = COL_PREFIX | HTREE_FULLSORT_KEY | HTREE_COMPRESSED_KEY;
+        unsigned flags = COL_PREFIX | HTREE_FULLSORT_KEY | HTREE_COMPRESSED_KEY | USE_TRAILING_HEADER;
         if(variableWidth)
             flags |= HTREE_VARSIZE;
         if(quickCompressed)

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -171,7 +171,17 @@ public:
             flags |= HTREE_TOPLEVEL_KEY;
         buildUserMetadata(metadata);                
         buildLayoutMetadata(metadata);
-        unsigned nodeSize = metadata ? metadata->getPropInt("_nodeSize", NODESIZE) : NODESIZE;
+        unsigned nodeSize = NODESIZE;
+        if (metadata)
+        {
+            nodeSize = metadata->getPropInt("_nodeSize", NODESIZE);
+            if (metadata->getPropBool("_noSeek", false))
+                flags |= TRAILING_HEADER_ONLY;
+            if (metadata->getPropBool("_useTrailingHeader", true))
+                flags |= USE_TRAILING_HEADER;
+        }
+        else
+            flags |= USE_TRAILING_HEADER;
         builder.setown(createKeyBuilder(out, flags, maxDiskRecordSize, nodeSize, helper->getKeyedSize(), isTopLevel ? 0 : totalCount, helper, !isTlk, isTlk));
     }
     void buildUserMetadata(Owned<IPropertyTree> & metadata)

tools/dumpkey/dumpkey.cpp

@@ -158,6 +158,12 @@ int main(int argc, const char **argv)
                 MemoryAttr block(sizeof(KeyHdr));
                 io->read(0, sizeof(KeyHdr), (void *)block.get());
                 header->load(*(KeyHdr*)block.get());
+                if (header->getKeyType() & USE_TRAILING_HEADER)
+                {
+                    if (io->read(in->size() - header->getNodeSize(), sizeof(KeyHdr), (void *)block.get()) != sizeof(KeyHdr))
+                        throw MakeStringException(4, "Invalid key %s: failed to read trailing key header", keyName);
+                    header->load(*(KeyHdr*)block.get());
+                }
 
                 printf("Key '%s'\nkeySize=%d keyedSize = %d NumParts=%x, Top=%d\n", keyName, key_size, keyedSize, index->numParts(), index->isTopLevelKey());
                 printf("File size = %" I64F "d, nodes = %" I64F "d\n", in->size(), in->size() / nodeSize - 1);