Revert "HPCC-3022 - Allow buildindex not to require maxlength if >4k"

This reverts commit 2703fa8f250d835e1b2a07f339695ad0cdd0693b.

system/jhtree/ctfile.cpp

@@ -242,12 +242,16 @@ CWriteNode::CWriteNode(offset_t _fpos, CKeyHdr *_keyHdr, bool isLeaf) : CWriteNo
 {
     hdr.leafFlag = isLeaf ? 1 : 0;
     if (!isLeaf)
+    {
         keyLen = keyHdr->getNodeKeyLength();
+    }
+    lastKeyValue = (char *) malloc(keyLen);
     lastSequence = 0;
 }
 
 CWriteNode::~CWriteNode()
 {
+    free(lastKeyValue);
 }
 
 bool CWriteNode::add(offset_t pos, const void *indata, size32_t insize, unsigned __int64 sequence)
@@ -360,12 +364,9 @@ bool CWriteNode::add(offset_t pos, const void *indata, size32_t insize, unsigned
         hdr.keyBytes += bytes;
     }
 
-    if (insize>lastKeyValue.length())
-    {
-        lastKeyValue.reallocate(insize);
-        keyHdr->checkMaxKeyLength(insize);
-    }
-    memcpy(lastKeyValue.bufferBase(), indata, insize);
+    if (insize>keyLen)
+        throw MakeStringException(0, "key+payload (%u) exceeds max length (%u)", insize, keyLen);
+    memcpy(lastKeyValue, indata, insize);
     lastSequence = sequence;
     hdr.numKeys++;
     return true;
@@ -374,15 +375,11 @@ bool CWriteNode::add(offset_t pos, const void *indata, size32_t insize, unsigned
 size32_t CWriteNode::compressValue(const char *keyData, size32_t size, char *result)
 {
     unsigned int pack = 0;
-    unsigned max = lastKeyValue.length();
-    if (max>255)
-        max = 255;
     if (hdr.numKeys)
     {
-        const char *lkv = (const char *)lastKeyValue.get();
-        for (; pack<size && pack<max; pack++)
+        for (; pack<size && pack<255; pack++)
         {
-            if (keyData[pack] != lkv[pack])
+            if (keyData[pack] != lastKeyValue[pack])
                 break;
         }
     }
@@ -535,7 +532,7 @@ unsigned CJHTreeNode::countAllocationsCurrent;
 unsigned CJHTreeNode::countAllocationsEver;
 SpinLock CJHTreeNode::spin;
 
-char *CJHTreeNode::expandKeys(void *src,size32_t &retsize, bool rowcompression)
+char *CJHTreeNode::expandKeys(void *src,unsigned keylength,size32_t &retsize, bool rowcompression)
 {
     Owned<IExpander> exp = rowcompression?createRDiffExpander():createLZWExpander(true);
     int len=exp->init(src);
@@ -614,7 +611,7 @@ void CJHTreeNode::unpack(const void *node, bool needCopy)
             if (!quick||!rowexp.get())
 #endif
             {
-                keyBuf = expandKeys(keys,expandedSize,quick);
+                keyBuf = expandKeys(keys,keyLen,expandedSize,quick);
             }
         }
         assertex(keyBuf||rowexp.get());

system/jhtree/ctfile.hpp

@@ -135,8 +135,6 @@ public:
     void write(IFileIOStream *, CRC32 *crc = NULL);
 
     unsigned int getMaxKeyLength();
-    void setMaxKeyLength(unsigned max) { hdr.length = max; }
-    void checkMaxKeyLength(unsigned max) { if (max>hdr.length) setMaxKeyLength(max); }
     bool isVariable();
     inline unsigned int getNodeKeyLength() 
     {
@@ -199,7 +197,7 @@ protected:
     size32_t expandedSize;
     Owned<IRandRowExpander> rowexp;  // expander for rand rowdiff   
 
-    static char *expandKeys(void *src,size32_t &retsize, bool rowcompression);
+    static char *expandKeys(void *src,unsigned keylength,size32_t &retsize, bool rowcompression);
     static IRandRowExpander *expandQuickKeys(void *src, bool needCopy);
 
     static void releaseMem(void *togo, size32_t size);
@@ -300,7 +298,7 @@ public:
 class jhtree_decl CWriteNode : public CWriteNodeBase
 {
 private:
-    MemoryAttr lastKeyValue;
+    char *lastKeyValue;
     unsigned __int64 lastSequence;
 
 public:
@@ -309,7 +307,7 @@ public:
 
     size32_t compressValue(const char *keyData, size32_t size, char *result);
     bool add(offset_t pos, const void *data, size32_t size, unsigned __int64 sequence);
-    const void *getLastKeyValue() const { return lastKeyValue.get(); }
+    const void *getLastKeyValue() const { return lastKeyValue; }
     unsigned __int64 getLastSequence() const { return lastSequence; }
 };
 

system/jhtree/keybuild.cpp

@@ -75,7 +75,7 @@ public:
 class CKeyBuilderBase : public CInterface
 {
 protected:
-    unsigned rawSize;
+    unsigned keyValueSize;
     count_t records;
     unsigned levels;
     offset_t nextPos;
@@ -88,18 +88,17 @@ protected:
     unsigned __int64 sequence;
     CRC32StartHT crcStartPosTable;
     CRC32EndHT crcEndPosTable;
-    bool doCrc, legacyCompat;
+    bool doCrc;
 
 public:
     IMPLEMENT_IINTERFACE;
 
-    CKeyBuilderBase(IFileIOStream *_out, unsigned flags, unsigned _rawSize, offset_t _fileSize, unsigned nodeSize, unsigned _keyedSize, unsigned __int64 _startSequence, bool _legacyCompat)
-        : out(_out), legacyCompat(_legacyCompat)
+    CKeyBuilderBase(IFileIOStream *_out, unsigned flags, unsigned rawSize, offset_t _fileSize, unsigned nodeSize, unsigned _keyedSize, unsigned __int64 _startSequence) : out(_out)
     {
         doCrc = false;
         sequence = _startSequence;
         keyHdr.setown(new CKeyHdr());
-        rawSize = _rawSize;
+        keyValueSize = rawSize;
         keyedSize = _keyedSize != (unsigned) -1 ? _keyedSize : rawSize;
 
         fileSize = _fileSize;
@@ -114,7 +113,7 @@ public:
         KeyHdr *hdr = keyHdr->getHdrStruct();
         hdr->nodeSize = nodeSize;
         hdr->extsiz = 4096;
-        hdr->length = 0; // fill in at end
+        hdr->length = keyValueSize; 
         hdr->ktype = flags; 
         hdr->timeid = 0;
         hdr->clstyp = 1;  // IDX_CLOSE
@@ -149,7 +148,7 @@ public:
         KeyHdr *hdr = keyHdr->getHdrStruct();
         records = hdr->nument;
         nextPos = hdr->nodeSize; // leaving room for header
-        rawSize = keyHdr->getMaxKeyLength();
+        keyValueSize = keyHdr->getMaxKeyLength();
         keyedSize = keyHdr->getNodeKeyLength();
     }
 
@@ -201,26 +200,12 @@ protected:
         return 0;
     }
 
-    void writeFileHeader(CRC32 *crc)
+    void writeFileHeader(bool fixHdr, CRC32 *crc)
     {
         if (out)
         {
             out->flush();
             out->seek(0, IFSbegin);
-
-#define LEGACY_DEFAULT_MAXLENGTH 4096
-            /* For legacy compatibility, where no explicit maxlength was provided.
-             * 1) Check if < default and set header to default - maintaining legacy key build/read behaviour.
-             * 2) Issue warning if runtime max exceeded default - key will not be compatible with legacy system, without MAXLENGTHs being added.
-             */
-            if (legacyCompat && rawSize==LEGACY_DEFAULT_MAXLENGTH)
-            {
-                if (keyHdr->getMaxKeyLength() < LEGACY_DEFAULT_MAXLENGTH)
-                    keyHdr->setMaxKeyLength(LEGACY_DEFAULT_MAXLENGTH);
-                else
-                    WARNLOG("Key created with legacy compatibility set, but key+payload size exceeds legacy default of 4096. Max size is: %d", keyHdr->getMaxKeyLength());
-            }
-
             keyHdr->write(out, crc);
         }
     }
@@ -359,8 +344,8 @@ private:
 public:
     IMPLEMENT_IINTERFACE;
 
-    CKeyBuilder(IFileIOStream *_out, unsigned flags, unsigned rawSize, offset_t fileSize, unsigned nodeSize, unsigned keyedSize, unsigned __int64 startSequence, bool legacyCompat)
-        : CKeyBuilderBase(_out, flags, rawSize, fileSize, nodeSize, keyedSize, startSequence, legacyCompat)
+    CKeyBuilder(IFileIOStream *_out, unsigned flags, unsigned rawSize, offset_t fileSize, unsigned nodeSize, unsigned keyedSize, unsigned __int64 startSequence) 
+        : CKeyBuilderBase(_out, flags, rawSize, fileSize, nodeSize, keyedSize, startSequence)
     {
         doCrc = true;
         activeNode = NULL;
@@ -395,7 +380,7 @@ public:
             writeMetadata(metaXML.str(), metaXML.length());
         }
         CRC32 headerCrc;
-        writeFileHeader(&headerCrc);
+        writeFileHeader(false, &headerCrc);
 
         if (fileCrc)
         {
@@ -504,9 +489,9 @@ protected:
     }
 };
 
-extern jhtree_decl IKeyBuilder *createKeyBuilder(IFileIOStream *_out, unsigned flags, unsigned rawSize, offset_t fileSize, unsigned nodeSize, unsigned keyFieldSize, unsigned __int64 startSequence, bool legacyCompat)
+extern jhtree_decl IKeyBuilder *createKeyBuilder(IFileIOStream *_out, unsigned flags, unsigned rawSize, offset_t fileSize, unsigned nodeSize, unsigned keyFieldSize, unsigned __int64 startSequence)
 {
-    return new CKeyBuilder(_out, flags, rawSize, fileSize, nodeSize, keyFieldSize, startSequence, legacyCompat);
+    return new CKeyBuilder(_out, flags, rawSize, fileSize, nodeSize, keyFieldSize, startSequence);
 }
 
 
@@ -559,7 +544,7 @@ public:
                 leafInfo.append(OLINK(nodes.item(idx2)));
         }
         buildTree(leafInfo);
-        writeFileHeader(NULL);
+        writeFileHeader(true, NULL);
     }
 
 protected:

system/jhtree/keybuild.hpp

@@ -101,7 +101,7 @@ interface IKeyBuilder : public IInterface
     virtual unsigned __int64 createBlob(size32_t size, const char * _ptr) = 0;
 };
 
-extern jhtree_decl IKeyBuilder *createKeyBuilder(IFileIOStream *_out, unsigned flags, unsigned rawSize, offset_t fileSize, unsigned nodeSize, unsigned keyFieldSize, unsigned __int64 startSequence, bool legacyCompat=false);
+extern jhtree_decl IKeyBuilder *createKeyBuilder(IFileIOStream *_out, unsigned flags, unsigned rawSize, offset_t fileSize, unsigned nodeSize, unsigned keyFieldSize, unsigned __int64 startSequence);
 
 interface IKeyDesprayer : public IInterface
 {

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -199,9 +199,7 @@ public:
         buildUserMetadata(metadata);                
         buildLayoutMetadata(metadata);
         unsigned nodeSize = metadata ? metadata->getPropInt("_nodeSize", NODESIZE) : NODESIZE;
-
-        bool legacyCompat = getOptBool(THOROPT_KEYBUILD_LEGACYCOMPAT, true);
-        builder.setown(createKeyBuilder(out, flags, maxDiskRecordSize, fileSize, nodeSize, helper->getKeyedSize(), isTopLevel ? 0 : totalCount, legacyCompat));
+        builder.setown(createKeyBuilder(out, flags, maxDiskRecordSize, fileSize, nodeSize, helper->getKeyedSize(), isTopLevel ? 0 : totalCount));
     }
 
 

thorlcr/thorutil/thormisc.hpp

@@ -61,7 +61,6 @@
 #define THOROPT_JOINHELPER_THREADS    "joinHelperThreads"       // Number of threads to use in threaded variety of join helper
 #define THOROPT_LKJOIN_LOCALFAILOVER  "lkjoin_localfailover"    // Force SMART to failover to distributed local lookup join (for testing only)   (default = false)
 #define THOROPT_LKJOIN_HASHJOINFAILOVER "lkjoin_hashjoinfailover" // Force SMART to failover to hash join (for testing only)                     (default = false)
-#define THOROPT_KEYBUILD_LEGACYCOMPAT "kbuild_legacycompat"     // Try to keep built key header max size compatible with legacy systems          (default = true)
 
 #define INITIAL_SELFJOIN_MATCH_WARNING_LEVEL 20000  // max of row matches before selfjoin emits warning