Browse Source

HPCC-10784 Keyed limits on stepped indexreads can be miscalculated

Fix regression in prior attempt to fix this (,LOCAL indexes broken).

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>
Richard Chapman 11 năm trước cách đây
mục cha
commit
89e0898edc
2 tập tin đã thay đổi với 23 bổ sung3 xóa
  1. 2 2
      roxie/ccd/ccdactivities.cpp
  2. 21 1
      system/jhtree/jhtree.cpp

+ 2 - 2
roxie/ccd/ccdactivities.cpp

@@ -3098,7 +3098,7 @@ protected:
     virtual void createSegmentMonitors() = 0;
     virtual void setPartNo(bool filechanged)
     {
-        if (!lastPartNo.partNo)
+        if (!lastPartNo.partNo)  // Check for ,LOCAL indexes
         {
             assertex(filechanged);
             Owned<IKeyIndexSet> allKeys = createKeyIndexSet();
@@ -3184,7 +3184,7 @@ protected:
         unsigned __int64 result = 0;
         unsigned inputsDone = 0;
         bool ret = true;
-        unsigned saveStepping = steppingOffset;
+        unsigned saveStepping = steppingOffset;  // Avoid using a stepping keymerger for the checkCount - they don't support it (and would be inefficient)
         steppingOffset = 0;
         while (!aborted && inputsDone < inputCount)
         {

+ 21 - 1
system/jhtree/jhtree.cpp

@@ -2764,6 +2764,8 @@ public:
 
     virtual unsigned __int64 getCount()
     {
+        assertex (!sortFieldOffset);  // we should have avoided using a stepping merger for precheck of limits, both for efficiency and because this code won't work
+                                      // as the sequence numbers are not in sequence
         unsigned __int64 ret = 0;
         if (resetPending)
             resetSort(NULL, 0, 0); // This is slightly suboptimal
@@ -2779,7 +2781,25 @@ public:
 
     virtual unsigned __int64 checkCount(unsigned __int64 max)
     {
-        throwUnexpected();  // Can't sensibly check counts on merged streams (and you don't want to even if you could)
+        assertex (!sortFieldOffset);  // we should have avoided using a stepping merger for precheck of limits, both for efficiency and because this code won't work
+                                      // as the sequence numbers are not in sequence
+        unsigned __int64 ret = 0;
+        if (resetPending)
+            resetSort(NULL, 0, 0); // this is a little suboptimal as we will not bail out early
+        for (unsigned i = 0; i < activekeys; i++)
+        {
+            unsigned key = mergeheap[i];
+            keyBuffer = buffers[key];
+            keyCursor = cursors[key];
+            ret += CKeyLevelManager::checkCount(max);
+            if (max)
+            {
+                if (ret > max)
+                    return ret;
+                max -= ret;
+            }
+        }
+        return ret;
     }
 
     virtual void serializeCursorPos(MemoryBuffer &mb)