Merge remote-tracking branch 'origin/candidate-3.8.x' into candidate-3.10.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/base/dadfs.cpp

@@ -3459,7 +3459,7 @@ public:
         {
         protected:
             CriticalSection &crit;
-            CIStringArray *&newNames;
+            CIArrayOf<CIStringArray> &newNames;
             IDistributedFile *file;
             unsigned width;
             IMultiException *mexcept;
@@ -3471,7 +3471,7 @@ public:
             bool * donerep;
             IException *except;
 
-            casyncforbase(IDistributedFile *_file,CIStringArray *&_newNames,unsigned _width,IMultiException *_mexcept,CriticalSection &_crit,bool *_ignoreprim,bool *_ignorerep)
+            casyncforbase(IDistributedFile *_file,CIArrayOf<CIStringArray> &_newNames,unsigned _width,IMultiException *_mexcept,CriticalSection &_crit,bool *_ignoreprim,bool *_ignorerep)
                 : newNames(_newNames),crit(_crit)
             {
                 width = _width;
@@ -3518,7 +3518,7 @@ public:
                     IException *ex = NULL;
                     RemoteFilename oldrfn;
                     part->getFilename(oldrfn,(unsigned)copy);
-                    const char *newfn = newNames[idx].item(copy);
+                    const char *newfn = newNames.item(idx).item(copy);
                     if (!newfn||!*newfn)
                         continue;
                     RemoteFilename newrfn;
@@ -3553,7 +3553,7 @@ public:
         class casyncfor1: public casyncforbase
         {
         public:
-            casyncfor1(IDistributedFile *_file,CIStringArray *&_newNames,unsigned _width,IMultiException *_mexcept,CriticalSection &_crit,bool *_ignoreprim,bool *_ignorerep)
+            casyncfor1(IDistributedFile *_file,CIArrayOf<CIStringArray> &_newNames,unsigned _width,IMultiException *_mexcept,CriticalSection &_crit,bool *_ignoreprim,bool *_ignorerep)
                 : casyncforbase(_file,_newNames,_width,_mexcept,_crit,_ignoreprim,_ignorerep)
             {
             }
@@ -3584,7 +3584,7 @@ public:
                 return true;
             }
 
-        } afor1 (this,*newNames.getArray(),width,mexcept,crit,NULL,NULL);
+        } afor1 (this,newNames,width,mexcept,crit,NULL,NULL);
         afor1.For(width,10,false,true);
         if (afor1.except)
             throw afor1.except; // no recovery needed
@@ -3613,7 +3613,7 @@ public:
         class casyncfor2: public casyncforbase
         {
         public:
-            casyncfor2(IDistributedFile *_file,CIStringArray *&_newNames,unsigned _width,IMultiException *_mexcept,CriticalSection &_crit,bool *_ignoreprim,bool *_ignorerep)
+            casyncfor2(IDistributedFile *_file,CIArrayOf<CIStringArray> &_newNames,unsigned _width,IMultiException *_mexcept,CriticalSection &_crit,bool *_ignoreprim,bool *_ignorerep)
                 : casyncforbase(_file,_newNames,_width,_mexcept,_crit,_ignoreprim,_ignorerep)
             {
             }
@@ -3633,7 +3633,7 @@ public:
                 return true;;
             }
 
-        } afor2 (this,*newNames.getArray(),width,mexcept,crit,ignoreprim,ignorerep);
+        } afor2 (this,newNames,width,mexcept,crit,ignoreprim,ignorerep);
         afor2.For(width,10,false,true);
         if (afor2.ok) {
             // now rename directory and partmask

roxie/roxiemem/roxiemem.cpp

@@ -92,7 +92,6 @@ static unsigned *heapBitmap;
 static unsigned heapBitmapSize;
 static unsigned heapTotalPages; // derived from heapBitmapSize - here for code clarity
 static unsigned heapLWM;
-static unsigned heapHWM;
 static unsigned heapAllocated;
 static unsigned heapLargeBlocks;
 static unsigned heapLargeBlockGranularity;
@@ -154,7 +153,6 @@ static void initializeHeap(unsigned pages, unsigned largeBlockGranularity, ILarg
     memset(heapBitmap, 0xff, heapBitmapSize*sizeof(unsigned));
     heapLargeBlocks = 1;
     heapLWM = 0;
-    heapHWM = heapBitmapSize;
 
     if (memTraceLevel)
         DBGLOG("RoxieMemMgr: %u Pages successfully allocated for the pool - memsize=%"I64F"u base=%p alignment=%"I64F"u bitmapSize=%u", 
@@ -347,7 +345,7 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
             lastStatsCycles = cyclesNow;
             StringBuffer s;
             memstats(s);
-            s.appendf(", heapLWM %u..%u, dataBuffersActive=%d, dataBufferPages=%d", heapLWM, heapHWM, atomic_read(&dataBuffersActive), atomic_read(&dataBufferPages));
+            s.appendf(", heapLWM %u, dataBuffersActive=%d, dataBufferPages=%d", heapLWM, atomic_read(&dataBuffersActive), atomic_read(&dataBufferPages));
             DBGLOG("RoxieMemMgr: %s", s.str());
         }
     }
@@ -406,7 +404,7 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
     else
     {
         // Usage pattern is such that we expect normally to succeed immediately.
-        unsigned i = heapHWM;
+        unsigned i = heapBitmapSize;
         unsigned matches = 0;
         while (i)
         {
@@ -440,7 +438,6 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
                             heapBitmap[i] = hbi;
                             if (memTraceLevel >= 2)
                                 DBGLOG("RoxieMemMgr: suballoc_aligned() %u pages ok - addr=%p", pages, ret);
-                            heapHWM = i+1;
                             heapAllocated += pages;
                             return ret;
                         }
@@ -513,8 +510,6 @@ static void subfree_aligned(void *ptr, unsigned pages = 1)
 
         if (wordOffset < heapLWM)
             heapLWM = wordOffset;
-        if (nextPageOffset > heapHWM)
-            heapHWM = nextPageOffset;
         loop
         {
             unsigned prev = heapBitmap[wordOffset];
@@ -534,7 +529,7 @@ static void subfree_aligned(void *ptr, unsigned pages = 1)
         }
     }
     if (memTraceLevel >= 2)
-        DBGLOG("RoxieMemMgr: subfree_aligned() %u pages ok - addr=%p heapLWM=%u..%u totalPages=%u", _pages, ptr, heapLWM, heapHWM, heapTotalPages);
+        DBGLOG("RoxieMemMgr: subfree_aligned() %u pages ok - addr=%p heapLWM=%u totalPages=%u", _pages, ptr, heapLWM, heapTotalPages);
 }
 
 static inline unsigned getRealActivityId(unsigned allocatorId, const IRowAllocatorCache *allocatorCache)
@@ -3251,7 +3246,6 @@ protected:
             _heapBitmapSize = heapBitmapSize;
             _heapTotalPages = heapTotalPages;
             _heapLWM = heapLWM;
-            _heapHWM = heapHWM;
             _heapAllocated = heapAllocated;
         }
         ~HeapPreserver()
@@ -3261,7 +3255,6 @@ protected:
             heapBitmapSize = _heapBitmapSize;
             heapTotalPages = _heapTotalPages;
             heapLWM = _heapLWM;
-            heapHWM = _heapHWM;
             heapAllocated = _heapAllocated;
         }
         char *_heapBase;
@@ -3269,7 +3262,6 @@ protected:
         unsigned _heapBitmapSize;
         unsigned _heapTotalPages;
         unsigned _heapLWM;
-        unsigned _heapHWM;
         unsigned _heapAllocated;
     };
     void initBitmap(unsigned size)
@@ -3280,7 +3272,6 @@ protected:
         heapBitmapSize = size;
         heapTotalPages = heapBitmapSize * UNSIGNED_BITS;
         heapLWM = 0;
-        heapHWM = size;
         heapAllocated = 0;
     }
 
@@ -4146,7 +4137,8 @@ const memsize_t memorySize = 0x60000000;
 class RoxieMemStressTests : public CppUnit::TestFixture
 {
     CPPUNIT_TEST_SUITE( RoxieMemStressTests );
-        CPPUNIT_TEST(testSequential);
+    CPPUNIT_TEST(testFragmenting);
+    CPPUNIT_TEST(testSequential);
     CPPUNIT_TEST_SUITE_END();
     const IContextLogger &logctx;
 
@@ -4184,6 +4176,32 @@ protected:
         ReleaseRoxieRow(rows);
         DBGLOG("Time for sequential allocate = %d", endTime - startTime);
     }
+
+    void testFragmenting()
+    {
+        unsigned requestSize = 32;
+        Owned<IRowManager> rowManager = createRowManager(0, NULL, logctx, NULL);
+        unsigned startTime = msTick();
+        void * prev = rowManager->allocate(requestSize, 1);
+        try
+        {
+            loop
+            {
+                void *next = rowManager->allocate(requestSize*1.25, 1);
+                requestSize *= 1.25;
+                ReleaseRoxieRow(prev);
+                prev = next;
+            }
+        }
+        catch (IException *E)
+        {
+            E->Release();
+        }
+        ReleaseRoxieRow(prev);
+        unsigned endTime = msTick();
+        DBGLOG("Time for fragmenting allocate = %d, max allocation=%u, limit = %"I64F"u", endTime - startTime, requestSize, (unsigned __int64) memorySize);
+        ASSERT(requestSize > memorySize/4);
+    }
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION( RoxieMemTests );