Merge pull request #6578 from ghalliday/issue12452

HPCC-12452 Allow configurable bucket sizes for the heap allocator

Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Mark Kelly <mark.kelly@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/roxierow.cpp

@@ -697,7 +697,7 @@ protected:
 
     void testSetup()
     {
-        setTotalMemoryLimit(false, 40*HEAP_ALIGNMENT_SIZE, 0, NULL);
+        setTotalMemoryLimit(false, 40*HEAP_ALIGNMENT_SIZE, 0, NULL, NULL);
     }
 
     void testCleanup()

ecl/eclagent/eclagent.cpp

@@ -2012,8 +2012,6 @@ void EclAgent::runProcess(IEclProcess *process)
 {
     assertex(rowManager==NULL);
     allocatorMetaCache.setown(createRowAllocatorCache(this));
-    rowManager.setown(roxiemem::createRowManager(0, NULL, queryDummyContextLogger(), allocatorMetaCache, false));
-    setHThorRowManager(rowManager.get());
 
     //Get memory limit. Workunit specified value takes precedence over config file
     int memLimitMB = agentTopology->getPropInt("@defaultMemoryLimitMB", DEFAULT_MEM_LIMIT);
@@ -2032,8 +2030,10 @@ void EclAgent::runProcess(IEclProcess *process)
     }
 #endif
     memsize_t memLimitBytes = (memsize_t)memLimitMB * 1024 * 1024;
-    roxiemem::setTotalMemoryLimit(allowHugePages, memLimitBytes, 0, NULL);
+    roxiemem::setTotalMemoryLimit(allowHugePages, memLimitBytes, 0, NULL, NULL);
 
+    rowManager.setown(roxiemem::createRowManager(0, NULL, queryDummyContextLogger(), allocatorMetaCache, false));
+    setHThorRowManager(rowManager.get());
     rowManager->setActivityTracking(queryWorkUnit()->getDebugValueBool("traceRoxiePeakMemory", false));
 
     if (debugContext)

roxie/ccd/ccdmain.cpp

@@ -758,7 +758,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         bool allowHugePages = topology->getPropBool("@heapUseHugePages", false);
         if (!totalMemoryLimit)
             totalMemoryLimit = 1024 * 0x100000;  // 1 Gb;
-        roxiemem::setTotalMemoryLimit(allowHugePages, totalMemoryLimit, 0, NULL);
+        roxiemem::setTotalMemoryLimit(allowHugePages, totalMemoryLimit, 0, NULL, NULL);
 
         traceStartStop = topology->getPropBool("@traceStartStop", false);
         traceServerSideCache = topology->getPropBool("@traceServerSideCache", false);

roxie/ccd/ccdserver.cpp

@@ -26549,7 +26549,7 @@ protected:
 
     void testSetup()
     {
-        roxiemem::setTotalMemoryLimit(false, 100 * 1024 * 1024, 0, NULL);
+        roxiemem::setTotalMemoryLimit(false, 100 * 1024 * 1024, 0, NULL, NULL);
     }
 
     void testCleanup()

roxie/roxiemem/roxiemem.cpp

@@ -116,6 +116,8 @@ inline VALUE_TYPE align_pow2(VALUE_TYPE value, ALIGN_TYPE alignment)
 
 #define PAGES(x, alignment)    (((x) + ((alignment)-1)) / (alignment))           // hope the compiler converts to a shift
 
+//---------------------------------------------------------------------------------------------------------------------
+
 typedef MapBetween<unsigned, unsigned, memsize_t, memsize_t> MapActivityToMemsize;
 
 static CriticalSection heapBitCrit;
@@ -123,6 +125,7 @@ static CriticalSection heapBitCrit;
 static void initializeHeap(bool allowHugePages, unsigned pages, unsigned largeBlockGranularity, ILargeMemCallback * largeBlockCallback)
 {
     if (heapBase) return;
+
     // CriticalBlock b(heapBitCrit); // unnecessary - must call this exactly once before any allocations anyway!
     heapBitmapSize = (pages + UNSIGNED_BITS - 1) / UNSIGNED_BITS;
     heapTotalPages = heapBitmapSize * UNSIGNED_BITS;
@@ -2573,7 +2576,7 @@ protected:
 };
 
 //Constants are here to ensure they can all be constant folded
-const unsigned roundupDoubleLimit = 2048;  // Values up to this limit are rounded to the nearest power of 2
+const unsigned roundupDoubleLimit = MAX_SIZE_DIRECT_BUCKET;  // Values up to this limit are directly mapped to a bucket size
 const unsigned roundupStepSize = 4096;  // Above the roundupDoubleLimit memory for a row is allocated in this size step
 const unsigned limitStepBlock = FixedSizeHeaplet::maxHeapSize()/MAX_FRAC_ALLOCATOR;  // until it gets to this size
 const unsigned numStepBlocks = PAGES(limitStepBlock, roundupStepSize); // how many step blocks are there?
@@ -2597,6 +2600,36 @@ public:
 };
 
 
+//---------------------------------------------------------------------------------------------------------------------
+
+static unsigned numDirectBuckets;
+//This array contains details of the actual sizes that are used to allocate an item of size bytes.
+//The entry allocSize((size-1)/ALLOC_ALIGNMENT is a value that indicates which heap and the size of that heap
+//NOTE: using "size-1" ensures that values X*ALLOC_ALIGNMENT-(ALLOC_ALIGNMENT-1)..X*ALLOC_ALIGNMENT are mapped to the same bin
+static unsigned allocSizeMapping[MAX_SIZE_DIRECT_BUCKET/ALLOC_ALIGNMENT+1];
+
+const static unsigned defaultAllocSizes[] = { 16, 32, 64, 128, 256, 512, 1024, 2048, 0 };
+
+void initAllocSizeMappings(const unsigned * sizes)
+{
+    size32_t bucketSize = sizes[0];
+    unsigned bucket = 0;
+    for (unsigned size=ALLOC_ALIGNMENT; size <= MAX_SIZE_DIRECT_BUCKET; size += ALLOC_ALIGNMENT)
+    {
+        if (size > bucketSize)
+        {
+            bucket++;
+            dbgassertex(bucketSize < sizes[bucket]);
+            bucketSize = sizes[bucket];
+        }
+        allocSizeMapping[(size-1)/ALLOC_ALIGNMENT] = ROUNDED(bucket, bucketSize);
+    }
+    assertex(sizes[bucket+1] == 0);
+    numDirectBuckets = bucket+1;
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
 
 class CChunkingRowManager : public CInterface, implements IRowManager
 {
@@ -2838,38 +2871,16 @@ public:
     {
         dbgassertex((size >= FixedSizeHeaplet::chunkHeaderSize) && (size <= FixedSizeHeaplet::maxHeapSize() + FixedSizeHeaplet::chunkHeaderSize));
         //MORE: A binary chop on sizes is likely to be better.
-        if (size<=256)
-        {
-            if (size<=64)
-            {
-                if (size<=32)
-                {
-                    if (size<=16)
-                        return ROUNDED(0, 16);
-                    return ROUNDED(1, 32);
-                }
-                return ROUNDED(2, 64);
-            }
-            if (size<=128)
-                return ROUNDED(3, 128);
-            return ROUNDED(4, 256);
-        }
-        if (size<=1024)
-        {
-            if (size<=512)
-                return ROUNDED(5, 512);
-            return ROUNDED(6, 1024);
-        }
-        if (size<=2048)
-            return ROUNDED(7, 2048);
+        if (size <= MAX_SIZE_DIRECT_BUCKET)
+            return allocSizeMapping[(size-1)/ALLOC_ALIGNMENT];
 
         if (size <= maxStepSize)
         {
             size32_t blocks = PAGES(size, roundupStepSize);
-            return ROUNDED(7 + blocks, blocks * roundupStepSize);
+            return ROUNDED((numDirectBuckets-1) + blocks, blocks * roundupStepSize);
         }
 
-        unsigned baseBlock = 7;
+        unsigned baseBlock = (numDirectBuckets-1);
         if (hasAnyStepBlocks)
             baseBlock += numStepBlocks;
 
@@ -4136,6 +4147,9 @@ void DataBufferBottom::_setDestructorFlag(const void *ptr) { throwUnexpected();
 
 extern IRowManager *createRowManager(memsize_t memLimit, ITimeLimiter *tl, const IContextLogger &logctx, const IRowAllocatorCache *allocatorCache, bool ignoreLeaks, bool outputOOMReports)
 {
+    if (numDirectBuckets == 0)
+        throw MakeStringException(ROXIEMM_HEAP_ERROR, "createRowManager() called before setTotalMemoryLimit()");
+
     return new CChunkingRowManager(memLimit, tl, logctx, allocatorCache, ignoreLeaks, outputOOMReports);
 }
 
@@ -4145,7 +4159,7 @@ extern void setMemoryStatsInterval(unsigned secs)
     lastStatsCycles = get_cycles_now();
 }
 
-extern void setTotalMemoryLimit(bool allowHugePages, memsize_t max, memsize_t largeBlockSize, ILargeMemCallback * largeBlockCallback)
+extern void setTotalMemoryLimit(bool allowHugePages, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback)
 {
     assertex(largeBlockSize == align_pow2(largeBlockSize, HEAP_ALIGNMENT_SIZE));
     unsigned totalMemoryLimit = (unsigned) (max / HEAP_ALIGNMENT_SIZE);
@@ -4155,6 +4169,7 @@ extern void setTotalMemoryLimit(bool allowHugePages, memsize_t max, memsize_t la
     if (memTraceLevel)
         DBGLOG("RoxieMemMgr: Setting memory limit to %"I64F"d bytes (%u pages)", (unsigned __int64) max, totalMemoryLimit);
     initializeHeap(allowHugePages, totalMemoryLimit, largeBlockGranularity, largeBlockCallback);
+    initAllocSizeMappings(allocSizes ? allocSizes : defaultAllocSizes);
 }
 
 extern memsize_t getTotalMemoryLimit()
@@ -4407,6 +4422,7 @@ protected:
 
         memsize_t memory = (useLargeMemory ? largeMemory : smallMemory) * (unsigned __int64)0x100000U;
         initializeHeap(false, (unsigned)(memory / HEAP_ALIGNMENT_SIZE), 0, NULL);
+        initAllocSizeMappings(defaultAllocSizes);
     }
 
     void testCleanup()
@@ -5343,6 +5359,7 @@ protected:
     void testRoundup()
     {
         Owned<IRowManager> rowManager = createRowManager(1, NULL, logctx, NULL);
+        CChunkingRowManager * managerObject = static_cast<CChunkingRowManager *>(rowManager.get());
         const unsigned maxFrac = firstFractionalHeap;
 
         const void * tempRow[MAX_FRAC_ALLOCATOR];
@@ -5653,7 +5670,7 @@ public:
 protected:
     void testSetup()
     {
-        setTotalMemoryLimit(false, memorySize, 0, NULL);
+        setTotalMemoryLimit(false, memorySize, 0, NULL, NULL);
     }
 
     void testCleanup()
@@ -5849,7 +5866,7 @@ public:
 protected:
     void testSetup()
     {
-        setTotalMemoryLimit(false, hugeMemorySize, 0, NULL);
+        setTotalMemoryLimit(false, hugeMemorySize, 0, NULL, NULL);
     }
 
     void testCleanup()

roxie/roxiemem/roxiemem.hpp

@@ -58,6 +58,7 @@
 #define UNKNOWN_ROWSET_ID               0x000F8421              // Use as the allocatorId for a rowset from an unknown activity
 #define UNKNOWN_ACTIVITY                123456789
 
+#define MAX_SIZE_DIRECT_BUCKET          2048                    // Sizes below this are directly mapped to a particular bucket
 #define ALLOC_ALIGNMENT                 sizeof(void *)          // Minimum alignment of data allocated from the heap manager
 #define PACKED_ALIGNMENT                4                       // Minimum alignment of packed blocks
 
@@ -497,7 +498,7 @@ interface IDataBufferManager : extends IInterface
 
 extern roxiemem_decl IDataBufferManager *createDataBufferManager(size32_t size);
 extern roxiemem_decl void setMemoryStatsInterval(unsigned secs);
-extern roxiemem_decl void setTotalMemoryLimit(bool allowHugePages, memsize_t max, memsize_t largeBlockSize, ILargeMemCallback * largeBlockCallback);
+extern roxiemem_decl void setTotalMemoryLimit(bool allowHugePages, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback);
 extern roxiemem_decl memsize_t getTotalMemoryLimit();
 extern roxiemem_decl void releaseRoxieHeap();
 extern roxiemem_decl bool memPoolExhausted();

roxie/udplib/uttest.cpp

@@ -754,7 +754,7 @@ int main(int argc, char * argv[] )
             printf("ERROR: my ip does not appear to be in range\n");
             usage();
         }
-        roxiemem::setTotalMemoryLimit(false, 1048576000, 0, NULL);
+        roxiemem::setTotalMemoryLimit(false, 1048576000, 0, NULL, NULL);
         testNxN();
         roxiemem::releaseRoxieHeap();
     }

thorlcr/master/thmastermain.cpp

@@ -688,7 +688,7 @@ int main( int argc, char *argv[]  )
         globals->setPropInt("@masterMemorySize", mmemSize);
 
         PROGLOG("Global memory size = %d MB", mmemSize);
-        roxiemem::setTotalMemoryLimit(gmemAllowHugePages, ((memsize_t)mmemSize) * 0x100000, 0, NULL);
+        roxiemem::setTotalMemoryLimit(gmemAllowHugePages, ((memsize_t)mmemSize) * 0x100000, 0, thorAllocSizes, NULL);
 
         const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
         const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");

thorlcr/slave/slavmain.cpp

@@ -667,7 +667,7 @@ void slaveMain()
     {
         // should prob. error here
     }
-    roxiemem::setTotalMemoryLimit(gmemAllowHugePages, ((memsize_t)gmemSize) * 0x100000, 0, NULL);
+    roxiemem::setTotalMemoryLimit(gmemAllowHugePages, ((memsize_t)gmemSize) * 0x100000, 0, thorAllocSizes, NULL);
 
     CJobListener jobListener;
     CThorResourceSlave slaveResource;

thorlcr/thorutil/thmem.hpp

@@ -547,4 +547,7 @@ extern graph_decl ILargeMemLimitNotify *createMultiThorResourceMutex(const char
 
 extern graph_decl void setThorVMSwapDirectory(const char *swapdir);
 
+//The following array defines the buckets that are created for pooling row allocations.  The sizes must be in ascending order, and terminated with 0
+const static unsigned thorAllocSizes[] = { 16, 32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, 0 };
+
 #endif