HPCC-26443 Allow query vs other (third party) memory requirements

Containerized only.

In order to explictly define how much of the resourced memory should be
dedicated to Thor's internal memory (roxiemem) vs other uses within the
same container, e.g. by 3rd parties, add a memory section which can
define memory characteristics.

Also allow these settings to be defined per workunit.

May expand at some point to also define things like amount to use for
different caching sizes etc.

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

helm/hpcc/values.schema.json

@@ -473,6 +473,19 @@
     "resources": {
       "type": "object"
     },
+    "memory": {
+      "type": "object",
+      "properties": {
+        "query": {
+          "type": "string",
+          "description": "The amount of overall resourced memory to dedicate to the query"
+        },
+        "thirdParties": {
+          "type": "string",
+          "description": "The amount of overall resource memory to reserve for 3rd party use"
+        }
+      }
+    },
     "secrets": {
       "oneOf": [
         {
@@ -1192,6 +1205,12 @@
           "type": "object",
           "additionalProperties": { "type": "string" }
         },
+        "workerMemory": {
+          "$ref": "#/definitions/memory"
+        },
+        "managerMemory": {
+          "$ref": "#/definitions/memory"
+        },
         "managerResources": {
           "$ref": "#/definitions/resources"
         },

helm/hpcc/values.yaml

@@ -526,6 +526,9 @@ thor:
   #workerResources:
   #  cpu: "4"
   #  memory: "4G"
+  #workerMemory:
+  #  query: "3G"
+  #  thirdParty: "500M"
   #eclAgentResources:
   #  cpu: "1"
   #  memory: "2G"

thorlcr/graph/thgraph.cpp

@@ -2714,7 +2714,7 @@ CJobBase::CJobBase(ILoadedDllEntry *_querySo, const char *_graphName) : querySo(
     maxDiskUsage = diskUsage = 0;
     dirty = true;
     aborted = false;
-    globalMemoryMB = globals->getPropInt("@globalMemorySize"); // in MB
+    queryMemoryMB = 0;
     channelsPerSlave = globals->getPropInt("@channelsPerSlave", 1);
     numChannels = channelsPerSlave;
     pluginMap = new SafePluginMap(&pluginCtx, true);
@@ -2740,6 +2740,66 @@ CJobBase::CJobBase(ILoadedDllEntry *_querySo, const char *_graphName) : querySo(
         throwUnexpected();
 }
 
+void CJobBase::applyMemorySettings(float recommendReservePercentage, const char *context)
+{
+    VStringBuffer totalMemorySetting("%sMemory/@total", context);
+    unsigned totalMemoryMB = globals->getPropInt(totalMemorySetting);
+
+    unsigned recommendReservedMemoryMB = totalMemoryMB * recommendReservePercentage / 100;
+#ifdef _CONTAINERIZED
+    /* only "query" memory is actually used (if set configures Thor roxiemem limit)
+     * others are only advisory, but totalled and checked if within the total limit.
+     */
+    std::vector<std::string> memorySettings = { "query", "thirdParty" };
+    offset_t totalRequirements = 0;
+    for (const auto &setting : memorySettings)
+    {
+        VStringBuffer workunitSettingName("%smemory.%s", context, setting.c_str()); // NB: workunit options are case insensitive
+        StringBuffer memString;
+        getWorkUnitValue(workunitSettingName, memString);
+        if (0 == memString.length())
+        {
+            VStringBuffer globalSettingName("%sMemory/@%s", context, setting.c_str());
+            globals->getProp(globalSettingName, memString);
+        }
+        if (memString.length())
+        {
+            offset_t memBytes = friendlyStringToSize(memString);
+            if (streq("query", setting.c_str()))
+                queryMemoryMB = (unsigned)(memBytes / 0x100000);
+            totalRequirements += memBytes;
+        }
+    }
+    unsigned totalRequirementsMB = (unsigned)(totalRequirements / 0x100000);
+    if (totalRequirementsMB > totalMemoryMB)
+        throw makeStringExceptionV(0, "The total memory requirements of the query (%u MB) exceeds the %s memory limit (%u MB)", totalRequirementsMB, context, totalMemoryMB);
+
+    unsigned remainingMB = totalMemoryMB - totalRequirementsMB;
+    if (remainingMB < recommendReservedMemoryMB)
+    {
+        WARNLOG("The total memory requirements of the query (%u MB) exceed the recommended reserve limits for %s (total memory: %u MB, reserve recommendation: %.2f%%)", totalRequirementsMB, context, totalMemoryMB, recommendReservePercentage);
+        if (0 == queryMemoryMB)
+            queryMemoryMB = remainingMB; // probably not recommended - use all of remaining in this case for query memory.
+    }
+    else if (0 == queryMemoryMB)
+        queryMemoryMB = remainingMB - recommendReservedMemoryMB;
+#else
+    if (totalMemoryMB < recommendReservedMemoryMB)
+        throw makeStringExceptionV(0, "The total memory (%u MB) is less than recommendReservedMemoryMB (%u MB), recommendReservePercentage (%.2f%%)", totalMemoryMB, recommendReservedMemoryMB, recommendReservePercentage);
+
+    unsigned remainingMB = totalMemoryMB;
+    queryMemoryMB = totalMemoryMB - recommendReservedMemoryMB;
+#endif
+
+    bool gmemAllowHugePages = globals->getPropBool("@heapUseHugePages", false);
+    gmemAllowHugePages = globals->getPropBool("@heapMasterUseHugePages", gmemAllowHugePages);
+    bool gmemAllowTransparentHugePages = globals->getPropBool("@heapUseTransparentHugePages", true);
+    bool gmemRetainMemory = globals->getPropBool("@heapRetainMemory", false);
+    roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, gmemRetainMemory, ((memsize_t)queryMemoryMB) * 0x100000, 0, thorAllocSizes, NULL);
+
+    PROGLOG("Total memory = %u MB, query memory = %u MB, memory spill at = %u (reserve = %.2f%%, reserveMB = %u, remainingMB = %u)", totalMemoryMB, queryMemoryMB, memorySpillAtPercentage, recommendReservePercentage, recommendReservedMemoryMB, remainingMB);
+}
+
 void CJobBase::init()
 {
     StringBuffer tmp;
@@ -2768,13 +2828,10 @@ void CJobBase::init()
     crcChecking = 0 != getWorkUnitValueInt("THOR_ROWCRC", globals->getPropBool("@THOR_ROWCRC", false));
     usePackedAllocator = 0 != getWorkUnitValueInt("THOR_PACKEDALLOCATOR", globals->getPropBool("@THOR_PACKEDALLOCATOR", true));
     memorySpillAtPercentage = (unsigned)getWorkUnitValueInt("memorySpillAt", globals->getPropInt("@memorySpillAt", 80));
-    sharedMemoryLimitPercentage = (unsigned)getWorkUnitValueInt("globalMemoryLimitPC", globals->getPropInt("@sharedMemoryLimit", 90));
-    sharedMemoryMB = globalMemoryMB*sharedMemoryLimitPercentage/100;
     failOnLeaks = getOptBool("failOnLeaks");
     maxLfnBlockTimeMins = getOptInt(THOROPT_MAXLFN_BLOCKTIME_MINS, DEFAULT_MAXLFN_BLOCKTIME_MINS);
     soapTraceLevel = getOptInt("soapTraceLevel", 1);
 
-    PROGLOG("Global memory size = %d MB, shared memory = %d%%, memory spill at = %d%%", globalMemoryMB, sharedMemoryLimitPercentage, memorySpillAtPercentage);
     StringBuffer tracing("maxActivityCores = ");
     if (maxActivityCores)
         tracing.append(maxActivityCores);

thorlcr/graph/thgraph.hpp

@@ -69,6 +69,23 @@
 #define THORDATALINK_COUNT_MASK         (RCMAX>>2)                                  // mask to extract count value only
 
 
+/* These percentages are used to determine the amount roxiemem allocated
+ * from total system memory.
+ *
+ * For historical reasons the default in bare-metal has always been a
+ * conservative reserve of 25%.
+ *
+ * NB: These percentages do not apply if the memory amount has been configured
+ * manually via 'globalMemorySize' and 'masterMemorySize'
+ */
+
+#ifdef _CONTAINERIZED
+constexpr float roxieMemPercentage = 10.0;
+#else
+constexpr float roxieMemPercentage = 25.0;
+#endif
+
+
 
 enum ActivityAttributes { ActAttr_Source=1, ActAttr_Sink=2 };
 const static roxiemem::RoxieHeapFlags defaultHeapFlags = roxiemem::RHFscanning;
@@ -829,7 +846,7 @@ protected:
     bool timeActivities;
     unsigned channelsPerSlave;
     unsigned numChannels;
-    unsigned maxActivityCores, globalMemoryMB, sharedMemoryMB;
+    unsigned maxActivityCores, queryMemoryMB, sharedMemoryMB;
     unsigned forceLogGraphIdMin, forceLogGraphIdMax;
     Owned<IContextLogger> logctx;
     Owned<IPerfMonHook> perfmonhook;
@@ -840,7 +857,7 @@ protected:
     bool usePackedAllocator;
     rank_t myNodeRank;
     Owned<IPropertyTree> graphXGMML;
-    unsigned memorySpillAtPercentage, sharedMemoryLimitPercentage;
+    unsigned memorySpillAtPercentage;
     CriticalSection sharedAllocatorCrit;
     Owned<IThorAllocator> sharedAllocator;
     bool jobEnded = false;
@@ -899,6 +916,7 @@ public:
     virtual IGraphTempHandler *createTempHandler(bool errorOnMissing) = 0;
     void addDependencies(IPropertyTree *xgmml, bool failIfMissing=true);
     void addSubGraph(IPropertyTree &xgmml);
+    void applyMemorySettings(float recommendReservePercentage, const char *context);
 
     void checkAndReportLeaks(roxiemem::IRowManager *rowManager);
     bool queryUseCheckpoints() const;

thorlcr/graph/thgraphmaster.cpp

@@ -1301,7 +1301,6 @@ CJobMaster::CJobMaster(IConstWorkUnit &_workunit, const char *graphName, ILoaded
     user.set(workunit->queryUser());
     token.append(_token.str());
     scope.append(_scope.str());
-    globalMemoryMB = globals->getPropInt("@masterMemorySize", globals->getPropInt("@globalMemorySize")); // in MB
     numChannels = 1;
     init();
 
@@ -1347,7 +1346,15 @@ CJobMaster::CJobMaster(IConstWorkUnit &_workunit, const char *graphName, ILoaded
         plugin.getPluginName(name);
         loadPlugin(pluginMap, pluginsDir.str(), name.str());
     }
-    sharedAllocator.setown(::createThorAllocator(globalMemoryMB, 0, 1, memorySpillAtPercentage, *logctx, crcChecking, usePackedAllocator));
+
+    unsigned recommendReservePercentage = roxieMemPercentage;
+#ifndef _CONTAINERIZED
+    // Weird @localThor mode, where it 50% of memory is dedicated to slaves, 25% to manager and 25% reserved (for OS etc.)
+    if (globals->getPropBool("@localThor") && (0 == globals->getPropInt("@masterMemorySize")))
+        recommendReservePercentage = 25 + 50;
+#endif
+    applyMemorySettings(recommendReservePercentage, "manager");
+    sharedAllocator.setown(::createThorAllocator(queryMemoryMB, 0, 1, memorySpillAtPercentage, *logctx, crcChecking, usePackedAllocator));
     Owned<IMPServer> mpServer = getMPServer();
     CJobChannel *channel = addChannel(mpServer);
     channel->reservePortKind(TPORT_mp); 

thorlcr/graph/thgraphslave.cpp

@@ -1686,7 +1686,27 @@ CJobSlave::CJobSlave(ISlaveWatchdog *_watchdog, IPropertyTree *_workUnitInfo, co
         pluginMap->loadFromList(pluginsList.str());
     }
     tmpHandler.setown(createTempHandler(true));
-    sharedAllocator.setown(::createThorAllocator(globalMemoryMB, sharedMemoryMB, numChannels, memorySpillAtPercentage, *logctx, crcChecking, usePackedAllocator));
+
+    float recommendReservePercentage = roxieMemPercentage;
+#ifndef _CONTAINERIZED
+    unsigned numWorkers = globals->getPropInt("@slavesPerNode", 1);
+
+    // Weird @localThor mode, where <roxieMemPercentage>(25%) of memory is reserved for OS, <roxieMemPercentage>(25%) is reserved for master, and reset fo slaves
+    if (globals->getPropBool("@localThor") && (0 == globals->getPropInt("@masterMemorySize")))
+    {
+        // reserve is <roxieMemPercentage>% (for OS) + <roxieMemPercentage>% (for manager) + [other workers * slave of rest(50%)]
+        recommendReservePercentage = roxieMemPercentage + roxieMemPercentage + ((numWorkers-1) * ((100-roxieMemPercentage-roxieMemPercentage) / ((float)numWorkers)));
+    }
+    else
+        recommendReservePercentage = roxieMemPercentage + ((numWorkers-1) * ((100-roxieMemPercentage) / ((float)numWorkers)));
+#endif
+    applyMemorySettings(recommendReservePercentage, "worker");
+
+    unsigned sharedMemoryLimitPercentage = (unsigned)getWorkUnitValueInt("globalMemoryLimitPC", globals->getPropInt("@sharedMemoryLimit", 90));
+    unsigned sharedMemoryMB = queryMemoryMB*sharedMemoryLimitPercentage/100;
+    PROGLOG("Shared memory = %d%%", sharedMemoryLimitPercentage);
+
+    sharedAllocator.setown(::createThorAllocator(queryMemoryMB, sharedMemoryMB, numChannels, memorySpillAtPercentage, *logctx, crcChecking, usePackedAllocator));
 
     StringBuffer remoteCompressedOutput;
     getOpt("remoteCompressedOutput", remoteCompressedOutput);

thorlcr/master/thmastermain.cpp

@@ -72,19 +72,6 @@
 #define SHUTDOWN_IN_PARALLEL 20
 
 
-/* These percentages are used to determine the amount roxiemem allocated
- * from total system memory.
- *
- * For historical reasons the default in bare-metal has always been a
- * conservative 75%.
- *
- * NB: These percentages do not apply if the memory amount has been configured
- * manually via 'globalMemorySize' and 'masterMemorySize'
- */
-
-static constexpr unsigned bareMetalRoxieMemPC = 75;
-static constexpr unsigned containerRoxieMemPC = 90;
-
 
 class CThorEndHandler : implements IThreaded
 {
@@ -742,7 +729,8 @@ int main( int argc, const char *argv[]  )
             Owned<IPropertyTree> masterNasFilters = envGetInstallNASHooks(nasConfig, &thorEp);
         }
 #endif
-        
+
+
         HardwareInfo hdwInfo;
         getHardwareInfo(hdwInfo);
         globals->setPropInt("@masterTotalMem", hdwInfo.totalMemory);
@@ -756,62 +744,41 @@ int main( int argc, const char *argv[]  )
             {
                 offset_t sizeBytes = friendlyStringToSize(workerResourcedMemory);
                 gmemSize = (unsigned)(sizeBytes / 0x100000);
-                gmemSize = gmemSize * containerRoxieMemPC / 100;
             }
             else
             {
-                unsigned maxMem = hdwInfo.totalMemory;
+                gmemSize = hdwInfo.totalMemory;
 #ifdef _WIN32
-                if (maxMem > 2048)
-                    maxMem = 2048;
+                if (gmemSize > 2048)
+                    gmemSize = 2048;
 #else
 #ifndef __64BIT__
-                if (maxMem > 2048)
+                if (gmemSize > 2048)
                 {
                     // 32 bit OS doesn't handle whole physically installed RAM
-                    maxMem = 2048;
+                    gmemSize = 2048;
                 }
 #ifdef __ARM_ARCH_7A__
                 // For ChromeBook with 2GB RAM
-                if (maxMem <= 2048)
+                if (gmemSize <= 2048)
                 {
                     // Decrease max memory to 2/3 
-                    maxMem = maxMem * 2 / 3; 
+                    gmemSize = gmemSize * 2 / 3; 
                 }
 #endif            
 #endif
 #endif
-                if (isContainerized())
-                    gmemSize = maxMem * containerRoxieMemPC / 100; // NB: MB's
-                else
-                {
-                    if (globals->getPropBool("@localThor") && 0 == mmemSize)
-                    {
-                        gmemSize = maxMem / 2; // 50% of total for slaves
-                        mmemSize = maxMem / 4; // 25% of total for master
-                    }
-                    else
-                        gmemSize = maxMem * bareMetalRoxieMemPC / 100; // NB: MB's
-                }
             }
-            unsigned perSlaveSize = gmemSize;
-#ifndef _CONTAINERIZED
-            if (slavesPerNode>1)
-            {
-                PROGLOG("Sharing globalMemorySize(%d MB), between %d slave processes. %d MB each", perSlaveSize, slavesPerNode, perSlaveSize / slavesPerNode);
-                perSlaveSize /= slavesPerNode;
-            }
-#endif
-            globals->setPropInt("@globalMemorySize", perSlaveSize);
         }
-        else
+        IPropertyTree *workerMemory = ensurePTree(globals, "workerMemory");
+        workerMemory->setPropInt("@total", gmemSize);
+
+        if (mmemSize)
         {
-            if (gmemSize >= hdwInfo.totalMemory)
-            {
-                // should prob. error here
-            }
+            if (mmemSize > hdwInfo.totalMemory)
+                OWARNLOG("Configured manager memory size (%u MB) is greater than total hardware memory (%u MB)", mmemSize, hdwInfo.totalMemory);
         }
-        if (0 == mmemSize)
+        else
         {
             // NB: This could be in a isContainerized(), but the 'managerResources' section only applies to containerized setups
             const char *managerResourcedMemory = globals->queryProp("managerResources/@memory");
@@ -819,22 +786,13 @@ int main( int argc, const char *argv[]  )
             {
                 offset_t sizeBytes = friendlyStringToSize(managerResourcedMemory);
                 mmemSize = (unsigned)(sizeBytes / 0x100000);
-                mmemSize = mmemSize * containerRoxieMemPC / 100;
             }
             else
                 mmemSize = gmemSize; // default to same as slaves
         }
 
-        bool gmemAllowHugePages = globals->getPropBool("@heapUseHugePages", false);
-        gmemAllowHugePages = globals->getPropBool("@heapMasterUseHugePages", gmemAllowHugePages);
-        bool gmemAllowTransparentHugePages = globals->getPropBool("@heapUseTransparentHugePages", true);
-        bool gmemRetainMemory = globals->getPropBool("@heapRetainMemory", false);
-
-        // if @masterMemorySize and @globalMemorySize unspecified gmemSize will be default based on h/w
-        globals->setPropInt("@masterMemorySize", mmemSize);
-
-        PROGLOG("Global memory size = %d MB", mmemSize);
-        roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, gmemRetainMemory, ((memsize_t)mmemSize) * 0x100000, 0, thorAllocSizes, NULL);
+        IPropertyTree *managerMemory = ensurePTree(globals, "managerMemory");
+        managerMemory->setPropInt("@total", mmemSize);
 
 #ifndef _CONTAINERIZED
         const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");

thorlcr/slave/slavmain.cpp

@@ -2341,16 +2341,6 @@ void slaveMain(bool &jobListenerStopped, ILogMsgHandler *logHandler)
     getHardwareInfo(hdwInfo);
     if (hdwInfo.totalMemory < masterMemMB)
         OWARNLOG("Slave has less memory than master node");
-    unsigned gmemSize = globals->getPropInt("@globalMemorySize");
-    bool gmemAllowHugePages = globals->getPropBool("@heapUseHugePages", false);
-    bool gmemAllowTransparentHugePages = globals->getPropBool("@heapUseTransparentHugePages", true);
-    bool gmemRetainMemory = globals->getPropBool("@heapRetainMemory", false);
-
-    if (gmemSize >= hdwInfo.totalMemory)
-    {
-        // should prob. error here
-    }
-    roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, gmemRetainMemory, ((memsize_t)gmemSize) * 0x100000, 0, thorAllocSizes, NULL);
 
     CThorResourceSlave slaveResource;
     CJobListener jobListener(jobListenerStopped);