HPCC-26600 Use resourced memory in k8s in hthor

1) Base default query memory on config resource specifications
(replacing unconfigurable 300MB limit).
2) Add a memory section (like Thor's, see HPCC-26443) to allow
specification of how much memory is user for the "query" vs "thirdParty"
3) Allow options to overriden at workunit level

Also fixes a bug introduced by HPCC-26443 which meant
that when globalMemorySize was specified in Thor (bare-metal),
only a percentage of that specific requirement was
allocated to the query/roxiemem.

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

common/thorhelper/hpccconfig.cpp

@@ -15,9 +15,13 @@
     limitations under the License.
 ############################################################################## */
 
+#include <string>
+#include <unordered_map>
 
+#include "jlog.hpp"
 #include "jptree.hpp"
 #include "jstring.hpp"
+#include "hpccconfig.hpp"
 
 bool getService(StringBuffer &serviceAddress, const char *serviceName, bool failIfNotFound)
 {
@@ -35,3 +39,54 @@ bool getService(StringBuffer &serviceAddress, const char *serviceName, bool fail
         throw makeStringExceptionV(-1, "Service '%s' not found", serviceName);
     return false;
 }
+
+void getMemorySpecifications(std::unordered_map<std::string, __uint64> &memorySpecifications, const IPropertyTree *config, const char *context, unsigned maxMB, GetJobValueFunction getJobValueFunction)
+{
+    /* fills the memorySpecifications map with memory settings retreived from either
+     * the helper function (typically a workunit value fetch function) or the config.
+     * Helper function values take priority.
+     * "total" is computed and filled with the aggregate sum of all memory settings.
+     * "recommendedMaxMemory" is computed and filled in based on max and the "maxMemPercentage" setting.
+     */
+
+    auto getSetting = [&](const char *setting, StringBuffer &result) -> bool
+    {
+        VStringBuffer workunitSettingName("%s.%s", context, setting); // NB: workunit options are case insensitive
+        if (getJobValueFunction(workunitSettingName, result))
+            return true;
+        VStringBuffer configSettingName("%s/@%s", context, setting);
+        return config->getProp(configSettingName, result);
+    };
+    std::initializer_list<const char *> memorySettings = { "query", "thirdParty" };
+    offset_t totalRequirements = 0;
+    for (auto setting : memorySettings)
+    {
+        StringBuffer value;
+        if (getSetting(setting, value))
+        {
+            offset_t memBytes = friendlyStringToSize(value);
+            memorySpecifications[setting] = memBytes;
+            totalRequirements += memBytes;
+        }
+    }
+    offset_t maxBytes = ((offset_t)maxMB) * 0x100000;
+    if (totalRequirements > maxBytes)
+        throw makeStringExceptionV(0, "The total memory requirements of the query (%u MB) in '%s' exceed the memory limit (%u MB)", (unsigned)(totalRequirements / 0x100000), context, maxMB);
+    memorySpecifications["total"] = totalRequirements;
+
+    float maxPercentage = 100.0;
+    offset_t recommendedMaxMemory = maxBytes;
+    StringBuffer value;
+    if (getSetting("maxMemPercentage", value))
+    {
+        maxPercentage = atof(value);
+        verifyex((maxPercentage > 0.0) && (maxPercentage <= 100.0));
+        if (maxPercentage < 100.0)
+        {
+            recommendedMaxMemory = maxBytes / 100.0 * maxPercentage;
+            if (totalRequirements > recommendedMaxMemory)
+                WARNLOG("The total memory requirements of the query (%u MB) exceed the recommended limits for '%s' (total memory: %u MB, recommended max percentage : %.2f%%)", (unsigned)(totalRequirements / 0x100000), context, maxMB, maxPercentage);
+        }
+    }
+    memorySpecifications["recommendedMaxMemory"] = recommendedMaxMemory;
+}

common/thorhelper/hpccconfig.hpp

@@ -26,5 +26,8 @@
 
 extern THORHELPER_API bool getService(StringBuffer &serviceAddress, const char *serviceName, bool failIfNotFound);
 
+typedef std::function<bool(const char *prop, StringBuffer &result)> GetJobValueFunction;
+extern THORHELPER_API void getMemorySpecifications(std::unordered_map<std::string, __uint64> &memorySpecifications, const IPropertyTree *config, const char *context, unsigned maxMB, GetJobValueFunction getJobValueFunction);
+
 #endif // _HPCCCONFIG_HPP_
 

dali/base/dacsds.cpp

@@ -948,6 +948,18 @@ void CClientRemoteTree::setPropInt64(const char *xpath, __int64 val)
     CRemoteTreeBase::setPropInt64(xpath, val);
 }
 
+void CClientRemoteTree::addPropReal(const char *xpath, double val)
+{
+    CConnectionLock b(connection);
+    CRemoteTreeBase::addPropReal(xpath, val);
+}
+
+void CClientRemoteTree::setPropReal(const char *xpath, double val)
+{
+    CConnectionLock b(connection);
+    CRemoteTreeBase::setPropReal(xpath, val);
+}
+
 void CClientRemoteTree::setPropBin(const char *xpath, size32_t size, const void *data)
 {
     CConnectionLock b(connection);

dali/base/dacsds.ipp

@@ -342,6 +342,8 @@ public:
     virtual void setProp(const char *xpath, const char *val) override;
     virtual void addPropInt64(const char *xpath, __int64 val) override;
     virtual void setPropInt64(const char *xpath, __int64 val) override;
+    virtual void addPropReal(const char *xpath, double val) override;
+    virtual void setPropReal(const char *xpath, double val) override;
     virtual void setPropBin(const char *xpath, size32_t size, const void *data) override;
     virtual IPropertyTree *setPropTree(const char *xpath, IPropertyTree *val) override;
     virtual IPropertyTree *addPropTree(const char *xpath, IPropertyTree *val) override;

ecl/eclagent/eclagent.cpp

@@ -53,6 +53,7 @@
 #include "roxiehelper.hpp"
 #include "jlzw.hpp"
 #include "anawu.hpp"
+#include "hpccconfig.hpp"
 
 using roxiemem::OwnedRoxieString;
 
@@ -2112,16 +2113,6 @@ void EclAgent::runProcess(IEclProcess *process)
     assertex(rowManager==NULL);
     allocatorMetaCache.setown(createRowAllocatorCache(this));
 
-    //Get memory limit. Workunit specified value takes precedence over config file
-    int memLimitMB = agentTopology->getPropInt("@defaultMemoryLimitMB", DEFAULT_MEM_LIMIT);
-    memLimitMB = queryWorkUnit()->getDebugValueInt("hthorMemoryLimit", memLimitMB);
-
-    bool allowHugePages = agentTopology->getPropBool("@heapUseHugePages", false);
-
-    bool allowTransparentHugePages = agentTopology->getPropBool("@heapUseTransparentHugePages", true);
-
-    bool retainMemory = agentTopology->getPropBool("@heapRetainMemory", false);
-
     if (agentTopology->hasProp("@httpGlobalIdHeader"))
         updateDummyContextLogger().setHttpIdHeaders(agentTopology->queryProp("@httpGlobalIdHeader"), agentTopology->queryProp("@httpCallerIdHeader"));
 
@@ -2150,6 +2141,32 @@ void EclAgent::runProcess(IEclProcess *process)
         }
     }
 
+    // a component may specify an alternate name for the agent/workflow memory area,
+    // e.g. Thor specifies in "eclAgentMemory"
+    const char *jobMemorySectionName = agentTopology->queryProp("@jobMemorySectionName");
+    if (isEmptyString(jobMemorySectionName))
+        jobMemorySectionName = "jobMemory";
+    //Get memory limit. Workunit specified value takes precedence over config
+    unsigned memLimitMB = queryWorkUnit()->getDebugValueInt("hthorMemoryLimit", 0);
+    if (0 == memLimitMB)
+    {
+        if (isContainerized())
+        {
+            const char *resourcedMemory = agentTopology->queryProp("resources/@memory");
+            if (!isEmptyString(resourcedMemory))
+            {
+                offset_t sizeBytes = friendlyStringToSize(resourcedMemory);
+                memLimitMB = (unsigned)(sizeBytes / 0x100000);
+            }
+            else
+                memLimitMB = DEFAULT_MEM_LIMIT;
+            static constexpr float defaultPctSysMemForRoxie = 90.0;
+            if (!agentTopology->hasProp(VStringBuffer("%s/@maxMemPercentage", jobMemorySectionName)))
+                ensurePTree(agentTopology, jobMemorySectionName)->setPropReal("@maxMemPercentage", defaultPctSysMemForRoxie);
+        }
+        else
+            memLimitMB = (unsigned)agentTopology->getPropInt("@defaultMemoryLimitMB", DEFAULT_MEM_LIMIT);
+    }
 #ifndef __64BIT__
     if (memLimitMB > 4096)
     {
@@ -2158,12 +2175,64 @@ void EclAgent::runProcess(IEclProcess *process)
         fail(0, errmsg.str());
     }
 #endif
-    memsize_t memLimitBytes = (memsize_t)memLimitMB * 1024 * 1024;
+
+    auto getWorkUnitValueFunc = [this](const char *prop, StringBuffer &result)
+    {
+        SCMStringBuffer value;
+        queryWorkUnit()->getDebugValue(prop, value);
+        if (0 == value.length())
+            return false;
+        result.append(value.str());
+        return true;
+    };
+    std::unordered_map<std::string, __uint64> memorySpecifications;
+    getMemorySpecifications(memorySpecifications, agentTopology, jobMemorySectionName, memLimitMB, getWorkUnitValueFunc);
+
+    unsigned queryMemoryMB = (unsigned)(memorySpecifications["query"] / 0x100000);
+
+    if (0 == queryMemoryMB)
+    {
+        unsigned totalRequirementsMB = (unsigned)(memorySpecifications["total"] / 0x100000);
+        unsigned recommendedMaxMB = (unsigned)(memorySpecifications["recommendedMaxMemory"] / 0x100000);
+        unsigned remainingMemoryMB = recommendedMaxMB - totalRequirementsMB;
+        if (agentTopology->getPropBool("@useChildProcesses"))
+        {
+            /* If using child proceses and there is no per query specification of memory usage
+             * divide up the available memory between the maxAcive instances
+             * if < than the historical default, use that and hence over commit.
+             */
+            unsigned maxActive = agentTopology->getPropInt("@maxActive");
+            queryMemoryMB = remainingMemoryMB / maxActive;
+            if (queryMemoryMB < DEFAULT_MEM_LIMIT)
+                queryMemoryMB = DEFAULT_MEM_LIMIT;
+        }
+        else
+            queryMemoryMB = remainingMemoryMB;
+    }
+
+    // a simple helper used below, to fetch bool from workunit, or 'jobMemory/' or legacy location
+    auto getBoolSetting = [&](const char *setting, bool defaultValue)
+    {
+        VStringBuffer attrSetting("@%s", setting);
+        return queryWorkUnit()->getDebugValueBool(setting,
+            agentTopology->getPropBool(VStringBuffer("jobMemory/%s", attrSetting.str()),
+            agentTopology->getPropBool(attrSetting,
+            defaultValue
+                                 )));
+    };
+
+    bool allowHugePages = getBoolSetting("heapUseHugePages", false);
+    bool allowTransparentHugePages = getBoolSetting("heapUseTransparentHugePages", true);
+    bool retainMemory = getBoolSetting("heapRetainMemory", false);
+
+    memsize_t memLimitBytes = (memsize_t)queryMemoryMB * 1024 * 1024;
     roxiemem::setTotalMemoryLimit(allowHugePages, allowTransparentHugePages, retainMemory, memLimitBytes, 0, NULL, NULL);
 
+    PROGLOG("Total memory = %u MB, query memory = %u MB", memLimitMB, queryMemoryMB);
+
     rowManager.setown(roxiemem::createRowManager(0, NULL, queryDummyContextLogger(), allocatorMetaCache, false));
     setHThorRowManager(rowManager.get());
-    rowManager->setActivityTracking(queryWorkUnit()->getDebugValueBool("traceRoxiePeakMemory", false));
+    rowManager->setActivityTracking(getBoolSetting("traceRoxiePeakMemory", false));
 
     if (debugContext)
         debugContext->checkBreakpoint(DebugStateReady, NULL, NULL);

helm/hpcc/templates/thor.yaml

@@ -30,7 +30,8 @@ Pass in dict with root and me
 {{- $hthorName := printf "%s-%s" .me.name $eclAgentType }}
 {{- $eclAgentScope := dict "name" .eclAgentName "type" $eclAgentType "useChildProcesses" .eclAgentUseChildProcesses "replicas" .eclAgentReplicas "maxActive" .me.maxJobs | merge (pick .me "keepJobs") }}
 {{- $thorAgentScope := dict "name" .thorAgentName "replicas" .thorAgentReplicas "maxActive" .me.maxGraphs | merge (pick .me "keepJobs") }}
-{{- $hthorScope := dict "name" $hthorName | merge (pick .me "multiJobLinger") }}
+{{- $eclAgentResources := .me.eclAgentResources | default dict -}}
+{{- $hthorScope := dict "name" $hthorName "jobMemorySectionName" "eclAgentMemory" | merge (pick .me "multiJobLinger") | merge (dict "resources" (deepCopy $eclAgentResources)) }}
 {{- $thorScope := omit .me "eclagent" "thoragent" "hthor" "logging" "env" "eclAgentResources" "eclAgentUseChildProcesses" "eclAgentReplicas" "thorAgentReplicas" "eclAgentType" }}
 {{- $misc := .root.Values.global.misc | default dict }}
 {{- $postJobCommand := $misc.postJobCommand | default "" }}

helm/hpcc/values.schema.json

@@ -499,9 +499,13 @@
           "type": "string",
           "description": "The amount of overall resourced memory to dedicate to the query"
         },
-        "thirdParties": {
+        "thirdParty": {
           "type": "string",
           "description": "The amount of overall resource memory to reserve for 3rd party use"
+        },
+        "maxMemPercentage": {
+          "type": "number",
+          "description": "The default maximum percentage of resource memory to dedicate to HPCC"
         }
       },
       "additionalProperties": false
@@ -1022,6 +1026,9 @@
         },
         "resources": {
           "$ref": "#/definitions/resources"
+        },
+        "jobMemory": {
+          "$ref": "#/definitions/memory"
         }
       }
     },
@@ -1255,6 +1262,9 @@
           "type": "object",
           "additionalProperties": { "type": "string" }
         },
+        "eclAgentMemory": {
+          "$ref": "#/definitions/memory"
+        },
         "workerMemory": {
           "$ref": "#/definitions/memory"
         },

system/jlib/jptree.cpp

@@ -1772,6 +1772,67 @@ void PTree::addPropInt64(const char *xpath, __int64 val)
     }
 }
 
+void PTree::setPropReal(const char * xpath, double val)
+{
+    if (!xpath || '\0' == *xpath)
+    {
+        std::string s = std::to_string(val);
+        setLocal((size32_t)s.length()+1, s.c_str());
+    }
+    else if (isAttribute(xpath))
+    {
+        std::string s = std::to_string(val);
+        setAttribute(xpath, s.c_str(), false);
+    }
+    else
+    {
+        const char *prop;
+        IPropertyTree *branch = splitBranchProp(xpath, prop, true);
+
+        if (isAttribute(prop))
+            branch->setPropReal(prop, val);
+        else
+        {
+            IPropertyTree *propBranch = queryCreateBranch(branch, prop);
+            propBranch->setPropReal(NULL, val);
+        }
+    }
+}
+
+void PTree::addPropReal(const char *xpath, double val)
+{
+    if (!xpath || '\0' == *xpath)
+    {
+        std::string s = std::to_string(val);
+        addLocal((size32_t)s.length()+1, s.c_str());
+    }
+    else if (isAttribute(xpath))
+    {
+        std::string s = std::to_string(val);
+        setAttribute(xpath, s.c_str(), false);
+    }
+    else if ('[' == *xpath)
+    {
+        std::string s = std::to_string(val);
+        aindex_t pos = getChildMatchPos(xpath);
+        if ((aindex_t) -1 == pos)
+            throw MakeIPTException(-1, "addPropInt64: qualifier unmatched %s", xpath);
+        addLocal((size32_t)s.length()+1, s.c_str(), false, pos);
+    }
+    else
+    {
+        IPropertyTree *parent, *child;
+        StringAttr path, qualifier;
+        resolveParentChild(xpath, parent, child, path, qualifier);
+        if (parent != this)
+            parent->addPropReal(path, val);
+        else if (child)
+            child->addPropReal(qualifier, val);
+        else
+            setPropReal(path, val);
+    }
+}
+
 int PTree::getPropInt(const char *xpath, int dft) const
 {
     return (int) getPropInt64(xpath, dft); // underlying type always __int64 (now)

system/jlib/jptree.hpp

@@ -103,6 +103,8 @@ interface jlib_decl IPropertyTree : extends serializable
     virtual void setPropInt64(const char *xpath, __int64 val) = 0;
     virtual void addPropInt64(const char *xpath, __int64 val) = 0;
 
+    virtual void setPropReal(const char *xpath, double val) = 0;
+    virtual void addPropReal(const char *xpath, double val) = 0;
     virtual double getPropReal(const char *xpath, double dft=0.0) const = 0;
 
     virtual bool getPropBin(const char *xpath, MemoryBuffer &ret) const = 0;

system/jlib/jptree.ipp

@@ -672,6 +672,8 @@ public:
     virtual __int64 getPropInt64(const char *xpath, __int64 dft=0) const override;
     virtual void setPropInt64(const char * xpath, __int64 val) override;
     virtual void addPropInt64(const char *xpath, __int64 val) override;
+    virtual void setPropReal(const char * xpath, double val) override;
+    virtual void addPropReal(const char *xpath, double val) override;
     virtual int getPropInt(const char *xpath, int dft=0) const override;
     virtual void setPropInt(const char *xpath, int val) override;
     virtual void addPropInt(const char *xpath, int val) override;

thorlcr/graph/thgraph.cpp

@@ -29,6 +29,7 @@
 #include "thorsoapcall.hpp"
 #include "thorport.hpp"
 #include "roxiehelper.hpp"
+#include "hpccconfig.hpp"
 
 
 
@@ -2740,59 +2741,38 @@ CJobBase::CJobBase(ILoadedDllEntry *_querySo, const char *_graphName) : querySo(
         throwUnexpected();
 }
 
-void CJobBase::applyMemorySettings(float recommendedMaxPercentage, const char *context)
+void CJobBase::applyMemorySettings(const char *context)
 {
     // NB: 'total' memory has been calculated in advance from either resource settings or from system memory.
-    VStringBuffer totalMemorySetting("%sMemory/@total", context);
-    unsigned totalMemoryMB = globals->getPropInt(totalMemorySetting);
-
-    unsigned recommendedMaxMemoryMB = totalMemoryMB * recommendedMaxPercentage / 100;
-#ifdef _CONTAINERIZED
-    /* only "query" memory is actually used (if set, configures Thor roxiemem limit)
-     * others are only advisory, but totalled and checked to ensure within the total limit.
-     */
-    std::initializer_list<const char *> memorySettings = { "query", "thirdParty" };
-    offset_t totalRequirements = 0;
-    for (auto setting : memorySettings)
-    {
-        VStringBuffer workunitSettingName("%smemory.%s", context, setting); // NB: workunit options are case insensitive
-        StringBuffer memString;
-        getWorkUnitValue(workunitSettingName, memString);
-        if (0 == memString.length())
-        {
-            VStringBuffer globalSettingName("%sMemory/@%s", context, setting);
-            globals->getProp(globalSettingName, memString);
-        }
-        if (memString.length())
-        {
-            offset_t memBytes = friendlyStringToSize(memString);
-            if (streq("query", setting))
-                queryMemoryMB = (unsigned)(memBytes / 0x100000);
-            totalRequirements += memBytes;
-        }
-    }
-    unsigned totalRequirementsMB = (unsigned)(totalRequirements / 0x100000);
-    if (totalRequirementsMB > totalMemoryMB)
-        throw makeStringExceptionV(0, "The total memory requirements of the query (%u MB) exceeds the %s memory limit (%u MB)", totalRequirementsMB, context, totalMemoryMB);
-
-    if (totalRequirementsMB > recommendedMaxMemoryMB)
-    {
-        WARNLOG("The total memory requirements of the query (%u MB) exceed the recommended reserve limits for %s (total memory: %u MB, recommended max percentage : %.2f%%)", totalRequirementsMB, context, totalMemoryMB, recommendedMaxPercentage);
-    
-        // if "query" memory has not been defined, then use the remaining memory
-        if (0 == queryMemoryMB)
-            queryMemoryMB = totalMemoryMB - totalRequirementsMB;
-    }
-    else if (0 == queryMemoryMB)
-        queryMemoryMB = recommendedMaxMemoryMB - totalRequirementsMB;
-#else
-    queryMemoryMB = recommendedMaxMemoryMB;
-#endif
-
-    bool gmemAllowHugePages = globals->getPropBool("@heapUseHugePages", false);
-    gmemAllowHugePages = globals->getPropBool("@heapMasterUseHugePages", gmemAllowHugePages);
-    bool gmemAllowTransparentHugePages = globals->getPropBool("@heapUseTransparentHugePages", true);
-    bool gmemRetainMemory = globals->getPropBool("@heapRetainMemory", false);
+    VStringBuffer memoryContext("%sMemory", context);
+    unsigned totalMemoryMB = globals->getPropInt(VStringBuffer("%s/@total", memoryContext.str()));
+    dbgassertex(totalMemoryMB);
+
+    auto getWorkUnitValueFunc = [this](const char *prop, StringBuffer &result) { getWorkUnitValue(prop, result); return result.length()>0;};
+    std::unordered_map<std::string, __uint64> memorySpecifications;
+    getMemorySpecifications(memorySpecifications, globals, memoryContext, totalMemoryMB, getWorkUnitValueFunc);
+    queryMemoryMB = (unsigned)(memorySpecifications["query"] / 0x100000);
+    if (0 == queryMemoryMB)
+    {
+        unsigned totalRequirementsMB = (unsigned)(memorySpecifications["total"] / 0x100000);
+        unsigned recommendedMaxMB = (unsigned)(memorySpecifications["recommendedMaxMemory"] / 0x100000);
+        queryMemoryMB = recommendedMaxMB - totalRequirementsMB;
+    }
+
+    // a simple helper used below, to fetch bool from workunit, or the memory settings (either managerMemory or workerMemory) or legacy location
+    auto getBoolSetting = [&](const char *setting, bool defaultValue)
+    {
+        VStringBuffer attrSetting("@%s", setting);
+        return getWorkUnitValueBool(setting,
+            globals->getPropBool(VStringBuffer("%s/%s", memoryContext.str(), attrSetting.str()),
+            globals->getPropBool(attrSetting,
+            defaultValue
+                                 )));
+    };
+    bool gmemAllowHugePages = getBoolSetting("heapUseHugePages", false);
+    gmemAllowHugePages = getBoolSetting("heapMasterUseHugePages", gmemAllowHugePages);
+    bool gmemAllowTransparentHugePages = getBoolSetting("heapUseTransparentHugePages", true);
+    bool gmemRetainMemory = getBoolSetting("heapRetainMemory", false);
     roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, gmemRetainMemory, ((memsize_t)queryMemoryMB) * 0x100000, 0, thorAllocSizes, NULL);
 
     PROGLOG("Total memory = %u MB, query memory = %u MB, memory spill at = %u", totalMemoryMB, queryMemoryMB, memorySpillAtPercentage);

thorlcr/graph/thgraph.hpp

@@ -922,7 +922,7 @@ public:
     virtual IGraphTempHandler *createTempHandler(bool errorOnMissing) = 0;
     void addDependencies(IPropertyTree *xgmml, bool failIfMissing=true);
     void addSubGraph(IPropertyTree &xgmml);
-    void applyMemorySettings(float recommendReservePercentage, const char *context);
+    void applyMemorySettings(const char *context);
 
     void checkAndReportLeaks(roxiemem::IRowManager *rowManager);
     bool queryUseCheckpoints() const;

thorlcr/graph/thgraphmaster.cpp

@@ -1347,13 +1347,7 @@ CJobMaster::CJobMaster(IConstWorkUnit &_workunit, const char *graphName, ILoaded
         loadPlugin(pluginMap, pluginsDir.str(), name.str());
     }
 
-    float recommendedMaxPercentage = defaultPctSysMemForRoxie;
-#ifndef _CONTAINERIZED
-    // @localThor mode - 25% is used for manager and 50% is used for workers
-    if (globals->getPropBool("@localThor") && (0 == globals->getPropInt("@masterMemorySize")))
-        recommendedMaxPercentage = 25.0;
-#endif
-    applyMemorySettings(recommendedMaxPercentage, "manager");
+    applyMemorySettings("manager");
     sharedAllocator.setown(::createThorAllocator(queryMemoryMB, 0, 1, memorySpillAtPercentage, *logctx, crcChecking, usePackedAllocator));
     Owned<IMPServer> mpServer = getMPServer();
     CJobChannel *channel = addChannel(mpServer);

thorlcr/graph/thgraphslave.cpp

@@ -1687,40 +1687,7 @@ CJobSlave::CJobSlave(ISlaveWatchdog *_watchdog, IPropertyTree *_workUnitInfo, co
     }
     tmpHandler.setown(createTempHandler(true));
 
-    /*
-     * Calculate maximum recommended memory for this worker.
-     * In container mode, there is only ever 1 worker per container,
-     * recommendedMaxPercentage = defaultPctSysMemForRoxie
-     * In bare-metal slavesPerNode is taken into account and @localThor if used.
-     * 
-     * recommendedMaxPercentage is used by applyMemorySettings to calculate the
-     * max amount of meemory that should be used (allowing enough left for heap/OS etc.)
-     */
-#ifdef _CONTAINERIZED
-    float recommendedMaxPercentage = defaultPctSysMemForRoxie;
-#else
-    // bare-metal only
-
-    float recommendedMaxPercentage = defaultPctSysMemForRoxie;
-    unsigned numWorkersPerNode = globals->getPropInt("@slavesPerNode", 1);
-
-    // @localThor mode - 25% is used for manager and 50% is used for workers
-    if (globals->getPropBool("@localThor") && (0 == globals->getPropInt("@masterMemorySize")))
-    {
-        /* In this mode, 25% is reserved for manager,
-         * 50% for the workers.
-         * Meaning this workers' recommendedMaxPercentage is remaining percentage */
-        float pctPerWorker = 50.0 / numWorkersPerNode;
-        recommendedMaxPercentage = pctPerWorker;
-    }
-    else
-    {
-        // deduct percentage for all other workers from max percentage
-        float pctPerWorker = defaultPctSysMemForRoxie / numWorkersPerNode;
-        recommendedMaxPercentage = pctPerWorker;
-    }
-#endif
-    applyMemorySettings(recommendedMaxPercentage, "worker");
+    applyMemorySettings("worker");
 
     unsigned sharedMemoryLimitPercentage = (unsigned)getWorkUnitValueInt("globalMemoryLimitPC", globals->getPropInt("@sharedMemoryLimit", 90));
     unsigned sharedMemoryMB = queryMemoryMB*sharedMemoryLimitPercentage/100;

thorlcr/master/thmastermain.cpp

@@ -732,6 +732,8 @@ int main( int argc, const char *argv[]  )
         }
 #endif
 
+        IPropertyTree *managerMemory = ensurePTree(globals, "managerMemory");
+        IPropertyTree *workerMemory = ensurePTree(globals, "workerMemory");
 
         HardwareInfo hdwInfo;
         getHardwareInfo(hdwInfo);
@@ -771,8 +773,20 @@ int main( int argc, const char *argv[]  )
 #endif
 #endif
             }
+
+#ifndef _CONTAINERIZED
+            // @localThor mode - 25% is used for manager and 50% is used for workers
+            // overrides recommended max percentage preferences if present
+            if (globals->getPropBool("@localThor") && (0 == mmemSize))
+            {
+                managerMemory->setProp("@maxMemPercentage", "25.0");
+                workerMemory->setPropReal("@maxMemPercentage", 50.0 / slavesPerNode);
+            }
+            else
+#endif
+                if (!workerMemory->hasProp("@maxMemPercentage"))
+                    workerMemory->setPropReal("@maxMemPercentage", defaultPctSysMemForRoxie);
         }
-        IPropertyTree *workerMemory = ensurePTree(globals, "workerMemory");
         workerMemory->setPropInt("@total", gmemSize);
 
         if (mmemSize)
@@ -791,9 +805,13 @@ int main( int argc, const char *argv[]  )
             }
             else
                 mmemSize = gmemSize; // default to same as slaves
+            if (!globals->hasProp("@globalMemorySize"))
+            {
+                if (!managerMemory->hasProp("@maxMemPercentage"))
+                    managerMemory->setProp("@maxMemPercentage", VStringBuffer("%.2f", defaultPctSysMemForRoxie));
+            }
         }
 
-        IPropertyTree *managerMemory = ensurePTree(globals, "managerMemory");
         managerMemory->setPropInt("@total", mmemSize);
 
         char thorPath[1024];