HPCC-23888 Add support for cost parameters in helm config

Signed-off-by: Shamser Ahmed <shamser.ahmed@lexisnexis.com>

common/workunit/workunit.cpp

@@ -13383,17 +13383,46 @@ extern WORKUNIT_API void addTimeStamp(IWorkUnit * wu, StatisticScopeType scopeTy
     wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scopestr, kind, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
 }
 
-extern WORKUNIT_API cost_type calculateThorCost(unsigned __int64 ms, unsigned clusterWidth)
+static double getCpuSize(const char *resourceName)
 {
-    IPropertyTree *costs = queryCostsConfiguration();
-    if (costs)
-    {
-        cost_type thor_master_rate = money2cost_type(costs->getPropReal("thor/@master"));
-        cost_type thor_slave_rate = money2cost_type(costs->getPropReal("thor/@slave"));
+    const char * cpuRequestedStr = queryComponentConfig().queryProp(resourceName);
+    if (!cpuRequestedStr)
+        return 0.0;
+    char * endptr;
+    double cpuRequested = strtod(cpuRequestedStr, &endptr);
+    if (cpuRequested < 0.0)
+        return 0.0;
+    if (*endptr == 'm')
+        cpuRequested /= 1000;
+    return cpuRequested;
+}
 
-        return calcCost(thor_master_rate, ms) + calcCost(thor_slave_rate, ms) * clusterWidth;
-    }
-    return 0;
+static double getCostCpuHour()
+{
+    double costCpuHour = queryGlobalConfig().getPropReal("cost/@perCpu");
+    if (costCpuHour < 0.0)
+        return 0.0;
+    return costCpuHour;
+}
+
+extern WORKUNIT_API double getMachineCostRate()
+{
+    return getCostCpuHour() * getCpuSize("resources/@cpu") ;
+};
+
+extern WORKUNIT_API double getThorManagerRate()
+{
+    return getCostCpuHour() * getCpuSize("managerResources/@cpu");
+}
+
+extern WORKUNIT_API double getThorWorkerRate()
+{
+    return getCostCpuHour() * getCpuSize("workerResources/@cpu");
+}
+
+extern WORKUNIT_API double calculateThorCost(unsigned __int64 ms, unsigned clusterWidth)
+{
+    return calcCost(getThorManagerRate(), ms) + calcCost(getThorWorkerRate(), ms) * clusterWidth;
 }
 
 void aggregateStatistic(StatsAggregation & result, IConstWorkUnit * wu, const WuScopeFilter & filter, StatisticKind search)

common/workunit/workunit.hpp

@@ -1714,7 +1714,10 @@ extern WORKUNIT_API const char * getWorkunitActionStr(WUAction action);
 extern WORKUNIT_API WUAction getWorkunitAction(const char * actionStr);
 
 extern WORKUNIT_API void addTimeStamp(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind, unsigned wfid=0);
-extern WORKUNIT_API cost_type calculateThorCost(unsigned __int64 ms, unsigned clusterWidth);
+extern WORKUNIT_API double getMachineCostRate();
+extern WORKUNIT_API double getThorManagerRate();
+extern WORKUNIT_API double getThorWorkerRate();
+extern WORKUNIT_API double calculateThorCost(unsigned __int64 ms, unsigned clusterWidth);
 
 extern WORKUNIT_API IPropertyTree * getWUGraphProgress(const char * wuid, bool readonly);
 
@@ -1743,7 +1746,7 @@ inline bool isGlobalScope(const char * scope) { return scope && (streq(scope, GL
 extern WORKUNIT_API bool isValidPriorityValue(const char * priority);
 extern WORKUNIT_API bool isValidMemoryValue(const char * memoryUnit);
 
-inline cost_type calcCost(cost_type ratePerHour, unsigned __int64 ms) { return ratePerHour * ms / 1000 / 3600; }
+inline double calcCost(double ratePerHour, unsigned __int64 ms) { return ratePerHour * ms / 1000 / 3600; }
 
 extern WORKUNIT_API void executeThorGraph(const char * graphName, IConstWorkUnit &workunit, const IPropertyTree &config);
 

ecl/eclagent/agentctx.hpp

@@ -122,7 +122,7 @@ struct IAgentContext : extends IGlobalCodeContext
     virtual void updateWULogfile(IWorkUnit *outputWU) = 0;
     virtual bool forceNewDiskReadActivity() const = 0;
     virtual void addWuExceptionEx(const char * text, unsigned code, unsigned severity, unsigned audience, char const * source) = 0;
-    virtual cost_type queryAgentMachineCost() const = 0;
+    virtual double queryAgentMachineCost() const = 0;
 };
 
 #endif // AGENTCTX_HPP_INCL

ecl/eclagent/eclagent.cpp

@@ -565,11 +565,11 @@ EclAgent::EclAgent(IConstWorkUnit *wu, const char *_wuid, bool _checkVersion, bo
             w->setXmlParams(_queryXML);
         updateSuppliedXmlParams(w);
     }
-    IPropertyTree *costs = queryCostsConfiguration();
-    if (costs)
+    agentMachineCost = getMachineCostRate();
+    if (agentMachineCost > 0.0)
     {
-        agentMachineCost = money2cost_type(costs->getPropReal("@agent"));
-        if (agentMachineCost)
+        IPropertyTree *costs = queryCostsConfiguration();
+        if (costs)
         {
             double softCostLimit = costs->getPropReal("@limit");
             double guillotineCost = wu->getDebugValueReal("maxCost", softCostLimit);
@@ -2371,7 +2371,7 @@ void EclAgentWorkflowMachine::noteTiming(unsigned wfid, timestamp_type startTime
     updateWorkunitStat(wu, SSTworkflow, scope, StWhenStarted, nullptr, startTime, 0);
     updateWorkunitStat(wu, SSTworkflow, scope, StTimeElapsed, nullptr, elapsedNs, 0);
 
-    const cost_type cost = calcCost(agent.queryAgentMachineCost(), nanoToMilli(elapsedNs)) + aggregateCost(wu, scope, true);
+    const cost_type cost = money2cost_type(calcCost(agent.queryAgentMachineCost(), nanoToMilli(elapsedNs))) + aggregateCost(wu, scope, true);
     if (cost)
         wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTworkflow, scope, StCostExecute, NULL, cost, 1, 0, StatsMergeReplace);
 }

ecl/eclagent/eclagent.ipp

@@ -247,7 +247,7 @@ public:
     {
         ctx->addWuExceptionEx(text, code, severity, audience, source);
     }
-    virtual cost_type queryAgentMachineCost() const override
+    virtual double queryAgentMachineCost() const override
     {
         return ctx->queryAgentMachineCost();
     };
@@ -397,7 +397,7 @@ private:
     StringAttr agentTempDir;
     Owned<IOrderedOutputSerializer> outputSerializer;
     int retcode;
-    cost_type agentMachineCost = 0;
+    double agentMachineCost = 0;
 
 private:
     void doSetResultString(type_t type, const char * stepname, unsigned sequence, int len, const char *val);
@@ -704,7 +704,7 @@ public:
     {
         return createRoxieRowAllocator(cache, *rowManager, meta, activityId, id, flags);
     }
-    virtual cost_type queryAgentMachineCost() const
+    virtual double queryAgentMachineCost() const
     {
         return agentMachineCost;
     }

ecl/eclagent/eclgraph.cpp

@@ -890,7 +890,7 @@ void EclSubGraph::updateProgress()
             {
                 unsigned __int64 elapsedTime = cycle_to_nanosec(elapsedGraphCycles);
                 parent.updateWUStatistic(lockedwu, SSTsubgraph, subgraphid, StTimeElapsed, nullptr, elapsedTime);
-                const cost_type cost = calcCost(agent->queryAgentMachineCost(), nanoToMilli(elapsedTime));
+                const cost_type cost = money2cost_type(calcCost(agent->queryAgentMachineCost(), nanoToMilli(elapsedTime)));
                 if (cost)
                 {
                     StringBuffer scope;
@@ -1268,7 +1268,7 @@ void EclGraph::execute(const byte * parentExtract)
             unsigned __int64 elapsedNs = milliToNano(elapsed);
             updateWorkunitStat(wu, SSTgraph, queryGraphName(), StTimeElapsed, description.str(), elapsedNs, wfid);
 
-            const cost_type cost = calcCost(agent->queryAgentMachineCost(), nanoToMilli(elapsedNs));
+            const cost_type cost = money2cost_type(calcCost(agent->queryAgentMachineCost(), elapsed));
             if (cost)
             {
                 StringBuffer scope;

ecl/eclcc/eclcc.cpp

@@ -1495,9 +1495,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     instance.stats.generateTime = (unsigned)nanoToMilli(totalTimeNs) - instance.stats.parseTime;
     updateWorkunitStat(instance.wu, SSTcompilestage, "compile", StTimeElapsed, NULL, totalTimeNs);
 
-    IPropertyTree *costs = queryCostsConfiguration();
-    const cost_type machineCost = costs ? money2cost_type(costs->getPropReal("@eclcc")) : 0;
-    const cost_type cost = calcCost(machineCost, nanoToMilli(totalTimeNs));
+    const cost_type cost = money2cost_type(calcCost(getMachineCostRate(), nanoToMilli(totalTimeNs)));
     if (cost)
         instance.wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StCostExecute, NULL, cost, 1, 0, StatsMergeReplace);
 

helm/hpcc/templates/_helpers.tpl

@@ -90,6 +90,10 @@ storage:
   - name: hpcc-spill-plane
     mount: {{ .Values.global.defaultSpillPath | default "/var/lib/HPCCSystems/hpcc-spill" | quote }}
 {{- end }}
+{{- if .Values.global.cost }}
+cost:
+{{ toYaml .Values.global.cost | indent 2 }}
+{{- end }}
 {{- end -}}
 
 {{/*

helm/hpcc/values.schema.json

@@ -165,7 +165,18 @@
         },
         "defaultMirrorPath": {
           "type": "string"
+        },
+        "cost": {
+          "description": "resource cost",
+          "type": "object",
+          "properties": {
+            "perCpu": {
+              "description": "cost of a single cpu",
+              "type": "number"
+            }
+          }
         }
+
       },
       "additionalProperties": false
     },

helm/hpcc/values.yaml

@@ -34,6 +34,9 @@ global:
     # kubeApiCidr: 172.17.0.3/32  
     # kubeApiPort: 7443
 
+  #cost:
+  #  perCpu: 0.126
+
 ## storage:
 ##
 ## If storage.[type].existingClaim is defined, a Persistent Volume Claim must

thorlcr/master/thdemonserver.cpp

@@ -38,8 +38,8 @@ private:
     unsigned reportRate;
     CIArrayOf<CGraphBase> activeGraphs;
     UnsignedArray graphStarts;
-    cost_type thorMasterCostRate = 0;
-    cost_type thorSlaveCostRate = 0;
+    double thorManagerRate = 0;
+    double thorWorkerRate = 0;
     cost_type costLimit = 0;
     cost_type workunitCost = 0;
 
@@ -91,12 +91,12 @@ private:
         if (costLimit || finished)
         {
             const unsigned clusterWidth = queryNodeClusterWidth();
-            const cost_type sgCost = calcCost(thorMasterCostRate, duration) + calcCost(thorSlaveCostRate, duration) * clusterWidth;
+            const cost_type sgCost = money2cost_type(calcCost(thorManagerRate, duration) + calcCost(thorWorkerRate, duration) * clusterWidth);
             if (finished)
                 wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTsubgraph, graphScope, StCostExecute, NULL, sgCost, 1, 0, StatsMergeReplace);
 
             const cost_type totalCost = workunitCost + sgCost;
-            if (totalCost > costLimit)
+            if (costLimit>0.0 && totalCost > costLimit)
             {
                 LOG(MCwarning, thorJob, "ABORT job cost exceeds limit");
                 graph.fireException(MakeThorException(TE_CostExceeded, "Job cost exceeds limit"));
@@ -198,12 +198,8 @@ public:
     {
         lastReport = msTick();
         reportRate = globals->getPropInt("@watchdogProgressInterval", 30);
-        IPropertyTree *costs = queryCostsConfiguration();
-        if (costs)
-        {
-            thorMasterCostRate = money2cost_type(costs->getPropReal("thor/@master"));
-            thorSlaveCostRate = money2cost_type(costs->getPropReal("thor/@slave"));
-        }
+        thorManagerRate = getThorManagerRate();
+        thorWorkerRate = getThorWorkerRate();
     }
 
     virtual void takeHeartBeat(MemoryBuffer &progressMb)

thorlcr/master/thgraphmanager.cpp

@@ -986,7 +986,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName,
         updateWorkunitStat(wu, SSTgraph, graphName, StTimeElapsed, graphTimeStr, graphTimeNs, wfid);
 
         addTimeStamp(wu, SSTgraph, graphName, StWhenFinished, wfid);
-        double cost = calculateThorCost(nanoToMilli(graphTimeNs), queryNodeClusterWidth());
+        cost_type cost = money2cost_type(calculateThorCost(nanoToMilli(graphTimeNs), queryNodeClusterWidth()));
         if (cost)
             wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTgraph, graphScope, StCostExecute, NULL, cost, 1, 0, StatsMergeReplace);
 

thorlcr/shared/thor.hpp

@@ -60,10 +60,6 @@ thor:
   daliServers: dali
   watchdogEnabled: true
   watchdogProgressEnabled: true
-  cost:
-    thor:
-      master: "0.000002"
-      slave: "0.00001"
 )!!";