Merge pull request #15277 from jakesmith/hpcc-26406-stub-memory-scale

HPCC-26406 Scale stub memory resource by number of instances

Reviewed-By: Gavin Halliday <gavin.halliday@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

helm/hpcc/templates/_helpers.tpl

@@ -827,12 +827,13 @@ resources:
 
 {{/*
 Add resources object for stub pods
+Pass in dict with instances defined
 */}}
 {{- define "hpcc.addStubResources" }}
 resources:
   limits:
     cpu: "50m"
-    memory: "100M"
+    memory: {{ (printf "%dM" (mul .instances 100)) | quote }}
 {{- end -}}
 
 {{/*

helm/hpcc/templates/eclagent.yaml

@@ -163,7 +163,7 @@ spec:
 {{- if .useChildProcesses }}
 {{- include "hpcc.addResources" (dict "me" .resources) | indent 8 }}
 {{- else if not $.Values.global.privileged }}
-{{- include "hpcc.addStubResources" . | indent 8 }}
+{{- include "hpcc.addStubResources" (dict "instances" .maxActive) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:

helm/hpcc/templates/eclccserver.yaml

@@ -165,7 +165,7 @@ spec:
 {{- if .useChildProcesses }}
 {{- include "hpcc.addResources" (dict "me" .resources) | indent 8 }}
 {{- else if not $.Values.global.privileged }}
-{{- include "hpcc.addStubResources" . | indent 8 }}
+{{- include "hpcc.addStubResources" (dict "instances" .maxActive) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:

helm/hpcc/templates/eclscheduler.yaml

@@ -94,7 +94,7 @@ spec:
 {{ include "hpcc.addSentinelProbes" . | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
 {{- if not $.Values.global.privileged }}
-{{- include "hpcc.addStubResources" . | indent 8 }}
+{{- include "hpcc.addStubResources" (dict "instances" 1) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:

helm/hpcc/templates/roxie.yaml

@@ -121,7 +121,7 @@ spec:
 {{ include "hpcc.addSentinelProbes" $toposerver | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
 {{- if not $.Values.global.privileged }}
-{{- include "hpcc.addStubResources" . | indent 8 }}
+{{- include "hpcc.addStubResources" (dict "instances" 1) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         workingDir: /var/lib/HPCCSystems

helm/hpcc/templates/thor.yaml

@@ -342,7 +342,7 @@ spec:
 {{- if $commonCtx.eclAgentUseChildProcesses }}
 {{- include "hpcc.addResources" (dict "me" .eclAgentResources) | indent 8 }}
 {{- else if not $.Values.global.privileged }}
-{{- include "hpcc.addStubResources" . | indent 8 }}
+{{- include "hpcc.addStubResources" (dict "instances" .maxJobs) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
@@ -401,7 +401,7 @@ spec:
           value: "/tmp/{{ $commonCtx.thorAgentName }}.sentinel"
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
 {{- if not $.Values.global.privileged }}
-{{- include "hpcc.addStubResources" . | indent 8 }}
+{{- include "hpcc.addStubResources" (dict "instances" .maxGraphs) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:

helm/hpcc/values.schema.json

@@ -766,13 +766,19 @@
     },
     "eclccserver": {
       "type": "object",
-      "required": [ "name" ],
+      "required": [ "name", "maxActive" ],
       "additionalProperties": { "type": ["integer", "string", "boolean"] },
       "properties": {
         "name": {
           "type": "string",
           "description": "The name of the eclccserver process"
         },
+        "maxActive": {
+          "type": "integer",
+          "minimum": 1,
+          "description": "Maximum number of compile jobs that can be active simultaneously",
+          "default": 4
+        },
         "replicas": {
           "type": "integer"
         },
@@ -909,6 +915,7 @@
     "eclagent": {
       "type": "object",
       "additionalProperties": { "type": ["integer", "string", "boolean"] },
+      "required": [ "maxActive" ],
       "properties": {
         "replicas": {
           "type": "integer"

testing/helm/tests/env.yaml

@@ -38,6 +38,7 @@ dfuserver:
     
 eclagent:
 - name: eclagent
+  maxActive: 1
   env:
   - name: ealv1
     value: ealv1v
@@ -58,6 +59,7 @@ thor:
 
 eclccserver: 
 - name: eclccserver
+  maxActive: 1
   env:
   - name: esav1
     value: esav1v

testing/helm/tests/labels.yaml

@@ -44,6 +44,7 @@ dfuserver:
     
 eclagent:
 - name: eclagent
+  maxActive: 1
   annotations:
     eaa1: eaav1
   labels:
@@ -67,6 +68,7 @@ thor:
 
 eclccserver: 
 - name: eclccserver
+  maxActive: 1
   annotations:
     esa1: esav1
   labels:
@@ -89,4 +91,3 @@ dali:
     dl1: dlv1
   services: # internal house keeping services
     coalescer:
-