Merge pull request #14874 from jakesmith/hpcc-25753-filter-configSHA

HPCC-25753 Avoid unnecessary pod restarts on configMap changes

Reviewed-by: Richard Chapman
Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>
Merged-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/workunit/workunit.cpp

@@ -14236,17 +14236,17 @@ bool applyK8sYaml(const char *componentName, const char *wuid, const char *job,
         E->Release();
         return false;
     }
-    jobYaml.replaceString("%jobname", jobname.str());
+    jobYaml.replaceString("_HPCC_JOBNAME_", jobname.str());
 
     VStringBuffer args("\"--workunit=%s\"", wuid);
     for (const auto &p: extraParams)
     {
-        if ('%' == p.first[0]) // jobspec substituion
+        if (hasPrefix(p.first.c_str(), "_HPCC_", false)) // jobspec substituion
             jobYaml.replaceString(p.first.c_str(), p.second.c_str());
         else
             args.append(" \"--").append(p.first.c_str()).append('=').append(p.second.c_str()).append("\"");
     }
-    jobYaml.replaceString("%args", args.str());
+    jobYaml.replaceString("_HPCC_ARGS_", args.str());
 
 // Disable ability change resources from within workunit
 // - all values are unquoted by toYAML.  This caused problems when previous string values are

dali/base/dafdesc.cpp

@@ -3306,8 +3306,19 @@ static void generateHosts(IPropertyTree * storage, GroupInfoArray & groups)
     }
 }
 
+
+static std::atomic<unsigned> normalizeHostGroupUpdateCBId{(unsigned)-1};
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    return true;
+}
+MODULE_EXIT()
+{
+    if ((unsigned)-1 != normalizeHostGroupUpdateCBId)
+        removeConfigUpdateHook(normalizeHostGroupUpdateCBId);
+}
 static CriticalSection storageCS;
-void initializeStorageGroups(bool createPlanesFromGroups)
+static void doInitializeStorageGroups(bool createPlanesFromGroups)
 {
     CriticalBlock block(storageCS);
     Owned<IPropertyTree> globalConfig = getGlobalConfig();
@@ -3418,6 +3429,21 @@ void initializeStorageGroups(bool createPlanesFromGroups)
     setupContainerizedStorageLocations();
 }
 
+void initializeStorageGroups(bool createPlanesFromGroups)
+{
+    doInitializeStorageGroups(createPlanesFromGroups);
+    unsigned uninitialized = (unsigned)-1;
+    if (normalizeHostGroupUpdateCBId.compare_exchange_strong(uninitialized, 0))
+    {
+        auto updateFunc = [createPlanesFromGroups](const IPropertyTree *oldComponentConfiguration, const IPropertyTree *oldGlobalConfiguration)
+        {
+            PROGLOG("initializeStorageGroups update");
+            doInitializeStorageGroups(createPlanesFromGroups);
+        };
+        normalizeHostGroupUpdateCBId = installConfigUpdateHook(updateFunc);
+    }
+}
+
 bool getDefaultStoragePlane(StringBuffer &ret)
 {
     // If the plane is specified for the component, then use that

helm/hpcc/templates/_helpers.tpl

@@ -826,7 +826,7 @@ apiVersion: v1
 metadata:
   name: {{ printf "%s-configmap" $configMapName }}
 data:
-  {{ $configMapName }}.yaml: |
+  {{ $configMapName }}.yaml:
     version: 1.0
     sasha:
 {{ toYaml (omit .me "logging") | indent 6 }}
@@ -1248,3 +1248,86 @@ Add a secret volume for a roxie udp key
 {{ end -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+A template to filter out a set of keys from a generated config yaml.
+Used to regenerate a configmap without the exclusions, so that it can be
+used to form an SHA as an annotation in a pod.
+This means pods only auto-restart if the non-excluded parts change.
+
+Pass in root, me, configMapHelper, component, excludeSectionRegexList and excludeKeyList
+excludeSectionRegexList is a list of regexp's that filter out top-level sections, e.g. [".*spec.yaml$" ]
+excludeKeyList is a list of key values to exclude from each section, e.g. [ "global", "esp.services" "esp.queues"]
+
+The configMap data section is reconstructed based on filtering out matches.
+
+Used to exclude parts of the config which are always allowed to change without causing a pod restart.
+e.g. a cache of secrets, with an auto reload/refresh mechanism, or 'replicas'.
+*/}}
+{{- define "hpcc.filterConfig" }}
+{{- $config := fromYaml (include .configMapHelper .) -}}
+{{- $configCtx := dict -}}
+{{- $excludeSectionRegexList := .excludeSectionRegexList -}}
+{{- $excludeKeyList := .excludeKeyList -}}
+{{- range $configElementName, $configElementDict := $config.data -}}
+  {{- $_ := set $configCtx "excludeSection" false -}}
+  {{- range $regex := $excludeSectionRegexList -}}
+    {{- if (regexMatch $regex $configElementName) -}}
+      {{- $_ := set $configCtx "excludeSection" true -}}
+    {{- end -}}
+  {{- end -}}
+  {{- if not $configCtx.excludeSection -}}
+    {{- $configDictCtx := dict -}}
+    {{- range $key := $excludeKeyList -}}
+      {{- $_ := set $configDictCtx "keyDictStr" (regexReplaceAll "(.*)\\..*$" $key "${1}") -}}
+      {{- if eq $configDictCtx.keyDictStr $key -}}{{/* single component key, e.g. "global"*/}}
+        {{- $configElementDict := (unset $configElementDict $key) -}}
+      {{- else -}}{{/* scopes component key, e.g. "eclccserver.queue"*/}}
+        {{- $_ := set $configDictCtx "keyKeyStr" (regexReplaceAll ".*\\.(.*)$" $key "${1}") -}}
+        {{- $subDict := get $configElementDict $configDictCtx.keyDictStr -}}
+        {{- if $subDict -}}
+          {{- $_ := set $configElementDict $configDictCtx.keyDictStr (unset $subDict $configDictCtx.keyKeyStr) -}}
+        {{- end -}}
+      {{- end -}}
+    {{- end -}}{{/*range $key*/}}
+    {{- $configYaml := toYaml $configElementDict -}}
+    {{- $_ := set $config.data $configElementName $configYaml -}}
+  {{- else -}}
+    {{- $configData := (unset $config.data $configElementName) -}}
+    {{- $_ := set $config "data" $configData -}}
+  {{- end -}}
+{{- end -}}{{/*range $configElementName*/}}
+{{ toYaml $config }}
+{{- end -}}
+
+{{/*
+A template to generate a component config
+Pass in root, me, configMapHelper
+*/}}
+{{- define "hpcc.generateConfig" }}
+{{- $config := fromYaml (include .configMapHelper .) -}}
+{{- range $configElementName, $configElementDict := $config.data -}}
+  {{- $configYaml := toYaml $configElementDict -}}
+  {{- $_ := set $config.data $configElementName $configYaml -}}
+{{- end }}
+{{ toYaml $config }}
+{{- end -}}
+
+{{/*
+A template to generate an SHA from a component config, to be used to annotate a Deployment,
+such that it will auto restart if the SHA changes.
+Uses filterConfig helper to select pertinent parts of the config to be part of the SHA.
+Pass in root, me, configMapHelper, component and excludeKeys
+excludeKeys is a comma separated list of key values to exclude from each section, e.g. "global,esp.services,esp.queues"
+
+globalExcludeSectionRegexList below is hard-coded list of section regexp's to exclude.
+globalExcludeList below is a hard-coded list of global keys to exclude.
+
+*/}}
+{{- define "hpcc.getConfigSHA" }}
+{{- $globalExcludeList := list (printf "%s.replicas" .component) -}}
+{{- $globalExcludeSectionRegexList := list ".*spec.yaml$" -}}
+{{- $combinedExcludeKeyList := concat (splitList "," (.excludeKeys | default "")) $globalExcludeList -}}
+{{- $ctx := merge (omit . "excludeKeys") (dict "excludeSectionRegexList" $globalExcludeSectionRegexList "excludeKeyList" $combinedExcludeKeyList) -}}
+{{- include "hpcc.filterConfig" $ctx | sha256sum }}
+{{- end -}}

helm/hpcc/templates/dali.yaml

@@ -102,9 +102,9 @@ data:
 {{- if not $sasha.disabled -}}
 {{- $_ := set $sasha "name" $sashaName -}}
 kind: ConfigMap
-{{- with ($sasha | merge (pick $dali "logging") | merge (dict "inDaliPod" true "access" (splitList " " (include "hpcc.getSashaServiceAccess" $sasha)))) }}
-{{- $sashaSecretsCategories := append ((or (has "data" .access) (has "dalidata" .access)) | ternary (list "storage") list) "system" }}
-{{ include "hpcc.sashaConfigMap" (dict "root" $ "me" . "secretsCategories" $sashaSecretsCategories ) }}
+{{- with ($sasha | merge (pick $dali "logging") | merge (dict "inDaliPod" true "access" (splitList " " (include "hpcc.getSashaServiceAccess" $sasha)))) -}}
+{{- $sashaSecretsCategories := append ((or (has "data" .access) (has "dalidata" .access)) | ternary (list "storage") list) "system" -}}
+{{ include "hpcc.generateConfig" (dict "root" $ "me" . "secretsCategories" $sashaSecretsCategories "configMapHelper" "hpcc.sashaConfigMap") }}
 {{- end }}
 ---
 {{- if $sasha.servicePort -}}

helm/hpcc/templates/dfuserver.yaml

@@ -3,7 +3,7 @@ apiVersion: v1
 metadata:
   name: {{ .me.name }}-configmap
 data:
-  {{ .me.name }}.yaml: |
+  {{ .me.name }}.yaml:
     version: 1.0
     dfuserver:
 {{ toYaml (omit .me "logging") | indent 6 }}
@@ -16,7 +16,7 @@ data:
 {{- if not .disabled -}}
 {{- $secretsCategories := list "system" "storage" -}}
 {{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories "includeLabels" (list "lz" "data" "") }}
-{{- $configSHA := include "hpcc.dfuServerConfigMap" $commonCtx | sha256sum }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.dfuServerConfigMap" "component" "dfuserver" "excludeKeys" "global")) -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -57,7 +57,7 @@ spec:
 {{- include "hpcc.addVolumes" $commonCtx | indent 6 }}
 ---
 kind: ConfigMap
-{{ include "hpcc.dfuServerConfigMap" $commonCtx }}
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.dfuServerConfigMap")) }}
 ---
 {{- end }}
 {{- end }}

helm/hpcc/templates/eclagent.yaml

@@ -4,12 +4,12 @@ Pass in dict with root and me
 */}}
 {{- define "hpcc.agentConfigMap" }}
 {{- $apptype := .me.type | default "hthor" -}}
-{{- $appJobName := printf "%s-%%jobname" $apptype }}
+{{- $appJobName := printf "%s-_HPCC_JOBNAME_" $apptype }}
 apiVersion: v1
 metadata:
   name: {{ .me.name }}-configmap
 data:
-  {{ .me.name }}.yaml: |
+  {{ .me.name }}.yaml:
     version: 1.0
     eclagent:
 {{ toYaml (omit .me "logging") | indent 6 }}
@@ -22,7 +22,7 @@ data:
 {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
 {{- if not .me.useChildProcesses }}
 {{- $misc := .root.Values.global.misc | default dict }}
-  {{ $apptype }}-jobspec.yaml: |
+  {{ $apptype }}-jobspec.yaml:
     apiVersion: batch/v1
     kind: Job
     metadata:
@@ -48,7 +48,7 @@ data:
 {{- include "hpcc.addSecurityContext" . | indent 12 }}
 {{ include "hpcc.addImageAttrs" . | indent 12 }}
 {{- include "hpcc.addResources" (dict "me" .me.resources) | indent 12 }}
-{{- $appCmd := printf "%s %s %s %%args" $apptype (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
+{{- $appCmd := printf "%s %s %s _HPCC_ARGS_" $apptype (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
 {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $appCmd)) | indent 12 }}
             workingDir: /var/lib/HPCCSystems
             volumeMounts:
@@ -56,7 +56,7 @@ data:
 {{ include "hpcc.addDataVolumeMount" . | indent 12 }}
 {{ include "hpcc.addDllVolumeMount" . | indent 12 }}
 {{ include "hpcc.addSecretVolumeMounts" . | indent 12 }}
-{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" .me.type) | indent 12 }}
+{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" $apptype) | indent 12 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolumeMount" . | indent 12 }}
 {{- end }}
@@ -65,7 +65,7 @@ data:
 {{ include "hpcc.addDataVolume" . | indent 10 }}
 {{ include "hpcc.addDllVolume" . | indent 10 }}
 {{ include "hpcc.addSecretVolumes" . | indent 10 }}
-{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" .me.type) | indent 10 }}
+{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" $apptype) | indent 10 }}
 {{- if $misc.postJobCommandViaSidecar }}
 {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }}
 {{- end }}
@@ -76,9 +76,10 @@ data:
 
 {{ range $.Values.eclagent -}}
 {{- if not .disabled -}}
+{{- $apptype := .type | default "hthor" -}}
 {{- $secretsCategories := list "system" "ecl-user" "ecl" "storage" }}
 {{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories }}
-{{- $configSHA := include "hpcc.agentConfigMap" $commonCtx | sha256sum }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.agentConfigMap" "component" "eclagent" "excludeKeys" (print "global," $apptype ".replicas"))) }}
 {{- include "hpcc.checkDefaultStoragePlane" $commonCtx }}
 apiVersion: apps/v1
 kind: Deployment
@@ -132,10 +133,10 @@ spec:
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "eclagent") | indent 6 }}
 ---
 kind: ConfigMap 
-{{ include "hpcc.agentConfigMap" $commonCtx }}
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.agentConfigMap")) }}
 ---
 {{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "eclagent") }}
-{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" .type) }}
+{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" $apptype) }}
 
 {{- end }}
 {{- end }}

helm/hpcc/templates/eclccserver.yaml

@@ -3,12 +3,12 @@ EclccServer configmap
 Pass in dict with root and me
 */}}
 {{- define "hpcc.eclccServerConfigMap" -}}
-{{- $compileJobName := printf "compile-%%jobname" }}
+{{- $compileJobName := printf "compile-_HPCC_JOBNAME_" }}
 apiVersion: v1
 metadata:
   name: {{ .me.name }}-configmap
 data:
-  {{ .me.name }}.yaml: |
+  {{ .me.name }}.yaml:
     version: 1.0
     eclccserver:
 {{ toYaml (omit .me "logging") | indent 6 }}
@@ -21,7 +21,7 @@ data:
 {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
 {{- if not .me.useChildProcesses }}
 {{- $misc := .root.Values.global.misc | default dict }}
-  compile-jobspec.yaml: |
+  compile-jobspec.yaml:
     apiVersion: batch/v1
     kind: Job
     metadata:
@@ -49,7 +49,7 @@ data:
 {{ include "hpcc.addImageAttrs" . | indent 12 }}
 {{- $misc := .root.Values.global.misc | default dict -}}
 {{- $postJobCommand := $misc.postJobCommand | default "" }}
-{{- $eclccserverCmd := printf "eclccserver %s %s %%args" (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
+{{- $eclccserverCmd := printf "eclccserver %s %s _HPCC_ARGS_" (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
 {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $eclccserverCmd)) | indent 12 }}
             workingDir: /tmp
             volumeMounts:
@@ -80,7 +80,7 @@ data:
 {{- if not .disabled -}}
 {{- $secretsCategories := list "system" "codeVerify" }}
 {{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories }}
-{{- $configSHA := include "hpcc.eclccServerConfigMap" $commonCtx | sha256sum }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.eclccServerConfigMap" "component" "eclccserver" "excludeKeys" "global")) }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -139,8 +139,7 @@ spec:
         emptyDir: {}
 ---
 kind: ConfigMap 
-{{ include "hpcc.eclccServerConfigMap" $commonCtx }}
-
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.eclccServerConfigMap")) }}
 ---
 {{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "eclccserver") }}
 {{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "compile") }}

helm/hpcc/templates/eclscheduler.yaml

@@ -7,7 +7,7 @@ apiVersion: v1
 metadata:
   name: {{ .me.name }}-configmap
 data:
-  {{ .me.name }}.yaml: |
+  {{ .me.name }}.yaml:
     version: 1.0
     eclscheduler:
 {{ toYaml (omit .me "logging") | indent 6 }}
@@ -24,7 +24,7 @@ data:
 {{- if not .disabled -}}
 {{- $secretsCategories := list "system" }}
 {{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories }}
-{{- $configSHA := include "hpcc.eclSchedulerConfigMap" $commonCtx | sha256sum }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.eclSchedulerConfigMap" "component" "eclscheduler" "excludeKeys" "global")) -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -76,8 +76,7 @@ spec:
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "eclscheduler") | indent 6 }}
 ---
 kind: ConfigMap
-{{ include "hpcc.eclSchedulerConfigMap" $commonCtx }}
-
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.eclSchedulerConfigMap")) }}
 ---
 
 {{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "eclscheduler") }}

helm/hpcc/templates/esp.yaml

@@ -7,7 +7,7 @@ apiVersion: v1
 metadata:
   name: {{ .me.name }}-configmap
 data:
-  {{ .me.name }}.yaml: |
+  {{ .me.name }}.yaml:
     version: 1.0
     esp:
 {{ toYaml (omit .me "logging" "metrics") | indent 6 }}
@@ -52,7 +52,7 @@ data:
 {{- $secretsCategories := ternary (list "storage" "esp" "codeSign" "codeVerify")  (list "storage" "esp") (eq $application "eclwatch") -}}
 {{- $includeStorageLabels := ternary (list "lz" "data" "") (list "data" "") (eq $application "eclwatch") -}}
 {{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories  "includeLabels" $includeStorageLabels -}}
-{{- $configSHA := include "hpcc.espConfigMap" $commonCtx | sha256sum }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.espConfigMap" "component" "esp" "excludeKeys" "global,esp.services,esp.queues")) }}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -101,8 +101,7 @@ spec:
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "certificate" .certificate "external" (and (hasKey . "public") .public)) | indent 6 }}
 ---
 kind: ConfigMap
-{{ include "hpcc.espConfigMap" $commonCtx }}
-
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.espConfigMap")) }}
 ---
 apiVersion: v1
 kind: Service

helm/hpcc/templates/localroxie.yaml

@@ -7,7 +7,7 @@ apiVersion: v1
 metadata:
   name: {{ .me.name }}-configmap
 data:
-  {{ .me.name }}.yaml: |
+  {{ .me.name }}.yaml:
     version: 1.0
     roxie:
 {{ toYaml (omit .me "logging") | indent 6 }}
@@ -21,7 +21,7 @@ data:
 {{- if not $roxie.disabled  -}}
 {{- $secretsCategories := list "system" "ecl-user" "ecl" "storage" }}
 {{- $commonCtx := dict "root" $ "me" $roxie "secretsCategories" $secretsCategories }}
-{{- $configSHA := include "hpcc.localroxieConfigMap" $commonCtx | sha256sum }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.localroxieConfigMap" "component" "roxie" "excludeKeys" "global")) }}
 {{- include "hpcc.checkDefaultStoragePlane" $commonCtx }}
 {{- if $roxie.localAgent -}}
 {{- $name := $roxie.name -}}
@@ -112,7 +112,7 @@ spec:
 {{- end }}
 {{- end }}
 kind: ConfigMap 
-{{ include "hpcc.localroxieConfigMap" $commonCtx }}
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.localroxieConfigMap")) }}
 ---
 {{ include "hpcc.addCertificate" (dict "root" $ "name" $roxie.name "services" $roxie.services "component" "localroxie" "external" false) }}
 {{ include "hpcc.addCertificate" (dict "root" $ "name" $roxie.name "services" $roxie.services "component" "localroxie" "external" true) }}

helm/hpcc/templates/roxie.yaml

@@ -7,7 +7,7 @@ apiVersion: v1
 metadata:
   name: {{ .me.name }}-configmap
 data:
-  {{ .me.name }}.yaml: |
+  {{ .me.name }}.yaml:
     version: 1.0
     roxie:
 {{ toYaml ( omit .me "logging" "topoServer" "encryptInTransit") | indent 6 }}
@@ -37,7 +37,7 @@ apiVersion: v1
 metadata:
   name: {{ .toponame }}-configmap
 data:
-  {{ .toponame }}.yaml: |
+  {{ .toponame }}.yaml:
     version: 1.0
     toposerver:
       port: {{ .topoport }}
@@ -54,8 +54,8 @@ data:
 {{- $_ := set $commonCtx "numChannels" ($roxie.numChannels | int | default 1) -}}
 {{- $_ := set $commonCtx "topoport" ($toposerver.port | int | default 9004) -}}
 {{- $_ := set $toposerver "name" $commonCtx.toponame -}}
-{{- $configSHA := include "hpcc.roxieConfigMap" $commonCtx | sha256sum }}
-{{- $topoconfigSHA := include "hpcc.roxieTopoConfigMap" $commonCtx | sha256sum }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.roxieConfigMap" "component" "roxie" "excludeKeys" "global")) }}
+{{- $topoconfigSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.roxieTopoConfigMap" "component" "toposerver" "excludeKeys" "global")) }}
 {{- include "hpcc.checkDefaultStoragePlane" $commonCtx }}
 {{- if not $roxie.localAgent -}}
 {{- $_ := set $roxie "localAgent" false -}}
@@ -164,10 +164,10 @@ spec:
 
 ---
 kind: ConfigMap 
-{{ include "hpcc.roxieConfigMap" $commonCtx }}
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.roxieConfigMap")) }}
 ---
 kind: ConfigMap 
-{{ include "hpcc.roxieTopoConfigMap" $commonCtx }}
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.roxieTopoConfigMap")) }}
 ---
 
 {{ if $roxie.serverReplicas -}}

helm/hpcc/templates/sasha.yaml

@@ -7,7 +7,7 @@
 {{- $_ := set $sasha "access" (splitList " " (include "hpcc.getSashaServiceAccess" $sasha)) -}}
 {{- $secretsCategories := append ((or (has "data" $sasha.access) (has "dalidata" $sasha.access)) | ternary (list "storage") list) "system" -}}
 {{- $commonCtx := dict "root" $ "me" $sasha "secretsCategories" $secretsCategories -}}
-{{- $configSHA := include "hpcc.sashaConfigMap" $commonCtx | sha256sum -}}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.sashaConfigMap" "component" "sasha" "excludeKeys" "global")) -}}
 {{- $serviceName := printf "sasha-%s" $sashaName -}}
 apiVersion: apps/v1
 kind: Deployment
@@ -42,7 +42,7 @@ spec:
 ---
 {{- end }}
 kind: ConfigMap 
-{{ include "hpcc.sashaConfigMap" $commonCtx }}
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.sashaConfigMap")) }}
 ---
 {{- if $sasha.storage }}
 {{- if and (not $sasha.storage.existingClaim) (not $sasha.storage.plane) }}

helm/hpcc/templates/thor.yaml

@@ -11,14 +11,15 @@ Pass in dict with root and me
 {{- $thorScope := omit .me "eclagent" "thoragent" "hthor" "logging" "eclAgentResources" "eclAgentUseChildProcesses" "eclAgentReplicas" "thorAgentReplicas" "eclAgentType" }}
 {{- $misc := .root.Values.global.misc | default dict }}
 {{- $postJobCommand := $misc.postJobCommand | default "" }}
-{{- $eclAgentJobName := printf "%s-%%jobname" $eclAgentType }}
-{{- $thorManagerJobName := printf "thormanager-%%jobname" }}
-{{- $thorWorkerJobName := printf "thorworker-%%jobname" }}
+{{- $eclAgentJobName := printf "%s-_HPCC_JOBNAME_" $eclAgentType }}
+{{- $thorManagerJobName := printf "thormanager-_HPCC_JOBNAME_" }}
+{{- $thorWorkerJobName := printf "thorworker-_HPCC_JOBNAME_" }}
+{{- $thorNetworkPolicyName := printf "thornetworkpolicy-_HPCC_JOBNAME_" }}
 apiVersion: v1 
 metadata:
   name: {{ $thorScope.name }}-configmap
 data:
-  {{ $thorScope.name }}.yaml: |
+  {{ $thorScope.name }}.yaml:
     version: 1.0
     thor:
 {{ toYaml $thorScope | indent 6 }}
@@ -42,7 +43,7 @@ data:
 {{ include "hpcc.generateGlobalConfigMap" .root| indent 6 }}
 
 {{- if not .eclAgentUseChildProcesses }}
-  {{ $eclAgentType }}-jobspec.yaml: |
+  {{ $eclAgentType }}-jobspec.yaml:
     apiVersion: batch/v1
     kind: Job
     metadata:
@@ -70,7 +71,7 @@ data:
 {{- include "hpcc.addSecurityContext" . | indent 12 }}
 {{ include "hpcc.addImageAttrs" . | indent 12 }}
 {{- include "hpcc.addResources" (dict "me" .eclAgentResources) | indent 12 }}
-{{- $agentCmd := printf "%s %s %s %%args" $eclAgentType (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
+{{- $agentCmd := printf "%s %s %s _HPCC_ARGS_" $eclAgentType (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
 {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $agentCmd)) | indent 12 }}
             workingDir: /var/lib/HPCCSystems
             volumeMounts:
@@ -95,7 +96,7 @@ data:
       backoffLimit: 0
 {{- end }}
 
-  thormanager-jobspec.yaml: |
+  thormanager-jobspec.yaml:
     apiVersion: batch/v1
     kind: Job
     metadata:
@@ -108,7 +109,7 @@ data:
             app: thor
             accessDali: "yes"
             accessEsp: "yes"
-            job: %jobname
+            job: "_HPCC_JOBNAME_"
         spec:
           {{- include "hpcc.placementsByJobTargetType" (dict "root" .root "job" $thorManagerJobName "target" .me.name "type" "thor") | indent 10 }}
           serviceAccountName: hpcc-agent
@@ -125,7 +126,7 @@ data:
 {{- include "hpcc.addSecurityContext" . | indent 12 }}
 {{ include "hpcc.addImageAttrs" . | indent 12 }}
 {{- include "hpcc.addResources" (dict "me" $thorScope.managerResources) | indent 12 }}
-{{- $thorManagerCmd := printf "thormaster_lcr %s %s %%args" (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
+{{- $thorManagerCmd := printf "thormaster_lcr %s %s _HPCC_ARGS_" (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
 {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $thorManagerCmd)) | indent 12 }}
             workingDir: /var/lib/HPCCSystems
             volumeMounts:
@@ -149,20 +150,20 @@ data:
           restartPolicy: Never
       backoffLimit: 0
 
-  thorworker-jobspec.yaml: |
+  thorworker-jobspec.yaml:
     apiVersion: batch/v1
     kind: Job
     metadata:
       name: {{ $thorWorkerJobName }}
     spec:
-      parallelism: %numWorkers
+      parallelism: _HPCC_NUM_WORKERS_
       ttlSecondsAfterFinished: 100
       template:
         metadata:
           labels:
             app: thor
             accessEsp: "true"
-            job: %jobname
+            job: "_HPCC_JOBNAME_"
         spec:
           {{- include "hpcc.placementsByJobTargetType" (dict "root" .root "job" $thorWorkerJobName "target" .me.name "type" "thor") | indent 10 }}
           serviceAccountName: hpcc-default
@@ -177,7 +178,7 @@ data:
 {{- include "hpcc.addSecurityContext" . | indent 12 }}
 {{ include "hpcc.addImageAttrs" . | indent 12 }}
 {{- include "hpcc.addResources" (dict "me" $thorScope.workerResources) | indent 12 }}
-{{- $thorWorkerCmd := printf "thorslave_lcr %s %s %%args" (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
+{{- $thorWorkerCmd := printf "thorslave_lcr %s %s _HPCC_ARGS_" (include "hpcc.configArg" .me) (include "hpcc.daliArg" .root) }}
 {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $thorWorkerCmd)) | indent 12 }}
             workingDir: /var/lib/HPCCSystems
             volumeMounts:
@@ -201,28 +202,28 @@ data:
           restartPolicy: Never
       backoffLimit: 0
 
-  thormanager-networkspec.yaml: |
+  thor-networkspec.yaml:
     apiVersion: networking.k8s.io/v1
     kind: NetworkPolicy
     metadata:
-      name: {{ $thorManagerJobName }}
+      name: {{ $thorNetworkPolicyName }}
     spec:
       podSelector:
         matchLabels:
           app: thor
-          job: %jobname
+          job: "_HPCC_JOBNAME_"
       ingress:
       - from:
         - podSelector:
             matchLabels:
               app: thor
-              job: %jobname
+              job: "_HPCC_JOBNAME_"
       egress:
       - to:
         - podSelector:
             matchLabels:
               app: thor
-              job: %jobname
+              job: "_HPCC_JOBNAME_"
 {{- end -}}
 
 {{- $local := dict "first" true }}
@@ -235,7 +236,7 @@ data:
 {{- $_ := set $commonCtx "eclAgentUseChildProcesses" (hasKey . "eclAgentUseChildProcesses" | ternary .eclAgentUseChildProcesses true) }}
 {{- $_ := set $commonCtx "eclAgentReplicas" (.eclAgentReplicas | default 1) }}
 {{- $_ := set $commonCtx "thorAgentReplicas" (.thorAgentReplicas | default 1) }}
-{{- $configSHA := include "hpcc.thorConfigMap" $commonCtx | sha256sum }}
+{{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.thorConfigMap" "component" "thor" "excludeKeys" "global")) }}
 {{- include "hpcc.checkDefaultStoragePlane" $commonCtx }}
 apiVersion: apps/v1
 kind: Deployment
@@ -335,7 +336,7 @@ spec:
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent") | indent 6 }}
 ---
 kind: ConfigMap
-{{ include "hpcc.thorConfigMap" $commonCtx }}
+{{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.thorConfigMap")) }}
 ---
 {{ include "hpcc.addCertificate" (dict "root" $ "name" $commonCtx.eclAgentName "component" "eclagent") }}
 {{ include "hpcc.addCertificate" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent") }}

thorlcr/master/thmastermain.cpp

@@ -949,7 +949,7 @@ int main( int argc, const char *argv[]  )
         StringBuffer myEp;
         queryMyNode()->endpoint().getUrlStr(myEp);
 
-        applyK8sYaml("thorworker", workunit, cloudJobName, "jobspec", { { "graphName", graphName}, { "master", myEp.str() }, { "%numWorkers", std::to_string(numWorkers)} }, false);
+        applyK8sYaml("thorworker", workunit, cloudJobName, "jobspec", { { "graphName", graphName}, { "master", myEp.str() }, { "_HPCC_NUM_WORKERS_", std::to_string(numWorkers)} }, false);
 #else
         StringBuffer thorEpStr;
         LOG(MCdebugProgress, thorJob, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getUrlStr(thorEpStr).str());