HPCC-23723 Reduce over-use of templates in helm charts

Further changes

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dockerfiles/hpcc/templates/_helpers.tpl

@@ -42,14 +42,14 @@ Translate a port list to a comma-separated list
  {{- end -}}
 {{- end -}}
 
-{{- /* Generate a ConfigMap for a component */ -}}
-{{- /* Pass in a dictionary with root, component and me defined */ -}}
-{{- define "hpcc.generateConfigMap" }}
-  global.yaml: |
-    version: "1.0"
-    global:
-      imageVersion: {{ .root.Values.global.image.version | quote }}
-      singleNode: {{ .root.Values.global.singleNode }}
+{{/*
+Generate global ConfigMap info
+
+Pass in root as .
+*/}}
+{{- define "hpcc.generateGlobalConfigMap" -}}
+imageVersion: {{ .Values.global.image.version | quote }}
+singleNode: {{ .Values.global.singleNode }}
 {{- end -}}
 
 {{- /* Add data volume mount for a component */ -}}
@@ -80,7 +80,7 @@ volumeMounts:
 
 {{- /* Add config arg for a component */ -}}
 {{- define "hpcc.configArg" -}}
-"--config=/etc/config/{{ .name }}.yaml", "--global=/etc/config/global.yaml"
+"--config=/etc/config/{{ .name }}.yaml"
 {{- end -}}
 
 {{- /* Add dali arg for a component */ -}}

dockerfiles/hpcc/templates/dali.yaml

@@ -40,11 +40,12 @@ apiVersion: v1
 metadata:
   name: {{ .name }}-configmap 
 data:
-{{- include "hpcc.generateConfigMap" (dict "root" $ "component" "dali" "me" .) }}
   {{ .name }}.yaml: |
     version: 1.0
     dali:
 {{ toYaml . | indent 6 }}
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
 ---
 apiVersion: v1
 kind: Service

dockerfiles/hpcc/templates/eclagent.yaml

@@ -37,13 +37,14 @@ apiVersion: v1
 metadata:
   name: {{ .name }}-configmap 
 data:
-{{- include "hpcc.generateConfigMap" (dict "root" $ "component" "eclagent" "me" .) }}
   {{ .name }}.yaml: |
     version: 1.0
     eclagent:
 {{ toYaml . | indent 6 }}
     {{ $apptype }}:
 {{ toYaml . | indent 6 }}
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
   eclagent-jobspec.yaml: |
     apiVersion: batch/v1
     kind: Job

dockerfiles/hpcc/templates/eclccserver.yaml

@@ -37,13 +37,14 @@ apiVersion: v1
 metadata:
   name: {{ .name }}-configmap 
 data:
-{{- include "hpcc.generateConfigMap" (dict "root" $ "component" "eclccserver" "me" .) }}
   {{ .name }}.yaml: |
     version: 1.0
     eclccserver:
 {{ toYaml . | indent 6 }}
       queues:
 {{ include "hpcc.generateConfigMapQueues" $ | indent 6 }}
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
   eclccserver-jobspec.yaml: |
     apiVersion: batch/v1
     kind: Job

dockerfiles/hpcc/templates/esp.yaml

@@ -34,13 +34,14 @@ apiVersion: v1
 metadata:
   name: {{ .name }}-configmap 
 data:
-{{- include "hpcc.generateConfigMap" (dict "root" $ "component" "esp" "me" .) }}
   {{ .name }}.yaml: |
     version: 1.0
     esp:
 {{ toYaml . | indent 6 }}
       queues:
 {{ include "hpcc.generateConfigMapQueues" $ | indent 6 }}
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
 ---
 apiVersion: v1
 kind: Service

dockerfiles/hpcc/templates/localroxie.yaml

@@ -41,11 +41,12 @@ apiVersion: v1
 metadata:
   name: {{ $roxie.name }}-configmap 
 data:
-{{- include "hpcc.generateConfigMap" (dict "root" $ "component" "roxie" "me" $roxie ) }}
   {{ $roxie.name }}.yaml: |
     version: 1.0
     roxie:
 {{ toYaml $roxie | indent 6 }}
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
 ---
 {{- end }}
 {{- end }}

dockerfiles/hpcc/templates/roxie.yaml

@@ -51,11 +51,12 @@ apiVersion: v1
 metadata:
   name: {{ $roxie.name }}-configmap 
 data:
-{{- include "hpcc.generateConfigMap" (dict "root" $ "component" "roxie" "me" $roxie) }}
   {{ $roxie.name }}.yaml: |
     version: 1.0
     roxie:
 {{ toYaml $roxie | indent 6 }}
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
 ---
 
 {{ if $roxie.serverReplicas -}}

dockerfiles/hpcc/templates/thor.yaml

@@ -83,24 +83,35 @@ apiVersion: v1
 metadata:
   name: {{ $thorScope.name }}-configmap 
 data:
-{{- include "hpcc.generateConfigMap" (dict "root" $ "component" "thor" "me" $thorScope) }}
   {{ $thorScope.name }}.yaml: |
     version: 1.0
     thor:
 {{ toYaml $thorScope | indent 6 }}
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
+
   {{ $eclAgentScope.name }}.yaml: |
     version: 1.0
     eclagent:
 {{ toYaml $eclAgentScope | indent 6 }}
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
+
   {{ $thorAgentScope.name }}.yaml: |
     version: 1.0
     eclagent:
 {{ toYaml $thorAgentScope | indent 6 }}
       type: thor
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
+
   {{ $hthorScope.name }}.yaml: |
     version: 1.0
     hthor:
 {{ toYaml $hthorScope | indent 6 }}
+    global:
+{{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
+
   hthor-jobspec.yaml: |
     apiVersion: batch/v1
     kind: Job

roxie/ccd/ccdmain.cpp

@@ -574,7 +574,6 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
     }
     init_signals();
 
-    bool dumpArgs = false;
     for (unsigned i=0; i<(unsigned)argc; i++)
     {
         if (stricmp(argv[i], "--help")==0 ||
@@ -589,8 +588,6 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             argv[i] = "--csv";
         else if (strsame(argv[i], "-raw"))
             argv[i] = "--raw";
-        else if (strsame(argv[i], "-v"))
-            dumpArgs = true;
     }
 
     #ifdef _USE_CPPUNIT
@@ -653,16 +650,6 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         topologyFile.append(codeDirectory).append(PATHSEPCHAR).append("RoxieTopology.xml");
         useOldTopology = checkFileExists(topologyFile.str());
         topology = loadConfiguration(useOldTopology ? nullptr : defaultYaml, argv, "roxie", "ROXIE", topologyFile, nullptr);
-        if (dumpArgs)
-        {
-            for (unsigned i=0; i<(unsigned)argc; i++)
-            {
-                DBGLOG("Arg: %s", argv[i]);
-            }
-            StringBuffer yamlText;
-            regenerateConfig(yamlText, topology, "Roxie");
-            DBGLOG("Configuration: %s", yamlText.str());
-        }
         saveTopology();
         const char *channels = topology->queryProp("@channels");
         if (channels)

system/jlib/jptree.cpp

@@ -7662,7 +7662,7 @@ void mergeConfiguration(IPropertyTree & target, IPropertyTree & source)
  * If there is an extends tag in the root of the file then this file is applied as a delta to the base file
  * the configuration is the contents of the tag within the file that matches the component tag.
 */
-static IPropertyTree * loadConfiguration(const char * filename, const char * componentTag)
+static IPropertyTree * loadConfiguration(const char * filename, const char * componentTag, bool required)
 {
     if (!checkFileExists(filename))
         throw makeStringExceptionV(99, "Configuration file %s not found", filename);
@@ -7677,8 +7677,11 @@ static IPropertyTree * loadConfiguration(const char * filename, const char * com
     assert(configTree);
     IPropertyTree * config = configTree->queryPropTree(componentTag);
     if (!config)
-        throw makeStringExceptionV(99, "Section %s is missing from file %s", componentTag, filename);
-
+    {
+        if (required)
+            throw makeStringExceptionV(99, "Section %s is missing from file %s", componentTag, filename);
+        return nullptr;
+    }
     const char * base = configTree->queryProp("@extends");
     if (!base)
         return LINK(config);
@@ -7688,7 +7691,7 @@ static IPropertyTree * loadConfiguration(const char * filename, const char * com
     addNonEmptyPathSepChar(baseFilename);
     baseFilename.append(base);
 
-    Owned<IPropertyTree> baseTree = loadConfiguration(baseFilename, componentTag);
+    Owned<IPropertyTree> baseTree = loadConfiguration(baseFilename, componentTag, required);
     mergeConfiguration(*baseTree, *config);
     return LINK(baseTree);
 }
@@ -7753,7 +7756,7 @@ static const char * extractOption(const char * option, const char * cur)
 
 static bool ignoreOption(const char * name)
 {
-    return streq(name, "config") || streq(name, "global") || streq(name, "outputconfig");
+    return streq(name, "config") || streq(name, "outputconfig");
 }
 
 static void applyCommandLineOption(IPropertyTree * config, const char * option, const char * value)
@@ -7790,16 +7793,6 @@ static void applyCommandLineOption(IPropertyTree * config, const char * option)
     }
 }
 
-jlib_decl StringBuffer & regenerateConfig(StringBuffer &yamlText, IPropertyTree * config, const char * componentTag)
-{
-    Owned<IPropertyTree> recreated = createPTree();
-    recreated->setProp("@version", currentVersion);
-    recreated->addPropTree(componentTag, LINK(config));
-
-    toYAML(recreated, yamlText, 0, YAML_SortTags);
-    return yamlText;
-}
-
 static Owned<IPropertyTree> componentConfiguration;
 static Owned<IPropertyTree> globalConfiguration;
 MODULE_INIT(INIT_PRIORITY_STANDARD)
@@ -7855,17 +7848,13 @@ jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char
 
     Linked<IPropertyTree> config(componentDefault);
     const char * optConfig = nullptr;
-    const char * optGlobal = nullptr;
     bool outputConfig = false;
     for (const char * * pArg = argv; *pArg; pArg++)
     {
         const char * cur = *pArg;
         const char * matchConfig = extractOption("--config", cur);
-        const char * matchGlobal = extractOption("--global", cur);
         if (matchConfig)
             optConfig = matchConfig;
-        else if (matchGlobal)
-            optGlobal = matchGlobal;
         else if (strsame(cur, "--help"))
         {
             //MORE: displayHelp(config);
@@ -7897,16 +7886,15 @@ jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char
         if (streq(optConfig, "1"))
             throw makeStringExceptionV(99, "Name of configuration file omitted (use --config=<filename>)");
 
+        StringBuffer fullpath;
         if (!isAbsolutePath(optConfig))
         {
-            StringBuffer fullpath;
             appendCurrentDirectory(fullpath, false);
             addNonEmptyPathSepChar(fullpath);
-            fullpath.append(optConfig);
-            delta.setown(loadConfiguration(fullpath, componentTag));
         }
-        else
-            delta.setown(loadConfiguration(optConfig, componentTag));
+        fullpath.append(optConfig);
+        delta.setown(loadConfiguration(fullpath, componentTag, true));
+        globalConfiguration.setown(loadConfiguration(fullpath, "global", false));
     }
     else
     {
@@ -7930,14 +7918,17 @@ jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char
 
     if (outputConfig)
     {
+        Owned<IPropertyTree> recreated = createPTree();
+        recreated->setProp("@version", currentVersion);
+        recreated->addPropTree(componentTag, LINK(config));
+        if (globalConfiguration)
+            recreated->addPropTree("global", globalConfiguration.getLink());
         StringBuffer yamlText;
-        regenerateConfig(yamlText, config, componentTag);
+        toYAML(recreated, yamlText, 0, YAML_SortTags);
         printf("%s\n", yamlText.str());
         exit(0);
     }
 
-    if (optGlobal)
-        globalConfiguration.setown(loadConfiguration(optGlobal, "global"));
     if (!globalConfiguration)
         globalConfiguration.setown(createPTree("global"));
 

system/jlib/jptree.hpp

@@ -295,7 +295,6 @@ inline static bool isValidXPathChr(char c)
 
 jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv);
 jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *));
-jlib_decl StringBuffer & regenerateConfig(StringBuffer &jsonText, IPropertyTree * config, const char * componentTag);
 jlib_decl IPropertyTree * queryCostsConfiguration();
 
 //The following can only be called after loadConfiguration has been called.  All components must call loadConfiguration().