HPCC-23354 Supply an example helm chart and other docs

Work in progress, but I think the concept is promising enough to review and
merge as a basis for future development.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dockerfiles/README.md

@@ -1,3 +1,6 @@
+Docker images
+=============
+
 Docker images related to HPCC are structured as follows
 
 hpccsystems/platform-build-base
@@ -6,7 +9,7 @@ This image contains all the development packages required to build the hpcc plat
 but no HPCC code or sources. It changes rarely. The current version is tagged 7.8 and
 is based on Ubuntu 18.04 base image
 
-hpccsystems/platform-build
+**hpccsystems/platform-build**
 
 Building this image builds an installation package (.deb file) for a specified git tag
 of the HPCC platform sources. The Dockerfile takes two arguments, naming the version of
@@ -19,17 +22,15 @@ working on a branch that is not yet tagged or merged into upstream, that uses
 hpccsystems/platform-build as a base in order to avoid the need for full rebuilds each time
 the image is built.
 
-hpccsystems/plaform-core
+**hpccsystems/plaform-core**
 
 This uses the .deb file from a hpccsystems/plaform-build image to install a copy of the
 full platform code, without specialization to a specific component.
 
-hpccsystems/dali
-hpccsystems/roxie
-hpccsystems/esp
-hpccsystems/eclagent
-hpccsystems/eclcc
-etc
+**hpccsystems/dali**  
+**hpccsystems/roxie**  
+**hpccsystems/esp**  
+**etc**  
 
 These are specializations of the platform-core image to run a specific component.
 Portions of the platform-core that are not needed by this component may be removed.
@@ -39,3 +40,34 @@ a cloud cluster.
 If launched without further parameters or configuration, a system with default
 settings can be started, but it will be more normal to apply some configuration at
 container launch time.
+
+---
+
+Helm chart
+==========
+
+The Helm chart in hpcc/ can be used to deploy an entire HPCC environment to a K8s cluster.
+
+Roxie modes under K8s
+---------------------
+
+When running under K8s, Roxie has 3 fundamental modes of operation:
+
+  1. Scalable array of one-way roxie servers
+
+     Set localSlave=true, replicas=initial number of pods
+
+  2. Per-channel-scalable array of combined servers/slaves
+
+     localSlave=false, numChannels=nn, replicas=initial number of pods per channel (default 2)
+
+     There will be numChannels*replicas pods in total
+
+  3. Scalable array of servers with per-channel-scalable array of slaves
+
+     localSlave=false, numChannels=nn, replicas=pods/channel, serverReplicas=initial number of server pods
+
+     There will be numChannels*replicas slave pods and serverReplicas server pods in total
+  
+     This mode is somewhat experimental at present!
+  

dockerfiles/cleanup.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# clean up old images
+
+[[ "$1" == "-all" ]] && docker rm $(docker ps -q -f 'status=exited')
+docker rmi $(docker images -q -f "dangling=true")
+
+HEAD=$(git rev-parse --short HEAD)
+PREV=$(git describe --abbrev=0 --tags)
+for f in `docker images  --format "{{.Repository}}:{{.Tag}}" | grep hpccsystems/ | grep -v $HEAD | grep -v $PREV` ; do
+  docker rmi $f
+done

dockerfiles/hpcc/Chart.yaml

@@ -1,6 +1,7 @@
 apiVersion: v2
 name: hpcc
-description: An example  Helm chart for launching a HPCC cluster using Kubernetes
+description: An example Helm chart for launching a HPCC cluster using Kubernetes
+icon: https://hpccsystems.com/sites/default/files/header-logo_0.png
 type: application
 
 # This is the chart version. This version number should be incremented each time you make changes

dockerfiles/hpcc/files/roxie.yaml

@@ -0,0 +1,3 @@
+poo: "{{ $.Values.global.image.version }}"
+poo2: "{{ $.Values.global.image.version }}"
+poo3: "{{ $.Values.global.image.version }}"

dockerfiles/hpcc/templates/_util.tpl

@@ -0,0 +1,86 @@
+{{- /* Translate a port list to a comma-separated list */ -}}
+{{- define "hpcc.utils.portListToCommas" -}}
+ {{- if hasPrefix "[]" (typeOf .) -}}
+  {{- $local := dict "first" true -}}
+  {{- range $key, $value := . -}}{{- if not $local.first -}},{{- end -}}{{- $value -}}{{- $_ := set $local "first" false -}}{{- end -}}
+ {{- else -}}
+  {{- . -}} 
+ {{- end -}}
+{{- end -}}
+
+{{- /* Generate local config info into config section */ -}}
+{{- /* Pass in a dictionary with root and me defined */ -}}
+{{- define "hpcc.utils.generateConfigMapFromFile" -}}
+{{- if hasKey .me "configFile" -}}
+{{- $filename := (printf "files/%s" .me.configFile) -}}
+{{- .me.name -}}.json: |
+{{ tpl (.root.Files.Get $filename) .root | indent 2 -}}
+{{- else if hasKey .me "config" -}}
+{{- .me.name -}}.json: |
+{{ .me.config | indent 2 -}}
+{{- end -}}
+{{- end -}}
+
+{{- /* Generate a ConfigMap for a component */ -}}
+{{- /* Pass in a dictionary with root and me defined */ -}}
+{{- define "hpcc.utils.generateConfigMap" }}
+kind: ConfigMap 
+apiVersion: v1 
+metadata:
+  name: {{ .me.name }}-configmap 
+data:
+  global.json: |
+    {
+      "version": {{ .root.Values.global.image.version | quote }}
+    }
+{{ include "hpcc.utils.generateConfigMapFromFile" . | indent 2 }}
+{{ end -}}
+
+{{- /* Add a ConfigMap volume for a component */ -}}
+{{- define "hpcc.utils.addConfigVolume" -}}
+- name: {{ .name }}-configmap-volume
+  configMap:
+    name: {{ .name }}-configmap
+{{- end -}}
+
+{{- /* Add a ConfigMap volume mount for a component */ -}}
+{{- define "hpcc.utils.addConfigVolumeMount" -}}
+- name: {{ .name }}-configmap-volume
+  mountPath: /etc/config
+{{- end -}}
+
+{{- /* Add standard volumes for a component */ -}}
+{{- define "hpcc.utils.addVolumes" -}}
+volumes:
+{{ include "hpcc.utils.addConfigVolume" . }}
+- name: dllserver-pv-storage
+  persistentVolumeClaim:
+    claimName: dllserver-pv-claim
+{{- end -}}
+
+{{- /* Add standard volume mounts for a component */ -}}
+{{- define "hpcc.utils.addVolumeMounts" -}}
+volumeMounts:
+{{ include "hpcc.utils.addConfigVolumeMount" . }}
+- name: dllserver-pv-storage
+  mountPath: "/var/lib/HPCCSystems/queries"
+{{- end -}}
+
+{{- /* Add config arg for a component */ -}}
+{{- define "hpcc.utils.configArg" -}}
+{{- if or (hasKey . "configFile") (hasKey . "config") -}}
+"--config=/etc/config/{{ .name }}.json", {{ end -}}
+"--global=/etc/config/global.json"
+{{- end -}}
+
+{{- /* Add dali arg for a component */ -}}
+{{- define "hpcc.utils.daliArg" -}}
+"--daliServers={{ (index .Values.dali 0).name }}"
+{{- end -}}
+
+{{- /* Add image attributes for a component */ -}}
+{{- /* Pass in a dictionary with root and imagename defined */ -}}
+{{- define "hpcc.utils.addImageAttrs" -}}
+image: "{{ .root.Values.global.image.root | default "hpccsystems" }}/{{ .imagename }}:{{ .root.Values.global.image.version }}"
+imagePullPolicy: {{ .root.Values.global.image.pullPolicy }}
+{{- end -}}

dockerfiles/hpcc/templates/dali.yaml

@@ -1,31 +1,42 @@
+{{ range $.Values.dali -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: dali
+  name: {{ .name | quote }}
 spec:
   replicas: 1
   selector:
     matchLabels:
-      run: dali
+      run: {{ .name | quote }}
   template:
     metadata:
       labels:
-        run: dali
+        run: {{ .name | quote }}
     spec:
       containers:
-      - name: dali
-        image: "hpccsystems/dali:{{ .Values.hpcc.image.version }}"
-        imagePullPolicy: {{ .Values.hpcc.image.pullPolicy }}
+      - name: {{ .name | quote }}
+        args: [
+                # {{ include "hpcc.utils.configArg" . }} - dali does not support this yet
+              ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "dali") | indent 8 }}
+        volumeMounts:
+{{ include "hpcc.utils.addConfigVolumeMount" . | indent 8 }}
+      volumes:
+{{ include "hpcc.utils.addConfigVolume" . | indent 6 }}
+---
+{{- include "hpcc.utils.generateConfigMap" (dict "root" $ "me" .) -}}
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: dali
+  name: {{ .name | quote }}
 spec:
   ports:
   - port: 7070
     protocol: TCP
     targetPort: 7070
   selector:
-    run: dali
+    run: {{ .name | quote }}
   type: ClusterIP
+---
+{{- end }}

dockerfiles/hpcc/templates/dllserver-claim.yaml

@@ -8,4 +8,4 @@ spec:
     - ReadWriteMany
   resources:
     requests:
-      storage: {{ .Values.hpcc.dllserver.storageSize }}
+      storage: {{ .Values.global.dllserver.storageSize }}

dockerfiles/hpcc/templates/eclagent.yaml

@@ -1,33 +1,28 @@
+{{ range $.Values.eclagent -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: eclagent
+  name: {{ .name | quote }}
 spec:
-  replicas: {{ .Values.hpcc.eclagent.replicas }}
+  replicas: {{ .replicas | default 1 }}
   selector:
     matchLabels:
-      run: eclagent
+      run: {{ .name | quote }}
   template:
     metadata:
       labels:
-        run: eclagent
+        run: {{ .name | quote }}
     spec:
-      volumes:
-      - name: dllserver-pv-storage
-        persistentVolumeClaim:
-          claimName: dllserver-pv-claim
-      initContainers:
-      - name: volume-mount-hack
-        image: busybox
-        command: ["sh", "-c", "chown -R 999:1000 /var/lib/HPCCSystems/queries"]
-        volumeMounts:
-        - mountPath: "/var/lib/HPCCSystems/queries"
-          name: dllserver-pv-storage
       containers:
-      - name: eclagent
-        image: "hpccsystems/eclagent:{{ .Values.hpcc.image.version }}"
-        imagePullPolicy: {{ .Values.hpcc.image.pullPolicy }}
-        volumeMounts:
-        - mountPath: "/var/lib/HPCCSystems/queries"
-          name: dllserver-pv-storage
-
+      - name: {{ .name | quote }}
+        args: [ 
+                {{ include "hpcc.utils.configArg" . }},
+                {{ include "hpcc.utils.daliArg" $ }}
+              ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "eclagent") | indent 8 }}
+{{ include "hpcc.utils.addVolumeMounts" . | indent 8 }}
+{{ include "hpcc.utils.addVolumes" . | indent 6 }}
+---
+{{- include "hpcc.utils.generateConfigMap" (dict "root" $ "me" .) -}}
+---
+{{- end }}

dockerfiles/hpcc/templates/eclccserver.yaml

@@ -1,22 +1,19 @@
+{{ range $.Values.eclccserver -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: eclccserver
+  name: {{ .name | quote }}
 spec:
-  replicas: 1
+  replicas: {{ .replicas | default 1 }}
   selector:
     matchLabels:
-      run: eclccserver
+      run: {{ .name | quote }}
   template:
     metadata:
       labels:
-        run: eclccserver
+        run: {{ .name | quote }}
     spec:
-      volumes:
-      - name: dllserver-pv-storage
-        persistentVolumeClaim:
-          claimName: dllserver-pv-claim
-
+      {{ if $.Values.global.singleNode | default false -}}
       initContainers:
       # This is a bit of a hack, to ensure that the persistent storage mounted for dllserver
       # is writable. This is not something we would want to do if using anything other than
@@ -27,12 +24,17 @@ spec:
         volumeMounts:
         - mountPath: "/var/lib/HPCCSystems/queries"
           name: dllserver-pv-storage
-
+      {{- end }}
       containers:
-      - name: eclccserver
-        image: "hpccsystems/eclccserver:{{ .Values.hpcc.image.version }}"
-        imagePullPolicy: {{ .Values.hpcc.image.pullPolicy }}
-        volumeMounts:
-        - mountPath: "/var/lib/HPCCSystems/queries"
-          name: dllserver-pv-storage
-
+      - name: {{ .name | quote }}
+        args: [
+                {{ include "hpcc.utils.configArg" . }},
+                {{ include "hpcc.utils.daliArg" $ }}
+              ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "eclccserver") | indent 8 }}
+{{ include "hpcc.utils.addVolumeMounts" . | indent 8 }}
+{{ include "hpcc.utils.addVolumes" . | indent 6 }}
+---
+{{- include "hpcc.utils.generateConfigMap" (dict "root" $ "me" .) -}}
+---
+{{- end }}

dockerfiles/hpcc/templates/esp.yaml

@@ -1,31 +1,40 @@
+{{ range $.Values.esp -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: esp
+  name: {{ .name | quote }}
 spec:
-  replicas: {{ .Values.hpcc.esp.replicas }}
+  replicas: {{ .replicas | default 1 }}
   selector:
     matchLabels:
-      run: esp
+      run: {{ .name | quote }}
   template:
     metadata:
       labels:
-        run: esp
+        run: {{ .name | quote }}
     spec:
       containers:
-      - name: esp
-        image: "hpccsystems/esp:{{ .Values.hpcc.image.version }}"
-        imagePullPolicy: {{ .Values.hpcc.image.pullPolicy }}
+      - name: {{ .name | quote }}
+        args: [
+                # {{ include "hpcc.utils.configArg" . }},
+                # {{ include "hpcc.utils.daliArg" $ }}
+              ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "esp") | indent 8 }}
+{{ include "hpcc.utils.addVolumeMounts" . | indent 8 }}
+{{ include "hpcc.utils.addVolumes" . | indent 6 }}
+---
+{{- include "hpcc.utils.generateConfigMap" (dict "root" $ "me" .) -}}
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: esp
+  name: {{ .name | quote }}
 spec:
   ports:
   - port: 8010
     protocol: TCP
     targetPort: 8010
   selector:
-    run: esp
+    run: {{ .name | quote }}
   type: LoadBalancer
+{{- end }}

dockerfiles/hpcc/templates/localroxie.yaml

@@ -0,0 +1,35 @@
+{{ range $roxie := $.Values.roxie -}}
+{{- if $roxie.localSlave -}}
+{{- $name := $roxie.name -}}
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $roxie.name | quote }}
+spec:
+  replicas: {{ $roxie.serverReplicas | default 1 }}
+  selector:
+    matchLabels:
+      run: {{ $roxie.name | quote }}
+  template:
+    metadata:
+      labels:
+        run: {{ $roxie.name | quote }}
+    spec:
+      containers:
+      - name: {{ $roxie.name | quote }}
+        args: [
+                {{ include "hpcc.utils.configArg" $roxie }},
+                {{ include "hpcc.utils.daliArg" $ }},
+                "--serverPorts={{ template "hpcc.utils.portListToCommas" $roxie.ports }}", 
+                "--localSlave=true"
+              ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "roxie") | indent 8 }}
+{{ include "hpcc.utils.addVolumeMounts" . | indent 8 }}
+{{ include "hpcc.utils.addVolumes" . | indent 6 }}
+---
+{{- include "hpcc.utils.generateConfigMap" (dict "root" $ "me" $roxie ) -}}
+---
+{{- end }}
+{{- end }}
+ 

dockerfiles/hpcc/templates/roxie.yaml

@@ -1,19 +1,121 @@
+{{ range $roxie := $.Values.roxie -}}
+{{- if not $roxie.localSlave -}}
+{{- $toponame := printf "%s-toposerver" $roxie.name -}}
+{{- $numChannels := $roxie.numChannels | int | default 1 -}}
+{{- $topoport := $roxie.topoport | int | default 9004 -}}
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: roxie
+  name: {{ $toponame | quote }}
 spec:
-  replicas: 1
+  replicas: {{ $roxie.topoReplicas | default 1 }}
   selector:
     matchLabels:
-      run: roxie
+      run: {{ $toponame | quote }}
   template:
     metadata:
       labels:
-        run: roxie
+        run: {{ $toponame | quote }}
     spec:
       containers:
-      - name: roxie
-        image: "hpccsystems/roxie:{{ .Values.hpcc.image.version }}"
-        imagePullPolicy: {{ .Values.hpcc.image.pullPolicy }}
-        args: [ "--serverPort=9870" ]
+      - name: {{ $toponame | quote }}
+        args: [ "--port={{ $topoport }}" ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "toposerver") | indent 8 }}
+        volumeMounts:
+{{ include "hpcc.utils.addConfigVolumeMount" . | indent 8 }}
+      volumes:
+{{ include "hpcc.utils.addConfigVolume" . | indent 6 }}
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $toponame | quote }}
+spec:
+  ports:
+  - port: {{ $topoport  }}
+    protocol: TCP
+    targetPort: {{ $topoport }}
+  selector:
+    run: {{ $toponame | quote }}
+  clusterIP: None # Headless service
+
+---
+
+{{- include "hpcc.utils.generateConfigMap" (dict "root" $ "me" $roxie) -}}
+
+---
+
+{{ if $roxie.serverReplicas -}}
+{{- $servername := printf "%s-server" $roxie.name -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $servername | quote }}
+spec:
+  replicas: {{ $roxie.serverReplicas }}
+  selector:
+    matchLabels:
+      run: {{ $servername | quote }}
+  template:
+    metadata:
+      labels:
+        run: {{ $servername | quote }}
+    spec:
+      containers:
+      - name: {{ $servername | quote }}
+        args: [
+                {{ include "hpcc.utils.configArg" . }},
+                {{ include "hpcc.utils.daliArg" $ }},
+                "--numChannels={{ $numChannels }}",
+                "--serverPorts={{ template "hpcc.utils.portListToCommas" $roxie.ports }}", 
+                "--topologyServers={{ $toponame }}:{{ $roxie.topoport }}",
+                "--localSlave=false"
+              ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "roxie") | indent 8 }}
+{{ include "hpcc.utils.addVolumeMounts" . | indent 8 }}
+{{ include "hpcc.utils.addVolumes" . | indent 6 }}
+---
+
+{{ end -}}
+{{ range $c, $e := until ($roxie.numChannels|int) -}}
+{{- $channel := add $c 1 -}}
+{{- $name := printf "%s-slave-%d" $roxie.name $channel -}}
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $name | quote}}
+spec:
+  replicas: {{ $roxie.channelReplicas | default 2 }}
+  selector:
+    matchLabels:
+      run: {{ $name | quote}}
+  template:
+    metadata:
+      labels:
+        run: {{ $name | quote}}
+    spec:
+      containers:
+      - name: {{ $name | quote}}
+        args: [ 
+                {{ include "hpcc.utils.configArg" $roxie }},
+                {{ include "hpcc.utils.daliArg" $ }},
+                "--channels={{ $channel }}", 
+                "--serverPorts={{ if not $roxie.serverReplicas }}{{ template "hpcc.utils.portListToCommas" $roxie.ports }}{{ end }}",
+                "--numChannels={{ $numChannels }}",
+                "--localSlave=false", 
+                "--topologyServers={{ $toponame }}:{{ $roxie.topoport }}",
+              ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "roxie") | indent 8 }}
+{{ include "hpcc.utils.addVolumeMounts" $roxie | indent 8 }}
+{{ include "hpcc.utils.addVolumes" $roxie | indent 6 }}
+
+---
+
+{{- end }}
+{{- end }}
+{{- end }}
+ 

dockerfiles/hpcc/values.yaml

@@ -1,74 +1,75 @@
 # Default values for hpcc.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
 
-replicaCount: 1
-
-hpcc:
-  eclagent:
-    replicas: 2
-  esp:
-    replicas: 2
+global:
+  # Settings in the global section apply to all HPCC components in all subcharts
+  
   dllserver:
     storageSize: 3Gi
 
   image:
+    # This should probably say "latest" or "stable" in the example script, but set to a specific version in production systems
+    # We should also support overriding the version within a component
     version: master-2020-02-10
+    root: "hpccsystems"    # change this if you want to pull your images from somewhere other than DockerHub hpccsystems/*
     pullPolicy: IfNotPresent
 
-imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""
-
-serviceAccount:
-  # Specifies whether a service account should be created
-  create: true
-  # The name of the service account to use.
-  # If not set and create is true, a name is generated using the fullname template
-  name:
-
-podSecurityContext: {}
-  # fsGroup: 2000
-
-securityContext: {}
-  # capabilities:
-  #   drop:
-  #   - ALL
-  # readOnlyRootFilesystem: true
-  # runAsNonRoot: true
-  # runAsUser: 1000
-
-service:
-  type: LoadBalancer
-  port: 8010
+  # Remove or override this if you don't want the simplified assumptions (including
+  # persistent storage that is local to the node) used for single-node installations
+  
+  singleNode: true
 
-ingress:
-  enabled: false
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  hosts:
-    - host: chart-example.local
-      paths: []
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - chart-example.local
+dali:
+- name: mydali
 
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
+eclagent:
+- name: myeclagent
+  replicas: 1
+    
+eclccserver:
+- name: myeclccserver
+  replicas: 1
+    
+esp:
+- name: myesp
+  replicas: 1
 
-nodeSelector: {}
+notroxie:
+  - name: roxie1
+    ports: [ 9876, 0 ]
+    localSlave: 1
+    Xconfig: |
+     {
+      "array": [
+        1,
+        2,
+        3
+      ],
+      "boolean": true,
+      "number": 123,
+      "object": {
+        "a": "b",
+        "c": "d",
+        "e": "f"
+      },
+      "string": "Hello World"
+     }
+    
+roxie:
+  - name: roxie2
+    ports: [ 9876, 0 ]
+    numChannels: 2
+    topoport: 9004
+    XconfigFile: "roxie.yaml"
+   
+notroxie: 
+  - name: roxie3
+    ports: 9876
+    numChannels: 2
+    serverReplicas: 1
+    topoReplicas: 2
+    topoport: 9004
+    Xconfig: |
+      hello there
 
-tolerations: []
 
-affinity: {}
+    

dockerfiles/incr.sh

@@ -11,7 +11,7 @@ PREV=$1
 [[ -z ${PREV} ]] && PREV=$(git log --format=format:%h $(git describe --abbrev=0 --tags)..HEAD | grep `docker images hpccsystems/platform-core --format {{.Tag}} | head -n 1`)
 [[ -z ${PREV} ]] && PREV=$(git describe --abbrev=0 --tags)
 
-if [[ "$HEAD" == "$PREV$FORCE" ]]
+if [[ "$HEAD" == "$PREV$FORCE" ]]  # set environment variable FORCE before running to override this check
 then
     echo Docker image hpccsystems/platform-core:${HEAD} already exists
 else
@@ -23,7 +23,7 @@ else
     echo Building local incremental images based on ${PREV}
         
     docker image build -t hpccsystems/platform-build:${HEAD} --build-arg BUILD_VER=${PREV} --build-arg COMMIT=${HEAD} platform-build-incremental/
-    docker image build -t hpccsystems/platform-core:${HEAD} --build-arg BUILD_VER=${HEAD} platform-core/  
+    docker image build -t hpccsystems/platform-core:${HEAD} --build-arg BUILD_VER=${HEAD} platform-core-debug/  
     
     docker image build -t hpccsystems/roxie:${HEAD} --build-arg BUILD_VER=${HEAD} roxie/
     docker image build -t hpccsystems/dali:${HEAD} --build-arg BUILD_VER=${HEAD} dali/

dockerfiles/platform-core-debug/Dockerfile

@@ -0,0 +1,16 @@
+# A version of the platform-core Dockerfile that keeps all the build artefacts in place, for use by developers wanting to debug the platform
+
+ARG BUILD_VER
+FROM hpccsystems/platform-build:${BUILD_VER}
+
+RUN apt-get install -y \
+    dnsutils \
+    nano 
+
+RUN dpkg -i /hpcc-dev/build/*.deb ; \
+    apt-get install -f -y
+
+USER hpcc
+ENV PATH="/opt/HPCCSystems/bin:${PATH}"
+ENV HPCC_containerized=1
+ENV HPCC_DLLSERVER_PATH=/var/lib/HPCCSystems/queries

dockerfiles/startall.sh

@@ -2,13 +2,7 @@
 
 HEAD=$(git rev-parse --short HEAD)
 
-kubectl run dali --image=hpccsystems/dali:${HEAD} --image-pull-policy=Never
-kubectl expose deployment dali --port=7070
-kubectl run esp --image=hpccsystems/esp:${HEAD} --image-pull-policy=Never
-kubectl expose deployment esp --port=8010 --type=LoadBalancer
-kubectl run roxie --image=hpccsystems/roxie:${HEAD} --image-pull-policy=Never
-kubectl run eclcc --image=hpccsystems/eclccserver:${HEAD} --image-pull-policy=Never
-kubectl run eclagent --image=hpccsystems/eclagent:${HEAD} --image-pull-policy=Never
-
-#kubectl logs mydali-759f975769-v6tmm
+helm install mycluster hpcc/ --set global.image.version=$HEAD 
+sleep 1
+kubectl get pods
 

dockerfiles/stopall.sh

@@ -1,6 +1,3 @@
 #!/bin/bash
 
-kubectl delete deployments.apps dali roxie esp eclcc eclagent
-kubectl delete service dali
-kubectl delete service esp
-
+helm uninstall mycluster