HPCC-23709 - Add volume mount for dali persisted storage

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dockerfiles/dali/daliconf.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<DALI name="mydali">
+<DALI name="mydali" dataPath="/var/lib/HPCCSystems/dalistore">
+  <Directories/>
   <SDS store="dalisds.xml" caseInsensitive="0" recoverFromIncErrors="true" 
        environment="/etc/HPCCSystems/environment.xml">
   </SDS>

dockerfiles/hpcc/templates/_helpers.tpl

@@ -170,6 +170,13 @@ initContainers:
 {{- end }}
 {{- end }}
 
+{{- /* Check dalistore host mount point, using hpcc.changeHostMountPerms */ -}}
+{{- define "hpcc.checkDaliStoreHostMount" -}}
+{{- if .root.Values.global.hostStorage | default false }}
+{{ include "hpcc.changeHostMountPerms" (dict "root" .root "volumeName" "dalistore-pv" "volumePath" "/var/lib/HPCCSystems/dalistore") }}
+{{- end }}
+{{- end }}
+
 {{- /* Add security context */ -}}
 {{- /* Pass in a dictionary with root and me defined */ -}}
 {{- define "hpcc.addSecurityContext" -}}

dockerfiles/hpcc/templates/dali.yaml

@@ -14,6 +14,7 @@ spec:
       labels:
         run: {{ .name | quote }}
     spec:
+      {{- include "hpcc.checkDaliStoreHostMount" (dict "root" $) | indent 6 }}
       containers:
       - name: {{ .name | quote }}
         args: [
@@ -23,8 +24,13 @@ spec:
 {{ include "hpcc.addImageAttrs" (dict "root" $ "me" . "imagename" "dali") | indent 8 }}
         volumeMounts:
 {{ include "hpcc.addConfigVolumeMount" . | indent 8 }}
+        - name: dalistore-pv
+          mountPath: "/var/lib/HPCCSystems/dalistore"
       volumes:
 {{ include "hpcc.addConfigVolume" . | indent 6 }}
+      - name: dalistore-pv
+        persistentVolumeClaim:
+          claimName: {{ .storage.existingClaim | default (printf "%s-%s-dalistore-pv-claim" (include "hpcc.fullname" $) .name) }}
 ---
 {{- include "hpcc.generateConfigMap" (dict "root" $ "component" "dali" "me" .) }}
 ---
@@ -41,5 +47,29 @@ spec:
     run: {{ .name | quote }}
   type: ClusterIP
 ---
+{{- if (not .storage.existingClaim) }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ printf "%s-%s-dalistore-pv-claim" (include "hpcc.fullname" $) .name }}
+  labels:
+    app.kubernetes.io/name: {{ printf "%s-%s-dalistore-pv-claim" (include "hpcc.fullname" $) .name }}
+    app.kubernetes.io/instance: {{ $.Release.Name }}
+    app.kubernetes.io/managed-by: {{ $.Release.Service }}
+    helm.sh/chart: {{ include "hpcc.chart" $ }}
+spec:
+  accessModes:
+    - ReadWriteMany  # accessed from dali and sasha
+  resources:
+    requests:
+      storage: {{ .storage.storageSize }}
+{{- if .storage.storageClass }}
+{{- if (eq "-" .storage.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .storage.storageClass }}"
+{{- end }}
+{{- end }}
+{{- end }}
 {{- end }}
 {{- end }}

dockerfiles/hpcc/values.schema.json

@@ -62,6 +62,9 @@
         "hostStorage": {
           "type": "boolean"
         },
+        "privileged": {
+          "type": "boolean"
+        },
         "dataStorage": {
           "$ref": "#/definitions/storage"
         },
@@ -106,7 +109,7 @@
     },
     "dali": {
       "type": "object",
-      "required": [ "name" ],
+      "required": [ "name", "storage" ],
       "additionalProperties": { "type": ["integer", "string", "boolean"] },
       "properties": {
         "name": {
@@ -115,6 +118,9 @@
         },
         "image": {
           "$ref": "#/definitions/image"
+        },
+        "storage": {
+          "$ref": "#/definitions/storage"
         }
       }
     },

dockerfiles/hpcc/values.yaml

@@ -39,6 +39,17 @@ global:
 
 dali:
 - name: mydali
+  storage:
+    storageSize: 1Gi
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+    ##   GKE, AWS & OpenStack)
+    ##
+    storageClass: ""
+    # existingClaim: ""
+
 
 eclagent:
 - name: hthor
@@ -60,7 +71,7 @@ esp:
   replicas: 1
 
 roxie:
-- name: roxie
+- name: roxie-cluster
   disabled: false
   ports: [9876,0]
   numChannels: 2

dockerfiles/startall.sh

@@ -19,9 +19,10 @@
 
 # Utility script for starting a local cluster corresponding to current git branch
 
-HEAD=$(git rev-parse --short HEAD)
+LABEL=$1
+[[ -z ${LABEL} ]] && LABEL=$(docker image ls | fgrep 'hpccsystems/platform-core' | head -n 1 | awk '{print $2}')
 
-helm install mycluster hpcc/ --set global.image.version=$HEAD-Debug --set global.privileged=true
+helm install mycluster hpcc/ --set global.image.version=$LABEL --set global.privileged=true
 sleep 1
 kubectl get pods