Merge pull request #15944 from ghalliday/issue27421

HPCC-27421 Update schema to allow definition of alternative plane access

Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

dockerfiles/startall.sh

@@ -25,6 +25,7 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 restArgs=()
 CLUSTERNAME=mycluster
 PVFILE=$scriptdir/../helm/examples/local/hpcc-localfile/values.yaml
+MKDIR=1
 
 MANAGED_ELK_SUBPATH="managed/logging/elastic"
 MANAGED_PROM_SUBPATH="managed/metrics/prometheus"
@@ -87,12 +88,18 @@ while [ "$#" -gt 0 ]; do
          ;;
       u) CMD="upgrade"
          ;;
-      p) shift
-         if [[ $arg == '-pv' ]] ; then
-           PERSISTVALUES="--values=$1"
-           PVFILE=$1
+      p) if [[ $arg == '-pm' ]] ; then
+           #Option for minikube to avoid trying to create the directories (because they are mounted elsewhere)
+           MKDIR=0
          else
-           PERSIST=$1
+           shift
+           if [[ $arg == '-pv' ]] ; then
+             PERSISTVALUES="--values=$1"
+             echo $PERSISTVALUES
+             PVFILE=$1
+           else
+             PERSIST=$1
+           fi
          fi
          ;;
       c) DEP_UPDATE=1
@@ -143,10 +150,12 @@ fi
 if [[ -n ${PERSIST} ]] ; then
   PERSIST=$(realpath -q $PERSIST || echo $PERSIST)
   PERSIST_PATH=$(echo $PERSIST | sed 's/\\//g')
-  for subdir in `grep subPath: $PVFILE | awk '{ print $2 }'` ; do
-    echo mkdir -p ${PERSIST_PATH}/$subdir
-    mkdir -p ${PERSIST_PATH}/$subdir
-  done
+  if [[ $MKDIR == '1' ]] ; then
+    for subdir in `grep subPath: $PVFILE | awk '{ print $2 }'` ; do
+      echo mkdir -p ${PERSIST_PATH}/$subdir
+      mkdir -p ${PERSIST_PATH}/$subdir
+    done
+  fi
   helm ${CMD} localfile $scriptdir/../helm/examples/local/hpcc-localfile --set common.hostpath=${PERSIST} $PERSISTVALUES | tee lsfull.yaml | grep -A1000 storage: > localstorage.yaml && \
   grep "##" lsfull.yaml  && \
   helm ${CMD} $CLUSTERNAME $scriptdir/../helm/hpcc/ --set global.image.root="${DOCKER_REPO}" --set global.image.version=$LABEL $DEVELOPER_OPTIONS ${restArgs[@]} -f localstorage.yaml ${PROMETHEUS_METRICS_SINK_ARG} ${ELASTIC_LOG_ACCESS_ARG}

helm/examples/local/hpcc-localfile/templates/NOTES.txt

@@ -12,6 +12,9 @@ storage:
     pvc: {{ printf "%s-%s-pvc" $plane.name (include "hpcc-localfile.fullname" $) }}
     prefix: {{ printf "%s%s%s" $.Values.common.mountPrefix (regexMatch ".*/$" $.Values.common.mountPrefix | ternary "" "/") $plane.subPath | quote }}
     category: {{ $plane.category }}
+{{- if hasKey $plane "numDevices" }}
+    numDevices: {{ $plane.numDevices }}
+{{- end }}
 {{- end }}
 
 {{ range $plane := .Values.planes -}}

helm/examples/local/hpcc-localfile/templates/create-pv.yaml

@@ -1,10 +1,14 @@
 {{- range $plane := .Values.planes -}}
+ {{- $num := int ( $plane.numDevices | default 1 ) -}}
+ {{- range $elem := untilStep 1 (int (add $num 1)) 1 }}
+  {{- $suffix := (eq $num 1) | ternary "" (printf "-many-%d" $elem) -}}
+  {{- $deviceDir := (eq $num 1) | ternary "" (printf "/d%d" $elem) -}}
 apiVersion: v1
 kind: PersistentVolume
 metadata:
-  name: {{ printf "%s-%s-pv" $plane.name (include "hpcc-localfile.fullname" $) }}
+  name: {{ printf "%s-%s-pv%s" $plane.name (include "hpcc-localfile.fullname" $) $suffix }}
   labels:
-    store: {{ printf "%s-%s-pv" $plane.name (include "hpcc-localfile.fullname" $) }}
+    store: {{ printf "%s-%s-pv%s" $plane.name (include "hpcc-localfile.fullname" $) $suffix }}
 spec:
   storageClassName: "manual"
   capacity:
@@ -15,6 +19,7 @@ spec:
   persistentVolumeReclaimPolicy: Retain
   hostPath:
     type: Directory
-    path: {{ printf "%s%s%s" $.Values.common.hostpath (regexMatch ".*/$" $.Values.common.hostpath | ternary "" "/") $plane.subPath | quote }}
+    path: {{ printf "%s%s%s%s" $.Values.common.hostpath (regexMatch ".*/$" $.Values.common.hostpath | ternary "" "/") $plane.subPath $deviceDir | quote }}
 ---
+ {{ end -}}
 {{ end }}

helm/examples/local/hpcc-localfile/templates/create-pvc.yaml

@@ -1,11 +1,15 @@
 {{- range $plane := .Values.planes -}}
+ {{- $num := int ( $plane.numDevices | default 1 ) -}}
+ {{- range $elem := untilStep 1 (int (add $num 1)) 1 }}
+  {{- $pvcsuffix := (eq $num 1) | ternary "" (printf "-%d" $elem) -}}
+  {{- $suffix := (eq $num 1) | ternary "" (printf "-many-%d" $elem) -}}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: {{ printf "%s-%s-pvc" $plane.name (include "hpcc-localfile.fullname" $) }}
+  name: {{ printf "%s-%s-pvc%s" $plane.name (include "hpcc-localfile.fullname" $) $pvcsuffix }}
 spec:
   storageClassName: "manual"
-  volumeName: {{ printf "%s-%s-pv" $plane.name (include "hpcc-localfile.fullname" $) }}
+  volumeName: {{ printf "%s-%s-pv%s" $plane.name (include "hpcc-localfile.fullname" $) $suffix }}
   accessModes:
     - {{ $plane.rwmany | default false | ternary "ReadWriteMany" "ReadWriteOnce" }}
   resources:
@@ -13,6 +17,7 @@ spec:
       storage: {{ $plane.size }}
   selector:
     matchLabels:
-      store: {{ printf "%s-%s-pv" $plane.name (include "hpcc-localfile.fullname" $) }}
+      store: {{ printf "%s-%s-pv%s" $plane.name (include "hpcc-localfile.fullname" $) $suffix }}
 ---
+ {{ end -}}
 {{ end }}

helm/examples/local/hpcc-localfile/values.schema.json

@@ -55,6 +55,10 @@
         "category": {
           "description": "the category this plane is usd for, e.g. lz, data",
           "type": "string"
+        },
+        "numDevices": {
+          "description": "the number of pvcs to create (allowing sub-directories on different devices)",
+          "type": "integer"
         }
       },
       "required": [ "name", "subPath", "size" ],

helm/hpcc/templates/_helpers.tpl

@@ -214,15 +214,25 @@ storage:
 {{- range $plane := $planes }}
  {{- if not $plane.disabled }}
   - name: {{ $plane.name | quote }}
-  {{- $planeYaml := omit $plane "name" "pvc" "storageClass" "storageSize" "subPath" -}}
+  {{- $planeYaml := omit $plane "name" "pvc" "storageClass" "storageSize" "subPath" "numMounts" -}}
   {{- if $plane.subPath -}}
    {{- $_ := set $planeYaml "prefix" (printf "%s/%s" $planeYaml.prefix $plane.subPath) -}}
   {{- end -}}
   {{- if and (eq "data" $plane.category) (not $plane.defaultSprayParts) -}}
    {{- $_ := set $planeYaml "defaultSprayParts" (include "hpcc.getMaxNumWorkers" $ | int) -}}
   {{- end -}}
+
+  {{- /* Remove pvc-related properties from the aliases*/ -}}
+  {{- if $plane.aliases }}
+   {{- $_ := set $planeYaml "aliases" (deepCopy $plane.aliases) -}}
+   {{- range $alias := $planeYaml.aliases -}}
+    {{- $_ := unset $alias "pvc" }}
+    {{- $_ := unset $alias "numMounts" }}
+   {{- end -}}
+  {{- end -}}
   {{- toYaml $planeYaml | nindent 4 }}
  {{- end }}
+
 {{- end }}
 {{- if not (include "hpcc.hasPlaneForCategory" (dict "root" $ "category" "spill")) }}
   - name: hpcc-spill-plane
@@ -314,24 +324,49 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup
 {{- $includeNames := .includeNames | default list -}}
 {{- $previousMounts := dict -}}
 {{- range $plane := $planes -}}
- {{- if not $plane.disabled -}}
-  {{- if or ($plane.pvc) (hasKey $plane "storageClass") -}}
-   {{- if not (hasKey $previousMounts $plane.prefix) -}}
-    {{- $mountpath := $plane.prefix -}}
-    {{- if or (has $plane.category $includeCategories) (has $plane.name $includeNames) }}
-     {{- $num := int ( $plane.numDevices | default 1 ) -}}
-     {{- if le $num 1 }}
+ {{- if not $plane.disabled }}
+  {{- if or (has $plane.category $includeCategories) (has $plane.name $includeNames) }}
+
+   {{- /*This plane is required - generate a mount if it has not already been created, and any aliases*/ -}}
+   {{- if or ($plane.pvc) (hasKey $plane "storageClass") }}
+    {{- if not (hasKey $previousMounts $plane.prefix) }}
+     {{- $mountPath := $plane.prefix }}
+     {{- $numMounts := int ( $plane.numMounts | default $plane.numDevices | default 1 ) }}
+     {{- if le $numMounts 1 }}
 - name: {{ lower $plane.name }}-pv
-  mountPath: {{ $mountpath | quote }}
+  mountPath: {{ $mountPath | quote }}
      {{- else }}
-      {{- range $elem := untilStep 1 (int (add $num 1)) 1 }}
-- name: {{ lower $plane.name }}-pv-many-{{- $elem }}
-  mountPath: {{ printf "%s/d%d" $mountpath $elem | quote }}
+      {{- range $elem := untilStep 1 (int (add $numMounts 1)) 1 }}
+- name: {{ lower $plane.name }}-pv-many-{{ $elem }}
+  mountPath: {{ printf "%s/d%d" $mountPath $elem | quote }}
       {{- end }}
      {{- end }}
     {{- end }}
     {{- $_ := set $previousMounts $plane.prefix true -}}
    {{- end }}
+
+   {{- /*Generate entries for each alias of the plane*/ -}}
+   {{- $curAlias := dict "num" 1 -}}
+   {{- range $alias := $plane.aliases | default list }}
+    {{- if $alias.pvc }}
+     {{- if not (hasKey $previousMounts $alias.prefix) }}
+      {{- $mountPath := $alias.prefix }}
+      {{- $numMounts := int ( $alias.numMounts | default $plane.numDevices | default 1 ) }}
+      {{- if le $numMounts 1 }}
+- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}
+  mountPath: {{ $mountPath | quote }}
+      {{- else }}
+       {{- range $elem := untilStep 1 (int (add $numMounts 1)) 1 }}
+- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}-many-{{ $elem }}
+  mountPath: {{ printf "%s/d%d" $mountPath $elem | quote }}
+       {{- end }}
+      {{- end }}
+     {{- end }}
+     {{- $_ := set $previousMounts $plane.prefix true -}}
+     {{- $_ := set $curAlias "num" (add $curAlias.num 1) }}
+    {{- end }}
+   {{- end }}
+
   {{- end }}
  {{- end }}
 {{- end }}
@@ -351,26 +386,51 @@ The plane will generate a volume if it matches either an includeLabel or an incl
 {{- $previousMounts := dict -}}
 {{- range $plane := $planes -}}
  {{- if not $plane.disabled -}}
-  {{- if or ($plane.pvc) (hasKey $plane "storageClass") -}}
-   {{- if not (hasKey $previousMounts $plane.prefix) -}}
-    {{- $mountpath := $plane.prefix -}}
-    {{- if or (has $plane.category $includeCategories) (has $plane.name $includeNames) }}
-     {{- $pvc := hasKey $plane "pvc" | ternary $plane.pvc (printf "%s-%s-pvc" (include "hpcc.fullname" $) $plane.name) -}}
-     {{- $num := int ( $plane.numDevices | default 1 ) -}}
-     {{- if le $num 1 }}
+  {{- if or (has $plane.category $includeCategories) (has $plane.name $includeNames) }}
+
+   {{- if or ($plane.pvc) (hasKey $plane "storageClass") -}}
+    {{- if not (hasKey $previousMounts $plane.prefix) }}
+     {{- $pvc := hasKey $plane "pvc" | ternary $plane.pvc (printf "%s-%s-pvc" (include "hpcc.fullname" $) $plane.name) }}
+     {{- $numMounts := int ( $plane.numMounts | default $plane.numDevices | default 1 ) }}
+     {{- if le $numMounts 1 }}
 - name: {{ lower $plane.name }}-pv
   persistentVolumeClaim:
     claimName: {{ $pvc }}
      {{- else }}
-      {{- range $elem := until $num }}
-- name: {{ lower $plane.name }}-pv-many-{{- add $elem 1 }}
+      {{- range $elem := until $numMounts }}
+- name: {{ lower $plane.name }}-pv-many-{{ add $elem 1 }}
   persistentVolumeClaim:
-    claimName: {{ $pvc }}-{{- add $elem 1 }}
+    claimName: {{ $pvc }}-{{ add $elem 1 }}
       {{- end }}
-     {{- end -}}
+     {{- end }}
+     {{- $_ := set $previousMounts $plane.prefix true }}
     {{- end }}
-    {{- $_ := set $previousMounts $plane.prefix true -}}
    {{- end }}
+
+   {{- /*Generate entries for each alias of the plane*/ -}}
+   {{- $curAlias := dict "num" 1 -}}
+   {{- range $alias := $plane.aliases | default list }}
+    {{- if $alias.pvc -}}
+     {{- if not (hasKey $previousMounts $alias.prefix) }}
+      {{- $pvc := $alias.pvc }}
+      {{- $numMounts := int ( $alias.numMounts | default $plane.numDevices | default 1 ) }}
+      {{- if le $numMounts 1 }}
+- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}
+  persistentVolumeClaim:
+    claimName: {{ $pvc }}
+      {{- else }}
+       {{- range $elem := until $numMounts }}
+- name: {{ lower $plane.name }}-pv-alias-{{ $curAlias.num }}-many-{{ add $elem 1 }}
+  persistentVolumeClaim:
+    claimName: {{ $pvc }}-{{ add $elem 1 }}
+       {{- end }}
+      {{- end }}
+      {{- $_ := set $previousMounts $alias.prefix true }}
+      {{- $_ := set $curAlias "num" (add $curAlias.num 1) }}
+     {{- end }}
+    {{- end }}
+   {{- end }}
+
   {{- end }}
  {{- end }}
 {{- end -}}

helm/hpcc/templates/storage.yaml

@@ -36,6 +36,7 @@
   {{- end -}}
   {{- $_ := set $previousPlanes $plane.name true -}}
   {{- $rawPlane := omit $plane "name" "category" "subPath" -}}
+  {{- $numDevices := int ($plane.numDevices | default 1) -}}
   {{- if and (not (hasKey $plane "hosts")) (not (hasKey $plane "hostGroup")) -}}
    {{- if hasKey $previousMounts $plane.prefix -}}
     {{- /* Should this restrict if has a pvc or a storage class? */ -}}
@@ -43,8 +44,19 @@
      {{- required (printf "Multiple incompatible planes refer to prefix '%s'" $plane.prefix) nil -}}
     {{- end -}}
    {{- end -}}
+   {{- $numMounts := int ($plane.numMounts | default $numDevices) -}}
+   {{- if and (ne $numMounts 1) (ne $numMounts $numDevices) }}
+     {{- required (printf "Storage plane '%s': numMounts should either be 1 or match numDevices" $plane.name) nil -}}
+   {{- end -}}
+   {{- $previousMounts := set $previousMounts $plane.prefix $rawPlane -}}
    {{- $_ := set $previousMounts $plane.prefix $rawPlane -}}
   {{- end -}}
+  {{ range $alias := $plane.aliases -}}
+   {{- $numMounts := int ($alias.numMounts | default $numDevices) -}}
+   {{- if and (ne $numMounts 1) (ne $numMounts $numDevices) }}
+     {{- required (printf "Storage plane '%s' alias '%s': numMounts (%d) should either be 1 or match numDevices (%d)" $plane.name $alias.name $numMounts $numDevices) nil -}}
+   {{- end -}}
+  {{- end -}}
  {{- end -}}
 {{- end -}}
 {{- /* check that planes exist for each of the required planes */ -}}

helm/hpcc/values.schema.json

@@ -444,15 +444,80 @@
     "storagePlane": {
       "description": "information about an individual storage plane",
       "type": "object",
+      "allOf": [
+        { "$ref": "#/definitions/storagePlaneCommon" },
+        {
+          "properties": {
+            "name": {
+              "description": "the name of the storage plane",
+              "type": "string"
+            },
+            "disabled": {
+              "description": "disable this plane definition",
+              "type": "boolean"
+            },
+            "numDevices": {
+              "description": "optional number of devices in the storage plane (default 1)",
+              "type": "integer"
+            },
+            "includeDeviceInPath": {
+              "description": "is a directory based on the part number appended to the path.  May not be needed",
+              "type": "boolean"
+            },
+            "replication": {
+              "description": "which planes (if any) the data is replicated onto (primarily bare metal)",
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "category": {
+              "description": "the category this plane is used for, e.g. lz, data",
+              "type": "string",
+              "enum": ["data", "lz", "dali", "sasha", "dll", "spill", "temp", "git", "remote", "debug" ]
+            },
+            "defaultSprayParts" : {
+              "description": "Number of parts sprayed by default",
+              "type" : "integer"
+            },
+            "subDirPerFilePart" : {
+              "description": "Place each logical file part in its own subdirectory",
+              "type": "boolean",
+              "default": true
+            },
+            "aliases" : {
+              "description": "A list of additional ways of accessing the data, along with the reasons why",
+              "type": "array",
+              "items": { "$ref": "#/definitions/storagePlaneAlias" }
+            },
+            "storageSize": {
+              "type": "string"
+            },
+            "storageClass": {
+              "type": "string"
+            },
+            "storageMode": {
+              "type": "string"
+            },
+            "forcePermissions": {
+              "type": "boolean"
+            },
+            "prefix": {},
+            "subPath": {},
+            "secret": {},
+            "pvc": {},
+            "hostGroup": {},
+            "hosts": {},
+            "umask": {},
+            "numMounts": {},
+            "cost": {}
+          },
+          "additionalProperties": false
+        }
+      ],
+      "required": [ "name", "prefix", "category" ]
+    },
+    "storagePlaneCommon": {
+      "$comment": "The following properties make sense either on a storage plane or an an alias.  For that reason they do not include properties that are used to create implement pvcs e.g. storageClass.  All new entries must be added as null entries in storagePlane and storagePlaneAlias",
       "properties": {
-        "name": {
-          "description": "the name of the storage plane",
-          "type": "string"
-        },
-        "disabled": {
-          "description": "disable this plane definition",
-          "type": "boolean"
-        },
         "prefix": {
           "description": "either the path for a local mount, or the url prefix",
           "type": "string"
@@ -478,43 +543,13 @@
           "type": "array",
           "items": { "type": "string" }
         },
-        "numDevices": {
-          "description": "optional number of devices in the storage plane (default 1)",
-          "type": "integer"
-        },
-        "includeDeviceInPath": {
-          "description": "is a directory based on the part number appended to the path.  May not be needed",
-          "type": "boolean"
-        },
-        "replication": {
-          "description": "which planes (if any) the data is replicated onto (primarily bare metal)",
-          "type": "array",
-          "items": { "type": "string" }
-        },
-        "category": {
-          "description": "the category this plane is usd for, e.g. lz, data",
-          "type": "string",
-          "enum": ["data", "lz", "dali", "sasha", "dll", "spill", "temp", "git", "remote", "debug" ]
-        },
         "umask" : {
           "description": "file creation mask (used by despray)",
           "type" : "string"
         },
-        "defaultSprayParts" : {
-          "description": "Number of parts sprayed by default",
-          "type" : "integer"
-        },
-        "storageSize": {
-          "type": "string"
-        },
-        "storageClass": {
-          "type": "string"
-        },
-        "storageMode": {
-          "type": "string"
-        },
-        "forcePermissions": {
-          "type": "boolean"
+        "numMounts": {
+          "description": "the number of mounts for this pvc - defaults to numDevices",
+          "type": "integer"
         },
         "cost" : {
           "description": "Costs associated with the storage and use of the plane",
@@ -525,15 +560,41 @@
               "type": "number"
             }
           }
-        },
-        "subDirPerFilePart" : {
-          "description": "Place each logical file part in its own subdirectory",
-          "type": "boolean",
-          "default": true
         }
-      },
-      "required": [ "name", "prefix", "category" ],
-      "additionalProperties": false
+      }
+    },
+    "storagePlaneAlias": {
+      "$comment": "The following properties are required for providing an alternative path to a data plane",
+      "allOf": [
+        { "$ref": "#/definitions/storagePlaneCommon" },
+        {
+          "properties": {
+            "name": {
+              "description": "the name of the alias",
+              "type": "string"
+            },
+            "mode": {
+              "description": "A list of access modes the engines would use to select this alias",
+              "type": "array",
+              "items": {
+                "type": "string",
+                "enum": [ "write", "sequential", "random", "api"]
+              }
+            },
+            "prefix": {},
+            "subPath": {},
+            "secret": {},
+            "pvc": {},
+            "hostGroup": {},
+            "hosts": {},
+            "umask": {},
+            "numMounts": {},
+            "cost": {}
+          },
+          "additionalProperties": false,
+          "required": [ "name", "mode", "prefix" ]
+        }
+      ]
     },
     "remoteStorage": {
       "description": "remote storage definitions",

testing/helm/errtests/manypvc.yaml

@@ -0,0 +1,45 @@
+storage:
+  planes:
+  - name: dali
+    pvc: dali-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/dalistorage"
+    category: dali
+  - name: dll
+    pvc: dll-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: dll
+  - name: sasha
+    pvc: sasha-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/sasha"
+    category: sasha
+  - name: debug
+    pvc: debug-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/debug"
+    category: debug
+  - name: manydata
+    pvc: manydata-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/manydata"
+    category: data
+    numDevices: 5
+    aliases:
+    - name: cache
+      pvc: cachedata-localmany-hpcc-localfile-pvc
+      prefix: "/var/lib/HPCCSystems/manydatacache"
+      mode: [ random ]
+      numMounts: 1
+    - name: cache2
+      pvc: cachedata-localmany-hpcc-localfile-pvc
+      prefix: "/var/lib/HPCCSystems/manydatacache2"
+      mode: [ sequential ]
+      numMounts: 3
+  - name: mydropzone
+    pvc: mydropzone-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/dropzone"
+    category: lz
+
+
+sasha:
+  wu-archiver:
+    plane: sasha
+  dfuwu-archiver:
+    plane: sasha

testing/helm/errtests/nodataplane.yaml

@@ -0,0 +1,22 @@
+storage:
+  planes:
+  - name: dali
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/dalistorage"
+    category: dali
+  - name: sasha
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/sashastorage"
+    category: sasha
+  - name: dll
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: dll
+  - name: mydropzone
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/mydropzone"
+    category: lz

testing/helm/errtests/nomode.yaml

@@ -0,0 +1,41 @@
+storage:
+  planes:
+  - name: dali
+    pvc: dali-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/dalistorage"
+    category: dali
+  - name: dll
+    pvc: dll-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: dll
+  - name: sasha
+    pvc: sasha-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/sasha"
+    category: sasha
+  - name: debug
+    pvc: debug-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/debug"
+    category: debug
+  - name: manydata
+    pvc: manydata-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/manydata"
+    category: data
+    numDevices: 5
+    aliases:
+    - pvc: cachedata-localmany-hpcc-localfile-pvc
+      mode: [ random ]
+      numMounts: 1
+    - pvc: cachedata-localmany-hpcc-localfile-pvc
+      prefix: "/var/lib/HPCCSystems/manydatacache2"
+      numMounts: 3
+  - name: mydropzone
+    pvc: mydropzone-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/dropzone"
+    category: lz
+
+
+sasha:
+  wu-archiver:
+    plane: sasha
+  dfuwu-archiver:
+    plane: sasha

testing/helm/run.sh

@@ -25,6 +25,15 @@ do
       cat errors.txt
       cat results.txt
       failed=1
+   else
+      helm template $hpccchart ${options} --values $file > results.txt 2> errors.txt
+      if [ $? -ne 0 ]
+      then
+         echo $file failed
+         cat errors.txt
+         cat results.txt
+         failed=1
+      fi
    fi
 done
 

testing/helm/tests/aliasplanes.yaml

@@ -0,0 +1,32 @@
+storage:
+  planes:
+  - name: dali
+    pvc: dali-gcpstorage-hpcc-filestore-pvc
+    prefix: "/var/lib/HPCCSystems/dalistorage"
+    category: dali
+
+  - name: dll
+    pvc: dll-gcpstorage-hpcc-filestore-pvc
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: dll
+
+  - name: sasha
+    pvc: sasha-gcpstorage-hpcc-filestore-pvc
+    prefix: "/var/lib/HPCCSystems/sasha"
+    category: sasha
+
+  - name: data
+    pvc: data-blob-nfs-pvc
+    prefix: "/var/lib/HPCCSystems/hpcc-data"
+    category: data
+    numDevices: 5
+    aliases:
+    - name: cache
+      mode: [ random ]
+      pvc: data-cache-pvc
+      prefix: "/var/lib/HPCCSystems/hpcc-data-cache"
+      numMounts: 1
+    - name: direct
+      mode: [ api ]
+      prefix: azure://ghallidaystorage  # not really sure what this syntax should be....
+      secret: azure-ghallidaystorage

testing/helm/tests/baremetal.yaml

@@ -38,7 +38,7 @@ storage:
     hostGroup: thor100_4
     replication: [ azureBlobPlane ]
 
-  - name: azureBlobPlane
+  - name: azureblobplane
     category: data
     prefix: azure://ghallidaystorage      # Not sure if this should be different from the mount.
     secret: azure-ghallidaystorage

testing/helm/tests/baremetal2.yaml

@@ -18,31 +18,31 @@ storage:
       delta: 1
 
   planes:
-  - name: demoOne
+  - name: demo-one
     category: data
     prefix: /home/gavin/temp
     hostGroup: demoOne
-  - name: myDropZone
+  - name: my-drop-zone
     prefix: /home/gavin/temp
     hosts: [ '192.168.86.202']
     category: 'lz'
-  - name: demoTwo
+  - name: demo-two
     prefix: /home/gavin/temp/demoTwo
     hostGroup: demoTwo
     category: data
-  - name: demoTwoReplica
+  - name: demo-two-replica
     prefix: /home/gavin/temp/demoTwo
     hostGroup: demoThree
     category: data
-  - name: demoTwoA
+  - name: demo-two-a
     prefix: /home/gavin/temp/demoTwoA
     hostGroup: demoTwoA
     category: data
-  - name: demoTwoB
+  - name: demo-two-b
     prefix: /home/gavin/temp/demoTwoB
     hostGroup: demoTwoB
     category: data
-  - name: myInlinePlane
+  - name: my-inline-plane
     prefix: /home/gavin/temp
     hosts: [ 'here', 'there', 'everywhere']
     category: data

testing/helm/tests/complex.yaml

@@ -2,40 +2,40 @@ storage:
   planes:
   #Some examples of the different kinds of storage planes that are supported
   #Store data on azure blobs
-  - name: azureBlobPlane
+  - name: azureblobplane
     prefix: azure://ghallidaystorage      # Not sure if this should be different from the mount.
     secret: azure-ghallidaystorage
     category: data
 
   #Store data on aws s3 buckets
-  - name: s3BucketPlane
+  - name: s3bucketplane
     prefix: s3://...
     category: data
 
   #Single node with data mounted, and mirror mounted at a different locations (could be a different disk)
-  - name: localDataPlane
+  - name: localdataplane
     prefix: /var/lib/hpccsystems/hpcc-data
     replication: [ localMirrorPlane, localMirror2Plane ]
     pvc: local-data-pvc
     category: data
-  - name: localMirrorPlane
+  - name: localmirrorplane
     prefix: /var/lib/hpccsystems/hpcc-mirror
     pvc: local-mirror-pvc
     category: data
-  - name: localMirror2Plane
+  - name: localmirror2plane
     prefix: /var/lib/hpccsystems/hpcc-mirror2
     pvc: local-mirror2-pvc
     category: data
 
   #Multiple nodes, data on a local mounts (all nodes mount the same logical file system).
   #Essentially identical to localDataPlane above
-  - name: nasPlane
-    prefix: /var/lib/hpccsystems/hpcc-data
+  - name: nasplane
+    prefix: /var/lib/hpccsystems/hpcc-data2
     replication: [ nasMirrorPlane ]
     pvc: nas-data-pvc
     category: data
-  - name: nasMirrorPlane
-    prefix: /var/lib/hpccsystems/hpcc-mirror
+  - name: nasmirrorplane
+    prefix: /var/lib/hpccsystems/hpcc-mirror3
     pvc: nas-mirror-pvc
     category: data
 
@@ -43,8 +43,8 @@ storage:
   #Allows an array of NFS servers to be used to store the data.  Would also potentially work for
   #local case with large numbers of local disks.
   #Mount point is /var/lib/hpccsystems/hpcc-data/d<device-number>
-  - name: nasArrayPlane
-    prefix: /var/lib/hpccsystems/hpcc-data
+  - name: nasarrayplane
+    prefix: /var/lib/hpccsystems/hpcc-data3
     pvc: nas-array-data-pvc
     numDevices: 100
     includeDeviceInPath: true
@@ -55,3 +55,14 @@ storage:
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/sashastorage"
     category: sasha
+
+  - name: dali
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/sashastorage"
+    category: dali
+  - name: dll
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/sashastorage"
+    category: dll

testing/helm/tests/manypvc.yaml

@@ -0,0 +1,49 @@
+storage:
+  planes:
+  - name: dali
+    pvc: dali-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/dalistorage"
+    category: dali
+  - name: dll
+    pvc: dll-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: dll
+  - name: sasha
+    pvc: sasha-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/sasha"
+    category: sasha
+  - name: debug
+    pvc: debug-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/debug"
+    category: debug
+  - name: manydata
+    pvc: manydata-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/manydata"
+    category: data
+    numDevices: 5
+    aliases:
+    - name: cache
+      pvc: cachedata-localmany-hpcc-localfile-pvc
+      prefix: "/var/lib/HPCCSystems/manydatacache"
+      mode: [ random ]
+      numMounts: 1
+    - name: cache2
+      pvc: cachedata-localmany-hpcc-localfile-pvc
+      prefix: "/var/lib/HPCCSystems/manydatacache2"
+      mode: [ sequential ]
+    - name: direct
+      pvc: cachedata-localmany-hpcc-localfile-pvc
+      prefix: "/var/lib/HPCCSystems/manydatacache2"
+      mode: [ api ]
+      numMounts: 5
+  - name: mydropzone
+    pvc: mydropzone-localmany-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/dropzone"
+    category: lz
+
+
+sasha:
+  wu-archiver:
+    plane: sasha
+  dfuwu-archiver:
+    plane: sasha

testing/helm/tests/newvalues.yaml

@@ -0,0 +1,28 @@
+storage:
+  planes:
+  - name: hpcc-data-plane
+    prefix: "/var/lib/HPCCSystems/hpcc-data"
+    storageSize: 1Gi
+    storageClass: ""
+    category: data
+  - name: hpcc-dll-plane
+    prefix: "/var/lib/HPCCSystems/queries"
+    storageSize: 3Gi
+    storageClass: ""
+    forcePermissions: true
+    category: dll
+  - name: hpcc-dali-plane
+    prefix: "/var/lib/HPCCSystems/dalistorage"
+    storageSize: 1Gi
+    storageClass: ""
+    category: dali
+  - name: landingzone
+    prefix: "/var/lib/HPCCSystems/landingzone"
+    storageSize: 1Gi
+    storageClass: ""
+    category: lz
+  - name: sasha
+    prefix: "/var/lib/HPCCSystems/sasha"
+    storageSize: 1Gi
+    storageClass: ""
+    category: sasha