HPCC-26134 Final helm file storage plane changes

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

dali/base/dafdesc.cpp

@@ -3456,12 +3456,13 @@ void initializeStorageGroups(bool createPlanesFromGroups)
 bool getDefaultStoragePlane(StringBuffer &ret)
 {
     // If the plane is specified for the component, then use that
-    if (getComponentConfigSP()->getProp("@storagePlane", ret))
+    if (getComponentConfigSP()->getProp("@dataPlane", ret))
         return true;
 
     //Otherwise check what the default plane for data storage is configured to be
-    if (getGlobalConfigSP()->getProp("storage/@dataPlane", ret))
-        return true;
+    Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[@category='data']");
+    if (dataPlanes->first())
+        return dataPlanes->query().getProp("@name", ret);
 
 #ifdef _CONTAINERIZED
     throwUnexpectedX("Default data plane not specified"); // The default should always have been configured by the helm charts

esp/smc/SMCLib/TpContainer.cpp

@@ -797,7 +797,7 @@ StringBuffer & getRoxieDefaultPlane(StringBuffer & plane, const char * roxieName
     if (!queue)
         throw makeStringExceptionV(ECLWATCH_INVALID_CLUSTER_NAME, "Unknown queue name %s", roxieName);
 
-    if (queue->getProp("@storagePlane", plane))
+    if (queue->getProp("@dataPlane", plane))
         return plane;
 
     //Find the first data plane - better if it was retrieved from roxie config

helm/examples/azure/values-auto-azurefile.yaml

@@ -34,19 +34,8 @@ storage:
     prefix: "/var/lib/HPCCSystems/mydropzone"
     category: lz
 
-  daliStorage:
-    plane: dali
-
-  dllStorage:
-    plane: dll
-
-  dataStorage:
-    plane: data
-
 sasha:
   wu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha
   dfuwu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha

helm/examples/azure/values-retained-azurefile.yaml

@@ -37,20 +37,8 @@ storage:
     prefix: "/var/lib/HPCCSystems/mydropzone"
     category: lz
 
-
-  daliStorage:
-    plane: dali
-
-  dllStorage:
-    plane: dll
-
-  dataStorage:
-    plane: data
-
 sasha:
   wu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha
   dfuwu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha

helm/examples/efs/values-auto-efs.yaml

@@ -34,19 +34,8 @@ storage:
     prefix: "/var/lib/HPCCSystems/mydropzone"
     category: lz
 
-  daliStorage:
-    plane: dali
-
-  dllStorage:
-    plane: dll
-
-  dataStorage:
-    plane: data
-
 sasha:
   wu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha
   dfuwu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha

helm/examples/efs/values-retained-efs.yaml

@@ -32,20 +32,8 @@ storage:
     prefix: "/var/lib/HPCCSystems/mydropzone"
     category: lz
 
-
-  dllStorage:
-    plane: dll
-
-  daliStorage:
-    plane: dali
-
-  dataStorage:
-    plane: data
-
 sasha:
   wu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha
   dfuwu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha

helm/examples/filestore/values-filestore.yaml

@@ -27,20 +27,8 @@ storage:
     prefix: "/var/lib/HPCCSystems/hpcc-data"
     category: data # NB: if not set, this would be the default
 
-
-  dllStorage:
-    plane: dll
-
-  daliStorage:
-    plane: dali
-
-  dataStorage:
-    plane: data
-
 sasha:
   wu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha
   dfuwu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha

helm/examples/local/values-localfile.yaml

@@ -37,20 +37,8 @@ storage:
     prefix: "/var/lib/HPCCSystems/mydropzone"
     category: lz
 
-
-  daliStorage:
-    plane: dali
-
-  dllStorage:
-    plane: dll
-
-  dataStorage:
-    plane: data
-
 sasha:
   wu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha
   dfuwu-archiver:
-    storage:
-      plane: sasha
+    plane: sasha

helm/examples/nfs/values-nfs.yaml

@@ -21,32 +21,15 @@ storage:
     pvc: sasha-nfsstorage-hpcc-nfs-pvc
     prefix: "/var/lib/HPCCSystems/sasha"
     category: sasha
+    forcePermissions: true
 
   - name: data
     pvc: data-nfsstorage-hpcc-nfs-pvc
     prefix: "/var/lib/HPCCSystems/hpcc-data"
     category: data
 
-
-  dllStorage:
-    plane: dll
-    forcePermissions: true
-
-  daliStorage:
-    plane: dali
-    forcePermissions: true
-
-  dataStorage:
-    plane: data
-    forcePermissions: true
-
-
 sasha:
   wu-archiver:
-    storage:
-      plane: sasha
-      forcePermissions: true
+    plane: sasha
   dfuwu-archiver:
-    storage:
-      plane: sasha
-      forcePermissions: true
+    plane: sasha

helm/examples/storageplanes/complex.yaml

@@ -6,26 +6,33 @@ storage:
   - name: azureBlobPlane
     prefix: azure://ghallidaystorage      # Not sure if this should be different from the mount.
     secret: azure-ghallidaystorage
+    category: data
 
   - name: azureDllPlane
     prefix: azure://ghallidaydllstore      # Not sure if this should be different from the mount.
     secret: azure-ghallidaydllstore
+    category: dll
 
   #Store data on aws s3 buckets
   - name: s3BucketPlane
     prefix: s3://...
+    category: data
 
   #Single node with data mounted, and mirror mounted at a different locations (could be a different disk)
   - name: localDataPlane
     prefix: /var/lib/hpccsystems/hpcc-data
     replication: [ localMirrorPlane, localMirror2Plane ]
     pvc: "local-data-pvc"
+    category: data
+
   - name: localMirrorPlane
     prefix: /var/lib/hpccsystems/hpcc-mirror
     pvc: "local-mirror-pvc"
+    category: data
   - name: localMirror2Plane
     prefix: /var/lib/hpccsystems/hpcc-mirror2
     pvc: "local-mirror2-pvc"
+    category: data
 
   #Multiple nodes, data on a local mounts (all nodes mount the same logical file system).
   #Essentially identical to localDataPlane above
@@ -33,9 +40,11 @@ storage:
     prefix: /var/lib/hpccsystems/hpcc-data
     replication: [ nasMirrorPlane ]
     pvc: "nas-data-pvc"
+    category: data
   - name: nasMirrorPlane
     prefix: /var/lib/hpccsystems/hpcc-mirror
     pvc: "nas-mirror-pvc"
+    category: data
 
   #Multiple nodes, data on multiple local mounts (all nodes mount the same logical file system).
   #Allows an array of NFS servers to be used to store the data.  Would also potentially work for
@@ -46,13 +55,9 @@ storage:
     pvc: "nas-array-data-pvc"
     numDevices: 100
     includeDeviceInPath: true
+    category: data
 
   # A locally mounted directory
   - name: localNvmePlane
     prefix: /tmp/hpcc/spill
-
-  #The following defines the default data plane (used if a component does not specify one)
-  dataStorage:
-    plane: localDataPlane
-  dllStorage:
-    plane: azureDllPlane
+    category: spill

helm/hpcc/docs/changes.md

@@ -129,8 +129,15 @@ In 8.2.x all storage for data, dlls, landing zones etc. is defined in the list o
 
 Previously daliStorage, dataStorage and dllStorage could either refer to an existing plane, or could implicitly
 define a storage plane.  That implicit plane could be based on a pre-existing persistent volume claim (by setting
-existingClaim) or ephemeral storage (by setting storageClass and storageSize).  In version 8.2. these sections can
-only refer to a plane defined in the plane list.
+existingClaim) or ephemeral storage (by setting storageClass and storageSize).  In version 8.2. these sections are no longer
+supported.
+
+Implicit planes that were defined within those sections must now use an explicit plane definition.  The plane definitions
+have been extended to allow ephemeral storage to be defined (see dali example below).
+
+Previously the default storage plane for a particular storage category could be specified by naming it in the appropriate storage
+section.  E.g. storage.dataStorage.plane could name the default data storage plane. The default is now the first plane with the
+corresponding category.
 
 Another change is that previously a storage plane had a labels: attribute to indicate what kind of data was stored on
 the plane.  It was a list, and if blank defaulted to \[ data \].  This has now become a single valued "category" attribute.
@@ -154,9 +161,6 @@ storage:
     pvc: my-pvc
     prefix: "/var/lib/HPCCSystems/dalistorage"
     category: dali
-
-  daliStorage:
-     name: dali
 ```
 
 And similarly a definition that uses ephemeral storage:
@@ -177,9 +181,29 @@ storage:
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/dalistorage"
     category: dali
+```
 
-  daliStorage:
-     name: dali
+The default values.yaml and example files have been updated to reflect these changes.
+
+## Default data plane changes
+
+Previously the storagePlane property could be set on an engine to change the default data plane.  This has now been
+renamed to dataPlane for consistency with the category name.  In future other types of planes will be configurable
+in the same way (spill, dll, temp).
+
+Sasha services used to define their storage plane within a nested storage attribute:
+
+```sasha:
+  wu-archiver:
+    storage:
+      plane: sasha
+```
+
+This is now simplified to:
+
+```sasha:
+  wu-archiver:
+    plane: sasha
 ```
 
-The default values.yaml and example files have been updated to reflect this change.
+A value of "" can be used mean use the 1st plane with the sasha catagory.

helm/hpcc/templates/_helpers.tpl

@@ -68,6 +68,10 @@ Translate a port list to a comma-separated list
  {{- end -}}
 {{- end -}}
 
+{{/*
+Return the name of the first plane that matches a particular category
+Pass in dict with root, category
+*/}}
 {{- define "hpcc.getFirstPlaneForCategory" -}}
 {{- $root := .root -}}
 {{- $category := .category -}}
@@ -93,23 +97,35 @@ true
 {{- end -}}
 
 {{/*
+Return the name of the default plane for this component that matches a particular category
+Pass in dict with root, category, me
+*/}}
+{{- define "hpcc.getDefaultPlane" -}}
+{{- $storage := (.root.Values.storage | default dict) -}}
+{{- $planeKey := (printf "%sPlane" .category) -}}
+{{- $firstPlane := (include "hpcc.getFirstPlaneForCategory" .) -}}
+{{- get .me $planeKey | default $firstPlane -}}
+{{- end -}}
+
+{{/*
 Get default data plane
 */}}
 {{- define "hpcc.getDefaultDataPlane" -}}
-{{- $storage := ($.Values.storage | default dict) -}}
-{{- $dataStorage := ($storage.dataStorage | default dict) -}}
-{{- $firstPlane := (include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "data")) -}}
-{{- $dataStorage.plane | default $firstPlane | default "hpcc-data-plane" -}}
+{{- include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "data") -}}
+{{- end -}}
+
+{{/*
+Get default spill plane
+*/}}
+{{- define "hpcc.getDefaultSpillPlane" -}}
+{{- include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "spill") -}}
 {{- end -}}
 
 {{/*
 Get default dll plane
 */}}
 {{- define "hpcc.getDefaultDllPlane" -}}
-{{- $storage := ($.Values.storage | default dict) -}}
-{{- $dllStorage := ($storage.dllStorage | default dict) -}}
-{{- $firstPlane := (include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "dll")) -}}
-{{- $dllStorage.plane | default $firstPlane | default "hpcc-dll-plane" -}}
+{{- include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "dll") -}}
 {{- end -}}
 
 {{/*
@@ -120,10 +136,6 @@ Pass in root as .
 {{- /*Create local variables which always exist to avoid having to check if intermediate key values exist*/ -}}
 {{- $storage := (.Values.storage | default dict) -}}
 {{- $planes := ($storage.planes | default list) -}}
-{{- $dataStorage := ($storage.dataStorage | default dict) -}}
-{{- $spillStorage := ($storage.spillStorage | default dict) -}}
-{{- $daliStorage := ($storage.daliStorage | default dict) -}}
-{{- $dllStorage := ($storage.dllStorage | default dict) -}}
 {{- $certificates := (.Values.certificates | default dict) -}}
 {{- $issuers := ($certificates.issuers | default dict) -}}
 mtls: {{ and ($certificates.enabled) (hasKey $issuers "local") }}
@@ -153,20 +165,10 @@ storage:
     {{- toYaml $planeYaml | nindent 4 }}
  {{- end }}
 {{- end }}
-{{- /* Add implicit planes if data or spill storage plane not specified*/ -}}
-{{- if not $dataStorage.plane }}
-{{- if not (include "hpcc.hasPlaneForCategory" (dict "root" $ "category" "data")) }}
-  - name: hpcc-data-plane
-    category: data
-    prefix: {{ .Values.global.defaultDataPath | default "/var/lib/HPCCSystems/hpcc-data" | quote }}
-{{- end }}
-{{- end }}
-{{- if not $spillStorage.plane }}
 {{- if not (include "hpcc.hasPlaneForCategory" (dict "root" $ "category" "spill")) }}
   - name: hpcc-spill-plane
     prefix: {{ .Values.global.defaultSpillPath | default "/var/lib/HPCCSystems/hpcc-spill" | quote }}
 {{- end }}
-{{- end }}
 {{- if .Values.global.cost }}
 cost:
 {{ toYaml .Values.global.cost | indent 2 }}
@@ -236,7 +238,7 @@ Add ConfigMap volume for a component
 
 {{/*
 Add volume mounts
-Pass in root and includeCategories (optional)
+Pass in root and includeCategories (optional) and/or includeNames (optional)
 Note: if there are multiple planes (other than dll, dali and spill planes), they should be all called with a single call
 to addVolumeMounts so that if a plane can be used for multiple purposes then duplicate volume mounts are not created.
 */}}
@@ -245,12 +247,13 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup
 {{- $storage := (.root.Values.storage | default dict) -}}
 {{- $planes := ($storage.planes | default list) -}}
 {{- $includeCategories := .includeCategories | default list -}}
+{{- $includeNames := .includeNames | default list -}}
 {{- $previousMounts := dict -}}
 {{- range $plane := $planes -}}
  {{- if or ($plane.pvc) (hasKey $plane "storageClass") -}}
   {{- if not (hasKey $previousMounts $plane.prefix) -}}
    {{- $mountpath := $plane.prefix -}}
-   {{- if has $plane.category $includeCategories }}
+   {{- if or (has $plane.category $includeCategories) (has $plane.name $includeNames) }}
     {{- $num := int ( $plane.numDevices | default 1 ) -}}
     {{- if le $num 1 }}
 - name: {{ lower $plane.name }}-pv
@@ -266,18 +269,6 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup
   {{- end }}
  {{- end }}
 {{- end }}
-{{- /*
-Create a data volume mount if data plane have not been specified in storage.planes
-Note: Some services used addVolumeMounts to add data planes and other types of plane using addVolumeMounts, so this code has
-to be located here rather than in addDataVolumeMount.
-*/ -}}
-{{- $dataStorage := ($storage.dataStorage | default dict) -}}
-{{- if and (has "data" $includeCategories) (not $dataStorage.plane) }}
-{{- if (not (include "hpcc.hasPlaneForCategory" (dict "root" .root "category" "data"))) }}
-- name: datastorage
-  mountPath: "/var/lib/HPCCSystems/hpcc-data"
-{{- end }}
-{{- end }}
 {{- end -}}
 
 {{/*
@@ -296,7 +287,6 @@ The plane will generate a volume if it matches either an includeLabel or an incl
 {{- define "hpcc.addVolumes" -}}
 {{- /*Create local variables which always exist to avoid having to check if intermediate key values exist*/ -}}
 {{- $storage := (.root.Values.storage | default dict) -}}
-{{- $dataStorage := ($storage.dataStorage | default dict) -}}
 {{- $planes := ($storage.planes | default list) -}}
 {{- $includeCategories := .includeCategories | default list -}}
 {{- $includeNames := .includeNames | default list -}}
@@ -336,37 +326,20 @@ Pass in dict with root
 
 {{/*
 Add a volume mount - if default plane is used, or the storage plane specifies a pvc
-Pass in dict with root, me, name, and optional path
+Pass in dict with root, planeName
 */}}
-{{- define "hpcc.getVolumeMountPrefix" -}}
+{{- define "hpcc.getPlanePrefix" -}}
 {{- $storage := (.root.Values.storage | default dict) -}}
 {{- $planes := ($storage.planes | default list) -}}
-{{- if .me.plane -}}
-{{- $me := .me -}}
- {{- range $plane := $planes -}}
-  {{- if and (or ($plane.pvc) (hasKey $plane "storageClass")) (eq $plane.name $me.plane) -}}
-{{ $plane.prefix }}
-  {{- end -}}
+{{- $name := .planeName -}}
+{{- range $plane := $planes -}}
+ {{- if (eq $plane.name $name) -}}
+  {{- $plane.prefix -}}
  {{- end -}}
-{{- else -}}
- {{- $_ := fail (printf "Volume mount %s does not define a storage plane" .name ) -}}
 {{- end -}}
 {{- end -}}
 
 {{/*
-Add a volume mount - if default plane is used, or the storage plane specifies a pvc
-Pass in dict with root, me, name, and optional path
-*/}}
-{{- define "hpcc.addVolumeMount" -}}
-{{- $mountPath := include "hpcc.getVolumeMountPrefix" . }}
-{{- if not $mountPath -}}
- {{- $_ := fail (printf "Missing plane definition for plane: %s" .me.plane ) -}}
-{{- end -}}
-- name: {{ .name }}
-  mountPath: {{ $mountPath }}
-{{- end -}}
-
-{{/*
 Add dll volume mount - if default plane is used, or the dll storage plane specifies a pvc
 Pass in dict with root
 */}}
@@ -485,9 +458,7 @@ Check whether a storage plane is defined or not.
 {{- $category := .category -}}
 {{- $storage := (.root.Values.storage | default dict) -}}
 {{- $planes := ($storage.planes | default list) -}}
-{{- $dataStorage := ($storage.dataStorage | default dict) -}}
-{{- /* If storage.dataStorage.plane is defined, the implicit plane hpcc-dataplane is not defined */ -}}
-{{- $done := dict "matched" (and (not $dataStorage.plane) (eq $search "hpcc-dataplane")) "all" "" -}}
+{{- $done := dict -}}
 {{- range $plane := $planes -}}
  {{- if eq $category $plane.category -}}
   {{- if eq $search $plane.name -}}
@@ -505,8 +476,8 @@ Check whether a storage plane is defined or not.
 Check that the storage and spill planes for a component exist
 */}}
 {{- define "hpcc.checkDefaultStoragePlane" -}}
-{{- if (hasKey .me "storagePlane") }}
- {{- $search := .me.storagePlane -}}
+{{- if (hasKey .me "dataPlane") }}
+ {{- $search := .me.dataPlane -}}
  {{- include "hpcc.checkValidStoragePlane" (dict "search" $search "root" .root "category" "data" "type" "storage data" "for" .me.name) -}}
 {{- end }}
 {{- if (hasKey .me "spillPlane") }}
@@ -726,7 +697,7 @@ Generate instance queue names
   type: roxie 
   prefix: {{ .prefix | default "null" }}
   queriesOnly: true
-  storagePlane: {{ .storagePlane | default (include "hpcc.getDefaultDataPlane" $) }}
+  dataPlane: {{ .dataPlane | default (include "hpcc.getDefaultDataPlane" $) }}
  {{- end }}
 {{ end -}}
 {{- range $.Values.thor -}}
@@ -850,8 +821,10 @@ data:
 {{ toYaml (omit .me "logging") | indent 6 }}
 {{- include "hpcc.generateLoggingConfig" . | indent 6 }}
 {{ include "hpcc.generateVaultConfig" . | indent 6 }}
-{{- if .me.storage }}
-      storagePath: {{ include "hpcc.getVolumeMountPrefix" (dict "root" .root "me" .me.storage "name" (printf "sasha-%s" .me.name) ) }}
+{{- if hasKey .me "plane" }}
+ {{- $sashaStoragePlane := .me.plane | default (include "hpcc.getFirstPlaneForCategory" (dict "root" .root "category" "sasha")) }}
+ {{- $_ := set .me "plane" $sashaStoragePlane }}
+      storagePath: {{ include "hpcc.getPlanePrefix" (dict "root" .root "planeName" $sashaStoragePlane) }}
 {{- end }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
@@ -890,9 +863,9 @@ Pass in dict with root and me
 */}}
 {{- define "hpcc.addSashaVolumeMounts" }}
 {{- $serviceName := printf "sasha-%s" .me.name -}}
-{{- if .me.storage }}
-{{- $volumeName := (hasKey .me.storage "plane") | ternary (printf "%s-pv" .me.storage.plane) $serviceName }}
-{{ include "hpcc.addVolumeMount" (dict "root" .root "me" .me.storage "name" $volumeName) -}}
+{{- if hasKey .me "plane" }}
+{{- $sashaStoragePlane := .me.plane | default (include "hpcc.getFirstPlaneForCategory" (dict "root" .root "category" "sasha")) }}
+{{ include "hpcc.addVolumeMounts" (dict "root" .root "includeNames" (list $sashaStoragePlane)) -}}
 {{- end }}
 {{ with (dict "name" $serviceName ) -}}
 {{ include "hpcc.addConfigMapVolumeMount" . }}
@@ -915,8 +888,9 @@ Pass in dict with root and me
 */}}
 {{- define "hpcc.addSashaVolumes" }}
 {{- $serviceName := printf "sasha-%s" .me.name -}}
-{{- if .me.storage }}
-{{ include "hpcc.addVolumes" (dict "root" .root "includeNames" (list .me.storage.plane) ) }}
+{{- if hasKey .me "plane" }}
+{{- $sashaStoragePlane := .me.plane | default (include "hpcc.getFirstPlaneForCategory" (dict "root" .root "category" "sasha")) }}
+{{ include "hpcc.addVolumes" (dict "root" .root "includeNames" (list $sashaStoragePlane) ) }}
 {{- end }}
 {{ with (dict "name" $serviceName) -}}
 {{ include "hpcc.addConfigMapVolume" . }}

helm/hpcc/templates/dali.yaml

@@ -60,8 +60,9 @@ spec:
       {{- if not ( $sashaServices.disabled ) -}}
       {{- range $sashaName, $_sasha := $dali.services -}}
       {{- $sasha := ($_sasha | default dict) -}}
-      {{- if and (not $sasha.disabled) ($sasha.storage) }}
-        {{- $_ := append $sashaPlanes $sasha.storage.plane }}
+      {{- if and (not $sasha.disabled) (hasKey $sasha "plane") }}
+        {{- $sashaStoragePlane := $sasha.plane | default (include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "sasha")) }}
+        {{- $_ := append $sashaPlanes $sashaStoragePlane }}
       {{- end }}
       {{- end }}
       {{- end }}
@@ -126,8 +127,7 @@ spec:
 {{- include "hpcc.addSecretVolumes" (dict "root" $ "secretsCategories" $tmpDaliScope.aggregateSashaSecretsCategories) | indent 6 }}
 {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "dali" "external" false) | indent 6 }}
 ---
-{{- $storage := ($.Values.storage | default dict) -}}
-{{- $daliStorage := ($storage.daliStorage | default dict) }}
+{{- $daliStoragePlane := $dali.daliPlane | default (include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "dali")) }}
 kind: ConfigMap 
 apiVersion: v1 
 metadata:
@@ -137,7 +137,7 @@ data:
     version: 1.0
     dali:
 {{ toYaml (omit $dali "logging" "services" "env") | indent 6 }}
-      dataPath: {{ include "hpcc.getVolumeMountPrefix" (dict "root" $ "me" $daliStorage "path" "dalistorage") }}
+      dataPath: {{ include "hpcc.getPlanePrefix" (dict "root" $ "planeName" $daliStoragePlane) }}
 {{- include "hpcc.generateLoggingConfig" $commonCtx | indent 6 }}
 {{ include "hpcc.generateVaultConfig" (dict "root" $ "secretsCategories" $daliSecretsCategories ) | indent 6 }}
     global:

helm/hpcc/templates/sasha.yaml

@@ -25,8 +25,7 @@
 {{- if not $sashaServices.disabled -}}
 {{ range $sashaName, $_sasha := $sashaServices -}}
 {{ $sasha := ($_sasha | default dict) -}}
-{{- $sashaStorage := $sasha.storage | default dict -}}
-{{- $sashaPlane := $sashaStorage.plane -}}
+{{- $sashaPlane := $sasha.plane | default (include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "sasha")) }}
 {{- if not $sasha.disabled -}}
 {{- $_ := set $sasha "name" $sashaName -}}
 {{- $_ := set $sasha "access" (splitList " " (include "hpcc.getSashaServiceAccess" $sasha)) -}}
@@ -62,7 +61,7 @@ spec:
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $serviceName "type" "sasha") | indent 6 }}
       serviceAccountName: "hpcc-default"
-      {{- if $sasha.storage }}
+      {{- if hasKey $sasha "plane" }}
       initContainers:
       {{ include "hpcc.changePlaneMountPerms" (dict "root" $ "includeNames" (list $sashaPlane)) | indent 6}}
       {{- end }}

helm/hpcc/templates/storage.yaml

@@ -43,4 +43,51 @@
  {{- end -}}
  {{- $_ := set $previousMounts $plane.prefix $rawPlane -}}
 {{- end -}}
+{{- /* check that planes exist for each of the required planes */ -}}
+{{ $need := dict }}
+{{- range $.Values.dali -}}
+ {{- if not .disabled -}}
+   {{- $_ := set $need "dali" true -}}
+ {{- end }}
+{{ end -}}
+{{- range $.Values.eclccserver -}}
+ {{- if not .disabled -}}
+   {{- $_ := set $need "dll" true -}}
+ {{- end }}
+{{ end -}}
+{{- range $.Values.eclscheduler -}}
+ {{- if not .disabled -}}
+   {{- $_ := set $need "dali" true -}}
+   {{- $_ := set $need "dll" true -}}
+ {{- end }}
+{{ end -}}
+{{- range $.Values.esp -}}
+ {{- if not .disabled -}}
+  {{- /* placeholder .. if any planes are required */ -}}
+ {{- end }}
+{{ end -}}
+{{- range $.Values.eclagent -}}
+ {{- if not .disabled -}}
+   {{- $_ := set $need "data" true -}}
+   {{- $_ := set $need "dll" true -}}
+   {{- $_ := set $need "dali" true -}}
+ {{- end }}
+{{ end -}}
+{{- range $.Values.roxie -}}
+ {{- if not .disabled -}}
+   {{- $_ := set $need "data" true -}}
+   {{- $_ := set $need "dll" true -}}
+ {{- end }}
+{{ end -}}
+{{- range $.Values.thor -}}
+ {{- if not .disabled -}}
+   {{- $_ := set $need "data" true -}}
+   {{- $_ := set $need "dll" true -}}
+   {{- $_ := set $need "dali" true -}}
+ {{- end }}
+{{ end -}}
+{{- range $category, $value := $need }}
+ {{- $_ := required (printf "No plane defined for category '%s'" $category) (include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" $category)) }}
+{{- end -}}
+{{- /* finally generate the pvcs for each of the storage planes that need one */ -}}
 {{- include "hpcc.addPVCsFromPlanes" . }}

helm/hpcc/values.schema.json

@@ -26,25 +26,6 @@
     "storage": {
       "type": "object",
       "properties": {
-        "dllStorage": {
-          "$ref": "#/definitions/storageType"
-        },
-        "dataStorage": {
-          "$ref": "#/definitions/storageType"
-        },
-        "daliStorage": {
-          "$ref": "#/definitions/storageType"
-        },
-        "spillStorage": {
-          "oneOf": [
-            {
-              "$ref": "#/definitions/storageType"
-            },
-            {
-              "type": "null"
-            }
-          ]
-        },
         "hostGroups": {
           "$ref": "#/definitions/hostGroups"
         },
@@ -169,7 +150,7 @@
               "type": "string",
               "description": "The (optional) file prefix to add to relative filenames"
             },
-            "storagePlane": {
+            "dataPlane": {
               "description": "The default storage plane to write data files to",
               "type": "string"
             },
@@ -382,16 +363,6 @@
       },
       "additionalProperties": false
     },
-    "storageType": {
-      "type": "object",
-      "properties": {
-        "plane": {
-          "type": "string"
-        }
-      },
-      "required": [ "plane" ],
-      "additionalProperties": false
-    },
     "storagePlanes": {
       "description": "storage plane definitions",
       "type": "array",
@@ -847,7 +818,7 @@
         "logging": {
           "$ref": "#/definitions/logging"
         },
-        "storagePlane": {
+        "dataPlane": {
           "description": "The default storage plane to write data files to",
           "type": "string"
         },
@@ -905,7 +876,7 @@
           "type": "array",
           "items": { "$ref": "#/definitions/roxieservice" }
         },
-        "storagePlane": {
+        "dataPlane": {
           "description": "The default storage plane to write data files to",
           "type": "string"
         },
@@ -1078,7 +1049,7 @@
         "logging": {
           "$ref": "#/definitions/logging"
         },
-        "storagePlane": {
+        "dataPlane": {
           "description": "The default storage plane to write data files to",
           "type": "string"
         },
@@ -1165,8 +1136,9 @@
         "image": {
           "$ref": "#/definitions/image"
         },
-        "storage": {
-          "$ref": "#/definitions/storageType"
+        "plane": {
+          "type": "string",
+          "description": "which storage plane to store the sasha data on"
         },
         "resources": {
           "$ref": "#/definitions/resources"
@@ -1206,7 +1178,7 @@
                 "at" : {},
                 "throttle": {},
                 "image": {},
-                "storage": {},
+                "plane": {},
                 "resources": {},
                 "env": {},
                 "annotations": {},
@@ -1225,6 +1197,7 @@
       "oneOf": [
         {
           "type": "object",
+          "required": [ "plane" ],
           "allOf": [
             { "$ref": "#/definitions/sashacommon" },
             { "$ref": "#/definitions/sasha-limitcutoff" },
@@ -1257,7 +1230,7 @@
                 "at" : {},
                 "throttle": {},
                 "image": {},
-                "storage": {},
+                "plane": {},
                 "resources": {},
                 "env": {},
                 "annotations": {},
@@ -1278,6 +1251,7 @@
       "oneOf": [
         {
           "type": "object",
+          "required": [ "plane" ],
           "allOf": [ 
             { "$ref": "#/definitions/sashacommon" },
             { "$ref": "#/definitions/sasha-limitcutoff" },
@@ -1290,7 +1264,7 @@
                 "at" : {},
                 "throttle": {},
                 "image": {},
-                "storage": {},
+                "plane": {},
                 "resources": {},
                 "env": {},
                 "annotations": {},
@@ -1322,7 +1296,7 @@
                 "at" : {},
                 "throttle": {},
                 "image": {},
-                "storage": {},
+                "plane": {},
                 "resources": {},
                 "env": {},
                 "annotations": {},
@@ -1365,7 +1339,7 @@
             "at" : {},
             "throttle": {},
             "image": {},
-            "storage": {},
+            "plane": {},
             "resources": {},
             "annotations": {},
             "labels": {}

helm/hpcc/values.yaml

@@ -101,9 +101,8 @@ security:
 
 ## storage:
 ##
-## 1. If a component has the storagePlane property set, then that plane will be the default data location for that component.
-## 2. If storage.dataStorage.plane is defined then that plane will be the default location
-## 3. If there is a plane definition with no label or a label of "data" then the first matching plane will be the default data location
+## 1. If an engine component has the dataPlane property set, then that plane will be the default data location for that component.
+## 2. If there is a plane definition with a category of "data" then the first matching plane will be the default data location
 ##
 ## If a data plane contains the storageClass property then an implicit pvc will be created for that data plane.
 ##
@@ -167,15 +166,6 @@ storage:
     prefix: "/var/lib/HPCCSystems/mydropzone"
     category: lz
 
-  dllStorage:
-    plane: dll
-
-  daliStorage:
-    plane: dali
-
-  dataStorage:
-    plane: data
-
 ## The certificates section can be used to enable cert-manager to generate TLS certificates for each component in the hpcc.
 ## You must first install cert-manager to use this feature.
 ## https://cert-manager.io/docs/installation/kubernetes/#installing-with-helm
@@ -321,8 +311,7 @@ sasha:
     #disabled: true
     service:
       servicePort: 8877
-    storage:
-      plane: sasha
+    plane: sasha
     #interval: 6 # (hours)
     #limit: 1000 # threshold number of workunits before archiving starts (0 disables)
     #cutoff: 8 # minimum workunit age to archive (days)
@@ -337,8 +326,7 @@ sasha:
     #disabled: true
     service:
       servicePort: 8877
-    storage:
-      plane: sasha
+    plane: sasha
       #forcePermissions: false
     #limit: 1000 # threshold number of DFU workunits before archiving starts (0 disables)
     #cutoff: 14 # minimum DFU workunit age to archive (days)

testing/helm/errtests/badsasha.yaml

@@ -1,4 +1,3 @@
 sasha:
   wu-archiver:
-    storage:
-      plane: null     # required
+    plane: null     # required

testing/helm/tests/implicitplanes.yaml

@@ -33,10 +33,3 @@ storage:
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/sashastorage"
     category: sasha
-
-  dllStorage:
-    plane: my-dll-plane
-  daliStorage:
-    plane: my-dali-plane
-  dataStorage:
-    plane: my-data-plane

testing/helm/tests/implicitplanes2.yaml

@@ -44,7 +44,3 @@ storage:
   #   cost:                                 # The storage cost
   #     storageAtRest: 0.113                # Storage at rest cost: cost per GiB/month
   #   options:                              # not sure if it is needed
-
-  dllStorage: null
-  daliStorage: null
-  dataStorage: null

thorlcr/master/thmastermain.cpp

@@ -1025,7 +1025,7 @@ int main( int argc, const char *argv[]  )
                 StringBuffer uniqueGrpName;
                 queryNamedGroupStore().addUnique(&queryProcessGroup(), uniqueGrpName);
                 // change default plane
-                getComponentConfigSP()->setProp("storagePlane", uniqueGrpName);
+                getComponentConfigSP()->setProp("@dataPlane", uniqueGrpName);
                 PROGLOG("Persistent Thor group created with group name: %s", uniqueGrpName.str());
             }
 #endif