HPCC-25759 Add support for bare-metal storage planes to helm charts

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

dali/base/dadfs.cpp

@@ -39,6 +39,7 @@
 #include "dadfs.hpp"
 #include "eclhelper.hpp"
 #include "seclib.hpp"
+#include "dameta.hpp"
 
 #include <string>
 #include <vector>
@@ -10060,10 +10061,9 @@ public:
         return createClusterGroup(grp_unknown, hosts, path, nullptr, false, false);
     }
 
-    void ensureStorageGroup(bool force, const char * name, unsigned numDevices, const char * path, StringBuffer & messages)
+    void ensureConsistentStorageGroup(bool force, const char * name, IPropertyTree * newClusterGroup, StringBuffer & messages)
     {
         IPropertyTree *existingClusterGroup = queryExistingGroup(name);
-        Owned<IPropertyTree> newClusterGroup = createStorageGroup(name, numDevices, path);
         bool matchExisting = clusterGroupCompare(newClusterGroup, existingClusterGroup);
         if (!existingClusterGroup || !matchExisting)
         {
@@ -10072,14 +10072,14 @@ public:
                 VStringBuffer msg("New cluster layout for cluster %s", name);
                 UWARNLOG("%s", msg.str());
                 messages.append(msg).newline();
-                addClusterGroup(name, newClusterGroup.getClear(), false);
+                addClusterGroup(name, LINK(newClusterGroup), false);
             }
             else if (force)
             {
                 VStringBuffer msg("Forcing new group layout for storageplane %s", name);
                 UWARNLOG("%s", msg.str());
                 messages.append(msg).newline();
-                addClusterGroup(name, newClusterGroup.getClear(), false);
+                addClusterGroup(name, LINK(newClusterGroup), false);
             }
             else
             {
@@ -10090,12 +10090,20 @@ public:
         }
     }
 
+    void ensureStorageGroup(bool force, const char * name, unsigned numDevices, const char * path, StringBuffer & messages)
+    {
+        Owned<IPropertyTree> newClusterGroup = createStorageGroup(name, numDevices, path);
+        ensureConsistentStorageGroup(force, name, newClusterGroup, messages);
+    }
+
     void constructStorageGroups(bool force, StringBuffer &messages)
     {
         IPropertyTree & global = queryGlobalConfig();
         IPropertyTree * storage = global.queryPropTree("storage");
         if (storage)
         {
+            normalizeHostGroups();
+
             Owned<IPropertyTreeIterator> planes = storage->getElements("planes");
             ForEach(*planes)
             {
@@ -10104,25 +10112,45 @@ public:
                 if (isEmptyString(name))
                     continue;
 
-                //Lower case the group name - see CnamedGroupStore::dolookup which lower cases before resolving.
+                //Lower case the group name - see CNamedGroupStore::dolookup which lower cases before resolving.
                 StringBuffer gname;
                 gname.append(name).toLowerCase();
 
                 //Two main type of storage plane - with a host group (bare metal) and without.
-                IPropertyTree *existingGroup = queryExistingGroup(gname);
-                const char * hosts = plane.queryProp("@hosts");
+                const char * hostGroup = plane.queryProp("@hostGroup");
                 const char * prefix = plane.queryProp("@prefix");
-                if (hosts)
+                Owned<IPropertyTree> newClusterGroup;
+                if (hostGroup)
+                {
+                    IPropertyTree * match = queryHostGroup(hostGroup, true);
+                    std::vector<std::string> hosts;
+                    Owned<IPropertyTreeIterator> hostIter = match->getElements("hosts");
+                    ForEach (*hostIter)
+                        hosts.push_back(hostIter->query().queryProp(nullptr));
+
+                    //A bare-metal storage plane defined in terms of a hostGroup
+                    newClusterGroup.setown(createClusterGroup(grp_unknown, hosts, prefix, nullptr, false, false));
+                }
+                else if (plane.hasProp("hostGroup"))
                 {
-                    IPropertyTree *existingClusterGroup = queryExistingGroup(gname);
-                    if (!existingClusterGroup)
-                        UNIMPLEMENTED_X("Bare metal storage planes not yet supported");
+                    throw makeStringExceptionV(-1, "Use 'hosts' rather than 'hostGroup' for inline list of hosts for plane %s", name);
+                }
+                else if (plane.hasProp("hosts"))
+                {
+                    //A bare-metal storage plane defined by an explicit list of ips (useful for landing zones)
+                    std::vector<std::string> hosts;
+                    Owned<IPropertyTreeIterator> iter = plane.getElements("hosts");
+                    ForEach(*iter)
+                        hosts.push_back(iter->query().queryProp(nullptr));
+                    newClusterGroup.setown(createClusterGroup(grp_unknown, hosts, prefix, nullptr, false, false));
                 }
                 else
                 {
+                    //Locally mounted, or url accessed storage plane - no associated hosts, localhost used as a placeholder
                     unsigned numDevices = plane.getPropInt("@numDevices", 1);
-                    ensureStorageGroup(force, gname, numDevices, prefix, messages);
+                    newClusterGroup.setown(createStorageGroup(name, numDevices, prefix));
                 }
+                ensureConsistentStorageGroup(force, name, newClusterGroup, messages);
             }
         }
     }

dali/base/dafdesc.cpp

@@ -32,6 +32,7 @@
 
 #include "dafdesc.hpp"
 #include "dadfs.hpp"
+#include "dameta.hpp"
 
 #define INCLUDE_1_OF_1    // whether to use 1_of_1 for single part files
 
@@ -3303,7 +3304,6 @@ static void generateHosts(IPropertyTree * storage, GroupInfoArray & groups)
             }
         }
     }
-
 }
 
 static CriticalSection storageCS;
@@ -3314,6 +3314,9 @@ void initializeStorageGroups(bool createPlanesFromGroups)
     if (!storage)
         storage = queryGlobalConfig().addPropTree("storage");
 
+    //Ensure that host groups that are defined in terms of other host groups are expanded out so they have an explicit list of hosts
+    normalizeHostGroups();
+
 #ifndef _CONTAINERIZED
     if (createPlanesFromGroups && !storage->hasProp("planes"))
     {
@@ -3384,6 +3387,8 @@ void initializeStorageGroups(bool createPlanesFromGroups)
     }
 #endif
 
+    normalizeHostGroups();
+
     //Groups are case insensitve, so add an extra key to the storage items to allow them to be
     //searched by group name, and also check for duplicates.
     Owned<IPropertyTreeIterator> iter = storage->getElements("planes");

dali/base/dameta.cpp

@@ -22,13 +22,19 @@
 
 
 //More to a more central location
-IPropertyTree * queryHostGroup(const char * name)
+IPropertyTree * queryHostGroup(const char * name, bool required)
 {
-    if (isEmptyString(name))
-        return nullptr;
-    VStringBuffer xpath("storage/hostGroups[@name='%s']", name);
-    IPropertyTree & global = queryGlobalConfig();
-    return global.queryPropTree(xpath);
+    if (!isEmptyString(name))
+    {
+        VStringBuffer xpath("storage/hostGroups[@name='%s']", name);
+        IPropertyTree & global = queryGlobalConfig();
+        IPropertyTree * match = global.queryPropTree(xpath);
+        if (match)
+            return match;
+    }
+    if (required)
+        throw makeStringExceptionV(-1, "No entry found for hostGroup: '%s'", name ? name : "<null>");
+    return nullptr;
 }
 
 IPropertyTree * queryStoragePlane(const char * name)
@@ -38,6 +44,43 @@ IPropertyTree * queryStoragePlane(const char * name)
     return global.queryPropTree(xpath);
 }
 
+
+// Expand indirect hostGroups so each hostGroups has an expanded list of host names
+void normalizeHostGroups()
+{
+    Owned<IPropertyTreeIterator> hostGroupIter = queryGlobalConfig().getElements("storage/hostGroups");
+    //Process the groups in order - so that multiple levels of indirection are supported
+    ForEach (*hostGroupIter)
+    {
+        IPropertyTree & cur = hostGroupIter->query();
+        if (!cur.hasProp("hosts"))
+        {
+            const char * name = cur.queryProp("@name");
+            const char * baseGroup = cur.queryProp("@hostGroup");
+            IPropertyTree * match = queryHostGroup(baseGroup, true);
+            StringArray hosts;
+            Owned<IPropertyTreeIterator> hostIter = match->getElements("hosts");
+            ForEach (*hostIter)
+                hosts.append(hostIter->query().queryProp(nullptr));
+
+            if (hosts.ordinality() == 0)
+                throw makeStringExceptionV(-1, "Host group %s contains no hosts", baseGroup);
+
+            unsigned numHosts = cur.getPropInt("@count", hosts.ordinality());
+            unsigned offset = cur.getPropInt("@offset");
+            if (offset + numHosts > hosts.ordinality())
+                throw makeStringExceptionV(-1, "Group %s extends past the end of the base group %s", name, baseGroup);
+
+            unsigned delta = cur.getPropInt("@delta");
+            for (unsigned i=0; i < numHosts; i++)
+            {
+                unsigned baseIndex = offset + (i + delta) % numHosts;
+                cur.addPropTree("hosts")->setProp(nullptr, hosts.item(baseIndex));
+            }
+        }
+    }
+}
+
 //Cloned for now - export and use from elsewhere
 
 static void copyPropIfMissing(IPropertyTree & target, const char * targetName, IPropertyTree & source, const char * sourceName)
@@ -116,10 +159,7 @@ void LogicalFileResolver::ensureHostGroup(const char * name)
     if (storage->hasProp(xpath))
         return;
 
-    IPropertyTree * hosts = queryHostGroup(name);
-    if (!hosts)
-        throw makeStringExceptionV(0, "No entry found for hostGroup: '%s'", name);
-
+    IPropertyTree * hosts = queryHostGroup(name, true);
     storage->addPropTreeArrayItem("hostGroups", LINK(hosts));
 }
 

dali/base/dameta.hpp

@@ -58,5 +58,8 @@ constexpr ResolveOptions operator &(ResolveOptions l, ResolveOptions r) { return
  */
 
 extern da_decl IPropertyTree * resolveLogicalFilenameFromDali(const char * filename, IUserDescriptor * user, ResolveOptions options);
+extern da_decl IPropertyTree * queryHostGroup(const char * name, bool required);
+extern da_decl IPropertyTree * queryStoragePlane(const char * name);
+extern da_decl void normalizeHostGroups();  // Expand indirect hostGroups so each hostGroups has an expanded list of host names
 
 #endif

devdoc/NewFileProcessing.rst

@@ -11,20 +11,21 @@ storage:
   hostGroups:
   - name: <required>
     hosts: [ .... ]
+  - name: <required>
+    hostGroup: <name>
+    count: <unsigned:#hosts>    # how many hosts within the host group are used ?(default is number of hosts)
+    offset: <unsigned:0>        # index of first host included in the derived group
+    delta: <unsigned:0>         # first host within the range[offset..offset+count-1] in the derived group
 
   planes:
     name: <required>
     prefix: <path>              # Root directory for accessing the plane (if pvc defined), or url to access plane.
     numDevices: 1               # number of devices that are part of the plane
-    hosts: <name>               # Name of the host group for bare metal
+    hostGroup: <name>           # Name of the host group for bare metal
+    hosts: [ host-names ]       # A list of host names for bare metal
     secret: <secret-id>         # what secret is required to access the files.
     options:                    # not sure if it is needed
 
-The following options are only used for replication on bare metal systems:
-
-    start: <unsigned:0>         # first offset within hosts that is unsigned
-    size: <unsigned:#hosts>     # how many hosts within the host group are used ?(default is number of hosts). numDevices = size of hostGroup
-    offset: <unsigned:0>        # number added to the part number before mapping to a host/device
 
 Changes:
 * The replication information has been removed from the storage plane.  It will now be specified on the thor instance indicating where (if anywhere) files are replicated.

esp/services/ws_fs/ws_fsService.cpp

@@ -1822,6 +1822,36 @@ void CFileSprayEx::readAndCheckSpraySourceReq(MemoryBuffer& srcxml, const char*
     getStandardPosixPath(sourcePathReq, sourcePath.str());
 }
 
+static void checkValidDfuQueue(const char * dfuQueue)
+{
+#ifndef _CONTAINERIZED
+        Owned<IEnvironmentFactory> envFactory = getEnvironmentFactory(true);
+        Owned<IConstEnvironment> constEnv = envFactory->openEnvironment();
+        if (!isEmptyString(dfuQueue))
+        {
+            if (!constEnv->isValidDfuQueueName(dfuQueue))
+                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid DFU server queue name:'%s'", dfuQueue);
+        }
+#else
+        bool isValidDfuQueueName = false;
+        Owned<IPropertyTreeIterator> dfuServers = queryComponentConfig().getElements("dfuQueues");
+        ForEach(*dfuServers)
+        {
+            IPropertyTree & dfuServer = dfuServers->query();
+            const char * dfuServerName = dfuServer.queryProp("@name");
+            StringBuffer knownDfuQueueName;
+            getDfuQueueName(knownDfuQueueName, dfuServerName);
+            if (streq(dfuQueue, knownDfuQueueName))
+            {
+                isValidDfuQueueName = true;
+                break;
+            }
+        }
+        if (!isValidDfuQueueName)
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Invalid DFU server queue name: '%s'", dfuQueue);
+#endif
+}
+
 bool CFileSprayEx::onSprayFixed(IEspContext &context, IEspSprayFixed &req, IEspSprayFixedResponse &resp)
 {
     try
@@ -1870,32 +1900,7 @@ bool CFileSprayEx::onSprayFixed(IEspContext &context, IEspSprayFixed &req, IEspS
 
         wu->setJobName(destTitle.str());
         const char * dfuQueue = req.getDFUServerQueue();
-#ifndef _CONTAINERIZED
-        Owned<IEnvironmentFactory> envFactory = getEnvironmentFactory(true);
-        Owned<IConstEnvironment> constEnv = envFactory->openEnvironment();
-        if (!isEmptyString(dfuQueue))
-        {
-            if (!constEnv->isValidDfuQueueName(dfuQueue))
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid DFU server queue name:'%s'", dfuQueue);
-        }
-#else
-        bool isValidDfuQueueName = false;
-        Owned<IPropertyTreeIterator> dfuServers = queryComponentConfig().getElements("dfuQueues");
-        ForEach(*dfuServers)
-        {
-            IPropertyTree & dfuServer = dfuServers->query();
-            const char * dfuServerName = dfuServer.queryProp("@name");
-            StringBuffer knownDfuQueueName;
-            getDfuQueueName(knownDfuQueueName, dfuServerName);
-            if (streq(dfuQueue, knownDfuQueueName))
-            {
-                isValidDfuQueueName = true;
-                break;
-            }
-        }
-        if (!isValidDfuQueueName)
-            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Invalid DFU server queue name: '%s'", dfuQueue);
-#endif
+        checkValidDfuQueue(dfuQueue);
         setDFUServerQueueReq(dfuQueue, wu);
         setUserAuth(context, wu);
         wu->setCommand(DFUcmd_import);
@@ -2069,13 +2074,7 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
         wu->setJobName(destTitle.str());
 
         const char * dfuQueue = req.getDFUServerQueue();
-        Owned<IEnvironmentFactory> envFactory = getEnvironmentFactory(true);
-        Owned<IConstEnvironment> constEnv = envFactory->openEnvironment();
-        if (!isEmptyString(dfuQueue))
-        {
-            if (!constEnv->isValidDfuQueueName(dfuQueue))
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid DFU server queue name:'%s'", dfuQueue);
-        }
+        checkValidDfuQueue(dfuQueue);
         setDFUServerQueueReq(dfuQueue, wu);
         setUserAuth(context, wu);
         wu->setCommand(DFUcmd_import);

esp/smc/SMCLib/TpWrapper.cpp

@@ -27,6 +27,7 @@
 #include "portlist.h"
 #include "daqueue.hpp"
 #include "dautils.hpp"
+#include "dameta.hpp"
 
 const char* MSG_FAILED_GET_ENVIRONMENT_INFO = "Failed to get environment information.";
 
@@ -464,21 +465,50 @@ void CTpWrapper::getTpEspServers(IArrayOf<IConstTpEspServer>& list)
     }
 }
 
-static IEspTpMachine * createLocalHostTpMachine(const char *path)
+#ifdef _CONTAINERIZED
+static IEspTpMachine * createHostTpMachine(const char * hostname, const char *path)
 {
     Owned<IEspTpMachine> machine = createTpMachine();
     IpAddress ipAddr;
-    ipAddr.ipset("localhost");
+    ipAddr.ipset(hostname);
     StringBuffer localHost;
     ipAddr.getIpText(localHost);
     machine->setName(localHost.str());
     machine->setNetaddress(localHost.str());
-    machine->setConfigNetaddress("localhost");
+    machine->setConfigNetaddress(hostname);
     machine->setDirectory(path);
     machine->setOS(getPathSepChar(path) == '/' ? MachineOsLinux : MachineOsW2K);
     return machine.getClear();
 }
 
+static void gatherDropZoneMachinesFromHosts(IArrayOf<IEspTpMachine> & tpMachines, IPropertyTree & planeOrGroup, const char * prefix)
+{
+    Owned<IPropertyTreeIterator> iter = planeOrGroup.getElements("hosts");
+    ForEach(*iter)
+    {
+        const char * host = iter->query().queryProp(nullptr);
+        tpMachines.append(*createHostTpMachine(host, prefix));
+    }
+}
+
+static void gatherDropZoneMachines(IArrayOf<IEspTpMachine> & tpMachines, IPropertyTree & plane)
+{
+    const char * prefix = plane.queryProp("@prefix");
+    if (plane.hasProp("hosts"))
+    {
+        gatherDropZoneMachinesFromHosts(tpMachines, plane, prefix);
+    }
+    else if (plane.hasProp("@hostGroup"))
+    {
+        IPropertyTree * hostGroup = queryHostGroup(plane.queryProp("@hostGroup"), true);
+        gatherDropZoneMachinesFromHosts(tpMachines, *hostGroup, prefix);
+    }
+    else
+        tpMachines.append(*createHostTpMachine("localhost", prefix));
+}
+#endif
+
+
 void CTpWrapper::getTpDfuServers(IArrayOf<IConstTpDfuServer>& list)
 {
 #ifdef _CONTAINERIZED
@@ -496,12 +526,8 @@ void CTpWrapper::getTpDfuServers(IArrayOf<IConstTpDfuServer>& list)
         pService->setQueue(queue);
         pService->setType(eqDfu);
         IArrayOf<IEspTpMachine> tpMachines;
-        Owned<IPropertyTreeIterator> planes = getDropZonePlanesIterator();
-        ForEach(*planes)
-        {
-            IPropertyTree & plane = planes->query();
-            tpMachines.append(*createLocalHostTpMachine(plane.queryProp("@prefix")));
-        }
+        //MORE: The ip and directory don't make any sense on the cloud version
+        tpMachines.append(*createHostTpMachine("localhost", "/var/lib/HPCCSystems"));
         pService->setTpMachines(tpMachines);
         list.append(*pService.getClear());
     }
@@ -1715,7 +1741,6 @@ void CTpWrapper::getDropZoneMachineList(double clientVersion, bool ECLWatchVisib
 
 void CTpWrapper::getTpDropZones(double clientVersion, const char* name, bool ECLWatchVisibleOnly, IArrayOf<IConstTpDropZone>& list)
 {
-
 #ifdef _CONTAINERIZED
     Owned<IPropertyTreeIterator> planes = getDropZonePlanesIterator(name);
     ForEach(*planes)
@@ -1730,7 +1755,7 @@ void CTpWrapper::getTpDropZones(double clientVersion, const char* name, bool ECL
         dropZone->setBuild("");
         dropZone->setECLWatchVisible(true);
         IArrayOf<IEspTpMachine> tpMachines;
-        tpMachines.append(*createLocalHostTpMachine(path));
+        gatherDropZoneMachines(tpMachines, plane);
         dropZone->setTpMachines(tpMachines);
         list.append(*dropZone.getClear());
     }

helm/hpcc/templates/_helpers.tpl

@@ -75,6 +75,10 @@ esp:
 {{ end -}}
 secretTimeout: {{ .Values.secrets.timeout | default 300 }}
 storage:
+{{- if hasKey $storage "hostGroups" }}
+  hostGroups:
+{{ toYaml $storage.hostGroups | indent 2 }}
+{{- end }}
   daliPlane: {{ $daliStoragePlane }}
   dllsPlane: {{ $dllStoragePlane }}
   dataPlane: {{ $dataStoragePlane }}

helm/hpcc/values.schema.json

@@ -8,9 +8,6 @@
     "security" : {
       "$ref": "#/definitions/security"
     },
-    "hostgroups": {
-      "$ref": "#/definitions/hostgroups"
-    },
     "placements": {
       "type": "array",
       "items": {
@@ -48,6 +45,9 @@
             }
           ]
         },
+        "hostGroups": {
+          "$ref": "#/definitions/hostGroups"
+        },
         "planes": {
           "$ref": "#/definitions/storagePlanes"
         }
@@ -392,10 +392,15 @@
           "description": "optional name of the persistent volume claim for this plane",
           "type": "string"
         },
-        "hosts": {
+        "hostGroup": {
           "description": "optional name of the host group (for bare metal storage)",
           "type": "string"
         },
+        "hosts": {
+          "description": "a list of host names",
+          "type": "array",
+          "items": { "type": "string" }
+        },
         "numDevices": {
           "description": "optional number of devices in the storage plane (default 1)",
           "type": "integer"
@@ -476,51 +481,55 @@
       "required": [ "name", "url" ],
       "additionalProperties": false
     },
-    "hostgroups": {
+    "hostGroups": {
       "oneOf": [
         {
-          "type": "object",
-          "additionalProperties": {
-            "$ref": "#/definitions/hostgroup"
-          }
+          "type": "array",
+          "items": { "$ref": "#/definitions/hostGroup" }
         },
         {
           "type": "null"
         }
       ]
     },
-    "hostgroup": {
+    "hostGroup": {
+      "type": "object",
       "oneOf": [
         {
+          "required": [ "name", "hosts" ]
+        },
+        {
+          "required": [ "name", "hostGroup" ]
+        }
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "The name of the host group process"
+        },
+        "hosts": {
           "description": "a list of host names",
           "type": "array",
           "items": { "type": "string" }
         },
-        {
-          "description": "a subset of an existing host group",
-          "type": "object",
-          "properties": {
-            "group": {
-              "description": "Name of the hostgroup to create a subset of",
-              "type": "string"
-            },
-            "count": {
-              "description": "Number of hosts in the subset",
-              "type": "integer"
-            },
-            "offset": {
-              "description": "Offset of the first host within the group",
-              "type": "integer"
-            },
-            "delta": {
-              "type": "integer",
-              "description": "Cycle offsset to apply to the hosts"
-            }
-          },
-          "required": [ "group" ],
-          "additionalProperties": false
+        "hostGroup": {
+          "description": "Name of the hostgroup to create a subset of",
+          "type": "string"
+        },
+        "count": {
+          "description": "Number of hosts in the subset",
+          "type": "integer"
+        },
+        "offset": {
+          "description": "Offset of the first host within the group",
+          "type": "integer"
+        },
+        "delta": {
+          "type": "integer",
+          "description": "Cycle offset to apply to the hosts"
         }
-      ]
+      }
     },
     "logging": {
       "type": "object",

testing/helm/errtests/baremetalerr.yaml

@@ -0,0 +1,42 @@
+storage:
+  hostGroups:
+    - name: demoOne
+      hosts: ['one', 'two', 'three']
+    - name: demoTwo
+      hosts: [ '127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4', '127.0.0.5', '127.0.0.6', '127.0.0.7', '127.0.0.8', '127.0.0.9', '127.0.0.10' ]
+    - name: demoThree
+      hostGroup: demoTwo
+      delta: 1
+    - name: demoTwoA
+      hostGroup: demoTwo
+      count: 5
+      delta: 1
+    - name: demoTwoB
+      hosts: demoTwo                                # error - should be a list rather than a single value
+      count: 5
+      offset: 5
+      delta: 1
+
+  planes:
+  - name: demoOne
+    prefix: /home/gavin/temp
+    hostGroup: demoOne
+  - name: myDropZone
+    prefix: /home/gavin/temp
+    hosts: [ '192.168.86.202']
+    labels: ['lz']
+  - name: demoTwo
+    prefix: /home/gavin/temp/demoTwo
+    hostGroup: demoTwo
+  - name: demoTwoReplica
+    prefix: /home/gavin/temp/demoTwo
+    hostGroup: demoThree
+  - name: demoTwoA
+    prefix: /home/gavin/temp/demoTwoA
+    hostGroup: demoTwoA
+  - name: demoTwoB
+    prefix: /home/gavin/temp/demoTwoB
+    hostGroup: demoTwoB
+  - name: myInlinePlane
+    prefix: /home/gavin/temp
+    hostGroup: [ 'here', 'there', 'everywhere']     # error - list is not allowed here, use hosts

testing/helm/tests/baremetal.yaml

@@ -1,37 +1,38 @@
-hostgroups:
-  thor400: [ node1, node2, node3, node4, node5, node6, node400 ]
-  thor400m:
-    group: thor400
-    delta: 1
-  thor20_1:
-    group: thor400
-    count: 20
-    offset: 0
-  thor20_2:
-    group: thor400
-    count: 20
-    offset: 20
-  thor100_4:
-    group: thor400
-    count: 100
-    offset: 300
-
 storage:
+  hostGroups:
+    - name: thor400
+      hosts: [ node1, node2, node3, node4, node5, node6, node400 ]
+    - name: thor400m
+      hostGroup: thor400
+      delta: 1
+    - name: thor20_1
+      hostGroup: thor400
+      count: 20
+      offset: 0
+    - name: thor20_2
+      hostGroup: thor400
+      count: 20
+      offset: 20
+    - name: thor100_4
+      hostGroup: thor400
+      count: 100
+      offset: 300
+
   planes:
   #Bare metal system with attached storage
   - name: thor400
     prefix: /var/lib/hpccsystems/hpcc-data       # only used if the local host matches the host for the device
-    hosts: thor400
+    hostGroup: thor400
     replication: [ attachedThor400MirrorPlane ]
     #numDevices: count(hosts)
   - name: thor400mirror
     prefix: /var/lib/hpccsystems/hpcc-mirror       # only used if the local host matches the host for the device
-    hosts: thor400mirror
+    hostGroup: thor400mirror
     #Does any other information about the replication policy need to be included?  I don't think it does....
 
   - name: thor100_4
     prefix: /var/lib/hpccsystems/hpcc-data       # only used if the local host matches the host for the device
-    hosts: thor100_4
+    hostGroup: thor100_4
     replication: [ azureBlobPlane ]
 
   - name: azureBlobPlane

testing/helm/tests/baremetal2.yaml

@@ -0,0 +1,42 @@
+storage:
+  hostGroups:
+    - name: demoOne
+      hosts: ['one', 'two', 'three']
+    - name: demoTwo
+      hosts: [ '127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4', '127.0.0.5', '127.0.0.6', '127.0.0.7', '127.0.0.8', '127.0.0.9', '127.0.0.10' ]
+    - name: demoThree
+      hostGroup: demoTwo
+      delta: 1
+    - name: demoTwoA
+      hostGroup: demoTwo
+      count: 5
+      delta: 1
+    - name: demoTwoB
+      hostGroup: demoTwo
+      count: 5
+      offset: 5
+      delta: 1
+
+  planes:
+  - name: demoOne
+    prefix: /home/gavin/temp
+    hostGroup: demoOne
+  - name: myDropZone
+    prefix: /home/gavin/temp
+    hosts: [ '192.168.86.202']
+    labels: ['lz']
+  - name: demoTwo
+    prefix: /home/gavin/temp/demoTwo
+    hostGroup: demoTwo
+  - name: demoTwoReplica
+    prefix: /home/gavin/temp/demoTwo
+    hostGroup: demoThree
+  - name: demoTwoA
+    prefix: /home/gavin/temp/demoTwoA
+    hostGroup: demoTwoA
+  - name: demoTwoB
+    prefix: /home/gavin/temp/demoTwoB
+    hostGroup: demoTwoB
+  - name: myInlinePlane
+    prefix: /home/gavin/temp
+    hosts: [ 'here', 'there', 'everywhere']