HPCC-23364 Off-node storage

Whilst continuing to use DFS and published groups, detected
containerized/cloud setup and use pseudo groups populated with
localhost.

Also move pv/pvc to be shared (not per Thor).

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

dali/base/dadfs.cpp

@@ -7304,6 +7304,35 @@ public:
         }
     }
 
+    void addGroup(const char *logicalgroupname, const std::vector<std::string> &hosts, bool cluster, const char *dir, GroupType groupType, bool overwrite)
+    {
+        dbgassertex(hosts.size());
+        StringBuffer name(logicalgroupname);
+        name.toLowerCase();
+        name.trim();
+        StringBuffer prop;
+        prop.appendf("Group[@name=\"%s\"]",name.str());
+        CConnectLock connlock("CNamedGroup::add", SDS_GROUPSTORE_ROOT, true, false, false, defaultTimeout);
+        if (!overwrite && connlock.conn->queryRoot()->hasProp(prop.str()))
+            return;
+        connlock.conn->queryRoot()->removeProp(prop.str());
+        if (0 == hosts.size())
+            return;
+        Owned<IPropertyTree> groupTree = doAddHosts(connlock, name.str(), hosts, cluster, dir);
+        SocketEndpointArray eps;
+        if (!loadGroup(groupTree, eps, nullptr, nullptr))
+        {
+            IWARNLOG("CNamedGroupStore.add: failed to add group '%s', due to unresolved hosts", name.str());
+            return;
+        }
+        Owned<IGroup> group = createIGroup(eps);
+        {
+            CriticalBlock block(cachesect);
+            cache.kill();
+            cache.append(*new CNamedGroupCacheEntry(group, name, dir, groupType));
+        }
+    }
+
     virtual void addUnique(IGroup *group,StringBuffer &lname, const char *dir) override
     {
         if (group->ordinality()==1)
@@ -7358,29 +7387,26 @@ public:
 
     virtual void add(const char *logicalgroupname, const std::vector<std::string> &hosts, bool cluster, const char *dir, GroupType groupType) override
     {
-        dbgassertex(hosts.size());
-        StringBuffer name(logicalgroupname);
-        name.toLowerCase();
-        name.trim();
-        StringBuffer prop;
-        prop.appendf("Group[@name=\"%s\"]",name.str());
-        CConnectLock connlock("CNamedGroup::add", SDS_GROUPSTORE_ROOT, true, false, false, defaultTimeout);
-        connlock.conn->queryRoot()->removeProp(prop.str());
-        if (0 == hosts.size())
-            return;
-        Owned<IPropertyTree> groupTree = doAddHosts(connlock, name.str(), hosts, cluster, dir);
-        SocketEndpointArray eps;
-        if (!loadGroup(groupTree, eps, nullptr, nullptr))
-        {
-            IWARNLOG("CNamedGroupStore.add: failed to add group '%s', due to unresolved hosts", name.str());
-            return;
-        }
-        Owned<IGroup> group = createIGroup(eps);
-        {
-            CriticalBlock block(cachesect);
-            cache.kill();
-            cache.append(*new CNamedGroupCacheEntry(group, name, dir, groupType));
-        }
+        addGroup(logicalgroupname, hosts, cluster, dir, groupType, true);
+    }
+
+    virtual void ensure(const char *logicalgroupname, const std::vector<std::string> &hosts, bool cluster, const char *dir, GroupType groupType) override
+    {
+        addGroup(logicalgroupname, hosts, cluster, dir, groupType, false);
+    }
+
+    virtual void ensureNasGroup(size32_t size) override
+    {
+        std::vector<std::string> hosts;
+        for (unsigned n=0; n<size; n++)
+            hosts.push_back("localhost");
+        VStringBuffer nasGroupName("__nas__%u", size);
+        ensure(nasGroupName, hosts, false, nullptr, grp_unknown);
+    }
+
+    virtual StringBuffer &getNasGroupName(StringBuffer &groupName, size32_t size) const override
+    {
+        return groupName.append("__nas__").append(size);
     }
 
     virtual unsigned removeNode(const char *logicalgroupname, const char *nodeToRemove) override

dali/base/dadfs.hpp

@@ -717,6 +717,9 @@ interface INamedGroupStore : extends IGroupResolver
     virtual INamedGroupIterator *getIterator() = 0;
     virtual INamedGroupIterator *getIterator(IGroup *match, bool exact=false) = 0;
     virtual void add(const char *logicalgroupname, const std::vector<std::string> &hosts, bool cluster=false, const char *dir=NULL, GroupType groupType = grp_unknown) = 0;
+    virtual void ensure(const char *logicalgroupname, const std::vector<std::string> &hosts, bool cluster=false, const char *dir=NULL, GroupType groupType = grp_unknown) = 0;
+    virtual void ensureNasGroup(size32_t size) = 0;
+    virtual StringBuffer &getNasGroupName(StringBuffer &groupName, size32_t size) const = 0;
     virtual unsigned removeNode(const char *logicalgroupname, const char *nodeToRemove) = 0;
     virtual void remove(const char *logicalgroupname) = 0;
     virtual void addUnique(IGroup *group,StringBuffer &lname,const char *dir=NULL) = 0;

dali/base/dasds.cpp

@@ -8762,6 +8762,9 @@ public:
             LOG(MCdebugInfo(100), unknownJob, "Failed to load main store");
             throw;
         }
+        // In nas/non-local storage mode, create a published named group for 1-way files to use
+        if (isCloud())
+            queryNamedGroupStore().ensureNasGroup(1);
         storeLoaded = true;
         manager->start();
     }

dockerfiles/hpcc/templates/_util.tpl

@@ -50,6 +50,12 @@ data:
   mountPath: /etc/config
 {{- end -}}
 
+{{- /* Add data volume mount for a component */ -}}
+{{- define "hpcc.utils.addDataVolumeMount" -}}
+- name: datastorage-pv
+  mountPath: "/var/lib/HPCCSystems/hpcc-data"
+{{- end -}}
+
 {{- /* Add standard volumes for a component */ -}}
 {{- define "hpcc.utils.addVolumes" -}}
 volumes:
@@ -57,12 +63,16 @@ volumes:
 - name: dllserver-pv-storage
   persistentVolumeClaim:
     claimName: dllserver-pv-claim
+- name: datastorage-pv
+  persistentVolumeClaim:
+    claimName: datastorage-pv-claim
 {{- end -}}
 
 {{- /* Add standard volume mounts for a component */ -}}
 {{- define "hpcc.utils.addVolumeMounts" -}}
 volumeMounts:
 {{ include "hpcc.utils.addConfigVolumeMount" . }}
+{{ include "hpcc.utils.addDataVolumeMount" . }}
 - name: dllserver-pv-storage
   mountPath: "/var/lib/HPCCSystems/queries"
 {{- end -}}

dockerfiles/hpcc/templates/dllserver.yaml

@@ -7,9 +7,9 @@ metadata:
 spec:
   storageClassName: manual
   capacity:
-    storage: 10Gi
+    storage: {{ .Values.global.dllserver.storageSize }}
   accessModes:
     - ReadWriteMany
   hostPath:
-    path: {{ printf "%s/queries" $.Values.global.hostMountPath }}
+    path: {{ printf "%s/queries" $.Values.global.dataStorage.hostMountPath }}
     type: DirectoryOrCreate

dockerfiles/hpcc/templates/eclagent.yaml

@@ -13,6 +13,9 @@ spec:
       labels:
         run: {{ .name | quote }}
     spec:
+      {{- if $.Values.global.singleNode | default false }}
+{{ include "hpcc.utils.changeMountPerms" (dict "volumeName" "datastorage-pv" "volumePath" "/var/lib/HPCCSystems/hpcc-data") | indent 6 }}
+      {{- end }}
       serviceAccountName: hpcc
       containers:
       - name: {{ .name | quote }}

dockerfiles/hpcc/templates/localroxie.yaml

@@ -22,7 +22,8 @@ spec:
                 {{ include "hpcc.utils.configArg" $roxie }},
                 {{ include "hpcc.utils.daliArg" $ }},
                 "--serverPorts={{ template "hpcc.utils.portListToCommas" $roxie.ports }}", 
-                "--localSlave=true"
+                "--localSlave=true",
+                "--resolveLocally=false"
               ]
 {{ include "hpcc.utils.addImageAttrs" (dict "root" $ "me" . "imagename" "roxie") | indent 8 }}
 {{ include "hpcc.utils.addVolumeMounts" . | indent 8 }}
@@ -32,4 +33,4 @@ spec:
 ---
 {{- end }}
 {{- end }}
- 
+ 

dockerfiles/hpcc/templates/roxie.yaml

@@ -64,6 +64,9 @@ spec:
       labels:
         run: {{ $servername | quote }}
     spec:
+      {{- if $.Values.global.singleNode | default false }}
+{{ include "hpcc.utils.changeMountPerms" (dict "volumeName" "datastorage-pv" "volumePath" "/var/lib/HPCCSystems/hpcc-data") | indent 6 }}
+      {{- end }}
       containers:
       - name: {{ $servername | quote }}
         args: [
@@ -72,7 +75,8 @@ spec:
                 "--numChannels={{ $numChannels }}",
                 "--serverPorts={{ template "hpcc.utils.portListToCommas" $roxie.ports }}", 
                 "--topologyServers={{ $toponame }}:{{ $roxie.topoport }}",
-                "--localSlave=false"
+                "--localSlave=false", 
+                "--resolveLocally=false"
               ]
 {{ include "hpcc.utils.addImageAttrs" (dict "root" $ "me" $roxie "imagename" "roxie") | indent 8 }}
 {{ include "hpcc.utils.addVolumeMounts" . | indent 8 }}
@@ -118,4 +122,4 @@ spec:
 {{- end }}
 {{- end }}
 {{- end }}
- 
+ 

dockerfiles/hpcc/templates/thor.yaml

@@ -1,11 +1,8 @@
 {{ range $thor := $.Values.thor -}}
 {{- $masterPort := $thor.masterport | default 20000 -}}
 {{- $slavePort := $thor.slaveport | default 20100 -}}
-{{- $pvName  := printf "%s-pv" .name }}
-{{- $pvcName := printf "%s-pvc" .name }}
 {{- $slaveName := printf "%s-slave" .name }}
 {{- $serviceName := printf "%s-svc" .name }}
-
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -20,6 +17,9 @@ spec:
       labels:
         run: {{ .name | quote }}
     spec:
+      {{- if $.Values.global.singleNode | default false }}
+{{ include "hpcc.utils.changeMountPerms" (dict "volumeName" "datastorage-pv" "volumePath" "/var/lib/HPCCSystems/hpcc-data") | indent 6 }}
+      {{- end }}
       containers:
       - name: {{ .name | quote }}
         ports:
@@ -49,7 +49,7 @@ spec:
         run: {{ $slaveName | quote }}
     spec:
       {{- if $.Values.global.singleNode | default false }}
-{{ include "hpcc.utils.changeMountPerms" (dict "volumeName" "mythorstorage" "volumePath" "/var/lib/HPCCSystems/hpcc-data") | indent 6 }}
+{{ include "hpcc.utils.changeMountPerms" (dict "volumeName" "datastorage-pv" "volumePath" "/var/lib/HPCCSystems/hpcc-data") | indent 6 }}
       {{- end }}
       containers:
         - name: {{ $slaveName | quote }}
@@ -63,8 +63,6 @@ spec:
           ports:
             - containerPort: {{ $slavePort }}
 {{ include "hpcc.utils.addVolumeMounts" . | indent 10 }}
-          - name: "mythorstorage"
-            mountPath: "/var/lib/HPCCSystems/hpcc-data"
         {{- if $thor.startDafilesrv | default false }}
         - name: {{ printf "%s-dafilesrv" $slaveName | quote }}
 {{ include "hpcc.utils.addImageAttrs" (dict "root" $ "me" . "imagename" "thorslave") | indent 10 }}
@@ -72,13 +70,9 @@ spec:
           ports:
             - containerPort: 7100
           volumeMounts:
-            - name: "mythorstorage"
-              mountPath: "/var/lib/HPCCSystems/hpcc-data"
+{{ include "hpcc.utils.addDataVolumeMount" . | indent 12 }}
         {{- end }}
 {{ include "hpcc.utils.addVolumes" . | indent 6 }}
-      - name: "mythorstorage"
-        persistentVolumeClaim:
-          claimName: {{ $pvcName | quote }}
 ---
 apiVersion: v1
 kind: Service
@@ -93,34 +87,5 @@ spec:
     run: {{ .name | quote }}
   type: ClusterIP
 ---
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: {{ $pvName | quote }}
-  labels:
-    type: local
-spec:
-  storageClassName: manual
-  capacity:
-    storage: {{ $thor.storageSize }}
-  accessModes:
-    - ReadWriteMany
-  persistentVolumeReclaimPolicy: Retain
-  hostPath:
-    path: {{ printf "%s/hpcc-data" $.Values.global.hostMountPath | quote }}
-    type: DirectoryOrCreate
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: {{ $pvcName | quote }}
-spec:
-  storageClassName: manual
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: {{ $thor.storageSize }}
----
 {{- include "hpcc.utils.generateConfigMap" (dict "root" $ "me" .) -}}
 {{- end }}

dockerfiles/hpcc/values.yaml

@@ -5,6 +5,12 @@ global:
   
   dllserver:
     storageSize: 3Gi
+  
+  dataStorage:
+    storageSize: 1G
+    # For singleMode only. Host mount point root for persistent storage.
+    hostMountPath: "/docker-host-mnt"
+    startDafilesrv: false # Temporary until non attached storage functionality added
 
   image:
     # This should probably say "latest" or "stable" in the example script, but set to a specific version in production systems
@@ -18,8 +24,6 @@ global:
   
   singleNode: true
 
-  # For singleMode only. Host mount point root for persistent storage.
-  hostMountPath: "/docker-host-mnt"
 
 
 dali:
@@ -64,5 +68,3 @@ thor:
     masterport: 20000
     numSlaves: 2
     slaveport: 20100
-    startDafilesrv: true # Temporary until non attached storage functionality added
-    storageSize: 100G

ecl/hthor/hthor.cpp

@@ -352,6 +352,8 @@ ClusterWriteHandler *createClusterWriteHandler(IAgentContext &agent, IHThorIndex
         OwnedRoxieString cluster(iwHelper ? iwHelper->getCluster(clusterIdx++) : dwHelper->getCluster(clusterIdx++));
         if(!cluster)
             break;
+        if (isCloud())
+            throw makeStringException(0, "Output clusters not supported in cloud environment");
         if(!clusterHandler)
         {
             if(extend)
@@ -651,7 +653,17 @@ void CHThorDiskWriteActivity::publish()
     {
         // add cluster
         StringBuffer mygroupname;
-        Owned<IGroup> mygrp = agent.getHThorGroup(mygroupname);
+        Owned<IGroup> mygrp;
+        if (isCloud())
+        {
+            queryNamedGroupStore().getNasGroupName(mygroupname, 1);
+            mygrp.setown(queryNamedGroupStore().lookup(mygroupname));
+        }
+        else
+        {
+            if (!agent.queryResolveFilesLocally())
+                mygrp.setown(agent.getHThorGroup(mygroupname));
+        }
         ClusterPartDiskMapSpec partmap; // will get this from group at some point
         desc->setNumParts(1);
         desc->setPartMask(base.str());
@@ -1210,8 +1222,16 @@ void CHThorIndexWriteActivity::execute()
         // add cluster
         StringBuffer mygroupname;
         Owned<IGroup> mygrp = NULL;
-        if (!agent.queryResolveFilesLocally())
-            mygrp.setown(agent.getHThorGroup(mygroupname));
+        if (isCloud())
+        {
+            queryNamedGroupStore().getNasGroupName(mygroupname, 1);
+            mygrp.setown(queryNamedGroupStore().lookup(mygroupname));
+        }
+        else
+        {
+            if (!agent.queryResolveFilesLocally())
+                mygrp.setown(agent.getHThorGroup(mygroupname));
+        }
         ClusterPartDiskMapSpec partmap; // will get this from group at some point
         desc->setNumParts(1);
         desc->setPartMask(base.str());

roxie/ccd/ccdserver.cpp

@@ -11720,6 +11720,8 @@ protected:
             OwnedRoxieString cluster(helper.getCluster(clusterIdx));
             if(!cluster)
                 break;
+            if (isCloud())
+                throw makeStringException(0, "Output clusters not supported in cloud environment");
             clusters.append(cluster);
             clusterIdx++;
         }
@@ -11730,7 +11732,13 @@ protected:
         }
         else
         {
-            if (roxieName.length())
+            if (isCloud())
+            {
+                StringBuffer nasGroupName;
+                queryNamedGroupStore().getNasGroupName(nasGroupName, 1);
+                clusters.append(nasGroupName);
+            }
+            else if (roxieName.length())
                 clusters.append(roxieName.str());
             else
                 clusters.append(".");

thorlcr/mfilemanager/thmfilemanager.cpp

@@ -226,6 +226,13 @@ public:
     CFileManager()
     {
         replicateOutputs = globals->getPropBool("@replicateOutputs");
+
+        /* In nas/non-local storage mode, create a published named group for files to use
+         * that matches the width of the cluster.
+         * Also create a 1-way named group, that is used in special cases, e.g. BUILDINDEX,FEW
+         */
+        if (isCloud())
+            queryNamedGroupStore().ensureNasGroup(queryClusterWidth());
     }
     StringBuffer &mangleLFN(CJobBase &job, const char *lfn, StringBuffer &out)
     {
@@ -374,6 +381,19 @@ public:
 
     IFileDescriptor *create(CJobBase &job, const char *logicalName, StringArray &groupNames, IArrayOf<IGroup> &groups, bool overwriteok, unsigned helperFlags=0, bool nonLocalIndex=false, unsigned restrictedWidth=0)
     {
+        if (isCloud())
+        {
+            StringBuffer nasGroupName;
+            // NB: normally size = queryClusterWidth(), but can be 1 (e.g. if BUILDINDEX,FEW)
+            queryNamedGroupStore().getNasGroupName(nasGroupName, groups.item(0).ordinality());
+            IGroup *nasGroup = queryNamedGroupStore().lookup(nasGroupName);
+            assertex(nasGroup);
+            groups.clear();
+            groupNames.kill();
+            groupNames.append(nasGroupName);
+            groups.append(*LINK(nasGroup));
+        }
+
         bool temporary = 0 != (helperFlags&TDXtemporary);
         bool jobReplicate = 0 != job.getWorkUnitValueInt("replicateOutputs", replicateOutputs);
         bool replicate = 0 != jobReplicate && !temporary && 0==(helperFlags&TDWnoreplicate);
@@ -722,6 +742,8 @@ void fillClusterArray(CJobBase &job, const char *filename, StringArray &clusters
     }
     else
     {
+        if (isCloud())
+            throw makeStringException(0, "Output clusters not supported in cloud environment");
         const char *cluster = clusters.item(0);
         Owned<IGroup> group = queryNamedGroupStore().lookup(cluster);
         if (!group)