Merge branch 'candidate-8.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/dllserver/dllserver.cpp

@@ -756,7 +756,7 @@ IDllServer & queryDllServer()
         StringBuffer dir;
         if(dllserver_root == NULL)
         {
-            if (envGetConfigurationDirectory("temp","dllserver","dllserver",dir)) // not sure if different instance might be better but never separated in past
+            if (getConfigurationDirectory(nullptr, "temp","dllserver","dllserver",dir)) // not sure if different instance might be better but never separated in past
                 dllserver_root = dir.str();
             else
                 dllserver_root = DEFAULT_SERVER_ROOTDIR;

common/environment/dalienv.cpp

@@ -333,21 +333,6 @@ bool getRemoteRunInfo(const char * keyName, const char * exeName, const char * v
     return false;
 }
 
-bool envGetConfigurationDirectory(const char *category, const char *component,const char *instance, StringBuffer &dirout)
-{
-    SessionId sessid = myProcessSession();
-    if (!sessid)
-        return false;
-
-    Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
-    Owned<IConstEnvironment> env = factory->openEnvironment();
-    Owned<IPropertyTree> root = &env->getPTree();
-    IPropertyTree * child = root->queryPropTree("Software/Directories");
-    if (child)
-        return getConfigurationDirectory(child,category,component,instance,dirout);
-    return false;
-}
-
 IPropertyTree *envGetNASConfiguration(IPropertyTree *source)
 {
     if ((NULL==source) || !source->hasProp("NAS"))

common/environment/dalienv.hpp

@@ -42,8 +42,6 @@ extern ENVIRONMENT_API bool canSpawnChildProcess(const IpAddress & ip);
 
 extern ENVIRONMENT_API bool getRemoteRunInfo(const char * keyName, const char * exeName, const char * version, const IpAddress &ip, StringBuffer &progpath, StringBuffer &workdir,INode *remotedali, unsigned timeout);
 
-extern ENVIRONMENT_API bool envGetConfigurationDirectory(const char *category, const char *component,const char *instance, StringBuffer &dirout);
-
 extern ENVIRONMENT_API IPropertyTree *envGetNASConfiguration(); // return NAS config from environment
 extern ENVIRONMENT_API IPropertyTree *envGetNASConfiguration(IPropertyTree *source);
 // These methods filter the NAS hooks based on the callers IP, unless 'myEp' is supplied.

common/environment/environment.cpp

@@ -207,7 +207,7 @@ private:
         if (!clusterGroupKeyNameCache)
         {
             StringBuffer keysDir;
-            envGetConfigurationDirectory("keys",nullptr, nullptr, keysDir);
+            getConfigurationDirectory(nullptr, "keys",nullptr, nullptr, keysDir);
 
             Owned<IPropertyTreeIterator> keyPairIt = p->getElements("EnvSettings/Keys/KeyPair");
             ForEach(*keyPairIt)

common/fileview2/fileview.hpp

@@ -84,6 +84,7 @@ interface IResultSetMetaData : extends IInterface
 typedef double xdouble;
 interface INewResultSet;
 interface IXmlWriter;
+interface IXmlWriterExt;
 interface IResultSetCursor : extends IInterface
 {
     virtual bool absolute(__int64 row) = 0;
@@ -159,24 +160,24 @@ extern FILEVIEW_API IResultSetFactory * getSecResultSetFactory(ISecManager *secm
 //Formatting applied remotely, so it can be accessed between different operating systems...
 extern FILEVIEW_API int findResultSetColumn(const INewResultSet * results, const char * columnName);
 
-extern FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL, const IProperties *xmlns=NULL);
+extern FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL, const IProperties *xmlns=NULL, __uint64 maxSize=0);
 extern FILEVIEW_API unsigned getResultXml(IStringVal & ret, INewResultSet * cursor,  const char * name,
-    unsigned start=0, unsigned count=0, const char * schemaName=nullptr, const IProperties * xmlns=nullptr, IAbortRequestCallback * abortCheck=nullptr);
+    unsigned start=0, unsigned count=0, const char * schemaName=nullptr, const IProperties * xmlns=nullptr, IAbortRequestCallback * abortCheck=nullptr, __uint64 maxSize=0);
 extern FILEVIEW_API unsigned getResultJSON(IStringVal & ret, INewResultSet * cursor,  const char* name,
-    unsigned start=0, unsigned count=0, const char * schemaName=nullptr, IAbortRequestCallback * abortCheck=nullptr);
-extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSetCursor * cursor, const char * name,
+    unsigned start=0, unsigned count=0, const char * schemaName=nullptr, IAbortRequestCallback * abortCheck=nullptr, __uint64 maxSize=0);
+extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriterExt & writer, IResultSetCursor * cursor, const char * name,
     unsigned start=0, unsigned count=0, const char * schemaName=nullptr, const IProperties * xmlns=nullptr, bool flushContent=false,
-    IAbortRequestCallback * abortCheck=nullptr);
-extern FILEVIEW_API unsigned writeResultXml(IXmlWriter & writer, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL, const IProperties *xmlns = NULL);
+    IAbortRequestCallback * abortCheck=nullptr, __uint64 maxSize=0);
+extern FILEVIEW_API unsigned writeResultXml(IXmlWriterExt & writer, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL, const IProperties *xmlns = NULL);
 
-extern FILEVIEW_API unsigned getResultCursorBin(MemoryBuffer & ret, IResultSetCursor * cursor, unsigned start=0, unsigned count=0);
-extern FILEVIEW_API unsigned getResultBin(MemoryBuffer & ret, INewResultSet * cursor, unsigned start=0, unsigned count=0);
+extern FILEVIEW_API unsigned getResultCursorBin(MemoryBuffer & ret, IResultSetCursor * cursor, unsigned start=0, unsigned count=0, __uint64 maxSize=0);
+extern FILEVIEW_API unsigned getResultBin(MemoryBuffer & ret, INewResultSet * cursor, unsigned start=0, unsigned count=0, __uint64 maxSize=0);
 
 #define WorkUnitXML_InclSchema      0x0001
 #define WorkUnitXML_NoRoot          0x0002
 #define WorkUnitXML_SeverityTags    0x0004
 
-extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const char *password, const IConstWorkUnit *cw, IXmlWriter &writer, unsigned flags, ErrorSeverity minSeverity, const char *rootTag);
+extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const char *password, const IConstWorkUnit *cw, IXmlWriterExt &writer, unsigned flags, ErrorSeverity minSeverity, const char *rootTag);
 extern FILEVIEW_API IStringVal& getFullWorkUnitResultsXML(const char *user, const char *pw, const IConstWorkUnit *wu, IStringVal &str, unsigned flags=0, ErrorSeverity minSeverity=SeverityInformation);
 extern FILEVIEW_API IStringVal& getFullWorkUnitResultsJSON(const char *user, const char *pw, const IConstWorkUnit *wu, IStringVal &str, unsigned flags=0, ErrorSeverity minSeverity=SeverityInformation);
 

common/fileview2/fvresultset.cpp

@@ -2293,36 +2293,36 @@ int findResultSetColumn(const INewResultSet * results, const char * columnName)
 
 
 extern FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name,
-    unsigned start, unsigned count, const char * schemaName, const IProperties * xmlns, IAbortRequestCallback * abortCheck)
+    unsigned start, unsigned count, const char * schemaName, const IProperties * xmlns, IAbortRequestCallback * abortCheck, __uint64 maxSize)
 {
     Owned<CommonXmlWriter> writer = CreateCommonXmlWriter(XWFexpandempty);
-    unsigned rc = writeResultCursorXml(*writer, cursor, name, start, count, schemaName, xmlns, false, abortCheck);
+    unsigned rc = writeResultCursorXml(*writer, cursor, name, start, count, schemaName, xmlns, false, abortCheck, maxSize);
     ret.set(writer->str());
     return rc;
 
 }
 
 extern FILEVIEW_API unsigned getResultXml(IStringVal & ret, INewResultSet * result, const char * name,
-    unsigned start, unsigned count, const char * schemaName, const IProperties * xmlns, IAbortRequestCallback * abortCheck)
+    unsigned start, unsigned count, const char * schemaName, const IProperties * xmlns, IAbortRequestCallback * abortCheck, __uint64 maxSize)
 {
     Owned<IResultSetCursor> cursor = result->createCursor();
-    return getResultCursorXml(ret, cursor, name, start, count, schemaName, xmlns, abortCheck);
+    return getResultCursorXml(ret, cursor, name, start, count, schemaName, xmlns, abortCheck, maxSize);
 }
 
 extern FILEVIEW_API unsigned getResultJSON(IStringVal & ret, INewResultSet * result, const char * name,
-    unsigned start, unsigned count, const char * schemaName, IAbortRequestCallback * abortCheck)
+    unsigned start, unsigned count, const char * schemaName, IAbortRequestCallback * abortCheck, __uint64 maxSize)
 {
     Owned<IResultSetCursor> cursor = result->createCursor();
     Owned<CommonJsonWriter> writer = new CommonJsonWriter(0);
     writer->outputBeginRoot();
-    unsigned rc = writeResultCursorXml(*writer, cursor, name, start, count, schemaName, nullptr, false, abortCheck);
+    unsigned rc = writeResultCursorXml(*writer, cursor, name, start, count, schemaName, nullptr, false, abortCheck, maxSize);
     writer->outputEndRoot();
     ret.set(writer->str());
     return rc;
 }
 
-extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSetCursor * cursor, const char * name,
-    unsigned start, unsigned count, const char * schemaName, const IProperties * xmlns, bool flushContent, IAbortRequestCallback * abortCheck)
+extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriterExt & writer, IResultSetCursor * cursor, const char * name,
+    unsigned start, unsigned count, const char * schemaName, const IProperties * xmlns, bool flushContent, IAbortRequestCallback * abortCheck, __uint64 maxSize)
 {
     if (schemaName)
     {
@@ -2359,6 +2359,9 @@ extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSe
         if (flushContent)
             writer.flushContent(false);
 
+        if (maxSize && (writer.length() > maxSize))
+            throw makeStringExceptionV(-1, "writeResultCursorXml exceeded max size (%u MB)", (unsigned)(maxSize / 0x100000));
+
         c++;
         if(count && c>=count)
             break;
@@ -2370,34 +2373,38 @@ extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSe
     return c;
 }
 
-extern FILEVIEW_API unsigned writeResultXml(IXmlWriter & writer, INewResultSet * result, const char* name,unsigned start, unsigned count, const char * schemaName, const IProperties *xmlns)
+extern FILEVIEW_API unsigned writeResultXml(IXmlWriterExt & writer, INewResultSet * result, const char* name,unsigned start, unsigned count, const char * schemaName, const IProperties *xmlns)
 {
     Owned<IResultSetCursor> cursor = result->createCursor();
     return writeResultCursorXml(writer, cursor, name, start, count, schemaName, xmlns);
 }
 
-extern FILEVIEW_API unsigned getResultCursorBin(MemoryBuffer & ret, IResultSetCursor * cursor, unsigned start, unsigned count)
+extern FILEVIEW_API unsigned getResultCursorBin(MemoryBuffer & ret, IResultSetCursor * cursor, unsigned start, unsigned count, __uint64 maxSize)
 {
     const IResultSetMetaData & meta = cursor->queryResultSet()->getMetaData();
     unsigned numCols = meta.getColumnCount();
 
+    __uint64 startSize = ret.length();
     unsigned c=0;
     for(bool ok=cursor->absolute(start);ok;ok=cursor->next())
     {
         for (unsigned col=0; col < numCols; col++)
             cursor->getRaw(MemoryBuffer2IDataVal(ret), col);
-
         c++;
+
+        if (maxSize && (ret.length()-startSize > maxSize))
+            throw makeStringExceptionV(-1, "getResultCursorBin exceeded max size (%u MB)", (unsigned)(maxSize / 0x100000));
+
         if(count && c>=count)
             break;
     }
     return c;
 }
 
-extern FILEVIEW_API unsigned getResultBin(MemoryBuffer & ret, INewResultSet * result, unsigned start, unsigned count)
+extern FILEVIEW_API unsigned getResultBin(MemoryBuffer & ret, INewResultSet * result, unsigned start, unsigned count, __uint64 maxSize)
 {
     Owned<IResultSetCursor> cursor = result->createCursor();
-    return getResultCursorBin(ret, cursor, start, count);
+    return getResultCursorBin(ret, cursor, start, count, maxSize);
 }
 
 inline const char *getSeverityTagname(ErrorSeverity severity, unsigned flags)
@@ -2420,7 +2427,7 @@ inline const char *getSeverityTagname(ErrorSeverity severity, unsigned flags)
     return "Exception";
 }
 
-extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const char *password, const IConstWorkUnit *cw, IXmlWriter &writer, unsigned flags, ErrorSeverity minSeverity, const char *rootTag)
+extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const char *password, const IConstWorkUnit *cw, IXmlWriterExt &writer, unsigned flags, ErrorSeverity minSeverity, const char *rootTag)
 {
     if (rootTag && *rootTag && !(flags & WorkUnitXML_NoRoot))
         writer.outputBeginNested(rootTag, true);

common/thorhelper/thorsoapcall.cpp

@@ -507,10 +507,10 @@ void initPersistentHandler()
     if (!persistentInitDone)
     {
 #ifndef _CONTAINERIZED
-        int maxPersistentRequests = queryEnvironmentConf().getPropInt("maxPersistentRequests", DEFAULT_MAX_PERSISTENT_REQUESTS);
+        int maxPersistentRequests = queryEnvironmentConf().getPropInt("maxHttpCallPersistentRequests", 0);
 #else
         Owned<IPropertyTree> conf = getComponentConfig();
-        int maxPersistentRequests = conf->getPropInt("@maxPersistentRequests", DEFAULT_MAX_PERSISTENT_REQUESTS);
+        int maxPersistentRequests = conf->getPropInt("@maxHttpCallPersistentRequests", 0);
 #endif
         if (maxPersistentRequests != 0)
             persistentHandler = createPersistentHandler(nullptr, DEFAULT_MAX_PERSISTENT_IDLE_TIME, maxPersistentRequests, PersistentLogLevel::PLogMin, true);

common/workunit/workunit.cpp

@@ -14148,7 +14148,12 @@ void deleteK8sResource(const char *componentName, const char *job, const char *r
     if (error.length())
         DBGLOG("kubectl delete error: %s", error.trimRight().str());
     if (ret)
-        throw makeStringException(0, "Failed to run kubectl delete");
+    {
+        StringBuffer errorText("Failed to run kubectl delete");
+        if (error.length())
+            errorText.append(", error: ").append(error);
+        throw makeStringException(0, errorText);
+    }
 }
 
 void waitK8sJob(const char *componentName, const char *job, unsigned pendingTimeoutSecs, KeepK8sJobs keepJob)
@@ -14274,7 +14279,10 @@ bool applyK8sYaml(const char *componentName, const char *wuid, const char *job,
     if (ret)
     {
         DBGLOG("Using yaml %s", jobYaml.str());
-        throw makeStringException(0, "Failed to replace k8s resource");
+        StringBuffer errorText("Failed to replace k8s resource");
+        if (error.length())
+            errorText.append(", error: ").append(error);
+        throw makeStringException(0, errorText);
     }
     return true;
 }

dali/base/dafdesc.cpp

@@ -3245,8 +3245,8 @@ void GroupInformation::createStoragePlane(IPropertyTree * storage, unsigned copy
     else
         plane->setProp("@prefix", queryBaseDirectory(groupType, copy));
 
-    const char * label = (dropZoneIndex != 0) ? "lz" : "data";
-    addPTreeItem(plane, "labels", label);
+    const char * category = (dropZoneIndex != 0) ? "lz" : "data";
+    plane->setProp("@category", category);
 
     //MORE: If container is identical to this except for the name we could generate an information tag @alias
 }
@@ -3435,30 +3435,6 @@ static void doInitializeStorageGroups(bool createPlanesFromGroups)
     //Ensure that host groups that are defined in terms of other host groups are expanded out so they have an explicit list of hosts
     normalizeHostGroups();
 
-    //Groups are case insensitve, so add an extra key to the storage items to allow them to be
-    //searched by group name, and also check for duplicates.
-    Owned<IPropertyTreeIterator> iter = storage->getElements("planes");
-    StringBuffer group;
-    ForEach(*iter)
-    {
-        IPropertyTree & cur = iter->query();
-        //Check if this has already been done - so the function is safe to call more than once
-        const char * oldgroup = cur.queryProp("@group");
-        if (oldgroup)
-            continue;
-
-        const char * name = cur.queryProp("@name");
-        group.clear().append(name).toLowerCase();
-
-        //Check the storage plane does not match another one case-insensitiviely. (It is unlikely the Helm chart will have installed.)
-        VStringBuffer xpath("plane[@group='%s']", group.str());
-        IPropertyTree * match = storage->queryPropTree(xpath);
-        if (match)
-            throwStringExceptionV(DALI_DUPLICATE_STORAGE_PLANE, "Duplicate storage planes %s,%s (case insensitive)", name, match->queryProp("@name"));
-
-        cur.setProp("@group", group);
-    }
-
     //The following can be removed once the storage planes have better integration
     setupContainerizedStorageLocations();
 }
@@ -3513,7 +3489,7 @@ IStoragePlane * getStoragePlane(const char * name, bool required)
     StringBuffer group;
     group.append(name).toLowerCase();
 
-    VStringBuffer xpath("storage/planes[@group='%s']", group.str());
+    VStringBuffer xpath("storage/planes[@name='%s']", group.str());
     Owned<IPropertyTree> match = getGlobalConfigSP()->getPropTree(xpath);
     if (!match)
     {

dali/base/dameta.cpp

@@ -21,30 +21,6 @@
 #include "dautils.hpp"
 
 
-//More to a more central location
-IPropertyTree * getHostGroup(const char * name, bool required)
-{
-    if (!isEmptyString(name))
-    {
-        VStringBuffer xpath("storage/hostGroups[@name='%s']", name);
-        Owned<IPropertyTree> global = getGlobalConfig();
-        IPropertyTree * match = global->getPropTree(xpath);
-        if (match)
-            return match;
-    }
-    if (required)
-        throw makeStringExceptionV(-1, "No entry found for hostGroup: '%s'", name ? name : "<null>");
-    return nullptr;
-}
-
-IPropertyTree * getStoragePlane(const char * name)
-{
-    VStringBuffer xpath("storage/planes[@name='%s']", name);
-    Owned<IPropertyTree> global = getGlobalConfig();
-    return global->getPropTree(xpath);
-}
-
-
 // Expand indirect hostGroups so each hostGroups has an expanded list of host names
 void normalizeHostGroups()
 {

dali/base/dameta.hpp

@@ -58,8 +58,6 @@ constexpr ResolveOptions operator &(ResolveOptions l, ResolveOptions r) { return
  */
 
 extern da_decl IPropertyTree * resolveLogicalFilenameFromDali(const char * filename, IUserDescriptor * user, ResolveOptions options);
-extern da_decl IPropertyTree * getHostGroup(const char * name, bool required);
-extern da_decl IPropertyTree * getStoragePlane(const char * name);
 extern da_decl void normalizeHostGroups();  // Expand indirect hostGroups so each hostGroups has an expanded list of host names
 
 #endif

dali/base/dautils.cpp

@@ -48,7 +48,7 @@
 #define MIN_REDIRECTION_LOAD_INTERVAL 1000
 
 
-constexpr const char * lz_plane_path = "storage/planes[labels='lz']";
+constexpr const char * lz_plane_path = "storage/planes[@category='lz']";
 
 IPropertyTreeIterator * getDropZonePlanesIterator(const char * name)
 {

dali/dfu/dfurun.cpp

@@ -468,7 +468,6 @@ class CDFUengine: public CInterface, implements IDFUengine
             IPropertyTree & plane = planes->query();
             const char * fullDropZoneDir = plane.queryProp("@prefix");
             assertex(fullDropZoneDir);
-            // note: for bare-metal drop-zones, will need to compare ip address
             if (startsWith(pfilePath, fullDropZoneDir))
                 return;
         }
@@ -1323,7 +1322,7 @@ public:
 #ifdef _CONTAINERIZED
                         StringBuffer clusterName;
                         destination->getGroupName(0, clusterName);
-                        Owned<IPropertyTree> plane = getDropZonePlane(clusterName);
+                        Owned<IPropertyTree> plane = getStoragePlane(clusterName);
                         if (plane)
                         {
                             if (plane->hasProp("@defaultSprayParts"))

dali/dfu/dfurunkdp.cpp

@@ -47,6 +47,10 @@ class CDKDPitem : public CInterface
 #else
         querySlaveExecutable("DKCSlaveProcess", "dkcslave", NULL, ep, p, workdir);
 #endif
+        // This code is being used to execute run_keypatch and run_keydiff.  I suspect it does not work, because it
+        // is using the dkcslave entry to find the execution directory, which is unlikely to be configured since it is no
+        // longer part of the source.  Retain the code for the moment.
+        // If we did want to support this again it would be better to move the keydiff/keypatch functionality into dafilesrv.
         splitDirTail(p.str(),ret);
         addPathSepChar(ret).append(tail);
         if (getPathSepChar(ret.str())=='\\') {

dali/dfuplus/dfuplus.cpp

@@ -351,7 +351,9 @@ int CDfuPlusHelper::doit()
     return 0;
 }
 
-bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format, StringBuffer &retwuid, StringBuffer &except)
+bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const char* srcplane,
+                                const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,
+                                const char *format, StringBuffer &retwuid, StringBuffer &except)
 {
     int recordsize;
     if(stricmp(format, "recfmvb") == 0) {
@@ -374,18 +376,23 @@ bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char*
     }
 
     Owned<IClientSprayFixed> req = sprayclient->createSprayFixedRequest();
-    if(srcxml == nullptr)
+    if(isEmptyString(srcxml))
     {
-        req->setSourceIP(srcip);
+        info("\nFixed spraying from %s on %s to %s\n", srcfile, srcplane?srcplane:srcip, dstname);
+        if (!isEmptyString(srcip)) req->setSourceIP(srcip);
+        if (!isEmptyString(srcplane)) req->setSourcePlane(srcplane);
         req->setSourcePath(srcfile);
     }
     else
+    {
+        info("\nSpraying to %s\n", dstname);
         req->setSrcxml(xmlbuf);
+    }
 
     if(recordsize != 0)
         req->setSourceRecordSize(recordsize);
 
-    if(dstcluster != nullptr)
+    if(!isEmptyString(dstcluster))
         req->setDestGroup(dstcluster);
     req->setDestLogicalName(dstname);
     req->setOverwrite(globals->getPropBool("overwrite", false));
@@ -429,11 +436,6 @@ bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char*
     if(globals->hasProp("expireDays"))
         req->setExpireDays(globals->getPropInt("expireDays"));
 
-    if(srcxml == nullptr)
-        info("\nFixed spraying from %s on %s to %s\n", srcfile, srcip, dstname);
-    else
-        info("\nSpraying to %s\n", dstname);
-
     Owned<IClientSprayFixedResponse> result = sprayclient->SprayFixed(req);
     const char *wuid = result->getWuid();
     if (!wuid||!*wuid) {
@@ -444,21 +446,26 @@ bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char*
     return true;
 }
 
-bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid, StringBuffer &except)
+bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const char* srcplane,
+                                   const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,
+                                   const char *format,StringBuffer &retwuid, StringBuffer &except)
 {
     Owned<IClientSprayVariable> req = sprayclient->createSprayVariableRequest();
-    if(srcxml == nullptr)
+    if(isEmptyString(srcxml))
     {
-        req->setSourceIP(srcip);
+        info("\nVariable spraying from %s on %s to %s\n", srcfile, srcplane?srcplane:srcip, dstname);
+        if (!isEmptyString(srcip)) req->setSourceIP(srcip);
+        if (!isEmptyString(srcplane)) req->setSourcePlane(srcplane);
         req->setSourcePath(srcfile);
     }
     else
     {
+        info("\nSpraying to %s\n", dstname);
         req->setSrcxml(xmlbuf);
     }
 
     const char* mrsstr = globals->queryProp("maxRecordSize");
-    if(mrsstr != nullptr)
+    if(!isEmptyString(mrsstr))
         req->setSourceMaxRecordSize(atoi(mrsstr));
     else
         req->setSourceMaxRecordSize(8192);
@@ -480,23 +487,23 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
     }
     else if(stricmp(format, "xml") == 0)
     {
-        if(encoding == nullptr)
+        if(isEmptyString(encoding))
             encoding = "utf8";
         else if(stricmp(encoding, "ascii") == 0)
             throw MakeStringException(-1, "xml format only accepts utf encodings");
-        if(rowtag == nullptr || *rowtag == '\0')
+        if(isEmptyString(rowtag))
             throw MakeStringException(-1, "rowtag not specified.");
-        if(rowpath && *rowpath)
+        if(!isEmptyString(rowpath))
             throw MakeStringException(-1, "You can't use rowpath option with xml format");
     }
     else if(stricmp(format, "csv") == 0)
     {
-        if(encoding == nullptr)
+        if(isEmptyString(encoding))
             encoding = "ascii";
 
-        if(rowtag != nullptr && *rowtag != '\0')
+        if(!isEmptyString(rowtag))
             throw MakeStringException(-1, "You can't use rowtag option with csv/delimited format");
-        if(rowpath && *rowpath)
+        if(!isEmptyString(rowpath))
             throw MakeStringException(-1, "You can't use rowpath option with csv/delimited format");
 
         const char* separator = globals->queryProp("separator");
@@ -509,7 +516,7 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
                 req->setNoSourceCsvSeparator(true);
         }
         const char* terminator = globals->queryProp("terminator");
-        if(terminator && *terminator)
+        if(!isEmptyString(terminator))
             req->setSourceCsvTerminate(terminator);
         const char* quote = globals->queryProp("quote");
         if(quote)
@@ -519,17 +526,17 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
             req->setSourceCsvQuote(quote);
         }
         const char* escape = globals->queryProp("escape");
-        if(escape && *escape)
+        if(!isEmptyString(escape))
             req->setSourceCsvEscape(escape);
     }
     else
         encoding = format; // may need extra later
 
     req->setSourceFormat(CDFUfileformat::decode(encoding));
-    if(rowtag != nullptr)
+    if(!isEmptyString(rowtag))
         req->setSourceRowTag(rowtag);
 
-    if(dstcluster != nullptr)
+    if(!isEmptyString(dstcluster))
         req->setDestGroup(dstcluster);
     req->setDestLogicalName(dstname);
     req->setOverwrite(globals->getPropBool("overwrite", false));
@@ -578,13 +585,10 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
     if(globals->hasProp("expireDays"))
         req->setExpireDays(globals->getPropInt("expireDays"));
 
-    if(srcxml == nullptr)
-        info("\nVariable spraying from %s on %s to %s\n", srcfile, srcip, dstname);
-    else
-        info("\nSpraying to %s\n", dstname);
     Owned<IClientSprayResponse> result = sprayclient->SprayVariable(req);
     const char *wuid = result->getWuid();
-    if (!wuid||!*wuid) {
+    if (isEmptyString(wuid))
+    {
         result->getExceptions().errorMessage(except);
         return false;
     }
@@ -594,31 +598,36 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
 
 int CDfuPlusHelper::spray()
 {
+    bool usingSrcPlane = false;
     const char* srcxml = globals->queryProp("srcxml");
     const char* srcip = globals->queryProp("srcip");
+    const char* srcplane =  globals->queryProp("srcPlane");
     const char* srcfile = globals->queryProp("srcfile");
 
     bool nowait = globals->getPropBool("nowait", false);
 
+    if (!isEmptyString(srcplane))
+        usingSrcPlane=true;
     MemoryBuffer xmlbuf;
 
-    if(srcxml == nullptr)
+    if(isEmptyString(srcxml))
     {
-        if(srcfile == nullptr)
+        if(isEmptyString(srcfile))
             throw MakeStringException(-1, "srcfile not specified");
-        if(srcip == nullptr) {
+        if(!usingSrcPlane && isEmptyString(srcip))
+        {
 #ifdef DAFILESRV_LOCAL
             progress("srcip not specified - assuming spray from local machine\n");
             srcip = ".";
 #else
-            throw MakeStringException(-1, "srcip not specified");
+            throw MakeStringException(-1, "Neither srcip nor srcplane specified");
 #endif
         }
     }
     else
     {
-        if(srcip != nullptr || srcfile != nullptr)
-            throw MakeStringException(-1, "srcip/srcfile and srcxml can't be used at the same time");
+        if(!isEmptyString(srcip) || !isEmptyString(srcfile) || usingSrcPlane)
+            throw MakeStringException(-1, "srcip/srcfile/srcplane and srcxml can't be used at the same time");
         StringBuffer buf;
         buf.loadFile(srcxml);
         int len = buf.length();
@@ -626,10 +635,10 @@ int CDfuPlusHelper::spray()
     }
 
     const char* dstname = globals->queryProp("dstname");
-    if(dstname == nullptr)
+    if(isEmptyString(dstname))
         throw MakeStringException(-1, "dstname not specified");
     const char* dstcluster = globals->queryProp("dstcluster");
-    if(dstcluster == nullptr)
+    if(isEmptyString(dstcluster))
         throw MakeStringException(-1, "dstcluster not specified");
     const char* format = globals->queryProp("format");
     if(format == nullptr)
@@ -637,17 +646,20 @@ int CDfuPlusHelper::spray()
     else if (stricmp(format, "delimited") == 0)
         format="csv";
 
-    SocketEndpoint localep;
-    StringBuffer localeps;
-    if (checkLocalDaFileSvr(srcip,localep))
-        srcip = localep.getUrlStr(localeps).str();
+    if (!usingSrcPlane)
+    {
+        SocketEndpoint localep;
+        StringBuffer localeps;
+        if (checkLocalDaFileSvr(srcip,localep))
+            srcip = localep.getUrlStr(localeps).str();
+    }
     StringBuffer wuid;
     StringBuffer errmsg;
     bool ok;
     if ((stricmp(format, "fixed") == 0)||(stricmp(format, "recfmvb") == 0)||(stricmp(format, "recfmv") == 0)||(stricmp(format, "variablebigendian") == 0))
-        ok = fixedSpray(srcxml,srcip,srcfile,xmlbuf,dstcluster,dstname,format,wuid,errmsg);
+        ok = fixedSpray(srcxml,srcip,srcfile,srcplane,xmlbuf,dstcluster,dstname,format,wuid,errmsg);
     else if((stricmp(format, "csv") == 0)||(stricmp(format, "xml") == 0)||(stricmp(format, "json") == 0)||(stricmp(format, "variable") == 0))
-        ok = variableSpray(srcxml,srcip,srcfile,xmlbuf,dstcluster,dstname,format,wuid, errmsg);
+        ok = variableSpray(srcxml,srcip,srcfile,srcplane,xmlbuf,dstcluster,dstname,format,wuid, errmsg);
     else
         throw MakeStringException(-1, "format %s not supported", format);
     if (!ok) {
@@ -716,21 +728,22 @@ int CDfuPlusHelper::replicate()
 int CDfuPlusHelper::despray()
 {
     const char* srcname = globals->queryProp("srcname");
-    if(srcname == nullptr)
+    if(isEmptyString(srcname))
         throw MakeStringException(-1, "srcname not specified");
 
     const char* dstxml = globals->queryProp("dstxml");
     const char* dstip = globals->queryProp("dstip");
+    const char* dstplane = globals->queryProp("dstplane");
     const char* dstfile = globals->queryProp("dstfile");
 
     bool nowait = globals->getPropBool("nowait", false);
 
     MemoryBuffer xmlbuf;
-    if(dstxml == nullptr)
+    if(isEmptyString(dstxml))
     {
-        if(dstip == nullptr) {
+        if (isEmptyString(dstplane) && isEmptyString(dstip)) {
 #ifdef DAFILESRV_LOCAL
-            progress("dstip not specified - assuming spray from local machine\n");
+            progress("dstip and dstplane not specified - assuming despray to local machine\n");
             dstip = ".";
 #else
             throw MakeStringException(-1, "dstip not specified");
@@ -739,28 +752,36 @@ int CDfuPlusHelper::despray()
     }
     else
     {
-        if(dstip != nullptr || dstfile != nullptr)
-            throw MakeStringException(-1, "dstip/dstfile and dstxml can't be used at the same time");
+        if(!isEmptyString(dstip) || !isEmptyString(dstfile) || !isEmptyString(dstplane))
+            throw MakeStringException(-1, "dstip/dstfile/dstplane and dstxml can't be used at the same time");
         StringBuffer buf;
         buf.loadFile(dstxml);
         int len = buf.length();
         xmlbuf.setBuffer(len, buf.detach(), true);
     }
 
-    if(dstxml == nullptr)
-        info("\nDespraying %s to host %s file %s\n", srcname, dstip, dstfile ? dstfile : "");
-    else
-        info("\nDespraying %s\n", srcname);
-
     Owned<IClientDespray> req = sprayclient->createDesprayRequest();
     req->setSourceLogicalName(srcname);
-    if(dstxml == nullptr)
+    StringBuffer extrainfo;
+    if(isEmptyString(dstxml))
     {
-        req->setDestIP(dstip);
+        extrainfo.append(" to");
+        if (!isEmptyString(dstplane))
+        {
+            extrainfo.appendf(" storage plane %s", dstplane);
+            req->setDestPlane(dstplane);
+        }
+        if (!isEmptyString(dstip))
+        {
+            extrainfo.appendf(" host %s", dstip);
+            req->setDestIP(dstip);
+        }
+        extrainfo.appendf(" file %s", dstfile ? dstfile : "");
         req->setDestPath(dstfile);
     }
     else
         req->setDstxml(xmlbuf);
+    info("\nDespraying %s%s\n",srcname,extrainfo.str());
 
     req->setOverwrite(globals->getPropBool("overwrite", false));
     if(globals->hasProp("connect"))
@@ -794,14 +815,13 @@ int CDfuPlusHelper::despray()
     if(globals->hasProp("decrypt"))
         req->setDecrypt(globals->queryProp("decrypt"));
 
-
     SocketEndpoint localep;
     StringBuffer localeps;
-    if (checkLocalDaFileSvr(dstip,localep))
+    if (isEmptyString(dstplane) && checkLocalDaFileSvr(dstip,localep))
         dstip = localep.getUrlStr(localeps).str();
     Owned<IClientDesprayResponse> result = sprayclient->Despray(req);
     const char* wuid = result->getWuid();
-    if(wuid == nullptr || *wuid == '\0')
+    if(isEmptyString(wuid))
         exc(result->getExceptions(),"despraying");
     else
     {

dali/dfuplus/dfuplus.hpp

@@ -66,8 +66,8 @@ private:
     int listhistory();
     int erasehistory();
 
-    bool fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid,StringBuffer &except );
-    bool variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid,StringBuffer &except );
+    bool fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const char* srcplane,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid,StringBuffer &except );
+    bool variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const char* srcplane,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid,StringBuffer &except );
 
     int waitToFinish(const char* wuid);
     void info(const char *format, ...) __attribute__((format(printf, 2, 3)));

dali/dfuplus/main.cpp

@@ -63,6 +63,7 @@ void handleSyntax()
     out.append("                                   source file set is empty.\n");
     out.append("    spray options:\n");
     out.append("        srcip=<source-machine-ip>\n");
+    out.append("        srcplane=<source-storage-plane-name>\n");
     out.append("        srcfile=<source-file-path>\n");
     out.append("        srcxml=<xml-file> -- replaces srcip and srcfile\n");
     out.append("        dstname=<destination-logical-name>\n");
@@ -101,6 +102,7 @@ void handleSyntax()
     out.append("    despray options:\n");
     out.append("        srcname=<source-logical-name>\n");
     out.append("        dstip=<destination-machine-ip>\n");
+    out.append("        dstplane=<destination-storage-plane-name>\n");
     out.append("        dstfile=<destination-file-path>\n");
     out.append("        dstxml=<xml-file> -- replaces dstip and dstfile\n");
     out.append("        splitprefix=... use prefix (same format as /prefix) to split file up\n");

ecl/eclccserver/eclccserver.cpp

@@ -1058,7 +1058,7 @@ void openLogFile()
 {
 #ifndef _CONTAINERIZED
     StringBuffer logname;
-    envGetConfigurationDirectory("log","eclccserver", getComponentConfigSP()->queryProp("@name"),logname);
+    getConfigurationDirectory(nullptr, "log","eclccserver", getComponentConfigSP()->queryProp("@name"),logname);
     Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(logname.str(), "eclccserver");
     lf->beginLogging();
 #else

ecl/hql/hqllex.l

@@ -1074,6 +1074,10 @@ __STAND_ALONE__     {
                     }
 __TARGET_PLATFORM__ { RETURNSYM(__TARGET_PLATFORM__); }
 
+__CONTAINERIZED__   {   setupdatepos;
+                        return isContainerized() ? TOK_TRUE : TOK_FALSE;
+                    }
+
 {percent}{alphanumcolon}*{percent} {
                         setupdatepos;
                         if (lexer->skipNesting || lexer->macroGathering)

esp/scm/ws_fs.ecm

@@ -300,6 +300,7 @@ GetDFUExceptionsResponse
 ESPrequest [nil_remove] SprayFixed
 {
     string sourceIP;
+    [min_ver("1.22")] string sourcePlane;
     string sourcePath;
     binary srcxml;
     [min_ver("1.09")] string sourceFormat;
@@ -345,6 +346,7 @@ SprayFixedResponse
 ESPrequest [nil_remove] SprayVariable
 {
     string sourceIP;
+    [min_ver("1.22")] string sourcePlane;
     string sourcePath;
     binary srcxml;
 
@@ -420,6 +422,7 @@ ESPrequest Despray
 
     string destIP;
     string destPath;
+    [min_ver("1.22")] string destPlane;
     binary dstxml;
     bool   overwrite;
 
@@ -689,7 +692,7 @@ ESPresponse [exceptions_inline, nil_remove] GetDFUServerQueuesResponse
 
 ESPservice [
     auth_feature("DEFERRED"),
-    version("1.21"),
+    version("1.22"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPmethod EchoDateTime(EchoDateTime, EchoDateTimeResponse);

esp/services/ws_fs/ws_fsService.cpp

@@ -1826,19 +1826,70 @@ bool CFileSprayEx::onGetDFUExceptions(IEspContext &context, IEspGetDFUExceptions
     return true;
 }
 
-void CFileSprayEx::readAndCheckSpraySourceReq(MemoryBuffer& srcxml, const char* srcIP, const char* srcPath,
+void CFileSprayEx::readAndCheckSpraySourceReq(MemoryBuffer& srcxml, const char* srcIP, const char* srcPath, const char* srcPlane,
     StringBuffer& sourceIPReq, StringBuffer& sourcePathReq)
 {
-    StringBuffer sourcePath(srcPath);
-    sourceIPReq.set(srcIP);
-    sourceIPReq.trim();
-    sourcePath.trim();
+    StringBuffer sourcePath;
+
     if(srcxml.length() == 0)
     {
+        if (!isEmptyString(srcPlane))
+        {
+            Owned<IPropertyTree> dropZone = getDropZonePlane(srcPlane);
+            if (!dropZone)
+                throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Unknown landing zone: %s", srcPlane);
+            const char * dropZonePlanePath = dropZone->queryProp("@prefix");
+            if (isAbsolutePath(srcPath))
+            {
+                if (!startsWith(srcPath,dropZonePlanePath))
+                    throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid source path");
+                sourcePath.append(srcPath).trim();
+            }
+            else
+            {
+                sourcePath.append(dropZonePlanePath);
+                addNonEmptyPathSepChar(sourcePath);
+                sourcePath.append(srcPath).trim();
+            }
+            const char * hostGroup = dropZone->queryProp("@hostGroup");
+            if (hostGroup)
+            {
+                Owned<IPropertyTree> match = getHostGroup(hostGroup,true);
+                if (!isEmptyString(srcIP))
+                {
+                    // Already have srcIP. Just need to check that the ip is valid for storage plane.
+                    bool ipAddressMatches = false;
+                    Owned<IPropertyTreeIterator> hostIter = match->getElements("hosts");
+                    ForEach (*hostIter)
+                    {
+                        const char *knownIP = hostIter->query().queryProp(nullptr);
+                        if (strcmp(knownIP, srcIP)==0)
+                        {
+                            ipAddressMatches=true;
+                            break;
+                        }
+                    }
+                    if (!ipAddressMatches)
+                        throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "srcip %s is not valid storage plane %s", srcIP, srcPlane);
+                    sourceIPReq.set(srcIP);
+                }
+                else
+                    sourceIPReq.set(match->queryProp("hosts[1]"));
+            }
+            else
+            {
+                sourceIPReq.set("localhost"); // storage plane will be mounted when not using hostgroup
+            }
+        }
+        else
+        {
+            sourcePath.append(srcPath).trim();
+            sourceIPReq.set(srcIP).trim();
+        }
         if (sourceIPReq.isEmpty())
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Source network IP not specified.");
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Source network IP not specified.");
         if (sourcePath.isEmpty())
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Source path not specified.");
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Source path not specified.");
     }
     getStandardPosixPath(sourcePathReq, sourcePath.str());
 }
@@ -1881,20 +1932,20 @@ bool CFileSprayEx::onSprayFixed(IEspContext &context, IEspSprayFixed &req, IEspS
         StringBuffer destFolder, destTitle, defaultFolder, defaultReplicateFolder;
 
         const char* destNodeGroup = req.getDestGroup();
-        if(destNodeGroup == NULL || *destNodeGroup == '\0')
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination node group not specified.");
+        if (isEmptyString(destNodeGroup))
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Destination node group not specified.");
 
         MemoryBuffer& srcxml = (MemoryBuffer&)req.getSrcxml();
         StringBuffer sourceIPReq, sourcePathReq;
-        readAndCheckSpraySourceReq(srcxml, req.getSourceIP(), req.getSourcePath(), sourceIPReq, sourcePathReq);
+        readAndCheckSpraySourceReq(srcxml, req.getSourceIP(), req.getSourcePath(), req.getSourcePlane(), sourceIPReq, sourcePathReq);
         const char* srcip = sourceIPReq.str();
         const char* srcfile = sourcePathReq.str();
         const char* destname = req.getDestLogicalName();
-        if(!destname || !*destname)
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination file not specified.");
+        if(isEmptyString(destname))
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Destination file not specified.");
         CDfsLogicalFileName lfn;
         if (!lfn.setValidate(destname))
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid destination filename:'%s'", destname);
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Invalid destination filename:'%s'", destname);
         destname = lfn.get();
         PROGLOG("SprayFixed: DestLogicalName %s, DestGroup %s", destname, destNodeGroup);
 
@@ -1931,7 +1982,7 @@ bool CFileSprayEx::onSprayFixed(IEspContext &context, IEspSprayFixed &req, IEspS
             RemoteMultiFilename rmfn;
             SocketEndpoint ep(srcip);
             if (ep.isNull())
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "SprayFixed to %s: cannot resolve source network IP from %s.", destname, srcip);
+                throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "SprayFixed to %s: cannot resolve source network IP from %s.", destname, srcip);
 
             rmfn.setEp(ep);
             StringBuffer fnamebuf(srcfile);
@@ -2054,8 +2105,8 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
         StringBuffer destFolder, destTitle, defaultFolder, defaultReplicateFolder;
 
         const char* destNodeGroup = req.getDestGroup();
-        if(destNodeGroup == NULL || *destNodeGroup == '\0')
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination node group not specified.");
+        if (isEmptyString(destNodeGroup))
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Destination node group not specified.");
 
         StringBuffer gName, ipAddr;
         const char *pTr = strchr(destNodeGroup, ' ');
@@ -2069,15 +2120,15 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
 
         MemoryBuffer& srcxml = (MemoryBuffer&)req.getSrcxml();
         StringBuffer sourceIPReq, sourcePathReq;
-        readAndCheckSpraySourceReq(srcxml, req.getSourceIP(), req.getSourcePath(), sourceIPReq, sourcePathReq);
+        readAndCheckSpraySourceReq(srcxml, req.getSourceIP(), req.getSourcePath(), req.getSourcePlane(), sourceIPReq, sourcePathReq);
         const char* srcip = sourceIPReq.str();
         const char* srcfile = sourcePathReq.str();
         const char* destname = req.getDestLogicalName();
-        if(!destname || !*destname)
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination file not specified.");
+        if(isEmptyString(destname))
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Destination file not specified.");
         CDfsLogicalFileName lfn;
         if (!lfn.setValidate(destname))
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid destination filename:'%s'", destname);
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Invalid destination filename:'%s'", destname);
 
         destname = lfn.get();
         PROGLOG("SprayVariable: DestLogicalName %s, DestGroup %s", destname, destNodeGroup);
@@ -2108,7 +2159,7 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
             RemoteMultiFilename rmfn;
             SocketEndpoint ep(srcip);
             if (ep.isNull())
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "SprayVariable to %s: cannot resolve source network IP from %s.", destname, srcip);
+                throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "SprayVariable to %s: cannot resolve source network IP from %s.", destname, srcip);
 
             rmfn.setEp(ep);
             StringBuffer fnamebuf(srcfile);
@@ -2272,25 +2323,54 @@ bool CFileSprayEx::onReplicate(IEspContext &context, IEspReplicate &req, IEspRep
     return true;
 }
 
-void CFileSprayEx::getDropZoneInfoByDestPlane(double clientVersion, const char* destPlane, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask)
+void CFileSprayEx::getDropZoneInfoByDestPlane(double clientVersion, const char* destPlane, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask, StringBuffer & hostip)
 {
-    Owned<IPropertyTree> plane = getDropZonePlane(destPlane);
-    if (!plane)
-        throw makeStringExceptionV(ECLWATCH_DROP_ZONE_NOT_FOUND, "No drop zone matching plane %s", destPlane);
+    Owned<IPropertyTree> dropZone = getDropZonePlane(destPlane);
+    if (!dropZone)
+        throw makeStringExceptionV(ECLWATCH_DROP_ZONE_NOT_FOUND, "Unknown landing zone %s", destPlane);
 
-    StringBuffer fullDropZoneDir(plane->queryProp("@prefix"));
+    StringBuffer fullDropZoneDir(dropZone->queryProp("@prefix"));
     addPathSepChar(fullDropZoneDir);
     if (isAbsolutePath(destFileIn))
     {
-        if (strncmp(fullDropZoneDir, destFileIn, fullDropZoneDir.length()))
-            throw makeStringExceptionV(ECLWATCH_DROP_ZONE_NOT_FOUND, "No drop zone configured for %s:%s", destPlane, destFileIn);
-        destFileOut.set(destFileIn);
+        if (!startsWith(destFileIn, fullDropZoneDir))
+            throw makeStringExceptionV(ECLWATCH_DROP_ZONE_NOT_FOUND, "No landing zone configured for %s:%s", destPlane, destFileIn);
+    }
+    else
+    {
+        destFileOut.append(fullDropZoneDir);
+        addNonEmptyPathSepChar(destFileOut);
+    }
+    destFileOut.append(destFileIn).trim();
+    dropZone->getProp("@umask", umask);
+    const char * hostGroup = dropZone->queryProp("@hostGroup");
+    if (hostGroup)
+    {
+        Owned<IPropertyTree> match = getHostGroup(hostGroup,true);
+        if (!hostip.isEmpty())
+        {
+            // Already have hostip. Just need to check that the ip is valid for storage plane.
+            bool ipAddressMatches = false;
+            Owned<IPropertyTreeIterator> hostIter = match->getElements("hosts");
+            ForEach (*hostIter)
+            {
+                const char *knownIP = hostIter->query().queryProp(nullptr);
+                if (strcmp(knownIP, hostip)==0)
+                {
+                    ipAddressMatches=true;
+                    break;
+                }
+            }
+            if (!ipAddressMatches)
+                throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "destip %s is not valid storage plane %s", hostip.str(), destPlane);
+        }
+        else
+            hostip.set(match->queryProp("hosts[1]"));
     }
     else
     {
-        destFileOut.append(destFileIn);
+        hostip.set("localhost"); // storage plane will be mounted when not using hostgroup
     }
-    plane->getProp("@umask", umask);
 }
 
 void CFileSprayEx::getDropZoneInfoByIP(double clientVersion, const char* ip, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask)
@@ -2417,7 +2497,13 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
 
         PROGLOG("Despray %s", srcname);
         double version = context.getClientVersion();
-        const char* destip = req.getDestIP();
+        StringBuffer destip(req.getDestIP());
+        const char* destPlane = req.getDestPlane();
+#ifdef _CONTAINERIZED
+        if (isEmptyString(destPlane))
+            destPlane = req.getDestGroup();  // allow eclwatch to continue providing storage plane as 'destgroup' field
+#endif
+
         StringBuffer destPath;
         StringBuffer implicitDestFile;
         const char* destfile = getStandardPosixPath(destPath, req.getDestPath()).str();
@@ -2425,8 +2511,8 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
         MemoryBuffer& dstxml = (MemoryBuffer&)req.getDstxml();
         if(dstxml.length() == 0)
         {
-            if(!destip || !*destip)
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination network IP not specified.");
+            if(isEmptyString(destPlane) && destip.isEmpty())
+                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination network IP/storage plane not specified.");
 
             //If the destination filename is not provided, calculate a relative filename from the logical filename
             if(!destfile || !*destfile)
@@ -2456,18 +2542,17 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
 
         if(dstxml.length() == 0)
         {
+            StringBuffer destfileWithPath, umask;
+            if (!isEmptyString(destPlane))
+                getDropZoneInfoByDestPlane(version, destPlane, destfile, destfileWithPath, umask, destip);
+            else
+                getDropZoneInfoByIP(version, destip, destfile, destfileWithPath, umask);
+
             RemoteFilename rfn;
-            SocketEndpoint ep(destip);
+            SocketEndpoint ep(destip.str());
             if (ep.isNull())
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Despray %s: cannot resolve destination network IP from %s.", srcname, destip);
+                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Despray %s: cannot resolve destination network IP from %s.", srcname, destip.str());
 
-            StringBuffer destfileWithPath, umask;
-#ifdef _CONTAINERIZED
-            const char* destPlane = req.getDestGroup();
-            getDropZoneInfoByDestPlane(version, destPlane, destfile, destfileWithPath, umask);
-#else
-            getDropZoneInfoByIP(version, destip, destfile, destfileWithPath, umask);
-#endif
             //Ensure the filename is dependent on the file part if parts are being preserved
             if (preserveFileParts && !strstr(destfileWithPath, "$P$"))
                 destfileWithPath.append("._$P$_of_$N$");
@@ -2576,7 +2661,7 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
         if (!bRoxie)
         {
             if (!lfn.setValidate(dstname))
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid destination filename:'%s'", dstname);
+                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid destination filename:'%s'", dstname);
             dstname = lfn.get();
         }
 
@@ -3488,7 +3573,7 @@ bool CFileSprayEx::onGetSprayTargets(IEspContext &context, IEspGetSprayTargetsRe
         context.ensureFeatureAccess(FILE_SPRAY_URL, SecAccess_Read, ECLWATCH_FILE_SPRAY_ACCESS_DENIED, "Permission denied.");
 #ifdef _CONTAINERIZED
         IArrayOf<IEspGroupNode> sprayTargets;
-        Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[labels='data']");
+        Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[@category='data']");
         ForEach(*dataPlanes)
         {
             IPropertyTree & plane = dataPlanes->query();

esp/services/ws_fs/ws_fsService.hpp

@@ -66,7 +66,7 @@ public:
 
 class CFileSprayEx : public CFileSpray
 {
-    void readAndCheckSpraySourceReq(MemoryBuffer& srcxml, const char* srcIP, const char* srcPath,
+    void readAndCheckSpraySourceReq(MemoryBuffer& srcxml, const char* srcIP, const char* srcPath, const char* srcplane,
         StringBuffer& sourceIPReq, StringBuffer& sourcePathReq);
 
 public:
@@ -148,7 +148,7 @@ protected:
     void appendGroupNode(IArrayOf<IEspGroupNode>& groupNodes, const char* nodeName, const char* clusterType, bool replicateOutputs);
     bool getOneDFUWorkunit(IEspContext& context, const char* wuid, IEspGetDFUWorkunitsResponse& resp);
     void getDropZoneInfoByIP(double clientVersion, const char* destIP, const char* destFile, StringBuffer& path, StringBuffer& mask);
-    void getDropZoneInfoByDestPlane(double clientVersion, const char* destGroup, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask);
+    void getDropZoneInfoByDestPlane(double clientVersion, const char* destGroup, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask, StringBuffer & hostip);
     bool checkDropZoneIPAndPath(double clientVersion, const char* dropZone, const char* netAddr, const char* path);
     void addDropZoneFile(IEspContext& context, IDirectoryIterator* di, const char* name, const char pathSep, IArrayOf<IEspPhysicalFileStruct>&files);
     void searchDropZoneFiles(IEspContext& context, IpAddress& ip, const char* dir, const char* nameFilter, IArrayOf<IEspPhysicalFileStruct>& files, unsigned& filesFound);

esp/services/ws_topology/ws_topologyService.cpp

@@ -1466,7 +1466,6 @@ bool CWsTopologyEx::onTpServiceQuery(IEspContext &context, IEspTpServiceQueryReq
             m_TpWrapper.getTpGenesisServers( ServiceList.getTpGenesisServers() );
             m_TpWrapper.getTpLdapServers( ServiceList.getTpLdapServers() );
             m_TpWrapper.getTpFTSlaves( ServiceList.getTpFTSlaves() );
-            m_TpWrapper.getTpDkcSlaves( ServiceList.getTpDkcSlaves() );
 
             if (version > 1.15)
             {

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -471,7 +471,7 @@ void CWsWorkunitsEx::init(IPropertyTree *cfg, const char *process, const char *s
 
     recursiveCreateDirectory(ESP_WORKUNIT_DIR);
 
-    getConfigurationDirectory(directories, "data", "esp", process, dataDirectory);
+    getConfigurationDirectory(directories, "temp", "esp", process, tempDirectory);
     wuFactory.setown(getWorkUnitFactory());
 
 #ifdef _CONTAINERIZED
@@ -2754,7 +2754,7 @@ void getCSVHeaders(const IResultSetMetaData& metaIn, CommonCSVWriter* writer, un
     }
 }
 
-unsigned getResultCSV(IStringVal& ret, INewResultSet* result, const char* name, __int64 start, unsigned& count, IAbortRequestCallback* abortCheck)
+unsigned getResultCSV(IStringVal& ret, INewResultSet* result, const char* name, __int64 start, unsigned& count, IAbortRequestCallback* abortCheck, __uint64 maxSize)
 {
     unsigned headerLayer = 0;
     CSVOptions csvOptions;
@@ -2767,12 +2767,12 @@ unsigned getResultCSV(IStringVal& ret, INewResultSet* result, const char* name,
     writer->finishCSVHeaders();
 
     Owned<IResultSetCursor> cursor = result->createCursor();
-    count = writeResultCursorXml(*writer, cursor, name, start, count, nullptr, nullptr, false, abortCheck);
+    count = writeResultCursorXml(*writer, cursor, name, start, count, nullptr, nullptr, false, abortCheck, maxSize);
     ret.set(writer->str());
     return count;
 }
 
-void appendResultSet(IEspContext &context, MemoryBuffer& mb, INewResultSet* result, const char *name, __int64 start, unsigned& count, __int64& total, bool bin, bool xsd, ESPSerializationFormat fmt, const IProperties *xmlns)
+void appendResultSet(IEspContext &context, MemoryBuffer& mb, INewResultSet* result, const char *name, __int64 start, unsigned& count, __int64& total, bool bin, bool xsd, ESPSerializationFormat fmt, const IProperties *xmlns, __uint64 maxSize)
 {
     if (!result)
         return;
@@ -2780,7 +2780,7 @@ void appendResultSet(IEspContext &context, MemoryBuffer& mb, INewResultSet* resu
     total=result->getNumRows();
 
     if(bin)
-        count = getResultBin(mb, result, (unsigned)start, count);
+        count = getResultBin(mb, result, (unsigned)start, count, maxSize);
     else
     {
         struct MemoryBuffer2IStringVal : public CInterface, implements IStringVal
@@ -2798,11 +2798,11 @@ void appendResultSet(IEspContext &context, MemoryBuffer& mb, INewResultSet* resu
 
         CESPAbortRequestCallback abortCallback(&context);
         if (fmt==ESPSerializationCSV)
-            count = getResultCSV(adaptor, result, name, (unsigned) start, count, &abortCallback);
+            count = getResultCSV(adaptor, result, name, (unsigned) start, count, &abortCallback, maxSize);
         else if (fmt==ESPSerializationJSON)
-            count = getResultJSON(adaptor, result, name, (unsigned) start, count, (xsd) ? "myschema" : NULL, &abortCallback);
+            count = getResultJSON(adaptor, result, name, (unsigned) start, count, (xsd) ? "myschema" : NULL, &abortCallback, maxSize);
         else
-            count = getResultXml(adaptor, result, name, (unsigned) start, count, (xsd) ? "myschema" : NULL, xmlns, &abortCallback);
+            count = getResultXml(adaptor, result, name, (unsigned) start, count, (xsd) ? "myschema" : NULL, xmlns, &abortCallback, maxSize);
     }
 }
 
@@ -2836,27 +2836,6 @@ INewResultSet* createFilteredResultSet(INewResultSet* result, IArrayOf<IConstNam
     return filter->create();
 }
 
-
-static bool isResultRequestSzTooBig(unsigned __int64 start, unsigned requestCount, unsigned __int64 resultSz, unsigned resultRows, unsigned __int64 limitSz)
-{
-    if ((0 == start) && (0 == requestCount)) // all being requested
-        return resultSz > limitSz;
-    else if (0 == resultRows)
-    {
-        // if resultRows == 0, have to assume the row count of the file is unknown (e.g. because sprayed and no meta)
-        // There is insuficient information to validate if the result will be too big.
-        return false;
-    }
-    else
-    {
-        if (start+requestCount > resultRows)
-            requestCount = resultRows-start;
-        unsigned __int64 avgRecSize = resultSz / resultRows;
-        unsigned __int64 estSize = requestCount * avgRecSize;
-        return estSize > limitSz;
-    }
-}
-
 void CWsWorkunitsEx::getWsWuResult(IEspContext &context, const char *wuid, const char *name, const char *logical, unsigned index, __int64 start,
     unsigned &count, __int64 &total, IStringVal &resname, bool bin, IArrayOf<IConstNamedValue> *filterBy, MemoryBuffer &mb,
     WUState &wuState, bool xsd)
@@ -2897,39 +2876,15 @@ void CWsWorkunitsEx::getWsWuResult(IEspContext &context, const char *wuid, const
     result->getResultLogicalName(logicalName);
     Owned<INewResultSet> rs;
     if (logicalName.length())
-    {
         rs.setown(resultSetFactory->createNewFileResultSet(logicalName.str(), cw->queryClusterName())); //MORE is this wrong cluster?
-    }
     else
         rs.setown(resultSetFactory->createNewResultSet(result, wuid));
     if (!filterBy || !filterBy->length())
-    {
-        unsigned __int64 resultSz;
-
-        if (0 == logicalName.length()) // could be a workunit owned file (OUTPUT, THOR)
-            result->getResultFilename(logicalName);
-        if (logicalName.length())
-        {
-            Owned<IDistributedFile> df = lookupLogicalName(context, logicalName.str(), false, false, false, nullptr, defaultPrivilegedUser);
-            if (!df)
-                throw makeStringExceptionV(ECLWATCH_FILE_NOT_EXIST, "Cannot find file %s.", logicalName.str());
-            resultSz = df->getDiskSize(true, false);
-        }
-        else
-            resultSz = result->getResultRawSize(nullptr, nullptr);
-
-        if (isResultRequestSzTooBig(start, count, resultSz, rs->getNumRows(), wuResultMaxSize))
-        {
-            throw makeStringExceptionV(ECLWATCH_INVALID_ACTION, "Failed to get the result for %s. The size is bigger than %lld MB.",
-                                       wuid, wuResultMaxSize/0x100000);
-        }
-
-        appendResultSet(context, mb, rs, name, start, count, total, bin, xsd, context.getResponseFormat(), result->queryResultXmlns());
-    }
+        appendResultSet(context, mb, rs, name, start, count, total, bin, xsd, context.getResponseFormat(), result->queryResultXmlns(), wuResultMaxSize);
     else
     {
         Owned<INewResultSet> filteredResult = createFilteredResultSet(rs, filterBy);
-        appendResultSet(context, mb, filteredResult, name, start, count, total, bin, xsd, context.getResponseFormat(), result->queryResultXmlns());
+        appendResultSet(context, mb, filteredResult, name, start, count, total, bin, xsd, context.getResponseFormat(), result->queryResultXmlns(), wuResultMaxSize);
     }
 
     wuState = cw->getState();
@@ -3227,20 +3182,11 @@ void CWsWorkunitsEx::getFileResults(IEspContext &context, const char *logicalNam
     Owned<IResultSetFactory> resultSetFactory = getSecResultSetFactory(context.querySecManager(), context.queryUser(), context.queryUserId(), context.queryPassword());
     Owned<INewResultSet> result(resultSetFactory->createNewFileResultSet(df, cluster));
     if (!filterBy || !filterBy->length())
-    {
-        if (isResultRequestSzTooBig(start, count, df->getDiskSize(true, false), result->getNumRows(), wuResultMaxSize))
-        {
-            throw makeStringExceptionV(ECLWATCH_INVALID_ACTION, "Failed to get the result from file %s. The size is bigger than %lld MB.",
-                                       logicalName, wuResultMaxSize/0x100000);
-        }
-    
-        appendResultSet(context, buf, result, resname.str(), start, count, total, bin, xsd, context.getResponseFormat(), nullptr);
-    }
+        appendResultSet(context, buf, result, resname.str(), start, count, total, bin, xsd, context.getResponseFormat(), nullptr, wuResultMaxSize);
     else
     {
-        // NB: this could be still be very big, appendResultSet should be changed to ensure filtered result doesn't grow bigger than wuResultMaxSize
         Owned<INewResultSet> filteredResult = createFilteredResultSet(result, filterBy);
-        appendResultSet(context, buf, filteredResult, resname.str(), start, count, total, bin, xsd, context.getResponseFormat(), nullptr);
+        appendResultSet(context, buf, filteredResult, resname.str(), start, count, total, bin, xsd, context.getResponseFormat(), nullptr, wuResultMaxSize);
     }
 }
 
@@ -4562,7 +4508,7 @@ int CWsWorkunitsSoapBindingEx::onStartUpload(IEspContext &ctx, CHttpRequest* req
 
             VStringBuffer fileName("%s%s", zipFolder, fileNames.item(0));
             wswService->queryWUFactory()->importWorkUnit(fileName, password,
-                wswService->getDataDirectory(), "ws_workunits", ctx.queryUserId(), ctx.querySecManager(), ctx.queryUser());
+                wswService->getTempDirectory(), "ws_workunits", ctx.queryUserId(), ctx.querySecManager(), ctx.queryUser());
         }
         else
             throw MakeStringException(ECLWATCH_INVALID_INPUT, "WsWorkunits::%s does not support the upload_ option.", method);

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -36,7 +36,7 @@
 #define UFO_RELOAD_MAPPED_QUERIES                0x04
 #define UFO_REMOVE_QUERIES_NOT_IN_QUERYSET       0x08
 
-static const __uint64 defaultWUResultMaxSize = 10000000; //10M
+static const __uint64 defaultWUResultMaxSize = 0x100000*10; //10M
 
 class QueryFilesInUse : public CInterface, implements ISDSSubscription
 {
@@ -233,7 +233,7 @@ public:
     void checkAndSetClusterQueryState(IEspContext &context, const char* cluster, const char* querySetId, IArrayOf<IEspQuerySetQuery>& queries, bool checkAllNodes);
     void checkAndSetClusterQueryState(IEspContext &context, const char* cluster, StringArray& querySetIds, IArrayOf<IEspQuerySetQuery>& queries, bool checkAllNodes);
     IWorkUnitFactory *queryWUFactory() { return wuFactory; };
-    const char *getDataDirectory() const { return dataDirectory.str(); };
+    const char *getTempDirectory() const { return tempDirectory.str(); };
 
     bool onWUQuery(IEspContext &context, IEspWUQueryRequest &req, IEspWUQueryResponse &resp);
     bool onWULightWeightQuery(IEspContext &context, IEspWULightWeightQueryRequest &req, IEspWULightWeightQueryResponse &resp);
@@ -421,7 +421,7 @@ private:
     Owned<IThreadPool> clusterQueryStatePool;
     unsigned thorSlaveLogThreadPoolSize = THOR_SLAVE_LOG_THREAD_POOL_SIZE;
     Owned<IWorkUnitFactory> wuFactory;
-    StringBuffer dataDirectory;
+    StringBuffer tempDirectory;
     __uint64 wuResultMaxSize = defaultWUResultMaxSize;
 
 public:

esp/smc/SMCLib/TpContainer.cpp

@@ -164,11 +164,6 @@ void CTpWrapper::getTpFTSlaves(IArrayOf<IConstTpFTSlave>& list)
     throw MakeStringExceptionDirect(ECLWATCH_CANNOT_GET_ENV_INFO, MSG_FAILED_GET_ENVIRONMENT_INFO);
 }
 
-void CTpWrapper::getTpDkcSlaves(IArrayOf<IConstTpDkcSlave>& list)
-{
-    throw MakeStringExceptionDirect(ECLWATCH_CANNOT_GET_ENV_INFO, MSG_FAILED_GET_ENVIRONMENT_INFO);
-}
-
 void CTpWrapper::getTpGenesisServers(IArrayOf<IConstTpGenesisServer>& list)
 {
     throw MakeStringExceptionDirect(ECLWATCH_CANNOT_GET_ENV_INFO, MSG_FAILED_GET_ENVIRONMENT_INFO);
@@ -204,7 +199,7 @@ void CTpWrapper::getGroupList(double espVersion, const char* kindReq, IArrayOf<I
 {
     try
     {
-        Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[labels='data']");
+        Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[@category='data']");
         ForEach(*dataPlanes)
         {
             IPropertyTree & plane = dataPlanes->query();
@@ -746,7 +741,7 @@ static void refreshValidTargets()
 static void refreshValidDataPlaneNames()
 {
     validDataPlaneNames.clear();
-    Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[labels='data']");
+    Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[@category='data']");
     ForEach(*dataPlanes)
         validDataPlaneNames.insert(dataPlanes->query().queryProp("@name"));
 }
@@ -806,7 +801,7 @@ StringBuffer & getRoxieDefaultPlane(StringBuffer & plane, const char * roxieName
         return plane;
 
     //Find the first data plane - better if it was retrieved from roxie config
-    Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[labels='data']");
+    Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[@category='data']");
     if (!dataPlanes->first())
         throwUnexpectedX("No default data plane defined");
     return plane.append(dataPlanes->query().queryProp("@name"));

esp/smc/SMCLib/TpWrapper.cpp

@@ -596,30 +596,6 @@ void CTpWrapper::getTpFTSlaves(IArrayOf<IConstTpFTSlave>& list)
     }
 }
 
-void CTpWrapper::getTpDkcSlaves(IArrayOf<IConstTpDkcSlave>& list)
-{
-    Owned<IPropertyTree> root = getEnvironment("Software");
-    if (!root)
-        throw MakeStringExceptionDirect(ECLWATCH_CANNOT_GET_ENV_INFO, MSG_FAILED_GET_ENVIRONMENT_INFO);
-
-    Owned<IPropertyTreeIterator> services= root->getElements(eqDkcSlave);
-    ForEach(*services)
-    {
-        IPropertyTree& serviceTree = services->query();
-
-        Owned<IEspTpDkcSlave> pService =createTpDkcSlave("","");
-        pService->setName(serviceTree.queryProp("@name"));
-        pService->setDescription(serviceTree.queryProp("@description"));
-        pService->setBuild(serviceTree.queryProp("@build"));
-
-        IArrayOf<IEspTpMachine> tpMachines;
-        fetchInstances(eqDkcSlave, serviceTree, tpMachines);
-        pService->setTpMachines(tpMachines);
-
-        list.append(*pService.getLink());
-    }
-}
-
 void CTpWrapper::getTpGenesisServers(IArrayOf<IConstTpGenesisServer>& list)
 {
     Owned<IPropertyTree> root = getEnvironment("Software");

esp/smc/SMCLib/TpWrapper.hpp

@@ -93,7 +93,6 @@ using std::string;
 #define eqAgentExec         "AgentExecProcess"
 #define eqEsp                   "EspProcess"
 #define eqDfu                   "DfuServerProcess"
-#define eqDkcSlave          "DKCSlaveProcess"
 #define eqSashaServer      "SashaServerProcess"
 #define eqLdapServer       "LDAPServerProcess"
 #define eqFTSlave          "FTSlaveProcess"
@@ -183,7 +182,6 @@ public:
     void getTpLdapServers(IArrayOf<IConstTpLdapServer>& list);
     void getTpDropZones(double clientVersion, const char* name, bool ECLWatchVisibleOnly, IArrayOf<IConstTpDropZone>& list);
     void getTpFTSlaves(IArrayOf<IConstTpFTSlave>& list);
-    void getTpDkcSlaves(IArrayOf<IConstTpDkcSlave>& list);
     void getTpGenesisServers(IArrayOf<IConstTpGenesisServer>& list);
     void getTpSparkThors(double clientVersion, const char* name, IArrayOf<IConstTpSparkThor>& list);
 

esp/src/src/FileSpray.ts

@@ -7,6 +7,49 @@ import * as ESPRequest from "./ESPRequest";
 
 declare const dojo;
 
+const lfEncode = (path: string) => {
+    let retVal = "";
+    for (let i = 0; i < path.length; ++i) {
+        switch (path[i]) {
+            case "/":
+            case "\\":
+                retVal += "::";
+                break;
+            case "A":
+            case "B":
+            case "C":
+            case "D":
+            case "E":
+            case "F":
+            case "G":
+            case "H":
+            case "I":
+            case "J":
+            case "K":
+            case "L":
+            case "M":
+            case "N":
+            case "O":
+            case "P":
+            case "Q":
+            case "R":
+            case "S":
+            case "T":
+            case "U":
+            case "V":
+            case "W":
+            case "X":
+            case "Y":
+            case "Z":
+                retVal += "^" + path[i];
+                break;
+            default:
+                retVal += path[i];
+        }
+    }
+    return retVal;
+};
+
 class FileListStore extends ESPRequest.Store {
 
     service = "FileSpray";
@@ -19,50 +62,8 @@ class FileListStore extends ESPRequest.Store {
 
     create(id) {
         const retVal = {
-            lfEncode(path) {
-                let retVal = "";
-                for (let i = 0; i < path.length; ++i) {
-                    switch (path[i]) {
-                        case "/":
-                        case "\\":
-                            retVal += "::";
-                            break;
-                        case "A":
-                        case "B":
-                        case "C":
-                        case "D":
-                        case "E":
-                        case "F":
-                        case "G":
-                        case "H":
-                        case "I":
-                        case "J":
-                        case "K":
-                        case "L":
-                        case "M":
-                        case "N":
-                        case "O":
-                        case "P":
-                        case "Q":
-                        case "R":
-                        case "S":
-                        case "T":
-                        case "U":
-                        case "V":
-                        case "W":
-                        case "X":
-                        case "Y":
-                        case "Z":
-                            retVal += "^" + path[i];
-                            break;
-                        default:
-                            retVal += path[i];
-                    }
-                }
-                return retVal;
-            },
             getLogicalFile() {
-                return "~file::" + this.NetAddress + this.lfEncode(this.fullPath);
+                return "~file::" + this.NetAddress + lfEncode(this.fullPath);
             }
         };
         retVal[this.idProperty] = id;
@@ -115,13 +116,21 @@ class LandingZonesFilterStore extends ESPRequest.Store {
     }
 
     preProcessRow(row) {
-        const fullPath = this.dropZone.machine.Directory + "/" + (row.Path === null ? "" : (row.Path + "/"));
+        const fullPath = this.dropZone.machine.Directory + "/" + (row.Path === null ? "" : (row.Path + "/")) + row.name;
+        const fullFolderPathParts = fullPath.split("/");
+        fullFolderPathParts.pop();
+        const netAddress = this.dropZone.machine.Netaddress;
+        const getLogicalFile = () => {
+            return `~file::${netAddress}${lfEncode(fullPath)}`;
+        };
         lang.mixin(row, {
             NetAddress: this.dropZone.machine.Netaddress,
             Directory: this.dropZone.machine.Directory,
             calculatedID: this.dropZone.machine.Netaddress + fullPath,
             OS: this.dropZone.machine.OS,
-            fullFolderPath: fullPath,
+            fullPath,
+            fullFolderPath: fullFolderPathParts.join("/"),
+            getLogicalFile,
             displayName: row.Path ? (row.Path + "/" + row.name) : row.name,
             type: row.isDir ? "filteredFolder" : "file"
         });
@@ -143,7 +152,7 @@ class LandingZonesStore extends ESPRequest.Store {
     }
 
     query(query, options) {
-        if (!query.filter) {
+        if (!query.filter || Object.entries(query.filter).length === 0) {
             return super.query(query, options);
         }
         const landingZonesFilterStore = new LandingZonesFilterStore({ dropZone: query.filter.__dropZone, server: query.filter.Server });

helm/examples/azure/hpcc-azurefile/values.schema.json

@@ -55,10 +55,9 @@
         "sku": {
           "type": "string"
         },
-        "labels": {
-          "description": "a list of labels associated with this plane, e.g. lz, data",
-          "type": "array",
-          "items": { "type": "string" }
+        "category": {
+          "description": "the category this plane is usd for, e.g. lz, data",
+          "type": "string"
         }
       },
       "required": [ "name", "subPath", "size" ],

helm/examples/azure/hpcc-azurefile/values.yaml

@@ -6,29 +6,29 @@ planes:
 - name: dali
   subPath: dalistorage
   size: 1Gi
-  labels: [ "dali" ]
+  category: dali
   #sku: "Standard_LRS"
 - name: dll
   subPath: queries # cannot currently be changed
   size: 1Gi
-  labels: [ "dll" ]
+  category: dll
   rwmany: true
   #sku: "Standard_LRS"
 - name: sasha
   subPath: sasha
   size: 1Gi
   rwmany: true
-  labels: [ "sasha" ]
+  category: sasha
   #sku: "Standard_LRS"
 - name: data
   subPath: hpcc-data # cannot currently be changed
   size: 3Gi
-  labels: [ "data" ] # NB: all "data" planes will be auto mounted by engine components and others that require access to data
+  category: data # NB: all "data" planes will be auto mounted by engine components and others that require access to data
   rwmany: true
   #sku: "Standard_LRS"
 - name: mydropzone
   subPath: dropzone
   size: 1Gi
   rwmany: true
-  labels: [ "lz" ]
+  category: lz
   #sku: "Standard_LRS"

helm/examples/azure/values-auto-azurefile.yaml

@@ -8,36 +8,31 @@ storage:
     storageSize: 1Gi
     storageClass: "azurefile"
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels:
-    - dali
+    category: dali
 
   - name: dll
     prefix: "/var/lib/HPCCSystems/queries"
     storageSize: 1Gi
     storageClass: "azurefile"
-    labels:
-    - dll
+    category: dll
 
   - name: sasha
     storageSize: 1Gi
     storageClass: "azurefile"
     prefix: "/var/lib/HPCCSystems/sashastorage"
-    labels:
-    - sasha
+    category: sasha
 
   - name: data
     storageSize: 1Gi
     storageClass: "azurefile"
     prefix: "/var/lib/HPCCSystems/hpcc-data"
-    labels:
-    - data # NB: if not set, this would be the default
+    category: data
 
   - name: mydropzone
     storageSize: 1Gi
     storageClass: "azurefile"
-    prefix: "/var/lib/HPCCSystems/dropzone"
-    labels:
-    - lz
+    prefix: "/var/lib/HPCCSystems/mydropzone"
+    category: lz
 
   daliStorage:
     plane: dali

helm/examples/azure/values-retained-azurefile.yaml

@@ -15,32 +15,27 @@ storage:
   - name: dali
     pvc: dali-azstorage-hpcc-azurefile-pvc
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels:
-    - dali
+    category: dali
 
   - name: dll
     pvc: dll-azstorage-hpcc-azurefile-pvc
     prefix: "/var/lib/HPCCSystems/queries"
-    labels:
-    - dll
+    category: dll
 
   - name: sasha
     pvc: sasha-azstorage-hpcc-azurefile-pvc
     prefix: "/var/lib/HPCCSystems/sasha"
-    labels:
-    - sasha
+    category: sasha
 
   - name: data
     pvc: data-azstorage-hpcc-azurefile-pvc
     prefix: "/var/lib/HPCCSystems/hpcc-data"
-    labels:
-    - data # NB: if not set, this would be the default
+    category: data
 
   - name: mydropzone
     pvc: mydropzone-azstorage-hpcc-azurefile-pvc
-    prefix: "/var/lib/HPCCSystems/dropzone"
-    labels:
-    - lz
+    prefix: "/var/lib/HPCCSystems/mydropzone"
+    category: lz
 
 
   daliStorage:

helm/examples/efs/hpcc-efs/values.schema.json

@@ -55,10 +55,9 @@
         "sku": {
           "type": "string"
         },
-        "labels": {
-          "description": "a list of labels associated with this plane, e.g. lz, data",
-          "type": "array",
-          "items": { "type": "string" }
+        "category": {
+          "description": "the category this plane is usd for, e.g. lz, data",
+          "type": "string"
         }
       },
       "required": [ "name", "subPath", "size" ],

helm/examples/efs/hpcc-efs/values.yaml

@@ -6,24 +6,24 @@ planes:
 - name: dali
   subPath: dalistorage
   size: 1Gi
-  labels: [ "dali" ]
+  category: dali
 - name: dll
   subPath: queries # cannot currently be changed
   size: 1Gi
-  labels: [ "dll" ]
+  category: dll
   rwmany: true
 - name: sasha
   subPath: sasha
   size: 1Gi
   rwmany: true
-  labels: [ "sasha" ]
+  category: sasha
 - name: data
   subPath: hpcc-data
   size: 3Gi
-  labels: [ "data" ] # NB: all "data" planes will be auto mounted by engine components and others that require access to data
+  category: data # NB: all "data" planes will be auto mounted by engine components and others that require access to data
   rwmany: true
 - name: mydropzone
   subPath: dropzone
   size: 1Gi
   rwmany: true
-  labels: [ "lz" ]
+  category: lz

helm/examples/efs/values-auto-efs.yaml

@@ -8,36 +8,31 @@ storage:
     storageSize: 1Gi
     storageClass: "aws-efs"
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels:
-    - dali
+    category: dali
 
   - name: dll
     prefix: "/var/lib/HPCCSystems/queries"
     storageSize: 1Gi
     storageClass: "aws-efs"
-    labels:
-    - dll
+    category: dll
 
   - name: sasha
     storageSize: 1Gi
     storageClass: "aws-efs"
     prefix: "/var/lib/HPCCSystems/sashastorage"
-    labels:
-    - sasha
+    category: sasha
 
   - name: data
     storageSize: 1Gi
     storageClass: "aws-efs"
     prefix: "/var/lib/HPCCSystems/hpcc-data"
-    labels:
-    - data # NB: if not set, this would be the default
+    category: data
 
   - name: mydropzone
     storageSize: 1Gi
     storageClass: "aws-efs"
-    prefix: "/var/lib/HPCCSystems/dropzone"
-    labels:
-    - lz
+    prefix: "/var/lib/HPCCSystems/mydropzone"
+    category: lz
 
   daliStorage:
     plane: dali

helm/examples/efs/values-retained-efs.yaml

@@ -10,32 +10,27 @@ storage:
   - name: dali
     pvc: dali-awsstorage-hpcc-efs-pvc
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels:
-    - dali
+    category: dali
 
   - name: dll
     pvc: dll-awsstorage-hpcc-efs-pvc
     prefix: "/var/lib/HPCCSystems/queries"
-    labels:
-    - dll
+    category: dll
 
   - name: sasha
     pvc: sasha-awsstorage-hpcc-efs-pvc
     prefix: "/var/lib/HPCCSystems/sasha"
-    labels:
-    - sasha
+    category: sasha
 
   - name: data
     pvc: data-awsstorage-hpcc-efs-pvc
     prefix: "/var/lib/HPCCSystems/hpcc-data"
-    labels:
-    - data # NB: if not set, this would be the default
+    category: data
 
   - name: mydropzone
     pvc: mydropzone-awsstorage-hpcc-efs-pvc
-    prefix: "/var/lib/HPCCSystems/dropzone"
-    labels:
-    - lz
+    prefix: "/var/lib/HPCCSystems/mydropzone"
+    category: lz
 
 
   dllStorage:

helm/examples/filestore/hpcc-filestore/values.schema.json

@@ -63,10 +63,9 @@
           "description": "Does the pvc require rw many access (data/dll currently)",
           "type": "boolean"
         },
-        "labels": {
-          "description": "a list of labels associated with this plane, e.g. lz, data",
-          "type": "array",
-          "items": { "type": "string" }
+        "category": {
+          "description": "the category this plane is usd for, e.g. lz, data",
+          "type": "string"
         }
       },
       "required": [ "name", "subPath", "size" ],

helm/examples/filestore/hpcc-filestore/values.yaml

@@ -13,19 +13,19 @@ planes:
 - name: dali
   subPath: dalistorage
   size: 1Gi
-  labels: [ "dali" ]
+  category: dali
 - name: dll
   subPath: queries # cannot currently be changed
   size: 1Gi
-  labels: [ "dll" ]
+  category: dll
   rwmany: true
 - name: sasha
   subPath: sasha
   size: 1Gi
   rwmany: true
-  labels: [ "sasha" ]
+  category: sasha
 - name: data
   subPath: hpcc-data
   size: 3Gi
-  labels: [ "data" ] # NB: all "data" planes will be auto mounted by engine components and others that require access to data
+  category: data # NB: all "data" planes will be auto mounted by engine components and others that require access to data
   rwmany: true

helm/examples/filestore/values-filestore.yaml

@@ -10,27 +10,22 @@ storage:
   - name: dali
     pvc: dali-gcpstorage-hpcc-filestore-pvc
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels:
-    - dali
+    category: dali
 
   - name: dll
     pvc: dll-gcpstorage-hpcc-filestore-pvc
     prefix: "/var/lib/HPCCSystems/queries"
-    labels:
-    - dll
+    category: dll
 
   - name: sasha
     pvc: sasha-gcpstorage-hpcc-filestore-pvc
     prefix: "/var/lib/HPCCSystems/sasha"
-    labels:
-    - sasha
+    category: sasha
 
   - name: data
     pvc: data-gcpstorage-hpcc-filestore-pvc
     prefix: "/var/lib/HPCCSystems/hpcc-data"
-    labels:
-    - data # NB: if not set, this would be the default
-
+    category: data # NB: if not set, this would be the default
 
 
   dllStorage:

helm/examples/local/hpcc-localfile/values.schema.json

@@ -52,10 +52,9 @@
           "description": "Does the pvc require rw many access (data/dll currently)",
           "type": "boolean"
         },
-        "labels": {
-          "description": "a list of labels associated with this plane, e.g. lz, data",
-          "type": "array",
-          "items": { "type": "string" }
+        "category": {
+          "description": "the category this plane is usd for, e.g. lz, data",
+          "type": "string"
         }
       },
       "required": [ "name", "subPath", "size" ],

helm/examples/local/hpcc-localfile/values.yaml

@@ -10,25 +10,25 @@ planes:
 - name: dali
   subPath: dalistorage
   size: 1Gi
-  labels: [ "dali" ]
+  category: dali
 - name: dll
   subPath: queries # cannot currently be changed
   size: 1Gi
-  labels: [ "dll" ]
+  category: dll
   rwmany: true
 - name: sasha
   subPath: sasha
   size: 1Gi
   rwmany: true
-  labels: [ "sasha" ]
+  category: sasha
 - name: data
   subPath: hpcc-data # cannot currently be changed
   size: 3Gi
-  labels: [ "data" ] # NB: all "data" planes will be auto mounted by engine components and others that require access to data
+  category: data # NB: all "data" planes will be auto mounted by engine components and others that require access to data
   rwmany: true
 - name: mydropzone
   subPath: dropzone
   size: 1Gi
   rwmany: true
-  labels: [ "lz" ]
+  category: lz
 

helm/examples/local/values-localfile.yaml

@@ -13,34 +13,29 @@
 storage:
   planes:
   - name: dali
-    pvc: dali-localfile-hpcc-localfile-pvc
+    pvc: dali-hpcc-localfile-pvc
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels:
-    - dali
+    category: dali
 
   - name: dll
-    pvc: dll-localfile-hpcc-localfile-pvc
+    pvc: dll-hpcc-localfile-pvc
     prefix: "/var/lib/HPCCSystems/queries"
-    labels:
-    - dll
+    category: dll
 
   - name: sasha
-    pvc: sasha-localfile-hpcc-localfile-pvc
+    pvc: sasha-hpcc-localfile-pvc
     prefix: "/var/lib/HPCCSystems/sasha"
-    labels:
-    - sasha
+    category: sasha
 
   - name: data
-    pvc: data-localfile-hpcc-localfile-pvc
+    pvc: data-hpcc-localfile-pvc
     prefix: "/var/lib/HPCCSystems/hpcc-data"
-    labels:
-    - data # NB: if not set, this would be the default
+    category: data
 
   - name: mydropzone
-    pvc: mydropzone-localfile-hpcc-localfile-pvc
-    prefix: "/var/lib/HPCCSystems/dropzone"
-    labels:
-    - lz
+    pvc: mydropzone-hpcc-localfile-pvc
+    prefix: "/var/lib/HPCCSystems/mydropzone"
+    category: lz
 
 
   daliStorage:

helm/examples/nfs/hpcc-nfs/values.schema.json

@@ -67,10 +67,9 @@
           "description": "Does the pvc require rw many access (data/dll currently)",
           "type": "boolean"
         },
-        "labels": {
-          "description": "a list of labels associated with this plane, e.g. lz, data",
-          "type": "array",
-          "items": { "type": "string" }
+        "category": {
+          "description": "the category this plane is usd for, e.g. lz, data",
+          "type": "string"
         }
       },
       "required": [ "name", "size" ],

helm/examples/nfs/hpcc-nfs/values.yaml

@@ -14,24 +14,24 @@ planes:
 - name: dali
   subPath: dalistorage
   size: 1Gi
-  labels: [ "dali" ]
+  category: dali
 - name: dll
   subPath: queries # cannot currently be changed
   size: 1Gi
-  labels: [ "dll" ]
+  category: dll
   rwmany: true
 - name: sasha
   subPath: sasha
   size: 1Gi
   rwmany: true
-  labels: [ "sasha" ]
+  category: sasha
 - name: data
   subPath: hpcc-data
   size: 3Gi
-  labels: [ "data" ] # NB: all "data" planes will be auto mounted by engine components and others that require access to data
+  category: data # NB: all "data" planes will be auto mounted by engine components and others that require access to data
   rwmany: true
 - name: mydropzone
   subPath: dropzone
   size: 1Gi
   rwmany: true
-  labels: [ "lz" ]
+  category: lz

helm/examples/nfs/values-nfs.yaml

@@ -10,26 +10,22 @@ storage:
   - name: dali
     pvc: dali-nfsstorage-hpcc-nfs-pvc
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels:
-    - dali
+    category: dali
 
   - name: dll
     pvc: dll-nfsstorage-hpcc-nfs-pvc
     prefix: "/var/lib/HPCCSystems/queries"
-    labels:
-    - dll
+    category: dll
 
   - name: sasha
     pvc: sasha-nfsstorage-hpcc-nfs-pvc
     prefix: "/var/lib/HPCCSystems/sasha"
-    labels:
-    - sasha
+    category: sasha
 
   - name: data
     pvc: data-nfsstorage-hpcc-nfs-pvc
     prefix: "/var/lib/HPCCSystems/hpcc-data"
-    labels:
-    - data # NB: if not set, this would be the default
+    category: data
 
 
   dllStorage:

helm/hpcc/Chart.yaml

@@ -6,9 +6,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 8.2.0-rc3
-
+version: 8.2.1-closedown
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 
-appVersion: 8.2.0-rc3
+appVersion: 8.2.1-closedown

helm/hpcc/docs/changes.md

@@ -132,6 +132,12 @@ define a storage plane.  That implicit plane could be based on a pre-existing pe
 existingClaim) or ephemeral storage (by setting storageClass and storageSize).  In version 8.2. these sections can
 only refer to a plane defined in the plane list.
 
+Another change is that previously a storage plane had a labels: attribute to indicate what kind of data was stored on
+the plane.  It was a list, and if blank defaulted to \[ data \].  This has now become a single valued "category" attribute.
+In the unlikely event of wanting to have multiple categories stored on the same mounted drive, it is possible to define
+a plane with a duplicate prefix and storage definition with a different category.  Planes also support the subPath
+attribute to allow a subdirectory to be used within a mount point.
+
 For instance if you have the following definition in 8.0.x:
 
 ```code
@@ -147,7 +153,7 @@ storage:
   - name: dali
     pvc: my-pvc
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels: [ "dali" ]
+    category: dali
 
   daliStorage:
      name: dali
@@ -170,7 +176,7 @@ storage:
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels: [ "dali" ]
+    category: dali
 
   daliStorage:
      name: dali

helm/hpcc/templates/_helpers.tpl

@@ -68,16 +68,15 @@ Translate a port list to a comma-separated list
  {{- end -}}
 {{- end -}}
 
-{{- define "hpcc.getFirstPlaneForLabel" -}}
+{{- define "hpcc.getFirstPlaneForCategory" -}}
 {{- $root := .root -}}
-{{- $label := .label -}}
+{{- $category := .category -}}
 {{- $storage := ($root.Values.storage | default dict) -}}
 {{- $planes := ($storage.planes | default list) -}}
 {{- $firstPlane := dict -}}
 {{- range $plane := $planes -}}
 {{- if not $firstPlane.plane -}}
-{{- $labels := $plane.labels | default (list "data") -}}
-{{- if (has $label $labels) -}}
+{{- if (eq $category $plane.category) -}}
 {{- $_ := set $firstPlane "plane" $plane.name -}}
 {{- end -}}
 {{- end -}}
@@ -87,8 +86,8 @@ Translate a port list to a comma-separated list
 {{- end -}}
 {{- end -}}
 
-{{- define "hpcc.hasPlaneForLabel" -}}
-{{- if (include "hpcc.getFirstPlaneForLabel" .) -}}
+{{- define "hpcc.hasPlaneForCategory" -}}
+{{- if (include "hpcc.getFirstPlaneForCategory" .) -}}
 true
 {{- end -}}
 {{- end -}}
@@ -99,7 +98,7 @@ Get default data plane
 {{- define "hpcc.getDefaultDataPlane" -}}
 {{- $storage := ($.Values.storage | default dict) -}}
 {{- $dataStorage := ($storage.dataStorage | default dict) -}}
-{{- $firstPlane := (include "hpcc.getFirstPlaneForLabel" (dict "root" $ "label" "data")) -}}
+{{- $firstPlane := (include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "data")) -}}
 {{- $dataStorage.plane | default $firstPlane | default "hpcc-data-plane" -}}
 {{- end -}}
 
@@ -109,7 +108,7 @@ Get default dll plane
 {{- define "hpcc.getDefaultDllPlane" -}}
 {{- $storage := ($.Values.storage | default dict) -}}
 {{- $dllStorage := ($storage.dllStorage | default dict) -}}
-{{- $firstPlane := (include "hpcc.getFirstPlaneForLabel" (dict "root" $ "label" "dll")) -}}
+{{- $firstPlane := (include "hpcc.getFirstPlaneForCategory" (dict "root" $ "category" "dll")) -}}
 {{- $dllStorage.plane | default $firstPlane | default "hpcc-dll-plane" -}}
 {{- end -}}
 
@@ -145,26 +144,25 @@ storage:
   planes:
 {{- /*Generate entries for each data plane (removing the pvc).  Exclude the planes used for dlls and dali.*/ -}}
 {{- range $plane := $planes -}}
- {{- if or (not $plane.labels) (or (has "data" $plane.labels) (has "lz" $plane.labels)) }}
+ {{- if or (eq "data" $plane.category) (eq "lz" $plane.category) }}
   - name: {{ $plane.name | quote }}
-  {{- if not $plane.labels }}
-    labels:
-    - data
-  {{- end }}
-{{ toYaml (omit $plane "name" "pvc" "storageClass" "storageSize") | indent 4 }}
+  {{- $planeYaml := omit $plane "name" "pvc" "storageClass" "storageSize" "subPath" -}}
+  {{- if $plane.subPath -}}
+   {{- $_ := set $planeYaml "prefix" (printf "%s/%s" $planeYaml.prefix $plane.subPath) -}}
+  {{- end -}}
+    {{- toYaml $planeYaml | nindent 4 }}
  {{- end }}
 {{- end }}
 {{- /* Add implicit planes if data or spill storage plane not specified*/ -}}
 {{- if not $dataStorage.plane }}
-{{- if not (include "hpcc.hasPlaneForLabel" (dict "root" $ "label" "data")) }}
+{{- if not (include "hpcc.hasPlaneForCategory" (dict "root" $ "category" "data")) }}
   - name: hpcc-data-plane
-    labels:
-    - data
+    category: data
     prefix: {{ .Values.global.defaultDataPath | default "/var/lib/HPCCSystems/hpcc-data" | quote }}
 {{- end }}
 {{- end }}
 {{- if not $spillStorage.plane }}
-{{- if not (include "hpcc.hasPlaneForLabel" (dict "root" $ "label" "spill")) }}
+{{- if not (include "hpcc.hasPlaneForCategory" (dict "root" $ "category" "spill")) }}
   - name: hpcc-spill-plane
     prefix: {{ .Values.global.defaultSpillPath | default "/var/lib/HPCCSystems/hpcc-spill" | quote }}
 {{- end }}
@@ -237,26 +235,8 @@ Add ConfigMap volume for a component
 {{- end -}}
 
 {{/*
-Returns a non empty string if any labels in the list includeLabels is in the plane.labels
-Note: the list includeLabels may contain an empty string (""), which matches planes that do not have a label
-Pass in plane and includeLabels
-Return: If there is any matching labels, there will be a non-empty string returned.  If there is no matching labels,
-        an empty string will be returned.
-*/}}
-{{- define "hpcc.doesStorageLabelsMatch" -}}
-{{- $plane := .plane -}}
-  {{- range $label := .includeLabels -}}
-    {{- if and (eq $label "data") (not $plane.labels) -}}
-      {{- print "T" -}}
-    {{- else if has $label $plane.labels -}}
-      {{- print "T" -}}
-    {{- end -}}
-  {{- end -}}
-{{- end -}}
-
-{{/*
 Add volume mounts
-Pass in root and includeLabels (optional)
+Pass in root and includeCategories (optional)
 Note: if there are multiple planes (other than dll, dali and spill planes), they should be all called with a single call
 to addVolumeMounts so that if a plane can be used for multiple purposes then duplicate volume mounts are not created.
 */}}
@@ -264,22 +244,25 @@ to addVolumeMounts so that if a plane can be used for multiple purposes then dup
 {{- /*Create local variables which always exist to avoid having to check if intermediate key values exist*/ -}}
 {{- $storage := (.root.Values.storage | default dict) -}}
 {{- $planes := ($storage.planes | default list) -}}
-{{- $includeLabels := .includeLabels | default list -}}
+{{- $includeCategories := .includeCategories | default list -}}
+{{- $previousMounts := dict -}}
 {{- range $plane := $planes -}}
  {{- if or ($plane.pvc) (hasKey $plane "storageClass") -}}
-  {{- $mountpath := $plane.prefix -}}
-  {{- $matchedLabels := include "hpcc.doesStorageLabelsMatch" (dict "plane" $plane "includeLabels" $includeLabels) }}
-  {{ if ne $matchedLabels "" }}
-   {{- $num := int ( $plane.numDevices | default 1 ) -}}
-   {{- if le $num 1 }}
+  {{- if not (hasKey $previousMounts $plane.prefix) -}}
+   {{- $mountpath := $plane.prefix -}}
+   {{- if has $plane.category $includeCategories }}
+    {{- $num := int ( $plane.numDevices | default 1 ) -}}
+    {{- if le $num 1 }}
 - name: {{ lower $plane.name }}-pv
   mountPath: {{ $mountpath | quote }}
-   {{- else }}
-    {{- range $elem := untilStep 1 (int (add $num 1)) 1 }}
+    {{- else }}
+     {{- range $elem := untilStep 1 (int (add $num 1)) 1 }}
 - name: {{ lower $plane.name }}-pv-many-{{- $elem }}
   mountPath: {{ printf "%s/d%d" $mountpath $elem | quote }}
+     {{- end }}
     {{- end }}
    {{- end }}
+   {{- $_ := set $previousMounts $plane.prefix true -}}
   {{- end }}
  {{- end }}
 {{- end }}
@@ -289,8 +272,8 @@ Note: Some services used addVolumeMounts to add data planes and other types of p
 to be located here rather than in addDataVolumeMount.
 */ -}}
 {{- $dataStorage := ($storage.dataStorage | default dict) -}}
-{{- if and (has "data" $includeLabels) (not $dataStorage.plane) }}
-{{- if (not (include "hpcc.hasPlaneForLabel" (dict "root" .root "label" "data"))) }}
+{{- if and (has "data" $includeCategories) (not $dataStorage.plane) }}
+{{- if (not (include "hpcc.hasPlaneForCategory" (dict "root" .root "category" "data"))) }}
 - name: datastorage
   mountPath: "/var/lib/HPCCSystems/hpcc-data"
 {{- end }}
@@ -302,12 +285,12 @@ Add data volume mount
 Pass in root
 */}}
 {{- define "hpcc.addDataVolumeMount" -}}
-{{- include "hpcc.addVolumeMounts" (dict "root" .root "includeLabels" (list "data" "lz")) -}}
+{{- include "hpcc.addVolumeMounts" (dict "root" .root "includeCategories" (list "data" "lz")) -}}
 {{- end -}}
 
 {{/*
 Add volumes
-Pass in root, includeLabels (optional) and includeNames (optional)
+Pass in root, includeCategories (optional) and includeNames (optional)
 The plane will generate a volume if it matches either an includeLabel or an includeName
 */}}
 {{- define "hpcc.addVolumes" -}}
@@ -315,25 +298,29 @@ The plane will generate a volume if it matches either an includeLabel or an incl
 {{- $storage := (.root.Values.storage | default dict) -}}
 {{- $dataStorage := ($storage.dataStorage | default dict) -}}
 {{- $planes := ($storage.planes | default list) -}}
-{{- $includeLabels := .includeLabels | default list -}}
+{{- $includeCategories := .includeCategories | default list -}}
 {{- $includeNames := .includeNames | default list -}}
+{{- $previousMounts := dict -}}
 {{- range $plane := $planes -}}
  {{- if or ($plane.pvc) (hasKey $plane "storageClass") -}}
-  {{- $matchedLabels := include "hpcc.doesStorageLabelsMatch" (dict "plane" $plane "includeLabels" $includeLabels) -}}
-  {{- if or ($matchedLabels) (has $plane.name $includeNames) }}
-   {{- $pvc := hasKey $plane "pvc" | ternary $plane.pvc (printf "%s-%s-pvc" (include "hpcc.fullname" $) $plane.name) -}}
-   {{- $num := int ( $plane.numDevices | default 1 ) -}}
-   {{- if le $num 1 }}
+  {{- if not (hasKey $previousMounts $plane.prefix) -}}
+   {{- $mountpath := $plane.prefix -}}
+   {{- if or (has $plane.category $includeCategories) (has $plane.name $includeNames) }}
+    {{- $pvc := hasKey $plane "pvc" | ternary $plane.pvc (printf "%s-%s-pvc" (include "hpcc.fullname" $) $plane.name) -}}
+    {{- $num := int ( $plane.numDevices | default 1 ) -}}
+    {{- if le $num 1 }}
 - name: {{ lower $plane.name }}-pv
   persistentVolumeClaim:
     claimName: {{ $pvc }}
-   {{- else }}
-    {{- range $elem := until $num }}
+    {{- else }}
+     {{- range $elem := until $num }}
 - name: {{ lower $plane.name }}-pv-many-{{- add $elem 1 }}
   persistentVolumeClaim:
     claimName: {{ $pvc }}-{{- add $elem 1 }}
-    {{- end }}
-   {{- end -}}
+     {{- end }}
+    {{- end -}}
+   {{- end }}
+   {{- $_ := set $previousMounts $plane.prefix true -}}
   {{- end }}
  {{- end }}
 {{- end -}}
@@ -344,7 +331,7 @@ Add data volume
 Pass in dict with root
 */}}
 {{- define "hpcc.addDataVolume" -}}
-{{- include "hpcc.addVolumes" (dict "root" .root "includeLabels" (list "data" "" "lz") ) -}}
+{{- include "hpcc.addVolumes" (dict "root" .root "includeCategories" (list "data" "lz") ) -}}
 {{- end -}}
 
 {{/*
@@ -384,7 +371,7 @@ Add dll volume mount - if default plane is used, or the dll storage plane specif
 Pass in dict with root
 */}}
 {{- define "hpcc.addDllVolumeMount" -}}
-{{- include "hpcc.addVolumeMounts" (dict "root" .root "includeLabels" (list "dll")) -}}
+{{- include "hpcc.addVolumeMounts" (dict "root" .root "includeCategories" (list "dll")) -}}
 {{- end -}}
 
 {{/*
@@ -392,7 +379,7 @@ Add dali volume mount - if default plane is used, or the dali storage plane spec
 Pass in dict with root
 */}}
 {{- define "hpcc.addDaliVolumeMount" -}}
-{{- include "hpcc.addVolumeMounts" (dict "root" .root "includeLabels" (list "dali")) -}}
+{{- include "hpcc.addVolumeMounts" (dict "root" .root "includeCategories" (list "dali")) -}}
 {{- end -}}
 
 {{/*
@@ -400,7 +387,7 @@ Add dll volume - if default plane is used, or the dll storage plane specifies a
 Pass in dict with root
 */}}
 {{- define "hpcc.addDllVolume" -}}
-{{- include "hpcc.addVolumes" (dict "root" .root "includeLabels" (list "dll") ) }}
+{{- include "hpcc.addVolumes" (dict "root" .root "includeCategories" (list "dll") ) }}
 {{- end -}}
 
 {{/*
@@ -408,7 +395,7 @@ Add dali volume - if default plane is used, or the dali storage plane specifies
 Pass in dict with root
 */}}
 {{- define "hpcc.addDaliVolume" -}}
-{{- include "hpcc.addVolumes" (dict "root" .root "includeLabels" (list "dali") ) }}
+{{- include "hpcc.addVolumes" (dict "root" .root "includeCategories" (list "dali") ) }}
 {{- end -}}
 
 {{/*
@@ -593,13 +580,12 @@ A kludge to ensure mounted storage (e.g. for nfs, minikube or docker for desktop
 {{- define "hpcc.changePlaneMountPerms" -}}
 {{- $storage := (.root.Values.storage | default dict) -}}
 {{- $planes := ($storage.planes | default list) -}}
-{{- $includeLabels := .includeLabels | default list -}}
+{{- $includeCategories := .includeCategories | default list -}}
 {{- $includeNames := .includeNames | default list -}}
 {{- range $plane := $planes -}}
  {{- if and ($plane.forcePermissions) (or ($plane.pvc) (hasKey $plane "storageClass")) -}}
   {{- $mountpath := $plane.prefix -}}
-  {{- $matchedLabels := include "hpcc.doesStorageLabelsMatch" (dict "plane" $plane "includeLabels" $includeLabels) }}
-  {{- if or ($matchedLabels) (has $plane.name $includeNames) }}
+  {{- if or (has $plane.category $includeCategories) (has $plane.name $includeNames) }}
 {{- $volumeName := (printf "%s-pv" $plane.name) -}}
 {{ include "hpcc.changeMountPerms" (dict "root" .root "volumeName" $volumeName "volumePath" $plane.prefix) }}
 {{- end }}
@@ -657,7 +643,7 @@ Add wait-and-run shared inter container volume
 Check dll mount point, using hpcc.changeMountPerms
 */}}
 {{- define "hpcc.checkDllMount" -}}
-{{ include "hpcc.changePlaneMountPerms" (dict "root" .root "includeLabels" (list "dll")) }}
+{{ include "hpcc.changePlaneMountPerms" (dict "root" .root "includeCategories" (list "dll")) }}
 {{- end }}
 
 {{/*
@@ -665,14 +651,14 @@ Check datastorage mount point, using hpcc.changeMountPerms
 Pass in a dictionary with root
 */}}
 {{- define "hpcc.checkDataMount" -}}
-{{ include "hpcc.changePlaneMountPerms" (dict "root" .root "includeLabels" (list "" "data" "lz")) }}
+{{ include "hpcc.changePlaneMountPerms" (dict "root" .root "includeCategories" (list "data" "lz")) }}
 {{- end }}
 
 {{/*
 Check dalistorage mount point, using hpcc.changeMountPerms
 */}}
 {{- define "hpcc.checkDaliMount" -}}
-{{ include "hpcc.changePlaneMountPerms" (dict "root" .root "includeLabels" (list "dali")) }}
+{{ include "hpcc.changePlaneMountPerms" (dict "root" .root "includeCategories" (list "dali")) }}
 {{- end }}
 
 
@@ -1063,17 +1049,21 @@ spec:
 {{- end -}}
 
 {{/*
-A template to generate PVCs for each storage plane that has storageSize defined and has the appropriate label
-Pass in dict with root, label.  optional name to restrict it to a single name.
+A template to generate PVCs for each storage plane that has storageSize defined and has the appropriate category
+Pass in dict with root, category.  optional name to restrict it to a single name.
 */}}
 {{- define "hpcc.addPVCsFromPlanes" }}
 {{- $storage := (.Values.storage | default dict) }}
 {{- $planes := ($storage.planes | default list) -}}
+{{- $previousMounts := dict -}}
 {{- range $plane := $planes -}}
-{{- if (hasKey $plane "storageClass") }}
-{{- $pvcname := (printf "%s-pvc" $plane.name) -}}
-{{- include "hpcc.addPVC" (dict "root" $ "name" $pvcname "me" $plane) }}
-{{- end }}
+ {{- if (hasKey $plane "storageClass") -}}
+  {{- if not (hasKey $previousMounts $plane.prefix) -}}
+   {{- $pvcname := (printf "%s-pvc" $plane.name) -}}
+   {{- include "hpcc.addPVC" (dict "root" $ "name" $pvcname "me" $plane) }}
+   {{- $_ := set $previousMounts $plane.prefix true -}}
+ {{- end }}
+ {{- end }}
 {{- end }}
 {{- end -}}
 

helm/hpcc/templates/dali.yaml

@@ -66,7 +66,7 @@ spec:
       {{- end }}
       {{- end }}
       initContainers: 
-        {{- include "hpcc.changePlaneMountPerms" (dict "root" $ "includeLabels" (list "dali") "includeNames" (uniq $sashaPlanes)) | indent 8 }}
+        {{- include "hpcc.changePlaneMountPerms" (dict "root" $ "includeCategories" (list "dali") "includeNames" (uniq $sashaPlanes)) | indent 8 }}
       containers:
       - name: {{ $dali.name | quote }}
         workingDir: /var/lib/HPCCSystems

helm/hpcc/templates/dfuserver.yaml

@@ -39,7 +39,7 @@ data:
 {{- if not .disabled -}}
 {{- $env := concat ($.Values.global.env | default list) (.env | default list) -}}
 {{- $secretsCategories := list "system" "storage" -}}
-{{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories "includeLabels" (list "lz" "data") "env" $env }}
+{{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories "includeCategories" (list "lz" "data") "env" $env }}
 {{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.dfuServerConfigMap" "component" "dfuserver" "excludeKeys" "global")) -}}
 apiVersion: apps/v1
 kind: Deployment

helm/hpcc/templates/esp.yaml

@@ -72,8 +72,8 @@ data:
 {{- $env := concat ($.Values.global.env | default list) (.env | default list) -}}
 {{- $application := .application | default "eclwatch" -}}
 {{- $secretsCategories := ternary (list "storage" "esp" "codeSign" "codeVerify")  (list "storage" "esp") (eq $application "eclwatch") -}}
-{{- $includeStorageLabels := ternary (list "lz" "data" "") (list "data" "") (eq $application "eclwatch") -}}
-{{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories  "includeLabels" $includeStorageLabels "env" $env -}}
+{{- $includeStorageCategories := ternary (list "lz" "data") (list "data") (eq $application "eclwatch") -}}
+{{- $commonCtx := dict "root" $ "me" . "secretsCategories" $secretsCategories  "includeCategories" $includeStorageCategories "env" $env -}}
 {{- $configSHA := include "hpcc.getConfigSHA" ($commonCtx | merge (dict "configMapHelper" "hpcc.espConfigMap" "component" "esp" "excludeKeys" "global,esp.queues")) }}
 
 apiVersion: apps/v1

helm/hpcc/templates/storage.yaml

@@ -21,4 +21,26 @@
 ##############################################################################
 
 */}}
+{{- /* check planes are unique, have valid names, and do not define the mount in multiple ways */ -}}
+{{- $storage := (.Values.storage | default dict) }}
+{{- $planes := ($storage.planes | default list) -}}
+{{- $previousPlanes := dict -}}
+{{- $previousMounts := dict -}}
+{{- range $plane := $planes -}}
+ {{- if (ne (lower $plane.name) $plane.name) -}}
+  {{- required (printf "Name of storage plane '%s' must not contain upper case" $plane.name) nil -}}
+ {{- end -}}
+ {{- if hasKey $previousPlanes $plane.name -}}
+  {{- required (printf "Storage plane '%s' is defined more than once" $plane.name) nil -}}
+ {{- end -}}
+ {{- $_ := set $previousPlanes $plane.name true -}}
+ {{- $rawPlane := omit $plane "name" "category" "subPath" -}}
+ {{- if hasKey $previousMounts $plane.prefix -}}
+  {{- /* Should this restrict if has a pvc or a storage class? */ -}}
+  {{- if not (deepEqual (get $previousMounts $plane.prefix) $rawPlane) -}}
+    {{- required (printf "Multiple incompatible planes refer to prefix '%s'" $plane.prefix) nil -}}
+  {{- end -}}
+ {{- end -}}
+ {{- $_ := set $previousMounts $plane.prefix $rawPlane -}}
+{{- end -}}
 {{- include "hpcc.addPVCsFromPlanes" . }}

helm/hpcc/values.schema.json

@@ -391,15 +391,8 @@
     },
     "storagePlanes": {
       "description": "storage plane definitions",
-      "oneOf": [
-        {
-          "type": "array",
-          "items": { "$ref": "#/definitions/storagePlane" }
-        },
-        {
-          "type": "null"
-        }
-      ]
+      "type": "array",
+      "items": { "$ref": "#/definitions/storagePlane" }
     },
     "storagePlane": {
       "description": "information about an individual storage plane",
@@ -413,6 +406,10 @@
           "description": "either the path for a local mount, or the url prefix",
           "type": "string"
         },
+        "subPath": {
+          "description": "optional subdirectory within the mount directory",
+          "type": "string"
+        },
         "secret": {
           "description": "optional name of any secret required to access this storage plane",
           "type": "string"
@@ -443,10 +440,10 @@
           "type": "array",
           "items": { "type": "string" }
         },
-        "labels": {
-          "description": "a list of labels associated with this plane, e.g. lz, data, dali, sasha, dlls, spill. NB: if blank/unset, treated as if 'data'",
-          "type": "array",
-          "items": { "type": "string" }
+        "category": {
+          "description": "the category this plane is usd for, e.g. lz, data",
+          "type": "string",
+          "enum": ["data", "lz", "dali", "sasha", "dll", "spill" ]
         },
         "umask" : {
           "description": "file creation mask (used by despray)",
@@ -480,7 +477,7 @@
         }
 
       },
-      "required": [ "name", "prefix" ],
+      "required": [ "name", "prefix", "category" ],
       "additionalProperties": false
     },
     "resources": {

helm/hpcc/values.yaml

@@ -117,6 +117,7 @@ storage:
   planes:
   #   name: <required>
   #   prefix: <path>                        # Root directory for accessing the plane (if pvc defined), or url to access plane.
+  #   subPath: <relative-path>              # Optional sub directory within <prefix> to use as the root directory
   #   pvc: <name>                           # The name of the persistant volume claim
   #   numDevices: 1                         # number of devices that are part of the plane
   #   replication: nullptr                  # a list or single item indicating which planes the data should be replicated onto
@@ -139,27 +140,27 @@ storage:
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/dalistorage"
-    labels: [ "dali" ]
+    category: dali
   - name: sasha
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/sashastorage"
-    labels: [ "sasha" ]
+    category: sasha
   - name: dll
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/queries"
-    labels: [ "dll" ]
+    category: dll
   - name: data
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/hpcc-data"
-    labels: [ "data" ]
+    category: data
   - name: mydropzone
     storageClass: ""
     storageSize: 1Gi
-    prefix: "/var/lib/HPCCSystems/dropzone"
-    labels: [ "lz" ]
+    prefix: "/var/lib/HPCCSystems/mydropzone"
+    category: lz
 
   dllStorage:
     plane: dll

helm/storage.rst

@@ -48,7 +48,7 @@ planes:
 
   numDevices: The number of logical storage devices.  An advanced option.  This allows data to be striped over multiple storage locations.  If it is set, then it assumes pvc defines a set of pvcs: ``pvc-1`` .. ``pvc-<n>``, which are mounted at locations ``prefix/d1`` .. ``prefix/d<n>``.  (Not yet implemented and likely to change!)
 
-  labels:  A list of uses which the plane is used for.  On a simple system it may make sense to place dali, dlls and sasha all on the same volume.
+  category:  The category of data which is stored on the plane - one of (data, lz, dali, sasha, dll, spill).  On a simple system it may make sense to place dali, dlls and sasha all on the same volume - in which case multiple planes can be defined with exactly the same definition, but different categories (and subPaths).
 
 
 The planes are used by refering to them in the plane tag in the appropriate storage section.  E.g. ``--set storage.daliStorage.plane=dali-plane``.

roxie/ccd/ccdmain.cpp

@@ -1243,6 +1243,9 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
             queryFileCache().start();
             loadPlugins();
         }
+#ifdef _CONTAINERIZED
+        initializeTopology(topoValues, myRoles);
+#endif
         createDelayedReleaser();
         globalPackageSetManager = createRoxiePackageSetManager(standAloneDll.getClear());
         globalPackageSetManager->load();
@@ -1411,7 +1414,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
                 queryFileCache().loadSavedOsCacheInfo();
                 queryFileCache().startCacheReporter();
 #ifdef _CONTAINERIZED
-                publishTopology(traceLevel, topoValues, myRoles);
+                publishTopology(traceLevel, myRoles);
 #endif
                 writeSentinelFile(sentinelFile);
                 DBGLOG("Startup completed - LPT=%u APT=%u", queryNumLocalTrees(), queryNumAtomTrees());

roxie/topo/toposerver.cpp

@@ -253,7 +253,8 @@ void doServer(ISocket *socket)
                         if (end != line.npos)
                         {
                             char *tail = nullptr;
-                            instance = strtoul(line.substr(end+1).c_str(), &tail, 10);
+                            std::string instanceStr = line.substr(end+1);
+                            instance = strtoul(instanceStr.c_str(), &tail, 10);
                             if (*tail)
                                 DBGLOG("Unexpected characters parsing instance value in topology entry %s", line.c_str());
                             line = line.substr(0, end);

roxie/udplib/udptopo.cpp

@@ -485,10 +485,14 @@ static std::thread topoThread;
 static Semaphore abortTopo;
 const unsigned topoUpdateInterval = 5000;
 
-extern UDPLIB_API void publishTopology(unsigned traceLevel, const StringArray &topoValues, const std::vector<RoxieEndpointInfo> &myRoles)
+extern UDPLIB_API void initializeTopology(const StringArray &topoValues, const std::vector<RoxieEndpointInfo> &myRoles)
 {
     topologyManager.setServers(topoValues);
     topologyManager.setRoles(myRoles);
+}
+
+extern UDPLIB_API void publishTopology(unsigned traceLevel, const std::vector<RoxieEndpointInfo> &myRoles)
+{
     if (topologyManager.numServers())
     {
         topoThread = std::thread([traceLevel, &myRoles]()

roxie/udplib/udptopo.hpp

@@ -125,7 +125,8 @@ struct RoxieEndpointInfo
     unsigned replicationLevel;
 };
 
-extern UDPLIB_API void publishTopology(unsigned traceLevel, const StringArray &topoValues, const std::vector<RoxieEndpointInfo> &myRoles);
+extern UDPLIB_API void initializeTopology(const StringArray &topoValues, const std::vector<RoxieEndpointInfo> &myRoles);
+extern UDPLIB_API void publishTopology(unsigned traceLevel, const std::vector<RoxieEndpointInfo> &myRoles);
 extern UDPLIB_API void stopTopoThread();
 
 #ifndef _CONTAINERIZED

system/jlib/jfile.cpp

@@ -7350,3 +7350,26 @@ IFileEventWatcher *createFileEventWatcher(FileWatchFunc callback)
 #endif // __linux__
 
 
+//---- Storage plane related functions ----------------------------------------------------
+
+IPropertyTree * getHostGroup(const char * name, bool required)
+{
+    if (!isEmptyString(name))
+    {
+        VStringBuffer xpath("storage/hostGroups[@name='%s']", name);
+        Owned<IPropertyTree> global = getGlobalConfig();
+        IPropertyTree * match = global->getPropTree(xpath);
+        if (match)
+            return match;
+    }
+    if (required)
+        throw makeStringExceptionV(-1, "No entry found for hostGroup: '%s'", name ? name : "<null>");
+    return nullptr;
+}
+
+IPropertyTree * getStoragePlane(const char * name)
+{
+    VStringBuffer xpath("storage/planes[@name='%s']", name);
+    Owned<IPropertyTree> global = getGlobalConfig();
+    return global->getPropTree(xpath);
+}

system/jlib/jfile.hpp

@@ -724,4 +724,9 @@ interface IFileEventWatcher : extends IInterface
 typedef std::function<void (const char *, FileWatchEvents)> FileWatchFunc;
 jlib_decl IFileEventWatcher *createFileEventWatcher(FileWatchFunc callback);
 
+//---- Storage plane related functions ----------------------------------------------------
+
+extern jlib_decl IPropertyTree * getHostGroup(const char * name, bool required);
+extern jlib_decl IPropertyTree * getStoragePlane(const char * name);
+
 #endif

system/jlib/jutil.cpp

@@ -2624,12 +2624,9 @@ jlib_decl bool queryMtlsBareMetalConfig()
 }
 #endif
 
+#ifndef _CONTAINERIZED
 static IPropertyTree *getOSSdirTree()
 {
-#ifdef _CONTAINERIZED
-    IERRLOG("getOSSdirTree() called from container system");
-    return nullptr;
-#endif
     Owned<IPropertyTree> envtree = getHPCCEnvironment();
     if (envtree) {
         IPropertyTree *ret = envtree->queryPropTree("Software/Directories");
@@ -2638,6 +2635,8 @@ static IPropertyTree *getOSSdirTree()
     }
     return NULL;
 }
+#endif
+
 
 StringBuffer &getFileAccessUrl(StringBuffer &out)
 {
@@ -2663,8 +2662,79 @@ StringBuffer &getFileAccessUrl(StringBuffer &out)
     return out;
 }
 
+
+#ifdef _CONTAINERIZED
+static bool getDefaultPlane(StringBuffer &ret, const char * componentOption, const char * globalOption)
+{
+    // If the plane is specified for the component, then use that
+    if (getComponentConfigSP()->getProp(componentOption, ret))
+        return true;
+
+    //Otherwise check what the default plane for data storage is configured to be
+    if (getGlobalConfigSP()->getProp(globalOption, ret))
+        return true;
+
+    return false;
+}
+
+static bool getDefaultPlaneDirectory(StringBuffer &ret, const char * componentOption, const char * globalOption)
+{
+    StringBuffer planeName;
+    if (!getDefaultPlane(planeName, componentOption, globalOption))
+        return false;
+
+    Owned<IPropertyTree> storagePlane = getStoragePlane(planeName);
+    return storagePlane->getProp("@prefix", ret);
+}
+#endif
+
 bool getConfigurationDirectory(const IPropertyTree *useTree, const char *category, const char *component, const char *instance, StringBuffer &dirout)
 {
+#ifdef _CONTAINERIZED
+    if (streq(category, "data"))
+    {
+        Owned<IPropertyTree> storagePlane = getStoragePlane(instance);
+        if (!storagePlane)
+            throw makeStringExceptionV(-1, "no default directory available for plane '%s'", instance);
+        return storagePlane->getProp("@prefix", dirout);
+    }
+    if (streq(category, "data2") || streq(category, "data3") || streq(category, "data4") || streq(category, "mirror"))
+        return false;
+    if (streq(category, "spill"))
+    {
+        return getDefaultPlaneDirectory(dirout, "@spillPlane", "storage/@spillPlane");
+    }
+    if (streq(category, "temp"))
+    {
+        if (getDefaultPlaneDirectory(dirout, "@tempPlane", "storage/@tempPlane"))
+            return true;
+        return getDefaultPlaneDirectory(dirout, "@spillPlane", "storage/@spillPlane");
+    }
+    if (streq(category, "log"))
+    {
+        return false;
+    }
+    if (streq(category, "dali"))
+    {
+        return getDefaultPlaneDirectory(dirout, "@daliPlane", "storage/@daliPlane");
+    }
+    if (streq(category, "query"))
+    {
+        return getDefaultPlaneDirectory(dirout, "@dllPlane", "storage/@dllPlane");
+    }
+    if (streq(category, "lock"))
+    {
+        //Called by NamedMutex.  Currently unused in the containerized system.
+        dirout.append("/var/lib/HPCCSystems/lock");
+        return true;
+    }
+    if (streq(category, "key") || streq(category, "run"))
+    {
+        throw makeStringExceptionV(-1, "Unexpected category '%s' requested in containerized mode", category);
+    }
+
+    throw makeStringExceptionV(-1, "Unrecognised configuration category %s", category);
+#else
     Linked<const IPropertyTree> dirtree = useTree;
     if (!dirtree)
         dirtree.setown(getOSSdirTree());
@@ -2726,6 +2796,7 @@ bool getConfigurationDirectory(const IPropertyTree *useTree, const char *categor
         }
     }
     return false;
+#endif
 }
 
 

testing/helm/errtests/baremetalerr.yaml

@@ -21,22 +21,28 @@ storage:
   - name: demoOne
     prefix: /home/gavin/temp
     hostGroup: demoOne
+    category: data
   - name: myDropZone
     prefix: /home/gavin/temp
     hosts: [ '192.168.86.202']
-    labels: ['lz']
+    category: 'lz'
   - name: demoTwo
     prefix: /home/gavin/temp/demoTwo
     hostGroup: demoTwo
+    category: data
   - name: demoTwoReplica
     prefix: /home/gavin/temp/demoTwo
     hostGroup: demoThree
+    category: data
   - name: demoTwoA
     prefix: /home/gavin/temp/demoTwoA
     hostGroup: demoTwoA
+    category: data
   - name: demoTwoB
     prefix: /home/gavin/temp/demoTwoB
     hostGroup: demoTwoB
+    category: data
   - name: myInlinePlane
     prefix: /home/gavin/temp
-    hostGroup: [ 'here', 'there', 'everywhere']     # error - list is not allowed here, use hosts
+    hostGroup: [ 'here', 'there', 'everywhere']     # error - list is not allowed here, use hosts
+    category: data

testing/helm/errtests/dupmount.yaml

@@ -0,0 +1,27 @@
+storage:
+  planes:
+  - name: dali
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/dalistorage"
+    category: dali
+  - name: sasha
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/sashastorage"
+    category: sasha
+  - name: dll
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: dll
+  - name: data
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/hpcc-data"
+    category: data
+  - name: mydropzone
+    storageClass: ""
+    storageSize: 3Gi
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: lz

testing/helm/errtests/dupplane.yaml

@@ -0,0 +1,27 @@
+storage:
+  planes:
+  - name: dali
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/dalistorage"
+    category: dali
+  - name: sasha
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/sashastorage"
+    category: sasha
+  - name: dll
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: dll
+  - name: data
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/hpcc-data"
+    category: data
+  - name: data
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/dropzone"
+    category: lz

testing/helm/errtests/nocategory.yaml

@@ -0,0 +1,21 @@
+storage:
+  planes:
+  - name: sasha
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/sashastorage"
+  - name: dll
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: "dll"
+  - name: data
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/hpcc-data"
+    category: "data"
+  - name: mydropzone
+    storageClass: ""
+    storageSize: 3Gi
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: "lz"

testing/helm/errtests/noplanes.yaml

@@ -0,0 +1,2 @@
+storage:
+  planes: null

testing/helm/errtests/unknowncategory.yaml

@@ -0,0 +1,22 @@
+storage:
+  planes:
+  - name: sasha
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/sashastorage"
+    category: unknown
+  - name: dll
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: dll
+  - name: data
+    storageClass: ""
+    storageSize: 1Gi
+    prefix: "/var/lib/HPCCSystems/hpcc-data"
+    category: data
+  - name: mydropzone
+    storageClass: ""
+    storageSize: 3Gi
+    prefix: "/var/lib/HPCCSystems/queries"
+    category: lz

testing/helm/tests/baremetal.yaml

@@ -21,40 +21,40 @@ storage:
   planes:
   #Bare metal system with attached storage
   - name: thor400
-    labels: [ data ]
+    category: data
     prefix: /var/lib/hpccsystems/hpcc-data       # only used if the local host matches the host for the device
     hostGroup: thor400
     replication: [ attachedThor400MirrorPlane ]
     #numDevices: count(hosts)
   - name: thor400mirror
-    labels: [ data ]
+    category: data
     prefix: /var/lib/hpccsystems/hpcc-mirror       # only used if the local host matches the host for the device
     hostGroup: thor400mirror
     #Does any other information about the replication policy need to be included?  I don't think it does....
 
   - name: thor100_4
-    labels: [ data ]
+    category: data
     prefix: /var/lib/hpccsystems/hpcc-data       # only used if the local host matches the host for the device
     hostGroup: thor100_4
     replication: [ azureBlobPlane ]
 
   - name: azureBlobPlane
-    labels: [ data ]
+    category: data
     prefix: azure://ghallidaystorage      # Not sure if this should be different from the mount.
     secret: azure-ghallidaystorage
 
   - name: dali
-    labels: [ dali ]
+    category: dali
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/dalistorage"
   - name: dll
-    labels: [ dll ]
+    category: dll
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/queries"
   - name: sasha
-    labels: [ sasha ]
+    category: sasha
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/sashastorage"

testing/helm/tests/baremetal2.yaml

@@ -19,40 +19,46 @@ storage:
 
   planes:
   - name: demoOne
+    category: data
     prefix: /home/gavin/temp
     hostGroup: demoOne
   - name: myDropZone
     prefix: /home/gavin/temp
     hosts: [ '192.168.86.202']
-    labels: ['lz']
+    category: 'lz'
   - name: demoTwo
     prefix: /home/gavin/temp/demoTwo
     hostGroup: demoTwo
+    category: data
   - name: demoTwoReplica
     prefix: /home/gavin/temp/demoTwo
     hostGroup: demoThree
+    category: data
   - name: demoTwoA
     prefix: /home/gavin/temp/demoTwoA
     hostGroup: demoTwoA
+    category: data
   - name: demoTwoB
     prefix: /home/gavin/temp/demoTwoB
     hostGroup: demoTwoB
+    category: data
   - name: myInlinePlane
     prefix: /home/gavin/temp
     hosts: [ 'here', 'there', 'everywhere']
+    category: data
 
   - name: dali
-    labels: [ dali ]
+    category: dali
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/dalistorage"
   - name: dll
-    labels: [ dll ]
+    category: dll
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/queries"
   - name: sasha
-    labels: [ sasha ]
+    category: sasha
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/sashastorage"

testing/helm/tests/complex.yaml

@@ -5,22 +5,27 @@ storage:
   - name: azureBlobPlane
     prefix: azure://ghallidaystorage      # Not sure if this should be different from the mount.
     secret: azure-ghallidaystorage
+    category: data
 
   #Store data on aws s3 buckets
   - name: s3BucketPlane
     prefix: s3://...
+    category: data
 
   #Single node with data mounted, and mirror mounted at a different locations (could be a different disk)
   - name: localDataPlane
     prefix: /var/lib/hpccsystems/hpcc-data
     replication: [ localMirrorPlane, localMirror2Plane ]
     pvc: local-data-pvc
+    category: data
   - name: localMirrorPlane
     prefix: /var/lib/hpccsystems/hpcc-mirror
     pvc: local-mirror-pvc
+    category: data
   - name: localMirror2Plane
     prefix: /var/lib/hpccsystems/hpcc-mirror2
     pvc: local-mirror2-pvc
+    category: data
 
   #Multiple nodes, data on a local mounts (all nodes mount the same logical file system).
   #Essentially identical to localDataPlane above
@@ -28,9 +33,11 @@ storage:
     prefix: /var/lib/hpccsystems/hpcc-data
     replication: [ nasMirrorPlane ]
     pvc: nas-data-pvc
+    category: data
   - name: nasMirrorPlane
     prefix: /var/lib/hpccsystems/hpcc-mirror
     pvc: nas-mirror-pvc
+    category: data
 
   #Multiple nodes, data on multiple local mounts (all nodes mount the same logical file system).
   #Allows an array of NFS servers to be used to store the data.  Would also potentially work for
@@ -41,9 +48,10 @@ storage:
     pvc: nas-array-data-pvc
     numDevices: 100
     includeDeviceInPath: true
+    category: data
 
   - name: sasha
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/sashastorage"
-    labels: [ sasha ]
+    category: sasha

testing/helm/tests/implicitplanes.yaml

@@ -4,34 +4,35 @@ storage:
     prefix: "/var/lib/HPCCSystems/hpcc-data"
     storageSize: 1Gi
     storageClass: ""
+    category: data
   - name: my-dll-plane
     prefix: "/var/lib/HPCCSystems/queries"
     storageSize: 3Gi
     storageClass: ""
     forcePermissions: true
-    labels: [ dll ]
+    category: dll
   - name: my-sasha-plane
     prefix: "/var/lib/HPCCSystems/sasha"
     storageSize: 3Gi
     storageClass: ""
     forcePermissions: true
-    labels: [ unknown ]
+    category: sasha
   - name: my-dali-plane
     prefix: "/var/lib/HPCCSystems/dalistorage"
     storageSize: 1Gi
     storageClass: ""
-    labels: [ dali ]
+    category: dali
   - name: landingzone
     prefix: "/var/lib/HPCCSystems/landingzone"
     storageSize: 1Gi
     storageClass: ""
-    labels: [ lz ]
+    category: lz
 
   - name: sasha
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/sashastorage"
-    labels: [ sasha ]
+    category: sasha
 
   dllStorage:
     plane: my-dll-plane

testing/helm/tests/implicitplanes2.yaml

@@ -4,33 +4,34 @@ storage:
     prefix: "/var/lib/HPCCSystems/hpcc-data"
     storageSize: 1Gi
     storageClass: ""
+    category: data
   - name: my-dll-plane
     prefix: "/var/lib/HPCCSystems/queries"
     storageSize: 3Gi
     storageClass: ""
     forcePermissions: true
-    labels: [ dll ]
+    category: dll
   - name: my-sasha-plane
     prefix: "/var/lib/HPCCSystems/sasha"
     storageSize: 3Gi
     storageClass: ""
     forcePermissions: true
-    labels: [ unknown ]
+    category: sasha
   - name: my-dali-plane
     prefix: "/var/lib/HPCCSystems/dalistorage"
     storageSize: 1Gi
     storageClass: ""
-    labels: [ dali ]
+    category: dali
   - name: landingzone
     prefix: "/var/lib/HPCCSystems/landingzone"
     storageSize: 1Gi
     storageClass: ""
-    labels: [ lz ]
+    category: lz
   - name: sasha
     storageClass: ""
     storageSize: 1Gi
     prefix: "/var/lib/HPCCSystems/sashastorage"
-    labels: [ sasha ]
+    category: sasha
 
   #   name: <required>
   #   prefix: <path>                        # Root directory for accessing the plane (if pvc defined), or url to access plane.

testing/regress/ecl/externalplane.ecl

@@ -0,0 +1,45 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nothor            HPCC-26144 plane:: currently not supported on thor
+
+import Std.File;
+
+rec := RECORD
+ string f;
+END;
+ds := DATASET([{'a'}], rec);
+
+dropzone := 'mydropzone' : STORED('dropzone');
+
+string getFName(string ext) := FUNCTION
+ RETURN '~PLANE::' + dropzone + '::' + 'external' + ext;
+END;
+
+external1 := getFName('1');
+external2 := getFName('2');
+
+SEQUENTIAL(
+File.DeleteLogicalFile(external1, true),
+File.DeleteLogicalFile(external2, true),
+OUTPUT(ds,,external1),
+OUTPUT(DATASET(external1, rec, FLAT), , external2),
+OUTPUT(DATASET(external2, rec, FLAT), , external1, OVERWRITE),
+OUTPUT(DATASET(external1, rec, FLAT)),
+File.DeleteLogicalFile(external1), // NB: will fail if doesn't exist
+File.DeleteLogicalFile(external2) // NB: will fail if doesn't exist
+);

testing/regress/ecl/key/externalplane.xml

@@ -0,0 +1,9 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><f>a</f></Row>
+</Dataset>

testing/regress/ecl/split_test.ecl

@@ -26,7 +26,7 @@ import Std.File AS FileServices;
 import $.setup;
 prefix := setup.Files(false, false).QueryFilePrefix;
 
-dropzonePath := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
+dropzonePath := '';// Use relative paths
 
 unsigned VERBOSE := 0;
 unsigned CLEAN_UP := 0;
@@ -70,7 +70,7 @@ end;
 desprayRec despray(desprayRec l) := TRANSFORM
   SELF.msg := FileServices.fDespray(
                        LOGICALNAME := sprayPrepFileName + l.suffix
-                      ,DESTINATIONIP := '.'
+                      ,DESTINATIONPLANE := 'mydropzone'
                       ,DESTINATIONPATH := desprayOutFileName + l.suffix
                       ,ALLOWOVERWRITE := True
                       );
@@ -107,13 +107,17 @@ c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(desprayRec,
 #end
 
 
-
+#if (__CONTAINERIZED__)
+SprayClusterName := 'data';
+CopySplitClusterName := 'data';
+CopyNoSplitClusterName := 'data';
+#else
 SprayClusterName := 'mythor';
 CopySplitClusterName := 'myroxie';
 CopyNoSplitClusterName := 'mythor';
+#end
 
 DestFile := 'split_test';
-ESPportIP := 'http://127.0.0.1:8010/FileSpray';
 
 gatherCounts(string filename, unsigned origRecCount, boolean noSplit, string operation = 'variable') := FUNCTION
 
@@ -147,53 +151,54 @@ sprayRec := RECORD
 end;
 
 // Variable spray tests
+#if (__CONTAINERIZED__)
+myRemoteEspFsURL := 'http://eclwatch:8010/FileSpray';
+#else
+myRemoteEspFsURL := 'http://127.0.0.1:8010/FileSpray';
+#end
 
 sprayRec spray(sprayRec l) := TRANSFORM
      SELF.msg :=
      if ( l.operation = 'variable',
             FileServices.fSprayVariable(
-                            SOURCEIP := '.',
+                            SOURCEPLANE := 'mydropzone',
                             SOURCEPATH := l.sourceFilename,
                             DESTINATIONGROUP := l.targetCluster,
                             DESTINATIONLOGICALNAME := l.targetFilename,
                             TIMEOUT := -1,
-                            ESPSERVERIPPORT := 'http://127.0.0.1:8010/FileSpray',
                             ALLOWOVERWRITE := true,
                             NOSPLIT :=  l.noSplit
                             ),
 
         if ( l.operation = 'fixed',
              FileServices.fSprayFixed(
-                                SOURCEIP := '.',
+                                SOURCEPLANE := 'mydropzone',
                                 SOURCEPATH :=  l.sourceFilename,
                                 RECORDSIZE := recSize,
                                 DESTINATIONGROUP := l.targetCluster,
                                 DESTINATIONLOGICALNAME := l.targetFilename,
                                 TIMEOUT := -1,
-                                ESPSERVERIPPORT := 'http://127.0.0.1:8010/FileSpray',
                                 ALLOWOVERWRITE := true,
                                 NOSPLIT :=  l.noSplit
                                 ),
              if ( l.operation = 'delimited',
                 FileServices.fSprayDelimited(
-                          SOURCEIP := '.',
+                          SOURCEPLANE := 'mydropzone',
                           SOURCEPATH :=  l.sourceFilename,
                           DESTINATIONGROUP := l.targetCluster,
                           DESTINATIONLOGICALNAME := l.targetFilename,
                           TIMEOUT := -1,
-                          ESPSERVERIPPORT := 'http://127.0.0.1:8010/FileSpray',
                           ALLOWOVERWRITE := true,
                           NOSPLIT :=  l.noSplit
                         ),
                     if ( l.operation = 'xml',
                         FileServices.fSprayXml(
-                            SOURCEIP := '.',
+                            SOURCEPLANE := 'mydropzone',
                             SOURCEPATH := l.sourceFilename,
                             SOURCEROWTAG := 'Rowtag',
                             DESTINATIONGROUP :=  l.targetCluster,
                             DESTINATIONLOGICALNAME := l.targetFilename,
                             TIMEOUT := -1,
-                            ESPSERVERIPPORT := 'http://127.0.0.1:8010/FileSpray',
                             ALLOWOVERWRITE := true,
                             NOSPLIT :=  l.noSplit
                             ),
@@ -202,15 +207,13 @@ sprayRec spray(sprayRec l) := TRANSFORM
                                     sourceLogicalName := l.sourceFilename,
                                     destinationGroup := l.targetCluster, //'myroxie', //ClusterName,
                                     destinationLogicalName := l.targetFilename,
-                                    sourceDali := '.',
                                     TIMEOUT := -1,
-                                    ESPSERVERIPPORT := 'http://127.0.0.1:8010/FileSpray',
                                     ALLOWOVERWRITE := true,
                                     NOSPLIT :=  l.noSplit
                                     ),
                                 if (l.operation = 'remotePull',
                                     FileServices.fRemotePull(
-                                         remoteEspFsURL := 'http://127.0.0.1:8010/FileSpray',
+                                         remoteEspFsURL := myRemoteEspFsURL,
                                          sourceLogicalName := l.sourceFilename,
                                          destinationGroup := l.targetCluster,
                                          destinationLogicalName := l.targetFilename,

thorlcr/activities/indexwrite/thindexwrite.cpp

@@ -280,6 +280,7 @@ public:
                 bloom->setProp("@bloomProbability", pval.str());
             }
             container.queryTempHandler()->registerFile(fileName, container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
+            props.setPropInt64("@numDiskWrites", statsCollection.getStatisticSum(StNumDiskWrites));
             if (!dlfn.isExternal())
                 queryThorFileManager().publish(container.queryJob(), fileName, *fileDesc);
         }

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -243,6 +243,7 @@ public:
             builder.clear();
             if (builderIFileIO)
             {
+                mergeStats(stats, builderIFileIO, diskWriteRemoteStatistics);
                 builderIFileIO->close();
                 builderIFileIO.clear();
             }

thorlcr/activities/thdiskbase.cpp

@@ -224,6 +224,7 @@ void CWriteMasterBase::publish()
     }
     if (TDWrestricted & diskHelperBase->getFlags())
         props.setPropBool("restricted", true );
+    props.setPropInt64("@numDiskWrites", statsCollection.getStatisticSum(StNumDiskWrites));
     container.queryTempHandler()->registerFile(fileName, container.queryOwner().queryGraphId(), diskHelperBase->getTempUsageCount(), TDXtemporary & diskHelperBase->getFlags(), getDiskOutputKind(diskHelperBase->getFlags()), &clusters);
     if (!dlfn.isExternal())
     {

thorlcr/graph/thgraphmaster.ipp

@@ -63,6 +63,14 @@ public:
             summary.merge(*nodeStats[n], n);
         summary.recordStatistics(result);
     }
+    stat_type getStatisticSum(StatisticKind kind)
+    {
+        stat_type total = 0;
+        unsigned index = mapping.getIndex(kind);
+        for (unsigned n=0; n < nodeStats.size(); n++) // NB: size is = queryClusterWidth()
+            total += nodeStats[n]->getValue(index);
+        return total;
+    }
 };
 
 class graphmaster_decl CThorEdgeCollection : public CThorStatsCollection

thorlcr/master/thmastermain.cpp

@@ -817,6 +817,7 @@ int main( int argc, const char *argv[]  )
         PROGLOG("Global memory size = %d MB", mmemSize);
         roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, gmemRetainMemory, ((memsize_t)mmemSize) * 0x100000, 0, thorAllocSizes, NULL);
 
+#ifndef _CONTAINERIZED
         const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
         const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
         StringBuffer datadir;
@@ -829,6 +830,8 @@ int main( int argc, const char *argv[]  )
             setBaseDirectory(overrideBaseDirectory, false);
         if (overrideReplicateDirectory&&*overrideBaseDirectory)
             setBaseDirectory(overrideReplicateDirectory, true);
+#endif
+
         StringBuffer tempDirStr;
         if (!getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"), tempDirStr))
         {

thorlcr/msort/tsorts1.cpp

@@ -84,7 +84,7 @@ public:
         }
 #endif // OPENSSL
 
-        stream = ConnectMergeRead(streamno,rowif,mergeep,startrec,numrecs,socket.getClear());
+        stream = ConnectMergeRead(streamno,rowif,mergeep,startrec,numrecs,socket);
 
         LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: Stream(%u) connected to %s",streamno,url);
     }

thorlcr/slave/thslavemain.cpp

@@ -447,6 +447,7 @@ int main( int argc, const char *argv[]  )
             else
                 globals->setProp("@externalProgDir", thorPath);
 
+#ifndef _CONTAINERIZED
             const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
             const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
             StringBuffer datadir;
@@ -459,6 +460,8 @@ int main( int argc, const char *argv[]  )
                 setBaseDirectory(overrideBaseDirectory, false);
             if (!isEmptyString(overrideReplicateDirectory))
                 setBaseDirectory(overrideReplicateDirectory, true);
+#endif
+
             StringBuffer tempDirStr;
             if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"), tempDirStr))
                 globals->setProp("@thorTempDirectory", tempDirStr.str());

thorlcr/thorutil/thormisc.cpp

@@ -76,7 +76,7 @@ const StatisticsMapping basicActivityStatistics({StTimeLocalExecute, StTimeBlock
 const StatisticsMapping groupActivityStatistics({StNumGroups, StNumGroupMax}, basicActivityStatistics);
 const StatisticsMapping hashJoinActivityStatistics({StNumLeftRows, StNumRightRows}, basicActivityStatistics);
 const StatisticsMapping indexReadActivityStatistics({StNumRowsProcessed, StNumIndexSeeks, StNumIndexScans, StNumPostFiltered, StNumIndexWildSeeks}, basicActivityStatistics);
-const StatisticsMapping indexWriteActivityStatistics({StPerReplicated}, basicActivityStatistics);
+const StatisticsMapping indexWriteActivityStatistics({StPerReplicated}, basicActivityStatistics, diskWriteRemoteStatistics);
 const StatisticsMapping keyedJoinActivityStatistics({ StNumIndexSeeks, StNumIndexScans, StNumIndexAccepted, StNumPostFiltered, StNumPreFiltered, StNumDiskSeeks, StNumDiskAccepted, StNumDiskRejected, StNumIndexWildSeeks}, basicActivityStatistics);
 const StatisticsMapping loopActivityStatistics({StNumIterations}, basicActivityStatistics);
 const StatisticsMapping lookupJoinActivityStatistics({StNumSmartJoinSlavesDegradedToStd, StNumSmartJoinDegradedToLocal}, basicActivityStatistics);