HPCC-25910 Improve the syntax for spraying from a k8s landingzone

Signed-off-by: Shamser Ahmed <shamser.ahmed@lexisnexis.com>

dali/dfu/dfurun.cpp

@@ -468,7 +468,6 @@ class CDFUengine: public CInterface, implements IDFUengine
             IPropertyTree & plane = planes->query();
             const char * fullDropZoneDir = plane.queryProp("@prefix");
             assertex(fullDropZoneDir);
-            // note: for bare-metal drop-zones, will need to compare ip address
             if (startsWith(pfilePath, fullDropZoneDir))
                 return;
         }
@@ -1317,7 +1316,7 @@ public:
 #ifdef _CONTAINERIZED
                         StringBuffer clusterName;
                         destination->getGroupName(0, clusterName);
-                        Owned<IPropertyTree> plane = getDropZonePlane(clusterName);
+                        Owned<IPropertyTree> plane = getStoragePlane(clusterName);
                         if (plane)
                         {
                             if (plane->hasProp("@defaultSprayParts"))

dali/dfuplus/dfuplus.cpp

@@ -351,7 +351,9 @@ int CDfuPlusHelper::doit()
     return 0;
 }
 
-bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format, StringBuffer &retwuid, StringBuffer &except)
+bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const char* srcplane,
+                                const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,
+                                const char *format, StringBuffer &retwuid, StringBuffer &except)
 {
     int recordsize;
     if(stricmp(format, "recfmvb") == 0) {
@@ -374,18 +376,23 @@ bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char*
     }
 
     Owned<IClientSprayFixed> req = sprayclient->createSprayFixedRequest();
-    if(srcxml == nullptr)
+    if(isEmptyString(srcxml))
     {
-        req->setSourceIP(srcip);
+        info("\nFixed spraying from %s on %s to %s\n", srcfile, srcplane?srcplane:srcip, dstname);
+        if (!isEmptyString(srcip)) req->setSourceIP(srcip);
+        if (!isEmptyString(srcplane)) req->setSourcePlane(srcplane);
         req->setSourcePath(srcfile);
     }
     else
+    {
+        info("\nSpraying to %s\n", dstname);
         req->setSrcxml(xmlbuf);
+    }
 
     if(recordsize != 0)
         req->setSourceRecordSize(recordsize);
 
-    if(dstcluster != nullptr)
+    if(!isEmptyString(dstcluster))
         req->setDestGroup(dstcluster);
     req->setDestLogicalName(dstname);
     req->setOverwrite(globals->getPropBool("overwrite", false));
@@ -429,11 +436,6 @@ bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char*
     if(globals->hasProp("expireDays"))
         req->setExpireDays(globals->getPropInt("expireDays"));
 
-    if(srcxml == nullptr)
-        info("\nFixed spraying from %s on %s to %s\n", srcfile, srcip, dstname);
-    else
-        info("\nSpraying to %s\n", dstname);
-
     Owned<IClientSprayFixedResponse> result = sprayclient->SprayFixed(req);
     const char *wuid = result->getWuid();
     if (!wuid||!*wuid) {
@@ -444,21 +446,26 @@ bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char*
     return true;
 }
 
-bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid, StringBuffer &except)
+bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const char* srcplane,
+                                   const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,
+                                   const char *format,StringBuffer &retwuid, StringBuffer &except)
 {
     Owned<IClientSprayVariable> req = sprayclient->createSprayVariableRequest();
-    if(srcxml == nullptr)
+    if(isEmptyString(srcxml))
     {
-        req->setSourceIP(srcip);
+        info("\nVariable spraying from %s on %s to %s\n", srcfile, srcplane?srcplane:srcip, dstname);
+        if (!isEmptyString(srcip)) req->setSourceIP(srcip);
+        if (!isEmptyString(srcplane)) req->setSourcePlane(srcplane);
         req->setSourcePath(srcfile);
     }
     else
     {
+        info("\nSpraying to %s\n", dstname);
         req->setSrcxml(xmlbuf);
     }
 
     const char* mrsstr = globals->queryProp("maxRecordSize");
-    if(mrsstr != nullptr)
+    if(!isEmptyString(mrsstr))
         req->setSourceMaxRecordSize(atoi(mrsstr));
     else
         req->setSourceMaxRecordSize(8192);
@@ -480,23 +487,23 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
     }
     else if(stricmp(format, "xml") == 0)
     {
-        if(encoding == nullptr)
+        if(isEmptyString(encoding))
             encoding = "utf8";
         else if(stricmp(encoding, "ascii") == 0)
             throw MakeStringException(-1, "xml format only accepts utf encodings");
-        if(rowtag == nullptr || *rowtag == '\0')
+        if(isEmptyString(rowtag))
             throw MakeStringException(-1, "rowtag not specified.");
-        if(rowpath && *rowpath)
+        if(!isEmptyString(rowpath))
             throw MakeStringException(-1, "You can't use rowpath option with xml format");
     }
     else if(stricmp(format, "csv") == 0)
     {
-        if(encoding == nullptr)
+        if(isEmptyString(encoding))
             encoding = "ascii";
 
-        if(rowtag != nullptr && *rowtag != '\0')
+        if(!isEmptyString(rowtag))
             throw MakeStringException(-1, "You can't use rowtag option with csv/delimited format");
-        if(rowpath && *rowpath)
+        if(!isEmptyString(rowpath))
             throw MakeStringException(-1, "You can't use rowpath option with csv/delimited format");
 
         const char* separator = globals->queryProp("separator");
@@ -509,7 +516,7 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
                 req->setNoSourceCsvSeparator(true);
         }
         const char* terminator = globals->queryProp("terminator");
-        if(terminator && *terminator)
+        if(!isEmptyString(terminator))
             req->setSourceCsvTerminate(terminator);
         const char* quote = globals->queryProp("quote");
         if(quote)
@@ -519,17 +526,17 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
             req->setSourceCsvQuote(quote);
         }
         const char* escape = globals->queryProp("escape");
-        if(escape && *escape)
+        if(!isEmptyString(escape))
             req->setSourceCsvEscape(escape);
     }
     else
         encoding = format; // may need extra later
 
     req->setSourceFormat(CDFUfileformat::decode(encoding));
-    if(rowtag != nullptr)
+    if(!isEmptyString(rowtag))
         req->setSourceRowTag(rowtag);
 
-    if(dstcluster != nullptr)
+    if(!isEmptyString(dstcluster))
         req->setDestGroup(dstcluster);
     req->setDestLogicalName(dstname);
     req->setOverwrite(globals->getPropBool("overwrite", false));
@@ -578,13 +585,10 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
     if(globals->hasProp("expireDays"))
         req->setExpireDays(globals->getPropInt("expireDays"));
 
-    if(srcxml == nullptr)
-        info("\nVariable spraying from %s on %s to %s\n", srcfile, srcip, dstname);
-    else
-        info("\nSpraying to %s\n", dstname);
     Owned<IClientSprayResponse> result = sprayclient->SprayVariable(req);
     const char *wuid = result->getWuid();
-    if (!wuid||!*wuid) {
+    if (isEmptyString(wuid))
+    {
         result->getExceptions().errorMessage(except);
         return false;
     }
@@ -594,31 +598,36 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
 
 int CDfuPlusHelper::spray()
 {
+    bool usingSrcPlane = false;
     const char* srcxml = globals->queryProp("srcxml");
     const char* srcip = globals->queryProp("srcip");
+    const char* srcplane =  globals->queryProp("srcPlane");
     const char* srcfile = globals->queryProp("srcfile");
 
     bool nowait = globals->getPropBool("nowait", false);
 
+    if (!isEmptyString(srcplane))
+        usingSrcPlane=true;
     MemoryBuffer xmlbuf;
 
-    if(srcxml == nullptr)
+    if(isEmptyString(srcxml))
     {
-        if(srcfile == nullptr)
+        if(isEmptyString(srcfile))
             throw MakeStringException(-1, "srcfile not specified");
-        if(srcip == nullptr) {
+        if(!usingSrcPlane && isEmptyString(srcip))
+        {
 #ifdef DAFILESRV_LOCAL
             progress("srcip not specified - assuming spray from local machine\n");
             srcip = ".";
 #else
-            throw MakeStringException(-1, "srcip not specified");
+            throw MakeStringException(-1, "Neither srcip nor srcplane specified");
 #endif
         }
     }
     else
     {
-        if(srcip != nullptr || srcfile != nullptr)
-            throw MakeStringException(-1, "srcip/srcfile and srcxml can't be used at the same time");
+        if(!isEmptyString(srcip) || !isEmptyString(srcfile) || usingSrcPlane)
+            throw MakeStringException(-1, "srcip/srcfile/srcplane and srcxml can't be used at the same time");
         StringBuffer buf;
         buf.loadFile(srcxml);
         int len = buf.length();
@@ -626,10 +635,10 @@ int CDfuPlusHelper::spray()
     }
 
     const char* dstname = globals->queryProp("dstname");
-    if(dstname == nullptr)
+    if(isEmptyString(dstname))
         throw MakeStringException(-1, "dstname not specified");
     const char* dstcluster = globals->queryProp("dstcluster");
-    if(dstcluster == nullptr)
+    if(isEmptyString(dstcluster))
         throw MakeStringException(-1, "dstcluster not specified");
     const char* format = globals->queryProp("format");
     if(format == nullptr)
@@ -637,17 +646,20 @@ int CDfuPlusHelper::spray()
     else if (stricmp(format, "delimited") == 0)
         format="csv";
 
-    SocketEndpoint localep;
-    StringBuffer localeps;
-    if (checkLocalDaFileSvr(srcip,localep))
-        srcip = localep.getUrlStr(localeps).str();
+    if (!usingSrcPlane)
+    {
+        SocketEndpoint localep;
+        StringBuffer localeps;
+        if (checkLocalDaFileSvr(srcip,localep))
+            srcip = localep.getUrlStr(localeps).str();
+    }
     StringBuffer wuid;
     StringBuffer errmsg;
     bool ok;
     if ((stricmp(format, "fixed") == 0)||(stricmp(format, "recfmvb") == 0)||(stricmp(format, "recfmv") == 0)||(stricmp(format, "variablebigendian") == 0))
-        ok = fixedSpray(srcxml,srcip,srcfile,xmlbuf,dstcluster,dstname,format,wuid,errmsg);
+        ok = fixedSpray(srcxml,srcip,srcfile,srcplane,xmlbuf,dstcluster,dstname,format,wuid,errmsg);
     else if((stricmp(format, "csv") == 0)||(stricmp(format, "xml") == 0)||(stricmp(format, "json") == 0)||(stricmp(format, "variable") == 0))
-        ok = variableSpray(srcxml,srcip,srcfile,xmlbuf,dstcluster,dstname,format,wuid, errmsg);
+        ok = variableSpray(srcxml,srcip,srcfile,srcplane,xmlbuf,dstcluster,dstname,format,wuid, errmsg);
     else
         throw MakeStringException(-1, "format %s not supported", format);
     if (!ok) {
@@ -716,21 +728,22 @@ int CDfuPlusHelper::replicate()
 int CDfuPlusHelper::despray()
 {
     const char* srcname = globals->queryProp("srcname");
-    if(srcname == nullptr)
+    if(isEmptyString(srcname))
         throw MakeStringException(-1, "srcname not specified");
 
     const char* dstxml = globals->queryProp("dstxml");
     const char* dstip = globals->queryProp("dstip");
+    const char* dstplane = globals->queryProp("dstplane");
     const char* dstfile = globals->queryProp("dstfile");
 
     bool nowait = globals->getPropBool("nowait", false);
 
     MemoryBuffer xmlbuf;
-    if(dstxml == nullptr)
+    if(isEmptyString(dstxml))
     {
-        if(dstip == nullptr) {
+        if (isEmptyString(dstplane) && isEmptyString(dstip)) {
 #ifdef DAFILESRV_LOCAL
-            progress("dstip not specified - assuming spray from local machine\n");
+            progress("dstip and dstplane not specified - assuming despray to local machine\n");
             dstip = ".";
 #else
             throw MakeStringException(-1, "dstip not specified");
@@ -739,28 +752,36 @@ int CDfuPlusHelper::despray()
     }
     else
     {
-        if(dstip != nullptr || dstfile != nullptr)
-            throw MakeStringException(-1, "dstip/dstfile and dstxml can't be used at the same time");
+        if(!isEmptyString(dstip) || !isEmptyString(dstfile) || !isEmptyString(dstplane))
+            throw MakeStringException(-1, "dstip/dstfile/dstplane and dstxml can't be used at the same time");
         StringBuffer buf;
         buf.loadFile(dstxml);
         int len = buf.length();
         xmlbuf.setBuffer(len, buf.detach(), true);
     }
 
-    if(dstxml == nullptr)
-        info("\nDespraying %s to host %s file %s\n", srcname, dstip, dstfile ? dstfile : "");
-    else
-        info("\nDespraying %s\n", srcname);
-
     Owned<IClientDespray> req = sprayclient->createDesprayRequest();
     req->setSourceLogicalName(srcname);
-    if(dstxml == nullptr)
+    StringBuffer extrainfo;
+    if(isEmptyString(dstxml))
     {
-        req->setDestIP(dstip);
+        extrainfo.append(" to");
+        if (!isEmptyString(dstplane))
+        {
+            extrainfo.appendf(" storage plane %s", dstplane);
+            req->setDestPlane(dstplane);
+        }
+        if (!isEmptyString(dstip))
+        {
+            extrainfo.appendf(" host %s", dstip);
+            req->setDestIP(dstip);
+        }
+        extrainfo.appendf(" file %s", dstfile ? dstfile : "");
         req->setDestPath(dstfile);
     }
     else
         req->setDstxml(xmlbuf);
+    info("\nDespraying %s%s\n",srcname,extrainfo.str());
 
     req->setOverwrite(globals->getPropBool("overwrite", false));
     if(globals->hasProp("connect"))
@@ -794,14 +815,13 @@ int CDfuPlusHelper::despray()
     if(globals->hasProp("decrypt"))
         req->setDecrypt(globals->queryProp("decrypt"));
 
-
     SocketEndpoint localep;
     StringBuffer localeps;
-    if (checkLocalDaFileSvr(dstip,localep))
+    if (isEmptyString(dstplane) && checkLocalDaFileSvr(dstip,localep))
         dstip = localep.getUrlStr(localeps).str();
     Owned<IClientDesprayResponse> result = sprayclient->Despray(req);
     const char* wuid = result->getWuid();
-    if(wuid == nullptr || *wuid == '\0')
+    if(isEmptyString(wuid))
         exc(result->getExceptions(),"despraying");
     else
     {

dali/dfuplus/dfuplus.hpp

@@ -66,8 +66,8 @@ private:
     int listhistory();
     int erasehistory();
 
-    bool fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid,StringBuffer &except );
-    bool variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid,StringBuffer &except );
+    bool fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const char* srcplane,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid,StringBuffer &except );
+    bool variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const char* srcplane,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid,StringBuffer &except );
 
     int waitToFinish(const char* wuid);
     void info(const char *format, ...) __attribute__((format(printf, 2, 3)));

dali/dfuplus/main.cpp

@@ -63,6 +63,7 @@ void handleSyntax()
     out.append("                                   source file set is empty.\n");
     out.append("    spray options:\n");
     out.append("        srcip=<source-machine-ip>\n");
+    out.append("        srcplane=<source-storage-plane-name>\n");
     out.append("        srcfile=<source-file-path>\n");
     out.append("        srcxml=<xml-file> -- replaces srcip and srcfile\n");
     out.append("        dstname=<destination-logical-name>\n");
@@ -101,6 +102,7 @@ void handleSyntax()
     out.append("    despray options:\n");
     out.append("        srcname=<source-logical-name>\n");
     out.append("        dstip=<destination-machine-ip>\n");
+    out.append("        dstplane=<destination-storage-plane-name>\n");
     out.append("        dstfile=<destination-file-path>\n");
     out.append("        dstxml=<xml-file> -- replaces dstip and dstfile\n");
     out.append("        splitprefix=... use prefix (same format as /prefix) to split file up\n");

esp/scm/ws_fs.ecm

@@ -300,6 +300,7 @@ GetDFUExceptionsResponse
 ESPrequest [nil_remove] SprayFixed
 {
     string sourceIP;
+    [min_ver("1.22")] string sourcePlane;
     string sourcePath;
     binary srcxml;
     [min_ver("1.09")] string sourceFormat;
@@ -345,6 +346,7 @@ SprayFixedResponse
 ESPrequest [nil_remove] SprayVariable
 {
     string sourceIP;
+    [min_ver("1.22")] string sourcePlane;
     string sourcePath;
     binary srcxml;
 
@@ -420,6 +422,7 @@ ESPrequest Despray
 
     string destIP;
     string destPath;
+    [min_ver("1.22")] string destPlane;
     binary dstxml;
     bool   overwrite;
 
@@ -689,7 +692,7 @@ ESPresponse [exceptions_inline, nil_remove] GetDFUServerQueuesResponse
 
 ESPservice [
     auth_feature("DEFERRED"),
-    version("1.21"),
+    version("1.22"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPmethod EchoDateTime(EchoDateTime, EchoDateTimeResponse);

esp/services/ws_fs/ws_fsService.cpp

@@ -1814,19 +1814,70 @@ bool CFileSprayEx::onGetDFUExceptions(IEspContext &context, IEspGetDFUExceptions
     return true;
 }
 
-void CFileSprayEx::readAndCheckSpraySourceReq(MemoryBuffer& srcxml, const char* srcIP, const char* srcPath,
+void CFileSprayEx::readAndCheckSpraySourceReq(MemoryBuffer& srcxml, const char* srcIP, const char* srcPath, const char* srcPlane,
     StringBuffer& sourceIPReq, StringBuffer& sourcePathReq)
 {
-    StringBuffer sourcePath(srcPath);
-    sourceIPReq.set(srcIP);
-    sourceIPReq.trim();
-    sourcePath.trim();
+    StringBuffer sourcePath;
+
     if(srcxml.length() == 0)
     {
+        if (!isEmptyString(srcPlane))
+        {
+            Owned<IPropertyTree> dropZone = getDropZonePlane(srcPlane);
+            if (!dropZone)
+                throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Unknown landing zone: %s", srcPlane);
+            const char * dropZonePlanePath = dropZone->queryProp("@prefix");
+            if (isAbsolutePath(srcPath))
+            {
+                if (!startsWith(srcPath,dropZonePlanePath))
+                    throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid source path");
+                sourcePath.append(srcPath).trim();
+            }
+            else
+            {
+                sourcePath.append(dropZonePlanePath);
+                addNonEmptyPathSepChar(sourcePath);
+                sourcePath.append(srcPath).trim();
+            }
+            const char * hostGroup = dropZone->queryProp("@hostGroup");
+            if (hostGroup)
+            {
+                Owned<IPropertyTree> match = getHostGroup(hostGroup,true);
+                if (!isEmptyString(srcIP))
+                {
+                    // Already have srcIP. Just need to check that the ip is valid for storage plane.
+                    bool ipAddressMatches = false;
+                    Owned<IPropertyTreeIterator> hostIter = match->getElements("hosts");
+                    ForEach (*hostIter)
+                    {
+                        const char *knownIP = hostIter->query().queryProp(nullptr);
+                        if (strcmp(knownIP, srcIP)==0)
+                        {
+                            ipAddressMatches=true;
+                            break;
+                        }
+                    }
+                    if (!ipAddressMatches)
+                        throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "srcip %s is not valid storage plane %s", srcIP, srcPlane);
+                    sourceIPReq.set(srcIP);
+                }
+                else
+                    sourceIPReq.set(match->queryProp("hosts[1]"));
+            }
+            else
+            {
+                sourceIPReq.set("localhost"); // storage plane will be mounted when not using hostgroup
+            }
+        }
+        else
+        {
+            sourcePath.append(srcPath).trim();
+            sourceIPReq.set(srcIP).trim();
+        }
         if (sourceIPReq.isEmpty())
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Source network IP not specified.");
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Source network IP not specified.");
         if (sourcePath.isEmpty())
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Source path not specified.");
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Source path not specified.");
     }
     getStandardPosixPath(sourcePathReq, sourcePath.str());
 }
@@ -1869,20 +1920,20 @@ bool CFileSprayEx::onSprayFixed(IEspContext &context, IEspSprayFixed &req, IEspS
         StringBuffer destFolder, destTitle, defaultFolder, defaultReplicateFolder;
 
         const char* destNodeGroup = req.getDestGroup();
-        if(destNodeGroup == NULL || *destNodeGroup == '\0')
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination node group not specified.");
+        if (isEmptyString(destNodeGroup))
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Destination node group not specified.");
 
         MemoryBuffer& srcxml = (MemoryBuffer&)req.getSrcxml();
         StringBuffer sourceIPReq, sourcePathReq;
-        readAndCheckSpraySourceReq(srcxml, req.getSourceIP(), req.getSourcePath(), sourceIPReq, sourcePathReq);
+        readAndCheckSpraySourceReq(srcxml, req.getSourceIP(), req.getSourcePath(), req.getSourcePlane(), sourceIPReq, sourcePathReq);
         const char* srcip = sourceIPReq.str();
         const char* srcfile = sourcePathReq.str();
         const char* destname = req.getDestLogicalName();
-        if(!destname || !*destname)
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination file not specified.");
+        if(isEmptyString(destname))
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Destination file not specified.");
         CDfsLogicalFileName lfn;
         if (!lfn.setValidate(destname))
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid destination filename:'%s'", destname);
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Invalid destination filename:'%s'", destname);
         destname = lfn.get();
         PROGLOG("SprayFixed: DestLogicalName %s, DestGroup %s", destname, destNodeGroup);
 
@@ -1919,7 +1970,7 @@ bool CFileSprayEx::onSprayFixed(IEspContext &context, IEspSprayFixed &req, IEspS
             RemoteMultiFilename rmfn;
             SocketEndpoint ep(srcip);
             if (ep.isNull())
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "SprayFixed to %s: cannot resolve source network IP from %s.", destname, srcip);
+                throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "SprayFixed to %s: cannot resolve source network IP from %s.", destname, srcip);
 
             rmfn.setEp(ep);
             StringBuffer fnamebuf(srcfile);
@@ -2042,8 +2093,8 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
         StringBuffer destFolder, destTitle, defaultFolder, defaultReplicateFolder;
 
         const char* destNodeGroup = req.getDestGroup();
-        if(destNodeGroup == NULL || *destNodeGroup == '\0')
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination node group not specified.");
+        if (isEmptyString(destNodeGroup))
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Destination node group not specified.");
 
         StringBuffer gName, ipAddr;
         const char *pTr = strchr(destNodeGroup, ' ');
@@ -2057,15 +2108,15 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
 
         MemoryBuffer& srcxml = (MemoryBuffer&)req.getSrcxml();
         StringBuffer sourceIPReq, sourcePathReq;
-        readAndCheckSpraySourceReq(srcxml, req.getSourceIP(), req.getSourcePath(), sourceIPReq, sourcePathReq);
+        readAndCheckSpraySourceReq(srcxml, req.getSourceIP(), req.getSourcePath(), req.getSourcePlane(), sourceIPReq, sourcePathReq);
         const char* srcip = sourceIPReq.str();
         const char* srcfile = sourcePathReq.str();
         const char* destname = req.getDestLogicalName();
-        if(!destname || !*destname)
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination file not specified.");
+        if(isEmptyString(destname))
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Destination file not specified.");
         CDfsLogicalFileName lfn;
         if (!lfn.setValidate(destname))
-            throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid destination filename:'%s'", destname);
+            throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Invalid destination filename:'%s'", destname);
 
         destname = lfn.get();
         PROGLOG("SprayVariable: DestLogicalName %s, DestGroup %s", destname, destNodeGroup);
@@ -2096,7 +2147,7 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
             RemoteMultiFilename rmfn;
             SocketEndpoint ep(srcip);
             if (ep.isNull())
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "SprayVariable to %s: cannot resolve source network IP from %s.", destname, srcip);
+                throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "SprayVariable to %s: cannot resolve source network IP from %s.", destname, srcip);
 
             rmfn.setEp(ep);
             StringBuffer fnamebuf(srcfile);
@@ -2260,25 +2311,54 @@ bool CFileSprayEx::onReplicate(IEspContext &context, IEspReplicate &req, IEspRep
     return true;
 }
 
-void CFileSprayEx::getDropZoneInfoByDestPlane(double clientVersion, const char* destPlane, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask)
+void CFileSprayEx::getDropZoneInfoByDestPlane(double clientVersion, const char* destPlane, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask, StringBuffer & hostip)
 {
-    Owned<IPropertyTree> plane = getDropZonePlane(destPlane);
-    if (!plane)
-        throw makeStringExceptionV(ECLWATCH_DROP_ZONE_NOT_FOUND, "No drop zone matching plane %s", destPlane);
+    Owned<IPropertyTree> dropZone = getDropZonePlane(destPlane);
+    if (!dropZone)
+        throw makeStringExceptionV(ECLWATCH_DROP_ZONE_NOT_FOUND, "Unknown landing zone %s", destPlane);
 
-    StringBuffer fullDropZoneDir(plane->queryProp("@prefix"));
+    StringBuffer fullDropZoneDir(dropZone->queryProp("@prefix"));
     addPathSepChar(fullDropZoneDir);
     if (isAbsolutePath(destFileIn))
     {
-        if (strncmp(fullDropZoneDir, destFileIn, fullDropZoneDir.length()))
-            throw makeStringExceptionV(ECLWATCH_DROP_ZONE_NOT_FOUND, "No drop zone configured for %s:%s", destPlane, destFileIn);
-        destFileOut.set(destFileIn);
+        if (!startsWith(destFileIn, fullDropZoneDir))
+            throw makeStringExceptionV(ECLWATCH_DROP_ZONE_NOT_FOUND, "No landing zone configured for %s:%s", destPlane, destFileIn);
+    }
+    else
+    {
+        destFileOut.append(fullDropZoneDir);
+        addNonEmptyPathSepChar(destFileOut);
+    }
+    destFileOut.append(destFileIn).trim();
+    dropZone->getProp("@umask", umask);
+    const char * hostGroup = dropZone->queryProp("@hostGroup");
+    if (hostGroup)
+    {
+        Owned<IPropertyTree> match = getHostGroup(hostGroup,true);
+        if (!hostip.isEmpty())
+        {
+            // Already have hostip. Just need to check that the ip is valid for storage plane.
+            bool ipAddressMatches = false;
+            Owned<IPropertyTreeIterator> hostIter = match->getElements("hosts");
+            ForEach (*hostIter)
+            {
+                const char *knownIP = hostIter->query().queryProp(nullptr);
+                if (strcmp(knownIP, hostip)==0)
+                {
+                    ipAddressMatches=true;
+                    break;
+                }
+            }
+            if (!ipAddressMatches)
+                throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "destip %s is not valid storage plane %s", hostip.str(), destPlane);
+        }
+        else
+            hostip.set(match->queryProp("hosts[1]"));
     }
     else
     {
-        destFileOut.append(destFileIn);
+        hostip.set("localhost"); // storage plane will be mounted when not using hostgroup
     }
-    plane->getProp("@umask", umask);
 }
 
 void CFileSprayEx::getDropZoneInfoByIP(double clientVersion, const char* ip, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask)
@@ -2405,7 +2485,13 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
 
         PROGLOG("Despray %s", srcname);
         double version = context.getClientVersion();
-        const char* destip = req.getDestIP();
+        StringBuffer destip(req.getDestIP());
+        const char* destPlane = req.getDestPlane();
+#ifdef _CONTAINERIZED
+        if (isEmptyString(destPlane))
+            destPlane = req.getDestGroup();  // allow eclwatch to continue providing storage plane as 'destgroup' field
+#endif
+
         StringBuffer destPath;
         StringBuffer implicitDestFile;
         const char* destfile = getStandardPosixPath(destPath, req.getDestPath()).str();
@@ -2413,8 +2499,8 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
         MemoryBuffer& dstxml = (MemoryBuffer&)req.getDstxml();
         if(dstxml.length() == 0)
         {
-            if(!destip || !*destip)
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination network IP not specified.");
+            if(isEmptyString(destPlane) && destip.isEmpty())
+                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Destination network IP/storage plane not specified.");
 
             //If the destination filename is not provided, calculate a relative filename from the logical filename
             if(!destfile || !*destfile)
@@ -2444,18 +2530,17 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
 
         if(dstxml.length() == 0)
         {
+            StringBuffer destfileWithPath, umask;
+            if (!isEmptyString(destPlane))
+                getDropZoneInfoByDestPlane(version, destPlane, destfile, destfileWithPath, umask, destip);
+            else
+                getDropZoneInfoByIP(version, destip, destfile, destfileWithPath, umask);
+
             RemoteFilename rfn;
-            SocketEndpoint ep(destip);
+            SocketEndpoint ep(destip.str());
             if (ep.isNull())
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Despray %s: cannot resolve destination network IP from %s.", srcname, destip);
+                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Despray %s: cannot resolve destination network IP from %s.", srcname, destip.str());
 
-            StringBuffer destfileWithPath, umask;
-#ifdef _CONTAINERIZED
-            const char* destPlane = req.getDestGroup();
-            getDropZoneInfoByDestPlane(version, destPlane, destfile, destfileWithPath, umask);
-#else
-            getDropZoneInfoByIP(version, destip, destfile, destfileWithPath, umask);
-#endif
             //Ensure the filename is dependent on the file part if parts are being preserved
             if (preserveFileParts && !strstr(destfileWithPath, "$P$"))
                 destfileWithPath.append("._$P$_of_$N$");
@@ -2564,7 +2649,7 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
         if (!bRoxie)
         {
             if (!lfn.setValidate(dstname))
-                throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid destination filename:'%s'", dstname);
+                throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid destination filename:'%s'", dstname);
             dstname = lfn.get();
         }
 

esp/services/ws_fs/ws_fsService.hpp

@@ -66,7 +66,7 @@ public:
 
 class CFileSprayEx : public CFileSpray
 {
-    void readAndCheckSpraySourceReq(MemoryBuffer& srcxml, const char* srcIP, const char* srcPath,
+    void readAndCheckSpraySourceReq(MemoryBuffer& srcxml, const char* srcIP, const char* srcPath, const char* srcplane,
         StringBuffer& sourceIPReq, StringBuffer& sourcePathReq);
 
 public:
@@ -148,7 +148,7 @@ protected:
     void appendGroupNode(IArrayOf<IEspGroupNode>& groupNodes, const char* nodeName, const char* clusterType, bool replicateOutputs);
     bool getOneDFUWorkunit(IEspContext& context, const char* wuid, IEspGetDFUWorkunitsResponse& resp);
     void getDropZoneInfoByIP(double clientVersion, const char* destIP, const char* destFile, StringBuffer& path, StringBuffer& mask);
-    void getDropZoneInfoByDestPlane(double clientVersion, const char* destGroup, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask);
+    void getDropZoneInfoByDestPlane(double clientVersion, const char* destGroup, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& umask, StringBuffer & hostip);
     bool checkDropZoneIPAndPath(double clientVersion, const char* dropZone, const char* netAddr, const char* path);
     void addDropZoneFile(IEspContext& context, IDirectoryIterator* di, const char* name, const char pathSep, IArrayOf<IEspPhysicalFileStruct>&files);
     void searchDropZoneFiles(IEspContext& context, IpAddress& ip, const char* dir, const char* nameFilter, IArrayOf<IEspPhysicalFileStruct>& files, unsigned& filesFound);