HPCC-9419 Pass data2 through ClusterPartDiskMapSpec for overloaded

When ESP service copies a file to a roxie, the service checks slaveConfig.
If it is 'overloaded', the service calls setRoxie() which is modified to
set a new flag CPDMSF_overloadedConfig into ClusterPartDiskMapSpec. If the
flag is set, a DFUServer copies 'non-primary' file parts into predefined
defaultReplicateDir (data2, ...).

Signed-off-by: Kevin Wang <kevin.wang@lexisnexis.com>

dali/base/dadfs.cpp

@@ -3314,8 +3314,12 @@ public:
         if (cluster!=NotFound) {
             unsigned n;
             unsigned d;
-            clusters.item(cluster).queryPartDiskMapping().calcPartLocation(partno,numParts(),rep,clusters.queryGroup(cluster)?clusters.queryGroup(cluster)->ordinality():numParts(),n,d);
-            setReplicateFilename(path,d);
+            ClusterPartDiskMapSpec& mspec = clusters.item(cluster).queryPartDiskMapping();
+            mspec.calcPartLocation(partno,numParts(),rep,clusters.queryGroup(cluster)?clusters.queryGroup(cluster)->ordinality():numParts(),n,d);
+            if ((d>0) && (mspec.flags&CPDMSF_overloadedConfig) && mspec.defaultReplicateDir.length())
+                path.set(mspec.defaultReplicateDir.get());
+            else
+                setReplicateFilename(path,d);
         }
     }
 

dali/base/dafdesc.cpp

@@ -103,6 +103,8 @@ void ClusterPartDiskMapSpec::setRoxie (unsigned redundancy, unsigned channelsPer
     if ((channelsPerNode>1)&&(redundancy==0)) {
         flags |= CPDMSF_wrapToNextDrv;
         maxDrvs = channelsPerNode;
+        if (_replicateOffset == 1)
+            flags |= CPDMSF_overloadedConfig;
     }
     else
         maxDrvs = (redundancy>1)?(redundancy+1):2;

dali/base/dafdesc.hpp

@@ -96,6 +96,7 @@ public:
 #define CPDMSF_repeatedPart     (0x08)      // if repeated parts included 
 #define CPDMSF_defaultBaseDir   (0x10)      // set if defaultBaseDir present
 #define CPDMSF_defaultReplicateDir  (0x20)      // set if defaultBaseDir present
+#define CPDMSF_overloadedConfig  (0x40)      // set if overloaded mode
 
 
 // ==PART DESCRIPTOR ==============================================================================================

esp/services/ws_fs/ws_fsService.cpp

@@ -522,7 +522,7 @@ bool CFileSprayEx::ParseLogicalPath(const char * pLogicalPath, StringBuffer &tit
     return true;
 }
 
-DFUclusterPartDiskMapping readClusterMappingSettings(const char *cluster, StringBuffer &dir, int& offset)
+DFUclusterPartDiskMapping readClusterMappingSettings(const char *cluster, StringBuffer &dir, int& offset, ClusterPartDiskMapSpec& mspec)
 {
     DFUclusterPartDiskMapping mapping = DFUcpdm_c_only;
     Owned<IEnvironmentFactory> envFactory = getEnvironmentFactory();
@@ -546,6 +546,7 @@ DFUclusterPartDiskMapping readClusterMappingSettings(const char *cluster, String
             else if (!strnicmp(slaveConfig, "overloaded", 10))
             {
                 mapping = DFUcpdm_c_then_d;
+                mspec.setRoxie(0, processe.getCount("RoxieSlave[1]/RoxieChannel"));
             }
             else if (!strnicmp(slaveConfig, "full", 4))
             {
@@ -2397,11 +2398,13 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
         wuFSpecDest->setFileMask(fileMask.str());
         wuOptions->setOverwrite(req.getOverwrite());
 
+        ClusterPartDiskMapSpec mspec;
+        wuFSpecDest->getClusterPartDiskMapSpec(destCluster.str(), mspec);
         if (bRoxie)
         {
             int offset;
             StringBuffer baseDir;
-            DFUclusterPartDiskMapping val = readClusterMappingSettings(destCluster.str(), baseDir, offset);
+            DFUclusterPartDiskMapping val = readClusterMappingSettings(destCluster.str(), baseDir, offset, mspec);
             wuFSpecDest->setWrap(true);                             // roxie always wraps
             if(req.getCompress())
                 wuFSpecDest->setCompressed(true);
@@ -2456,8 +2459,6 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
                 wuOptions->setIfNewer(true);
         }
 
-        ClusterPartDiskMapSpec mspec;
-        wuFSpecDest->getClusterPartDiskMapSpec(destCluster.str(), mspec);
         mspec.setDefaultBaseDir(defaultFolder.str());
         mspec.setDefaultReplicateDir(defaultReplicateFolder.str());
         wuFSpecDest->setClusterPartDiskMapSpec(destCluster.str(), mspec);