HPCC-27204 Support planes with multiple devices (striping)

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

common/thorhelper/roxiehelper.cpp

@@ -26,6 +26,7 @@
 #include "jfile.hpp"
 #include "mpbase.hpp"
 #include "dafdesc.hpp"
+#include "dautils.hpp"
 #include "dadfs.hpp"
 #include "zcrypt.hpp"
 
@@ -2399,7 +2400,10 @@ void ClusterWriteHandler::getPhysicalName(StringBuffer & name, const char * clus
 {
     Owned<IStoragePlane> plane = getDataStoragePlane(cluster, false);
     const char * prefix = plane ? plane->queryPrefix() : nullptr;
-    makePhysicalPartName(logicalName.get(), 1, 1, name, 0, DFD_OSdefault, prefix, false);
+    unsigned stripeNum = 0;
+    if (plane)
+        stripeNum = calcStripeNumber(0, logicalName.get(), plane->numDevices());
+    makePhysicalPartName(logicalName.get(), 1, 1, name, 0, DFD_OSdefault, prefix, false, stripeNum);
 }
 
 void ClusterWriteHandler::addCluster(char const * cluster)
@@ -2454,16 +2458,25 @@ void ClusterWriteHandler::getLocalPhysicalFilename(StringAttr & out) const
     PROGLOG("%s(CLUSTER) for logical filename %s writing to local file %s", activityType.get(), logicalName.get(), out.get());
 }
 
-void ClusterWriteHandler::splitPhysicalFilename(StringBuffer & dir, StringBuffer & base) const
+void ClusterWriteHandler::getDirAndFilename(StringBuffer & dir, StringBuffer & filename) const
 {
 #ifdef _CONTAINERIZED
     assertex(localClusterName.length());
 #endif
-    StringBuffer physicalName, physicalDir, physicalBase;
-    getPhysicalName(physicalName, localClusterName);
-    splitFilename(physicalName, &physicalDir, &physicalDir, &physicalBase, &physicalBase);
-    dir.append(physicalDir);
-    base.append(physicalBase);
+    Owned<IStoragePlane> plane = getDataStoragePlane(localClusterName, false);
+    unsigned stripeNum = 0;
+    const char *prefix = nullptr;
+    if (plane)
+    {
+        stripeNum = calcStripeNumber(0, logicalName.get(), plane->numDevices());
+        prefix = plane->queryPrefix();
+    }
+
+    makePhysicalDirectory(dir, logicalName.get(), 0, DFD_OSdefault, prefix);
+
+    StringBuffer fullPath;
+    makePhysicalPartName(logicalName.get(), 1, 1, fullPath, 0, DFD_OSdefault, prefix, false, stripeNum);
+    splitFilename(fullPath, nullptr, nullptr, &filename, &filename);
 }
 
 void ClusterWriteHandler::getTempFilename(StringAttr & out) const

common/thorhelper/roxiehelper.hpp

@@ -542,7 +542,7 @@ public:
     ClusterWriteHandler(char const * _logicalName, char const * _activityType);
     void addCluster(char const * cluster);
     void getLocalPhysicalFilename(StringAttr & out) const;
-    void splitPhysicalFilename(StringBuffer & dir, StringBuffer & base) const;
+    void getDirAndFilename(StringBuffer & dir, StringBuffer & filename) const;
     void copyPhysical(IFile * source, bool noCopy) const;
     void setDescriptorParts(IFileDescriptor * desc, char const * basename, IPropertyTree * attrs) const;
     void finish(IFile * file) const;

dali/base/dadfs.cpp

@@ -4311,12 +4311,19 @@ public:
         newdir.append(baseDir).append(newPath);
         StringBuffer fullname;
         CIArrayOf<CIStringArray> newNames;
+        unsigned existingLfnHash = queryAttributes().getPropInt("@lfnHash");
         unsigned i;
-        for (i=0;i<width;i++) {
+        for (i=0;i<width;i++)
+        {
             newNames.append(*new CIStringArray);
             CDistributedFilePart &part = parts.item(i);
-            for (unsigned copy=0; copy<part.numCopies(); copy++) {
-                makePhysicalPartName(newname, i+1, width, newPath.clear(), 0, os, myBase, hasDirPerPart());
+            for (unsigned copy=0; copy<part.numCopies(); copy++)
+            {
+                unsigned cn = copyClusterNum(i, copy, nullptr);
+                unsigned numStripedDevices = queryPartDiskMapping(cn).numStripedDevices;
+                unsigned stripeNum = calcStripeNumber(i, existingLfnHash, numStripedDevices);
+
+                makePhysicalPartName(newname, i+1, width, newPath.clear(), 0, os, myBase, hasDirPerPart(), stripeNum);
                 newPath.remove(0, strlen(myBase));
 
                 StringBuffer copyDir(baseDir);
@@ -4519,11 +4526,8 @@ public:
             root->setProp("@partmask",newmask.str());
             partmask.set(newmask.str());
             directory.set(newdir.str());
-            StringBuffer mask;
-            for (unsigned i=0;i<width;i++) {
-                mask.appendf("Part[%d]/@name",i+1);
+            for (unsigned i=0;i<width;i++)
                 parts.item(i).clearOverrideName();
-            }
             savePartsAttr(false);
         }
         else {
@@ -6951,6 +6955,13 @@ unsigned CDistributedFilePart::copyClusterNum(unsigned copy,unsigned *replicate)
 StringBuffer &CDistributedFilePart::getPartDirectory(StringBuffer &ret,unsigned copy)
 {
     const char *defdir = parent.queryDefaultDir();
+    StringBuffer stripeDir;
+    unsigned cn = copyClusterNum(copy, nullptr);
+    unsigned numStripedDevices = parent.queryPartDiskMapping(cn).numStripedDevices;
+    unsigned lfnHash = parent.queryRoot()->getPropInt("Attr/@lfnHash");
+    addStripeDirectory(stripeDir, defdir, parent.clusters.item(cn).queryGroupName(), partIndex, lfnHash, numStripedDevices);
+    if (stripeDir.isEmpty())
+        stripeDir.append(defdir);
     StringBuffer dir;
     const char *pn;
     if (overridename.isEmpty())
@@ -6966,11 +6977,11 @@ StringBuffer &CDistributedFilePart::getPartDirectory(StringBuffer &ret,unsigned
         if (odir.length()) {
             if (isAbsolutePath(pn))
                 dir.append(odir);
-            else if (defdir&&*defdir)
-                addPathSepChar(dir.append(defdir)).append(odir);
+            else if (!stripeDir.isEmpty())
+                addPathSepChar(dir.append(stripeDir)).append(odir);
         }
         else
-            dir.append(defdir);
+            dir.append(stripeDir);
     }
     if (dir.length()==0)
         IERRLOG("IDistributedFilePart::getPartDirectory unable to determine part directory");

dali/base/dafdesc.cpp

@@ -200,6 +200,7 @@ void ClusterPartDiskMapSpec::toProp(IPropertyTree *tree)
     setPropDef(tree,"@interleave",interleave,0);
     setPropDef(tree,"@mapFlags",flags,0);
     setPropDef(tree,"@repeatedPart",repeatedPart,(int)CPDMSRP_notRepeated);
+    setPropDef(tree, "@numStripedDevices", numStripedDevices, 1);
     if (defaultBaseDir.isEmpty())
         tree->removeProp("@defaultBaseDir");
     else
@@ -228,6 +229,7 @@ void ClusterPartDiskMapSpec::fromProp(IPropertyTree *tree)
     interleave = getPropDef(tree,"@interleave",0);
     flags = (byte)getPropDef(tree,"@mapFlags",0);
     repeatedPart = (unsigned)getPropDef(tree,"@repeatedPart",(int)CPDMSRP_notRepeated);
+    numStripedDevices = (unsigned)getPropDef(tree, "@numStripedDevices", 1);
     setDefaultBaseDir(tree->queryProp("@defaultBaseDir"));
     setDefaultReplicateDir(tree->queryProp("@defaultReplicateDir"));
 }
@@ -246,6 +248,8 @@ void ClusterPartDiskMapSpec::serialize(MemoryBuffer &mb)
         mb.append(defaultBaseDir);
     if (flags&CPDMSF_defaultReplicateDir)
         mb.append(defaultReplicateDir);
+    if (flags&CPDMSF_striped)
+        mb.append(numStripedDevices);
 }
 
 void ClusterPartDiskMapSpec::deserialize(MemoryBuffer &mb)
@@ -268,6 +272,8 @@ void ClusterPartDiskMapSpec::deserialize(MemoryBuffer &mb)
         mb.read(defaultReplicateDir);
     else
         defaultReplicateDir.clear();
+    if (flags&CPDMSF_striped)
+        mb.read(numStripedDevices);
 }
 
 
@@ -398,7 +404,23 @@ struct CClusterInfo: implements IClusterInfo, public CInterface
                 name.set(gname);
         }
     }
-
+    void checkStriped()
+    {
+        if (!name.isEmpty())
+        {
+            Owned<IStoragePlane> plane = getDataStoragePlane(name, false);
+#ifdef _CONTAINERIZED
+            mspec.numStripedDevices = plane ? plane->numDevices() : 1;
+            if (mspec.numStripedDevices>1)
+                mspec.flags |= CPDMSF_striped;
+            else
+                mspec.flags &= ~CPDMSF_striped;
+#else
+            // Bare-metal can have multiple devices per plane (e.g. data + mirror), but it doesn't stripe across them
+            mspec.numStripedDevices = 1;
+#endif
+        }
+    }
 public:
     IMPLEMENT_IINTERFACE;
     CClusterInfo(MemoryBuffer &mb,INamedGroupStore *resolver)
@@ -418,6 +440,7 @@ public:
         name.toLowerCase();
         mspec =_mspec;
         checkClusterName(resolver);
+        checkStriped();
     }
     CClusterInfo(IPropertyTree *pt,INamedGroupStore *resolver,unsigned flags)
     {
@@ -442,6 +465,7 @@ public:
         }
         else
             checkClusterName(resolver);
+        checkStriped();
     }
 
     const char *queryGroupName()
@@ -586,6 +610,7 @@ protected:
 public:
 
     StringAttr tracename;
+    unsigned lfnHash = 0;
     IArrayOf<IClusterInfo> clusters;
 
     Owned<IPropertyTree> attr;
@@ -930,6 +955,54 @@ void getClusterInfo(IPropertyTree &pt, INamedGroupStore *resolver, unsigned flag
     }
 }
 
+inline bool validFNameChar(char c)
+{
+    static const char *invalids = "*\"/:<>?\\|";
+    return (c>=32 && c<127 && !strchr(invalids, c));
+}
+
+inline const char *skipRoot(const char *lname)
+{
+    for (;;) {
+        while (*lname==' ')
+            lname++;
+        if (*lname!='.')
+            break;
+        const char *s = lname+1;
+        while (*s==' ')
+            s++;
+        if (!*s)
+            lname = s;
+        else if ((s[0]==':')&&(s[1]==':'))
+            lname = s+2;
+        else
+            break;
+    }
+    return lname;
+}
+
+// returns position in result buffer of tail lfn name
+static size32_t translateLFNToPath(StringBuffer &result, const char *lname, char pathSep)
+{
+    lname = skipRoot(lname);
+    char c;
+    size32_t l = result.length();
+    while ((c=*(lname++))!=0)
+    {
+        if ((c==':')&&(*lname==':'))
+        {
+            lname++;
+            result.clip().append(pathSep);
+            l = result.length();
+            lname = skipRoot(lname);
+        }
+        else if (validFNameChar(c))
+            result.append((char)tolower(c));
+        else
+            result.appendf("%%%.2X", (int) c);
+    }
+    return l;
+}
 
 class CFileDescriptor:  public CFileDescriptorBase, implements ISuperFileDescriptor
 {
@@ -1128,6 +1201,13 @@ class CFileDescriptor:  public CFileDescriptorBase, implements ISuperFileDescrip
     {
         unsigned n = numParts();
         if (idx<n) {
+            StringBuffer stripeDir;
+            unsigned lcopy;
+            IClusterInfo * cluster = queryCluster(idx,copy,lcopy);
+            if (cluster)
+                addStripeDirectory(stripeDir, directory, cluster->queryGroupName(), idx, lfnHash, cluster->queryPartDiskMapping().numStripedDevices);
+            if (stripeDir.isEmpty())
+                stripeDir.append(directory);
             StringBuffer fullpath;
             StringBuffer tmp1;
             CPartDescriptor &pt = *part(idx);
@@ -1138,23 +1218,23 @@ class CFileDescriptor:  public CFileDescriptorBase, implements ISuperFileDescrip
                     StringBuffer tmpon; // bit messy but need to ensure dir put back on before removing!
                     const char *on = pt.overridename.get();
                     if (on&&*on&&!isAbsolutePath(on)&&!directory.isEmpty())
-                        on = addPathSepChar(tmpon.append(directory)).append(on).str();
+                        on = addPathSepChar(tmpon.append(stripeDir)).append(on).str();
                     StringBuffer tmp2;
                     splitDirMultiTail(on,tmp1,tmp2);
                 }
                 else
                     splitDirTail(pt.overridename,tmp1);
-                if (directory.isEmpty()||(isAbsolutePath(tmp1.str())||(stdIoHandle(tmp1.str())>=0)))
+                if (stripeDir.isEmpty()||(isAbsolutePath(tmp1.str())||(stdIoHandle(tmp1.str())>=0)))
                     fullpath.swapWith(tmp1);
                 else {
-                    fullpath.append(directory);
+                    fullpath.append(stripeDir);
                     if (fullpath.length())
                         addPathSepChar(fullpath);
                     fullpath.append(tmp1);
                 }
             }
             else if (!partmask.isEmpty()) {
-                fullpath.append(directory);
+                fullpath.append(stripeDir);
                 if (containsPathSepChar(partmask)) {
                     if (fullpath.length())
                         addPathSepChar(fullpath);
@@ -1162,11 +1242,9 @@ class CFileDescriptor:  public CFileDescriptorBase, implements ISuperFileDescrip
                 }
             }
             else
-                fullpath.append(directory);
+                fullpath.append(stripeDir);
             replaceClusterDir(idx,copy, fullpath);
             StringBuffer baseDir, repDir;
-            unsigned lcopy;
-            IClusterInfo * cluster = queryCluster(idx,copy,lcopy);
             if (cluster)
             {
                 DFD_OS os = SepCharBaseOs(getPathSepChar(fullpath));
@@ -1337,7 +1415,9 @@ public:
             }
         }
         attr.setown(createPTree(mb));
-        if (!attr)
+        if (attr)
+            lfnHash = attr->getPropInt("@lfnHash");
+        else
             attr.setown(createPTree("Attr")); // doubt can happen
         fileFlags = static_cast<FileDescriptorFlags>(attr->getPropInt("@flags"));
         if (version == SERIALIZATION_VERSION2)
@@ -1447,7 +1527,13 @@ public:
             attr.setown(createPTreeFromIPT(at));
         else
             attr.setown(createPTree("Attr"));
-
+        if (attr->hasProp("@lfnHash")) // potentially missing for meta coming from a legacy Dali
+            lfnHash = attr->getPropInt("@lfnHash");
+        else if (tracename.length())
+        {
+            lfnHash = getFilenameHash(tracename.length(), tracename.str());
+            attr->setPropInt("@lfnHash", lfnHash);
+        }
         if (totalsize!=(offset_t)-1)
             attr->setPropInt64("@size",totalsize);
     }
@@ -1472,8 +1558,8 @@ public:
     void serializeTree(IPropertyTree &pt,unsigned flags)
     {
         closePending();
-//      if (!tracename.isEmpty())
-//          pt.setProp("@trace",tracename);             // don't include trace name in tree (may revisit later)
+        if (!tracename.isEmpty())
+            pt.setProp("@trace",tracename);
         if (!directory.isEmpty())
             pt.setProp("@directory",directory);
         if (!partmask.isEmpty())
@@ -1637,11 +1723,17 @@ public:
         doSetPart(idx,ep,localname.str(),pt);
     }
 
-
-
     void setTraceName(const char *trc)
     {
-        tracename.set(trc);
+        CDfsLogicalFileName logicalName;
+        logicalName.setAllowWild(true); // for cases where IFileDescriptor used to point to external files (e.g. during spraying)
+        logicalName.set(trc); // normalize
+        tracename.set(logicalName.get());
+        if (!queryProperties().hasProp("@lfnHash"))
+        {
+            lfnHash = hashc((const unsigned char *)tracename.str(), tracename.length(), 0);
+            queryProperties().setPropInt("@lfnHash", lfnHash);
+        }
     }
 
     unsigned numClusterCopies(unsigned cnum,unsigned partnum)
@@ -2323,30 +2415,6 @@ IPartDescriptor *deserializePartFileDescriptor(MemoryBuffer &mb)
     return LINK(&parts.item(0));
 }
 
-
-IFileDescriptor *createFileDescriptor(const char *lname,IGroup *grp,IPropertyTree *tree,DFD_OS os,unsigned width)
-{
-    // only handles 1 copy
-    IFileDescriptor *res = createFileDescriptor(tree);
-    res->setTraceName(lname);
-    StringBuffer dir;
-    getLFNDirectoryUsingDefaultBaseDir(dir, lname, os);
-    res->setDefaultDir(dir.str());
-    if (width==0)
-        width = grp->ordinality();
-    StringBuffer s;
-    for (unsigned i=0;i<width;i++) {
-        makePhysicalPartName(lname, i+1, width, s.clear(), 0, os, nullptr, false);
-        RemoteFilename rfn;
-        rfn.setPath(grp->queryNode(i%grp->ordinality()).endpoint(),s.str());
-        res->setPart(i,rfn,NULL);
-    }
-    ClusterPartDiskMapSpec map; // use defaults
-    map.defaultCopies = DFD_DefaultCopies;
-    res->endCluster(map);
-    return res;
-}
-
 IFileDescriptor *createFileDescriptor(const char *lname, const char *clusterType, const char *groupName, IGroup *group)
 {
     StringBuffer partMask;
@@ -2360,6 +2428,7 @@ IFileDescriptor *createFileDescriptor(const char *lname, const char *clusterType
         getLFNDirectoryUsingBaseDir(curDir, lname, defaultDir.str());
 
     Owned<IFileDescriptor> fileDesc = createFileDescriptor();
+    fileDesc->setTraceName(lname);
     fileDesc->setNumParts(parts);
     fileDesc->setPartMask(partMask);
     fileDesc->setDefaultDir(curDir);
@@ -2382,6 +2451,7 @@ IFileDescriptor *createFileDescriptor(const char *lname, const char *planeName,
     getLFNDirectoryUsingBaseDir(dir, lname, plane->queryPrefix());
 
     Owned<IFileDescriptor> fileDesc = createFileDescriptor();
+    fileDesc->setTraceName(lname);
     fileDesc->setNumParts(numParts);
     fileDesc->setPartMask(partMask);
     fileDesc->setDefaultDir(dir);
@@ -2404,12 +2474,6 @@ IFileDescriptor *deserializeFileDescriptorTree(IPropertyTree *tree, INamedGroupS
     return new CFileDescriptor(tree, resolver, flags);
 }
 
-inline bool validFNameChar(char c)
-{
-    static const char *invalids = "*\"/:<>?\\|";
-    return (c>=32 && c<127 && !strchr(invalids, c));
-}
-
 static const char * defaultWindowsBaseDirectories[__grp_size][MAX_REPLICATION_LEVELS] =
     {
             { "c:\\thordata", "d:\\thordata" },
@@ -2635,50 +2699,7 @@ StringBuffer &getPartMask(StringBuffer &ret,const char *lname,unsigned partmax)
     return ret;
 }
 
-inline const char *skipRoot(const char *lname)
-{
-    for (;;) {
-        while (*lname==' ')
-            lname++;
-        if (*lname!='.')
-            break;
-        const char *s = lname+1;
-        while (*s==' ')
-            s++;
-        if (!*s)
-            lname = s;
-        else if ((s[0]==':')&&(s[1]==':'))
-            lname = s+2;
-        else
-            break;
-    }
-    return lname;
-}
-
-// returns position in result buffer of tail lfn name
-static size32_t translateLFNToPath(StringBuffer &result, const char *lname, char pathSep)
-{
-    lname = skipRoot(lname);
-    char c;
-    size32_t l = result.length();
-    while ((c=*(lname++))!=0)
-    {
-        if ((c==':')&&(*lname==':'))
-        {
-            lname++;
-            result.clip().append(pathSep);
-            l = result.length();
-            lname = skipRoot(lname);
-        }
-        else if (validFNameChar(c))
-            result.append((char)tolower(c));
-        else
-            result.appendf("%%%.2X", (int) c);
-    }
-    return l;
-}
-
-StringBuffer &makePhysicalPartName(const char *lname, unsigned partno, unsigned partmax, StringBuffer &result, unsigned replicateLevel, DFD_OS os,const char *diroverride,bool dirPerPart)
+StringBuffer &makePhysicalPartName(const char *lname, unsigned partno, unsigned partmax, StringBuffer &result, unsigned replicateLevel, DFD_OS os,const char *diroverride,bool dirPerPart,unsigned stripeNum)
 {
     assertex(lname);
     if (strstr(lname,"::>")) { // probably query
@@ -2721,6 +2742,9 @@ StringBuffer &makePhysicalPartName(const char *lname, unsigned partno, unsigned
         l++;
     }
 
+    if (stripeNum)
+        result.append('d').append(stripeNum).append(OsSepChar(os));
+
     l = translateLFNToPath(result, lname, OsSepChar(os));
 
     char c;
@@ -2765,11 +2789,17 @@ StringBuffer &makePhysicalPartName(const char *lname, unsigned partno, unsigned
     return result.clip();
 }
 
+StringBuffer &makePhysicalDirectory(StringBuffer &result, const char *lname, unsigned replicateLevel, DFD_OS os,const char *diroverride)
+{
+    return makePhysicalPartName(lname, 0, 0, result, replicateLevel, os, diroverride, false, 0);
+}
+
+
 StringBuffer &makeSinglePhysicalPartName(const char *lname, StringBuffer &result, bool allowospath, bool &wasdfs,const char *diroverride)
 {
     wasdfs = !(allowospath&&(isAbsolutePath(lname)||(stdIoHandle(lname)>=0)));
     if (wasdfs)
-        return makePhysicalPartName(lname, 1, 1, result, 0, DFD_OSdefault, diroverride, false);
+        return makePhysicalPartName(lname, 1, 1, result, 0, DFD_OSdefault, diroverride, false, 0);
     return result.append(lname);
 }
 
@@ -3529,21 +3559,27 @@ void initializeStorageGroups(bool createPlanesFromGroups)
 bool getDefaultStoragePlane(StringBuffer &ret)
 {
 #ifdef _CONTAINERIZED
-    // If the plane is specified for the component, then use that
-    if (getComponentConfigSP()->getProp("@dataPlane", ret))
+    if (getDefaultPlane(ret, "@dataPlane", "data"))
         return true;
 
-    //Otherwise check what the default plane for data storage is configured to be
-    Owned<IPropertyTreeIterator> dataPlanes = getGlobalConfigSP()->getElements("storage/planes[@category='data']");
-    if (dataPlanes->first())
-        return dataPlanes->query().getProp("@name", ret);
-
     throwUnexpectedX("Default data plane not specified"); // The default should always have been configured by the helm charts
 #else
     return false;
 #endif
 }
 
+bool getDefaultSpillPlane(StringBuffer &ret)
+{
+#ifdef _CONTAINERIZED
+    if (getDefaultPlane(ret, "@spillPlane", "spill"))
+        return true;
+
+    throwUnexpectedX("Default spill plane not specified"); // The default should always have been configured by the helm charts
+#else
+    return false;
+#endif
+}
+
 //---------------------------------------------------------------------------------------------------------------------
 
 class CStoragePlaneInfo : public CInterfaceOf<IStoragePlane>

dali/base/dafdesc.hpp

@@ -80,6 +80,7 @@ public:
                             // ( | CPDMSRP_onlyRepeated for *only* repeats )
     StringAttr defaultBaseDir; // if set overrides *base* directory (i.e. /c$/dir/x/y becomes odir/x/y)
     StringAttr defaultReplicateDir;
+    unsigned numStripedDevices = 1;
 
     void setRoxie (unsigned redundancy, unsigned channelsPerNode, int replicateOffset=1);
     void setRepeatedCopies(unsigned partnum,bool onlyrepeats);
@@ -98,13 +99,14 @@ public:
     bool isReplicated() const;
 };
 
-#define CPDMSF_wrapToNextDrv    (0x01)      // whether should wrap to next drv
-#define CPDMSF_fillWidth        (0x02)      // replicate copies fill cluster serially (when num parts < clusterwidth/2)
-#define CPDMSF_packParts        (0x04)      // whether to save parts as binary
-#define CPDMSF_repeatedPart     (0x08)      // if repeated parts included
-#define CPDMSF_defaultBaseDir   (0x10)      // set if defaultBaseDir present
-#define CPDMSF_defaultReplicateDir  (0x20)      // set if defaultBaseDir present
-#define CPDMSF_overloadedConfig  (0x40)      // set if overloaded mode
+#define CPDMSF_wrapToNextDrv       (0x01) // whether should wrap to next drv
+#define CPDMSF_fillWidth           (0x02) // replicate copies fill cluster serially (when num parts < clusterwidth/2)
+#define CPDMSF_packParts           (0x04) // whether to save parts as binary
+#define CPDMSF_repeatedPart        (0x08) // if repeated parts included
+#define CPDMSF_defaultBaseDir      (0x10) // set if defaultBaseDir present
+#define CPDMSF_defaultReplicateDir (0x20) // set if defaultBaseDir present
+#define CPDMSF_overloadedConfig    (0x40) // set if overloaded mode
+#define CPDMSF_striped             (0x80) // set if parts striped over multiple devices
 
 
 // ==PART DESCRIPTOR ==============================================================================================
@@ -326,13 +328,16 @@ extern da_decl StringBuffer &makePhysicalPartName(
                                 unsigned replicateLevel,            // uses replication directory
                                 DFD_OS os,                          // os must be specified if no dir specified
                                 const char *diroverride,            // override default directory
-                                bool dirPerPart);                   // generate a subdirectory per part
+                                bool dirPerPart,                    // generate a subdirectory per part
+                                unsigned stripeNum);                // stripe number
 extern da_decl StringBuffer &makeSinglePhysicalPartName(const char *lname, // single part file
                                                         StringBuffer &result,
                                                         bool allowospath,   // allow an OS (absolute) file path
                                                         bool &wasdfs,       // not OS path
                                                         const char *diroverride=NULL
                                                         );
+extern da_decl StringBuffer &makePhysicalDirectory(StringBuffer &result, const char *lname, unsigned replicateLevel, DFD_OS os,const char *diroverride);
+
 // 
 extern da_decl StringBuffer &getLFNDirectoryUsingBaseDir(StringBuffer &result, const char *lname, const char *baseDir);
 extern da_decl StringBuffer &getLFNDirectoryUsingDefaultBaseDir(StringBuffer &result, const char *lname, DFD_OS os);
@@ -347,6 +352,7 @@ extern da_decl bool setReplicateDir(const char *name,StringBuffer &out, bool isr
 
 extern da_decl void initializeStorageGroups(bool createPlanesFromGroups);
 extern da_decl bool getDefaultStoragePlane(StringBuffer &ret);
+extern da_decl bool getDefaultSpillPlane(StringBuffer &ret);
 extern da_decl IStoragePlane * getDataStoragePlane(const char * name, bool required);
 extern da_decl IStoragePlane * getRemoteStoragePlane(const char * name, bool required);
 
@@ -358,7 +364,6 @@ extern da_decl IFileDescriptor *getExternalFileDescriptor(const char *logicalnam
 extern da_decl ISuperFileDescriptor *createSuperFileDescriptor(IPropertyTree *attr);        // ownership of attr tree is taken
 extern da_decl IFileDescriptor *deserializeFileDescriptor(MemoryBuffer &mb);
 extern da_decl IFileDescriptor *deserializeFileDescriptorTree(IPropertyTree *tree, INamedGroupStore *resolver=NULL, unsigned flags=0);  // flags IFDSF_*
-extern da_decl IFileDescriptor *createFileDescriptor(const char *lname,IGroup *grp,IPropertyTree *tree,DFD_OS os=DFD_OSdefault,unsigned width=0);  // creates default
 extern da_decl IPartDescriptor *deserializePartFileDescriptor(MemoryBuffer &mb);
 extern da_decl void deserializePartFileDescriptors(MemoryBuffer &mb,IArrayOf<IPartDescriptor> &parts);
 extern da_decl IFileDescriptor *createFileDescriptor(const char *lname, const char *planeName, unsigned numParts);
@@ -400,5 +405,4 @@ inline DFD_OS SepCharBaseOs(char c)
 
 extern da_decl void extractFilePartInfo(IPropertyTree &info, IFileDescriptor &file);
 
-
 #endif

dali/base/dautils.cpp

@@ -3240,6 +3240,7 @@ class CLocalOrDistributedFile: implements ILocalOrDistributedFile, public CInter
     Owned<IDistributedFile> dfile;
     CDfsLogicalFileName lfn;    // set if localpath but prob not useful
     StringAttr localpath;
+    StringAttr fileDescPath;
 public:
     IMPLEMENT_IINTERFACE;
     CLocalOrDistributedFile()
@@ -3247,11 +3248,12 @@ public:
         fileExists = false;
     }
 
-    const char *queryLogicalName()
+    virtual const char *queryLogicalName() override
     {
         return lfn.get();
     }
-    IDistributedFile * queryDistributedFile() 
+
+    virtual IDistributedFile * queryDistributedFile() override
     { 
         return dfile.get(); 
     }
@@ -3309,6 +3311,7 @@ public:
             }
 
             StringBuffer dir;
+            unsigned stripeNum = 0;
 #ifdef _CONTAINERIZED
             StringBuffer cluster;
             if (clusters)
@@ -3321,10 +3324,16 @@ public:
                 getDefaultStoragePlane(cluster);
             Owned<IStoragePlane> plane = getDataStoragePlane(cluster, true);
             dir.append(plane->queryPrefix());
+            unsigned numStripedDevices = plane->numDevices();
+            stripeNum = calcStripeNumber(0, lfn.get(), numStripedDevices);
 #endif
+            StringBuffer descPath;
+            makePhysicalDirectory(descPath, lfn.get(), 0, DFD_OSdefault, dir);
+            fileDescPath.set(descPath);
+
             // MORE - should we create the IDistributedFile here ready for publishing (and/or to make sure it's locked while we write)?
             StringBuffer physicalPath;
-            makePhysicalPartName(lfn.get(), 1, 1, physicalPath, 0, DFD_OSdefault, dir, false); // more - may need to override path for roxie
+            makePhysicalPartName(lfn.get(), 1, 1, physicalPath, 0, DFD_OSdefault, dir, false, stripeNum); // more - may need to override path for roxie
             localpath.set(physicalPath);
             fileExists = (dfile != NULL);
             return write;
@@ -3332,11 +3341,12 @@ public:
         return false;
     }
 
-    IFileDescriptor *getFileDescriptor()
+    virtual IFileDescriptor *getFileDescriptor() override
     {
         if (dfile.get())
             return dfile->getFileDescriptor();
         Owned<IFileDescriptor> fileDesc = createFileDescriptor();
+        fileDesc->setTraceName(lfn.get());
         StringBuffer dir;
         if (localpath.isEmpty()) { // e.g. external file
             StringBuffer tail;
@@ -3354,7 +3364,7 @@ public:
             }
         }
         else 
-            splitDirTail(localpath,dir);
+            splitDirTail(fileDescPath,dir);
         fileDesc->setDefaultDir(dir.str());
         RemoteFilename rfn;
         getPartFilename(rfn,0,0);
@@ -3363,7 +3373,7 @@ public:
         return fileDesc.getClear();
     }
 
-    bool getModificationTime(CDateTime &dt)
+    virtual bool getModificationTime(CDateTime &dt) override
     {
         if (dfile.get())
             return dfile->getModificationTime(dt);
@@ -3375,22 +3385,21 @@ public:
         return false;
     }
 
-    virtual unsigned numParts()
+    virtual unsigned numParts() override 
     {
         if (dfile.get()) 
             return dfile->numParts();
         return 1;
     }
 
-
-    unsigned numPartCopies(unsigned partnum)
+    virtual unsigned numPartCopies(unsigned partnum) override
     {
         if (dfile.get()) 
             return dfile->queryPart(partnum).numCopies();
         return 1;
     }
     
-    IFile *getPartFile(unsigned partnum,unsigned copy)
+    virtual IFile *getPartFile(unsigned partnum,unsigned copy) override
     {
         RemoteFilename rfn;
         if ((partnum==0)&&(copy==0))
@@ -3398,7 +3407,29 @@ public:
         return NULL;
     }
     
-    RemoteFilename &getPartFilename(RemoteFilename &rfn, unsigned partnum,unsigned copy)
+    virtual void getDirAndFilename(StringBuffer &dir, StringBuffer &filename) override
+    {
+        if (dfile.get())
+        {
+            dir.append(dfile->queryDefaultDir());
+            splitFilename(localpath, nullptr, nullptr, &filename, &filename);
+        }
+        else if (localpath.isEmpty())
+        {
+            RemoteFilename rfn;
+            lfn.getExternalFilename(rfn);
+            StringBuffer fullPath;
+            rfn.getLocalPath(fullPath);
+            splitFilename(localpath, nullptr, &dir, &filename, &filename);
+        }
+        else
+        {
+            dir.append(fileDescPath);
+            splitFilename(localpath, nullptr, nullptr, &filename, &filename);
+        }
+    }
+
+    virtual RemoteFilename &getPartFilename(RemoteFilename &rfn, unsigned partnum,unsigned copy) override
     {
         if (dfile.get()) 
             dfile->queryPart(partnum).getFilename(rfn,copy);
@@ -3425,7 +3456,7 @@ public:
         return path;
     }
 
-    bool getPartCrc(unsigned partnum, unsigned &crc)
+    virtual bool getPartCrc(unsigned partnum, unsigned &crc) override
     {
         if (dfile.get())  
             return dfile->queryPart(partnum).getCrc(crc);
@@ -3437,7 +3468,7 @@ public:
         return false;
     }
 
-    offset_t getPartFileSize(unsigned partnum)
+    virtual offset_t getPartFileSize(unsigned partnum) override
     {
         if (dfile.get()) 
             return dfile->queryPart(partnum).getFileSize(true,false);
@@ -3447,7 +3478,7 @@ public:
         return (offset_t)-1;
     }
 
-    offset_t getFileSize()
+    virtual offset_t getFileSize() override
     {
         if (dfile.get())
             dfile->getFileSize(true,false);
@@ -3458,16 +3489,15 @@ public:
         return ret;
     }
 
-    virtual bool exists() const
+    virtual bool exists() const override
     {
         return fileExists;
     }
 
-    virtual bool isExternal() const
+    virtual bool isExternal() const override
     {
         return lfn.isExternal();
     }
-
 };
 
 ILocalOrDistributedFile* createLocalOrDistributedFile(const char *fname,IUserDescriptor *user,bool onlylocal,bool onlydfs, bool iswrite, bool isPrivilegedUser, const StringArray *clusters)
@@ -3726,3 +3756,30 @@ extern da_decl IRemoteConnection* connectXPathOrFile(const char* path, bool safe
         xpath.append(path);
     return conn.getClear();
 }
+
+void addStripeDirectory(StringBuffer &out, const char *directory, const char *planeName, unsigned partNum, unsigned lfnHash, unsigned numStripes)
+{
+    if (numStripes <= 1)
+        return;
+    /* 'directory' is the prefix+logical file path, we need to know
+    * the plane prefix to manipulate it and insert the stripe directory.
+    */
+    Owned<IStoragePlane> plane = getDataStoragePlane(planeName, false);
+    if (plane)
+    {
+        const char *planePrefix = plane->queryPrefix();
+        if (!isEmptyString(planePrefix))
+        {
+            assertex(startsWith(directory, planePrefix));
+            const char *tail = directory+strlen(planePrefix);
+            if (isPathSepChar(*tail))
+                tail++;
+            out.append(planePrefix);
+            assertex(lfnHash);
+            unsigned stripeNum = calcStripeNumber(partNum, lfnHash, numStripes);
+            addPathSepChar(out).append('d').append(stripeNum);
+            if (*tail)
+                addPathSepChar(out).append(tail);
+        }
+    }
+}

dali/base/dautils.hpp

@@ -439,6 +439,7 @@ interface ILocalOrDistributedFile: extends IInterface
     virtual unsigned numParts() = 0;
     virtual unsigned numPartCopies(unsigned partnum) = 0;
     virtual IFile *getPartFile(unsigned partnum,unsigned copy=0) = 0;
+    virtual void getDirAndFilename(StringBuffer &dir, StringBuffer &filename) = 0;
     virtual RemoteFilename &getPartFilename(RemoteFilename &rfn, unsigned partnum,unsigned copy=0) = 0;
     virtual offset_t getPartFileSize(unsigned partnum)=0;   // NB expanded size             
     virtual bool getPartCrc(unsigned partnum, unsigned &crc) = 0;
@@ -544,5 +545,26 @@ extern da_decl void setPageCacheTimeoutMilliSeconds(unsigned timeoutSeconds);
 extern da_decl void setMaxPageCacheItems(unsigned _maxPageCacheItems);
 extern da_decl IRemoteConnection* connectXPathOrFile(const char* path, bool safe, StringBuffer& xpath);
 extern da_decl bool expandExternalPath(StringBuffer &dir, StringBuffer &tail, const char * filename, const char * s, bool iswin, IException **e);
-
+extern da_decl void addStripeDirectory(StringBuffer &out, const char *directory, const char *planeName, unsigned partNum, unsigned lfnHash, unsigned numStripes);
+inline unsigned getFilenameHash(size32_t len, const char *filename)
+{
+    return hashc((const unsigned char *)filename, len, 0);
+}
+inline unsigned getFilenameHash(const char *filename)
+{
+    return getFilenameHash(strlen(filename), filename);
+}
+inline unsigned calcStripeNumber(unsigned partNum, unsigned lfnHash, unsigned numStripes)
+{
+    if (numStripes <= 1)
+        return 0;
+    return ((partNum+lfnHash)%numStripes)+1;
+}
+inline unsigned calcStripeNumber(unsigned partNum, const char *lfnName, unsigned numStripes)
+{
+    if (numStripes <= 1)
+        return 0;
+    unsigned lfnHash = getFilenameHash(lfnName);
+    return ((partNum+lfnHash)%numStripes)+1;
+}
 #endif

dali/datest/dfuwutest.cpp

@@ -435,7 +435,7 @@ IFileDescriptor *createRoxieFileDescriptor(const char *cluster, const char *lfn,
             UERRLOG("dataDirectory not specified");
             return NULL;
         }
-        makePhysicalPartName(lfn,i+1,width,filename,0,DFD_OSdefault,dir,false);
+        makePhysicalPartName(lfn,i+1,width,filename,0,DFD_OSdefault,dir,false,0);
         RemoteFilename rfn;
         rfn.setPath(grp->queryNode(i).endpoint(),filename.str());
         ret->setPart(i,rfn,NULL);

dali/dfu/dfurun.cpp

@@ -1507,7 +1507,7 @@ public:
                             default:
                                 os = DFD_OSdefault;
                             };
-                            Owned<IFileDescriptor> dstpatchf = createFileDescriptor(lname.str(),grp,NULL,os,patchf->numParts());
+                            Owned<IFileDescriptor> dstpatchf = createFileDescriptor(lname.str(), gname.str(), patchf->numParts());
                             fsys.transfer(patchf, dstpatchf, NULL, NULL, NULL, opttree, &feedback, &abortnotify, dfuwuid);
                             removePartFiles(patchf);
                             Owned<IFileDescriptor> newf = dstFile->getFileDescriptor();

ecl/eclagent/eclagent.cpp

@@ -3238,7 +3238,7 @@ char *EclAgent::getFilePart(const char *lfn, bool create)
         expandLogicalName(full_lfn, lfn);
 
         StringBuffer physical;
-        makePhysicalPartName(full_lfn.str(), 1, 1, physical, 0, DFD_OSdefault, nullptr, false);//MORE: What do we do if local ?
+        makePhysicalPartName(full_lfn.str(), 1, 1, physical, 0, DFD_OSdefault, nullptr, false, 0);//MORE: What do we do if local ?
 
         StringBuffer dir,base;
         splitFilename(physical.str(), &dir, &dir, &base, &base);

ecl/hthor/hthor.cpp

@@ -651,7 +651,7 @@ void CHThorDiskWriteActivity::publish()
     StringBuffer dir,base;
     offset_t fileSize = file->size();
     if(clusterHandler)
-        clusterHandler->splitPhysicalFilename(dir, base);
+        clusterHandler->getDirAndFilename(dir, base);
     else
         splitFilename(filename, &dir, &dir, &base, &base);
 
@@ -1244,7 +1244,7 @@ void CHThorIndexWriteActivity::execute()
     StringBuffer dir,base;
     offset_t indexFileSize = file->size();
     if(clusterHandler)
-        clusterHandler->splitPhysicalFilename(dir, base);
+        clusterHandler->getDirAndFilename(dir, base);
     else
         splitFilename(filename, &dir, &dir, &base, &base);
 

roxie/ccd/ccdfile.cpp

@@ -1677,6 +1677,7 @@ public:
 
             bool defaultDirPerPart = false;
             StringBuffer defaultDir;
+            unsigned stripeNum = 0;
 #ifdef _CONTAINERIZED
             if (!dlfn.isExternal())
             {
@@ -1685,12 +1686,14 @@ public:
                 fileDesc.getClusterGroupName(0, planeName);
                 Owned<IStoragePlane> plane = getDataStoragePlane(planeName, true);
                 defaultDir.append(plane->queryPrefix());
+                unsigned numStripedDevices = plane->numDevices();
+                stripeNum = calcStripeNumber(partNo-1, dlfn.get(), numStripedDevices);
                 FileDescriptorFlags fileFlags = static_cast<FileDescriptorFlags>(fileDesc.queryProperties().getPropInt("@flags"));
                 if (FileDescriptorFlags::none != (fileFlags & FileDescriptorFlags::dirperpart))
                     defaultDirPerPart = true;
             }
 #endif
-            makePhysicalPartName(dlfn.get(), partNo, numParts, localLocation, replicationLevel, DFD_OSdefault, defaultDir.str(), defaultDirPerPart);
+            makePhysicalPartName(dlfn.get(), partNo, numParts, localLocation, replicationLevel, DFD_OSdefault, defaultDir.str(), defaultDirPerPart, stripeNum);
         }
         Owned<ILazyFileIO> ret;
         try
@@ -3627,13 +3630,11 @@ private:
         if (!dFile->isExternal())
         {
             Owned<IFileDescriptor> desc = createFileDescriptor();
+            desc->setTraceName(dFile->queryLogicalName());
             desc->setNumParts(1);
 
-            RemoteFilename rfn;
-            dFile->getPartFilename(rfn, 0, 0);
-            StringBuffer physicalName, physicalDir, physicalBase;
-            rfn.getLocalPath(physicalName);
-            splitFilename(physicalName, &physicalDir, &physicalDir, &physicalBase, &physicalBase);
+            StringBuffer physicalDir, physicalBase;
+            dFile->getDirAndFilename(physicalDir, physicalBase);
             desc->setDefaultDir(physicalDir.str());
             desc->setPartMask(physicalBase.str());
 

roxie/ccd/ccdserver.cpp

@@ -12336,34 +12336,17 @@ public:
     virtual void setFileProperties(IFileDescriptor *desc) const
     {
         // Now publish to name services
-        StringBuffer dir,base;
+        StringBuffer dir, base;
         offset_t indexFileSize = writer->queryFile()->size();
         if(clusterHandler)
-            clusterHandler->splitPhysicalFilename(dir, base);
-        else
-        {
-            // Check filename is URL and get localpath, only actually necessary if force remote reads are are on
-            RemoteFilename rfn;
-            rfn.setRemotePath(filename);
-            StringBuffer localPath;
-            rfn.getLocalPath(localPath);
-            splitFilename(localPath, &dir, &dir, &base, &base);
-        }
-
-        desc->setDefaultDir(dir.str());
+            clusterHandler->getDirAndFilename(dir, base);
 
         //properties of the first file part.
         Owned<IPropertyTree> attrs;
         if(clusterHandler)
             attrs.setown(createPTree("Part", ipt_fast));  // clusterHandler is going to set attributes
         else
-        {
-            // add cluster
-            StringBuffer mygroupname;
-            desc->setNumParts(1);
-            desc->setPartMask(base.str());
             attrs.set(&desc->queryPart(0)->queryProperties());
-        }
         attrs->setPropInt64("@size", indexFileSize);
         attrs->setPropInt64("@recordCount", reccount);
 

system/jlib/jutil.cpp

@@ -2711,7 +2711,7 @@ StringBuffer &getFileAccessUrl(StringBuffer &out)
 
 
 #ifdef _CONTAINERIZED
-static bool getDefaultPlane(StringBuffer &ret, const char * componentOption, const char * category)
+bool getDefaultPlane(StringBuffer &ret, const char * componentOption, const char * category)
 {
     // If the plane is specified for the component, then use that
     if (getComponentConfigSP()->getProp(componentOption, ret))

system/jlib/jutil.hpp

@@ -636,5 +636,9 @@ extern jlib_decl bool checkCreateDaemon(unsigned argc, const char * * argv);
 //Createpassword of specified length, containing UpperCaseAlphas, LowercaseAlphas, numerics and symbols
 extern jlib_decl const char * generatePassword(StringBuffer &pwd, int pwdLen);
 
+#ifdef _CONTAINERIZED
+extern jlib_decl bool getDefaultPlane(StringBuffer &ret, const char * componentOption, const char * category);
+#endif
+
 #endif
 

testing/unittests/unittests.cpp

@@ -114,7 +114,13 @@ void loadDlls(IArray &objects, const char * libDirectory)
     }
 }
 
-int main(int argc, char* argv[])
+static constexpr const char * defaultYaml = R"!!(
+version: "1.0"
+unittests:
+  name: unittests
+)!!";
+
+int main(int argc, const char *argv[])
 {
     InitModuleObjects();
 
@@ -127,6 +133,10 @@ int main(int argc, char* argv[])
     bool verbose = false;
     bool list = false;
     bool useDefaultLocations = true;
+
+    //NB: not actually used for now, but required initialization for anything that may call getGlobalConfig*() or getComponentConfig*()
+    Owned<IPropertyTree> globals = loadConfiguration(defaultYaml, argv, "unittests", nullptr, nullptr, nullptr, nullptr, false);
+
     for (int argNo = 1; argNo < argc; argNo++)
     {
         const char *arg = argv[argNo];

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -546,6 +546,7 @@ public:
                         getPartFilename(*tlkDesc, l, path, true);
                         if (0 == l)
                         {
+                            ensureDirectoryForFile(path.str());
                             OwnedIFile dstIFile = createIFile(path.str());
                             copyFile(dstIFile, existingTlkIFile);
                         }

thorlcr/activities/keydiff/thkeydiff.cpp

@@ -80,6 +80,9 @@ public:
         if (width > container.queryJob().querySlaves())
             throw MakeActivityException(this, 0, "Unsupported: keydiff(%s, %s) - Cannot diff a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), newIndexFile->queryLogicalName(), width, container.queryJob().querySlaves());
 
+        StringBuffer defaultCluster;
+        if (getDefaultStoragePlane(defaultCluster))
+            clusters.append(defaultCluster);
         IArrayOf<IGroup> groups;
         fillClusterArray(container.queryJob(), outputName, clusters, groups);
         patchDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));

thorlcr/activities/keypatch/thkeypatch.cpp

@@ -84,6 +84,9 @@ public:
         if (width > container.queryJob().querySlaves())
             throw MakeActivityException(this, 0, "Unsupported: keypatch(%s, %s) - Cannot patch a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), patchFile->queryLogicalName(), width, container.queryJob().querySlaves());
 
+        StringBuffer defaultCluster;
+        if (getDefaultStoragePlane(defaultCluster))
+            clusters.append(defaultCluster);
         IArrayOf<IGroup> groups;
         fillClusterArray(container.queryJob(), outputName, clusters, groups);
         newIndexDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));

thorlcr/activities/spill/thspill.cpp

@@ -43,7 +43,20 @@ public:
         queryThorFileManager().addScope(container.queryJob(), helperFileName, expandedFileName, true);
         fileName.set(expandedFileName);
 
-        fillClusterArray(container.queryJob(), fileName, clusters, groups);
+        if (TDXtemporary & helper->getFlags())
+        {
+            // NB: these temp IFileDescriptors are not published
+            // but we want to ensure they don't have the default data plane
+            // which would cause the paths to be manipulated if numDevices>1
+            StringBuffer planeName;
+            if (getDefaultSpillPlane(planeName))
+            {
+                clusters.append(planeName);
+                groups.append(*LINK(&queryLocalGroup()));
+            }
+        }
+        if (0 == groups.ordinality()) // may be filled if temp (see above)
+            fillClusterArray(container.queryJob(), fileName, clusters, groups);
         fileDesc.setown(queryThorFileManager().create(container.queryJob(), fileName, clusters, groups, true, TDWnoreplicate+TDXtemporary));
         IPropertyTree &props = fileDesc->queryProperties();
         bool blockCompressed=false;

thorlcr/activities/thdiskbase.cpp

@@ -170,15 +170,30 @@ void CWriteMasterBase::init()
             idx++;
         }
 
+        IArrayOf<IGroup> groups;
         if (idx == 0)
         {
-            StringBuffer defaultCluster;
-            if (getDefaultStoragePlane(defaultCluster))
-                clusters.append(defaultCluster);
+            if (TDXtemporary & diskHelperBase->getFlags())
+            {
+                // NB: these temp IFileDescriptors are not published
+                // but we want to ensure they don't have the default data plane
+                // which would cause the paths to be manipulated if numDevices>1
+                StringBuffer planeName;
+                if (getDefaultSpillPlane(planeName))
+                {
+                    clusters.append(planeName);
+                    groups.append(*LINK(&queryLocalGroup()));
+                }
+            }
+            else
+            {
+                StringBuffer defaultCluster;
+                if (getDefaultStoragePlane(defaultCluster))
+                    clusters.append(defaultCluster);
+            }
         }
-
-        IArrayOf<IGroup> groups;
-        fillClusterArray(container.queryJob(), fileName, clusters, groups);
+        if (0 == groups.ordinality()) // may be filled if temp (see above)
+            fillClusterArray(container.queryJob(), fileName, clusters, groups);
         fileDesc.setown(queryThorFileManager().create(container.queryJob(), fileName, clusters, groups, overwriteok, diskHelperBase->getFlags()));
         if (1 == groups.ordinality())
             targetOffset = getGroupOffset(groups.item(0), container.queryJob().querySlaveGroup());

thorlcr/mfilemanager/thmfilemanager.cpp

@@ -485,6 +485,8 @@ public:
             desc.setown(createFileDescriptor());
             if (temporary)
                 desc->queryProperties().setPropBool("@temporary", temporary);
+            else
+                desc->setTraceName(logicalName);
             if (persistent)
                 desc->queryProperties().setPropBool("@persistent", persistent);
             desc->queryProperties().setProp("@workunit", wuidStr.str());

thorlcr/thorutil/thormisc.cpp

@@ -62,6 +62,7 @@ static Owned<IGroup> nodeGroup;    // master + processGroup
 static Owned<IGroup> slaveGroup;   // group containing all channels
 static Owned<IGroup> clusterGroup; // master + slaveGroup
 static Owned<IGroup> dfsGroup;     // same as slaveGroup, but without ports
+static Owned<IGroup> localGroup;   // used as a placeholder in IFileDescriptors for local files (spills)
 static Owned<ICommunicator> nodeComm; // communicator based on nodeGroup (master+slave processes)
 
 
@@ -103,6 +104,7 @@ MODULE_EXIT()
     clusterGroup.clear();
     slaveGroup.clear();
     dfsGroup.clear();
+    localGroup.clear();
     nodeComm.clear();
     ClusterMPAllocator.clear();
 }
@@ -861,6 +863,10 @@ void setupGroups(INode *_masterNode, IGroup *_processGroup, IGroup *_slaveGroup)
         dfsGroupNodes.append(*createINodeIP(nodeIter->query().endpoint(), 0));
     dfsGroup.setown(createIGroup(dfsGroupNodes.ordinality(), dfsGroupNodes.getArray()));
 
+    Owned<INode> localNode = createINode("localhost");
+    INode *p = localNode;
+    localGroup.setown(createIGroup(1, &p));
+
     nodeComm.setown(createCommunicator(nodeGroup));
 }
     
@@ -934,6 +940,7 @@ IGroup &queryProcessGroup() { return *processGroup; }
 IGroup &queryClusterGroup() { return *clusterGroup; }
 IGroup &querySlaveGroup() { return *slaveGroup; }
 IGroup &queryDfsGroup() { return *dfsGroup; }
+IGroup &queryLocalGroup() { return *localGroup; }
 unsigned queryClusterWidth() { return clusterGroup->ordinality()-1; }
 unsigned queryNodeClusterWidth() { return nodeGroup->ordinality()-1; }
 

thorlcr/thorutil/thormisc.hpp

@@ -534,6 +534,7 @@ extern graph_decl ICommunicator &queryNodeComm();
 extern graph_decl IGroup &queryClusterGroup();
 extern graph_decl IGroup &querySlaveGroup();
 extern graph_decl IGroup &queryDfsGroup();
+extern graph_decl IGroup &queryLocalGroup();
 extern graph_decl unsigned queryClusterWidth();
 extern graph_decl unsigned queryNodeClusterWidth();