HPCC-27249 Allow getFileTree clients/server to conditionally remap

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

common/workunit/referencedfilelist.cpp

@@ -389,7 +389,7 @@ IPropertyTree *ReferencedFile::getRemoteFileTree(IUserDescriptor *user, INode *r
     StringBuffer remoteLFN;
     if (remotePrefix && *remotePrefix)
         remoteLFN.append(remotePrefix).append("::").append(logicalName);
-    return queryDistributedFileDirectory().getFileTree(remoteLFN.length() ? remoteLFN.str() : logicalName.str(), user, remote, WF_LOOKUP_TIMEOUT, false, false);
+    return queryDistributedFileDirectory().getFileTree(remoteLFN.length() ? remoteLFN.str() : logicalName.str(), user, remote, WF_LOOKUP_TIMEOUT, GetFileTreeOpts::none);
 }
 
 IPropertyTree *ReferencedFile::getSpecifiedOrRemoteFileTree(IUserDescriptor *user, INode *remote, const char *remotePrefix)

dali/base/dacoven.cpp

@@ -37,7 +37,7 @@ extern void closedownDFS();
 // base is saved in store whenever block exhausted, so replacement coven servers can restart 
 
 // server side versioning.
-#define ServerVersion    "3.16"
+#define ServerVersion    "3.17"
 #define MinClientVersion "1.5"
 
 

dali/base/dadfs.cpp

@@ -87,6 +87,7 @@ enum MDFSRequestKind
     MDFS_SET_FILE_PROTECT,
     MDFS_ITERATE_FILTEREDFILES,
     MDFS_ITERATE_FILTEREDFILES2,
+    MDFS_GET_FILE_TREE2,
     MDFS_MAX
 };
 
@@ -1114,7 +1115,7 @@ public:
     IDFScopeIterator *getScopeIterator(IUserDescriptor *user, const char *subscope,bool recursive,bool includeempty);
     bool loadScopeContents(const char *scopelfn,StringArray *scopes,    StringArray *supers,StringArray *files, bool includeemptyscopes);
 
-    IPropertyTree *getFileTree(const char *lname,IUserDescriptor *user,const INode *foreigndali,unsigned foreigndalitimeout,bool expandnodes=false, bool appendForeign=true);
+    IPropertyTree *getFileTree(const char *lname,IUserDescriptor *user,const INode *foreigndali,unsigned foreigndalitimeout,GetFileTreeOpts opts = GetFileTreeOpts::expandNodes|GetFileTreeOpts::appendForeign);
     void setFileAccessed(CDfsLogicalFileName &dlfn, IUserDescriptor *user,const CDateTime &dt,const INode *foreigndali=NULL,unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT);
     IFileDescriptor *getFileDescriptor(const char *lname,IUserDescriptor *user,const INode *foreigndali=NULL,unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT);
     IDistributedFile *getFile(const char *lname,IUserDescriptor *user,const INode *foreigndali=NULL,unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT);
@@ -8017,7 +8018,7 @@ bool CDistributedFileDirectory::isSuperFile(    const char *logicalname,
                                                 INode *foreigndali,
                                                 unsigned timeout)
 {
-    Owned<IPropertyTree> tree = getFileTree(logicalname, user, foreigndali,timeout, false);
+    Owned<IPropertyTree> tree = getFileTree(logicalname, user, foreigndali,timeout, GetFileTreeOpts::appendForeign);
     return tree.get()&&(strcmp(tree->queryName(),queryDfsXmlBranchName(DXB_SuperFile))==0);
 }
 
@@ -10769,33 +10770,59 @@ public:
         }
     }
 
-    void getFileTree(CMessageBuffer &mb,StringBuffer &trc)
+    void getFileTree(CMessageBuffer &mb, StringBuffer &trc, unsigned version)
     {
         TransactionLog transactionLog(*this, MDFS_GET_FILE_TREE, mb.getSender());
+
+        GetFileTreeOpts opts;
+        Owned<IUserDescriptor> udesc;
+
         StringAttr lname;
         mb.read(lname);
-        unsigned ver;
-        if (mb.length()<mb.getPos()+sizeof(unsigned))
-            ver = 0;
-        else
+        if (version >= 2)
         {
-            mb.read(ver);
-            // this is a bit of a mess - for backward compatibility where user descriptor specified
-            if (ver>MDFS_GET_FILE_TREE_V2)
+            unsigned _opts;        
+            mb.read(_opts);
+            opts = static_cast<GetFileTreeOpts>(_opts);
+            bool hasUser;
+            mb.read(hasUser);
+            if (hasUser)
             {
-                mb.reset(mb.getPos()-sizeof(unsigned));
-                ver = 0;
+                udesc.setown(createUserDescriptor());
+                udesc->deserialize(mb);
             }
         }
-        trc.appendf("getFileTree(%s,%d)",lname.str(),ver);
-        if (queryTransactionLogging())
-            transactionLog.log("%s", trc.str());
-        Owned<IUserDescriptor> udesc;
-        if (mb.getPos()<mb.length())
+        else // pre gft versioning/tidyup
         {
-            udesc.setown(createUserDescriptor());
-            udesc->deserialize(mb);
+            // for ancient backward version compatibility
+            if (mb.length()<mb.getPos()+sizeof(unsigned))
+                version = 0;
+            else
+            {
+                mb.read(version);
+                // this is a bit of a mess - for backward compatibility where user descriptor specified
+                if (version>MDFS_GET_FILE_TREE_V2)
+                {
+                    mb.reset(mb.getPos()-sizeof(unsigned));
+                    version = 0;
+                }
+                else
+                {
+                    // NB: just for clarity - MDFS_GET_FILE_TREE_V2 is hardwired to 1
+                    // version >= 2 is cleaned up version.
+                    version = 1;
+                }
+            }
+            if (mb.getPos()<mb.length())
+            {
+                udesc.setown(createUserDescriptor());
+                udesc->deserialize(mb);
+            }
         }
+        trc.appendf("getFileTree(%s, client gft version=%u)",lname.str(), version);
+        if (queryTransactionLogging())
+            transactionLog.log("%s", trc.str());
+
         mb.clear();
         CDfsLogicalFileName dlfn;
         dlfn.set(lname);
@@ -10808,72 +10835,85 @@ public:
             CScopeConnectLock sconnlock("getFileTree", *logicalname, false, false, false, defaultTimeout);
             IPropertyTree* sroot = sconnlock.conn()?sconnlock.conn()->queryRoot():NULL;
             logicalname->getTail(tail);
-            Owned<IPropertyTree> tree = getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_File),"@name",tail.str(),false);
-            if (tree)
+            if (version >= 2)
             {
-                if (ver>=MDFS_GET_FILE_TREE_V2)
+                Owned<IPropertyTree> tree = getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_File),"@name",tail.str(),false);
+                if (tree)
                 {
+                    //NB: for new clients, only clients specifically asking for remap
+                    if (hasMask(opts, GetFileTreeOpts::remapToService))
+                        remapGroupsToDafilesrv(tree, &queryNamedGroupStore());
+
                     Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(tree,&queryNamedGroupStore(),IFDSF_EXCLUDE_CLUSTERNAMES);
-                    if (fdesc)
+                    mb.append((int)1); // 1 == standard file
+                    fdesc->serialize(mb);
+
+                    /* not sure why this attribute is special/here at top level
+                        and not part of the IFileDescriptor serialization itself */
+                    StringBuffer dts;
+                    tree->getProp("@modified", dts);
+                    unsigned l = dts.length();
+                    mb.append(l).append(l, dts.str());
+
+                    break;
+                }
+                else
+                {
+                    tree.setown(getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_SuperFile),"@name",tail.str(),false));
+                    if (tree)
+                    {
+                        mb.append((int)2); // 2 == super
+                        tree->serialize(mb);
+                        break;
+                    }
+                }
+            }
+            else
+            {
+                Owned<IPropertyTree> tree = getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_File),"@name",tail.str(),false);
+                if (tree)
+                {
+                    if (version == MDFS_GET_FILE_TREE_V2)
                     {
 #ifdef _CONTAINERIZED
-                        unsigned nc = fdesc->numClusters();
-                        for (unsigned c=0; c<nc; c++)
-                        {
-                            IClusterInfo *clusterInfo = fdesc->queryClusterNum(c);
-                            const char *planeName = clusterInfo->queryGroupName();
-                            Owned<IStoragePlane> plane = getDataStoragePlane(planeName, true);
-                            if (!plane->queryHosts() && isAbsolutePath(plane->queryPrefix())) // if host group, or url, don't touch
-                            {
-                                {
-                                    CriticalBlock b(dafileSrvNodeCS);
-                                    if (nullptr == dafileSrvNode)
-                                    {
-                                        auto externalService = getDafileServiceFromConfig("directio");
-                                        VStringBuffer dafilesrvEpStr("%s:%u", externalService.first.c_str(), externalService.second);
-                                        dafileSrvNode.setown(createINode(dafilesrvEpStr));
-                                    }
-                                }
-                                IGroup *oldGroup = clusterInfo->queryGroup();
-                                std::vector<INode *> nodes;
-                                for (unsigned n=0; n<oldGroup->ordinality(); n++)
-                                    nodes.push_back(dafileSrvNode);
-                                Owned<IGroup> newGroup = createIGroup((rank_t)oldGroup->ordinality(), &nodes[0]);
-                                clusterInfo->setGroup(newGroup); // NB: links
-                            }
-                        }
+                        // NB: to be here, the client is by definition legacy, and should be foreign.
+                        // foreignAccess must also have been configured in this environment
+
+                        if (getComponentConfigSP()->getPropBool("@foreignAccess"))
+                            remapGroupsToDafilesrv(tree, &queryNamedGroupStore());
 #endif
 
-                        ver = MDFS_GET_FILE_TREE_V2;
-                        mb.append((int)-2).append(ver);
+                        Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(tree,&queryNamedGroupStore(),IFDSF_EXCLUDE_CLUSTERNAMES);
+
+                        mb.append((int)-2).append(version);
+
                         fdesc->serialize(mb);
                         StringBuffer dts;
-                        if (tree->getProp("@modified",dts)) {
+                        if (tree->getProp("@modified",dts))
+                        {
                             CDateTime dt;
                             dt.setString(dts.str());
                             dt.serialize(mb);
                         }
                     }
-                    else
-                        ver = 0;
-                }
-                if (ver==0)
-                {
-                    tree.setown(createPTreeFromIPT(tree));
-                    StringBuffer cname;
-                    logicalname->getCluster(cname);
-                    expandFileTree(tree,true,cname.str()); // resolve @node values that may not be set
-                    tree->serialize(mb);
+                    else if (version==0) // ancient version backward compatibility
+                    {
+                        tree.setown(createPTreeFromIPT(tree));
+                        StringBuffer cname;
+                        logicalname->getCluster(cname);
+                        expandFileTree(tree,true,cname.str()); // resolve @node values that may not be set
+                        tree->serialize(mb);
+                    }
+                    break;
                 }
-                break;
-            }
-            else
-            {
-                tree.setown(getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_SuperFile),"@name",tail.str(),false));
-                if (tree)
+                else
                 {
-                    tree->serialize(mb);
-                    break;
+                    tree.setown(getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_SuperFile),"@name",tail.str(),false));
+                    if (tree)
+                    {
+                        tree->serialize(mb);
+                        break;
+                    }
                 }
             }
             if (redmatch.get())
@@ -10950,7 +10990,14 @@ public:
                 }
                 case MDFS_GET_FILE_TREE:
                 {
-                    getFileTree(mb, trc);
+                    getFileTree(mb, trc, 0);
+                    break;
+                }
+                case MDFS_GET_FILE_TREE2:
+                {
+                    unsigned clientGFTVersion;
+                    mb.read(clientGFTVersion);
+                    getFileTree(mb, trc, clientGFTVersion);
                     break;
                 }
                 case MDFS_GET_GROUP_TREE:
@@ -11235,14 +11282,20 @@ void CDistributedFileDirectory::setFileProtect(CDfsLogicalFileName &dlfn,IUserDe
     checkDfsReplyException(mb);
 }
 
-IPropertyTree *CDistributedFileDirectory::getFileTree(const char *lname, IUserDescriptor *user, const INode *foreigndali,unsigned foreigndalitimeout, bool expandnodes, bool appendForeign)
+IPropertyTree *CDistributedFileDirectory::getFileTree(const char *lname, IUserDescriptor *user, const INode *foreigndali,unsigned foreigndalitimeout, GetFileTreeOpts opts)
 {
+    constexpr unsigned gftVersion = 2; // for future use (0 and 1 are reserved for legacy versions)
+    bool expandnodes = hasMask(opts, GetFileTreeOpts::expandNodes);
+    bool appendForeign = hasMask(opts, GetFileTreeOpts::appendForeign);
+
+    bool getFileTree2Support = queryDaliServerVersion().compare("3.17") >= 0;
     // this accepts either a foreign dali node or a foreign lfn
     Owned<INode> fnode;
     CDfsLogicalFileName dlfn;
     SocketEndpoint ep;
     dlfn.set(lname);
-    if (dlfn.isForeign()) {
+    if (dlfn.isForeign())
+    {
         if (!dlfn.getEp(ep))
             throw MakeStringException(-1,"cannot resolve dali ip in foreign file name (%s)",lname);
         fnode.setown(createINode(ep));
@@ -11252,53 +11305,115 @@ IPropertyTree *CDistributedFileDirectory::getFileTree(const char *lname, IUserDe
     if (isLocalDali(foreigndali))
         foreigndali = NULL;
     CMessageBuffer mb;
-    mb.append((int)MDFS_GET_FILE_TREE).append(lname);
-    mb.append(MDFS_GET_FILE_TREE_V2);
-    if (user)
+
+    if (getFileTree2Support)
     {
-        user->serializeWithoutPassword(mb);
-    }
+        mb.append((int)MDFS_GET_FILE_TREE2);
+        mb.append(gftVersion);
+        mb.append(lname);
+        // if it's a foreign dali, and unless explicitly requested to remap or explicitly requested to suppress foreign remapping
+        // ensure the remap flag is sent.
+        if (foreigndali && !hasMask(opts, GetFileTreeOpts::remapToService | GetFileTreeOpts::suppressForeignRemapToService))
+            opts |= GetFileTreeOpts::remapToService;
+
+        mb.append(static_cast<unsigned>(opts));
+
+        if (user)
+        {
+            mb.append(true);
+            user->serializeWithoutPassword(mb);
+        }
+        else
+        {
+            mb.append(false);
 #ifdef NULL_DALIUSER_STACKTRACE
+            DBGLOG("UNEXPECTED USER (NULL) in dadfs.cpp getFileTree() line %d",__LINE__);
+            PrintStackReport();
+#endif
+        }
+    }
     else
     {
-        DBGLOG("UNEXPECTED USER (NULL) in dadfs.cpp getFileTree() line %d",__LINE__);
-        PrintStackReport();
-    }
+        mb.append((int)MDFS_GET_FILE_TREE).append(lname);
+        mb.append(MDFS_GET_FILE_TREE_V2);
+        if (user)
+            user->serializeWithoutPassword(mb);
+#ifdef NULL_DALIUSER_STACKTRACE
+        else
+        {
+            DBGLOG("UNEXPECTED USER (NULL) in dadfs.cpp getFileTree() line %d",__LINE__);
+            PrintStackReport();
+        }
 #endif
+    }
     if (foreigndali)
         foreignDaliSendRecv(foreigndali,mb,foreigndalitimeout);
     else
         queryCoven().sendRecv(mb,RANK_RANDOM,MPTAG_DFS_REQUEST);
     checkDfsReplyException(mb);
     if (mb.length()==0)
-        return NULL;
-    unsigned ver = 0;
-    if ((mb.length()>=sizeof(int))&&(*(int *)mb.bufferBase()) == -2) { // version indicator
-        int i;
-        mb.read(i);
-        mb.read(ver);
-    }
+        return nullptr;
+
     Owned<IPropertyTree> ret;
-    if (ver==0)
-        ret.setown(createPTree(mb));
-    else {
-        Owned<IFileDescriptor> fdesc;
-        CDateTime modified;
-        if (ver==MDFS_GET_FILE_TREE_V2) { // no longer in use but support for back compatibility
+    Owned<IFileDescriptor> fdesc;
+    if (getFileTree2Support)
+    {
+        unsigned type; // 1 = regular file, 2 = super
+        mb.read(type);
+        if (1 == type)
+        {
             fdesc.setown(deserializeFileDescriptor(mb));
-            if (mb.remaining()>0)
-                modified.deserialize(mb);
+            ret.setown(createPTree(queryDfsXmlBranchName(DXB_File)));
+            fdesc->serializeTree(*ret,expandnodes?0:CPDMSF_packParts);
+            /* See server-side code, not sure why this attribute is special/here at top level
+            and not part of the IFileDescriptor serialization itself */
+            unsigned l;
+            mb.read(l);
+            if (l)
+            {
+                StringAttr v((const char *)mb.readDirect(l), l);
+                ret->setProp("@modified", v);
+            }
         }
         else
-            throw MakeStringException(-1,"Unknown GetFileTree serialization version %d",ver);
-        ret.setown(createPTree(queryDfsXmlBranchName(DXB_File)));
-        fdesc->serializeTree(*ret,expandnodes?0:CPDMSF_packParts);
-        if (!modified.isNull()) {
-            StringBuffer dts;
-            ret->setProp("@modified",modified.getString(dts).str());
+        {
+            verifyex(2 == type); // no other valid possibility
+            ret.setown(createPTree(mb));
+        }
+    }
+    else
+    {
+        unsigned ver = 0;
+        if ((mb.length()>=sizeof(int))&&(*(int *)mb.bufferBase()) == -2) // version indicator
+        {
+            int i;
+            mb.read(i);
+            mb.read(ver);
+        }
+        if (ver==0)
+            ret.setown(createPTree(mb));
+        else
+        {
+            CDateTime modified;
+            if (ver==MDFS_GET_FILE_TREE_V2) // no longer in use but support for back compatibility
+            {
+                fdesc.setown(deserializeFileDescriptor(mb));
+                if (mb.remaining()>0)
+                    modified.deserialize(mb);
+            }
+            else
+                throw MakeStringException(-1,"Unknown GetFileTree serialization version %d",ver);
+            ret.setown(createPTree(queryDfsXmlBranchName(DXB_File)));
+            fdesc->serializeTree(*ret,expandnodes?0:CPDMSF_packParts);
+            if (!modified.isNull())
+            {
+                StringBuffer dts;
+                ret->setProp("@modified",modified.getString(dts).str());
+            }
         }
     }
-    if (expandnodes) {
+    if (expandnodes)
+    {
         StringBuffer cname;
         dlfn.getCluster(cname);
         expandFileTree(ret,true,cname.str());
@@ -11315,7 +11430,7 @@ IPropertyTree *CDistributedFileDirectory::getFileTree(const char *lname, IUserDe
 
 IFileDescriptor *CDistributedFileDirectory::getFileDescriptor(const char *lname,IUserDescriptor *user,const INode *foreigndali,unsigned foreigndalitimeout)
 {
-    Owned<IPropertyTree> tree = getFileTree(lname,user,foreigndali,foreigndalitimeout,false);
+    Owned<IPropertyTree> tree = getFileTree(lname,user,foreigndali,foreigndalitimeout,GetFileTreeOpts::appendForeign);
     if (!tree)
         return NULL;
     if (strcmp(tree->queryName(),queryDfsXmlBranchName(DXB_SuperFile))==0) {
@@ -11336,7 +11451,7 @@ IFileDescriptor *CDistributedFileDirectory::getFileDescriptor(const char *lname,
 
 IDistributedFile *CDistributedFileDirectory::getFile(const char *lname,IUserDescriptor *user,const INode *foreigndali,unsigned foreigndalitimeout)
 {
-    Owned<IPropertyTree> tree = getFileTree(lname,user,foreigndali,foreigndalitimeout,false);
+    Owned<IPropertyTree> tree = getFileTree(lname,user,foreigndali,foreigndalitimeout,GetFileTreeOpts::appendForeign);
     if (!tree)
         return NULL;
     if (strcmp(tree->queryName(),queryDfsXmlBranchName(DXB_SuperFile))==0) {
@@ -12772,7 +12887,7 @@ bool CDistributedFileDirectory::publishMetaFileXML(const CDfsLogicalFileName &lo
 {
     if (logicalname.isExternal()||logicalname.isForeign()||logicalname.isQuery())
         return false;
-    Owned<IPropertyTree> file = getFileTree(logicalname.get(),user,NULL,FOREIGN_DALI_TIMEOUT,true);
+    Owned<IPropertyTree> file = getFileTree(logicalname.get(),user,NULL,FOREIGN_DALI_TIMEOUT,GetFileTreeOpts::expandNodes|GetFileTreeOpts::appendForeign);
     if (!file.get())
         return false;
     if (strcmp(file->queryName(),queryDfsXmlBranchName(DXB_SuperFile))==0)

dali/base/dadfs.hpp

@@ -567,6 +567,17 @@ typedef IIteratorOf<IFileRelationship> IFileRelationshipIterator;
 /**
  * A distributed directory. Can created, access and delete files, super-files and logic-files.
  */
+
+enum class GetFileTreeOpts
+{
+    none                          = 0x0,
+    expandNodes                   = 0x1,
+    appendForeign                 = 0x2,
+    remapToService                = 0x4,
+    suppressForeignRemapToService = 0x8
+};
+BITMASK_ENUM(GetFileTreeOpts);
+
 interface IDistributedFileDirectory: extends IInterface
 {
     virtual IDistributedFile *lookup(   const char *logicalname,
@@ -610,7 +621,7 @@ interface IDistributedFileDirectory: extends IInterface
     virtual bool exists(const char *logicalname,IUserDescriptor *user,bool notsuper=false,bool superonly=false) = 0;                           // logical name exists
     virtual bool existsPhysical(const char *logicalname,IUserDescriptor *user) = 0;                                                    // physical parts exists
 
-    virtual IPropertyTree *getFileTree(const char *lname, IUserDescriptor *user, const INode *foreigndali=NULL, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT, bool expandnodes=true, bool appendForeign=true) =0;
+    virtual IPropertyTree *getFileTree(const char *lname, IUserDescriptor *user, const INode *foreigndali=NULL, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT, GetFileTreeOpts opts = GetFileTreeOpts::expandNodes|GetFileTreeOpts::appendForeign) =0;
     virtual IFileDescriptor *getFileDescriptor(const char *lname,IUserDescriptor *user,const INode *foreigndali=NULL, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) =0;
 
     virtual IDistributedSuperFile *createSuperFile(const char *logicalname,IUserDescriptor *user,bool interleaved,bool ifdoesnotexist=false,IDistributedFileTransaction *transaction=NULL) = 0;

dali/base/dautils.cpp

@@ -3783,3 +3783,48 @@ void addStripeDirectory(StringBuffer &out, const char *directory, const char *pl
         }
     }
 }
+
+static CriticalSection dafileSrvNodeCS;
+static Owned<INode> dafileSrvNode;
+void remapGroupsToDafilesrv(IPropertyTree *file, INamedGroupStore *resolver)
+{
+    FileDescriptorFlags fileFlags = static_cast<FileDescriptorFlags>(file->getPropInt("Attr/@flags"));
+    Owned<IPropertyTreeIterator> iter = file->getElements("Cluster");
+    ForEach(*iter)
+    {
+        IPropertyTree &cluster = iter->query();
+        const char *planeName = cluster.queryProp("@name");
+        Owned<IStoragePlane> plane = getDataStoragePlane(planeName, true);
+        if (!plane->queryHosts() && isAbsolutePath(plane->queryPrefix())) // if host group, or url, don't touch
+        {
+            {
+                CriticalBlock b(dafileSrvNodeCS);
+                if (nullptr == dafileSrvNode)
+                {
+                    auto externalService = getDafileServiceFromConfig("directio");
+                    VStringBuffer dafilesrvEpStr("%s:%u", externalService.first.c_str(), externalService.second);
+                    dafileSrvNode.setown(createINode(dafilesrvEpStr));
+                }
+            }
+
+            Owned<IGroup> group;
+            if (cluster.hasProp("Group"))
+                group.setown(createIGroup(cluster.queryProp("Group")));
+            else
+            {
+                assertex(resolver);
+                StringBuffer defaultDir;
+                GroupType groupType;
+                group.setown(resolver->lookup(planeName, defaultDir, groupType));
+            }
+
+            std::vector<INode *> nodes;
+            for (unsigned n=0; n<group->ordinality(); n++)
+                nodes.push_back(dafileSrvNode);
+            Owned<IGroup> newGroup = createIGroup((rank_t)group->ordinality(), &nodes[0]);
+            StringBuffer groupText;
+            newGroup->getText(groupText);
+            cluster.setProp("Group", groupText);
+        }
+    }
+}

dali/base/dautils.hpp

@@ -567,4 +567,7 @@ inline unsigned calcStripeNumber(unsigned partNum, const char *lfnName, unsigned
     unsigned lfnHash = getFilenameHash(lfnName);
     return ((partNum+lfnHash)%numStripes)+1;
 }
+interface INamedGroupStore;
+extern da_decl void remapGroupsToDafilesrv(IPropertyTree *file, INamedGroupStore *resolver);
+
 #endif

dali/daliadmin/daadmin.cpp

@@ -481,7 +481,7 @@ bool dfsfile(const char *lname, IUserDescriptor *userDesc, StringBuffer &out, Un
     CDfsLogicalFileName lfn;
     lfn.set(lname);
     if (!lfn.isExternal()) {
-        Owned<IPropertyTree> tree = queryDistributedFileDirectory().getFileTree(lname,userDesc,NULL,daliConnectTimeoutMs,true); //,userDesc);
+        Owned<IPropertyTree> tree = queryDistributedFileDirectory().getFileTree(lname,userDesc,NULL,daliConnectTimeoutMs,GetFileTreeOpts::expandNodes|GetFileTreeOpts::appendForeign); //,userDesc);
         if (partslist)
             filterParts(tree,*partslist);
         if (!tree) {

dali/dfu/dfurun.cpp

@@ -936,7 +936,7 @@ public:
             slfn.clearForeign();
             srcdali.setown(createINode(ep));
         }
-        Owned<IPropertyTree> ftree = queryDistributedFileDirectory().getFileTree(srclfn,ctx.srcuser,srcdali, FOREIGN_DALI_TIMEOUT, false);
+        Owned<IPropertyTree> ftree = queryDistributedFileDirectory().getFileTree(srclfn,ctx.srcuser,srcdali, FOREIGN_DALI_TIMEOUT, GetFileTreeOpts::appendForeign);
         if (!ftree.get()) {
             StringBuffer s;
             throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)");

dali/dfu/dfuutil.cpp

@@ -625,7 +625,7 @@ public:
             slfn.clearForeign();
             srcdali.setown(createINode(ep));
         }
-        Owned<IPropertyTree> ftree = fdir->getFileTree(slfn.get(),foreignuserdesc,srcdali, FOREIGN_DALI_TIMEOUT, false);
+        Owned<IPropertyTree> ftree = fdir->getFileTree(slfn.get(),foreignuserdesc,srcdali, FOREIGN_DALI_TIMEOUT, GetFileTreeOpts::appendForeign);
         if (!ftree.get()) {
             StringBuffer s;
             throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)");
@@ -717,7 +717,7 @@ public:
             slfn.clearForeign();
             srcdali.setown(createINode(ep));
         }
-        Owned<IPropertyTree> ftree = fdir->getFileTree(slfn.get(), foreignuserdesc, srcdali, FOREIGN_DALI_TIMEOUT, false);
+        Owned<IPropertyTree> ftree = fdir->getFileTree(slfn.get(), foreignuserdesc, srcdali, FOREIGN_DALI_TIMEOUT, GetFileTreeOpts::appendForeign);
         if (!ftree.get()) {
             StringBuffer s;
             throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)");
@@ -858,7 +858,7 @@ public:
             srcdali.setown(createINode(ep));
         }
         StringBuffer s;
-        Owned<IPropertyTree> ftree = fdir->getFileTree(srcLFN.get(), foreignuserdesc, srcdali, FOREIGN_DALI_TIMEOUT, false);
+        Owned<IPropertyTree> ftree = fdir->getFileTree(srcLFN.get(), foreignuserdesc, srcdali, FOREIGN_DALI_TIMEOUT, GetFileTreeOpts::appendForeign);
         if (!ftree.get())
             throw MakeStringException(-1,"Source file %s could not be found in Dali %s",srcLFN.get(), getDaliEndPointStr(srcdali, s));
         IPropertyTree *attsrc = ftree->queryPropTree("Attr");
@@ -1145,7 +1145,7 @@ public:
                 return;
             }
         }
-        Owned<IPropertyTree> ftree = queryDistributedFileDirectory().getFileTree(srclfn,srcuser,srcnode, FOREIGN_DALI_TIMEOUT, false);
+        Owned<IPropertyTree> ftree = queryDistributedFileDirectory().getFileTree(srclfn,srcuser,srcnode, FOREIGN_DALI_TIMEOUT, GetFileTreeOpts::appendForeign);
         if (!ftree.get())
         {
             StringBuffer s;

esp/scm/ws_dfs.ecm

@@ -31,6 +31,7 @@ ESPrequest DFSFileLookupRequest
     string Name;                        // the logical file name
     int64 LeaseId;                      // lease to associate any locks to
     int RequestTimeout(300);            // Max seconds to block waiting (default = 5 mins)
+    bool AccessViaDafilesrv(false);     // DFS service will return file meta pointing to a dafilesrv service
 };
 
 ESPresponse [exceptions_inline] DFSFileLookupResponse

esp/services/ws_dfsservice/ws_dfsservice.cpp

@@ -42,11 +42,14 @@ static unsigned __int64 getLockId(unsigned __int64 leaseId)
     return ++nextLockID;
 }
 
-static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, IPropertyTree *metaRoot, IPropertyTree *meta)
+static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, bool remap, IPropertyTree *metaRoot, IPropertyTree *meta)
 {
     Owned<IPropertyTree> tree = queryDistributedFileDirectory().getFileTree(logicalName, nullptr);
     if (!tree)
         return;
+    if (remap)
+        remapGroupsToDafilesrv(tree, nullptr);
+
     CDfsLogicalFileName lfn;
     lfn.set(logicalName);
     if (lfn.isForeign())
@@ -95,7 +98,7 @@ static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, I
             {
                 IPropertyTree &sub = *(orderedSubFiles[f]);
                 sub.getProp("@name", subname.clear());
-                populateLFNMeta(subname, leaseId, metaRoot, fileMeta);
+                populateLFNMeta(subname, leaseId, remap, metaRoot, fileMeta);
             }
         }
     }
@@ -158,7 +161,8 @@ bool CWsDfsEx::onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &r
 
         // populate file meta data and lock id's
         Owned<IPropertyTree> responseTree = createPTree();
-        populateLFNMeta(logicalName, leaseId, responseTree, responseTree);
+        bool remap = req.getAccessViaDafilesrv();
+        populateLFNMeta(logicalName, leaseId, remap, responseTree, responseTree);
 
         // serialize response
         MemoryBuffer respMb, compressedRespMb;

helm/hpcc/templates/dali.yaml

@@ -21,6 +21,26 @@
 ##############################################################################
 
 */}}
+
+{{/*
+Return true if Dali setup as exposed service with directio enabled
+Pass in root and me
+*/}}
+{{- define "hpcc.isForeignAccessConfigured" }}
+{{- $service := .me.service | default dict -}}
+{{- $visibility := $service.visibility | default "cluster" -}}
+{{- if (not (eq $visibility "cluster")) -}}
+ {{ range .root.Values.dafilesrv -}}
+  {{- if not .disabled -}}
+   {{- if (eq "directio" .application) -}}
+true
+   {{- end -}}
+  {{- end -}}
+ {{- end -}}
+{{- end }}
+{{- end -}}
+
+
 {{ range $dali := $.Values.dali -}}
 {{- if not $dali.disabled -}}
 {{- $env := concat ($.Values.global.env | default list) (.env | default list) -}}
@@ -162,6 +182,7 @@ data:
 {{- include "hpcc.generateLoggingConfig" $commonCtx | indent 6 }}
 {{- include "hpcc.generateMetricsConfig" $commonCtx | indent 6 }}
 {{ include "hpcc.generateVaultConfig" (dict "root" $ "secretsCategories" $daliSecretsCategories ) | indent 6 }}
+      foreignAccess: {{ include "hpcc.isForeignAccessConfigured" (dict "root" $ "me" .) | default "false" }}
     global:
 {{ include "hpcc.generateGlobalConfigMap" $ | indent 6 }}
 ---
@@ -193,20 +214,13 @@ Expose dali as a external service, only if there is a service definition and daf
 {{- $service := deepCopy (.service | default dict) -}}
 {{- $_ := set $service "visibility" ($service.visibility | default "cluster") -}}
 {{- $_ := set $service "servicePort" ($service.servicePort | default 7070) -}}
-{{- if and (hasKey . "service") (not (eq $service.visibility "cluster")) -}}
- {{ range $.Values.dafilesrv -}}
-  {{- if not .disabled -}}
-   {{- if (eq "directio" .application) -}}
-    {{- $_ := set $dafilesrvCtx "directAccess" true -}}
-   {{- end -}}
-  {{- end -}}
- {{- end -}}
-{{- end }}
-{{- if $dafilesrvCtx.directAccess -}}
+{{- if (include "hpcc.isForeignAccessConfigured" (dict "root" $ "me" .)) -}}
  {{- if not (hasKey $service "labels") -}}
   {{- $_ := set $service "labels" dict -}}
  {{- end -}}
  {{- $_ := set $service "labels" (merge $service.labels (dict "server" .name)) -}}
+{{- else -}}
+ {{- $_ := set $service "visibility" "cluster" -}}
 {{- end }}
 {{- include "hpcc.addService" ( dict "root" $ "name" .name "service" $service "selector" .name "defaultPort" 7070 ) | nindent 0 }}
 ---

system/jlib/jlib.hpp

@@ -395,6 +395,6 @@ inline constexpr X operator | (X l, X r) { return (X)((unsigned)l | (unsigned)r)
 inline constexpr X operator & (X l, X r) { return (X)((unsigned)l & (unsigned)r); } \
 inline constexpr X operator ~ (X l) { return (X)(~(unsigned)l); } \
 inline X & operator |= (X & l, X r) { l = l | r; return l; } \
-inline X & operator &= (X & l, X r) { l = (X)(l & r); return l; }
-
+inline X & operator &= (X & l, X r) { l = (X)(l & r); return l; } \
+inline bool hasMask(X l, X r) { return (l & r) != (X)0; }
 #endif

system/jlib/jmisc.cpp

@@ -1023,10 +1023,13 @@ jlib_decl char **getSystemEnv()
 }
 
 
-#ifdef _CONTAINERIZED
-// NB: will fire an exception if command fails (returns non-zero exit code)
 void runKubectlCommand(const char *title, const char *cmd, const char *input, StringBuffer *output)
 {
+#ifndef _CONTAINERIZED
+    UNIMPLEMENTED_X("runKubectlCommand");
+#endif
+// NB: will fire an exception if command fails (returns non-zero exit code)
+
     StringBuffer _output, error;
     if (!output)
         output = &_output;
@@ -1047,6 +1050,9 @@ static CTimeLimitedCache<std::string, std::pair<std::string, unsigned>> external
 static CriticalSection externalServiceCacheCrit;
 std::pair<std::string, unsigned> getExternalService(const char *serviceName)
 {
+#ifndef _CONTAINERIZED
+    UNIMPLEMENTED_X("getExternalService");
+#endif
     {
         CriticalBlock b(externalServiceCacheCrit);
         std::pair<std::string, unsigned> cachedExternalSevice;
@@ -1085,9 +1091,11 @@ std::pair<std::string, unsigned> getExternalService(const char *serviceName)
     return servicePair;
 }
 
-
 std::pair<std::string, unsigned> getDafileServiceFromConfig(const char *application)
 {
+#ifndef _CONTAINERIZED
+    UNIMPLEMENTED_X("getDafileServiceFromConfig");
+#endif
     /* NB: For now expect 1 dafilesrv in configuration only
      * We could have multiple dafilesrv services with e.g. different specs./replicas etc. that
      * serviced different planes. At the moment dafilesrv mounts all data planes.
@@ -1108,4 +1116,4 @@ std::pair<std::string, unsigned> getDafileServiceFromConfig(const char *applicat
         throw makeStringExceptionV(-1, "dafilesrv '%s': external service port not defined", dafilesrvName.str());
     return externalService;
 }
-#endif
+

system/jlib/jmisc.hpp

@@ -22,6 +22,8 @@
 
 #include "platform.h"
 #include <stdio.h>
+#include <string>
+#include <utility>
 #include "jiface.hpp"
 #include "jcrc.hpp"
 #include "jlog.hpp"