Merge pull request #10689 from jakesmith/hpcc-18801

HPCC-18801 Add CLI options for listing and attaching found files.

Reviewed-By: Attila Vamos <attila.vamos@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

dali/dfuXRefLib/XRefFilesNode.cpp

@@ -45,25 +45,24 @@ bool CXRefFilesNode::IsChanged()
 void CXRefFilesNode::Commit()
 {
     if (m_bChanged)
-        Deserialize(getDataTree());
+        Deserialize(queryDataTree());
     m_bChanged = false;
 }
 
-StringBuffer& CXRefFilesNode::Serialize(StringBuffer& outStr)
+MemoryBuffer &CXRefFilesNode::queryData()
 {
-    if (!m_bChanged && _data.length() > 0)
-    {
-        outStr.append(_data);
-        return outStr;
-    }
-    _data.clear();
-    MemoryBuffer buff;
-    m_baseTree.getPropBin("data",buff);
-    if (buff.length())
+    if (m_bChanged || (0 == _data.length()))
     {
-        outStr.append(buff.length(),buff.toByteArray());
-        _data.append(outStr);
+        _data.clear();
+        m_baseTree.getPropBin("data", _data);
     }
+    return _data;
+}
+
+StringBuffer& CXRefFilesNode::Serialize(StringBuffer& outStr)
+{
+    MemoryBuffer &_data = queryData();
+    outStr.append(_data.length(), _data.toByteArray());
     return outStr;
 }
 
@@ -79,17 +78,22 @@ IPropertyTree* CXRefFilesNode::FindNode(const char* NodeName)
 {
     StringBuffer xpath;
     xpath.clear().appendf("File/[Partmask=\"%s\"]", NodeName);
-    StringBuffer tmpbuf;
-    return getDataTree().getBranch(xpath.str());
+    return queryDataTree().getBranch(xpath.str());
 }
         
-IPropertyTree& CXRefFilesNode::getDataTree()
+IPropertyTreeIterator *CXRefFilesNode::getMatchingFiles(const char *match, const char *type)
+{
+    StringBuffer xpath;
+    xpath.clear().appendf("File/[%s=\"%s\"]", type, match);
+    return queryDataTree().getElements(xpath.str());
+}
+
+IPropertyTree& CXRefFilesNode::queryDataTree()
 {
     if (m_DataTree.get() == 0)
     {
-        StringBuffer dataStr;
-        Serialize(dataStr);
-        m_DataTree.setown(createPTreeFromXMLString(dataStr.str()));
+        MemoryBuffer &data = queryData();
+        m_DataTree.setown(createPTreeFromXMLString(data.length(), data.toByteArray()));
     }
     return *m_DataTree.get();
 }
@@ -97,7 +101,8 @@ IPropertyTree& CXRefFilesNode::getDataTree()
 static bool checkPartsInCluster(const char *title,const char *clustername, IPropertyTree *subBranch, StringBuffer &errstr,bool exists)
 {
     Owned<IGroup> group = queryNamedGroupStore().lookup(clustername);
-    if (!group) {
+    if (!group)
+    {
         ERRLOG("%s cluster not found",clustername);
         errstr.appendf("ERROR: %s cluster not found",clustername);
         return false;
@@ -106,12 +111,15 @@ static bool checkPartsInCluster(const char *title,const char *clustername, IProp
     unsigned i;
     StringBuffer xpath;
     unsigned n = group->ordinality();
-    ForEach(*partItr) {
+    ForEach(*partItr)
+    {
         IPropertyTree& part = partItr->query();
         unsigned pn = part.getPropInt("Num");
-        for (int rep=0;rep<2;rep++) {
+        for (int rep=0;rep<2;rep++)
+        {
             i = 0;
-            for (;;) {
+            for (;;)
+            {
                 i++;
                 xpath.clear().appendf(rep?"RNode[%d]":"Node[%d]",i);
                 if (!part.hasProp(xpath.str())) 
@@ -119,14 +127,17 @@ static bool checkPartsInCluster(const char *title,const char *clustername, IProp
                 SocketEndpoint ep(part.queryProp(xpath.str()));
                 ep.port = 0;
                 rank_t gn = group->rank(ep);
-                if (group->rank(ep)==RANK_NULL) {
+                if (group->rank(ep)==RANK_NULL)
+                {
                     StringBuffer eps;
                     ERRLOG("%s %s Part %d on %s is not in cluster %s",title,rep?"Replicate":"Primary",pn,ep.getUrlStr(eps).str(),clustername);
                     errstr.appendf("ERROR: %s %s part %d on %s is not in cluster %s",title,rep?"Replicate":"Primary",pn,ep.getUrlStr(eps).str(),clustername);
                     return false;
                 }
-                if (exists) {
-                    if ((pn-1+rep)%n==gn) {
+                if (exists)
+                {
+                    if ((pn-1+rep)%n==gn)
+                    {
                         ERRLOG("Logical file for %s exists (part not orphaned?)",title);
                         errstr.appendf("Logical file for %s exists (part not orphaned?)",title);
                         return false;
@@ -145,7 +156,8 @@ bool CXRefFilesNode::RemovePhysical(const char *Partmask,IUserDescriptor* udesc,
 {   
     size32_t startlen = errstr.length();
     IPropertyTree* subBranch = FindNode(Partmask);
-    if (!subBranch) {
+    if (!subBranch)
+    {
         ERRLOG("%s branch not found",Partmask);
         errstr.appendf("ERROR: %s branch not found",Partmask);
         return false;
@@ -153,7 +165,8 @@ bool CXRefFilesNode::RemovePhysical(const char *Partmask,IUserDescriptor* udesc,
     // sanity check file doesn't (now) exist 
     bool exists = false;
     StringBuffer lfn;
-    if (LogicalNameFromMask(Partmask,lfn)) {
+    if (LogicalNameFromMask(Partmask,lfn))
+    {
         if (queryDistributedFileDirectory().exists(lfn.str(),udesc,true)) 
             exists = true;
     }
@@ -172,7 +185,8 @@ bool CXRefFilesNode::RemovePhysical(const char *Partmask,IUserDescriptor* udesc,
         /////////////////////////////////
         StringBuffer xpath;
         unsigned i = 0;
-        for (;;) {
+        for (;;)
+        {
             i++;
             xpath.clear().appendf("Node[%d]",i);
             if (!part.hasProp(xpath.str())) 
@@ -183,7 +197,8 @@ bool CXRefFilesNode::RemovePhysical(const char *Partmask,IUserDescriptor* udesc,
             files.append(rmtFile);
         }
         i = 0;
-        for (;;) {
+        for (;;)
+        {
             i++;
             xpath.clear().appendf("RNode[%d]",i);
             if (!part.hasProp(xpath.str())) 
@@ -197,7 +212,6 @@ bool CXRefFilesNode::RemovePhysical(const char *Partmask,IUserDescriptor* udesc,
                 rmtFile.setPath(ip,remoteFile.str()); 
             files.append(rmtFile);
         }
-
     }
         
     CriticalSection crit;
@@ -217,8 +231,10 @@ bool CXRefFilesNode::RemovePhysical(const char *Partmask,IUserDescriptor* udesc,
             try{
                 Owned<IFile> _remoteFile =  createIFile(files.item(idx));
                 DBGLOG("Removing physical part at %s",_remoteFile->queryFilename());
-                if (_remoteFile->exists()) {
-                    if (!_remoteFile->remove()) {
+                if (_remoteFile->exists())
+                {
+                    if (!_remoteFile->remove())
+                    {
                         StringBuffer errname;
                         files.item(idx).getRemotePath(errname);
                         ERRLOG("Could not delete file %s",errname.str());
@@ -272,20 +288,23 @@ bool CXRefFilesNode::RemoveLogical(const char* LogicalName,IUserDescriptor* udes
     StringBuffer tmpbuf;
         
 
-    IPropertyTree* pLogicalFileNode =  getDataTree().getBranch(xpath.str());
-    if (!pLogicalFileNode) {
+    IPropertyTree* pLogicalFileNode =  queryDataTree().getBranch(xpath.str());
+    if (!pLogicalFileNode)
+    {
         ERRLOG("Branch %s not found",xpath.str());
         errstr.appendf("Branch %s not found",xpath.str());
         return false;
     }
     if (!checkPartsInCluster(LogicalName,clustername,pLogicalFileNode,errstr,false))
         return false;
-    if (queryDistributedFileDirectory().existsPhysical(LogicalName,udesc)) {
+    if (queryDistributedFileDirectory().existsPhysical(LogicalName,udesc))
+    {
         ERRLOG("Logical file %s all parts exist (not lost?))",LogicalName);
         errstr.appendf("Logical file %s all parts exist (not lost?))",LogicalName);
         return false;
     }
-    if (!getDataTree().removeTree(pLogicalFileNode)) {                  
+    if (!queryDataTree().removeTree(pLogicalFileNode))
+    {
         ERRLOG("Removing XRef Branch %s", xpath.str());
         errstr.appendf("Removing XRef Branch %s", xpath.str());
         return false;
@@ -298,7 +317,8 @@ bool CXRefFilesNode::RemoveLogical(const char* LogicalName,IUserDescriptor* udes
 bool CXRefFilesNode::AttachPhysical(const char *Partmask,IUserDescriptor* udesc, const char *clustername, StringBuffer &errstr)
 {
     IPropertyTree* subBranch = FindNode(Partmask);
-    if (!subBranch) {
+    if (!subBranch)
+    {
         ERRLOG("%s node not found",Partmask);
         errstr.appendf("ERROR: %s node not found",Partmask);
         return false;
@@ -307,7 +327,8 @@ bool CXRefFilesNode::AttachPhysical(const char *Partmask,IUserDescriptor* udesc,
         return false;
 
     StringBuffer logicalName;
-    if (!LogicalNameFromMask(Partmask,logicalName)) {
+    if (!LogicalNameFromMask(Partmask,logicalName))
+    {
         ERRLOG("%s - could not attach",Partmask);
         errstr.appendf("ERROR: %s - could not attach",Partmask);
         return false;
@@ -441,7 +462,7 @@ bool CXRefFilesNode::RemoveTreeNode(const char* NodeName)
     if (!subBranch)
         return false;
     StringBuffer tmpbuf;
-    return getDataTree().removeTree(subBranch);
+    return queryDataTree().removeTree(subBranch);
 }
 
 bool CXRefFilesNode::RemoveRemoteFile(const char* fileName,  const char* ipAddress)
@@ -484,5 +505,5 @@ void CXRefOrphanFilesNode::CleanTree(IPropertyTree& inTree)
         Itr->next();
     }
     if(partcount != 0)
-            inTree.setPropInt("Partsfound",partcount);
+        inTree.setPropInt("Partsfound",partcount);
 }

dali/dfuXRefLib/XRefFilesNode.hpp

@@ -38,6 +38,7 @@ interface IXRefFilesNode : extends IInterface
     virtual bool RemovePhysical(const char* Partmask,IUserDescriptor* udesc,const char *clustername, StringBuffer &errstr) = 0;
     virtual bool AttachPhysical(const char* Partmask,IUserDescriptor* udesc,const char *clustername, StringBuffer &errstr) = 0;
     virtual bool RemoveLogical(const char* LogicalName,IUserDescriptor* udesc,const char *clustername,StringBuffer &errstr) = 0;
+    virtual IPropertyTreeIterator *getMatchingFiles(const char *match, const char *type) = 0;
     virtual bool IsChanged() = 0;
     virtual void Commit() = 0;
 };
@@ -48,29 +49,30 @@ protected:
     bool m_bChanged;
     IPropertyTree& m_baseTree;
     Owned<IPropertyTree> m_DataTree;
-    StringBuffer _data;
+    MemoryBuffer _data;
     StringBuffer prefixName;
     StringAttr rootdir;
 private:
     IPropertyTree* FindNode(const char* NodeName);
-    IPropertyTree& getDataTree();
+    IPropertyTree& queryDataTree();
     void DirectoryFromMask(const char* Partmask,StringBuffer& directory);
     bool LogicalNameFromMask(const char* Partmask,StringBuffer& logicalName);
     bool RemoveTreeNode(const char* NodeName);
     bool RemoveRemoteFile(const char* fileName,  const char* ipAddress);
+    MemoryBuffer &queryData();
     virtual void CleanTree(IPropertyTree& inTree){}
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
     CXRefFilesNode(IPropertyTree& baseNode,const char* cluster, const char *rootdir);
     virtual ~CXRefFilesNode(){};
-    virtual bool IsChanged();
-    void Commit();
-    virtual StringBuffer& Serialize(StringBuffer& outStr);
-    virtual void Deserialize(IPropertyTree& inTree);
-    virtual bool RemovePhysical(const char* Partmask,IUserDescriptor* udesc,const char *clustername,StringBuffer &errstr);
-    virtual bool RemoveLogical(const char* LogicalName,IUserDescriptor* udesc,const char *clustername,StringBuffer &errstr);
-    virtual bool AttachPhysical(const char* Partmask,IUserDescriptor* udesc,const char *clustername,StringBuffer &errstr);
-
+    virtual bool IsChanged() override;
+    void Commit() override;
+    virtual StringBuffer& Serialize(StringBuffer& outStr) override;
+    virtual void Deserialize(IPropertyTree& inTree) override;
+    virtual bool RemovePhysical(const char* Partmask,IUserDescriptor* udesc,const char *clustername,StringBuffer &errstr) override;
+    virtual bool RemoveLogical(const char* LogicalName,IUserDescriptor* udesc,const char *clustername,StringBuffer &errstr) override;
+    virtual bool AttachPhysical(const char* Partmask,IUserDescriptor* udesc,const char *clustername,StringBuffer &errstr) override;
+    virtual IPropertyTreeIterator *getMatchingFiles(const char *match, const char *type) override;
 };
 
 class CXRefOrphanFilesNode : public CXRefFilesNode

dali/dfuXRefLib/XRefNodeManager.cpp

@@ -107,6 +107,7 @@ CXRefNode::CXRefNode(const char* NodeName, IRemoteConnection *_conn)
     IPropertyTree* cluster_ptree = m_conn->queryRoot()->queryPropTree(xpath.str());
     m_XRefTree.set(cluster_ptree);
     m_XRefTree->getProp("@name",m_origName);
+    rootDir.set(m_XRefTree->queryProp("@rootdir"));
     //DBGLOG("returning from CXRefNode::CXRefNode(const char* NodeName)");
 
 }
@@ -117,6 +118,7 @@ CXRefNode::CXRefNode(IPropertyTree* pTreeRoot)
     try
     {
         m_XRefTree.set(pTreeRoot);
+        rootDir.set(m_XRefTree->queryProp("@rootdir"));
         pTreeRoot->getProp("@name",m_origName);
         //load up our tree with the data.....if there is data
         MemoryBuffer buff;
@@ -191,9 +193,8 @@ IXRefFilesNode* CXRefNode::getLostFiles()
             lostBranch = m_XRefTree->addPropTree("Lost",createPTree());
             commit();
         }
-        const char *rootdir = m_XRefTree.get()?m_XRefTree->queryProp("@rootdir"):NULL;
         StringBuffer tmpbuf;
-        m_lost.setown(new CXRefFilesNode(*lostBranch,getName(tmpbuf).str(),rootdir));
+        m_lost.setown(new CXRefFilesNode(*lostBranch,getName(tmpbuf).str(),rootDir));
     }
     return m_lost.getLink();
 }
@@ -208,9 +209,8 @@ IXRefFilesNode* CXRefNode::getFoundFiles()
             foundBranch = m_XRefTree->addPropTree("Found",createPTree());
             commit();
         }
-        const char *rootdir = m_XRefTree.get()?m_XRefTree->queryProp("@rootdir"):NULL;
         StringBuffer tmpbuf;
-        m_found.setown(new CXRefFilesNode(*foundBranch,getName(tmpbuf).str(),rootdir));
+        m_found.setown(new CXRefFilesNode(*foundBranch,getName(tmpbuf).str(),rootDir));
     }
     return m_found.getLink();
 }
@@ -225,9 +225,8 @@ IXRefFilesNode* CXRefNode::getOrphanFiles()
             orphanBranch = m_XRefTree->addPropTree("Orphans",createPTree());
             commit();
         }
-        const char *rootdir = m_XRefTree.get()?m_XRefTree->queryProp("@rootdir"):NULL;
         StringBuffer tmpbuf;
-        m_orphans.setown(new CXRefOrphanFilesNode(*orphanBranch,getName(tmpbuf).str(),rootdir));
+        m_orphans.setown(new CXRefOrphanFilesNode(*orphanBranch,getName(tmpbuf).str(),rootDir));
     }
     return m_orphans.getLink();
 }

dali/dfuXRefLib/XRefNodeManager.hpp

@@ -53,6 +53,7 @@ interface IConstXRefNode : extends IInterface
     virtual IXRefFilesNode* getFoundFiles() = 0;
     virtual IXRefFilesNode* getOrphanFiles() = 0;
     virtual StringBuffer& getCluster(StringBuffer& Cluster) = 0;
+    virtual const char *queryRootDir() const = 0;
     virtual bool useSasha() = 0;
 };
 
@@ -94,6 +95,7 @@ private:
     StringBuffer m_dataStr;
     StringBuffer m_ClusterName;
     Mutex commitMutex;
+    StringAttr rootDir;
 
 private:
     IPropertyTree& getDataTree();
@@ -106,29 +108,30 @@ public:
     virtual ~CXRefNode();
     bool IsChanged();
     void SetChanged(bool bChanged);
+    void setLastModified(IJlibDateTime& DateTime);
 
     //IXRefProgressCallback
     void progress(const char *text);
     void error(const char *text);
 
     //IConstXRefNode
-    StringBuffer & getName(StringBuffer & str);
-    StringBuffer & getLastModified(StringBuffer & str);
-    bool useSasha();
+    virtual StringBuffer & getName(StringBuffer & str) override;
+    virtual StringBuffer & getLastModified(StringBuffer & str) override;
+    virtual bool useSasha() override;
+    virtual const char *queryRootDir() const override { return rootDir; }
     
-    StringBuffer& getXRefData(StringBuffer & buff);
-    StringBuffer& getStatus(StringBuffer & buff);
-    void setLastModified(IJlibDateTime& DateTime);
-    IXRefFilesNode* getLostFiles();
-    IXRefFilesNode* getFoundFiles();
-    IXRefFilesNode* getOrphanFiles();
-    StringBuffer &serializeMessages(StringBuffer &buf);
-    void deserializeMessages(IPropertyTree& inTree);
-    StringBuffer &serializeDirectories(StringBuffer &buf);
-    void deserializeDirectories(IPropertyTree& inTree);
-    bool removeEmptyDirectories(StringBuffer &errstr);
+    virtual StringBuffer& getXRefData(StringBuffer & buff) override;
+    virtual StringBuffer& getStatus(StringBuffer & buff) override;
+    virtual IXRefFilesNode* getLostFiles() override;
+    virtual IXRefFilesNode* getFoundFiles() override;
+    virtual IXRefFilesNode* getOrphanFiles() override;
     //IXRefNode
-    void setName(const char* str);
+    virtual StringBuffer &serializeMessages(StringBuffer &buf) override;
+    virtual void deserializeMessages(IPropertyTree& inTree) override;
+    virtual StringBuffer &serializeDirectories(StringBuffer &buf) override;
+    virtual void deserializeDirectories(IPropertyTree& inTree) override;
+    virtual bool removeEmptyDirectories(StringBuffer &errstr) override;
+    virtual void setName(const char* str) override;
     
     void setXRefData(StringBuffer & buff);
     void setXRefData(IPropertyTree & pTree);

dali/dfuXRefLib/dfuxreflib.cpp

@@ -2732,30 +2732,171 @@ IPropertyTree * runXRefCluster(const char *cluster,IXRefNode *nodeToUpdate)
 }
 
 
-IPropertyTree * RunProcess(unsigned nclusters,const char **clusters,unsigned numdirs,const char **dirbaselist,unsigned flags,IXRefProgressCallback *_msgcallback,unsigned numthreads)
+IPropertyTree * RunProcess(XRefCmd cmd, unsigned nclusters,const char **clusters,unsigned numArgs,const char **args,unsigned flags,IXRefProgressCallback *_msgcallback,unsigned numthreads)
 {
     //Provide a wrapper for the command line
-    if (flags & PMupdateeclwatch) {
-        if (nclusters==1) {
-            const char *cluster = *clusters;
-            CXRefNodeManager nodemanager;
-            Owned<IPropertyTree> tree = runXRef(nclusters,clusters,NULL,numthreads);
-            if (tree) {
-                Owned<IXRefNode> xRefNode = nodemanager.getXRefNode(cluster);
-                if (!xRefNode.get())
-                    xRefNode.setown( nodemanager.CreateXRefNode(cluster));
-                xRefNode->setCluster(cluster);
-                xRefNode->BuildXRefData(*tree.get(),cluster);
-                xRefNode->commit();
+    switch (cmd)
+    {
+        case xrefUpdate:
+        {
+            if (flags & PMupdateeclwatch)
+            {
+                if (nclusters==1)
+                {
+                    const char *cluster = *clusters;
+                    CXRefNodeManager nodemanager;
+                    Owned<IPropertyTree> tree = runXRef(nclusters,clusters,NULL,numthreads);
+                    if (tree)
+                    {
+                        Owned<IXRefNode> xRefNode = nodemanager.getXRefNode(cluster);
+                        if (!xRefNode.get())
+                            xRefNode.setown( nodemanager.CreateXRefNode(cluster));
+                        xRefNode->setCluster(cluster);
+                        xRefNode->BuildXRefData(*tree.get(),cluster);
+                        xRefNode->commit();
+                    }
+                }
+                else
+                {
+                    // do clusters 1 at time
+                    for (unsigned i = 0; i<nclusters; i++)
+                        RunProcess(cmd,1,clusters+i,numArgs,args,flags,_msgcallback,numthreads);
+                }
             }
+            break;
         }
-        else {
-            // do clusters 1 at time
-            for (unsigned i = 0; i<nclusters; i++)
-                RunProcess(1,clusters+i,numdirs,dirbaselist,flags,_msgcallback,numthreads);
+        case xrefScan:
+        {
+            CXRefManager xrefmanager;
+            return xrefmanager.process(nclusters,clusters,numArgs,args,flags,_msgcallback,numthreads);
+        }
+        case xrefListFound:
+        case xrefAttachFound:
+        {
+            if (1 == nclusters)
+            {
+                const char *match = "*"; // default
+                if (numArgs)
+                    match = args[0];
+                const char *cluster = clusters[0];
+                Owned<IXRefNodeManager> XRefNodeManager = CreateXRefNodeFactory();
+                Owned<IConstXRefNode> xRefNode = XRefNodeManager->getXRefNode(cluster);
+                if (!xRefNode)
+                    WARNLOG("Cannot find XREF info for cluster: %s", cluster);
+                else
+                {
+                    StringBuffer partMask;
+                    if (streq("*", match))
+                        partMask.append(match);
+                    else
+                    {
+                        partMask.append(xRefNode->queryRootDir());
+                        addPathSepChar(partMask);
+                        const char *s = match;
+                        do
+                        {
+                            const char *next = strstr(s, "::");
+                            if (!next)
+                            {
+                                partMask.append(s);
+                                break;
+                            }
+                            else
+                                partMask.append(next-s, s).append('/');
+                            s = next+2;
+                        }
+                        while (true);
+                        partMask.append(".*");
+                    }
+                    Owned<IXRefFilesNode> foundFiles = xRefNode->getFoundFiles();
+                    Owned<IPropertyTreeIterator> iter = foundFiles->getMatchingFiles(partMask, "Partmask");
+                    unsigned processed = 0;
+                    StringArray matched;
+                    ForEach(*iter)
+                    {
+                        IPropertyTree &file = iter->query();
+                        const char *partMask = file.queryProp("Partmask");
+                        switch (cmd)
+                        {
+                            case xrefListFound:
+                            {
+                                CDfsLogicalFileName lfn;
+                                if (!lfn.setFromMask(partMask, xRefNode->queryRootDir()))
+                                {
+                                    fprintf(stderr, "Error processing partMask=%s, could not deduce logical filename\n", partMask);
+                                    continue;
+                                }
+                                const char *logicalName = lfn.get();
+                                unsigned __int64 size = file.getPropInt64("Size");
+                                const char *modified = file.queryProp("Modified");
+                                unsigned numParts = file.getPropInt("Numparts");
+                                if (!processed)
+                                {
+                                    PROGLOG("Name,Size,Modified,NumParts");
+                                    PROGLOG("===========================");
+                                }
+                                PROGLOG("%s,%" I64F "u,%s,%u", logicalName, size, modified, numParts);
+                                ++processed;
+                                break;
+                            }
+                            case xrefAttachFound:
+                            {
+                                matched.append(partMask);
+                                break;
+                            }
+                            default:
+                                throwUnexpected();
+                        }
+                    }
+                    switch (cmd)
+                    {
+                        case xrefListFound:
+                        {
+                            PROGLOG("%u files found", processed);
+                            break;
+                        }
+                        case xrefAttachFound:
+                        {
+                            iter.clear();
+                            PROGLOG("%u found files matched", matched.ordinality());
+                            ForEachItemIn(f, matched)
+                            {
+                                const char *partMask = matched.item(f);
+                                CDfsLogicalFileName lfn;
+                                if (!lfn.setFromMask(partMask, xRefNode->queryRootDir()))
+                                {
+                                    fprintf(stderr, "Error processing partMask=%s, could not deduce logical filename\n", partMask);
+                                    continue;
+                                }
+                                const char *logicalName = lfn.get();
+                                StringBuffer errStr;
+                                if (foundFiles->AttachPhysical(partMask, nullptr, cluster, errStr))
+                                {
+                                    PROGLOG("File '%s' attached", lfn.get());
+                                    processed++;
+                                }
+                                else
+                                    fprintf(stderr, "%s\n", errStr.str());
+                            }
+
+                            PROGLOG("%u files re-attached. Committing xref meta data changes..", processed);
+                            if (processed>0)
+                                foundFiles->Commit();
+                            break;
+                        }
+                        default:
+                            break;
+                    }
+                }
+            }
+            else
+            {
+                // do clusters 1 at time
+                for (unsigned i = 0; i<nclusters; i++)
+                    RunProcess(cmd,1,clusters+i,numArgs,args,flags,_msgcallback,numthreads);
+            }
+            break;
         }
-        return NULL;
     }
-    CXRefManager xrefmanager;
-    return xrefmanager.process(nclusters,clusters,numdirs,dirbaselist,flags,_msgcallback,numthreads);
+    return nullptr;
 }

dali/dfuXRefLib/dfuxreflib.hpp

@@ -32,9 +32,11 @@
 #define PMbackupoutput  0x08
 #define PMupdateeclwatch 0x10
 
+enum XRefCmd { xrefNulCmd, xrefScan, xrefUpdate, xrefListFound, xrefAttachFound };
+
 
 extern  DFUXREFLIB_API IPropertyTree * runXRef(unsigned numclusters,const char **clusternames,IXRefProgressCallback *callback,unsigned numthreads);
-extern  DFUXREFLIB_API IPropertyTree * RunProcess(unsigned nclusters,const char **clusters,unsigned numdirs,const char **dirbaselist,unsigned flags,IXRefProgressCallback *_msgcallback,unsigned numthreads);
+extern  DFUXREFLIB_API IPropertyTree * RunProcess(XRefCmd cmd,unsigned nclusters,const char **clusters,unsigned nargs,const char **args,unsigned flags,IXRefProgressCallback *_msgcallback,unsigned numthreads);
 
 extern  DFUXREFLIB_API IPropertyTree * runXRefCluster(const char *cluster,IXRefNode *nodeToUpdate);
 // this will use sasha if enabled

dali/dfuxref/dfuxrefmain.cpp

@@ -40,6 +40,8 @@ void usage(const char *progname)
 {
     printf("usage DFUXREF <dali-server> /c:<cluster-name> /d:<dir-name> [ /backupcheck ] \n");
     printf("or    DFUXREF <dali-server> /e:<cluster-name>  -- updates ECLwatch information\n");
+    printf("or    DFUXREF <dali-server> /lf:<cluster-name> [file-pattern] -- lists matching found files on cluster\n");
+    printf("or    DFUXREF <dali-server> /af:<cluster-name> [/u:username /p:password] [/y] [file-pattern] -- attach matching found files on cluster\n");
 }
 
 int main(int argc, char* argv[])
@@ -72,60 +74,130 @@ int main(int argc, char* argv[])
     ep.set(argv[1],DALI_SERVER_PORT);
     epa.append(ep);
     Owned<IGroup> group = createIGroup(epa); 
-    try {
+    try
+    {
         initClientProcess(group,DCR_Dfu);
         setPasswordsFromSDS();
-        unsigned ndirs = 0;
-        unsigned nclusters = 0;
-        const char **dirs = (const char **)malloc(argc*sizeof(const char *));
-        const char **clusters = (const char **)malloc(argc*sizeof(const char *));
+        StringArray args, clusters;
         bool backupcheck = false;
         unsigned mode = PMtextoutput|PMcsvoutput|PMtreeoutput;
-        for (i=2;i<(unsigned)argc;i++) {
-            if (argv[i][0]=='/') {
-                if (stricmp(argv[i],"/backupcheck")==0) {
+        const char *listMask = nullptr;
+        StringAttr username, password;
+        bool confirmed = false;
+        XRefCmd xrefCmd = xrefNulCmd;
+
+        for (i=2;i<(unsigned)argc;i++)
+        {
+            if (argv[i][0]=='/')
+            {
+                if (stricmp(argv[i],"/backupcheck")==0)
                     mode = PMbackupoutput;
+                else if (hasPrefix(argv[i], "/lf:", false))
+                {
+                    if (xrefCmd != xrefNulCmd)
+                        break;
+                    xrefCmd = xrefListFound;
+                    const char *arg = argv[i]+4;
+                    clusters.append(arg);
                 }
-                else  {
+                else if (hasPrefix(argv[i], "/af:", false))
+                {
+                    if (xrefCmd != xrefNulCmd)
+                        break;
+                    xrefCmd = xrefAttachFound;
+                    const char *arg = argv[i]+4;
+                    clusters.append(arg);
+                }
+                else
+                {
                     const char *arg="";
                     unsigned ni=i;
-                    if (argv[i][1]&&(argv[i][2]==':')) {
+                    if (argv[i][1]&&(argv[i][2]==':'))
                         arg = argv[i]+3;
-                    }
-                    else  if ((i+1<(unsigned)argc)&&argv[i+1][0]) {
+                    else if ((i+1<(unsigned)argc)&&(argv[i+1][0] != '/'))
+                    {
                         ni++;
                         arg = argv[ni];
                     }
-                    switch (toupper(argv[i][1])) {
+                    switch (toupper(argv[i][1]))
+                    {
                     case 'E':
-                        // fall through
-                        mode = PMtextoutput|PMupdateeclwatch;
+                        mode = PMtextoutput;
+                        if (xrefCmd != xrefNulCmd)
+                            break;
+                        xrefCmd = xrefUpdate;
+                        clusters.append(arg);
+                        break;
                     case 'C':
-                        clusters[nclusters++] = arg;
+                        if (xrefCmd != xrefNulCmd)
+                            break;
+                        xrefCmd = xrefScan;
+                        clusters.append(arg);
                         break;
                     case 'D':
-                        dirs[ndirs++] = arg;
+                        if ((xrefCmd != xrefScan))
+                            break;
+                        args.append(arg);
+                        break;
+                    case 'U':
+                        username.set(arg);
+                        break;
+                    case 'P':
+                        password.set(arg);
+                        break;
+                    case 'Y':
+                        confirmed = true;
                         break;
                     default:
-                        ni = argc;
-                        nclusters = 0;
+                        break;
                     }
                     i = ni;
                 }
             }
+            else
+                args.append(argv[i]);
         }
 
-        if (nclusters==0)
+        if (clusters.ordinality()==0 || xrefCmd == xrefNulCmd)
             usage(argv[0]);
-        else {
+        else
+        {
+            Owned<IUserDescriptor> userDesc;
+            if (username)
+            {
+                userDesc.setown(createUserDescriptor());
+                userDesc->set(username, password);
+                queryDistributedFileDirectory().setDefaultUser(userDesc);
+            }
+            if (xrefAttachFound == xrefCmd)
+            {
+                const char *match = "*"; // default
+                if (args.ordinality())
+                    match = args.item(0);
+                PROGLOG("This will re-attach all found file matching: %s", match);
+                int ch;
+                if (!confirmed)
+                {
+                    PROGLOG("Are you sure? [Y/N]");
+                    do
+                    {
+                        ch = toupper(ch = getchar());
+                    } while (ch != 'Y' && ch != 'N');
+                    PROGLOG("%c",ch);
+                    if (ch == 'Y')
+                        confirmed = true;
+                }
+                if (!confirmed)
+                    throw MakeStringException(0, "Aborted");
+            }
             DBGLOG("Starting%s",cmdline.str());
-            IPropertyTree * pReturnTree = RunProcess(nclusters,clusters,ndirs,dirs,mode,NULL,4);
-            if (pReturnTree) {
+            IPropertyTree * pReturnTree = RunProcess(xrefCmd, clusters.ordinality(), clusters.getArray(), args.ordinality(), args.getArray(), mode, NULL, 4);
+            if (pReturnTree)
                 saveXML("dfutree.xml",pReturnTree);
-            }
         }
     }
-    catch (IException *e) {
+    catch (IException *e)
+    {
         pexception("Exception",e);
         e->Release();
     }