HPCC-14312 Refactor ESP DFUXref service

1. Remove member variables (user_, password_) which are never
used;
2. Add getUserDescriptor() which are shared by 2 callers;
3. Add getXRefNodeByCluster() which are shared by multiple
callers;
4. Does not create a DFUXRefBuild request if the same request
has already been queued;
5. Replace skipTilda() calls with the DFS calls for file name
normalization;
6. Clean the existing code.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

esp/services/ws_dfu/ws_dfuXRefService.cpp

@@ -21,6 +21,7 @@
 
 #include "dadfs.hpp"
 #include "daft.hpp"
+#include "dautils.hpp"
 #include "wshelpers.hpp"
 #include "exception_util.hpp"
 #include "package.h"
@@ -80,23 +81,15 @@ static void dfuXrefXMLToJSON(StringBuffer& buf)
 
 void CWsDfuXRefEx::init(IPropertyTree *cfg, const char *process, const char *service)
 {
-    
-    StringBuffer xpath;
-    
     DBGLOG("Initializing %s service [process = %s]", service, process);
-    
-    xpath.clear().appendf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/User", process, service);
-    cfg->getProp(xpath.str(), user_);
-
-    xpath.clear().appendf("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]/Password", process, service);
-    cfg->getProp(xpath.str(), password_);
 
     if (!daliClientActive())
     {
         OERRLOG("No Dali Connection Active.");
         throw MakeStringException(-1, "No Dali Connection Active. Please Specify a Dali to connect to in you configuration file");
     }
-    XRefNodeManager.setown(CreateXRefNodeFactory());    
+
+    XRefNodeManager.setown(CreateXRefNodeFactory());
 
     //Start out builder thread......
     m_XRefbuilder.setown(new CXRefExBuilderThread());
@@ -107,130 +100,114 @@ bool CWsDfuXRefEx::onDFUXRefArrayAction(IEspContext &context, IEspDFUXRefArrayAc
 {
     try
     {
-        StringBuffer username;
-        context.getUserID(username);
-
-        Owned<IUserDescriptor> userdesc;
-        if(username.length() > 0)
-        {
-            userdesc.setown(createUserDescriptor());
-            userdesc->set(username.str(), context.queryPassword(), context.querySignature());
-        }
-
-        if(*req.getAction() == 0 || *req.getType() == 0 || *req.getCluster() == 0)
-        {
-            IERRLOG("Action, Type and Cluster must be specified in the DFUXRefArrayAction ");
-            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Action, cluster, or type not defined.");
-        }
-        
-        Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(req.getCluster());
-        if (xRefNode.get() == 0)
-        {
-            IERRLOG("Unable to resolve XRef cluster name %s",req.getCluster());
-            throw MakeStringException(ECLWATCH_CANNOT_RESOLVE_CLUSTER_NAME, "Unable to resolve cluster name %s",req.getCluster());
-        }
+        context.ensureFeatureAccess(FEATURE_URL, SecAccess_Full, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefArrayAction: Permission denied.");
 
-        
-        Owned<IXRefFilesNode> _fileNode = getFileNodeInterface(*xRefNode.get(),req.getType());
-        if (_fileNode.get() == 0)
+        const char *action = req.getAction();
+        const char *type = req.getType();
+        if (isEmptyString(action) || isEmptyString(type))
+            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Action or Type not defined.");
+        if (!streq("Attach", action) && !streq("Delete", action) && !streq("DeleteLogical", action))
+            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Invalid DFUXRefArrayAction: only Attach, Delete or DeleteLogical allowed.");
+
+        StringArray &xrefFiles = req.getXRefFiles();
+        if (xrefFiles.ordinality() == 0)
+            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "XRefFile not defined.");
+
+        const char *cluster = req.getCluster();
+        Owned<IXRefNode> xRefNode = getXRefNodeByCluster(cluster);
+        Owned<IXRefFilesNode> fileNode = getFileNodeInterface(*xRefNode, type);
+        if (!fileNode)
         {
-            IERRLOG("Unable to find a suitable IXRefFilesNode interface for %s",req.getType());
-            throw MakeStringException(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "Unable to find a suitable IXRefFilesNode interface for %s",req.getType());
+            IERRLOG("Unable to find a suitable IXRefFilesNode interface for %s", type);
+            throw MakeStringException(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "Unable to find a suitable IXRefFilesNode interface for %s", type);
         }
 
-        context.ensureFeatureAccess(FEATURE_URL, SecAccess_Full, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefArrayAction: Permission denied.");
-
-        StringBuffer returnStr,UserName;
-        const char* ActionType = req.getAction();
+        StringBuffer returnStr;
         ESPSerializationFormat fmt = context.getResponseFormat();
-        for(unsigned i = 0; i < req.getXRefFiles().length();i++)
+        Owned<IUserDescriptor> userDesc = getUserDescriptor(context);
+        ForEachItemIn(i, xrefFiles)
         {
-            StringBuffer errstr;
-            if (strcmp("Delete" ,ActionType) == 0)
+            StringBuffer err;
+            const char *file = xrefFiles.item(i);
+            if (streq("Attach", action))
             {
-                if (_fileNode->RemovePhysical(req.getXRefFiles().item(i),userdesc,req.getCluster(),errstr))
-                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr,NULL,"Removed Physical part %s",req.getXRefFiles().item(i));
+                if(fileNode->AttachPhysical(file, userDesc, cluster, err))
+                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr, nullptr,
+                        "Reattached Physical part %s", file);
                 else
-                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr,NULL,"Error(s) removing physical part %s\n%s",req.getXRefFiles().item(i),errstr.str());
+                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr, nullptr,
+                        "Error(s) attaching physical part %s\n%s", file, err.str());
             }
-            else if (strcmp("Attach" ,ActionType) == 0)
+            else if (streq("Delete", action))
             {
-                if(_fileNode->AttachPhysical(req.getXRefFiles().item(i),userdesc,req.getCluster(),errstr) )
-                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr,NULL,"Reattached Physical part %s",req.getXRefFiles().item(i));
+                if (fileNode->RemovePhysical(file, userDesc, cluster, err))
+                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr, nullptr,
+                        "Removed Physical part %s", file);
                 else
-                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr,NULL,"Error(s) attaching physical part %s\n%s",req.getXRefFiles().item(i),errstr.str());
+                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr, nullptr,
+                        "Error(s) removing physical part %s\n%s", file, err.str());
             }
-            if (strcmp("DeleteLogical" ,ActionType) == 0)
-            {
+            else 
+            {   // DeleteLogical:
                 // Note we don't want to physically delete 'lost' files - this will end up with orphans on next time round but that is safer
-                if (_fileNode->RemoveLogical(req.getXRefFiles().item(i),userdesc,req.getCluster(),errstr)) {
-                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr,NULL,"Removed Logical File %s",req.getXRefFiles().item(i));
-                }
+                if (fileNode->RemoveLogical(file, userDesc, cluster, err))
+                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr, nullptr,
+                        "Removed Logical File %s", file);
                 else
-                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr,NULL,"Error(s) removing File %s\n%s",req.getXRefFiles().item(i),errstr.str());
+                    appendReplyMessage(fmt==ESPSerializationJSON, returnStr, nullptr,
+                        "Error(s) removing File %s\n%s", file, err.str());
             }
         }
 
         xRefNode->commit();
-        resp.setDFUXRefArrayActionResult(returnStr.str());
+        resp.setDFUXRefArrayActionResult(returnStr);
     }
-    catch(IException* e)
+    catch(IException *e)
     {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
     }
     return true;
 }
 
-IXRefFilesNode* CWsDfuXRefEx::getFileNodeInterface(IXRefNode& XRefNode,const char* nodeType)
+IXRefFilesNode *CWsDfuXRefEx::getFileNodeInterface(IXRefNode &XRefNode, const char *nodeType)
 {
-    if (strcmp("Found" ,nodeType) == 0)
+    if (strieq("Found", nodeType))
         return XRefNode.getFoundFiles();
-    else if (strcmp("Lost" ,nodeType) == 0)
+    else if (strieq("Lost", nodeType))
         return XRefNode.getLostFiles();
-    else if (strcmp("Orphan" ,nodeType) == 0)
+    else if (strieq("Orphan", nodeType))
         return XRefNode.getOrphanFiles();
-    else
-        OWARNLOG("Unrecognized file node type %s",nodeType);
-    return 0;
+
+    OWARNLOG("Unrecognized file node type %s", nodeType);
+    return nullptr;
 }
 
-void CWsDfuXRefEx::readLostFileQueryResult(IEspContext &context, StringBuffer& buf)
+void CWsDfuXRefEx::readLostFileQueryResult(IEspContext &context, StringBuffer &buf)
 {
-    Owned<IPropertyTree> lostFilesQueryResult = createPTreeFromXMLString(buf.str());
+    Owned<IPropertyTree> lostFilesQueryResult = createPTreeFromXMLString(buf);
     if (!lostFilesQueryResult)
-    {
-        PROGLOG("readLostFileQueryResult() failed in creating PTree.");
-        return;
-    }
-
-    StringBuffer username;
-    Owned<IUserDescriptor> userdesc;
-    context.getUserID(username);
-    if(username.length() > 0)
-    {
-        userdesc.setown(createUserDescriptor());
-        userdesc->set(username.str(), context.queryPassword(), context.querySignature());
-    }
+        throw MakeStringException(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "readLostFileQueryResult() failed in creating PTree.");
 
+    Owned<IUserDescriptor> userDesc = getUserDescriptor(context);
     Owned<IPropertyTreeIterator> iter = lostFilesQueryResult->getElements("File");
     ForEach(*iter)
     {
-        IPropertyTree& item = iter->query();
-        const char* fileName = item.queryProp("Name");
-        if (!fileName || !*fileName)
+        IPropertyTree &item = iter->query();
+        const char *fileName = item.queryProp("Name");
+        if (isEmptyString(fileName))
             continue;
 
         try
         {
-            Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userdesc, false, false, false, NULL, 0);
+            Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userDesc, false, false, false, NULL, 0);
             if(df)
                 item.addPropInt64("Size", queryDistributedFileSystem().getSize(df));
         }
-        catch(IException* e)
+        catch(IException *e)
         {
             item.addProp("Status", "Warning: this file may be locked now. It can't be recovered as locked.");
             StringBuffer eMsg;
-            PROGLOG("Exception in readLostFileQueryResult(): %s", e->errorMessage(eMsg).str());
+            IERRLOG("Exception in readLostFileQueryResult(): %s", e->errorMessage(eMsg).str());
             e->Release();
         }
     }
@@ -241,33 +218,28 @@ void CWsDfuXRefEx::readLostFileQueryResult(IEspContext &context, StringBuffer& b
         toXML(lostFilesQueryResult, buf.clear());
 }
 
-
 bool CWsDfuXRefEx::onDFUXRefLostFiles(IEspContext &context, IEspDFUXRefLostFilesQueryRequest &req, IEspDFUXRefLostFilesQueryResponse &resp)
 {
     try
     {
         context.ensureFeatureAccess(FEATURE_URL, SecAccess_Read, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefLostFiles: Permission denied.");
 
-        if (!req.getCluster() || !*req.getCluster())
-            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Cluster not defined.");
-
-        Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(req.getCluster());
-        if (xRefNode.get() == 0)
-            throw MakeStringExceptionDirect(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "XRefNode not found.");
+        Owned<IXRefNode> xRefNode = getXRefNodeByCluster(req.getCluster());
+        Owned<IXRefFilesNode> lostFiles = xRefNode->getLostFiles();
+        if (!lostFiles)
+            return true;
 
         StringBuffer buf;
-        Owned<IXRefFilesNode> _lost = xRefNode->getLostFiles();
-        if (_lost)
-        {
-            _lost->Serialize(buf);
-            if (!buf.isEmpty())
-                readLostFileQueryResult(context, buf);
-        }
-        resp.setDFUXRefLostFilesQueryResult(buf.str());
+        lostFiles->Serialize(buf);
+        if (buf.isEmpty())
+            return true;
+
+        readLostFileQueryResult(context, buf);
+        resp.setDFUXRefLostFilesQueryResult(buf);
     }
-    catch(IException* e)
+    catch(IException *e)
     {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
     }
     return true;
 }
@@ -279,33 +251,23 @@ bool CWsDfuXRefEx::onDFUXRefFoundFiles(IEspContext &context, IEspDFUXRefFoundFil
     {
         context.ensureFeatureAccess(FEATURE_URL, SecAccess_Read, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefFoundFiles: Permission denied.");
 
-        StringBuffer username;
-        context.getUserID(username);
-
-        if (!req.getCluster() || !*req.getCluster())
-            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Cluster not defined.");
-
-        Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(req.getCluster());
-        if (xRefNode.get() == 0)
-            throw MakeStringExceptionDirect(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "XRefNode not found.");
+        Owned<IXRefNode> xRefNode = getXRefNodeByCluster(req.getCluster());
+        Owned<IXRefFilesNode> foundFiles = xRefNode->getFoundFiles();
+        if (!foundFiles)
+            return true;
 
         StringBuffer buf;
-        Owned<IXRefFilesNode> _found = xRefNode->getFoundFiles();
-        if (_found)
-        {
-            _found->Serialize(buf);
-            if (!buf.isEmpty())
-            {
-                ESPSerializationFormat fmt = context.getResponseFormat();
-                if (fmt == ESPSerializationJSON)
-                    dfuXrefXMLToJSON(buf);
-            }
-        }
-        resp.setDFUXRefFoundFilesQueryResult(buf.str());
+        foundFiles->Serialize(buf);
+        if (buf.isEmpty())
+            return true;
+
+        if (context.getResponseFormat() == ESPSerializationJSON)
+            dfuXrefXMLToJSON(buf);
+        resp.setDFUXRefFoundFilesQueryResult(buf);
     }
-    catch(IException* e)
+    catch(IException *e)
     {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
     }
     return true;
 }
@@ -316,33 +278,23 @@ bool CWsDfuXRefEx::onDFUXRefOrphanFiles(IEspContext &context, IEspDFUXRefOrphanF
     {
         context.ensureFeatureAccess(FEATURE_URL, SecAccess_Read, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefOrphanFiles: Permission denied.");
 
-        StringBuffer username;
-        context.getUserID(username);
-
-        if (!req.getCluster() || !*req.getCluster())
-            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Cluster not defined.");
-
-        Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(req.getCluster());
-        if (xRefNode.get() == 0)
-            throw MakeStringExceptionDirect(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "XRefNode not found.");
+        Owned<IXRefNode> xRefNode = getXRefNodeByCluster(req.getCluster());
+        Owned<IXRefFilesNode> orphanFiles = xRefNode->getOrphanFiles();
+        if (!orphanFiles)
+            return true;
 
         StringBuffer buf;
-        Owned<IXRefFilesNode> _orphan = xRefNode->getOrphanFiles();
-        if (_orphan)
-        {
-            _orphan->Serialize(buf);
-            if (!buf.isEmpty())
-            {
-                ESPSerializationFormat fmt = context.getResponseFormat();
-                if (fmt == ESPSerializationJSON)
-                    dfuXrefXMLToJSON(buf);
-            }
-        }
-        resp.setDFUXRefOrphanFilesQueryResult(buf.str());
+        orphanFiles->Serialize(buf);
+        if (buf.isEmpty())
+            return true;
+
+        if (context.getResponseFormat() == ESPSerializationJSON)
+            dfuXrefXMLToJSON(buf);
+        resp.setDFUXRefOrphanFilesQueryResult(buf);
     }
-    catch(IException* e)
+    catch(IException *e)
     {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
     }
     return true;
 }
@@ -353,29 +305,19 @@ bool CWsDfuXRefEx::onDFUXRefMessages(IEspContext &context, IEspDFUXRefMessagesQu
     {
         context.ensureFeatureAccess(FEATURE_URL, SecAccess_Read, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefMessages: Permission denied.");
 
-        StringBuffer username;
-        context.getUserID(username);
-
-        if (!req.getCluster() || !*req.getCluster())
-            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Cluster not defined.");
-
-        Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(req.getCluster());
-        if (xRefNode.get() == 0)
-            throw MakeStringExceptionDirect(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "XRefNode not found.");
-
         StringBuffer buf;
+        Owned<IXRefNode> xRefNode = getXRefNodeByCluster(req.getCluster());
         xRefNode->serializeMessages(buf);
-        if (!buf.isEmpty())
-        {
-            ESPSerializationFormat fmt = context.getResponseFormat();
-            if (fmt == ESPSerializationJSON)
-                dfuXrefXMLToJSON(buf);
-        }
-        resp.setDFUXRefMessagesQueryResult(buf.str());
+        if (buf.isEmpty())
+            return true;
+
+        if (context.getResponseFormat() == ESPSerializationJSON)
+            dfuXrefXMLToJSON(buf);
+        resp.setDFUXRefMessagesQueryResult(buf);
     }
-    catch(IException* e)
+    catch(IException *e)
     {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
     }
     return true;
 }
@@ -386,23 +328,17 @@ bool CWsDfuXRefEx::onDFUXRefCleanDirectories(IEspContext &context, IEspDFUXRefCl
     {
         context.ensureFeatureAccess(FEATURE_URL, SecAccess_Write, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefCleanDirectories: Permission denied.");
 
-        StringBuffer username;
-        context.getUserID(username);
+        StringBuffer err;
+        Owned<IXRefNode> xRefNode = getXRefNodeByCluster(req.getCluster());
+        xRefNode->removeEmptyDirectories(err);
+        if (!err.isEmpty())
+            throw MakeStringException(ECLWATCH_INTERNAL_ERROR, "Failed in DFUXRefCleanDirectories: %s", err.str());
 
-        if (!req.getCluster() || !*req.getCluster())
-            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Cluster not defined.");
-
-        Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(req.getCluster());
-        if (xRefNode.get() == 0)
-            throw MakeStringExceptionDirect(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "XRefNode not found.");
-
-        StringBuffer buf;
-        xRefNode->removeEmptyDirectories(buf);
         resp.setRedirectUrl(StringBuffer("/WsDFUXRef/DFUXRefDirectories?Cluster=").append(req.getCluster()));
     }
-    catch(IException* e)
+    catch(IException *e)
     {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
     }
     return true;
 }
@@ -413,76 +349,69 @@ bool CWsDfuXRefEx::onDFUXRefDirectories(IEspContext &context, IEspDFUXRefDirecto
     {
         context.ensureFeatureAccess(FEATURE_URL, SecAccess_Read, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefDirectories: Permission denied.");
 
-        StringBuffer username;
-        context.getUserID(username);
+        StringBuffer buf;
+        Owned<IXRefNode> xRefNode = getXRefNodeByCluster(req.getCluster());
+        xRefNode->serializeDirectories(buf);
+        if (buf.isEmpty())
+            return true;
+
+        Owned<IPropertyTree> dirs = createPTreeFromXMLString(buf);
+        if (!dirs)
+            throw MakeStringException(ECLWATCH_INVALID_COMPONENT_INFO,
+                "Failed in creating PTree for XRefNode Directories: %s.", req.getCluster());
+
+        Owned<IPropertyTreeIterator> iter = dirs->getElements("Directory");
+        ForEach(*iter)
+            updateSkew(iter->query());
+
+        if (context.getResponseFormat() == ESPSerializationJSON)
+            toJSON(dirs, buf.clear());
+        else
+            toXML(dirs, buf.clear());
 
-        if (!req.getCluster() || !*req.getCluster())
-            throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Cluster not defined.");
+        resp.setDFUXRefDirectoriesQueryResult(buf);
+    }
+    catch(IException *e)
+    {   
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}
 
-        Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(req.getCluster());
-        if (xRefNode.get() == 0)
-            throw MakeStringExceptionDirect(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "XRefNode not found.");
+void CWsDfuXRefEx::updateSkew(IPropertyTree &node)
+{
+    char *skew = (char*) node.queryProp("Skew");
+    if (isEmptyString(skew))
+        return;
 
-        StringBuffer buf, buf0;
-        xRefNode->serializeDirectories(buf0);
-        if (!buf0.isEmpty())
+    StringBuffer positive, negative;
+    char *skewPtr = strchr(skew, '/');
+    if (skewPtr)
+    {
+        if (skew[0] == '+' && (strlen(skew) > 1))
+            positive.append(skewPtr - skew - 1, skew+1);
+        else
+            positive.append(skewPtr - skew, skew);
+        skewPtr++;
+        if (skewPtr)
         {
-            Owned <IPropertyTree> dirs = createPTreeFromXMLString(buf0.str()); // Why are we doing this?
-            if (!dirs)
-                throw MakeStringExceptionDirect(ECLWATCH_INVALID_COMPONENT_INFO, "Failed in creating PTree for XRefNode Directories.");
-
-            Owned<IPropertyTreeIterator> iter = dirs->getElements("Directory");
-            ForEach(*iter)
-            {
-                IPropertyTree &node = iter->query();
-
-                StringBuffer positive, negative;
-                char* skew = (char*) node.queryProp("Skew");
-                if (!skew || !*skew)
-                    continue;
-
-                char* skewPtr = strchr(skew, '/');
-                if (skewPtr)
-                {
-                    if (skew[0] == '+' && (strlen(skew) > 1))
-                        positive.append(skewPtr - skew - 1, skew+1);
-                    else
-                        positive.append(skewPtr - skew, skew);
-                    skewPtr++;
-                    if (skewPtr)
-                    {
-                        if (skewPtr[0] == '-')
-                            negative.append(skewPtr+1);
-                        else
-                            negative.append(skewPtr);
-                    }
-                }
-                else
-                {
-                    if (skew[0] == '+' && (strlen(skew) > 1))
-                        positive.append(skew+1);
-                    else
-                        positive.append(skew);
-                }
-
-                node.removeProp("Skew");
-                node.addProp("PositiveSkew", positive);
-                node.addProp("NegativeSkew", negative);
-            }
-
-            ESPSerializationFormat fmt = context.getResponseFormat();
-            if (fmt == ESPSerializationJSON)
-                toJSON(dirs, buf);
+            if (skewPtr[0] == '-')
+                negative.append(skewPtr+1);
             else
-                toXML(dirs, buf);
+                negative.append(skewPtr);
         }
-        resp.setDFUXRefDirectoriesQueryResult(buf.str());
     }
-    catch(IException* e)
-    {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    else
+    {
+        if (skew[0] == '+' && (strlen(skew) > 1))
+            positive.append(skew+1);
+        else
+            positive.append(skew);
     }
-    return true;
+
+    node.removeProp("Skew");
+    node.addProp("PositiveSkew", positive);
+    node.addProp("NegativeSkew", negative);
 }
 
 bool CWsDfuXRefEx::onDFUXRefBuild(IEspContext &context, IEspDFUXRefBuildRequest &req, IEspDFUXRefBuildResponse &resp)
@@ -491,34 +420,40 @@ bool CWsDfuXRefEx::onDFUXRefBuild(IEspContext &context, IEspDFUXRefBuildRequest
     {
         context.ensureFeatureAccess(FEATURE_URL, SecAccess_Full, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefBuild: Permission denied.");
 
-        StringBuffer username;
-        context.getUserID(username);
-
-        if (!req.getCluster() || !*req.getCluster())
+        const char *cluster = req.getCluster();
+        if (isEmptyString(cluster))
             throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Cluster not defined.");
 
-        //create the node if it doesn;t exist
-        Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(req.getCluster());
-        if (xRefNode.get() == 0)
-        {
-            xRefNode.setown( XRefNodeManager->CreateXRefNode(req.getCluster()));
-        }
         StringBuffer returnStr;
         ESPSerializationFormat fmt = context.getResponseFormat();
-        if (m_XRefbuilder->isQueued(req.getCluster()) )
-            appendReplyMessage(fmt == ESPSerializationJSON, returnStr,"/WsDFUXRef/DFUXRefList","An XRef build for cluster %s is in process. Click here to return to the main XRef List.",req.getCluster());
-        else if (!m_XRefbuilder->isRunning())
-            appendReplyMessage(fmt == ESPSerializationJSON, returnStr,"/WsDFUXRef/DFUXRefList","Running XRef Process. Click here to return to the main XRef List.");
-        else
-            appendReplyMessage(fmt == ESPSerializationJSON, returnStr,"/WsDFUXRef/DFUXRefList","someone is currently running a Xref build. Your request will be added to the queue. Please click here to return to the main page.");
+        if (m_XRefbuilder->isQueued(cluster))
+        { //The XRef build request for this cluster has been queued. No need to queue again.
+            appendReplyMessage(fmt == ESPSerializationJSON, returnStr, "/WsDFUXRef/DFUXRefList",
+                "An XRef build for cluster %s is in process. Click here to return to the main XRef List.", cluster);
+            resp.setDFUXRefActionResult(returnStr);
+            return true;
+        }
 
+        //create the node if it doesn;t exist
+        Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(cluster);
+        if (!xRefNode)
+            xRefNode.setown(XRefNodeManager->CreateXRefNode(cluster));
+        if (!xRefNode)
+            throw MakeStringException(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "XRefNode not created for %s.", cluster);
+
+        if (!m_XRefbuilder->isRunning())
+            appendReplyMessage(fmt == ESPSerializationJSON, returnStr,"/WsDFUXRef/DFUXRefList",
+                "Running XRef Process. Click here to return to the main XRef List.");
+        else
+            appendReplyMessage(fmt == ESPSerializationJSON, returnStr,"/WsDFUXRef/DFUXRefList",
+                "Someone is currently running a Xref build. Your request will be added to the queue. Please click here to return to the main page.");
 
-        m_XRefbuilder->queueRequest(xRefNode,req.getCluster());
-        resp.setDFUXRefActionResult(returnStr.str());
+        m_XRefbuilder->queueRequest(xRefNode, cluster);
+        resp.setDFUXRefActionResult(returnStr);
     }
-    catch(IException* e)
+    catch(IException *e)
     {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
     }
     return true;
 }
@@ -529,13 +464,9 @@ bool CWsDfuXRefEx::onDFUXRefBuildCancel(IEspContext &context, IEspDFUXRefBuildCa
     {
         context.ensureFeatureAccess(FEATURE_URL, SecAccess_Full, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefBuildCancel: Permission denied.");
 
-        StringBuffer username;
-        context.getUserID(username);
-
         m_XRefbuilder->cancel();
         StringBuffer returnStr;
-        ESPSerializationFormat fmt = context.getResponseFormat();
-        if (fmt == ESPSerializationJSON)
+        if (context.getResponseFormat() == ESPSerializationJSON)
         {
             returnStr.append("{ \"Message\": { \"Value\": ");
             returnStr.append("\"All Queued items have been cleared. The current running job will continue to execute.\",");
@@ -545,9 +476,9 @@ bool CWsDfuXRefEx::onDFUXRefBuildCancel(IEspContext &context, IEspDFUXRefBuildCa
             returnStr.appendf("<Message><Value>All Queued items have been cleared. The current running job will continue to execute.</Value><href>/WsDFUXRef/DFUXRefList</href></Message>");
         resp.setDFUXRefBuildCancelResult(returnStr.str());
     }
-    catch(IException* e)
+    catch(IException *e)
     {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
     }
     return true;
 }
@@ -570,15 +501,15 @@ void CWsDfuXRefEx::addXRefNode(const char* name, IPropertyTree* pXRefNodeTree)
     }
 }
 
-bool CWsDfuXRefEx::addUniqueXRefNode(const char* processName, BoolHash& uniqueProcesses, IPropertyTree* pXRefNodeTree)
+bool CWsDfuXRefEx::addUniqueXRefNode(const char *processName, BoolHash &uniqueProcesses, IPropertyTree *xrefNodeTree)
 {
     if (isEmptyString(processName))
         return false;
-    bool* found = uniqueProcesses.getValue(processName);
+    bool *found = uniqueProcesses.getValue(processName);
     if (found && *found)
         return false;
     uniqueProcesses.setValue(processName, true);
-    addXRefNode(processName, pXRefNodeTree);
+    addXRefNode(processName, xrefNodeTree);
     return true;
 }
 
@@ -588,14 +519,11 @@ bool CWsDfuXRefEx::onDFUXRefList(IEspContext &context, IEspDFUXRefListRequest &r
     {
         context.ensureFeatureAccess(FEATURE_URL, SecAccess_Read, ECLWATCH_DFU_XREF_ACCESS_DENIED, "WsDfuXRef::DFUXRefList: Permission denied.");
 
-        StringBuffer username;
-        context.getUserID(username);
-
         CConstWUClusterInfoArray clusters;
         getEnvironmentClusterInfo(clusters);
 
         BoolHash uniqueProcesses;
-        Owned<IPropertyTree> pXRefNodeTree = createPTree("XRefNodes");
+        Owned<IPropertyTree> xrefNodeTree = createPTree("XRefNodes");
         ForEachItemIn(c, clusters)
         {
             IConstWUClusterInfo &cluster = clusters.item(c);
@@ -605,44 +533,37 @@ bool CWsDfuXRefEx::onDFUXRefList(IEspContext &context, IEspDFUXRefListRequest &r
                 {
                     const StringArray &primaryThorProcesses = cluster.getPrimaryThorProcesses();
                     ForEachItemIn(i, primaryThorProcesses)
-                        addUniqueXRefNode(primaryThorProcesses.item(i), uniqueProcesses, pXRefNodeTree);
+                        addUniqueXRefNode(primaryThorProcesses.item(i), uniqueProcesses, xrefNodeTree);
                 }
                 break;
             case RoxieCluster:
                 SCMStringBuffer roxieProcess;
-                addUniqueXRefNode(cluster.getRoxieProcess(roxieProcess).str(), uniqueProcesses, pXRefNodeTree);
+                addUniqueXRefNode(cluster.getRoxieProcess(roxieProcess).str(), uniqueProcesses, xrefNodeTree);
                 break;
             }
         }
-        addXRefNode("SuperFiles", pXRefNodeTree);
+        addXRefNode("SuperFiles", xrefNodeTree);
 
         StringBuffer buf;
-        ESPSerializationFormat fmt = context.getResponseFormat();
-        if (fmt == ESPSerializationJSON)
-            resp.setDFUXRefListResult(toJSON(pXRefNodeTree, buf).str());
+        if (context.getResponseFormat() == ESPSerializationJSON)
+            resp.setDFUXRefListResult(toJSON(xrefNodeTree, buf).str());
         else
-            resp.setDFUXRefListResult(toXML(pXRefNodeTree, buf).str());
+            resp.setDFUXRefListResult(toXML(xrefNodeTree, buf).str());
     }
-    catch(IException* e)
+    catch(IException *e)
     {   
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
     }
     return true;
 }
 
-inline const char *skipTilda(const char *lfn) //just in case
-{
-    if (lfn)
-        while (*lfn == '~' || *lfn == ' ')
-            lfn++;
-    return lfn;
-}
-
-inline void addLfnToUsedFileMap(MapStringTo<bool> &usedFileMap, const char *lfn)
+inline void addLfnToUsedFileMap(MapStringTo<bool> &usedFileMap, const char *fileName)
 {
-    lfn = skipTilda(lfn);
-    if (lfn)
-        usedFileMap.setValue(lfn, true);
+    //Normalize file name, including remove the leading tilda.
+    CDfsLogicalFileName lfn;
+    lfn.set(fileName);
+    if (lfn.get())
+        usedFileMap.setValue(lfn.get(), true);
 }
 
 void addUsedFilesFromPackageMaps(MapStringTo<bool> &usedFileMap, const char *process)
@@ -683,16 +604,18 @@ void findUnusedFilesInDFS(StringArray &unusedFiles, const char *process, const M
     Owned<IPropertyTreeIterator> files = root->getElements(xpath);
     ForEach(*files)
     {
-        const char *lfn = skipTilda(files->query().queryProp(NULL));
-        if (lfn && !usedFileMap.getValue(lfn))
-            unusedFiles.append(lfn);
+        CDfsLogicalFileName lfn;
+        lfn.set(files->query().queryProp(nullptr));
+        if (lfn.get() && !usedFileMap.getValue(lfn.get()))
+            unusedFiles.append(lfn.get());
     }
 }
+
 bool CWsDfuXRefEx::onDFUXRefUnusedFiles(IEspContext &context, IEspDFUXRefUnusedFilesRequest &req, IEspDFUXRefUnusedFilesResponse &resp)
 {
     const char *process = req.getProcessCluster();
-    if (!process || !*process)
-        throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "process cluster, not specified.");
+    if (isEmptyString(process))
+        throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "process cluster not specified.");
 
     SocketEndpointArray servers;
     getRoxieProcessServers(process, servers);
@@ -715,3 +638,27 @@ bool CWsDfuXRefEx::onDFUXRefUnusedFiles(IEspContext &context, IEspDFUXRefUnusedF
     resp.setUnusedFiles(unusedFiles);
     return true;
 }
+
+IXRefNode *CWsDfuXRefEx::getXRefNodeByCluster(const char* cluster)
+{
+    if (isEmptyString(cluster))
+        throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Cluster not defined.");
+
+    Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(cluster);
+    if (!xRefNode)
+        throw MakeStringException(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "XRefNode not found for %s.", cluster);
+
+    return xRefNode.getClear();
+}
+
+IUserDescriptor *CWsDfuXRefEx::getUserDescriptor(IEspContext &context)
+{
+    StringBuffer userName;
+    context.getUserID(userName);
+    if (userName.isEmpty())
+        return nullptr;
+
+    Owned<IUserDescriptor> userDesc = createUserDescriptor();
+    userDesc->set(userName, context.queryPassword(), context.querySignature());
+    return userDesc.getClear();
+}

esp/services/ws_dfu/ws_dfuXRefService.hpp

@@ -180,17 +180,16 @@ public:
 
 class CWsDfuXRefEx : public CWsDFUXRef
 {
-private:
-    StringBuffer  user_;
-    StringBuffer  password_;
     Owned<IXRefNodeManager> XRefNodeManager;
     Owned<CXRefExBuilderThread> m_XRefbuilder;
 
-private:
     IXRefFilesNode* getFileNodeInterface(IXRefNode& XRefNode,const char* nodeType);
     void addXRefNode(const char* name, IPropertyTree* pXRefNodeTree);
     void readLostFileQueryResult(IEspContext &context, StringBuffer& buf);
     bool addUniqueXRefNode(const char* processName, BoolHash& uniqueProcesses, IPropertyTree* pXRefNodeTree);
+    IXRefNode* getXRefNodeByCluster(const char* cluster);
+    IUserDescriptor* getUserDescriptor(IEspContext& context);
+    void updateSkew(IPropertyTree &node);
 public:
    IMPLEMENT_IINTERFACE;