HPCC-21181 Add GetFileDetails option to WsDfuXRef.DFUXRefUnusedFiles

1. The DFULogicalFile is moved from ws_dfu.ecm to ws_dfu_common.ecm
so that it can be shared with ws_dfuXRef.ecm;
2. The 5 CWsDfuEx methods related to getLogicalFilesSorted() are
moved to new namespace WsDFUHelpers so that they can be shared with
ws_dfuXRefService.cpp;
3. other changes.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

dali/dfuplus/CMakeLists.txt

@@ -28,6 +28,7 @@ include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
 
 set (    SRCS
          ../../esp/bindings/bindutil.cpp
+         ${ESPSCM_GENERATED_DIR}/ws_dfu_common_esp.cpp
          ${ESPSCM_GENERATED_DIR}/ws_dfu_esp.cpp
          ${ESPSCM_GENERATED_DIR}/ws_fs_esp.cpp
          dfuplus.cpp

ecl/eclcmd/roxie/CMakeLists.txt

@@ -18,6 +18,7 @@ include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
 set (    SRCS
          ${ESPSCM_GENERATED_DIR}/common_esp.cpp
          ${ESPSCM_GENERATED_DIR}/ws_smc_esp.cpp
+         ${ESPSCM_GENERATED_DIR}/ws_dfu_common_esp.cpp
          ${ESPSCM_GENERATED_DIR}/ws_dfuXref_esp.cpp
          ${ESPSCM_GENERATED_DIR}/ws_dfu_esp.cpp
          ecl-roxie.cpp

esp/clients/wsdfuaccess/CMakeLists.txt

@@ -30,6 +30,7 @@ include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
 
 set (    SRCS 
          wsdfuaccess.cpp 
+         ${ESPSCM_GENERATED_DIR}/ws_dfu_common_esp.cpp
          ${ESPSCM_GENERATED_DIR}/ws_dfu_esp.cpp
     )
 

esp/scm/smcscm.cmake

@@ -27,6 +27,7 @@ set ( ESPSCM_GENERATED_DIR ${CMAKE_BINARY_DIR}/generated )
 set ( ESPSCM_SRCS
       common.ecm
       ws_dfu.ecm
+      ws_dfu_common.ecm
       ws_dfuXref.ecm
       ws_fs.ecm
       ws_roxie.ecm

esp/scm/ws_dfu.ecm

@@ -16,6 +16,9 @@
 ############################################################################## */
 
 #include "xslprocessor.hpp"
+
+EspInclude(ws_dfu_common);
+
 //  ===========================================================================
 
 ESPenum DFUArrayActions : string
@@ -94,41 +97,6 @@ ESPStruct DFUFileBloom
     string Probability;
 };
 
-ESPStruct DFULogicalFile
-{
-    string Prefix;
-    [depr_ver("1.26")] string ClusterName;
-    [min_ver("1.26")] string NodeGroup;
-    string Directory;
-    string Description;
-    string Parts;
-    string Name;
-    string Owner;
-    string Totalsize;
-    string RecordCount;
-    string Modified;
-    string LongSize;
-    string LongRecordCount;
-    bool   isSuperfile;
-    [depr_ver("1.22")] bool   isZipfile;
-    bool   isDirectory(false);
-    bool   Replicate(false);
-    [min_ver("1.01")] int64 IntSize;
-    [min_ver("1.01")] int64 IntRecordCount;
-    [min_ver("1.02")] bool FromRoxieCluster;
-    [min_ver("1.03")] bool BrowseData;
-    [min_ver("1.14"), depr_ver("1.24")] bool IsKeyFile;
-    [min_ver("1.22")] bool IsCompressed;
-    [min_ver("1.24")] string ContentType;
-    [min_ver("1.22")] int64 CompressedFileSize;
-    [min_ver("1.30")] string SuperOwners;
-    [min_ver("1.30")] bool Persistent(false);
-    [min_ver("1.34")] bool IsProtected(false);
-    [min_ver("1.41")] string KeyType;
-    [min_ver("1.52")] int NumOfSubfiles;
-    [min_ver("1.55")] string Accessed;
-};
-
 ESPStruct DFUPart
 {
     int Id;

esp/scm/ws_dfuXref.ecm

@@ -15,7 +15,10 @@
     limitations under the License.
 ############################################################################## */
 
-#include "xslprocessor.hpp" 
+#include "xslprocessor.hpp"
+
+EspInclude(ws_dfu_common);
+
 //  ===========================================================================
 ESPrequest 
 [
@@ -209,19 +212,18 @@ ESPrequest DFUXRefUnusedFilesRequest
 {
     string ProcessCluster;
     bool CheckPackageMaps;
+    [min_ver("1.01")] bool GetFileDetails(false);
 };
 
 ESPresponse [exceptions_inline] DFUXRefUnusedFilesResponse
 {
     unsigned UnusedFileCount;
     ESParray<string, File> UnusedFiles;
+    [min_ver("1.01")] ESParray<ESPStruct DFULogicalFile> UnusedFilesWithDetails;
 };
 
-
-
-
 //  ===========================================================================
-ESPservice [auth_feature("DEFERRED"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsDFUXRef
+ESPservice [version("1.01"), default_client_version("1.01"), auth_feature("DEFERRED"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsDFUXRef
 {
     ///ESPmethod [resp_xsl_default("./smc_xslt/xref_main.xslt")] DFUXRefList(DFUXRefListRequest, DFUXRefListResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/xref_main.xslt")] DFUXRefList(DFUXRefListRequest, DFUXRefListResponse);

esp/scm/ws_dfu_common.ecm

@@ -0,0 +1,55 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "xslprocessor.hpp"
+
+EspInclude(common);
+
+ESPStruct DFULogicalFile
+{
+    string Prefix;
+    [depr_ver("1.26")] string ClusterName;
+    [min_ver("1.26")] string NodeGroup;
+    string Directory;
+    string Description;
+    string Parts;
+    string Name;
+    string Owner;
+    string Totalsize;
+    string RecordCount;
+    string Modified;
+    string LongSize;
+    string LongRecordCount;
+    bool   isSuperfile;
+    [depr_ver("1.22")] bool   isZipfile;
+    bool   isDirectory(false);
+    bool   Replicate(false);
+    [min_ver("1.01")] int64 IntSize;
+    [min_ver("1.01")] int64 IntRecordCount;
+    [min_ver("1.02")] bool FromRoxieCluster;
+    [min_ver("1.03")] bool BrowseData;
+    [min_ver("1.14"), depr_ver("1.24")] bool IsKeyFile;
+    [min_ver("1.22")] bool IsCompressed;
+    [min_ver("1.24")] string ContentType;
+    [min_ver("1.22")] int64 CompressedFileSize;
+    [min_ver("1.30")] string SuperOwners;
+    [min_ver("1.30")] bool Persistent(false);
+    [min_ver("1.34")] bool IsProtected(false);
+    [min_ver("1.41")] string KeyType;
+    [min_ver("1.52")] int NumOfSubfiles;
+    [min_ver("1.55")] string Accessed;
+};

esp/services/ws_dfu/CMakeLists.txt

@@ -28,11 +28,14 @@ include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
 
 set (    SRCS 
          ../../../dali/dfu/dfuutil.cpp 
+         ${ESPSCM_GENERATED_DIR}/ws_dfu_common_esp.cpp 
          ${ESPSCM_GENERATED_DIR}/ws_dfu_esp.cpp 
          ${ESPSCM_GENERATED_DIR}/ws_dfuXref_esp.cpp 
+         ${HPCC_SOURCE_DIR}/esp/scm/ws_dfu_common.ecm
          ${HPCC_SOURCE_DIR}/esp/scm/ws_dfu.ecm
          ${HPCC_SOURCE_DIR}/esp/scm/ws_dfuXref.ecm
          ${HPCC_SOURCE_DIR}/esp/clients/roxiecontrol.cpp
+         ws_dfuHelpers.cpp 
          ws_dfuPlugin.cpp 
          ws_dfuService.cpp 
          ws_dfuXRefService.cpp 

esp/services/ws_dfu/ws_dfuHelpers.cpp

@@ -0,0 +1,198 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "esp.hpp"
+#include "wshelpers.hpp"
+
+#include "ws_dfuHelpers.hpp"
+
+bool WsDFUHelpers::addDFUQueryFilter(DFUQResultField* filters, unsigned short& count, MemoryBuffer& buff, const char* value, DFUQResultField name)
+{
+    if (isEmptyString(value))
+        return false;
+    filters[count++] = name;
+    buff.append(value);
+    return true;
+}
+
+void WsDFUHelpers::appendDFUQueryFilter(const char* name, DFUQFilterType type, const char* value, StringBuffer& filterBuf)
+{
+    if (isEmptyString(name) || isEmptyString(value))
+        return;
+    filterBuf.append(type).append(DFUQFilterSeparator).append(name).append(DFUQFilterSeparator).append(value).append(DFUQFilterSeparator);
+}
+
+void WsDFUHelpers::appendDFUQueryFilter(const char* name, DFUQFilterType type, const char* value, const char* valueHigh, StringBuffer& filterBuf)
+{
+    if (isEmptyString(name) || isEmptyString(value))
+        return;
+    filterBuf.append(type).append(DFUQFilterSeparator).append(name).append(DFUQFilterSeparator).append(value).append(DFUQFilterSeparator);
+    filterBuf.append(valueHigh).append(DFUQFilterSeparator);
+}
+
+const char* WsDFUHelpers::getPrefixFromLogicalName(const char* logicalName, StringBuffer& prefix)
+{
+    if (isEmptyString(logicalName))
+        return nullptr;
+
+    const char *c=strstr(logicalName, "::");
+    if (c)
+        prefix.append(c-logicalName, logicalName);
+    else
+        prefix.append(logicalName);
+    return prefix.str();
+}
+
+bool WsDFUHelpers::addToLogicalFileList(IPropertyTree& file, const char* nodeGroup, double version, IArrayOf<IEspDFULogicalFile>& logicalFiles)
+{
+    const char* logicalName = file.queryProp(getDFUQResultFieldName(DFUQRFname));
+    if (isEmptyString(logicalName))
+        return false;
+
+    try
+    {
+        Owned<IEspDFULogicalFile> lFile = createDFULogicalFile();
+        lFile->setName(logicalName);
+        lFile->setOwner(file.queryProp(getDFUQResultFieldName(DFUQRFowner)));
+
+        StringBuffer buf(file.queryProp(getDFUQResultFieldName(DFUQRFtimemodified)));
+        lFile->setModified(buf.replace('T', ' '));
+        lFile->setPrefix(getPrefixFromLogicalName(logicalName, buf.clear()));
+        lFile->setDescription(file.queryProp(getDFUQResultFieldName(DFUQRFdescription)));
+
+        if (isEmptyString(nodeGroup))
+            nodeGroup = file.queryProp(getDFUQResultFieldName(DFUQRFnodegroup));
+        if (!isEmptyString(nodeGroup))
+        {
+            if (version < 1.26)
+                lFile->setClusterName(nodeGroup);
+            else
+                lFile->setNodeGroup(nodeGroup);
+        }
+
+        int numSubFiles = 0;
+        if (!file.hasProp(getDFUQResultFieldName(DFUQRFnumsubfiles)))
+        {
+            lFile->setIsSuperfile(false);
+            lFile->setDirectory(file.queryProp(getDFUQResultFieldName(DFUQRFdirectory)));
+            lFile->setParts(file.queryProp(getDFUQResultFieldName(DFUQRFnumparts)));
+        }
+        else
+        {
+            lFile->setIsSuperfile(true);
+            if (version >= 1.52)
+            {
+                numSubFiles = file.getPropInt(getDFUQResultFieldName(DFUQRFnumsubfiles));
+                lFile->setNumOfSubfiles(numSubFiles);
+            }
+        }
+        lFile->setBrowseData(numSubFiles > 1 ? false : true); ////Bug 41379 - ViewKeyFile Cannot handle superfile with multiple subfiles
+
+        if (version >= 1.30)
+        {
+            if (file.hasProp(getDFUQResultFieldName(DFUQRFprotect)))
+                lFile->setIsProtected(true);
+            if (file.getPropBool(getDFUQResultFieldName(DFUQRFpersistent), false))
+                lFile->setPersistent(true);
+            if (file.hasProp(getDFUQResultFieldName(DFUQRFsuperowners)))
+                lFile->setSuperOwners(file.queryProp(getDFUQResultFieldName(DFUQRFsuperowners)));
+        }
+
+        __int64 size = file.getPropInt64(getDFUQResultFieldName(DFUQRForigsize), 0);
+        if (size > 0)
+        {
+            StringBuffer s;
+            lFile->setIntSize(size);
+            lFile->setTotalsize(s<<comma(size));
+        }
+
+        __int64 records = file.getPropInt64(getDFUQResultFieldName(DFUQRFrecordcount), 0);
+        if (!records)
+            records = file.getPropInt64(getDFUQResultFieldName(DFUQRForigrecordcount), 0);
+        if (!records)
+        {
+            __int64 recordSize = file.getPropInt64(getDFUQResultFieldName(DFUQRFrecordsize), 0);
+            if(recordSize > 0)
+                records = size/recordSize;
+        }
+        if (records > 0)
+        {
+            StringBuffer s;
+            lFile->setIntRecordCount(records);
+            lFile->setRecordCount(s<<comma(records));
+        }
+
+        bool isKeyFile = false;
+        if (version > 1.13)
+        {
+            const char* kind = file.queryProp(getDFUQResultFieldName(DFUQRFkind));
+            if (!isEmptyString(kind))
+            {
+                if (strieq(kind, "key"))
+                    isKeyFile = true;
+                if (version >= 1.24)
+                    lFile->setContentType(kind);
+                else
+                    lFile->setIsKeyFile(isKeyFile);
+            }
+        }
+
+        if (isKeyFile && (version >= 1.41))
+        {
+            if (isFilePartitionKey(file))
+                lFile->setKeyType("Partitioned");
+            else if (isFileLocalKey(file))
+                lFile->setKeyType("Local");
+            else
+                lFile->setKeyType("Distributed");
+        }
+
+        bool isFileCompressed = file.getPropBool(getDFUQResultFieldName(DFUQRFiscompressed));
+        if (isFileCompressed)
+        {
+            if (version >= 1.22)
+            {
+                if (file.hasProp(getDFUQResultFieldName(DFUQRFcompressedsize)))
+                    lFile->setCompressedFileSize(file.getPropInt64(getDFUQResultFieldName(DFUQRFcompressedsize)));
+                else if (isKeyFile)
+                    lFile->setCompressedFileSize(size);
+            }
+        }
+        if (version < 1.22)
+            lFile->setIsZipfile(isFileCompressed);
+        else
+            lFile->setIsCompressed(isFileCompressed);
+
+        if (version >= 1.55)
+        {
+            StringBuffer accessed(file.queryProp(getDFUQResultFieldName(DFUQRFaccessed)));
+            if (!accessed.isEmpty())
+                lFile->setAccessed(accessed.replace('T', ' '));
+        }
+
+        logicalFiles.append(*lFile.getClear());
+    }
+    catch(IException* e)
+    {
+        VStringBuffer msg("Failed to retrieve data for logical file %s: ", logicalName);
+        int code = e->errorCode();
+        e->errorMessage(msg);
+        e->Release();
+        throw MakeStringException(code, "%s", msg.str());
+    }
+    return true;
+}

esp/services/ws_dfu/ws_dfuHelpers.hpp

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _WSDFU_HELPERS_HPP__
+#define _WSDFU_HELPERS_HPP__
+
+#include "dadfs.hpp"
+
+#include "jstring.hpp"
+#include "exception_util.hpp"
+#include "ws_dfu_common_esp.ipp"
+
+namespace WsDFUHelpers
+{
+    bool addDFUQueryFilter(DFUQResultField* filters, unsigned short& count, MemoryBuffer& buff, const char* value, DFUQResultField name);
+    void appendDFUQueryFilter(const char*name, DFUQFilterType type, const char* value, StringBuffer& filterBuf);
+    void appendDFUQueryFilter(const char*name, DFUQFilterType type, const char* value, const char* valueHigh, StringBuffer& filterBuf);
+    const char* getPrefixFromLogicalName(const char* logicalName, StringBuffer& prefix);
+    bool addToLogicalFileList(IPropertyTree& file, const char* nodeGroup, double version, IArrayOf<IEspDFULogicalFile>& logicalFiles);
+};
+
+#endif

esp/services/ws_dfu/ws_dfuService.cpp

@@ -2214,7 +2214,7 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
     if (version >= 1.28)
     {
         StringBuffer buf;
-        FileDetails.setPrefix(getPrefixFromLogicalName(lname, buf));
+        FileDetails.setPrefix(WsDFUHelpers::getPrefixFromLogicalName(lname, buf));
         if (cluster && *cluster)
             FileDetails.setNodeGroup(cluster);
         else if (clusters.length() == 1)
@@ -2709,7 +2709,7 @@ void CWsDfuEx::getLogicalFileAndDirectory(IEspContext &context, IUserDescriptor*
             throw MakeStringException(ECLWATCH_CANNOT_GET_FILE_ITERATOR,"Cannot get LogicalFile information from file system.");
 
         ForEach(*it)
-            addToLogicalFileList(it->query(), NULL, version, logicalFiles);
+            WsDFUHelpers::addToLogicalFileList(it->query(), NULL, version, logicalFiles);
         numFiles = totalFiles;
     }
 
@@ -3519,30 +3519,6 @@ void CWsDfuEx::getAPageOfSortedLogicalFile(IEspContext &context, IUserDescriptor
     return;
 }
 
-bool CWsDfuEx::addDFUQueryFilter(DFUQResultField *filters, unsigned short &count, MemoryBuffer &buff, const char* value, DFUQResultField name)
-{
-    if (!value || !*value)
-        return false;
-    filters[count++] = name;
-    buff.append(value);
-    return true;
-}
-
-void CWsDfuEx::appendDFUQueryFilter(const char *name, DFUQFilterType type, const char *value, StringBuffer& filterBuf)
-{
-    if (!name || !*name || !value || !*value)
-        return;
-    filterBuf.append(type).append(DFUQFilterSeparator).append(name).append(DFUQFilterSeparator).append(value).append(DFUQFilterSeparator);
-}
-
-void CWsDfuEx::appendDFUQueryFilter(const char *name, DFUQFilterType type, const char *value, const char *valueHigh, StringBuffer& filterBuf)
-{
-    if (!name || !*name || !value || !*value)
-        return;
-    filterBuf.append(type).append(DFUQFilterSeparator).append(name).append(DFUQFilterSeparator).append(value).append(DFUQFilterSeparator);
-    filterBuf.append(valueHigh).append(DFUQFilterSeparator);
-}
-
 void CWsDfuEx::setFileTypeFilter(const char* fileType, StringBuffer& filterBuf)
 {
     DFUQFileTypeFilter fileTypeFilter = DFUQFFTall;
@@ -3562,7 +3538,7 @@ void CWsDfuEx::setFileTypeFilter(const char* fileType, StringBuffer& filterBuf)
         fileTypeFilter = DFUQFFTall;
     filterBuf.append(DFUQFTspecial).append(DFUQFilterSeparator).append(DFUQSFFileType).append(DFUQFilterSeparator).append(fileTypeFilter).append(DFUQFilterSeparator);
     if (notInSuperfile)
-        appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFsuperowner), DFUQFThasProp, "0", filterBuf);
+        WsDFUHelpers::appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFsuperowner), DFUQFThasProp, "0", filterBuf);
 }
 
 void CWsDfuEx::setFileNameFilter(const char* fname, const char* prefix, StringBuffer &filterBuf)
@@ -3593,9 +3569,9 @@ void CWsDfuEx::setDFUQueryFilters(IEspDFUQueryRequest& req, StringBuffer& filter
 {
     setFileNameFilter(req.getLogicalName(), req.getPrefix(), filterBuf);
     setFileTypeFilter(req.getFileType(), filterBuf);
-    appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFattrowner), DFUQFTwildcardMatch, req.getOwner(), filterBuf);
-    appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFkind), DFUQFTwildcardMatch, req.getContentType(), filterBuf);
-    appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFgroup), DFUQFTcontainString, req.getNodeGroup(), ",", filterBuf);
+    WsDFUHelpers::appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFattrowner), DFUQFTwildcardMatch, req.getOwner(), filterBuf);
+    WsDFUHelpers::appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFkind), DFUQFTwildcardMatch, req.getContentType(), filterBuf);
+    WsDFUHelpers::appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFgroup), DFUQFTcontainString, req.getNodeGroup(), ",", filterBuf);
     if (!req.getIncludeSuperOwner_isNull() && req.getIncludeSuperOwner())
         filterBuf.append(DFUQFTincludeFileAttr).append(DFUQFilterSeparator).append(DFUQSFAOincludeSuperOwner).append(DFUQFilterSeparator);
 
@@ -3680,158 +3656,6 @@ void CWsDfuEx::setDFUQuerySortOrder(IEspDFUQueryRequest& req, StringBuffer& sort
     return;
 }
 
-const char* CWsDfuEx::getPrefixFromLogicalName(const char* logicalName, StringBuffer& prefix)
-{
-    if (!logicalName || !*logicalName)
-        return NULL;
-
-    const char *c=strstr(logicalName, "::");
-    if (c)
-        prefix.append(c-logicalName, logicalName);
-    else
-        prefix.append(logicalName);
-    return prefix.str();
-}
-
-bool CWsDfuEx::addToLogicalFileList(IPropertyTree& file, const char* nodeGroup, double version, IArrayOf<IEspDFULogicalFile>& logicalFiles)
-{
-    const char* logicalName = file.queryProp(getDFUQResultFieldName(DFUQRFname));
-    if (!logicalName || !*logicalName)
-        return false;
-
-    try
-    {
-        Owned<IEspDFULogicalFile> lFile = createDFULogicalFile("","");
-        lFile->setName(logicalName);
-        lFile->setOwner(file.queryProp(getDFUQResultFieldName(DFUQRFowner)));
-
-        StringBuffer buf(file.queryProp(getDFUQResultFieldName(DFUQRFtimemodified)));
-        lFile->setModified(buf.replace('T', ' ').str());
-        lFile->setPrefix(getPrefixFromLogicalName(logicalName, buf.clear()));
-        lFile->setDescription(file.queryProp(getDFUQResultFieldName(DFUQRFdescription)));
-
-        if (!nodeGroup || !*nodeGroup)
-                nodeGroup = file.queryProp(getDFUQResultFieldName(DFUQRFnodegroup));
-        if (nodeGroup && *nodeGroup)
-        {
-            if (version < 1.26)
-                lFile->setClusterName(nodeGroup);
-            else
-                lFile->setNodeGroup(nodeGroup);
-        }
-
-        int numSubFiles = file.hasProp(getDFUQResultFieldName(DFUQRFnumsubfiles));
-        if(numSubFiles)
-        {
-            lFile->setIsSuperfile(true);
-            if (version >= 1.52)
-            {
-                numSubFiles = file.getPropInt(getDFUQResultFieldName(DFUQRFnumsubfiles));
-                lFile->setNumOfSubfiles(numSubFiles);
-            }
-        }
-        else
-        {
-            lFile->setIsSuperfile(false);
-            lFile->setDirectory(file.queryProp(getDFUQResultFieldName(DFUQRFdirectory)));
-            lFile->setParts(file.queryProp(getDFUQResultFieldName(DFUQRFnumparts)));
-        }
-        lFile->setBrowseData(numSubFiles > 1 ? false : true); ////Bug 41379 - ViewKeyFile Cannot handle superfile with multiple subfiles
-
-        if (version >= 1.30)
-        {
-            bool persistent = file.getPropBool(getDFUQResultFieldName(DFUQRFpersistent), false);
-            if (persistent)
-                lFile->setPersistent(true);
-            if (file.hasProp(getDFUQResultFieldName(DFUQRFsuperowners)))
-                lFile->setSuperOwners(file.queryProp(getDFUQResultFieldName(DFUQRFsuperowners)));
-            if (file.hasProp(getDFUQResultFieldName(DFUQRFprotect)))
-                lFile->setIsProtected(true);
-        }
-
-        __int64 size = file.getPropInt64(getDFUQResultFieldName(DFUQRForigsize),0);
-        if (size > 0)
-        {
-            lFile->setIntSize(size);
-            lFile->setTotalsize((buf.clear()<<comma(size)).str());
-        }
-
-        __int64 records = file.getPropInt64(getDFUQResultFieldName(DFUQRFrecordcount),0);
-        if (!records)
-            records = file.getPropInt64(getDFUQResultFieldName(DFUQRForigrecordcount),0);
-        if (!records)
-        {
-            __int64 recordSize=file.getPropInt64(getDFUQResultFieldName(DFUQRFrecordsize),0);
-            if(recordSize > 0)
-                records = size/recordSize;
-        }
-        if (records > 0)
-        {
-            lFile->setIntRecordCount(records);
-            lFile->setRecordCount((buf.clear()<<comma(records)).str());
-        }
-
-        bool isKeyFile = false;
-        if (version > 1.13)
-        {
-            const char * kind = file.queryProp(getDFUQResultFieldName(DFUQRFkind));
-            if (kind && *kind)
-            {
-                if (strieq(kind, "key"))
-                    isKeyFile = true;
-                if (version >= 1.24)
-                    lFile->setContentType(kind);
-                else
-                    lFile->setIsKeyFile(isKeyFile);
-            }
-        }
-
-        if (isKeyFile && (version >= 1.41))
-        {
-            if (isFilePartitionKey(file))
-                lFile->setKeyType("Partitioned");
-            else if (isFileLocalKey(file))
-                lFile->setKeyType("Local");
-            else
-                lFile->setKeyType("Distributed");
-        }
-
-        bool isFileCompressed = file.getPropBool(getDFUQResultFieldName(DFUQRFiscompressed));
-        if (isFileCompressed)
-        {
-            if (version >= 1.22)
-            {
-                if (file.hasProp(getDFUQResultFieldName(DFUQRFcompressedsize)))
-                    lFile->setCompressedFileSize(file.getPropInt64(getDFUQResultFieldName(DFUQRFcompressedsize)));
-                else if (isKeyFile)
-                    lFile->setCompressedFileSize(size);
-            }
-        }
-        if (version < 1.22)
-            lFile->setIsZipfile(isFileCompressed);
-        else
-            lFile->setIsCompressed(isFileCompressed);
-
-        if (version >= 1.55)
-        {
-            StringBuffer accessed(file.queryProp(getDFUQResultFieldName(DFUQRFaccessed)));
-            if (!accessed.isEmpty())
-                lFile->setAccessed(accessed.replace('T', ' '));
-        }
-
-        logicalFiles.append(*lFile.getClear());
-    }
-    catch(IException* e)
-    {
-        VStringBuffer msg("Failed to retrieve data for logical file %s: ", logicalName);
-        int code = e->errorCode();
-        e->errorMessage(msg);
-        e->Release();
-        throw MakeStringException(code, "%s", msg.str());
-    }
-    return true;
-}
-
 void CWsDfuEx::setDFUQueryResponse(IEspContext &context, unsigned totalFiles, StringBuffer& sortBy, bool descending, unsigned pageStart, unsigned pageSize,
                                    IEspDFUQueryRequest& req, IEspDFUQueryResponse& resp)
 {
@@ -3952,7 +3776,7 @@ bool CWsDfuEx::doLogicalFileSearch(IEspContext &context, IUserDescriptor* udesc,
     unsigned short localFilterCount = 0;
     DFUQResultField localFilters[8];
     MemoryBuffer localFilterBuf;
-    addDFUQueryFilter(localFilters, localFilterCount, localFilterBuf, req.getNodeGroup(), DFUQRFnodegroup);
+    WsDFUHelpers::addDFUQueryFilter(localFilters, localFilterCount, localFilterBuf, req.getNodeGroup(), DFUQRFnodegroup);
     localFilters[localFilterCount] = DFUQRFterm;
 
     StringBuffer sortBy;
@@ -3996,7 +3820,7 @@ bool CWsDfuEx::doLogicalFileSearch(IEspContext &context, IUserDescriptor* udesc,
 
     IArrayOf<IEspDFULogicalFile> logicalFiles;
     ForEach(*it)
-        addToLogicalFileList(it->query(), NULL, version, logicalFiles);
+        WsDFUHelpers::addToLogicalFileList(it->query(), NULL, version, logicalFiles);
 
     if (!allMatchingFilesReceived)
     {

esp/services/ws_dfu/ws_dfuService.hpp

@@ -25,6 +25,7 @@
 #include "dadfs.hpp"
 #include "environment.hpp"
 #include <atomic>
+#include "ws_dfuHelpers.hpp"
 
 class CThorNodeGroup: public CInterface
 {
@@ -185,16 +186,11 @@ public:
     virtual bool onDFUFileCreate(IEspContext &context, IEspDFUFileCreateRequest &req, IEspDFUFileCreateResponse &resp);
 
 private:
-    const char* getPrefixFromLogicalName(const char* logicalName, StringBuffer& prefix);
-    bool addDFUQueryFilter(DFUQResultField *filters, unsigned short &count, MemoryBuffer &buff, const char* value, DFUQResultField name);
-    void appendDFUQueryFilter(const char *name, DFUQFilterType type, const char *value, StringBuffer& filterBuf);
-    void appendDFUQueryFilter(const char *name, DFUQFilterType type, const char *value, const char *valueHigh, StringBuffer& filterBuf);
     void setFileIterateFilter(unsigned maxFiles, StringBuffer &filterBuf);
     void setFileTypeFilter(const char* fileType, StringBuffer& filterBuf);
     void setFileNameFilter(const char* fname, const char* prefix, StringBuffer &buff);
     void setDFUQueryFilters(IEspDFUQueryRequest& req, StringBuffer& filterBuf);
     void setDFUQuerySortOrder(IEspDFUQueryRequest& req, StringBuffer& sortBy, bool& descending, DFUQResultField* sortOrder);
-    bool addToLogicalFileList(IPropertyTree& file, const char* nodeGroup, double version, IArrayOf<IEspDFULogicalFile>& logicalFiles);
     void setDFUQueryResponse(IEspContext &context, unsigned totalFiles, StringBuffer& sortBy, bool descending, unsigned pageStart,
         unsigned pageSize, IEspDFUQueryRequest & req, IEspDFUQueryResponse & resp);
     void getLogicalFileAndDirectory(IEspContext &context, IUserDescriptor* udesc, const char *dirname,

esp/services/ws_dfu/ws_dfuXRefService.cpp

@@ -611,6 +611,55 @@ void findUnusedFilesInDFS(StringArray &unusedFiles, const char *process, const M
     }
 }
 
+IDFAttributesIterator *CWsDfuXRefEx::getAllLogicalFilesInCluster(IEspContext &context, const char *cluster, bool &allMatchingFilesReceived)
+{
+    StringBuffer filterBuf;
+    //The filterBuf is sent to dali to retrieve the logical files whose @group attribute contains this cluster.
+    WsDFUHelpers::appendDFUQueryFilter(getDFUQFilterFieldName(DFUQFFgroup), DFUQFTcontainString, cluster, ",", filterBuf);
+
+    DFUQResultField localFilters[2];
+    MemoryBuffer localFilterBuf;
+    unsigned short localFilterCount = 0;
+    //If a logical file is for >1 clusters, the localFilterBuf is used to pick up the logical file which is for this cluster.
+    WsDFUHelpers::addDFUQueryFilter(localFilters, localFilterCount, localFilterBuf, cluster, DFUQRFnodegroup);
+    localFilters[localFilterCount] = DFUQRFterm;
+
+    DFUQResultField sortOrder[2] = {DFUQRFname, DFUQRFterm};
+
+    __int64 cacheHint = 0; //No paging is needed.
+    unsigned totalFiles = 0, pageStart = 0, pageSize = ITERATE_FILTEREDFILES_LIMIT;
+    Owned<IUserDescriptor> userDesc = getUserDescriptor(context);
+
+    PROGLOG("getLogicalFilesSorted() called");
+    Owned<IDFAttributesIterator> it = queryDistributedFileDirectory().getLogicalFilesSorted(userDesc, sortOrder, filterBuf,
+        localFilters, localFilterBuf.bufferBase(), pageStart, pageSize, &cacheHint, &totalFiles, &allMatchingFilesReceived);
+    PROGLOG("getLogicalFilesSorted() done");
+
+    return it.getClear();
+}
+
+void CWsDfuXRefEx::findUnusedFilesWithDetailsInDFS(IEspContext &context, const char *process, const MapStringTo<bool> &usedFileMap, IArrayOf<IEspDFULogicalFile> &unusedFiles)
+{
+    //Collect information about logical files in dali for the given cluster.
+    bool allMatchingFilesReceived = true;
+    Owned<IDFAttributesIterator> it = getAllLogicalFilesInCluster(context, process, allMatchingFilesReceived);
+    if (!it)
+        throw MakeStringException(ECLWATCH_CANNOT_GET_FILE_ITERATOR, "Failed to retrieve logical files for %s.", process);
+
+    if (!allMatchingFilesReceived)
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "WsDfu::DFURoxieUnusedFiles not supported for %s: too many files.", process);
+
+    //Find out unused Roxie logical files
+    double version = context.getClientVersion();
+    ForEach(*it)
+    {
+        IPropertyTree &file = it->query();
+        const char *fileName = file.queryProp(getDFUQResultFieldName(DFUQRFname));
+        if (!isEmptyString(fileName) && !usedFileMap.getValue(fileName))
+            WsDFUHelpers::addToLogicalFileList(file, nullptr, version, unusedFiles);
+    }
+}
+
 bool CWsDfuXRefEx::onDFUXRefUnusedFiles(IEspContext &context, IEspDFUXRefUnusedFilesRequest &req, IEspDFUXRefUnusedFilesResponse &resp)
 {
     const char *process = req.getProcessCluster();
@@ -630,12 +679,23 @@ bool CWsDfuXRefEx::onDFUXRefUnusedFiles(IEspContext &context, IEspDFUXRefUnusedF
     Owned<IPropertyTreeIterator> roxieFiles = controlXrefInfo->getElements("//File");
     ForEach(*roxieFiles)
         addLfnToUsedFileMap(usedFileMap, roxieFiles->query().queryProp("@name"));
+
     if (req.getCheckPackageMaps())
         addUsedFilesFromPackageMaps(usedFileMap, process);
-    StringArray unusedFiles;
-    findUnusedFilesInDFS(unusedFiles, process, usedFileMap);
-    resp.setUnusedFileCount(unusedFiles.length());
-    resp.setUnusedFiles(unusedFiles);
+    if (!req.getGetFileDetails())
+    {
+        StringArray unusedFiles;
+        findUnusedFilesInDFS(unusedFiles, process, usedFileMap);
+        resp.setUnusedFileCount(unusedFiles.length());
+        resp.setUnusedFiles(unusedFiles);
+    }
+    else
+    {
+        IArrayOf<IEspDFULogicalFile> unusedLFs;
+        findUnusedFilesWithDetailsInDFS(context, process, usedFileMap, unusedLFs);
+        resp.setUnusedFileCount(unusedLFs.length());
+        resp.setUnusedFilesWithDetails(unusedLFs);
+    }
     return true;
 }
 

esp/services/ws_dfu/ws_dfuXRefService.hpp

@@ -23,6 +23,8 @@
 #include "TpWrapper.hpp"
 #include "dfuxreflib.hpp"
 #include "jqueue.tpp"
+#include "ws_dfu_esp.ipp"
+#include "ws_dfuHelpers.hpp"
 
 class CXRefExBuilderThread : public Thread
 {
@@ -190,6 +192,8 @@ class CWsDfuXRefEx : public CWsDFUXRef
     IXRefNode* getXRefNodeByCluster(const char* cluster);
     IUserDescriptor* getUserDescriptor(IEspContext& context);
     void updateSkew(IPropertyTree &node);
+    IDFAttributesIterator* getAllLogicalFilesInCluster(IEspContext &context, const char *cluster, bool &allMatchingFilesReceived);
+    void findUnusedFilesWithDetailsInDFS(IEspContext &context, const char *process, const MapStringTo<bool> &usedFileMap, IArrayOf<IEspDFULogicalFile> &unusedFiles);
 public:
    IMPLEMENT_IINTERFACE;
 

plugins/fileservices/CMakeLists.txt

@@ -26,6 +26,7 @@ project( fileservices )
 include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
 
 set ( SRCS 
+      ${ESPSCM_GENERATED_DIR}/ws_dfu_common_esp.cpp
       ${ESPSCM_GENERATED_DIR}/ws_dfu_esp.cpp 
       ${ESPSCM_GENERATED_DIR}/ws_fs_esp.cpp 
       fileservices.cpp