Merge pull request #5486 from wangkx/h10920

HPCC-10920 Return ClusterName in WsDFU.DFUInfo response

Reviewed-By: Gordon Smith <gordon.smith@lexisnexis.com>
Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

esp/eclwatch/ws_XSLT/dfu_file.xslt

@@ -315,7 +315,7 @@
                     <col span="3" class="number"/>
                 </colgroup>
                 <tr class="grey"><th>Number</th><th>IP</th><th>Size</th><xsl:if test="string-length($actualSize)"><th>Actual Size</th></xsl:if></tr>
-                <xsl:apply-templates select="DFUFileParts/DFUPart">
+                <xsl:apply-templates select="DFUFilePartsOnClusters/DFUFilePartsOnCluster/DFUFileParts/DFUPart">
                     <xsl:sort select="Id" data-type="number"/>
                     <xsl:sort select="Copy" data-type="number"/>
                 </xsl:apply-templates>
@@ -357,13 +357,13 @@
     </xsl:template>
 
     <xsl:template match="DFUPart">
-    <xsl:if test="Copy mod  2 > 0">
+        <xsl:if test="Copy=1"> <!-- Copy=1: display primary copy only -->
             <tr>
-        <td><xsl:value-of select="Id"/></td>
-        <td><xsl:value-of select="Ip"/></td>
-        <td class="number"><xsl:value-of select="Partsize"/></td>
-        <xsl:if test="string-length($actualSize)"><td class="number"><xsl:value-of select="ActualSize"/></td></xsl:if>
-        </tr>
+                <td><xsl:value-of select="Id"/></td>
+                <td><xsl:value-of select="Ip"/></td>
+                <td class="number"><xsl:value-of select="Partsize"/></td>
+                <xsl:if test="string-length($actualSize)"><td class="number"><xsl:value-of select="ActualSize"/></td></xsl:if>
+            </tr>
         </xsl:if>
     </xsl:template>
 

esp/scm/ws_dfu.ecm

@@ -73,6 +73,12 @@ ESPStruct DFUFileStat
     string MaxSkew;
 };
 
+ESPStruct DFUFilePartsOnCluster
+{
+    string Cluster;
+    ESParray<ESPstruct DFUPart> DFUFileParts;
+};
+
 ESPStruct DFUFileDetail
 {
     string Name;
@@ -86,7 +92,7 @@ ESPStruct DFUFileDetail
     string RecordCount;
     string Wuid;
     string Owner;
-    string Cluster;
+    [depr_ver("1.25")] string Cluster;
     string JobName;
     string Persistent;
     string Format;
@@ -100,7 +106,8 @@ ESPStruct DFUFileDetail
     string Ecl;
     [depr_ver("1.22")] bool ZipFile(false);
     ESPstruct DFUFileStat Stat;
-    ESParray<ESPstruct DFUPart> DFUFileParts;
+    [depr_ver("1.25")] ESParray<ESPstruct DFUPart> DFUFileParts;
+    [min_ver("1.25")] ESParray<ESPstruct DFUFilePartsOnCluster> DFUFilePartsOnClusters;
     bool isSuperfile(false);
     bool ShowFileContent(true);
     ESParray<string> subfiles;
@@ -631,7 +638,7 @@ ESPresponse [exceptions_inline, nil_remove, http_encode(0)] DFUSearchDataRespons
 
 //  ===========================================================================
 ESPservice [
-    version("1.24"), default_client_version("1.24"),
+    version("1.25"), default_client_version("1.25"),
     noforms, 
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1623,6 +1623,62 @@ bool CWsDfuEx::getUserFilePermission(IEspContext &context, IUserDescriptor* udes
     return true;
 }
 
+void CWsDfuEx::getFilePartsOnClusters(IEspContext &context, const char* clusterReq, StringArray& clusters, IDistributedFile* df, IEspDFUFileDetail& FileDetails,
+    offset_t& mn, offset_t& mx, offset_t& sum, offset_t& count)
+{
+    IArrayOf<IConstDFUFilePartsOnCluster>& partsOnClusters = FileDetails.getDFUFilePartsOnClusters();
+    ForEachItemIn(i, clusters)
+    {
+        const char* clusterName = clusters.item(i);
+        if (!clusterName || !*clusterName || (clusterReq && *clusterReq && !strieq(clusterReq, clusterName)))
+            continue;
+
+        Owned<IEspDFUFilePartsOnCluster> partsOnCluster = createDFUFilePartsOnCluster("","");
+        partsOnCluster->setCluster(clusterName);
+        IArrayOf<IConstDFUPart>& filePartList = partsOnCluster->getDFUFileParts();
+
+        Owned<IFileDescriptor> fdesc = df->getFileDescriptor(clusterName);
+        Owned<IPartDescriptorIterator> pi = fdesc->getIterator();
+        ForEach(*pi)
+        {
+            IPartDescriptor *part = &pi->get();
+            unsigned partIndex = part->queryPartIndex();
+
+            StringBuffer partSizeStr;
+            IPropertyTree *partPropertyTree = part->getProperties();
+            if (!partPropertyTree)
+                partSizeStr.set("<N/A>");
+            else
+            {
+                __uint64 size = partPropertyTree->getPropInt("@size");
+                comma c4(size);
+                partSizeStr<<c4;
+
+                count++;
+                sum+=size;
+                if(size>mx) mx=size;
+                if(size<mn) mn=size;
+            }
+
+            for (unsigned int i=0; i<part->numCopies(); i++)
+            {
+                StringBuffer b;
+                part->queryNode(i)->endpoint().getUrlStr(b);
+
+                Owned<IEspDFUPart> FilePart = createDFUPart("","");
+                FilePart->setId(partIndex+1);
+                FilePart->setPartsize(partSizeStr.str());
+                FilePart->setIp(b.str());
+                FilePart->setCopy(i+1);
+
+                filePartList.append(*FilePart.getClear());
+            }
+        }
+
+        partsOnClusters.append(*partsOnCluster.getClear());
+    }
+}
+
 void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, const char *name, const char *cluster, 
     const char *description,IEspDFUFileDetail& FileDetails)
 {
@@ -1633,12 +1689,9 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, co
         throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s.",name);
 
     StringArray clusters;
-    if (cluster && *cluster)
-    {
-        df->getClusterNames(clusters);
-        if(!FindInStringArray(clusters, cluster))
-            throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s.",name);
-    }
+    df->getClusterNames(clusters);
+    if (cluster && *cluster && !FindInStringArray(clusters, cluster))
+        throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s on %s.", name, cluster);
 
     double version = context.getClientVersion();
     offset_t size=queryDistributedFileSystem().getSize(df), recordSize=df->queryAttributes().getPropInt64("@recordSize",0);
@@ -1785,24 +1838,22 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, co
         FileDetails.setEcl(df->queryAttributes().queryProp("ECL"));
 
     StringBuffer clusterStr;
-    if ((!cluster || !*cluster) && clusters.ordinality())
-    {
-        clusterStr.append(clusters.item(0));
-    }
-    else if (cluster && *cluster)
+    ForEachItemIn(i, clusters)
     {
-        clusterStr.append(cluster);
+        if (!clusterStr.length())
+            clusterStr.append(clusters.item(i));
+        else
+            clusterStr.append(",").append(clusters.item(i));
     }
 
     if (clusterStr.length() > 0)
     {
-        FileDetails.setCluster(clusterStr.str());
         if (!checkFileContent(context, udesc, name, clusterStr.str()))
             FileDetails.setShowFileContent(false);
 
         if (version > 1.05)
         {
-            FileDetails.setFromRoxieCluster(false);
+            bool fromRoxieCluster = false;
 
             StringArray roxieClusterNames;
             IArrayOf<IEspTpCluster> roxieclusters;
@@ -1812,62 +1863,69 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, co
             {
                 IEspTpCluster& r_cluster = roxieclusters.item(k);
                 StringBuffer sName = r_cluster.getName();
-                if (!stricmp(sName.str(), clusterStr.str()))
+                if (FindInStringArray(clusters, sName.str()))
                 {
-                    FileDetails.setFromRoxieCluster(true);
+                    fromRoxieCluster = true;
                     break;
                 }
             }
+            FileDetails.setFromRoxieCluster(fromRoxieCluster);
         }
     }
 
-    IArrayOf<IConstDFUPart>& PartList = FileDetails.getDFUFileParts();
-
-    Owned<IDistributedFilePartIterator> pi = df->getIterator();
     offset_t mn=LLC(0x7fffffffffffffff), mx=0, sum=0, count=0;
-    ForEach(*pi)
+    if (version >= 1.25)
+        getFilePartsOnClusters(context, cluster, clusters, df, FileDetails, mn, mx, sum, count);
+    else
     {
-        Owned<IDistributedFilePart> part = &pi->get();
-        for (unsigned int i=0; i<part->numCopies(); i++)
+        FileDetails.setCluster(clusters.item(0));
+        IArrayOf<IConstDFUPart>& PartList = FileDetails.getDFUFileParts();
+
+        Owned<IDistributedFilePartIterator> pi = df->getIterator();
+        ForEach(*pi)
         {
-            Owned<IEspDFUPart> FilePart = createDFUPart("","");
+            Owned<IDistributedFilePart> part = &pi->get();
+            for (unsigned int i=0; i<part->numCopies(); i++)
+            {
+                Owned<IEspDFUPart> FilePart = createDFUPart("","");
 
-            StringBuffer b;
-            part->queryNode(i)->endpoint().getUrlStr(b);
+                StringBuffer b;
+                part->queryNode(i)->endpoint().getUrlStr(b);
 
-            FilePart->setId(part->getPartIndex()+1);
-            FilePart->setCopy(i+1);
-            FilePart->setIp(b.str());
-            FilePart->setPartsize("<N/A>");
+                FilePart->setId(part->getPartIndex()+1);
+                FilePart->setCopy(i+1);
+                FilePart->setIp(b.str());
+                FilePart->setPartsize("<N/A>");
 
-           try
-            {
-                offset_t size=queryDistributedFileSystem().getSize(part);
-                comma c4(size);
-                tmpstr.clear();
-                tmpstr<<c4;
-                FilePart->setPartsize(tmpstr.str());
+               try
+                {
+                    offset_t size=queryDistributedFileSystem().getSize(part);
+                    comma c4(size);
+                    tmpstr.clear();
+                    tmpstr<<c4;
+                    FilePart->setPartsize(tmpstr.str());
 
-                if(size!=-1)
+                    if(size!=-1)
+                    {
+                        count+=1;
+                        sum+=size;
+                        if(size>mx) mx=size;
+                        if(size<mn) mn=size;
+                    }
+                }
+                catch(IException *e)
                 {
-                    count+=1;
-                    sum+=size;
-                    if(size>mx) mx=size;
-                    if(size<mn) mn=size;
+                    StringBuffer msg;
+                    ERRLOG("Exception %d:%s in WS_DFU queryDistributedFileSystem().getSize()", e->errorCode(), e->errorMessage(msg).str());
+                    e->Release();
+                }
+                catch(...)
+                {
+                    ERRLOG("Unknown exception in WS_DFU queryDistributedFileSystem().getSize()");
                 }
-            }
-            catch(IException *e)
-            {
-                StringBuffer msg;
-                ERRLOG("Exception %d:%s in WS_DFU queryDistributedFileSystem().getSize()", e->errorCode(), e->errorMessage(msg).str());
-                e->Release();
-            }
-            catch(...)
-            {
-                ERRLOG("Unknown exception in WS_DFU queryDistributedFileSystem().getSize()");
-            }
 
-            PartList.append(*FilePart.getClear());
+                PartList.append(*FilePart.getClear());
+            }
         }
     }
     if(count)

esp/services/ws_dfu/ws_dfuService.hpp

@@ -132,6 +132,8 @@ private:
     void parseStringArray(const char *input, StringArray& strarray);
     int superfileAction(IEspContext &context, const char* action, const char* superfile, StringArray& subfiles,
         const char* beforeSubFile, bool existingSuperfile, bool autocreatesuper, bool deleteFile, bool removeSuperfile =  true);
+    void getFilePartsOnClusters(IEspContext &context, const char* clusterReq, StringArray& clusters, IDistributedFile* df, IEspDFUFileDetail& FileDetails,
+        offset_t& mn, offset_t& mx, offset_t& sum, offset_t& count);
 private:
     bool         m_disableUppercaseTranslation;
     StringBuffer m_clusterName;