Merge pull request #11420 from wangkx/h19607

HPCC-20090 Return partition/bloom information in WsDfu.DFUInfo

Reviewed-By: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

esp/scm/ws_dfu.ecm

@@ -49,6 +49,20 @@ ESPStruct SpaceItem
     int64 SmallestSizeInt;
 };
 
+ESPStruct DFUFilePartition
+{
+    int64 FieldMask;
+    ESParray<string> FieldNames;
+};
+
+ESPStruct DFUFileBloom
+{
+    int64 FieldMask;
+    ESParray<string> FieldNames;
+    int64 Limit;
+    string Probability;
+};
+
 ESPStruct DFULogicalFile
 {
     string Prefix;
@@ -167,6 +181,8 @@ ESPStruct [nil_remove] DFUFileDetail
     [min_ver("1.37"), json_inline(1)] string jsonInfo;
     [min_ver("1.37")] binary binInfo;
     [min_ver("1.38")] string PackageID;
+    [min_ver("1.39")] ESPstruct DFUFilePartition Partition;
+    [min_ver("1.39")] ESParray<ESPstruct DFUFileBloom> Blooms;
 };
 
 ESPStruct DFUSpaceItem

esp/services/ws_dfu/ws_dfuService.cpp

@@ -2003,6 +2003,51 @@ void CWsDfuEx::getFilePartsOnClusters(IEspContext &context, const char* clusterR
     }
 }
 
+void CWsDfuEx::parseFieldMask(unsigned __int64 fieldMask, unsigned &fieldCount, IntArray &fieldIndexArray)
+{
+    while (fieldMask > 0)
+    {
+        if (fieldMask & 1)
+            fieldIndexArray.append(fieldCount); //index from 0
+        fieldMask >>= 1;
+        fieldCount++;
+    }
+}
+
+void CWsDfuEx::queryFieldNames(IEspContext &context, const char *fileName, const char *cluster,
+    unsigned __int64 fieldMask, StringArray &fieldNames)
+{
+    if (!fileName || !*fileName)
+        throw MakeStringException(ECLWATCH_MISSING_PARAMS, "File name required");
+
+    Owned<IResultSetFactory> resultSetFactory = getSecResultSetFactory(context.querySecManager(), context.queryUser(), context.queryUserId(), context.queryPassword());
+    Owned<INewResultSet> result = resultSetFactory->createNewFileResultSet(fileName, cluster);
+    if (!result)
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "Failed to access FileResultSet for %s.", fileName);
+
+    unsigned fieldCount = 0;
+    IntArray fieldIndexArray;
+    parseFieldMask(fieldMask, fieldCount, fieldIndexArray);
+
+    const IResultSetMetaData& metaData = result->getMetaData();
+    unsigned totalColumns = (unsigned) metaData.getColumnCount();
+    if (fieldCount > totalColumns)
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid FieldMask %" I64F "u: total fields %u, ask for %u.",
+            fieldMask, totalColumns, fieldCount);
+
+    ForEachItemIn(i, fieldIndexArray)
+    {
+        int fieldIndex = fieldIndexArray.item(i);
+
+        SCMStringBuffer columnLabel;
+        if (metaData.hasSetTranslation(fieldIndex))
+            metaData.getNaturalColumnLabel(columnLabel, fieldIndex);
+        if (columnLabel.length() < 1)
+            metaData.getColumnLabel(columnLabel, fieldIndex);
+        fieldNames.append(columnLabel.str());
+    }
+}
+
 void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, const char *name, const char *cluster,
     const char *querySet, const char *query, const char *description, bool includeJsonTypeInfo, bool includeBinTypeInfo,
     CDFUChangeProtection protect, IEspDFUFileDetail &FileDetails)
@@ -2143,6 +2188,40 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
     FileDetails.setOwner(df->queryAttributes().queryProp("@owner"));
     FileDetails.setJobName(df->queryAttributes().queryProp("@job"));
 
+    if (version >= 1.39)
+    {
+        if (df->queryAttributes().hasProp("@partitionFieldMask"))
+        {
+            StringArray partitionFieldNames;
+            unsigned __int64 partitionFieldMask = df->queryAttributes().getPropInt64("@partitionFieldMask");
+            queryFieldNames(context, name, cluster, partitionFieldMask, partitionFieldNames);
+
+            IEspDFUFilePartition &partition = FileDetails.updatePartition();
+            partition.setFieldMask(partitionFieldMask);
+            partition.setFieldNames(partitionFieldNames);
+        }
+
+        IArrayOf<IEspDFUFileBloom> bloomList;
+        Owned<IPropertyTreeIterator> itr= df->queryAttributes().getElements("Bloom");
+        ForEach(*itr)
+        {
+            IPropertyTree &bloomTree = itr->query();
+
+            StringArray bloomFieldNames;
+            unsigned __int64 bloomFieldMask = bloomTree.getPropInt64("@bloomFieldMask");
+            queryFieldNames(context, name, cluster, bloomFieldMask, bloomFieldNames);
+
+            Owned<IEspDFUFileBloom> bloom= createDFUFileBloom();
+            bloom->setFieldMask(bloomFieldMask);
+            bloom->setFieldNames(bloomFieldNames);
+            bloom->setLimit(bloomTree.getPropInt64("@bloomLimit"));
+            bloom->setProbability(bloomTree.queryProp("@bloomProbability"));
+            bloomList.append(*bloom.getLink());
+        }
+        if (bloomList.ordinality())
+            FileDetails.setBlooms(bloomList);
+    }
+
     //#14280
     IDistributedSuperFile *sf = df->querySuperFile();
     if(sf)

esp/services/ws_dfu/ws_dfuService.hpp

@@ -228,6 +228,9 @@ private:
     void getFilePartsOnClusters(IEspContext &context, const char* clusterReq, StringArray& clusters, IDistributedFile* df, IEspDFUFileDetail& FileDetails,
         offset_t& mn, offset_t& mx, offset_t& sum, offset_t& count);
     bool getQueryFile(const char *logicalName, const char *querySet, const char *queryID, IEspDFUFileDetail &fileDetails);
+    void queryFieldNames(IEspContext &context, const char *fileName, const char *cluster,
+        unsigned __int64 fieldMask, StringArray &fieldNames);
+    void parseFieldMask(unsigned __int64 fieldMask, unsigned &fieldCount, IntArray &fieldIndexArray);
     bool attachServiceToDali() override
     {
         m_daliDetached = false;