HPCC-12213 Add new WsDFU method to retrieve meta data

A new WsDFU method DFUGetFileMetaData is added to
retrieve MetaData from DFU IResultSetMetaData.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

esp/scm/ws_dfu.ecm

@@ -420,13 +420,18 @@ ESPresponse [exceptions_inline] DFUFileViewResponse
     ESParray<ESPstruct DFULogicalFile> DFULogicalFiles;
 };
 
-ESPStruct DFUDataColumn
+ESPStruct [nil_remove] DFUDataColumn
 {
+    [min_ver("1.29")] int ColumnID;
     string ColumnLabel;
     string ColumnType;
     [min_ver("1.05")] string ColumnValue;
     int ColumnSize;
     int MaxSize;
+    [min_ver("1.29")] string ColumnEclType;
+    [min_ver("1.29")] int ColumnRawSize;
+    [min_ver("1.29")] bool IsNaturalColumn;
+    [min_ver("1.29")] bool IsKeyedColumn;
 };
 
 ESPrequest DFUGetDataColumnsRequest
@@ -651,10 +656,30 @@ ESPresponse [exceptions_inline, nil_remove, http_encode(0)] DFUSearchDataRespons
 
 };
 
+ESPrequest DFUGetFileMetaDataRequest
+{
+    string LogicalFileName;
+    string ClusterName;
+    bool IncludeXmlSchema(false);
+    bool AddHeaderInXmlSchema(false);
+    bool IncludeXmlXPathSchema(false);
+    bool AddHeaderInXmlXPathSchema(false);
+};
+
+ESPresponse [exceptions_inline, nil_remove, http_encode(0)] DFUGetFileMetaDataResponse
+{
+    int TotalColumnCount;
+    int KeyedColumnCount;
+    ESParray<ESPstruct DFUDataColumn> DataColumns;
+    string XmlSchema;
+    string XmlXPathSchema;
+    int64 TotalResultRows;
+};
+
 
 //  ===========================================================================
 ESPservice [
-    version("1.28"), default_client_version("1.28"),
+    version("1.29"), default_client_version("1.29"),
     noforms, 
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {
@@ -673,6 +698,7 @@ ESPservice [
     ESPmethod [resp_xsl_default("/esp/xslt/dfu_getdatacolumns.xslt")] DFUGetDataColumns(DFUGetDataColumnsRequest, DFUGetDataColumnsResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/dfu_viewdata.xslt")] DFUBrowseData(DFUBrowseDataRequest, DFUBrowseDataResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/dfu_searchdata.xslt")] DFUSearchData(DFUSearchDataRequest, DFUSearchDataResponse);
+    ESPmethod DFUGetFileMetaData(DFUGetFileMetaDataRequest, DFUGetFileMetaDataResponse);
     ESPmethod DFUDefFile(DFUDefFileRequest, DFUDefFileResponse);
 
     ESPmethod [resp_xsl_default("/esp/xslt/addto_superfile.xslt")] AddtoSuperfile(AddtoSuperfileRequest, AddtoSuperfileResponse);

esp/services/ws_dfu/ws_dfuService.cpp

@@ -4249,6 +4249,139 @@ bool CWsDfuEx::onDFUSearchData(IEspContext &context, IEspDFUSearchDataRequest &r
     return true;
 }
 
+static const char * const columnTypes[] = { "Boolean", "Integer", "Unsigned Integer", "Real", "String",
+    "Data", "Unicode", "Unknown", "BeginIfBlock", "EndIfBlock", "BeginRecord", "EndRecord", "Set", "Dataset", NULL };
+
+bool CWsDfuEx::onDFUGetFileMetaData(IEspContext &context, IEspDFUGetFileMetaDataRequest & req, IEspDFUGetFileMetaDataResponse & resp)
+{
+    class CDFUFileMetaDataReader: public CInterface, implements IDFUFileMetaDataReader
+    {
+        int totalCcolumnCount;
+        int keyedColumnCount;
+        StringBuffer XmlSchema, XmlXPathSchema;
+        IArrayOf<IEspDFUDataColumn> dataColumns;
+        const IResultSetMetaData& meta;
+
+        bool readColumnLabel(const int columnID, IEspDFUDataColumn* out)
+        {
+            SCMStringBuffer columnLabel;
+            bool isNaturalColumn = true;
+            if (meta.hasSetTranslation(columnID))
+                meta.getNaturalColumnLabel(columnLabel, columnID);
+            if (columnLabel.length() < 1)
+            {
+                meta.getColumnLabel(columnLabel, columnID);
+                isNaturalColumn = false;
+            }
+            out->setColumnLabel(columnLabel.str());
+            out->setIsNaturalColumn(isNaturalColumn);
+            return isNaturalColumn;
+        }
+        void readColumnEclType(const int columnID, IEspDFUDataColumn* out)
+        {
+            SCMStringBuffer s;
+            out->setColumnEclType(meta.getColumnEclType(s, columnID).str());
+        }
+        void readColumnRawSize(const int columnID, IEspDFUDataColumn* out)
+        {
+            DisplayType columnType = meta.getColumnDisplayType(columnID);
+            if ((columnType == TypeUnicode) || columnType == TypeString)
+                out->setColumnRawSize(meta.getColumnRawSize(columnID));
+        }
+        void readColumn(const int columnID, const bool isKeyed, IArrayOf<IEspDFUDataColumn>& dataColumns)
+        {
+            Owned<IEspDFUDataColumn> dataItem = createDFUDataColumn();
+            dataItem->setColumnID(columnID+1);
+            dataItem->setIsKeyedColumn(isKeyed);
+
+            if (readColumnLabel(columnID, dataItem))
+                dataItem->setColumnType("Others");
+            else
+                dataItem->setColumnType(columnTypes[meta.getColumnDisplayType(columnID)]);
+            readColumnRawSize(columnID, dataItem);
+            readColumnEclType(columnID, dataItem);
+            dataColumns.append(*dataItem.getClear());
+        }
+
+    public:
+        IMPLEMENT_IINTERFACE;
+        CDFUFileMetaDataReader(const IResultSetMetaData& _meta) : meta(_meta)
+        {
+            totalCcolumnCount = meta.getColumnCount();
+            keyedColumnCount = meta.getNumKeyedColumns();
+            unsigned i = 0;
+            for (; i < keyedColumnCount; i++)
+                readColumn(i, true, dataColumns);
+            for (i = keyedColumnCount; i < totalCcolumnCount; i++)
+                readColumn(i, false, dataColumns);
+        };
+        inline unsigned getTotalColumnCount() { return totalCcolumnCount; }
+        inline unsigned getKeyedColumnCount() { return keyedColumnCount; }
+        inline IArrayOf<IEspDFUDataColumn>& getDataColumns() { return dataColumns; }
+        inline StringBuffer& getXmlSchema(StringBuffer& s, const bool addHeader)
+        {
+            StringBufferAdaptor schema(s);
+            meta.getXmlSchema(schema, addHeader);
+            return s;
+        }
+        inline StringBuffer& getXmlXPathSchema(StringBuffer& s, const bool addHeader)
+        {
+            StringBufferAdaptor XPathSchema(s);
+            meta.getXmlXPathSchema(XPathSchema, addHeader);
+            return s;
+        }
+    };
+
+    try
+    {
+        StringBuffer nameStr, fileNameStr = req.getLogicalFileName();
+        const char* fileName = fileNameStr.trim().str();
+        if (!fileName || !*fileName)
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "CWsDfuEx::onDFUGetFileMetaData: LogicalFileName not set");
+
+        {
+            Owned<IUserDescriptor> userdesc = createUserDescriptor();
+            userdesc->set(context.getUserID(nameStr).str(), context.queryPassword());
+            Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userdesc);
+            if(!df)
+                throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"CWsDfuEx::onDFUGetFileMetaData: Could not find file %s.", fileName);
+
+            IDistributedSuperFile *sf = df->querySuperFile();
+            if (sf && (sf->numSubFiles() > 1))
+                throw MakeStringException(ECLWATCH_INVALID_ACTION, "CWsDfuEx::onDFUGetFileMetaData: This feature is not designed to work with a superfile which contains multiple subfiles.");
+        }
+
+        const char* cluster = NULL;
+        StringBuffer clusterNameStr = req.getClusterName();
+        if (clusterNameStr.trim().length() > 0)
+            cluster = clusterNameStr.str();
+
+        Owned<IResultSetFactory> resultSetFactory = getSecResultSetFactory(context.querySecManager(), context.queryUser(), context.queryUserId(), context.queryPassword());
+        Owned<INewResultSet> result = resultSetFactory->createNewFileResultSet(fileName, cluster);
+        if (!result)
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "CWsDfuEx::onDFUGetFileMetaData: Failed to access FileResultSet for %s.", fileName);
+
+        Owned<IResultSetCursor> cursor = result->createCursor();
+        Owned<IDFUFileMetaDataReader> dataReader = new CDFUFileMetaDataReader(cursor->queryResultSet()->getMetaData());
+        resp.setTotalColumnCount(dataReader->getTotalColumnCount());
+        resp.setKeyedColumnCount(dataReader->getKeyedColumnCount());
+        resp.setDataColumns(dataReader->getDataColumns());
+
+        StringBuffer s, s1;
+        if (req.getIncludeXmlSchema())
+            resp.setXmlSchema(dataReader->getXmlSchema(s, req.getAddHeaderInXmlSchema()).str());
+        if (req.getIncludeXmlXPathSchema())
+            resp.setXmlXPathSchema(dataReader->getXmlXPathSchema(s1, req.getAddHeaderInXmlXPathSchema()).str());
+
+        resp.setTotalResultRows(result->getNumRows());
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}
+
 bool CWsDfuEx::onDFUBrowseData(IEspContext &context, IEspDFUBrowseDataRequest &req, IEspDFUBrowseDataResponse &resp)
 {
     try

esp/services/ws_dfu/ws_dfuService.hpp

@@ -30,6 +30,15 @@
 #include "fvrelate.hpp"
 #include "dadfs.hpp"
 
+interface IDFUFileMetaDataReader : extends IInterface
+{
+    virtual unsigned getTotalColumnCount() = 0;
+    virtual unsigned getKeyedColumnCount() = 0;
+    virtual IArrayOf<IEspDFUDataColumn>& getDataColumns() = 0;
+    virtual StringBuffer& getXmlSchema(StringBuffer& s, const bool addHeader) = 0;
+    virtual StringBuffer& getXmlXPathSchema(StringBuffer& s, const bool addHeader) = 0;
+};
+
 class CWsDfuSoapBindingEx : public CWsDfuSoapBinding
 {
 public:
@@ -64,6 +73,7 @@ public:
     bool onDFUGetDataColumns(IEspContext &context, IEspDFUGetDataColumnsRequest &req, IEspDFUGetDataColumnsResponse &resp);
     bool onDFUBrowseData(IEspContext &context, IEspDFUBrowseDataRequest &req, IEspDFUBrowseDataResponse &resp);
     bool onDFUSearchData(IEspContext &context, IEspDFUSearchDataRequest &req, IEspDFUSearchDataResponse &resp);
+    bool onDFUGetFileMetaData(IEspContext &context, IEspDFUGetFileMetaDataRequest &req, IEspDFUGetFileMetaDataResponse &resp);
 
     virtual bool onSavexml(IEspContext &context, IEspSavexmlRequest &req, IEspSavexmlResponse &resp);
     virtual bool onAdd(IEspContext &context, IEspAddRequest &req, IEspAddResponse &resp);