Merge pull request #10480 from shamser/dfurecordtypeinfo

HPCC-18419 Service to retrieve serialized typeInfo information

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/hql/hqlexpr.cpp

@@ -14060,6 +14060,20 @@ void exportMap(IPropertyTree *dataNode, IHqlExpression *destTable, IHqlExpressio
     maps->addPropTree("MapTables", map);
 }
 
+void exportJsonType(StringBuffer &ret, IHqlExpression *table)
+{
+    Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
+    const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
+    dumpTypeInfo(ret, typeInfo);
+}
+
+void exportBinaryType(MemoryBuffer &ret, IHqlExpression *table)
+{
+    Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
+    const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
+    dumpTypeInfo(ret, typeInfo);
+}
+
 void exportData(IPropertyTree *data, IHqlExpression *table, bool flatten)
 {
     IPropertyTree *tt = NULL;

ecl/hql/hqlexpr.hpp

@@ -1830,6 +1830,8 @@ extern HQL_API StringBuffer& getFriendlyTypeStr(ITypeInfo* type, StringBuffer& s
         for (unsigned idx = first; idx < numOfChildren##idx; idx++) 
 
 extern HQL_API void exportData(IPropertyTree *data, IHqlExpression *table, bool flatten=false);
+extern HQL_API void exportJsonType(StringBuffer &ret, IHqlExpression *table);
+extern HQL_API void exportBinaryType(MemoryBuffer &ret, IHqlExpression *table);
 
 extern HQL_API void clearCacheCounts();
 extern HQL_API void displayHqlCacheStats();

esp/scm/ws_dfu.ecm

@@ -145,6 +145,8 @@ ESPStruct [nil_remove] DFUFileDetail
     [min_ver("1.34")] string PercentCompressed;
     [min_ver("1.22")] bool IsCompressed(false);
     [min_ver("1.28")] bool BrowseData(true);
+    [min_ver("1.37"), json_inline(1)] string jsonInfo;
+    [min_ver("1.37")] binary binInfo;
 };
 
 ESPStruct DFUSpaceItem
@@ -292,6 +294,8 @@ DFUInfoRequest
     bool UpdateDescription(false);
     string FileName;
     string FileDesc;
+    [min_ver("1.37")] bool IncludeJsonTypeInfo(false);
+    [min_ver("1.37")] bool IncludeBinTypeInfo(false);
 };
 
 ESPresponse
@@ -319,6 +323,19 @@ DFUDefFileResponse
     [http_content("application/octet-stream")] binary defFile;
 };
 
+ESPrequest DFURecordTypeInfoRequest
+{
+    string Name;
+    bool IncludeJsonTypeInfo(true);
+    bool IncludeBinTypeInfo(true);
+};
+
+ESPresponse DFURecordTypeInfoResponse
+{
+    [json_inline(1)] string jsonInfo;
+    binary binInfo;
+};
+
 ESPrequest SuperfileActionRequest
 {
     string action;
@@ -745,7 +762,7 @@ ESPresponse [exceptions_inline, nil_remove] EraseHistoryResponse
 //  ===========================================================================
 ESPservice [
     auth_feature("DEFERRED"),
-    version("1.36"),
+    version("1.37"),
     noforms,
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {
@@ -771,6 +788,7 @@ ESPservice [
 
     ESPmethod ListHistory(ListHistoryRequest, ListHistoryResponse);
     ESPmethod EraseHistory(EraseHistoryRequest, EraseHistoryResponse);
+    ESPmethod DFURecordTypeInfo(DFURecordTypeInfoRequest, DFURecordTypeInfoResponse);
 };
 
 SCMexportdef(WSDFU);

esp/services/ws_dfu/ws_dfuService.cpp

@@ -346,11 +346,13 @@ bool CWsDfuEx::onDFUInfo(IEspContext &context, IEspDFUInfoRequest &req, IEspDFUI
 
         if (req.getUpdateDescription())
         {
-            doGetFileDetails(context, userdesc.get(), req.getFileName(), req.getCluster(), req.getFileDesc(), resp.updateFileDetail());
+            doGetFileDetails(context, userdesc.get(), req.getFileName(), req.getCluster(), req.getFileDesc(),
+                             req.getIncludeJsonTypeInfo(), req.getIncludeBinTypeInfo(), resp.updateFileDetail());
         }
         else
         {
-            doGetFileDetails(context, userdesc.get(), req.getName(), req.getCluster(), NULL, resp.updateFileDetail());
+            doGetFileDetails(context, userdesc.get(), req.getName(), req.getCluster(), NULL,
+                             req.getIncludeJsonTypeInfo(), req.getIncludeBinTypeInfo(), resp.updateFileDetail());
         }
     }
     catch(IException* e)
@@ -1531,21 +1533,23 @@ bool CWsDfuEx::onDFUDefFile(IEspContext &context,IEspDFUDefFileRequest &req, IEs
     return true;
 }
 
-void CWsDfuEx::xsltTransformer(const char* xsltPath,StringBuffer& source,StringBuffer& returnStr)
+IHqlExpression * getEclRecordDefinition(const char * ecl)
 {
-    if (m_xsl.get() == 0)
+    MultiErrorReceiver errs;
+    OwnedHqlExpr record = parseQuery(ecl, &errs);
+    if (errs.errCount())
     {
-        m_xsl.setown(getXslProcessor());
+        StringBuffer errtext;
+        IError *first = errs.firstError();
+        first->toString(errtext);
+        throw MakeStringException(ECLWATCH_CANNOT_PARSE_ECL_QUERY, "Failed in parsing ECL record definition: %s @ %d:%d.", errtext.str(), first->getColumn(), first->getLine());
     }
-
-    Owned<IXslTransform> xform = m_xsl->createXslTransform();
-
-    xform->loadXslFromFile(xsltPath);
-    xform->setXmlSource(source.str(), source.length()+1);
-    xform->transform(returnStr.clear());
+    if(!record)
+        throw MakeStringException(ECLWATCH_CANNOT_PARSE_ECL_QUERY, "Failed in parsing ECL record definition.");
+    return record.getClear();
 }
 
-void CWsDfuEx::getDefFile(IUserDescriptor* udesc, const char* FileName,StringBuffer& returnStr)
+IHqlExpression * getEclRecordDefinition(IUserDescriptor* udesc, const char* FileName)
 {
     Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(FileName, udesc);
     if(!df)
@@ -1553,22 +1557,66 @@ void CWsDfuEx::getDefFile(IUserDescriptor* udesc, const char* FileName,StringBuf
     if(!df->queryAttributes().hasProp("ECL"))
         throw MakeStringException(ECLWATCH_MISSING_PARAMS,"No record definition for file %s.",FileName);
 
-    StringBuffer text;
-    text.append(df->queryAttributes().queryProp("ECL"));
+    return getEclRecordDefinition(df->queryAttributes().queryProp("ECL"));
+}
 
-    MultiErrorReceiver errs;
-    OwnedHqlExpr record = parseQuery(text.str(), &errs);
-    if (errs.errCount())
+bool CWsDfuEx::onDFURecordTypeInfo(IEspContext &context, IEspDFURecordTypeInfoRequest &req, IEspDFURecordTypeInfoResponse &resp)
+{
+    try
     {
-        StringBuffer errtext;
-        IError *first = errs.firstError();
-        first->toString(errtext);
-        throw MakeStringException(ECLWATCH_CANNOT_PARSE_ECL_QUERY, "Failed in parsing ECL query: %s @ %d:%d.", errtext.str(), first->getColumn(), first->getLine());
+        if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Read, false))
+            throw MakeStringException(ECLWATCH_DFU_ACCESS_DENIED, "Failed to access DFURecordTypeInfo. Permission denied.");
+
+        const char* fileName = req.getName();
+        if (!fileName || !*fileName)
+            throw MakeStringException(ECLWATCH_MISSING_PARAMS, "File name required");
+        PROGLOG("DFURecordTypeInfo file: %s", fileName);
+
+        const char* userId = context.queryUserId();
+        Owned<IUserDescriptor> userdesc;
+        if(userId && *userId)
+        {
+            userdesc.setown(createUserDescriptor());
+            userdesc->set(userId, context.queryPassword(), context.querySessionToken(), context.querySignature());
+        }
+
+        OwnedHqlExpr record = getEclRecordDefinition(userdesc, fileName);
+        if (req.getIncludeJsonTypeInfo())
+        {
+            StringBuffer jsonFormat;
+            exportJsonType(jsonFormat,record);
+            resp.setJsonInfo(jsonFormat);
+        }
+        if (req.getIncludeBinTypeInfo())
+        {
+            MemoryBuffer binFormat;
+            exportBinaryType(binFormat,record);
+            resp.setBinInfo(binFormat);
+        }
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}
+void CWsDfuEx::xsltTransformer(const char* xsltPath,StringBuffer& source,StringBuffer& returnStr)
+{
+    if (m_xsl.get() == 0)
+    {
+        m_xsl.setown(getXslProcessor());
     }
 
-    if(!record)
-        throw MakeStringException(ECLWATCH_CANNOT_PARSE_ECL_QUERY, "Failed in parsing ECL query.");
+    Owned<IXslTransform> xform = m_xsl->createXslTransform();
+
+    xform->loadXslFromFile(xsltPath);
+    xform->setXmlSource(source.str(), source.length()+1);
+    xform->transform(returnStr.clear());
+}
 
+void CWsDfuEx::getDefFile(IUserDescriptor* udesc, const char* FileName,StringBuffer& returnStr)
+{
+    OwnedHqlExpr record = getEclRecordDefinition(udesc, FileName);
     Owned<IPropertyTree> data = createPTree("Table", ipt_caseInsensitive);
     exportData(data, record);
 
@@ -1819,7 +1867,7 @@ void CWsDfuEx::getFilePartsOnClusters(IEspContext &context, const char* clusterR
 }
 
 void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, const char *name, const char *cluster,
-    const char *description,IEspDFUFileDetail& FileDetails)
+    const char *description, bool includeJsonTypeInfo, bool includeBinTypeInfo, IEspDFUFileDetail& FileDetails)
 {
     if (!name || !*name)
         throw MakeStringException(ECLWATCH_MISSING_PARAMS, "File name required");
@@ -2186,6 +2234,22 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, co
             }
         }
     }
+    if (df->queryAttributes().hasProp("ECL") && (includeJsonTypeInfo||includeBinTypeInfo) )
+    {
+        OwnedHqlExpr record = getEclRecordDefinition(df->queryAttributes().queryProp("ECL"));
+        if (includeJsonTypeInfo)
+        {
+            StringBuffer jsonFormat;
+            exportJsonType(jsonFormat,record);
+            FileDetails.setJsonInfo(jsonFormat);
+        }
+        if (includeBinTypeInfo)
+        {
+            MemoryBuffer binFormat;
+            exportBinaryType(binFormat,record);
+            FileDetails.setBinInfo(binFormat);
+        }
+    }
     PROGLOG("doGetFileDetails: %s done", name);
 }
 

esp/services/ws_dfu/ws_dfuService.hpp

@@ -131,7 +131,8 @@ public:
     bool onDFUBrowseData(IEspContext &context, IEspDFUBrowseDataRequest &req, IEspDFUBrowseDataResponse &resp);
     bool onDFUSearchData(IEspContext &context, IEspDFUSearchDataRequest &req, IEspDFUSearchDataResponse &resp);
     bool onDFUGetFileMetaData(IEspContext &context, IEspDFUGetFileMetaDataRequest &req, IEspDFUGetFileMetaDataResponse &resp);
-
+    bool onDFURecordTypeInfo(IEspContext &context, IEspDFURecordTypeInfoRequest &req, IEspDFURecordTypeInfoResponse &resp);
+	
     virtual bool onSavexml(IEspContext &context, IEspSavexmlRequest &req, IEspSavexmlResponse &resp);
     virtual bool onAdd(IEspContext &context, IEspAddRequest &req, IEspAddResponse &resp);
     virtual bool onAddRemote(IEspContext &context, IEspAddRemoteRequest &req, IEspAddRemoteResponse &resp);
@@ -157,7 +158,7 @@ private:
         bool includeSuperOwner, IArrayOf<IEspDFULogicalFile>& LogicalFiles, int& numFiles, int& numDirs);
     bool doLogicalFileSearch(IEspContext &context, IUserDescriptor* udesc, IEspDFUQueryRequest & req, IEspDFUQueryResponse & resp);
     void doGetFileDetails(IEspContext &context, IUserDescriptor* udesc, const char *name,const char *cluster,
-        const char *description,IEspDFUFileDetail& FileDetails);
+        const char *description, bool includeJsonTypeInfo, bool includeBinTypeInfo, IEspDFUFileDetail& FileDetails);
     bool createSpaceItemsByDate(IArrayOf<IEspSpaceItem>& SpaceItems, StringBuffer interval, unsigned& yearFrom,
         unsigned& monthFrom, unsigned& dayFrom, unsigned& yearTo, unsigned& monthTo, unsigned& dayTo);
     bool setSpaceItemByScope(IArrayOf<IEspSpaceItem>& SpaceItems64, const char*scopeName, const char*logicalName, __int64 size);

rtl/eclrtl/rtldynfield.cpp

@@ -678,6 +678,11 @@ extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInf
     return CRtlFieldTypeSerializer::serialize(ret, t);
 }
 
+extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t)
+{
+    return CRtlFieldTypeBinSerializer::serialize(ret, t);
+}
+
 extern ECLRTL_API void dumpRecordType(size32_t & __lenResult,char * & __result,IOutputMetaData &metaVal)
 {
     StringBuffer ret;

rtl/eclrtl/rtldynfield.hpp

@@ -96,6 +96,7 @@ interface IRtlFieldTypeDeserializer : public IInterface
 extern ECLRTL_API IRtlFieldTypeDeserializer *createRtlFieldTypeDeserializer();
 
 extern ECLRTL_API StringBuffer &dumpTypeInfo(StringBuffer &ret, const RtlTypeInfo *t);
+extern ECLRTL_API MemoryBuffer &dumpTypeInfo(MemoryBuffer &ret, const RtlTypeInfo *t);
 
 /**
  * Serialize metadata of supplied record to JSON, and return it to ECL caller as a string. Used for testing serializer.