Merge pull request #14365 from wangkx/h25032

HPCC-25032 Add framework and first method for ESP Dali service

Reviewed-By: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

dali/base/dautils.cpp

@@ -3568,3 +3568,36 @@ ILockInfoCollection *deserializeLockInfoCollection(MemoryBuffer &mb)
 {
     return new CLockInfoCollection(mb);
 }
+
+static const char* remLeading(const char* s)
+{
+    if (*s == '/')
+        s++;
+    return s;
+}
+
+static unsigned daliConnectTimeoutMs = 5000;
+extern da_decl IRemoteConnection* connectXPathOrFile(const char* path, bool safe, StringBuffer& xpath)
+{
+    CDfsLogicalFileName lfn;
+    StringBuffer lfnPath;
+    if ((strstr(path, "::") != nullptr) && !strchr(path, '/'))
+    {
+        lfn.set(path);
+        lfn.makeFullnameQuery(lfnPath, DXB_File);
+        path = lfnPath.str();
+    }
+    else if (strchr(path + ((*path == '/') ? 1 : 0),'/') == nullptr)
+        safe = true;    // all root trees safe
+
+    Owned<IRemoteConnection> conn = querySDS().connect(remLeading(path), myProcessSession(), safe ? 0 : RTM_LOCK_READ, daliConnectTimeoutMs);
+    if (!conn && !lfnPath.isEmpty())
+    {
+        lfn.makeFullnameQuery(lfnPath.clear(), DXB_SuperFile);
+        path = lfnPath.str();
+        conn.setown(querySDS().connect(remLeading(path), myProcessSession(), safe? 0 : RTM_LOCK_READ, daliConnectTimeoutMs));
+    }
+    if (conn.get())
+        xpath.append(path);
+    return conn.getClear();
+}

dali/base/dautils.hpp

@@ -536,5 +536,6 @@ extern da_decl ILockInfoCollection *deserializeLockInfoCollection(MemoryBuffer &
 
 extern da_decl void setPageCacheTimeoutMilliSeconds(unsigned timeoutSeconds);
 extern da_decl void setMaxPageCacheItems(unsigned _maxPageCacheItems);
+extern da_decl IRemoteConnection* connectXPathOrFile(const char* path, bool safe, StringBuffer& xpath);
 
 #endif

dali/daliadmin/daliadmin.cpp

@@ -220,30 +220,6 @@ static unsigned __int64 hextoll(const char *str, bool &error)
     return rolling;
 }
 
-
-static IRemoteConnection *connectXPathOrFile(const char *path,bool safe,StringBuffer &xpath)
-{
-    CDfsLogicalFileName lfn;
-    StringBuffer lfnpath;
-    if ((strstr(path,"::")!=NULL)&&!strchr(path,'/')) {
-        lfn.set(path);
-        lfn.makeFullnameQuery(lfnpath,DXB_File);
-        path = lfnpath.str();
-    }
-    else if (strchr(path+((*path=='/')?1:0),'/')==NULL)
-        safe = true;    // all root trees safe
-    Owned<IRemoteConnection> conn = querySDS().connect(remLeading(path),myProcessSession(),safe?0:RTM_LOCK_READ, daliConnectTimeoutMs);
-    if (!conn&&lfnpath.length()) {
-        lfn.makeFullnameQuery(lfnpath.clear(),DXB_SuperFile);
-        path = lfnpath.str();
-        conn.setown(querySDS().connect(remLeading(path),myProcessSession(),safe?0:RTM_LOCK_READ, daliConnectTimeoutMs));
-    }
-    if (conn.get())
-        xpath.append(path);
-    return conn.getClear();
-}
-
-
 //=============================================================================
 
 static void _export_(const char *path,const char *dst,bool safe=false)

esp/applications/eclwatch/application.yaml

@@ -16,3 +16,4 @@ application:
    - ws_esdlconfig
    - ws_store
    - ws_codesign
+   - WSDali

esp/applications/eclwatch/ldap_authorization_map.yaml

@@ -152,3 +152,4 @@ ldap:
       ws_esdlconfig: []
       ws_elk: []
       ws_store: []
+      WSDali: []

esp/applications/eclwatch/plugins.yaml

@@ -20,6 +20,7 @@ service_plugins:
   WsTopology: ws_topology
   WsWorkunits: ws_workunits
   ws_codesign: ws_codesign
+  WSDali: ws_dali
 
 binding_plugins:
   FileSpray: ws_fs
@@ -39,4 +40,5 @@ binding_plugins:
   WsSMC: ws_smc
   WSESPControl: wsespcontrol
   ws_codesign: ws_codesign
+  WSDali: ws_dali
 

esp/scm/espscm.cmake

@@ -48,6 +48,7 @@ set ( ESPSCM_SRCS
       ws_store.ecm
       ws_codesign.ecm
       ws_decoupledlogging.ecm
+      ws_dali.ecm
     )
 
 foreach ( loop_var ${ESPSCM_SRCS} )

esp/scm/ws_dali.ecm

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+EspInclude(common);
+
+ESPservice [auth_feature("NONE"), //This declares that the method logic handles feature level authorization
+    version("1.00"), default_client_version("1.00"), exceptions_inline("./smc_xslt/exceptions.xslt")] WSDali
+{
+};
+
+SCMexportdef(WSDali);
+SCMapi(WSDali) IClientWSDali *createWSDaliClient();

esp/services/CMakeLists.txt

@@ -38,6 +38,7 @@ HPCC_ADD_SUBDIRECTORY (espcontrol "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_elk "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_store "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_codesign "PLATFORM")
+HPCC_ADD_SUBDIRECTORY (ws_dali "PLATFORM")
 if (NOT CONTAINERIZED)
   HPCC_ADD_SUBDIRECTORY (ws_config "PLATFORM")
   HPCC_ADD_SUBDIRECTORY (WsDeploy "PLATFORM")

esp/services/ws_dali/CMakeLists.txt

@@ -0,0 +1,65 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# Component: ws_dali
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for ws_dali
+#####################################################
+
+project( ws_dali )
+
+include(${HPCC_SOURCE_DIR}/esp/scm/espscm.cmake)
+
+set (    SRCS
+         ${ESPSCM_GENERATED_DIR}/ws_dali_esp.cpp
+         ws_daliplugin.cpp
+         ws_daliservice.cpp
+    )
+
+include_directories (
+         ${HPCC_SOURCE_DIR}/esp/platform
+         ${HPCC_SOURCE_DIR}/esp/bindings
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
+         ${HPCC_SOURCE_DIR}/esp/clients
+         ${HPCC_SOURCE_DIR}/esp/smc/SMCLib
+         ${HPCC_SOURCE_DIR}/system/include
+         ${HPCC_SOURCE_DIR}/system/jlib
+         ${HPCC_SOURCE_DIR}/system/xmllib
+         ${HPCC_SOURCE_DIR}/system/security/securesocket
+         ${HPCC_SOURCE_DIR}/system/security/LdapSecurity
+         ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/system/mp 
+         ${HPCC_SOURCE_DIR}/common/thorhelper
+         ${HPCC_SOURCE_DIR}/dali/base
+    )
+
+ADD_DEFINITIONS( -D_USRDLL )
+
+HPCC_ADD_LIBRARY( ws_dali SHARED ${SRCS} )
+install ( TARGETS ws_dali RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+target_link_libraries ( ws_dali
+         jlib
+         xmllib
+         esphttp
+    )
+    
+IF (USE_OPENSSL)
+    target_link_libraries ( ws_dali 
+        securesocket
+    )
+ENDIF()

esp/services/ws_dali/ws_daliplugin.cpp

@@ -0,0 +1,63 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#pragma warning (disable : 4786)
+
+//ESP Bindings
+#include "http/platform/httpprot.hpp"
+
+//ESP Service
+#include "ws_daliservice.hpp"
+
+#include "espplugin.hpp"
+
+extern "C"
+{
+
+//when we aren't loading dynamically
+// Change the function names when we stick with dynamic loading.
+ESP_FACTORY IEspService* esp_service_factory(const char* name, const char* type, IPropertyTree* cfg, const char* process)
+{
+    if (strieq(type, "WSDali"))
+    {
+        CWSDaliEx* service = new CWSDaliEx;
+        service->init(cfg, process, name);
+        return service;
+    }
+    return nullptr;
+}
+
+ESP_FACTORY IEspRpcBinding* esp_binding_factory(const char* name, const char* type, IPropertyTree* cfg, const char* process)
+{
+    //binding names of the form <servicetype>_http are being added so the names can be made more consistent and can therefore be automatically generated
+    //  the name also better reflects that these bindings are for all HTTP based protocols, not just SOAP
+    //  both "SoapBinding" and "_http" names instantiate the same objects.
+    if (strieq(type, "ws_daliSoapBinding") || strieq(type, "WSDali_http"))
+    {
+        CWSDaliSoapBinding* binding = new CWSDaliSoapBindingEx(cfg, name, process);
+        return binding;
+    }
+
+    return nullptr;
+}
+
+ESP_FACTORY IEspProtocol* esp_protocol_factory(const char* name, const char* type, IPropertyTree* cfg, const char* process)
+{
+    return http_protocol_factory(name, type, cfg, process);
+}
+
+};

esp/services/ws_dali/ws_daliservice.cpp

@@ -0,0 +1,98 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifdef _USE_OPENLDAP
+#include "ldapsecurity.ipp"
+#endif
+
+#include "ws_daliservice.hpp"
+#include "jlib.hpp"
+#include "dautils.hpp"
+#include "dasds.hpp"
+
+#define REQPATH_EXPORTSDSDATA "/WSDali/Export"
+
+const char* daliFolder = "tempdalifiles" PATHSEPSTR;
+const unsigned daliFolderLength = strlen(daliFolder);
+
+void CWSDaliEx::init(IPropertyTree* cfg, const char* process, const char* service)
+{
+    espProcess.set(process);
+}
+
+int CWSDaliSoapBindingEx::onGet(CHttpRequest* request, CHttpResponse* response)
+{
+    try
+    {
+#ifdef _USE_OPENLDAP
+        request->queryContext()->ensureSuperUser(ECLWATCH_SUPER_USER_ACCESS_DENIED, "Access denied, administrators only.");
+#endif
+        if (wsdService->isDaliDetached())
+            throw makeStringException(ECLWATCH_CANNOT_CONNECT_DALI, "Dali detached.");
+
+        StringBuffer path;
+        request->getPath(path);
+
+        if (!strnicmp(path.str(), REQPATH_EXPORTSDSDATA, sizeof(REQPATH_EXPORTSDSDATA) - 1))
+        {
+            exportSDSData(request, response);
+            return 0;
+        }
+    }
+    catch(IException* e)
+    {
+        onGetException(*request->queryContext(), request, response, *e);
+        FORWARDEXCEPTION(*request->queryContext(), e,  ECLWATCH_INTERNAL_ERROR);
+    }
+
+    return CWSDaliSoapBinding::onGet(request,response);
+}
+
+void CWSDaliSoapBindingEx::exportSDSData(CHttpRequest* request, CHttpResponse* response)
+{
+    StringBuffer path, xpath, safeReq;
+    request->getParameter("Path", path);
+    request->getParameter("Safe", safeReq);
+    Owned<IRemoteConnection> conn = connectXPathOrFile(path, strToBool(safeReq), xpath);
+    if (!conn)
+        throw makeStringException(ECLWATCH_CANNOT_CONNECT_DALI, "Failed to connect Dali.");
+
+    Owned<IPropertyTree> root = conn->getRoot();
+
+    Owned<IFile> workingDir = createIFile(daliFolder);
+    if (!workingDir->exists())
+        workingDir->createDirectory();
+
+    StringBuffer peer, outFileNameWithPath;
+    VStringBuffer prefix("sds_for_%s", request->getPeer(peer).str());
+    Owned<IFileIO> io = createUniqueFile(daliFolder, prefix, nullptr, outFileNameWithPath, IFOcreaterw);
+    { //Force the fios to finish
+        Owned<IFileIOStream> fios = createBufferedIOStream(io);
+        toXML(root, *fios);
+    }
+
+    VStringBuffer headerStr("attachment;filename=%s", outFileNameWithPath.str() + daliFolderLength);
+    IEspContext* context = request->queryContext();
+    context->addCustomerHeader("Content-disposition", headerStr.str());
+
+    response->setContent(createIOStream(io));
+    response->setContentType(HTTP_TYPE_OCTET_STREAM);
+    response->send();
+
+    io.clear();
+    removeFileTraceIfFail(outFileNameWithPath);
+}

esp/services/ws_dali/ws_daliservice.hpp

@@ -0,0 +1,72 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _ESPWIZ_ws_dali_HPP__
+#define _ESPWIZ_ws_dali_HPP__
+
+#include "ws_dali_esp.ipp"
+#include "exception_util.hpp"
+
+class CWSDaliEx : public CWSDali
+{
+    StringAttr espProcess;
+    std::atomic<bool> daliDetached{false};
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    bool isDaliDetached()
+    {
+        return daliDetached;
+    }
+
+    virtual bool attachServiceToDali() override
+    {
+        daliDetached = false;
+        return true;
+    }
+
+    virtual bool detachServiceFromDali() override
+    {
+        daliDetached = true;
+        return true;
+    }
+
+    virtual void init(IPropertyTree* cfg, const char* process, const char* service) override;
+};
+
+class CWSDaliSoapBindingEx : public CWSDaliSoapBinding
+{
+    CWSDaliEx* wsdService = nullptr;
+
+    void exportSDSData(CHttpRequest* request, CHttpResponse* response);
+
+public:
+    CWSDaliSoapBindingEx(http_soap_log_level level = hsl_none) : CWSDaliSoapBinding(level) { }
+    CWSDaliSoapBindingEx(IPropertyTree* cfg, const char* bindName, const char* procName, http_soap_log_level level = hsl_none)
+        : CWSDaliSoapBinding(cfg, bindName, procName, level) { }
+
+    virtual int onGet(CHttpRequest* request, CHttpResponse* response) override;
+
+    virtual void addService(const char* name, const char* host, unsigned short port, IEspService& service) override
+    {
+        wsdService = dynamic_cast<CWSDaliEx*>(&service);
+        CWSDaliSoapBinding::addService(name, host, port, service);
+    }
+};
+
+#endif //_ESPWIZ_ws_dali_HPP__

initfiles/componentfiles/configxml/@temp/esp_service_WsSMC.xsl

@@ -133,6 +133,10 @@ This is required by its binding with ESP service '<xsl:value-of select="$espServ
             <xsl:with-param name="bindingNode" select="$bindingNode"/>
             <xsl:with-param name="authNode" select="$authNode"/>
         </xsl:apply-templates>
+        <xsl:apply-templates select="." mode="ws_dali">
+            <xsl:with-param name="bindingNode" select="$bindingNode"/>
+            <xsl:with-param name="authNode" select="$authNode"/>
+        </xsl:apply-templates>
     </xsl:template>
 
     <!-- WS-SMC -->
@@ -728,6 +732,37 @@ This is required by its binding with ESP service '<xsl:value-of select="$espServ
          <xsl:apply-templates select="@*[string(.) != '']|node()" mode="copy"/>
       </xsl:copy>
    </xsl:template>
+
+    <!-- ws_dali -->
+    <xsl:template match="EspService" mode="ws_dali">
+        <xsl:param name="bindingNode"/>
+        <xsl:param name="authNode"/>
+
+        <xsl:variable name="serviceType" select="'WSDali'"/>
+        <xsl:variable name="serviceName" select="concat($serviceType, '_', @name, '_', $process)"/>
+        <xsl:variable name="bindName" select="concat($serviceType, '_', $bindingNode/@name, '_', $process)"/>
+        <xsl:variable name="bindType" select="'ws_daliSoapBinding'"/>
+        <xsl:variable name="servicePlugin">
+            <xsl:call-template name="defineServicePlugin">
+                <xsl:with-param name="plugin" select="'ws_dali'"/>
+            </xsl:call-template>
+        </xsl:variable>
+        <EspService name="{$serviceName}" type="{$serviceType}" plugin="{$servicePlugin}">
+        </EspService>
+        <EspBinding name="{$bindName}" service="{$serviceName}" protocol="{$bindingNode/@protocol}" type="{$bindType}"
+            plugin="{$servicePlugin}" netAddress="0.0.0.0" port="{$bindingNode/@port}">
+            <xsl:call-template name="bindAuthentication">
+                <xsl:with-param name="bindingNode" select="$bindingNode"/>
+                <xsl:with-param name="authMethod" select="$authNode/@method"/>
+                <xsl:with-param name="service" select="'ws_dali'"/>
+            </xsl:call-template>
+        </EspBinding>
+    </xsl:template>
+   <xsl:template match="*" mode="copy">
+      <xsl:copy>
+         <xsl:apply-templates select="@*[string(.) != '']|node()" mode="copy"/>
+      </xsl:copy>
+   </xsl:template>
    
    <xsl:template match="@*" mode="copy">
       <xsl:if test="string(.) != ''">

system/jlib/jfile.cpp

@@ -5638,7 +5638,7 @@ bool unmountDrive(const char *drv)
 
 }
 
-IFileIO *createUniqueFile(const char *dir, const char *prefix, const char *ext, StringBuffer &filename)
+IFileIO *createUniqueFile(const char *dir, const char *prefix, const char *ext, StringBuffer &filename, IFOmode mode)
 {
     CDateTime dt;
     dt.setNow();
@@ -5662,7 +5662,7 @@ IFileIO *createUniqueFile(const char *dir, const char *prefix, const char *ext,
         OwnedIFile iFile = createIFile(filename.str());
         if (!iFile->exists())
         {
-            try { return iFile->openShared(IFOcreate, IFSHnone); } // NB: could be null if path not found
+            try { return iFile->openShared(mode, IFSHnone); } // NB: could be null if path not found
             catch (IException *e)
             {
                 EXCLOG(e, "createUniqueFile");

system/jlib/jfile.hpp

@@ -600,7 +600,7 @@ extern jlib_decl void extractBlobElements(const char * prefix, const RemoteFilen
 extern jlib_decl bool mountDrive(const char *drv,const RemoteFilename &rfn); // linux only currently
 extern jlib_decl bool unmountDrive(const char *drv); // linux only currently
 
-extern jlib_decl IFileIO *createUniqueFile(const char *dir, const char *prefix, const char *ext, StringBuffer &tmpName);
+extern jlib_decl IFileIO *createUniqueFile(const char *dir, const char *prefix, const char *ext, StringBuffer &tmpName, IFOmode mode=IFOcreate);
 
 
 // used by remote copy