HPCC-27004 Split Esp DFS service into it's own service

For redundancy, updates etc., split the ESP DFS service into
it's own service deployment (i.e. out of eclwatch).

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

docs/EN_US/ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml

@@ -72,6 +72,7 @@ esp.eclservices
 esp.eclqueries
 esp.esdl-sandbox
 esp.sql2ecl
+esp.dfs
 roxie.roxie
 thor.thor
 sasha.dfurecovery-archiver
@@ -104,6 +105,7 @@ sasha-dfuwu-archiver-576b978cc7-b47v7         1/1     Running   0          2m6s
 sasha-file-expiry-8496d87879-xct7f            1/1     Running   0          2m6s
 sasha-wu-archiver-5f64594948-xjblh            1/1     Running   0          2m6s
 sql2ecl-5c8c94d55-tj4td                       1/1     Running   0          2m6s
+dfs-4a9f12621-jabc1                           1/1     Running   0          2m6s
 thor-eclagent-6b8f564f9c-qnczz                1/1     Running   0          2m6s
 thor-thoragent-56d788869f-7trxk               1/1     Running   0          2m6s</programlisting></para>
 
@@ -156,7 +158,8 @@ roxie                 LoadBalancer 10.100.134.125  localhost    9876:30480/TCP
 roxie-toposerver      ClusterIP    None            &lt;none&gt;       9004/TCP          2m6s
 sasha-dfuwu-archiver  ClusterIP    10.110.200.110  &lt;none&gt;       8877/TCP          2m6s
 sasha-wu-archiver     ClusterIP    10.111.34.240   &lt;none&gt;       8877/TCP          2m6s
-sql2ecl               LoadBalancer 10.107.177.180  localhost    8510:30054/TCP    2m6s</programlisting></para>
+sql2ecl               LoadBalancer 10.107.177.180  localhost    8510:30054/TCP    2m6s
+dfs                   LoadBalancer 10.100.52.9     localhost    8520:30184/TCP    2m6s</programlisting></para>
 
     <para>Notice the <emphasis role="strong">eclwatch</emphasis> service is
     running on <emphasis role="strong">localhost:8010</emphasis>. Use that

docs/PT_BR/ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml

@@ -74,6 +74,7 @@ esp.eclservices
 esp.eclqueries
 esp.esdl-sandbox
 esp.sql2ecl
+esp.dfs
 roxie.roxie
 thor.thor
 sasha.dfurecovery-archiver
@@ -106,6 +107,7 @@ sasha-dfuwu-archiver-576b978cc7-b47v7         1/1     Running   0          2m6s
 sasha-file-expiry-8496d87879-xct7f            1/1     Running   0          2m6s
 sasha-wu-archiver-5f64594948-xjblh            1/1     Running   0          2m6s
 sql2ecl-5c8c94d55-tj4td                       1/1     Running   0          2m6s
+dfs-4a9f12621-jabc1                           1/1     Running   0          2m6s
 thor-eclagent-6b8f564f9c-qnczz                1/1     Running   0          2m6s
 thor-thoragent-56d788869f-7trxk               1/1     Running   0          2m6s</programlisting></para>
 
@@ -158,7 +160,8 @@ roxie                 LoadBalancer 10.100.134.125  localhost    9876:30480/TCP
 roxie-toposerver      ClusterIP    None            &lt;none&gt;       9004/TCP          2m6s
 sasha-dfuwu-archiver  ClusterIP    10.110.200.110  &lt;none&gt;       8877/TCP          2m6s
 sasha-wu-archiver     ClusterIP    10.111.34.240   &lt;none&gt;       8877/TCP          2m6s
-sql2ecl               LoadBalancer 10.107.177.180  localhost    8510:30054/TCP    2m6s</programlisting></para>
+sql2ecl               LoadBalancer 10.107.177.180  localhost    8510:30054/TCP    2m6s
+dfs                   LoadBalancer 10.100.52.9     localhost    8520:30184/TCP    2m6s</programlisting></para>
 
     <para>Observe que o serviço <emphasis role="strong">eclwatch</emphasis>
     está sendo executado em <emphasis role="strong">localhost:8010</emphasis>.

esp/applications/CMakeLists.txt

@@ -21,3 +21,4 @@ HPCC_ADD_SUBDIRECTORY (esdl)
 HPCC_ADD_SUBDIRECTORY (esdl-sandbox)
 HPCC_ADD_SUBDIRECTORY (loggingservice)
 HPCC_ADD_SUBDIRECTORY (sql2ecl)
+HPCC_ADD_SUBDIRECTORY (dfs)

esp/applications/dfs/CMakeLists.txt

@@ -0,0 +1,26 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2022 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+set ( ESP_APPLICATION_FILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/esp.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/application.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/ldap_authorization_map.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/plugins.yaml
+)
+
+FOREACH( iFile ${ESP_APPLICATION_FILES} )
+    Install( FILES ${iFile} DESTINATION componentfiles/applications/dfs COMPONENT Runtime )
+ENDFOREACH ( iFile )

esp/applications/dfs/application.yaml

@@ -0,0 +1,4 @@
+application:
+  name: dfs
+  services:
+  - ws_dfsservice

esp/applications/dfs/esp.yaml

@@ -0,0 +1,27 @@
+esp:
+   instance: dfs
+   description: DFS Service
+   daliServers: dali
+   loadDaliBindings: false
+   auth: ldap
+   tls: true
+   service:
+     port: 8880
+   enableSEHMapping: true
+   httpConfigAccess: true
+   logLevel: 1
+   maxBacklogQueueSize: 200
+   portalurl: http://hpccsystems.com/download
+   logDir: "-"
+
+   tls_config:
+     certificate: /opt/HPCCSystems/secrets/certificates/public/tls.crt
+     privatekey: /opt/HPCCSystems/secrets/certificates/public/tls.key
+     cipherList: "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5"
+     verify:
+       enable: false
+       address_match: false
+       accept_selfsigned: false
+       trusted_peers: [ anyone ]
+       ca_certificates:
+       - path: "/opt/HPCCSystems/secrets/certificates/public/ca.crt"

esp/applications/dfs/ldap_authorization_map.yaml

@@ -0,0 +1,11 @@
+ldap:
+  root_access:
+    resource: WsDfsAccess
+    required: Read
+    description: Base access to EclWatch
+  resource_map:
+    ws_dfsservice:
+      Feature:
+      - path: DfsAccess
+        resource: DfsAccess
+        description: Base access to DFS services

esp/applications/dfs/plugins.yaml

@@ -0,0 +1,9 @@
+protocol_plugins:
+  http_protocol: esphttp
+  secure_http_protocol: esphttp
+
+service_plugins:
+  ws_dfsservice: ws_dfsservice
+
+binding_plugins:
+  ws_dfsservice: ws_dfsservice

esp/applications/eclservices/application.yaml

@@ -5,7 +5,6 @@ application:
    - WsWorkunits
    - WsTopology
    - WsDfu
-   - ws_dfsservice
    - WsDfuXRef
    - WsFileIO
    - WsPackageProcess

esp/applications/eclservices/eclservices.yaml

@@ -36,7 +36,6 @@ eclservices:
         xslt:
         - name: def_file
           ^: "./smc_xslt/def_file.xslt"
-   ws_dfsservice:
    WsDfuXRef:
       ViewTimeout: 1000
       LayoutProgram: dot/dot -Tsvg -Gordering=out

esp/applications/eclservices/ldap_authorization_map.yaml

@@ -105,11 +105,6 @@ ldap:
          -  path: DfuAccess
             resource: DfuAccess
             description: Access to DFU
-      ws_dfsservice:
-         Feature:
-         -  path: DfsAccess
-            resource: DfsAccess
-            description: Access to DFS
       WsDfuXRef:
          Feature:
          -  path: DfuXrefAccess

esp/applications/eclservices/plugins.yaml

@@ -7,7 +7,6 @@ service_plugins:
   ws_access: ws_access
   ws_account: ws_account
   WsDfu: ws_dfu
-  ws_dfsservice: ws_dfsservice
   WsDfuXRef: ws_dfu
   ws_elk: ws_elk
   ws_esdlconfig: ws_esdlconfig
@@ -26,7 +25,6 @@ binding_plugins:
   ws_access: ws_access
   ws_account: ws_account
   WsDfu: ws_dfu
-  ws_dfsservice: ws_dfsservice
   WsDfuXRef: ws_dfu
   ws_elk: ws_elk
   ws_esdlconfig: ws_esdlconfig

esp/applications/eclwatch/application.yaml

@@ -5,7 +5,6 @@ application:
    - WsWorkunits
    - WsTopology
    - WsDfu
-   - ws_dfsservice
    - WsDfuXRef
    - WsFileIO
    - WsPackageProcess

esp/applications/eclwatch/eclwatch.yaml

@@ -34,7 +34,6 @@ eclwatch:
         xslt:
         - name: def_file
           ^: "./smc_xslt/def_file.xslt"
-   ws_dfsservice:
    WsDfuXRef:
       ViewTimeout: 1000
       LayoutProgram: dot/dot -Tsvg -Gordering=out

esp/applications/eclwatch/ldap_authorization_map.yaml

@@ -106,11 +106,6 @@ ldap:
          -  path: DfuAccess
             resource: DfuAccess
             description: Access to DFU
-      ws_dfsservice:
-         Feature:
-         -  path: DfsAccess
-            resource: DfsAccess
-            description: Access to DFS
       WsDfuXRef:
          Feature:
          -  path: DfuXrefAccess

esp/applications/eclwatch/plugins.yaml

@@ -7,7 +7,6 @@ service_plugins:
   ws_access: ws_access
   ws_account: ws_account
   WsDfu: ws_dfu
-  ws_dfsservice: ws_dfsservice
   WsDfuXRef: ws_dfu
   ws_ecl: ws_ecl
   ws_elk: ws_elk
@@ -29,7 +28,6 @@ binding_plugins:
   ws_access: ws_access
   ws_account: ws_account
   WsDfu: ws_dfu
-  ws_dfsservice: ws_dfsservice
   WsDfuXRef: ws_dfu
   ws_ecl: ws_ecl
   ws_elk: ws_elk

esp/clients/ws_dfsclient/ws_dfsclient.cpp

@@ -630,18 +630,18 @@ IDFSFile *lookupDFSFile(const char *logicalName, unsigned timeoutSecs, unsigned
     {
 #ifdef _CONTAINERIZED
         // NB: only expected to be here if experimental option #option('dfsesp-localfiles', true); is in use.
-        // This finds and uses local eclwatch service for local read lookukup.
-        Owned<IPropertyTreeIterator> eclWatchServices = getGlobalConfigSP()->getElements("services[@type='eclwatch']");
+        // This finds and uses local dfs service for local read lookukup.
+        Owned<IPropertyTreeIterator> eclWatchServices = getGlobalConfigSP()->getElements("services[@type='dfs']");
         if (!eclWatchServices->first())
-            throw makeStringException(-1, "No eclwatch service not defined");
+            throw makeStringException(-1, "Dfs service not defined in esp services");
         const IPropertyTree &eclWatch = eclWatchServices->query();
         StringBuffer eclWatchName;
         eclWatch.getProp("@name", eclWatchName);
         auto result = getExternalService(eclWatchName);
         if (result.first.empty())
-            throw makeStringExceptionV(-1, "eclwatch '%s': service not found", eclWatchName.str());
+            throw makeStringExceptionV(-1, "dfs '%s': service not found", eclWatchName.str());
         if (0 == result.second)
-            throw makeStringExceptionV(-1, "eclwatch '%s': service port not defined", eclWatchName.str());
+            throw makeStringExceptionV(-1, "dfs '%s': service port not defined", eclWatchName.str());
         const char *protocol = eclWatch.getPropBool("@tls") ? "https" : "http";
         serviceUrl.appendf("%s://%s:%u", protocol, result.first.c_str(), result.second);
 #else

esp/scm/ws_dfs.ecm

@@ -60,5 +60,5 @@ ESPservice [
     ESPmethod [auth_feature("DfsAccess:READ"), min_ver("1.01")] DFSFileLookup(DFSFileLookupRequest, DFSFileLookupResponse);
 };
 
-SCMexportdef(WSDFS);
-SCMapi(WSDFS) IClientWsDfs *createWsDfsClient();
+SCMexportdef(WsDfs);
+SCMapi(WsDfs) IClientWsDfs *createWsDfsClient();

helm/examples/certmanager/README-vault-pki.md

@@ -284,6 +284,7 @@ roxie-agent-local-roxie-agent-1-cert      True    roxie-agent-local-roxie-agent-
 roxie-agent-local-roxie-agent-2-cert      True    roxie-agent-local-roxie-agent-2-tls      85s
 roxie-local-roxie-workunit-cert           True    roxie-local-roxie-workunit-tls           85s
 sql2ecl-public-sql2ecl-cert               True    sql2ecl-public-sql2ecl-tls               85s
+dfs-public-dfs-cert                       True    dfs-public-dfs-tls                       85s
 thoragent-local-thor-thoragent-cert       True    thoragent-local-thor-thoragent-tls       85s
 thormanager-local-thormanager-w-cert      True    thormanager-local-thormanager-w-tls      85s
 thorworker-local-thorworker-w-cert        True    thorworker-local-thorworker-w-tls        85s
@@ -331,6 +332,7 @@ roxie-local-roxie-workunit-tls           kubernetes.io/tls                     3
 sh.helm.release.v1.cert-manager.v1       helm.sh/release.v1                    1      3m52s
 sh.helm.release.v1.myhpcc.v1             helm.sh/release.v1                    1      2m58s
 sql2ecl-public-sql2ecl-tls               kubernetes.io/tls                     3      2m55s
+dfs-public-dfs-tls                       kubernetes.io/tls                     3      2m55s
 thoragent-local-thor-thoragent-tls       kubernetes.io/tls                     3      2m52s
 thormanager-local-thormanager-w-tls      kubernetes.io/tls                     3      2m51s
 thorworker-local-thorworker-w-tls        kubernetes.io/tls                     3      2m51s

helm/examples/certmanager/README.md

@@ -121,6 +121,7 @@ roxie-agent-local-roxie-agent-1-cert      True    roxie-agent-local-roxie-agent-
 roxie-agent-local-roxie-agent-2-cert      True    roxie-agent-local-roxie-agent-2-tls      85s
 roxie-local-roxie-workunit-cert           True    roxie-local-roxie-workunit-tls           85s
 sql2ecl-public-sql2ecl-cert               True    sql2ecl-public-sql2ecl-tls               85s
+dfs-public-dfs-cert                       True    dfs-public-dfs-tls                       85s
 thoragent-local-thor-thoragent-cert       True    thoragent-local-thor-thoragent-tls       85s
 thormanager-local-thormanager-w-cert      True    thormanager-local-thormanager-w-tls      85s
 thorworker-local-thorworker-w-cert        True    thorworker-local-thorworker-w-tls        85s
@@ -168,6 +169,7 @@ roxie-local-roxie-workunit-tls           kubernetes.io/tls                     3
 sh.helm.release.v1.cert-manager.v1       helm.sh/release.v1                    1      3m52s
 sh.helm.release.v1.myhpcc.v1             helm.sh/release.v1                    1      2m58s
 sql2ecl-public-sql2ecl-tls               kubernetes.io/tls                     3      2m55s
+dfs-public-dfs-tls                       kubernetes.io/tls                     3      2m55s
 thoragent-local-thor-thoragent-tls       kubernetes.io/tls                     3      2m52s
 thormanager-local-thormanager-w-tls      kubernetes.io/tls                     3      2m51s
 thorworker-local-thorworker-w-tls        kubernetes.io/tls                     3      2m51s

helm/hpcc/values.yaml

@@ -557,6 +557,17 @@ esp:
   #resources:
   #  cpu: "250m"
   #  memory: "1G"
+- name: dfs
+  application: dfs
+  auth: none
+  replicas: 1
+  service:
+    visibility: local
+    servicePort: 8520
+  #resources:
+  #  cpu: "250m"
+  #  memory: "1G"
+
 
 roxie:
 - name: roxie