HPCC-24742 Kubernetes support for dynamic esdl services

1. Add containerized, k8s, and helm support for esdl services
2. Move ESP application security to a common directory
3. Add ESP application support for SecurityManager plugins

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexisrisk.com>

esp/applications/CMakeLists.txt

@@ -13,7 +13,10 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
+HPCC_ADD_SUBDIRECTORY (common)
 HPCC_ADD_SUBDIRECTORY (eclwatch)
 HPCC_ADD_SUBDIRECTORY (eclservices)
 HPCC_ADD_SUBDIRECTORY (eclqueries)
+HPCC_ADD_SUBDIRECTORY (esdl)
+HPCC_ADD_SUBDIRECTORY (esdl-sandbox)
 HPCC_ADD_SUBDIRECTORY (sql2ecl)

esp/applications/common/CMakeLists.txt

@@ -0,0 +1,25 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+set ( ESP_APPLICATION_FILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/ldap.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/configsec.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/directories.yaml
+)
+
+FOREACH( iFile ${ESP_APPLICATION_FILES} )
+    Install( FILES ${iFile} DESTINATION componentfiles/applications/common COMPONENT Runtime )
+ENDFOREACH ( iFile )

esp/applications/common/configsec.yaml

@@ -0,0 +1,10 @@
+authNZ:
+  configsec:
+    SecurityManager:
+      type: AccurintSecurity
+      name: accurintsecmgr
+    buildSet: accurintsecmgr
+    InstanceFactoryName: newLNSecurityManager
+    libName: accurintsecmgr
+    name: accurintsecmgr
+    typeSelector: cfg

esp/applications/eclqueries/CMakeLists.txt

@@ -18,10 +18,8 @@ set ( ESP_APPLICATION_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/esp.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/application.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/eclqueries.yaml
-    ${CMAKE_CURRENT_SOURCE_DIR}/ldap.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/ldap_authorization_map.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/plugins.yaml
-    ${CMAKE_CURRENT_SOURCE_DIR}/directories.yaml
 )
 
 FOREACH( iFile ${ESP_APPLICATION_FILES} )

esp/applications/eclqueries/directories.yaml

@@ -1,11 +0,0 @@
-directories:
-  log: "/var/log/[NAME]/[INST]"
-  run: "/var/lib/[NAME]/[INST]"
-  conf: "/etc/[NAME]/[INST]"
-  temp: "/var/lib/[NAME]/[INST]/temp"
-  data: "/var/lib/[NAME]/hpcc-data/[COMPONENT]"
-  data2: "/var/lib/[NAME]/hpcc-data2/[COMPONENT]"
-  data3: "/var/lib/[NAME]/hpcc-data3/[COMPONENT]"
-  mirror: "/var/lib/[NAME]/hpcc-mirror/[COMPONENT]"
-  query: "/var/lib/[NAME]/queries/[INST]"
-  lock: "/var/lock/[NAME]/[INST]"

esp/applications/eclqueries/esp.yaml

@@ -2,6 +2,7 @@ esp:
    instance: eclqueries
    description: ECL Queries
    daliServers: dali
+   loadDaliBindings: false
    auth: ldap
    tls: true
    port: 8880

esp/applications/eclservices/CMakeLists.txt

@@ -18,10 +18,8 @@ set ( ESP_APPLICATION_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/esp.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/application.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/eclservices.yaml
-    ${CMAKE_CURRENT_SOURCE_DIR}/ldap.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/ldap_authorization_map.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/plugins.yaml
-    ${CMAKE_CURRENT_SOURCE_DIR}/directories.yaml
 )
 
 FOREACH( iFile ${ESP_APPLICATION_FILES} )

esp/applications/eclservices/directories.yaml

@@ -1,11 +0,0 @@
-directories:
-  log: "/var/log/[NAME]/[INST]"
-  run: "/var/lib/[NAME]/[INST]"
-  conf: "/etc/[NAME]/[INST]"
-  temp: "/var/lib/[NAME]/[INST]/temp"
-  data: "/var/lib/[NAME]/hpcc-data/[COMPONENT]"
-  data2: "/var/lib/[NAME]/hpcc-data2/[COMPONENT]"
-  data3: "/var/lib/[NAME]/hpcc-data3/[COMPONENT]"
-  mirror: "/var/lib/[NAME]/hpcc-mirror/[COMPONENT]"
-  query: "/var/lib/[NAME]/queries/[INST]"
-  lock: "/var/lock/[NAME]/[INST]"

esp/applications/eclservices/esp.yaml

@@ -2,6 +2,7 @@ esp:
    instance: eclservices
    description: ECL Services
    api_only: true
+   loadDaliBindings: false
    auth: ldap
    tls: true
    port: 8880

esp/applications/eclservices/ldap.yaml

@@ -1,25 +0,0 @@
-ldap:
-   objname: ldapserver
-   description: LDAP server process
-   ldapProtocol: ldaps
-   authMethod: kerberos
-   localDomain: localdomain
-   ldapPort: 389
-   ldapSecurePort: 636
-   systemCommonName: hpcc_admin2
-   systemPassword: ""
-   systemUser: hpcc_admin2
-   adminGroupName: HPCCAdmin
-   maxConnections: 10
-   passwordExpirationWarningDays: 10
-   cacheTimeout: 5
-   ldapTimeoutSecs: 131
-   sharedCache: true
-   checkViewPermissions: ''
-   filesBasedn: ou=files,ou=ecl
-   groupsBasedn: ou=groups,ou=ecl
-   sudoersBasedn: ou=SUDOers
-   systemBasedn: cn=Users
-   usersBasedn: ou=users,ou=ecl
-   resourcesBasedn: ou=EclWatch,ou=EspServices,ou=ecl
-   workunitsBasedn: ou=workunits,ou=ecl

esp/applications/eclwatch/CMakeLists.txt

@@ -18,10 +18,8 @@ set ( ESP_APPLICATION_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/esp.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/application.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/eclwatch.yaml
-    ${CMAKE_CURRENT_SOURCE_DIR}/ldap.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/ldap_authorization_map.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/plugins.yaml
-    ${CMAKE_CURRENT_SOURCE_DIR}/directories.yaml
 )
 
 FOREACH( iFile ${ESP_APPLICATION_FILES} )

esp/applications/eclwatch/ldap.yaml

@@ -1,25 +0,0 @@
-ldap:
-   objname: ldapserver
-   description: LDAP server process
-   ldapProtocol: ldaps
-   authMethod: kerberos
-   localDomain: localdomain
-   ldapPort: 389
-   ldapSecurePort: 636
-   systemCommonName: hpcc_admin2
-   systemPassword: ""
-   systemUser: hpcc_admin2
-   adminGroupName: HPCCAdmin
-   maxConnections: 10
-   passwordExpirationWarningDays: 10
-   cacheTimeout: 5
-   ldapTimeoutSecs: 131
-   sharedCache: true
-   checkViewPermissions: ''
-   filesBasedn: ou=files,ou=ecl
-   groupsBasedn: ou=groups,ou=ecl
-   sudoersBasedn: ou=SUDOers
-   systemBasedn: cn=Users
-   usersBasedn: ou=users,ou=ecl
-   resourcesBasedn: ou=EclWatch,ou=EspServices,ou=ecl
-   workunitsBasedn: ou=workunits,ou=ecl

esp/applications/esdl-sandbox/CMakeLists.txt

@@ -0,0 +1,27 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+set ( ESP_APPLICATION_FILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/esp.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/application.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/esdl-sandbox.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/ldap_authorization_map.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/plugins.yaml
+)
+
+FOREACH( iFile ${ESP_APPLICATION_FILES} )
+    Install( FILES ${iFile} DESTINATION componentfiles/applications/esdl-sandbox COMPONENT Runtime )
+ENDFOREACH ( iFile )

esp/applications/esdl-sandbox/application.yaml

@@ -0,0 +1,4 @@
+application:
+  name: esdl-sandbox
+  services:
+   - esdl_svc_engine

esp/applications/esdl-sandbox/esdl-sandbox.yaml

@@ -0,0 +1,2 @@
+esdl-sandbox:
+   esdl_svc_engine: []

esp/applications/esdl-sandbox/esp.yaml

@@ -0,0 +1,28 @@
+esp:
+   instance: esdl-sandbox
+   description: ESDL Sandbox
+   daliServers: mydali
+   loadDaliBindings: true
+   auth: ldap
+   tls: true
+   port: 8880
+   enableSEHMapping: true
+   httpConfigAccess: true
+   logLevel: 1
+   maxBacklogQueueSize: 200
+   portalurl: http://hpccsystems.com/download
+   logDir: "-"
+
+   tls_config:
+     certificate: /etc/HPCCSystems/certificates/{$instance}/server.crt
+     privatekey: /etc/HPCCSystems/certificates/{$instance}/private.key
+     passphrase: JHjdi6DDptMwYmJNTZsqjg==
+     cipherList:
+       verify:
+          enable: false
+          address_match: false
+          accept_selfsigned: true
+          ca_certificates:
+            - path: "ca.pem"
+          trusted_peers:
+            - anyone

esp/applications/esdl-sandbox/ldap_authorization_map.yaml

@@ -0,0 +1,11 @@
+ldap:
+   root_access:
+      resource: EsdlAccess
+      required: Read
+      description: Base access to ESDL Services
+   resource_map:
+      ws_ecl:
+         Feature:
+         -  path: EsdlAccess
+            resource: EsdlAccess
+            description: Base access to ESDL services

esp/applications/esdl-sandbox/plugins.yaml

@@ -0,0 +1,9 @@
+protocol_plugins:
+  http_protocol: esphttp
+  secure_http_protocol: esphttp
+
+service_plugins:
+  esdl_svc_engine: esdl_svc_engine
+
+binding_plugins:
+  esdl_svc_engine: esdl_svc_engine

esp/applications/esdl/CMakeLists.txt

@@ -0,0 +1,27 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+set ( ESP_APPLICATION_FILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/esp.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/application.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/configsec_authorization_map.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/ldap_authorization_map.yaml
+    ${CMAKE_CURRENT_SOURCE_DIR}/plugins.yaml
+)
+
+FOREACH( iFile ${ESP_APPLICATION_FILES} )
+    Install( FILES ${iFile} DESTINATION componentfiles/applications/esdl COMPONENT Runtime )
+ENDFOREACH ( iFile )

esp/applications/esdl/application.yaml

@@ -0,0 +1,4 @@
+application:
+  name: esdl
+  services:
+   - esdl

esp/applications/esdl/configsec_authorization_map.yaml

@@ -0,0 +1,6 @@
+authNZ:
+  configsec:
+   root_access:
+      resource: EsdlAccess
+      required: Read
+      description: Base access to ESDL Services

esp/applications/esdl/esp.yaml

@@ -0,0 +1,29 @@
+esp:
+   instance: esdl-integration
+   description: ESDL integration
+   daliServers: mydali
+   auth: ldap
+   tls: true
+   port: 8880
+   bindings: "/opt/HPCCSystems/bindings"
+   loadDaliBindings: false
+   enableSEHMapping: true
+   httpConfigAccess: true
+   logLevel: 1
+   maxBacklogQueueSize: 200
+   portalurl: http://hpccsystems.com/download
+   logDir: "-"
+
+   tls_config:
+     certificate: /etc/HPCCSystems/certificates/{$instance}/server.crt
+     privatekey: /etc/HPCCSystems/certificates/{$instance}/private.key
+     passphrase: JHjdi6DDptMwYmJNTZsqjg==
+     cipherList:
+       verify:
+          enable: false
+          address_match: false
+          accept_selfsigned: true
+          ca_certificates:
+            - path: "ca.pem"
+          trusted_peers:
+            - anyone

esp/applications/esdl/ldap_authorization_map.yaml

@@ -0,0 +1,5 @@
+ldap:
+   root_access:
+      resource: EsdlAccess
+      required: Read
+      description: Base access to ESDL Services

esp/applications/esdl/plugins.yaml

@@ -0,0 +1,9 @@
+protocol_plugins:
+  http_protocol: esphttp
+  secure_http_protocol: esphttp
+
+service_plugins:
+  esdl: esdl_svc_engine
+
+binding_plugins:
+  esdl: esdl_svc_engine

esp/applications/sql2ecl/CMakeLists.txt

@@ -17,10 +17,8 @@
 set ( ESP_APPLICATION_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/esp.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/application.yaml
-    ${CMAKE_CURRENT_SOURCE_DIR}/ldap.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/ldap_authorization_map.yaml
     ${CMAKE_CURRENT_SOURCE_DIR}/plugins.yaml
-    ${CMAKE_CURRENT_SOURCE_DIR}/directories.yaml
 )
 
 FOREACH( iFile ${ESP_APPLICATION_FILES} )

esp/applications/sql2ecl/directories.yaml

@@ -1,11 +0,0 @@
-directories:
-  log: "/var/log/[NAME]/[INST]"
-  run: "/var/lib/[NAME]/[INST]"
-  conf: "/etc/[NAME]/[INST]"
-  temp: "/var/lib/[NAME]/[INST]/temp"
-  data: "/var/lib/[NAME]/hpcc-data/[COMPONENT]"
-  data2: "/var/lib/[NAME]/hpcc-data2/[COMPONENT]"
-  data3: "/var/lib/[NAME]/hpcc-data3/[COMPONENT]"
-  mirror: "/var/lib/[NAME]/hpcc-mirror/[COMPONENT]"
-  query: "/var/lib/[NAME]/queries/[INST]"
-  lock: "/var/lock/[NAME]/[INST]"

esp/applications/sql2ecl/esp.yaml

@@ -2,6 +2,7 @@ esp:
    instance: sql2ecl
    description: SQL2ECL Service
    daliServers: dali
+   loadDaliBindings: false
    auth: ldap
    tls: true
    port: 8880

esp/applications/sql2ecl/ldap.yaml

@@ -1,25 +0,0 @@
-ldap:
-   objname: ldapserver
-   description: LDAP server process
-   ldapProtocol: ldaps
-   authMethod: kerberos
-   localDomain: localdomain
-   ldapPort: 389
-   ldapSecurePort: 636
-   systemCommonName: hpcc_admin2
-   systemPassword: ""
-   systemUser: hpcc_admin2
-   adminGroupName: HPCCAdmin
-   maxConnections: 10
-   passwordExpirationWarningDays: 10
-   cacheTimeout: 5
-   ldapTimeoutSecs: 131
-   sharedCache: true
-   checkViewPermissions: ''
-   filesBasedn: ou=files,ou=ecl
-   groupsBasedn: ou=groups,ou=ecl
-   sudoersBasedn: ou=SUDOers
-   systemBasedn: cn=Users
-   usersBasedn: ou=users,ou=ecl
-   resourcesBasedn: ou=WsEcl,ou=EspServices,ou=ecl
-   workunitsBasedn: ou=workunits,ou=ecl

esp/platform/application_config.cpp

@@ -40,17 +40,27 @@ static void appendPTreeFromYamlFile(IPropertyTree *tree, const char *file)
 
 IPropertyTree *loadApplicationConfig(const char *application, const char* argv[])
 {
+    Owned<IPropertyTree> applicationConfig = createPTree(application);
+    IPropertyTree *defaultConfig = applicationConfig->addPropTree("esp");
+
     char sepchar = getPathSepChar(COMPONENTFILES_DIR);
+
     StringBuffer path(COMPONENTFILES_DIR);
+    addPathSepChar(path, sepchar).append("applications").append(sepchar).append("common").append(sepchar);
+    if (checkDirExists(path))
+    {
+        Owned<IDirectoryIterator> common_dir = createDirectoryIterator(path, "*.yaml", false, false);
+        ForEach(*common_dir)
+            appendPTreeFromYamlFile(defaultConfig, common_dir->query().queryFilename());
+    }
+
+    path.set(COMPONENTFILES_DIR);
     addPathSepChar(path, sepchar).append("applications").append(sepchar).append(application).append(sepchar);
     if (!checkDirExists(path))
         throw MakeStringException(-1, "Can't find esp application %s (dir %s)", application, path.str());
-
-    Owned<IPropertyTree> applicationConfig = createPTree(application);
-    IPropertyTree *defaultConfig = applicationConfig->addPropTree("esp");
-    Owned<IDirectoryIterator> dir = createDirectoryIterator(path, "*.yaml", false, false);
-    ForEach(*dir)
-        appendPTreeFromYamlFile(defaultConfig, dir->query().queryFilename());
+    Owned<IDirectoryIterator> application_dir = createDirectoryIterator(path, "*.yaml", false, false);
+    ForEach(*application_dir)
+        appendPTreeFromYamlFile(defaultConfig, application_dir->query().queryFilename());
 
     //apply provided config to the application
     Owned<IPropertyTree> config = loadConfiguration(defaultConfig, argv, "esp", "ESP", nullptr, nullptr);
@@ -78,14 +88,8 @@ static void copyDirectories(IPropertyTree *target, IPropertyTree *src)
     }
 }
 
-//returns true if ldap is enabled.
-//  only support ldap or "none" for now.
-//  "none" must be explicit, don't want to accidentally turn off security
-bool addSecurity(IPropertyTree *legacyEsp, IPropertyTree *appEsp, StringBuffer &bindAuth)
+bool addLdapSecurity(IPropertyTree *legacyEsp, IPropertyTree *appEsp, StringBuffer &bindAuth)
 {
-    const char *auth = appEsp->queryProp("@auth");
-    if (auth && streq(auth, "none"))
-        return false;
     const char *ldapAddress = appEsp->queryProp("@ldapAddress");
     if (isEmptyString(ldapAddress))
         throw MakeStringException(-1, "LDAP not configured.  To run without security set auth=none");
@@ -107,16 +111,68 @@ bool addSecurity(IPropertyTree *legacyEsp, IPropertyTree *appEsp, StringBuffer &
 
     VStringBuffer authenticationXml("<Authentication htpasswdFile='/etc/HPCCSystems/.htpasswd' ldapAuthMethod='kerberos' ldapConnections='10' ldapServer='%s' method='ldaps' passwordExpirationWarningDays='10'/>", configname.str());
     legacyEsp->addPropTree("Authentication", createPTreeFromXMLString(authenticationXml));
+    return true;
+}
 
+bool addAuthNZSecurity(const char *name, IPropertyTree *legacyEsp, IPropertyTree *appEsp, StringBuffer &bindAuth)
+{
+    IPropertyTree *authNZ = appEsp->queryPropTree("authNZ");
+    if (!authNZ)
+        throw MakeStringException(-1, "can't find application AuthNZ section.  To run without security set auth=none");
+    authNZ = authNZ->queryPropTree(name);
+    if (!authNZ)
+        throw MakeStringException(-1, "can't find application %s AuthNZ settings.  To run without security set auth=none", name);
+    IPropertyTree *appSecMgr = authNZ->queryPropTree("SecurityManager");
+    if (!appSecMgr)
+        throw MakeStringException(-1, "can't find SecurityManager settings.  To run without security set auth=none");
+    const char *method = appSecMgr->queryProp("@name");
+    const char *tag = appSecMgr->queryProp("@type");
+    if (isEmptyString(tag))
+        throw MakeStringException(-1, "SecurityManager type attribute required.  To run without security set auth=none");
+
+    legacyEsp->addPropTree("AuthDomains", createPTreeFromXMLString("<AuthDomains><AuthDomain authType='AuthPerRequestOnly' clientSessionTimeoutMinutes='120' domainName='default' invalidURLsAfterAuth='/esp/login' loginLogoURL='/esp/files/eclwatch/img/Loginlogo.png' logonURL='/esp/files/Login.html' logoutURL='' serverSessionTimeoutMinutes='240' unrestrictedResources='/favicon.ico,/esp/files/*,/esp/xslt/*'/></AuthDomains>"));
+
+    IPropertyTree *legacy = legacyEsp->addPropTree("SecurityManagers");
+    legacy = legacy->addPropTree("SecurityManager");
+    copyAttributes(legacy, appSecMgr);
+    legacy = legacy->addPropTree(tag);
+    mergePTree(legacy, authNZ); //extra info clean up later
+    legacy->removeProp("SecurityManager"); //already copied these attributes above, don't need this as a child
+
+    bindAuth.setf("<Authenticate method='%s'/>", method ? method : "unknown");
     return true;
 }
 
-void bindAuthResources(IPropertyTree *legacyAuthenticate, IPropertyTree *app, const char *service)
+//  auth "none" must be explicit, default to secure mode, don't want to accidentally turn off security
+bool addSecurity(IPropertyTree *legacyEsp, IPropertyTree *appEsp, StringBuffer &bindAuth)
 {
-    IPropertyTree *appLdap = app->queryPropTree("ldap");
-    if (!appLdap)
-        throw MakeStringException(-1, "can't find application LDAP settings.  To run without security set auth=none");
-    IPropertyTree *root_access = appLdap->queryPropTree("root_access");
+    const char *auth = appEsp->queryProp("@auth");
+    if (isEmptyString(auth) || streq(auth, "none"))
+        return false;
+    if (streq(auth, "ldap"))
+        return addLdapSecurity(legacyEsp, appEsp, bindAuth);
+    return addAuthNZSecurity(auth, legacyEsp, appEsp, bindAuth);
+}
+
+void bindAuthResources(IPropertyTree *legacyAuthenticate, IPropertyTree *app, const char *service, const char *auth)
+{
+    IPropertyTree *appAuth = nullptr;
+    if (isEmptyString(auth) || streq(auth, "ldap"))
+        appAuth = app->queryPropTree("ldap");
+    else if (streq(auth, "none"))
+        return;
+    else
+    {
+        appAuth = app->queryPropTree("authNZ");
+        if (!appAuth)
+            return;
+        appAuth = appAuth->queryPropTree(auth);
+        if (!appAuth)
+            return;
+    }
+    if (!appAuth)
+        throw MakeStringException(-1, "can't find application Auth settings.  To run without security set auth=none");
+    IPropertyTree *root_access = appAuth->queryPropTree("root_access");
     StringAttr required(root_access->queryProp("@required"));
     StringAttr description(root_access->queryProp("@description"));
     StringAttr resource(root_access->queryProp("@resource"));
@@ -124,12 +180,12 @@ void bindAuthResources(IPropertyTree *legacyAuthenticate, IPropertyTree *app, co
     legacyAuthenticate->addPropTree("Location", createPTreeFromXMLString(locationXml));
 
     VStringBuffer featuresPath("resource_map/%s/Feature", service);
-    Owned<IPropertyTreeIterator> features = appLdap->getElements(featuresPath);
+    Owned<IPropertyTreeIterator> features = appAuth->getElements(featuresPath);
     ForEach(*features)
         legacyAuthenticate->addPropTree("Feature", LINK(&features->query()));
 }
 
-void bindService(IPropertyTree *legacyEsp, IPropertyTree *app, const char *service, const char *protocol, const char *netAddress, unsigned port, const char *auth, int seq)
+void bindService(IPropertyTree *legacyEsp, IPropertyTree *app, const char *service, const char *protocol, const char *netAddress, unsigned port, const char *bindAuth, int seq)
 {
     VStringBuffer xpath("binding_plugins/@%s", service);
     const char *binding_plugin = app->queryProp(xpath);
@@ -139,11 +195,12 @@ void bindService(IPropertyTree *legacyEsp, IPropertyTree *app, const char *servi
     if (seq==0)
         bindingEntry->setProp("@defaultBinding", "true");
 
-    if (!isEmptyString(auth))
+    if (!isEmptyString(bindAuth))
     {
-        IPropertyTree *authenticate = bindingEntry->addPropTree("Authenticate", createPTreeFromXMLString(auth));
+        const char *auth = app->queryProp("@auth");
+        IPropertyTree *authenticate = bindingEntry->addPropTree("Authenticate", createPTreeFromXMLString(bindAuth));
         if (authenticate)
-            bindAuthResources(authenticate, app, service);
+            bindAuthResources(authenticate, app, service, auth);
     }
 }
 
@@ -166,7 +223,7 @@ static void mergeServicePTree(IPropertyTree *target, IPropertyTree *toMerge)
     }
 }
 
-void addService(IPropertyTree *legacyEsp, IPropertyTree *app, const char *application, const char *service, const char *protocol, const char *netAddress, unsigned port, const char *auth, int seq)
+void addService(IPropertyTree *legacyEsp, IPropertyTree *app, const char *application, const char *service, const char *protocol, const char *netAddress, unsigned port, const char *bindAuth, int seq)
 {
     VStringBuffer plugin_xpath("service_plugins/@%s", service);
     const char *service_plugin = app->queryProp(plugin_xpath);
@@ -179,7 +236,7 @@ void addService(IPropertyTree *legacyEsp, IPropertyTree *app, const char *applic
     if (serviceConfig && serviceEntry)
         mergeServicePTree(serviceEntry, serviceConfig);
 
-    bindService(legacyEsp, app, service, protocol, netAddress, port, auth, seq);
+    bindService(legacyEsp, app, service, protocol, netAddress, port, bindAuth, seq);
 }
 
 bool addProtocol(IPropertyTree *legacyEsp, IPropertyTree *app)
@@ -249,9 +306,9 @@ IPropertyTree *buildApplicationLegacyConfig(const char *application, const char*
 
     bool tls = addProtocol(legacyEsp, appEspConfig);
 
-    StringBuffer auth;
-    addSecurity(legacyEsp, appEspConfig, auth);
-    addServices(legacyEsp, appEspConfig, application, auth, tls);
+    StringBuffer bindAuth;
+    addSecurity(legacyEsp, appEspConfig, bindAuth);
+    addServices(legacyEsp, appEspConfig, application, bindAuth, tls);
 
     IPropertyTree *legacyDirectories = legacy->queryPropTree("Software/Directories");
     IPropertyTree *appDirectories = appEspConfig->queryPropTree("directories");

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -1488,7 +1488,7 @@ EsdlBindingImpl::EsdlBindingImpl()
         m_isAttached = true;
 }
 
-EsdlBindingImpl::EsdlBindingImpl(IPropertyTree* cfg, const char *binding,  const char *process) : CHttpSoapBinding(cfg, binding, process)
+EsdlBindingImpl::EsdlBindingImpl(IPropertyTree* cfg, IPropertyTree* esdlArchive, const char *binding,  const char *process) : CHttpSoapBinding(cfg, binding, process)
 {
     m_pCentralStore.setown(createEsdlCentralStore());
     m_bindingName.set(binding);
@@ -1515,7 +1515,10 @@ EsdlBindingImpl::EsdlBindingImpl(IPropertyTree* cfg, const char *binding,  const
 
     try
     {
-        m_esdlBndCfg.setown(fetchESDLBinding(process, binding, m_esdlStateFilesLocation));
+        if (esdlArchive)
+            m_esdlBndCfg.set(esdlArchive->queryPropTree("Binding"));
+        else
+            m_esdlBndCfg.setown(fetchESDLBinding(process, binding, m_esdlStateFilesLocation));
 
         if (!m_esdlBndCfg.get())
             DBGLOG("ESDL Binding: Could not fetch ESDL binding %s for ESP Process %s", binding, process);
@@ -1571,6 +1574,67 @@ IPropertyTree* EsdlBindingImpl::fetchESDLBinding(const char *process, const char
     }
 }
 
+bool EsdlBindingImpl::loadLocalDefinitions(IPropertyTree *esdlArchive, const char * espServiceName, Owned<IEsdlDefinition>& esdl, IPropertyTree * esdl_binding, StringBuffer & loadedServiceName)
+{
+    if (!esdl || !esdl_binding || !esdlArchive)
+        return false;
+
+    //Loading first ESDL definition encountered, informed that espServiceName is to be treated as arbitrary
+    IPropertyTree * esdl_binding_definition = esdl_binding->queryPropTree("Definition[1]");
+
+    if (esdl_binding_definition)
+    {
+        try
+        {
+            const char * id = esdl_binding_definition->queryProp("@id");
+            PROGLOG("Loading esdl definition for ID %s", id);
+            loadedServiceName.set(esdl_binding_definition->queryProp("@esdlservice"));
+            IEsdlShare* esdlshare = queryEsdlShare();
+            Linked<IEsdlDefinition> shareddef = esdlshare->lookup(id);
+            if (shareddef)
+            {
+                PROGLOG("Found esdl definition %s in shared cache, use it directly.", id);
+                esdl.set(shareddef);
+                return true;
+            }
+            else
+                PROGLOG("Esdl definition %s not found in shared cache, loading it from archive", id);
+
+            StringBuffer esdlXML;
+            if (esdlArchive)
+            {
+                esdlXML.set(esdlArchive->queryProp("Definitions"));
+                if (esdlXML.isEmpty())
+                    throw MakeStringException(-1, "Could not load ESDL definition: '%s' assigned to esp service name '%s'", id, espServiceName);
+            }
+
+#ifdef _DEBUG
+            DBGLOG("\nESDL Definition to be loaded:\n%s", esdlXML.str());
+#endif
+            esdl->addDefinitionFromXML(esdlXML, id);
+
+            if (strcmp(loadedServiceName.str(), espServiceName)!=0)
+                DBGLOG("ESDL Binding: ESP service %s now based off of ESDL Service def: %s", espServiceName, loadedServiceName.str());
+
+            esdlshare->add(id, esdl.get());
+        }
+        catch (IException * e)
+        {
+            StringBuffer msg;
+            e->errorMessage(msg);
+            DBGLOG("Error while loading ESDL definitions: %s", msg.str());
+            e->Release();
+        }
+    }
+    else
+    {
+        DBGLOG("ESDL Binding: Could not find any ESDL definition while loading ESDL Binding: %s", esdl_binding->queryProp("@id"));
+        return false;
+    }
+
+    return true;
+}
+
 /* if the target ESDL binding contains an ESDL service definition matching this espServiceName, load it.
  * Otherwise, load the first definition available, and report it via the loadedServiceName
  */
@@ -1807,7 +1871,7 @@ void EsdlBindingImpl::clearBindingState()
     clearState(m_esdlStateFilesLocation.str());
 }
 
-void EsdlBindingImpl::addService(const char * name,
+void EsdlBindingImpl::addService(IPropertyTree *esdlArchive, const char * name,
                                  const char * host,
                                  unsigned short port,
                                  IEspService & service)
@@ -1828,7 +1892,13 @@ void EsdlBindingImpl::addService(const char * name,
             if (m_esdl)
             {
                 StringBuffer loadedservicename;
-                if (!loadDefinitions(name, m_esdl, m_esdlBndCfg, loadedservicename, m_esdlStateFilesLocation.str()))
+                bool loaded = false;
+                if (esdlArchive)
+                    loaded = loadLocalDefinitions(esdlArchive, name, m_esdl, m_esdlBndCfg, loadedservicename);
+                else
+                    loaded = loadDefinitions(name, m_esdl, m_esdlBndCfg, loadedservicename, m_esdlStateFilesLocation.str());
+
+                if (!loaded)
                 {
                     DBGLOG("ESDL Binding: Error adding ESP service '%s': Could not fetch ESDL definition", name);
                     return;

esp/services/esdl_svc_engine/esdl_binding.hpp

@@ -244,7 +244,7 @@ public:
     IMPLEMENT_IINTERFACE;
 
     EsdlBindingImpl();
-    EsdlBindingImpl(IPropertyTree* cfg, const char *bindname=NULL, const char *procname=NULL);
+    EsdlBindingImpl(IPropertyTree* cfg, IPropertyTree *esdlArchive, const char *bindname=NULL, const char *procname=NULL);
 
     virtual ~EsdlBindingImpl()
     {
@@ -257,7 +257,7 @@ public:
 
     virtual void initEsdlServiceInfo(IEsdlDefService &srvdef);
 
-    virtual void addService(const char * name, const char * host, unsigned short port, IEspService & service);
+    void addService(IPropertyTree *esdlArchive, const char * name, const char * host, unsigned short port, IEspService & service);
 
     int onGetInstantQuery(IEspContext &context, CHttpRequest* request, CHttpResponse* response,    const char *serviceName, const char *methodName);
     int HandleSoapRequest(CHttpRequest* request, CHttpResponse* response);
@@ -373,6 +373,8 @@ private:
     void saveDESDLState();
     IPropertyTree * fetchESDLBinding(const char *process, const char *bindingName, const char * stateFileName);
     bool loadDefinitions(const char * espServiceName, Owned<IEsdlDefinition>& esdl, IPropertyTree * config, StringBuffer & loadedServiceName, const char * stateFileName);
+    bool loadLocalDefinitions(IPropertyTree *esdlArchive, const char * espServiceName, Owned<IEsdlDefinition>& esdl, IPropertyTree * config, StringBuffer & loadedServiceName);
+
 };
 
 #endif //_EsdlBinding_HPP__

esp/services/esdl_svc_engine/esdl_monitor.cpp

@@ -176,6 +176,8 @@ public:
 
     void setupSubscription()
     {
+        if (false == queryComponentConfig().getPropBool("@loadDaliBindings", true))
+            return;
         m_isSubscribed = true;
         m_pSubscription.setown(createEsdlSubscription(this));
     }
@@ -211,8 +213,69 @@ public:
         }
     }
 
+    void addLocalBinding(const char *process, unsigned int port, const char *filepath, IFile &ifile)
+    {
+        if (isEmptyString(process))
+            process = "esdl";
+        Owned<IPropertyTree> tree = createPTree(ifile);
+        if (!tree)
+            return;
+        StringBuffer id(tree->queryProp("@id"));
+        if (id.isEmpty())
+            splitFilename(filepath, nullptr, nullptr, &id, nullptr);
+        const char *static_binding = tree->queryProp("@static_binding");
+        if (isEmptyString(static_binding))
+            static_binding = "esdl_binding";
+
+        DBGLOG("Creating new binding %s", id.str());
+        CriticalBlock cb(m_CritSect);
+
+        VStringBuffer portStr("%d", port);
+        StringBuffer protocol, serviceName;
+        Owned<IPropertyTree> envpt = getEnvpt(static_binding, id, portStr, protocol, serviceName);
+        if (protocol.length() == 0)
+            protocol.set("http");
+        StringBuffer envptxml;
+        toXML(envpt, envptxml);
+        DBGLOG("Use the following config tree to create the binding and service:\n%s\n", envptxml.str());
+        IEspServer* server = queryEspServer();
+        IEspProtocol* espProtocol = server->queryProtocol(protocol.str());
+        Owned<EsdlBindingImpl> esdlbinding = new CEsdlSvcEngineSoapBindingEx(envpt, tree, static_binding, process);
+        Owned<EsdlServiceImpl> esdlservice = new CEsdlSvcEngine();
+        esdlservice->init(envpt, process, serviceName.str());
+        esdlbinding->addService(tree, esdlservice->getServiceType(), nullptr, port, *esdlservice.get());
+        esdlbinding->addProtocol(protocol.str(), *espProtocol);
+        server->addBinding(id.str(), nullptr, port, *espProtocol, *esdlbinding.get(), false, envpt);
+        DBGLOG("Successfully instantiated new DESDL binding %s and service", id.str());
+    }
+
+    void loadLocalBindings()
+    {
+        const char *paths = queryComponentConfig().queryProp("@bindings");
+        if (isEmptyString(paths))
+            return;
+        StringArray entries;
+        entries.appendList(paths, ";");
+
+        const char *process = queryComponentConfig().queryProp("@instance");
+        unsigned int port  = queryComponentConfig().getPropInt("@port");
+        ForEachItemIn(i, entries)
+        {
+            const char *path = entries.item(i);
+            Owned<IDirectoryIterator> dir = createDirectoryIterator(path, nullptr, false, false);
+            ForEach(*dir)
+            {
+                StringBuffer filename;
+                dir->getName(filename);
+                addLocalBinding(process, port, filename, dir->query());
+            }
+        }
+    }
+
     void loadDynamicBindings()
     {
+        if (false == queryComponentConfig().getPropBool("@loadDaliBindings", true))
+            return;
         Owned<IPropertyTree> esdlBindings = m_pCentralStore->getBindings();
         if (!esdlBindings)
            throw MakeStringException(-1, "Unable to retrieve ESDL bindings information");
@@ -380,10 +443,10 @@ private:
         DBGLOG("Use the following config tree to create the binding and service:\n%s\n", envptxml.str());
         IEspServer* server = queryEspServer();
         IEspProtocol* espProtocol = server->queryProtocol(protocol.str());
-        Owned<EsdlBindingImpl> esdlbinding = new CEsdlSvcEngineSoapBindingEx(envpt,  data->name.str(), data->espProcess.str());
+        Owned<EsdlBindingImpl> esdlbinding = new CEsdlSvcEngineSoapBindingEx(envpt, nullptr, data->name.str(), data->espProcess.str());
         Owned<EsdlServiceImpl> esdlservice = new CEsdlSvcEngine();
         esdlservice->init(envpt, data->espProcess.str(), serviceName.str());
-        esdlbinding->addService(esdlservice->getServiceType(), nullptr, data->port, *esdlservice.get());
+        esdlbinding->addService(nullptr, esdlservice->getServiceType(), nullptr, data->port, *esdlservice.get());
         esdlbinding->addProtocol(protocol.str(), *espProtocol);
         server->addBinding(data->name.str(), nullptr, data->port, *espProtocol, *esdlbinding.get(), false, envpt);
         DBGLOG("Successfully instantiated new DESDL binding %s and service", data->id.str());
@@ -412,11 +475,10 @@ private:
     IPropertyTree* getEnvpt(EsdlNotifyData* notifyData, StringBuffer& protocol, StringBuffer& serviceName)
     {
         VStringBuffer portStr("%d", notifyData->port);
-        return getEnvpt(notifyData->espProcess.str(), notifyData->name.str(), notifyData->id.str(),
-                portStr.str(), protocol, serviceName);
+        return getEnvpt(notifyData->name.str(), notifyData->id.str(), portStr.str(), protocol, serviceName);
     }
 
-    IPropertyTree* getEnvpt(const char* espProcess, const char* bindingName, const char* bindingId, const char* port, StringBuffer& protocol, StringBuffer& serviceName)
+    IPropertyTree* getEnvpt(const char* bindingName, const char* bindingId, const char* port, StringBuffer& protocol, StringBuffer& serviceName)
     {
         if (!bindingName || !*bindingName)
             bindingName = bindingId;
@@ -477,6 +539,7 @@ extern "C" void startEsdlMonitor()
         CEsdlMonitor* monitor = new CEsdlMonitor();
         gEsdlMonitor.setown(monitor);
         isEsdlMonitorStarted = true;
+        monitor->loadLocalBindings();
         monitor->loadDynamicBindings();
         monitor->setupSubscription();
     }

esp/services/esdl_svc_engine/esdl_store.cpp

@@ -729,77 +729,99 @@ public:
         return true;
     }
 
-    virtual int bindService(const char* bindingName,
+    void validatePort(const char *espPort)
+    {
+        while(*espPort == '0')
+            espPort++;
+
+        int ind = 0;
+        for ( ; espPort[ind]; ind++)
+        {
+            if (!isdigit(espPort[ind]))
+                throw MakeStringException(-1, "Esp port can only be a positive integer.");
+        }
+        if(ind > 5)
+            throw MakeStringException(-1, "Esp port should be between 1 and 65535");
+
+        int port = atoi(espPort);
+        if(port <= 0 || port > 65535)
+            throw MakeStringException(-1, "Esp port should be between 1 and 65535");
+    }
+
+    bool validateBindServiceParameters(const char* bindingName,
                                              IPropertyTree* methodsConfig,
                                              const char* espProcName,
                                              const char* espPort,
                                              const char* definitionId,
                                              const char* esdlServiceName,
-                                             StringBuffer& message,
-                                             bool overwrite,
-                                             const char* user) override
+                                             StringBuffer& lcDefId,
+                                             StringBuffer& message)
     {
         if (!espProcExists(espProcName))
         {
             message.setf("Esp process %s not found in the environment, please double check the case-sensitive spelling", espProcName);
-            return -1;
+            return false;
         }
 
-        if ((!bindingName || !*bindingName) && (!espPort || !*espPort))
+        if (isEmptyString(bindingName) && isEmptyString(espPort))
         {
             message.setf("Could not configure '%s' - need target binding name or port", esdlServiceName);
-            return -1;
+            return false;
         }
 
-        if(espPort && *espPort)
+        if (isEmptyString(definitionId))
         {
-            while(*espPort == '0')
-                espPort++;
-
-            int ind = 0;
-            for ( ; espPort[ind]; ind++)
-            {
-                if (!isdigit(espPort[ind]))
-                    throw MakeStringException(-1, "Esp port can only be a positive integer.");
-            }
-            if(ind > 5)
-                throw MakeStringException(-1, "Esp port should be between 1 and 65535");
-
-            int port = atoi(espPort);
-            if(port <= 0 || port > 65535)
-                throw MakeStringException(-1, "Esp port should be between 1 and 65535");
+            message.set("Could not configure DESDL service: Target Esdl definition id required");
+            return false;
         }
 
-        if (!definitionId || !*definitionId)
-        {
-            message.set("Could not configure DESDL service: Target Esdl definition id not available");
-            return -1;
-        }
-        StringBuffer lcDefId(definitionId);
+        if(espPort && *espPort)
+            validatePort(espPort);
+
+        lcDefId.set(definitionId);
         lcDefId.trim().toLowerCase();
         definitionId = lcDefId.str();
+
         if (!definitionExists(definitionId))
         {
             message.setf("Esdl definition %s doesn't exist, please double check the spelling", definitionId);
-            return -1;
+            return false;
         }
 
-        if (!esdlServiceName || !*esdlServiceName)
+        if (isEmptyString(esdlServiceName))
         {
-            message.set("Could not configure DESDL service: Target Esdl definition service name not available");
-            return -1;
+            message.set("Could not configure DESDL service: Target Esdl definition name reqired");
+            return false;
         }
 
         if (!isEsdlServiceDefined(definitionId, esdlServiceName))
         {
             message.setf("Esdl service %s is not defined in %s, please double check the case-sensitive service name", esdlServiceName, definitionId);
-            return -1;
+            return false;
         }
+        return true;
+    }
+
+    virtual int bindService(const char* bindingName,
+                                             IPropertyTree* methodsConfig,
+                                             const char* espProcName,
+                                             const char* espPort,
+                                             const char* definitionId,
+                                             const char* esdlServiceName,
+                                             StringBuffer& message,
+                                             bool overwrite,
+                                             const char* user) override
+    {
+        StringBuffer lcDefId;
+        if (!validateBindServiceParameters(bindingName, methodsConfig, espProcName, espPort, definitionId, esdlServiceName, lcDefId, message))
+            return -1;
 
         //Get static binding if exists
-        Owned<IPropertyTree> bindingcfg = getEspBindingConfig(espProcName, espPort, bindingName);
-        StringBuffer duplicateBindingId;
-        if (bindingName && *bindingName)
+        Owned<IPropertyTree> staticCfg;
+        if (!isContainerized())
+            staticCfg.setown(getStaticBindingConfig(espProcName, espPort, bindingName));
+        StringBuffer existingBindingId;
+        if (!isEmptyString(bindingName))
         {
             StringBuffer msg;
             Owned<IPropertyTree> esdlbindingtree = getBindingTree(espProcName, bindingName, msg);
@@ -822,13 +844,22 @@ public:
                 }
                 if (!espPort || !*espPort)
                     espPort = existingPort;
-                duplicateBindingId.set(esdlbindingtree->queryProp("@id"));
+                existingBindingId.set(esdlbindingtree->queryProp("@id"));
             }
-            else if (bindingcfg)
+            else if (isContainerized())
+            {
+                //currently always the same for containerized esdl-sandbox
+                const char* bn = "esdl_svc_engine_binding";
+                StringBuffer msg;
+                Owned<IPropertyTree> esdlbindingtree = getBindingTree(espProcName, bn, msg);
+                if (!esdlbindingtree)
+                    bindingName = bn;
+            }
+            else if (staticCfg)
             {
                 DBGLOG("Static esp binding configured for %s", bindingName);
                 if (!espPort || !*espPort)
-                    espPort = bindingcfg->queryProp("@port");
+                    espPort = staticCfg->queryProp("@port");
             }
             else if (!espPort || !*espPort)
             {
@@ -838,9 +869,13 @@ public:
         }
         else //espPort provided
         {
-            if (bindingcfg)
+            const char *bn = nullptr;
+            if (isContainerized())
+                bn = "esdl_svc_engine_binding";
+            else if (staticCfg)
+                bn = staticCfg->queryProp("@name");
+            if (bn)
             {
-                const char* bn = bindingcfg->queryProp("@name");
                 DBGLOG("Static esp binding %s configured for port %s", bn, espPort);
                 StringBuffer msg;
                 Owned<IPropertyTree> esdlbindingtree = getBindingTree(espProcName, bn, msg);
@@ -852,8 +887,8 @@ public:
             }
             else
             {
-                bindingcfg.setown(getEspBindingConfig(espProcName, "0", nullptr));
-                if (!bindingcfg)
+                staticCfg.setown(getStaticBindingConfig(espProcName, "0", nullptr));
+                if (!staticCfg)
                 {
                     message.setf("Could not configure service '%s' because there's no template esp binding configured on port %s or port 0 for esp process '%s'", esdlServiceName, espPort, espProcName);
                     OWARNLOG("%s", message.str());
@@ -866,29 +901,28 @@ public:
            throw MakeStringException(-1, "Unexpected error while attempting to access ESDL definition dali registry.");
 
         IPropertyTree * bindings = conn->queryRoot();
-
-        IPropertyTree* duplicateBinding = nullptr;
-        if (duplicateBindingId.length() == 0)
+        IPropertyTree* existingBinding = nullptr;
+        if (existingBindingId.length() == 0)
         {
             VStringBuffer xpath("%s[@espprocess='%s'][@port='%s'][Definition/@esdlservice='%s']", ESDL_BINDING_ENTRY, espProcName, espPort, esdlServiceName);
-            duplicateBinding = bindings->queryPropTree(xpath.str());
+            existingBinding = bindings->queryPropTree(xpath.str());
         }
         else
         {
-            VStringBuffer xpath("%s[@id='%s']", ESDL_BINDING_ENTRY, duplicateBindingId.str());
-            duplicateBinding = bindings->queryPropTree(xpath.str());
+            VStringBuffer xpath("%s[@id='%s']", ESDL_BINDING_ENTRY, existingBindingId.str());
+            existingBinding = bindings->queryPropTree(xpath.str());
         }
 
         StringBuffer origTimestamp;
         StringBuffer origOwner;
 
-        if (duplicateBinding)
+        if (existingBinding)
         {
             if (overwrite)
             {
-                origTimestamp.set(duplicateBinding->queryProp("@created"));
-                origOwner.set(duplicateBinding->queryProp("@publishedBy"));
-                bindings->removeTree(duplicateBinding);
+                origTimestamp.set(existingBinding->queryProp("@created"));
+                origOwner.set(existingBinding->queryProp("@publishedBy"));
+                bindings->removeTree(existingBinding);
             }
             else
             {
@@ -1123,9 +1157,9 @@ public:
             IPropertyTree &binding = iter->query();
             if (binding.getPropInt("@port", 0) == 0)
             {
-                Owned<IPropertyTree> espbindingcfg = getEspBindingConfig(binding.queryProp("@espprocess"), nullptr, binding.queryProp("@espbinding"));
-                if(espbindingcfg)
-                    binding.setPropInt("@port", espbindingcfg->getPropInt("@port", 0));
+                Owned<IPropertyTree> staticCfg = getStaticBindingConfig(binding.queryProp("@espprocess"), nullptr, binding.queryProp("@espbinding"));
+                if(staticCfg)
+                    binding.setPropInt("@port", staticCfg->getPropInt("@port", 0));
             }
         }
         return bindings.getLink();
@@ -1162,7 +1196,7 @@ private:
         return isDefinitionBound(id);
     }
 
-    IPropertyTree * getEspBindingConfig(const char * espprocname, const char * espbindingport, const char * bindingname)
+    IPropertyTree * getStaticBindingConfig(const char * espprocname, const char * espbindingport, const char * bindingname)
     {
         if (!espprocname || !*espprocname)
             return nullptr;
@@ -1209,6 +1243,17 @@ private:
 
     bool espProcExists(const char * espprocname)
     {
+#ifdef _CONTAINERIZED
+        VStringBuffer xpath("services[@name='%s']", espprocname);
+        IPropertyTree *service = queryComponentConfig().queryPropTree(xpath);
+        if (service)
+        {
+            const char *serviceType = service->queryProp("@type");
+            if (!serviceType || streq(serviceType, "esdl-sandbox"))
+                return true;
+        }
+        return false;
+#else
         if (!espprocname || !*espprocname)
             return false;
         VStringBuffer xpath("/Environment/Software/EspProcess[@name='%s']", espprocname);
@@ -1217,6 +1262,7 @@ private:
             return true;
         else
             return false;
+#endif
     }
 
     bool isEsdlServiceDefined(const char* definitionId, const char* serviceName)

esp/services/esdl_svc_engine/esdl_svc_engine.cpp

@@ -35,7 +35,7 @@ CEsdlSvcEngineSoapBindingEx::CEsdlSvcEngineSoapBindingEx()
 {
 }
 
-CEsdlSvcEngineSoapBindingEx::CEsdlSvcEngineSoapBindingEx(IPropertyTree* cfg, const char *bindname, const char *procname) : EsdlBindingImpl(cfg, bindname, procname)
+CEsdlSvcEngineSoapBindingEx::CEsdlSvcEngineSoapBindingEx(IPropertyTree* cfg, IPropertyTree *esdlArchive, const char *bindname, const char *procname) : EsdlBindingImpl(cfg, esdlArchive, bindname, procname)
 {
 }
 

esp/services/esdl_svc_engine/esdl_svc_engine.hpp

@@ -47,14 +47,14 @@ public:
     IMPLEMENT_IINTERFACE;
 
     CEsdlSvcEngineSoapBindingEx();
-    CEsdlSvcEngineSoapBindingEx(IPropertyTree* cfg, const char *bindname=NULL, const char *procname=NULL);
+    CEsdlSvcEngineSoapBindingEx(IPropertyTree* cfg, IPropertyTree *esdlArchive, const char *bindname=NULL, const char *procname=NULL);
 
     virtual ~CEsdlSvcEngineSoapBindingEx() {}
 
     virtual void addService(const char * name, const char * host, unsigned short port, IEspService & service)
     {
         m_pESDLService = dynamic_cast<CEsdlSvcEngine*>(&service);
-        EsdlBindingImpl::addService(name, host, port, service);
+        EsdlBindingImpl::addService(nullptr, name, host, port, service);
     }
 
     virtual const char *queryServiceType(){return  m_pESDLService->getServiceType();}

esp/services/esdl_svc_engine/esdl_svc_engine_plugin.cpp

@@ -37,7 +37,7 @@ ESP_FACTORY IEspService * esp_service_factory(const char *name, const char* type
 
 ESP_FACTORY IEspRpcBinding * esp_binding_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
 {
-    return new CEsdlSvcEngineSoapBindingEx(cfg, name, process);
+    return new CEsdlSvcEngineSoapBindingEx(cfg, nullptr, name, process);
 }
 
 

helm/examples/esdl-sandbox/README.md

@@ -0,0 +1,41 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+Example of esdl-sandbox service.  This service is an alpha release.  It may change quite a bit before the final release.
+
+After deploying a helm k8s hpcc..
+
+From helm/examples/esdl-sandbox folder:
+
+1. Generate ecl layouts:
+esdl ecl esdl_example.esdl .
+
+2. Publish example roxie query:
+ecl publish roxie RoxieEchoPersonInfo.ecl
+
+3. Publish the esdl defined service to dynamicESDL:
+esdl publish esdl_example.esdl EsdlExample
+
+4. Bind both java and roxie implementations to DynamicESDL (note that here, port is internal pod port not service port)
+esdl bind-service esdl-sandbox 8880 esdlexample.1 EsdlExample --config esdl_binding.xml
+
+5. Test calling the service:
+soapplus -url http://.:8899/EsdlExample -i roxierequest.xml
+
+6. Interact with both services by browsing to:
+
+http://<DynamicEsdlIP>:8899

helm/examples/esdl-sandbox/RoxieEchoPersonInfo.ecl

@@ -0,0 +1,3 @@
+import $.esdl_example;
+request := dataset([], esdl_example.t_RoxieEchoPersonInfoRequest) : stored('RoxieEchoPersonInfoRequest', few);
+output(request, named('RoxieEchoPersonInfoResponse'));

helm/examples/esdl-sandbox/esdl_binding.xml

@@ -0,0 +1,27 @@
+ <Methods>
+       <xsdl:CustomRequestTransform>
+          <xsdl:choose>
+             <xsdl:when test="$clientversion=1.9">
+                <xsdl:SetValue target="Row/Name/First"  value="'service'" />
+             </xsdl:when>
+             <xsdl:otherwise>
+                <xsdl:SetValue target="Row/Name/Last" value="'zz'"/>
+             </xsdl:otherwise>
+          </xsdl:choose>
+       </xsdl:CustomRequestTransform>
+    <Method name="RoxieEchoPersonInfo" querytype="roxie" url="http://roxie:9876/roxie" queryname="RoxieEchoPersonInfo">
+         <xsdl:CustomRequestTransform target="soap:Body/{$query}">
+            <xsdl:choose>
+               <xsdl:when test="$clientversion=1.9">
+                  <xsdl:SetValue target="vertest"  value="'v1.9'"/>
+                  <xsdl:SetValue target="RoxieEchoPersonInfoRequest/Row/Name/First"  value="'v1.9'"/>
+               </xsdl:when>
+               <xsdl:otherwise>
+                  <xsdl:SetValue target="vertest"  value="concat('v', $clientversion)"/>
+                  <xsdl:SetValue target="RoxieEchoPersonInfoRequest/Row/Name/Last" value="concat('v', $clientversion)"/>
+               </xsdl:otherwise>
+            </xsdl:choose>
+         </xsdl:CustomRequestTransform>
+   </Method>
+</Methods>
+

helm/examples/esdl-sandbox/esdl_example.ecl

@@ -0,0 +1,50 @@
+/*** Not to be hand edited (changes will be lost on re-generation) ***/
+/*** ECL Interface generated by esdl2ecl version 1.0 from esdl_example.xml. ***/
+/*===================================================*/
+
+
+EXPORT esdl_example := MODULE
+
+EXPORT t_NameInfo := RECORD
+	UTF8 First {XPATH('First')};
+	UTF8 Last {XPATH('Last')};
+	SET OF UTF8 Aliases {XPATH('Aliases/Alias'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! 
+END;
+
+EXPORT t_AddressInfo := RECORD
+	UTF8 Line1 {XPATH('Line1')};
+	UTF8 Line2 {XPATH('Line2')};
+	UTF8 City {XPATH('City')};
+	UTF8 State {XPATH('State')};
+	INTEGER Zip {XPATH('Zip')};
+	UTF8 _type {XPATH('type')}; //values['Home','Work','Hotel','']
+END;
+
+EXPORT t_RoxieEchoPersonInfoRequest := RECORD
+	t_NameInfo Name {XPATH('Name')};
+	DATASET(t_AddressInfo) Addresses {XPATH('Addresses/Address'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! 
+END;
+
+/*Empty record generated from empty EsdlRequest
+EXPORT t_EsdlExamplePingRequest := RECORD
+END;
+*/
+
+EXPORT t_RoxieEchoPersonInfoResponse := RECORD
+	INTEGER count {XPATH('count')};
+	t_NameInfo Name {XPATH('Name')};
+	DATASET(t_AddressInfo) Addresses {XPATH('Addresses/Address'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! 
+END;
+
+/*Empty record generated from empty EsdlResponse
+EXPORT t_EsdlExamplePingResponse := RECORD
+END;
+*/
+
+
+END;
+
+/*** Not to be hand edited (changes will be lost on re-generation) ***/
+/*** ECL Interface generated by esdl2ecl version 1.0 from esdl_example.xml. ***/
+/*===================================================*/
+

helm/examples/esdl-sandbox/esdl_example.esdl

@@ -0,0 +1,61 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+////////////////////////////////////////////////////////////
+
+ESPenum AddressType : string
+{
+    HOME("Home"),
+    WORK("Work"),
+    HOTEL("Hotel")
+};
+
+ESPStruct NameInfo
+{
+    string First("Joe");
+    string Last("Doe");
+    ESParray<string, Alias> Aliases;
+};
+
+ESPStruct AddressInfo
+{
+    ESPenum AddressType type("Home");
+    string Line1;
+    string Line2;
+    string City;
+    string State;
+    int Zip(33487);  
+};
+
+ESPrequest RoxieEchoPersonInfoRequest
+{
+     ESPstruct NameInfo Name;
+     ESParray<ESPstruct AddressInfo, Address> Addresses;
+};
+
+ESPresponse RoxieEchoPersonInfoResponse
+{
+     int count(0);
+     ESPstruct NameInfo Name;
+     ESParray<ESPstruct AddressInfo, Address> Addresses;
+};
+
+ESPservice [version("0.01")] EsdlExample
+{
+    ESPmethod RoxieEchoPersonInfo(RoxieEchoPersonInfoRequest, RoxieEchoPersonInfoResponse);
+};
+

helm/examples/esdl-sandbox/roxierequest.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:SOAP-ENC="http://schemas.xmlsoap.org/soap/encoding/" xmlns="urn:hpccsystems:ws:esdlexample:roxieechopersoninfo@ver=0">
+                                                                                                                                        urn:hpccsystems:ws:esdlexample:roxieechopersoninfo@ver=0.01
+ <soap:Body>
+  <RoxieEchoPersonInfoRequest>
+   <Name>
+    <First>Joe</First>
+    <Last>Doe</Last>
+    <Aliases>
+     <Alias>John</Alias>
+     <Alias>James</Alias>
+     <Alias>Joseph</Alias>
+     <Alias>Jessy</Alias>
+    </Aliases>
+   </Name>
+   <Addresses>
+    <Address>
+     <Line1>105 Main St</Line1>
+     <City>Boca Raton</City>
+     <State>Florida</State>
+     <Zip>33487</Zip>
+     <type>Home</type>
+    </Address>
+    <Address>
+     <Line1>901 Main St</Line1>
+     <City>Orlando</City>
+     <State>Florida</State>
+     <Zip>77777</Zip>
+     <type>Work</type>
+    </Address>
+   </Addresses>
+  </RoxieEchoPersonInfoRequest>
+ </soap:Body>
+</soap:Envelope>

helm/examples/esdl-sandbox/share.ecl

@@ -0,0 +1,11 @@
+export share := module
+
+export t_IntegerArrayItem := record
+    integer value { xpath('')};
+end;
+
+export t_StringArrayItem := record
+    string value { xpath(''), MAXLENGTH(8192) };
+end;
+
+end;

helm/examples/esdl-sandbox/soaprequest.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:SOAP-ENC="http://schemas.xmlsoap.org/soap/encoding/" xmlns="urn:hpccsystems:ws:esdlexample:roxieechopersoninfo@ver=0">
+ <soap:Body>
+  <RoxieEchoPersonInfoRequest>
+   <Name>
+    <First>Joe</First>
+    <Last>Doe</Last>
+    <Aliases>
+     <Alias>John</Alias>
+     <Alias>James</Alias>
+     <Alias>Joseph</Alias>
+     <Alias>Jessy</Alias>
+    </Aliases>
+   </Name>
+   <Addresses>
+    <Address>
+     <Line1>105 Main St</Line1>
+     <City>Boca Raton</City>
+     <State>Florida</State>
+     <Zip>33487</Zip>
+     <type>Home</type>
+    </Address>
+    <Address>
+     <Line1>901 Main St</Line1>
+     <City>Orlando</City>
+     <State>Florida</State>
+     <Zip>77777</Zip>
+     <type>Work</type>
+    </Address>
+   </Addresses>
+  </RoxieEchoPersonInfoRequest>
+ </soap:Body>
+</soap:Envelope>

helm/hpcc/values.yaml

@@ -192,6 +192,14 @@ esp:
   public: true
   servicePort: 8002
 
+- name: esdl-sandbox
+  application: esdl-sandbox
+  auth: none
+  tls: off
+  replicas: 1
+  public: true
+  servicePort: 8899
+
 - name: sql2ecl
   application: sql2ecl
   auth: none