HPCC-25491 Support Azure LDAP Helm configuration for DALI

Enable LDAP support in Dali, including specifying LDAP credentials as secrets.
Provide legacy and Azure LDAP defaults in dali/ldap/yaml files.
Ensure support for Dali config parm checkScopeScans

Signed-off-by: Russ Whitehead <william.whitehead@lexisnexisrisk.com>

dali/CMakeLists.txt

@@ -30,3 +30,4 @@ HPCC_ADD_SUBDIRECTORY (server "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (treeview "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (updtdalienv "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (sasha "PLATFORM")
+HPCC_ADD_SUBDIRECTORY (ldap "PLATFORM")

dali/ldap/CMakeLists.txt

@@ -0,0 +1,23 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+set ( COMMON_LDAP_FILES
+    ${HPCC_SOURCE_DIR}/esp/applications/common/ldap/ldap.yaml
+)
+
+FOREACH( iFile ${COMMON_LDAP_FILES} )
+    Install( FILES ${iFile} DESTINATION componentfiles/applications/common/ldap COMPONENT Runtime )
+ENDFOREACH ( iFile )

dali/server/daldap.cpp

@@ -59,7 +59,7 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
     void createDefaultScopes()
     {
         try {
-            ISecUser* user = NULL;
+            Owned<ISecUser> user = ldapsecurity->createUser(nullptr);
             StringBuffer userTempFileScope(queryDfsXmlBranchName(DXB_Internal));
             if (ldapsecurity->addResourceEx(RT_FILE_SCOPE, *user, userTempFileScope.str(),PT_ADMINISTRATORS_ONLY, NULL))
                 PROGLOG("LDAP: Created default '%s' scope", userTempFileScope.str());

dali/server/daserver.cpp

@@ -392,6 +392,45 @@ static StringBuffer &formatDaliRole(StringBuffer &out, unsigned __int64 role)
     return out.append(queryRoleName((DaliClientRole)role));
 }
 
+#ifdef _CONTAINERIZED
+static IPropertyTree * getContainerLDAPConfiguration(const IPropertyTree *appConfig)
+{
+    const char * authMethod = appConfig->queryProp("@auth");
+    if (streq(authMethod, "none"))
+    {
+        WARNLOG("ECLWatch is unsafe, no security manager specified in configuration (auth: none)");
+        return nullptr; //no security manager
+    }
+
+    if (!streq(authMethod, "ldap"))
+    {
+        throw makeStringExceptionV(-1, "Unrecognized auth method specified, (auth: %s)", authMethod);
+    }
+
+    const char *ldapAddress = appConfig->queryProp("@ldapAddress");
+    if (isEmptyString(ldapAddress))
+        throw makeStringException(-1, "LDAP not configured (missing 'ldapAddress').  To run without security set 'auth: none'");
+
+    //Get default LDAP attributes from ldap.yaml
+    StringBuffer ldapDefaultsFile(hpccBuildInfo.componentDir);
+    char sepchar = getPathSepChar(ldapDefaultsFile.str());
+    addPathSepChar(ldapDefaultsFile, sepchar).append("applications").append(sepchar).append("common").append(sepchar).append("ldap").append(sepchar).append("ldap.yaml");
+    Owned<IPropertyTree> defaults;
+    if (!checkFileExists(ldapDefaultsFile))
+    {
+        throw makeStringExceptionV(-1, "Unable to locate LDAP defaults file '%s'", ldapDefaultsFile.str());
+    }
+    defaults.setown(createPTreeFromYAMLFile(ldapDefaultsFile.str()));
+
+    //Build merged configuration
+    Owned<IPropertyTree> mergedConfig = defaults->getPropTree("ldap");
+    mergePTree(mergedConfig, appConfig->queryPropTree("ldap"));//overlay defaults with config settings
+    mergedConfig->addProp("@ldapAddress", ldapAddress);
+
+    return LINK(mergedConfig);
+}
+#endif
+
 static constexpr const char * defaultYaml = R"!!(
 version: 1.0
 dali:
@@ -736,7 +775,11 @@ int main(int argc, const char* argv[])
             }
             else
             {
-                setLDAPconnection(createDaliLdapConnection(serverConfig->getPropTree("Coven/ldapSecurity")));
+#ifdef _CONTAINERIZED
+                setLDAPconnection(createDaliLdapConnection(getContainerLDAPConfiguration(serverConfig)));//container configuration
+#else
+                setLDAPconnection(createDaliLdapConnection(serverConfig->getPropTree("Coven/ldapSecurity")));//legacy configuration
+#endif
             }
 #endif
         }

esp/applications/common/ldap/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -14,11 +14,10 @@
 #    limitations under the License.
 ################################################################################
 
-set ( ESP_LDAP_FILES
+set ( COMMON_LDAP_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/ldap.yaml
-    ${CMAKE_CURRENT_SOURCE_DIR}/azure_ldap.yaml
 )
 
-FOREACH( iFile ${ESP_LDAP_FILES} )
+FOREACH( iFile ${COMMON_LDAP_FILES} )
     Install( FILES ${iFile} DESTINATION componentfiles/applications/common/ldap COMPONENT Runtime )
 ENDFOREACH ( iFile )

esp/applications/common/ldap/azure_ldap.yaml

@@ -1,21 +0,0 @@
-ldap:
-  serverType: AzureActiveDirectory
-  description: LDAP server process
-  ldapProtocol: ldaps
-  ldapPort: 389
-  ldapSecurePort: 636
-  adminGroupName: HPCCAdmins
-  maxConnections: 10
-  passwordExpirationWarningDays: 10
-  cacheTimeout: 5
-  ldapTimeoutSecs: 131
-  sharedCache: true
-  filesBasedn: ou=files,ou=ecl
-  groupsBasedn: ou=AADDC Users
-  sudoersBasedn: ou=SUDOers
-  systemBasedn: ou=AADDC Users
-  usersBasedn: ou=AADDC Users
-  resourcesBasedn: ou=WsEcl,ou=EspServices,ou=ecl
-  workunitsBasedn: ou=workunits,ou=ecl
-  ldapAdminSecretKey: "ldapadmincredskey"
-  ldapAdminVaultId: ""

esp/applications/common/ldap/ldap.yaml

@@ -19,3 +19,4 @@ ldap:
   workunitsBasedn: ou=workunits,ou=ecl
   ldapAdminSecretKey: "ldapadmincredskey"
   ldapAdminVaultId: ""
+  checkScopeScans: true

esp/platform/application_config.cpp

@@ -97,11 +97,7 @@ bool addLdapSecurity(IPropertyTree *legacyEsp, IPropertyTree *appEsp, StringBuff
 
     StringBuffer path(hpccBuildInfo.componentDir);
     char sepchar = getPathSepChar(hpccBuildInfo.componentDir);
-    addPathSepChar(path, sepchar).append("applications").append(sepchar).append("common").append(sepchar).append("ldap").append(sepchar);
-    if (ldapType == LdapType::LegacyAD)
-        path.append("ldap.yaml");
-    else
-        path.append("azure_ldap.yaml");
+    addPathSepChar(path, sepchar).append("applications").append(sepchar).append("common").append(sepchar).append("ldap").append(sepchar).append("ldap.yaml");
     if (checkFileExists(path))
         appendPTreeFromYamlFile(appEsp, path.str(), false);
 

helm/hpcc/templates/dali.yaml

@@ -27,7 +27,7 @@
 {{- $daliPlaneIncludeCategories := list "dali" -}}
 {{- $commonCtx := dict "root" $ "me" $dali "includeCategories" $daliPlaneIncludeCategories "env" $env -}}
 {{- $sashaServices := $dali.services | default dict -}}
-{{- $daliSecretsCategories := list "system" -}}
+{{- $daliSecretsCategories := list "system" "authn" -}}
 {{- $tmpDaliScope := dict "aggregateSashaSecretsCategories" list "aggregatePlaneCategories" $daliPlaneIncludeCategories "aggregateSashaNamedPlanes" list -}}
 {{- if not ($sashaServices.disabled) -}}
  {{- range $sashaName, $_sasha := $dali.services -}}

helm/hpcc/values.schema.json

@@ -591,6 +591,11 @@
         }
       }
     },
+    "auth": {
+      "type": "string",
+      "description": "Authentication method",
+      "enum": ["none", "ldap"]
+    },
     "ldap": {
       "type": "object",
       "additionalProperties": { "type": ["integer", "string", "boolean"] },
@@ -625,7 +630,7 @@
         },
         "systemBasedn": {
           "type": "string",
-          "description": "The base distinguished name of the Active Directory systemUser"
+          "description": "The base distinguished name of the Active Directory Administrator"
         },
         "resourcesBasedn": {
           "type": "string",
@@ -637,11 +642,11 @@
         },
         "ldapAdminSecretKey": {
           "type": "string",
-          "description": "The key name to be used to look up the Active Directory system administrator account Username/Password"
+          "description": "The key name to be used to look up the Active Directory Administrator account Username/Password"
         },
         "ldapAdminVaultId": {
           "type": "string",
-          "description": "The optional vault name to be used to look up the Active Directory system administrator account Username/Password, using ldapAdminSecretKey"
+          "description": "The optional vault name to be used to look up the Active Directory Administrator account Username/Password, using ldapAdminSecretKey"
         },
         "ldapPort": {
           "type": "integer",
@@ -670,6 +675,10 @@
         "sharedCache": {
           "type": "boolean",
           "description": "Use a single, shared LDAP cache"
+        },
+        "checkScopeScans": {
+          "type": "boolean",
+          "description": "Only return iterated logical file metadata for files that user has scope permission to access"
         }
       }
     },
@@ -778,7 +787,7 @@
     },
     "dali": {
       "type": "object",
-      "required": [ "name" ],
+      "required": [ "name", "auth" ],
       "additionalProperties": { "type": ["integer", "string", "boolean"] },
       "properties": {
         "name": {
@@ -794,6 +803,12 @@
         "logging": {
           "$ref": "#/definitions/logging"
         },
+        "auth": {
+          "$ref": "#/definitions/auth"
+        },
+        "ldap": {
+          "$ref": "#/definitions/ldap"
+        },
         "annotations": {
           "type": "object",
           "additionalProperties": { "type": "string" }
@@ -903,7 +918,7 @@
     },
     "esp": {
       "type": "object",
-      "required": [ "name", "service" ],
+      "required": [ "name", "service", "auth" ],
       "additionalProperties": { "type": ["integer", "string", "boolean"] },
       "properties": {
         "name": {
@@ -933,6 +948,9 @@
           "type": "object",
           "additionalProperties": { "type": "string" }
         },
+        "auth": {
+          "$ref": "#/definitions/auth"
+        },
         "ldap": {
           "$ref": "#/definitions/ldap"
         },

helm/hpcc/values.yaml

@@ -308,6 +308,7 @@ dafilesrv:
 
 dali:
 - name: mydali
+  auth: none
   services: # internal house keeping services
     coalescer:
       service:

testing/helm/tests/env.yaml

@@ -1,5 +1,6 @@
 esp:
 - name: eclwatch
+  auth: none
   application: eclwatch
   service:
     servicePort: 8880
@@ -74,6 +75,7 @@ sasha:
 
 dali:
 - name: mydali
+  auth: none
   services: # internal house keeping services
     coalescer:
   env:

testing/helm/tests/labels.yaml

@@ -1,5 +1,6 @@
 esp:
 - name: eclwatch
+  auth: none
   application: eclwatch
   service:
     servicePort: 8880
@@ -85,6 +86,7 @@ sasha:
 
 dali:
 - name: mydali
+  auth: none
   annotations:
     da1: dav1
   labels: