HPCC-9215 DALI should not check LDAP for non managed scopes

Currently dali makes time consuming calls to LDAP to check the permissions
for every single file, even when that file not within a managed scope.
This was causing dali to delay workunits that used to run in seconds,
to over an hour or more. Additionally, DALI would call out to LDAP to check
a scope, even if that scope was a child of a scope with known permissions.

This fix corrects behavior in two ways.

First, checks if the scope is managed (if it was added to the ESP "FileScopes"
permissions), and if not, assign the default LDAP permission without a call
to LDAP.

Second, if it is part of a managed scope (exact match or a child of another
managed scope) then returns the cached permissions of the closest parent
scope, with an early exit in case of a DENY permission. If the scope is not
in the cache, a call to LDAP is made and the results are added to the cache
for subsequent calls.

Signed-off-by: William Whitehead <william.whitehead@lexisnexis.com>

Whitespace fixes

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

system/security/LdapSecurity/ldapconnection.cpp

@@ -5178,6 +5178,32 @@ private:
         Owned<ISecResource> resource = new CLdapSecResource(resName.str());
         return addResource(RT_FILE_SCOPE, user, resource, PT_ADMINISTRATORS_AND_USER, m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE));
     }
+
+    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes)
+    {
+        getResourcesEx(RT_FILE_SCOPE, m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE), NULL, NULL, scopes);
+        return scopes.length();
+    }
+
+    virtual int queryDefaultPermission(ISecUser& user)
+    {
+        const char* basedn = m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE);
+        if(basedn == NULL || *basedn == '\0')
+        {
+            DBGLOG("corresponding basedn is not defined");
+            return -2;
+        }
+        const char* basebasedn = strchr(basedn, ',') + 1;
+        StringBuffer baseresource;
+        baseresource.append(basebasedn-basedn-4, basedn+3);
+        IArrayOf<ISecResource> base_resources;
+        base_resources.append(*new CLdapSecResource(baseresource.str()));
+        bool baseok = authorizeScope(user, base_resources, basebasedn);
+        if(baseok)
+            return base_resources.item(0).getAccessFlags();
+        else
+            return -2;
+    }
 };
 
 int LdapUtils::getServerInfo(const char* ldapserver, int ldapport, StringBuffer& domainDN, LdapServerType& stype, const char* domainname)

system/security/LdapSecurity/ldapconnection.hpp

@@ -188,6 +188,8 @@ interface ILdapClient : extends IInterface
     virtual ILdapConfig* queryConfig() = 0;
     virtual const char* getPasswordStorageScheme() = 0;
     virtual bool createUserScope(ISecUser& user) = 0;
+    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) = 0;
+    virtual int queryDefaultPermission(ISecUser& user) = 0;
 };
 
 ILdapClient* createLdapClient(IPropertyTree* cfg);

system/security/LdapSecurity/ldapsecurity.cpp

@@ -530,6 +530,7 @@ void CLdapSecManager::init(const char *serviceName, IPropertyTree* cfg)
     int cachetimeout = cfg->getPropInt("@cacheTimeout", 5);
     m_permissionsCache.setCacheTimeout( 60 * cachetimeout);
     m_permissionsCache.setTransactionalEnabled(true);
+    m_permissionsCache.setSecManager(this);
     m_passwordExpirationWarningDays = cfg->getPropInt(".//@passwordExpirationWarningDays", 10); //Default to 10 days
 };
 
@@ -822,9 +823,22 @@ int CLdapSecManager::authorizeFileScope(ISecUser & user, const char * filescope)
     if(filescope == 0 || filescope[0] == '\0')
         return SecAccess_Full;
 
+    StringBuffer managedFilescope;
+    if(m_permissionsCache.isCacheEnabled() && !m_usercache_off)
+    {
+        int accessFlags;
+        //See if file scope in question is managed by LDAP permissions.
+        //  If not, return default file permission (dont call out to LDAP)
+        //  If is, look in cache for permission of longest matching managed scope strings. If found return that permission (no call to LDAP),
+        //  otherwise a call to LDAP "authorizeFileScope" is necessary, specifying the longest matching managed scope string
+        bool gotPerms = m_permissionsCache.queryPermsManagedFileScope(user, filescope, managedFilescope, &accessFlags);
+        if (gotPerms)
+            return accessFlags;
+    }
+
     Owned<ISecResourceList> rlist;
     rlist.setown(createResourceList("FileScope"));
-    rlist->addResource(filescope);
+    rlist->addResource(managedFilescope.length() ? managedFilescope.str() : filescope );
     
     bool ok = authorizeFileScope(user, rlist.get());
     if(ok)
@@ -1230,6 +1244,18 @@ bool CLdapSecManager::createUserScopes()
     return true;
 }
 
+
+aindex_t CLdapSecManager::getManagedFileScopes(IArrayOf<ISecResource>& scopes)
+{
+    return m_ldap_client->getManagedFileScopes(scopes);
+}
+
+int CLdapSecManager::queryDefaultPermission(ISecUser& user)
+{
+    return m_ldap_client->queryDefaultPermission(user);
+}
+
+
 extern "C"
 {
 LDAPSECURITY_API ISecManager * newLdapSecManager(const char *serviceName, IPropertyTree &config)

system/security/LdapSecurity/ldapsecurity.ipp

@@ -247,7 +247,6 @@ public:
     IMPLEMENT_IINTERFACE
 
     CLdapSecResource(const char *name);
-
     void addAccess(int flags);
     void setAccessFlags(int flags);
     virtual void setRequiredAccessFlags(int flags);
@@ -439,6 +438,8 @@ public:
         return m_passwordExpirationWarningDays;
     }
     virtual bool createUserScopes();
+    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes);
+    virtual int queryDefaultPermission(ISecUser& user);
 };
 
 #endif

system/security/shared/basesecurity.hpp

@@ -285,7 +285,8 @@ public:
     }
 
     virtual bool createUserScopes() {UNIMPLEMENTED; return false;}
-
+    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) {UNIMPLEMENTED; }
+    virtual int queryDefaultPermission(ISecUser& user) {UNIMPLEMENTED; }
 protected:
     const char* getServer(){return m_dbserver.toCharArray();}
     const char* getUser(){return m_dbuser.toCharArray();}

system/security/shared/caching.cpp

@@ -178,6 +178,7 @@ void CResPermissionsCache::removeStaleEntries(time_t tstamp)
         }
         m_resAccessMap.erase(cachekey);
     }
+
     m_timestampMap.erase(iBegin, itL);
     m_tLastCleanup = tstamp;
 }
@@ -213,6 +214,8 @@ CPermissionsCache::~CPermissionsCache()
     MapUserCache::const_iterator uiEnd = m_userCache.end(); 
     for (ui = m_userCache.begin(); ui != uiEnd; ui++)
         delete (*ui).second;
+
+    removeAllManagedFileScopes();
 }
 
 int CPermissionsCache::lookup( ISecUser& sec_user, IArrayOf<ISecResource>& resources, 
@@ -378,3 +381,200 @@ void CPermissionsCache::removeFromUserCache(ISecUser& sec_user)
     }
 }
 
+bool CPermissionsCache::addManagedFileScopes(IArrayOf<ISecResource>& scopes)
+{
+    synchronized block(m_managedFileScopesCacheMonitor);
+    ForEachItemIn(x, scopes)
+    {
+        ISecResource* scope = &scopes.item(x);
+        if(!scope)
+            continue;
+        const char* cachekey = scope->getName();
+        if(cachekey == NULL)
+            continue;
+        map<string, ISecResource*>::iterator it = m_managedFileScopesMap.find(cachekey);
+        if (it != m_managedFileScopesMap.end())
+        {
+            ISecResource *res = (*it).second;
+            res->Release();
+            m_managedFileScopesMap.erase(it);
+        }
+#ifdef _DEBUG
+        DBGLOG("Caching Managed File Scope %s",cachekey);
+#endif
+        m_managedFileScopesMap.insert( pair<string, ISecResource*>(cachekey, LINK(scope)));
+    }
+    return true;
+}
+
+inline void CPermissionsCache::removeManagedFileScopes(IArrayOf<ISecResource>& scopes)
+{
+    synchronized block(m_managedFileScopesCacheMonitor);
+    ForEachItemIn(x, scopes)
+    {
+        ISecResource* scope = &scopes.item(x);
+        if(!scope)
+            continue;
+        const char* cachekey = scope->getName();
+        if(cachekey == NULL)
+            continue;
+        map<string, ISecResource*>::iterator it = m_managedFileScopesMap.find(cachekey);
+        if (it != m_managedFileScopesMap.end())
+        {
+            ISecResource *res = (*it).second;
+            res->Release();
+            m_managedFileScopesMap.erase(it);
+        }
+    }
+}
+
+inline void CPermissionsCache::removeAllManagedFileScopes()
+{
+    synchronized block(m_managedFileScopesCacheMonitor);
+    map<string, ISecResource*>::const_iterator cit;
+    map<string, ISecResource*>::const_iterator iEnd = m_managedFileScopesMap.end();
+
+    for (cit = m_managedFileScopesMap.begin(); cit != iEnd; cit++)
+    {
+        ISecResource *res = (*cit).second;
+        res->Release();
+    }
+    m_managedFileScopesMap.clear();
+}
+
+/*
+    if perms set on 'scopeA::scopeB' only and lookup of 'scopeA::scopeB::scopeC::scopeD'
+    need to lookup:
+        'scopeA'
+    no match=>continue
+    match=>continue if read permissions (if no read, implies can't "see" child scopes)
+        'scopeA::scopeB'
+    no match=>continue
+    match=>continue if read permissions (if no read, implies can't "see" child scopes)
+
+    etc. Until full scope path checked, or no read permissions hit on ancestor scope.
+*/
+bool CPermissionsCache::queryPermsManagedFileScope(ISecUser& sec_user, const char * fullScope, StringBuffer& managedScope, int * accessFlags)
+{
+    if (!fullScope || !*fullScope)
+    {
+        *accessFlags = queryDefaultPermission(sec_user);
+        return true;
+    }
+
+    time_t now;
+    time(&now);
+    if (m_secMgr && (0 == m_lastManagedFileScopesRefresh || m_lastManagedFileScopesRefresh < (now - m_cacheTimeout)))
+    {
+        removeAllManagedFileScopes();
+        IArrayOf<ISecResource> scopes;
+        aindex_t count = m_secMgr->getManagedFileScopes(scopes);
+        if (count)
+            addManagedFileScopes(scopes);
+        m_defaultPermission = SecAccess_Unknown;//trigger refresh
+        m_lastManagedFileScopesRefresh = now;
+    }
+
+    if (m_managedFileScopesMap.empty())
+    {
+        *accessFlags = queryDefaultPermission(sec_user);
+        return true;
+    }
+
+    StringArray scopes;
+    {
+        StringBuffer scope;
+        const char * p = fullScope;
+        while (*p)
+        {
+            if (*p == ':')
+            {
+                if (*(p+1) != ':')
+                    return false;//Malformed scope string, let LDAP figure it out
+                scopes.append(scope.str());
+                scope.append(*(p++));
+            }
+            scope.append(*(p++));
+        }
+        scopes.append(scope.str());
+    }
+    synchronized block(m_managedFileScopesCacheMonitor);
+    ISecResource *matchedRes = NULL;
+    ISecResource *res = NULL;
+    bool isManaged = false;
+
+    for(unsigned i = 0; i < scopes.length(); i++)
+    {
+        const char* scope = scopes.item(i);
+        map<string, ISecResource*>::const_iterator it = m_managedFileScopesMap.find(scope);
+        if (it != m_managedFileScopesMap.end())
+        {
+            isManaged = true;
+            res = (*it).second;
+            res->setResourceType(RT_FILE_SCOPE);
+            LINK(res);
+            IArrayOf<ISecResource> secResArr;
+            secResArr.append(*res);
+            bool found;
+            int nFound = lookup(sec_user, secResArr, &found);
+            if (nFound && found)
+            {
+                if (0 == (res->getAccessFlags() & SecAccess_Read))
+                {
+                    *accessFlags = res->getAccessFlags();
+                    managedScope.append(const_cast<char *>(res->getName()));
+#ifdef _DEBUG
+                    DBGLOG("FileScope %s for %s(%s) access denied %d",fullScope, sec_user.getName(), res->getName(), *accessFlags);
+#endif
+                    return true;
+                }
+                else
+                    matchedRes = res;//allowed at this scope, but must also look at child scopes
+            }
+        }
+    }
+    bool rc;
+    if (isManaged)
+    {
+        if (matchedRes)
+        {
+            *accessFlags = matchedRes->getAccessFlags();
+            managedScope.append(const_cast<char *>(matchedRes->getName()));
+#ifdef _DEBUG
+            DBGLOG("FileScope %s for %s(%s) access granted %d", fullScope, sec_user.getName(), matchedRes->getName(), *accessFlags);
+#endif
+            rc = true;
+        }
+        else
+        {
+            managedScope.append(const_cast<char *>(res->getName()));
+
+#ifdef _DEBUG
+            DBGLOG("FileScope %s for %s(%s) managed but not cached", fullScope, sec_user.getName(), res->getName());
+#endif
+            rc = false;//need to go to LDAP to check
+        }
+    }
+    else
+    {
+        *accessFlags = queryDefaultPermission(sec_user);
+#ifdef _DEBUG
+        DBGLOG("FileScope %s for %s not managed, using default %d", fullScope, sec_user.getName(),*accessFlags);
+#endif
+        rc = true;
+    }
+    return rc;
+}
+
+int CPermissionsCache::queryDefaultPermission(ISecUser& user)
+{
+    if (m_defaultPermission == SecAccess_Unknown)
+    {
+        if (m_secMgr)
+            m_defaultPermission = m_secMgr->queryDefaultPermission(user);
+        else
+            m_defaultPermission = SecAccess_None;
+    }
+    return m_defaultPermission;
+
+}

system/security/shared/caching.hpp

@@ -137,10 +137,13 @@ public:
 class CPermissionsCache
 {
 public:
-    CPermissionsCache() 
+    CPermissionsCache()
     { 
         m_cacheTimeout = 300;
         m_transactionalEnabled = false;
+        m_secMgr = NULL;
+        m_lastManagedFileScopesRefresh = 0;
+        m_defaultPermission = SecAccess_Unknown;
     }
     virtual ~CPermissionsCache();
 
@@ -165,7 +168,14 @@ public:
     const int getCacheTimeout() { return m_cacheTimeout; }
     bool  isCacheEnabled() { return m_cacheTimeout > 0; }
     void setTransactionalEnabled(bool enable) { m_transactionalEnabled = enable; }
-    bool isTransactionalEnabled() { return m_transactionalEnabled;} 
+    bool isTransactionalEnabled() { return m_transactionalEnabled;}
+
+    bool addManagedFileScopes(IArrayOf<ISecResource>& scopes);
+    void removeManagedFileScopes(IArrayOf<ISecResource>& scopes);
+    void removeAllManagedFileScopes();
+    bool queryPermsManagedFileScope(ISecUser& sec_user, const char * fullScope, StringBuffer& managedScope, int * accessFlags);
+    void setSecManager(ISecManager * secMgr) { m_secMgr = secMgr; }
+    int  queryDefaultPermission(ISecUser& user);
 private:
 
     typedef std::map<string, CResPermissionsCache*> MapResPermissionsCache;
@@ -180,6 +190,14 @@ private:
 
     MapUserCache m_userCache;
     Monitor m_userCacheMonitor;
+
+
+    //Managed File Scope support
+    int                         m_defaultPermission;
+    map<string, ISecResource*>  m_managedFileScopesMap;
+    Monitor                     m_managedFileScopesCacheMonitor;
+    ISecManager *               m_secMgr;
+    time_t                      m_lastManagedFileScopesRefresh;
 };
 
 time_t getThreadCreateTime();

system/security/shared/defaultsecuritymanager.hpp

@@ -80,6 +80,8 @@ public:
         return true;
     }
     virtual bool createUserScopes() { return false; }
+    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) { return 0; }
+    virtual int queryDefaultPermission(ISecUser& user) { return SecAccess_Full; }
 };
 
 class CLocalSecurityManager : public CDefaultSecurityManager

system/security/shared/seclib.hpp

@@ -296,6 +296,8 @@ interface ISecManager : extends IInterface
     virtual const char * getDescription() = 0;
     virtual unsigned getPasswordExpirationWarningDays() = 0;
     virtual bool createUserScopes() = 0;
+    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) = 0;
+    virtual int queryDefaultPermission(ISecUser& user) = 0;
 };
 
 interface IExtSecurityManager