HPCC-3130 changing perms on eclwatch takes 5 minutes to take affect

Currently if you change permissions via ECLWatch, the ESP cache is updated
but the DALI cache does not update until the timeout period (default 5
minutes, configurable via configmgr). This makes it difficult to test things
like file permissions which are verified by DALI.
This commit adds a new "Flush Permissions Cache" button to the permissions
screens in ECLWatch, and when selected, after presenting a scary prompt,
if the admin user chooses to continue the local (ESP) cache is cleared,
and a message is sent to DALI requesting it to do the same
Signed-off-by: William Whitehead <william.whitehead@lexisnexis.com>

dali/base/dasess.cpp

@@ -77,6 +77,7 @@ interface ISessionManagerServer: implements IConnectionMonitor
     virtual SessionId lookupProcessSession(INode *node) = 0;
     virtual INode *getProcessSessionNode(SessionId id) =0;
     virtual int getPermissionsLDAP(const char *key,const char *obj,IUserDescriptor *udesc,unsigned flags, int *err)=0;
+    virtual bool clearPermissionsCache(IUserDescriptor *udesc) = 0;
     virtual void stopSession(SessionId sessid,bool failed) = 0;
     virtual void setClientAuth(IDaliClientAuthConnection *authconn) = 0;
     virtual void setLDAPconnection(IDaliLdapConnection *_ldapconn) = 0;
@@ -364,6 +365,7 @@ enum MSessionRequestKind {
     MSR_STOP_SESSION,
     MSR_IMPORT_CAPABILITIES,
     MSR_LOOKUP_LDAP_PERMISSIONS,
+    MSR_CLEAR_PERMISSIONS_CACHE,
     MSR_EXIT // TBD
 };
 
@@ -527,6 +529,14 @@ public:
                 coven.reply(mb);
             }
             break;
+        case MSR_CLEAR_PERMISSIONS_CACHE: {
+                Owned<IUserDescriptor> udesc=createUserDescriptor();
+                udesc->deserialize(mb);
+                bool ok = manager.clearPermissionsCache(udesc);
+                mb.append(ok);
+                coven.reply(mb);
+            }
+            break;
         }
     }
 
@@ -800,6 +810,19 @@ public:
         return ret;
     }
 
+    bool clearPermissionsCache(IUserDescriptor *udesc)
+    {
+        if (securitydisabled)
+            return true;
+        if (queryDaliServerVersion().compare("1.8") < 0) {
+            securitydisabled = true;
+            return true;
+        }
+        CMessageBuffer mb;
+        mb.append((int)MSR_CLEAR_PERMISSIONS_CACHE);
+        udesc->serialize(mb);
+        return queryCoven().sendRecv(mb,RANK_RANDOM,MPTAG_DALI_SESSION_REQUEST,SESSIONREPLYTIMEOUT);
+    }
     bool checkScopeScansLDAP()
     {
         assertex(!"checkScopeScansLDAP called on client");
@@ -1265,6 +1288,17 @@ public:
 #endif
     }
 
+    virtual bool clearPermissionsCache(IUserDescriptor *udesc)
+    {
+#ifdef _NO_LDAP
+        bool ok = true;
+#else
+        bool ok = true;
+        if (ldapconn->getLDAPflags() & DLF_ENABLED)
+            ok = ldapconn->clearPermissionsCache(udesc);
+#endif
+        return ok;
+    }
     virtual bool checkScopeScansLDAP()
     {
 #ifdef _NO_LDAP

dali/base/dasess.hpp

@@ -110,6 +110,7 @@ interface ISessionManager: extends IInterface
     virtual bool checkScopeScansLDAP()=0;
     virtual unsigned getLDAPflags()=0;
     virtual void setLDAPflags(unsigned flags)=0;
+    virtual bool clearPermissionsCache(IUserDescriptor *udesc)=0;
 
 };
 

dali/server/daldap.cpp

@@ -175,6 +175,18 @@ public:
         return 255;
     }
 
+    bool clearPermissionsCache(IUserDescriptor *udesc)
+    {
+        if (!ldapsecurity || ((getLDAPflags() & DLF_ENABLED) == 0))
+            return true;
+        StringBuffer username;
+        StringBuffer password;
+        udesc->getUserName(username);
+        udesc->getPassword(password);
+        Owned<ISecUser> user = ldapsecurity->createUser(username);
+        user->credentials().setPassword(password);
+        return ldapsecurity->clearPermissionsCache(*user);
+    }
     bool checkScopeScans()
     {
         return (ldapflags&DLF_SCOPESCANS)!=0;

dali/server/daldap.hpp

@@ -30,7 +30,7 @@ interface IDaliLdapConnection: extends IInterface
     virtual bool checkScopeScans() = 0;
     virtual unsigned getLDAPflags() = 0;
     virtual void setLDAPflags(unsigned flags) = 0;
-
+    virtual bool clearPermissionsCache(IUserDescriptor *udesc) = 0;
 };
 
 extern IDaliLdapConnection *createDaliLdapConnection(IPropertyTree *proptree);

esp/eclwatch/ws_XSLT/CMakeLists.txt

@@ -138,6 +138,7 @@ FOREACH ( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/access_accountpermissions.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/access_adduser.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/access_basedns.xslt
+    ${CMAKE_CURRENT_SOURCE_DIR}/access_clearpermissionscache.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/access_filepermission.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/access_groupadd.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/access_groupdelete.xslt

esp/eclwatch/ws_XSLT/access_clearpermissionscache.xslt

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:fo="http://www.w3.org/1999/XSL/Format">
+  <xsl:output method="html"/>
+  <xsl:template match="/">
+    <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+      <head>
+        <script type="text/javascript" src="/esp/files/scripts/espdefault.js">&#160;</script>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+        <title>Permissions Cache</title>
+      </head>
+      <body class="yui-skin-sam" onload="nof5();">
+        <xsl:apply-templates/>
+      </body>
+    </html>
+  </xsl:template>
+
+  <xsl:template match="ClearPermissionsCacheResponse">
+    <table>
+      <tbody>
+        <th align="left">
+          <h2>Clear Permissions Cache Result</h2>
+        </th>
+        <tr>
+          <td>
+            <xsl:choose>
+              <xsl:when test="retcode=0">
+                Permissions cache cleared.
+              </xsl:when>
+              <xsl:otherwise>
+                Permissions cache clear unsuccessful
+              </xsl:otherwise>
+            </xsl:choose>
+            <xsl:value-of select="retmsg"/>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+
+  </xsl:template>
+</xsl:stylesheet>

esp/eclwatch/ws_XSLT/access_resources.xslt

@@ -188,6 +188,11 @@
               </form>
             </td>
           </xsl:if>
+            <td>
+                <form action="/ws_access/ClearPermissionsCache">
+                  <input id="clearPermissionsCacheBtn" class="sbutton" type="submit" name="action" value="Clear Permissions Cache" onclick="return confirm('Are you sure you want to clear the DALI and ESP permissions caches? Running workunit performance might degrade significantly until the caches have been refreshed.')"/>
+                </form>
+            </td>
                 </tr>
             </table>
       <form action="/ws_access/ResourceAddInput" id="addform">

esp/scm/ws_access.ecm

@@ -553,6 +553,14 @@ ESPresponse PermissionsResetResponse
     string retmsg;
 };
 
+ESPrequest ClearPermissionsCacheRequest
+{
+};
+
+ESPresponse ClearPermissionsCacheResponse
+{
+    int retcode;
+};
 ESPrequest PermissionActionRequest
 {
     string basedn;
@@ -681,6 +689,7 @@ ESPservice [version("1.07"), default_client_version("1.07"), exceptions_inline("
     ESPmethod [client_xslt("/esp/xslt/access_filepermission.xslt")] FilePermission(FilePermissionRequest, FilePermissionResponse);
     ESPmethod [client_xslt("/esp/xslt/access_permissionresetinput.xslt")] PermissionsResetInput(PermissionsResetInputRequest, PermissionsResetInputResponse);
     ESPmethod [client_xslt("/esp/xslt/access_permissionsreset.xslt")] PermissionsReset(PermissionsResetRequest, PermissionsResetResponse);
+    ESPmethod [client_xslt("/esp/xslt/access_clearpermissionscache.xslt")] ClearPermissionsCache(ClearPermissionsCacheRequest, ClearPermissionsCacheResponse);
     //ESPmethod [client_xslt("/esp/xslt/access_useraccountexport.xslt")] UserAccountExport(UserAccountExportRequest, UserAccountExportResponse);
     ESPmethod UserAccountExport(UserAccountExportRequest, UserAccountExportResponse);
 

esp/services/ws_access/CMakeLists.txt

@@ -46,6 +46,8 @@ include_directories (
          ${HPCC_SOURCE_DIR}/esp/bindings
          ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
          ${HPCC_SOURCE_DIR}/esp/smc/SMCLib
+         ${HPCC_SOURCE_DIR}/dali/base
+         ${HPCC_SOURCE_DIR}/system/mp
     )
 
 # NOTE - this should not be needed, it's the result of poor encapsulation and using CLdapSecManager directly 

esp/services/ws_access/ws_accessService.cpp

@@ -21,6 +21,7 @@
 
 #include "ws_accessService.hpp"
 #include "exception_util.hpp"
+#include "dasess.hpp"
 
 #include <set>
 
@@ -1893,6 +1894,30 @@ bool Cws_accessEx::onPermissionsResetInput(IEspContext &context, IEspPermissions
     return true;
 }
 
+bool Cws_accessEx::onClearPermissionsCache(IEspContext &context, IEspClearPermissionsCacheRequest &req, IEspClearPermissionsCacheResponse &resp)
+{
+    ISecManager* secmgr = context.querySecManager();
+    if(secmgr == NULL)
+        throw MakeStringException(ECLWATCH_INVALID_SEC_MANAGER, MSG_SEC_MANAGER_IS_NULL);
+
+    //Clear local cache
+    Owned<ISecUser> user = secmgr->createUser(context.queryUserId());
+    ISecCredentials& cred = user->credentials();
+    cred.setPassword(context.queryPassword());
+    bool ok = secmgr->clearPermissionsCache(*user);
+
+    //Request DALI to clear its cache
+    if (ok)
+    {
+        Owned<IUserDescriptor> userdesc;
+        userdesc.setown(createUserDescriptor());
+        userdesc->set(context.queryUserId(), context.queryPassword());
+        ok = querySessionManager().clearPermissionsCache(userdesc);
+    }
+
+    resp.setRetcode(ok ? 0 : -1);
+    return true;
+}
 bool Cws_accessEx::permissionsReset(CLdapSecManager* ldapsecmgr, const char* basedn, const char* rtype0, const char* prefix,
         const char* resourceName, ACT_TYPE accountType, const char* accountName,
         bool allow_access, bool allow_read, bool allow_write, bool allow_full,

esp/services/ws_access/ws_accessService.hpp

@@ -116,6 +116,7 @@ public:
     virtual bool onPermissionsResetInput(IEspContext &context, IEspPermissionsResetInputRequest &req, IEspPermissionsResetInputResponse &resp);
     virtual bool onPermissionsReset(IEspContext &context, IEspPermissionsResetRequest &req, IEspPermissionsResetResponse &resp);
     virtual bool onUserAccountExport(IEspContext &context, IEspUserAccountExportRequest &req, IEspUserAccountExportResponse &resp);
+    virtual bool onClearPermissionsCache(IEspContext &context, IEspClearPermissionsCacheRequest &req, IEspClearPermissionsCacheResponse &resp);
 };
 
 #endif //_ESPWIZ_ws_access_HPP__

system/security/LdapSecurity/ldapconnection.cpp

@@ -1325,21 +1325,7 @@ public:
 
         if(rtype == RT_FILE_SCOPE)
         {
-            int defaultFileScopePermission = -2;
-            //if(m_defaultFileScopePermission == -2)
-            {
-                const char* basebasedn = strchr(basedn, ',') + 1;
-                StringBuffer baseresource;
-                baseresource.append(basebasedn-basedn-4, basedn+3);
-                IArrayOf<ISecResource> base_resources;
-                base_resources.append(*(new CLdapSecResource(baseresource.str())));
-                bool baseok = authorizeScope(user, base_resources, basebasedn);
-                if(baseok)
-                {
-                    //m_defaultFileScopePermission = base_resources.item(0).getAccessFlags();
-                    defaultFileScopePermission = base_resources.item(0).getAccessFlags();
-                }
-            }
+            int defaultFileScopePermission = queryDefaultPermission(user);
             IArrayOf<ISecResource> non_emptylist;
             ForEachItemIn(x, resources)
             {

system/security/LdapSecurity/ldapsecurity.cpp

@@ -1255,6 +1255,16 @@ int CLdapSecManager::queryDefaultPermission(ISecUser& user)
     return m_ldap_client->queryDefaultPermission(user);
 }
 
+bool CLdapSecManager::clearPermissionsCache(ISecUser& user)
+{
+    if(m_permissionsCache.isCacheEnabled())
+    {
+        if (!authenticate(&user))
+            return false;
+        m_permissionsCache.flush();
+    }
+    return true;
+}
 
 extern "C"
 {

system/security/LdapSecurity/ldapsecurity.ipp

@@ -440,6 +440,7 @@ public:
     virtual bool createUserScopes();
     virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes);
     virtual int queryDefaultPermission(ISecUser& user);
+    virtual bool clearPermissionsCache(ISecUser &user);
 };
 
 #endif

system/security/shared/basesecurity.hpp

@@ -287,6 +287,7 @@ public:
     virtual bool createUserScopes() {UNIMPLEMENTED; return false;}
     virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) {UNIMPLEMENTED; }
     virtual int queryDefaultPermission(ISecUser& user) {UNIMPLEMENTED; }
+    virtual bool clearPermissionsCache(ISecUser& user) {return false;}
 protected:
     const char* getServer(){return m_dbserver.toCharArray();}
     const char* getUser(){return m_dbuser.toCharArray();}

system/security/shared/caching.cpp

@@ -202,20 +202,7 @@ void CResPermissionsCache::remove(SecResourceType rtype, const char* resourcenam
 
 CPermissionsCache::~CPermissionsCache()
 {
-    //delete all user-specific caches
-    //
-    MapResPermissionsCache::const_iterator i;
-    MapResPermissionsCache::const_iterator iEnd = m_resPermissionsMap.end(); 
-
-    for (i = m_resPermissionsMap.begin(); i != iEnd; i++)
-        delete (*i).second;
-
-    MapUserCache::const_iterator ui;
-    MapUserCache::const_iterator uiEnd = m_userCache.end(); 
-    for (ui = m_userCache.begin(); ui != uiEnd; ui++)
-        delete (*ui).second;
-
-    removeAllManagedFileScopes();
+    flush();
 }
 
 int CPermissionsCache::lookup( ISecUser& sec_user, IArrayOf<ISecResource>& resources, 
@@ -578,3 +565,24 @@ int CPermissionsCache::queryDefaultPermission(ISecUser& user)
     return m_defaultPermission;
 
 }
+void CPermissionsCache::flush()
+{
+    {
+        synchronized block(m_cachemonitor);
+        MapResPermissionsCache::const_iterator i;
+        MapResPermissionsCache::const_iterator iEnd = m_resPermissionsMap.end();
+        for (i = m_resPermissionsMap.begin(); i != iEnd; i++)
+            delete (*i).second;
+        m_resPermissionsMap.clear();
+    }
+    {
+        synchronized block(m_userCacheMonitor);
+        MapUserCache::const_iterator ui;
+        MapUserCache::const_iterator uiEnd = m_userCache.end();
+        for (ui = m_userCache.begin(); ui != uiEnd; ui++)
+            delete (*ui).second;
+        m_userCache.clear();
+    }
+    m_lastManagedFileScopesRefresh = 0;
+    m_defaultPermission = SecAccess_Unknown;//trigger refresh
+}

system/security/shared/caching.hpp

@@ -169,7 +169,7 @@ public:
     bool  isCacheEnabled() { return m_cacheTimeout > 0; }
     void setTransactionalEnabled(bool enable) { m_transactionalEnabled = enable; }
     bool isTransactionalEnabled() { return m_transactionalEnabled;}
-
+    void flush();
     bool addManagedFileScopes(IArrayOf<ISecResource>& scopes);
     void removeManagedFileScopes(IArrayOf<ISecResource>& scopes);
     void removeAllManagedFileScopes();

system/security/shared/seclib.hpp

@@ -298,6 +298,7 @@ interface ISecManager : extends IInterface
     virtual bool createUserScopes() = 0;
     virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) = 0;
     virtual int queryDefaultPermission(ISecUser& user) = 0;
+    virtual bool clearPermissionsCache(ISecUser & user) = 0;
 };
 
 interface IExtSecurityManager