Merge branch 'candidate-6.2.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

rtl/eclrtl/eclrtl.cpp

@@ -4495,7 +4495,7 @@ int rtlCompareUtf8Utf8Strength(size32_t llen, const char * left, size32_t rlen,
     return rtlCompareUnicodeUnicodeStrength(llen, uleft.getustr(), rlen, uright.getustr(), locale, strength);
 }
 
-void rtlDecPushUtf8(size32_t len, const void * data)
+void rtlDecPushUtf8(size32_t len, const char * data)
 {
     DecPushString(len, (const char *)data); // good enough for the moment
 }

rtl/eclrtl/eclrtl.hpp

@@ -652,7 +652,7 @@ ECLRTL_API void rtlUtf8ToUtf8(size32_t outlen, char * out, size32_t inlen, const
 ECLRTL_API void rtlUtf8ToUtf8X(size32_t & outlen, char * & out, size32_t inlen, const char *in);
 ECLRTL_API int rtlCompareUtf8Utf8(size32_t llen, const char * left, size32_t rlen, const char * right, const char * locale);
 ECLRTL_API int rtlCompareUtf8Utf8Strength(size32_t llen, const char * left, size32_t rlen, const char * right, const char * locale, unsigned strength);
-ECLRTL_API void rtlDecPushUtf8(size32_t len, const void * data);
+ECLRTL_API void rtlDecPushUtf8(size32_t len, const char * data);
 ECLRTL_API bool rtlUtf8ToBool(size32_t inlen, const char * in);
 ECLRTL_API __int64 rtlUtf8ToInt(size32_t inlen, const char * in);
 ECLRTL_API double rtlUtf8ToReal(size32_t inlen, const char * in);

system/security/LdapSecurity/ldapconnection.cpp

@@ -127,92 +127,105 @@ public:
     }
 };
 
-class CLoadBalancer : public CInterface
+
+class CHostManager
 {
 private:
-    StringArray hostArray;
-    unsigned    nextIndex;
-    Mutex       m_mutex;
+    StringArray m_hostArray;
+    Mutex       m_HMMutex;
+    unsigned    m_curHostIdx;
+    bool        m_populated;
 
 public:
-    CLoadBalancer(const char* addrlist)
+    CHostManager()
+    {
+        m_populated = false;
+    }
+
+    void populateHosts(const char* addrlist)
     {
-        char *copyFullText = strdup(addrlist);
-        char *saveptr;
-        char *ip = strtok_r(copyFullText, "|", &saveptr);
-        while (ip != NULL)
+        if (m_populated)
+            return;
+
+        synchronized block(m_HMMutex);
+        if (!m_populated)
         {
-            if (isdigit(*ip))
+            char *copyFullText = strdup(addrlist);
+            char *saveptr;
+            char *ip = strtok_r(copyFullText, "|", &saveptr);
+            while (ip != NULL)
             {
-                char *dash = strrchr(ip, '-');
-                if (dash)
+                if (isdigit(*ip))
                 {
-                    *dash = 0;
-                    int last = atoi(dash+1);
-                    char *dot = strrchr(ip, '.');
-                    *dot = 0;
-                    int first = atoi(dot+1);
-                    for (int i = first; i <= last; i++)
+                    char *dash = strrchr(ip, '-');
+                    if (dash)
                     {
-                        StringBuffer t;
-                        t.append(ip).append('.').append(i);
-                        hostArray.append(t.str());
+                        *dash = 0;
+                        int last = atoi(dash+1);
+                        char *dot = strrchr(ip, '.');
+                        *dot = 0;
+                        int first = atoi(dot+1);
+                        for (int i = first; i <= last; i++)
+                        {
+                            StringBuffer t;
+                            t.append(ip).append('.').append(i);
+                            m_hostArray.append(t.str());
+                        }
+                    }
+                    else
+                    {
+                        m_hostArray.append(ip);
                     }
                 }
                 else
                 {
-                    hostArray.append(ip);
+                    m_hostArray.append(ip);
                 }
+                DBGLOG("Added ldap server %s", m_hostArray.item(m_hostArray.ordinality()-1));
+                ip = strtok_r(NULL, "|", &saveptr);
             }
-            else
+            free(copyFullText);
+
+            if(m_hostArray.length() == 0)
             {
-                hostArray.append(ip);
+                throw MakeStringException(-1, "No valid ldap server address specified");
             }
-            ip = strtok_r(NULL, "|", &saveptr);
-        }
-        free(copyFullText);
-
-        if(hostArray.length() == 0)
-        {
-            throw MakeStringException(-1, "No valid ldap server address specified");
-        }
-
-        Owned<IRandomNumberGenerator> random = createRandomNumberGenerator();
-        random->seed((unsigned)get_cycles_now());
 
-        unsigned i = hostArray.ordinality();
-        while (i > 1)
-        {
-            unsigned j = random->next() % i;
-            i--;
-            hostArray.swap(i, j);
+            m_curHostIdx = 0;
+            m_populated = true;
         }
+    }
 
-        nextIndex = 0;
-
-        for(unsigned ind = 0; ind < hostArray.length(); ind++)
-        {
-            DBGLOG("Added ldap server %s", hostArray.item(ind));
-        }
+    int queryNumHosts()
+    {
+        return m_hostArray.ordinality();
     }
 
-    virtual ~CLoadBalancer()
+    const char * queryCurrentHost()
     {
+        synchronized block(m_HMMutex);
+        return m_hostArray.item(m_curHostIdx);
     }
 
-    const char* next()
+    void blacklistHost(const char * blockedHost)
     {
-        unsigned curindex = 0;
-        
+        if (m_hostArray.ordinality() == 1)
+        {
+            DBGLOG("Cannot blacklist the only configured ldap server %s", m_hostArray.item(m_curHostIdx));
+            return;
+        }
+
+        //If blockedHost is not already blacklisted, do so
+        synchronized block(m_HMMutex);
+        if (0 == strcmp(blockedHost, m_hostArray.item(m_curHostIdx)))
         {
-            synchronized block(m_mutex);
-            curindex = nextIndex;
-            ++nextIndex %= hostArray.ordinality();
+            DBGLOG("Blacklisting ldap server %s", m_hostArray.item(m_curHostIdx));
+            if (++m_curHostIdx == m_hostArray.ordinality())
+                m_curHostIdx = 0;//start over at begin of host array
         }
 
-        return (hostArray.item(curindex));
     }
-};
+} s_hostManager;
 
 
 inline bool LdapServerDown(int rc)
@@ -228,7 +241,6 @@ private:
 
     Owned<IPropertyTree> m_cfg;
 
-    Owned<CLoadBalancer> m_ldaphosts;
     int                  m_ldapport;
     int                  m_ldap_secure_port;
     StringBuffer         m_protocol;
@@ -295,7 +307,7 @@ public:
         {
             throw MakeStringException(-1, "ldapAddress not found in config");
         }
-        m_ldaphosts.setown(new CLoadBalancer(hostsbuf.str()));
+        s_hostManager.populateHosts(hostsbuf.str());
 
         cfg->getProp(".//@ldapProtocol", m_protocol);
         if(m_protocol.length() == 0)
@@ -317,24 +329,31 @@ public:
         else
             m_ldap_secure_port = atoi(portbuf.str());
 
+        int rc;
         StringBuffer hostbuf, dcbuf;
-        int rc = 0;
-        getLdapHost(hostbuf);
-        for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++)
+        const char * ldapDomain = cfg->queryProp(".//@ldapDomain");
+        for (int numHosts=0; numHosts < getHostCount(); numHosts++)
         {
-            rc = LdapUtils::getServerInfo(hostbuf.str(), m_ldapport, dcbuf, m_serverType, cfg->queryProp(".//@ldapDomain"));
-            if(!LdapServerDown(rc) || retries > LDAPSEC_MAX_RETRIES)
-                break;
-            sleep(LDAPSEC_RETRY_WAIT);
-            if(retries < LDAPSEC_MAX_RETRIES)
+            getLdapHost(hostbuf);
+            for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++)
             {
-                DBGLOG("Server %s temporarily unreachable.", hostbuf.str());
-                // Retrying next ldap sever, might be the same server
-                hostbuf.clear();
-                getLdapHost(hostbuf);
-                DBGLOG("Retrying with %s...", hostbuf.str());
+                rc = LdapUtils::getServerInfo(hostbuf.str(), m_ldapport, dcbuf, m_serverType, ldapDomain);
+                if(!LdapServerDown(rc) || retries >= LDAPSEC_MAX_RETRIES)
+                    break;
+                sleep(LDAPSEC_RETRY_WAIT);
+                if(retries < LDAPSEC_MAX_RETRIES)
+                {
+                    DBGLOG("Server %s temporarily unreachable, retrying...", hostbuf.str());
+                }
+            }
+            if (rc != LDAP_SUCCESS)
+            {
+                blacklistHost(hostbuf);
             }
+            else
+                break;
         }
+
         if(rc != LDAP_SUCCESS)
         {
             throw MakeStringException(-1, "getServerInfo error - %s", ldap_err2string(rc));
@@ -492,28 +511,22 @@ public:
         return m_sdfieldname.str();
     }
 
-    virtual StringBuffer& getLdapHost(StringBuffer& hostbuf)
+    virtual int getHostCount()
     {
-        hostbuf.clear();
-        try
-        {
-            hostbuf.append(m_ldaphosts->next());
-        }
-        catch(IException* e)
-        {
-            StringBuffer emsg;
-            e->errorMessage(emsg);
-            DBGLOG("getLdapHost exception - %s", emsg.str());
-            e->Release();
-        }
-        catch(...)
-        {
-            DBGLOG("getLdapHost unknown exception");
-        }
+        return s_hostManager.queryNumHosts();
+    }
 
+    virtual StringBuffer& getLdapHost(StringBuffer& hostbuf)
+    {
+        hostbuf.set(s_hostManager.queryCurrentHost());
         return hostbuf;
     }
 
+    virtual void blacklistHost(const char * host)
+    {
+        s_hostManager.blacklistHost(host);
+    }
+
     virtual void markDown(const char* ldaphost)
     {
         //SocketEndpoint ep(ldaphost, 0);
@@ -738,9 +751,6 @@ private:
 public:
     virtual bool connect(bool force_ssl = false)
     {
-        StringBuffer hostbuf;
-        m_ldapconfig->getLdapHost(hostbuf);
-        int rc = LDAP_SERVER_DOWN;
 
         const char* proto;
         if(force_ssl)
@@ -754,29 +764,41 @@ public:
             m_useSSL = (0 == stricmp(proto, "ldaps") ? true : false);
         }
 
-        for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++)
+        int rc;
+        StringBuffer hostbuf;
+        for (int numHosts=0; numHosts < m_ldapconfig->getHostCount(); numHosts++)
         {
-            rc = connect(hostbuf.str(), proto);
-            if(!LdapServerDown(rc) || retries > LDAPSEC_MAX_RETRIES)
-                break;
-            sleep(LDAPSEC_RETRY_WAIT);
-            if(retries < LDAPSEC_MAX_RETRIES)
-                DBGLOG("Server temporarily unreachable, retrying ...");
-            // Retrying next ldap sever, might be the same server
-            hostbuf.clear();
             m_ldapconfig->getLdapHost(hostbuf);
-        }
 
-        if(rc == LDAP_SERVER_DOWN)
-        {
-            StringBuffer dc;
-            LdapUtils::getDcName(m_ldapconfig->getDomain(), dc);
-            if(dc.length() > 0)
+            for(int retries = 0; retries <= LDAPSEC_MAX_RETRIES; retries++)
             {
-                WARNLOG("Using automatically obtained LDAP Server %s", dc.str());
-                rc = connect(dc.str(), proto);
+                rc = connect(hostbuf.str(), proto);
+                if(!LdapServerDown(rc) || retries > LDAPSEC_MAX_RETRIES)
+                    break;
+                sleep(LDAPSEC_RETRY_WAIT);
+                if(retries < LDAPSEC_MAX_RETRIES)
+                    DBGLOG("Server temporarily unreachable, retrying ...");
             }
+
+            if(rc == LDAP_SERVER_DOWN)
+            {
+                StringBuffer dc;
+                LdapUtils::getDcName(m_ldapconfig->getDomain(), dc);
+                if(dc.length() > 0)
+                {
+                    WARNLOG("Using automatically obtained LDAP Server %s", dc.str());
+                    rc = connect(dc.str(), proto);
+                }
+            }
+
+            if (rc != LDAP_SUCCESS)
+            {
+                m_ldapconfig->blacklistHost(hostbuf);
+            }
+            else
+                break;
         }
+
         if(rc == LDAP_SUCCESS)
             return true;
         else
@@ -1614,14 +1636,6 @@ public:
             {
                 DBGLOG("LdapBind for user %s (retries=%d).", username, retries);
                 {
-#ifdef NULL_DALIUSER_STACKTRACE
-                    //following debug code to be removed
-                    if (!username)
-                    {
-                        DBGLOG("UNEXPECTED USER (NULL) in ldapconnection.cpp authenticate() line %d", __LINE__);
-                        PrintStackReport();
-                    }
-#endif
                     LDAP* user_ld = LdapUtils::LdapInit(m_ldapconfig->getProtocol(), hostbuf.str(), m_ldapconfig->getLdapPort(), m_ldapconfig->getLdapSecurePort());
                     rc = LdapUtils::LdapBind(user_ld, m_ldapconfig->getDomain(), username, password, userdnbuf.str(), m_ldapconfig->getServerType(), m_ldapconfig->getAuthMethod());
                     if(rc != LDAP_SUCCESS)
@@ -1635,7 +1649,6 @@ public:
                 if(retries < LDAPSEC_MAX_RETRIES)
                     DBGLOG("Server temporarily unreachable, retrying ...");
                 // Retrying next ldap sever, might be the same server
-                hostbuf.clear();
                 m_ldapconfig->getLdapHost(hostbuf);
             }
 

system/security/LdapSecurity/ldapconnection.hpp

@@ -81,7 +81,11 @@ ldap_compare_ext_s LDAP_P((
     typedef struct timeval TIMEVAL;
 #endif
 
-#define LDAPTIMEOUT 60 //20 second connection/search timeout
+#ifdef _DEBUG
+	#define LDAPTIMEOUT 5
+#else
+	#define LDAPTIMEOUT 20
+#endif
 #define DEFAULT_LDAP_POOL_SIZE 10
 
 // 1 for ActiveDirectory, 2 for iPlanet, 3 for openLdap
@@ -161,6 +165,8 @@ interface ILdapConfig : extends IInterface
     virtual LdapServerType getServerType() = 0;
     virtual const char * getCfgServerType() const = 0;
     virtual StringBuffer& getLdapHost(StringBuffer& hostbuf) = 0;
+    virtual int getHostCount() = 0;
+    virtual void blacklistHost(const char * host) = 0;
     virtual void markDown(const char* ldaphost) = 0;
     virtual int getLdapPort() = 0;
     virtual int getLdapSecurePort() = 0;