HPCC-23880 Establish contention on /Sessions/Process if timeout

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

esp/bindings/http/platform/httpbinding.cpp

@@ -329,7 +329,7 @@ EspHttpBinding::EspHttpBinding(IPropertyTree* tree, const char *bindname, const
 void EspHttpBinding::setSDSSession()
 {
     espSessionSDSPath.setf("%s/%s[@name=\"%s\"]", PathSessionRoot, PathSessionProcess, processName.get());
-    Owned<IRemoteConnection> conn = querySDS().connect(espSessionSDSPath.str(), myProcessSession(), RTM_LOCK_WRITE, SESSION_SDS_LOCK_TIMEOUT);
+    Owned<IRemoteConnection> conn = getSDSConnectionWithRetry(espSessionSDSPath, RTM_LOCK_WRITE, SDSSESSION_CONNECT_TIMEOUTMS);
     if (!conn)
         throw MakeStringException(-1, "Failed to connect SDS ESP Session.");
 

esp/platform/espcfg.cpp

@@ -125,7 +125,7 @@ int CSessionCleaner::run()
         {
             if (!m_isDetached)
             {
-                Owned<IRemoteConnection> conn = querySDS().connect(espSessionSDSPath.get(), myProcessSession(), RTM_LOCK_WRITE, SESSION_SDS_LOCK_TIMEOUT);
+                Owned<IRemoteConnection> conn = getSDSConnectionWithRetry(espSessionSDSPath.get(), RTM_LOCK_WRITE, SDSSESSION_CONNECT_TIMEOUTMS);
                 if (!conn)
                     throw MakeStringException(-1, "Failed to connect to %s.", PathSessionRoot);
 
@@ -221,7 +221,7 @@ void CEspConfig::ensureSDSSessionDomains()
     //Ensure SDS Session tree if there is session auth or there is no AuthDomain setting (ex. old environment.xml)
     if (hasSessionAuth || !hasAuthDomainSettings)
     {
-        Owned<IRemoteConnection> conn = querySDS().connect(PathSessionRoot, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SESSION_SDS_LOCK_TIMEOUT);
+        Owned<IRemoteConnection> conn = getSDSConnectionWithRetry(PathSessionRoot, RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDSSESSION_CONNECT_TIMEOUTMS);
         if (!conn)
             throw MakeStringException(-1, "Failed to connect to %s.", PathSessionRoot);
 

esp/platform/espcontext.cpp

@@ -27,6 +27,7 @@
 #include "espprotocol.hpp"
 #include "espsecurecontext.hpp"
 #include "ldapsecurity.ipp"
+#include "dasds.hpp"
 
 class CEspContext : public CInterface, implements IEspContext
 {
@@ -1018,3 +1019,28 @@ IEspServer* queryEspServer()
 {
     return dynamic_cast<IEspServer*>(getESPContainer());
 }
+
+IRemoteConnection* getSDSConnectionWithRetry(const char* xpath, unsigned mode, unsigned timeoutMs)
+{
+    CTimeMon timer(timeoutMs);
+    unsigned remaining;
+    while (!timer.timedout(&remaining))
+    {
+        try
+        {
+            unsigned connTimeoutMs = remaining > SESSION_SDS_LOCK_TIMEOUT ? SESSION_SDS_LOCK_TIMEOUT : remaining;
+            Owned<IRemoteConnection> conn = querySDS().connect(xpath, myProcessSession(), mode, connTimeoutMs);
+            if (!conn)
+                throw MakeStringException(-1, "getSDSConnectionWithRetry() : unabled to establish connection to : %s", xpath);
+            return conn.getClear();
+        }
+        catch (ISDSException* e)
+        {
+            if (SDSExcpt_LockTimeout != e->errorCode())
+                throw;
+            IERRLOG(e, "getSDSConnectionWithRetry()");
+            e->Release();
+        }
+    }
+    return nullptr;
+}

esp/platform/espcontext.hpp

@@ -145,5 +145,9 @@ esp_http_decl const char* getBuildVersion();
 esp_http_decl void setBuildLevel(const char* buildLevel);
 esp_http_decl const char* getBuildLevel();
 esp_http_decl IEspServer* queryEspServer();
+
+#define SDSSESSION_CONNECT_TIMEOUTMS (180*1000)
+interface IRemoteConnection;
+esp_http_decl IRemoteConnection* getSDSConnectionWithRetry(const char* xpath, unsigned mode, unsigned timeoutMs);
 #endif