HPCC-24429 Support Decoupled Logging in ESP logging manager

1. Configure logging manager not add log request to agent;
2. Also let logging agent not pick up log request from the
FailSafe folder.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

esp/logging/logginglib/logthread.cpp

@@ -27,6 +27,7 @@ const char* const PropMaxLogQueueLength = "MaxLogQueueLength";
 const char* const PropQueueSizeSignal = "QueueSizeSignal";
 const char* const PropMaxTriesRS = "MaxTriesRS";
 const char* const PropFailSafe = "FailSafe";
+const char* const PropDisableFailSafe = "DisableFailSafe";
 const char* const PropAckedFiles = "AckedFiles";
 const char* const PropDefaultAckedFiles = "AckedFiles";
 const char* const PropAckedLogRequests = "AckedLogRequests";
@@ -73,9 +74,14 @@ CLogThread::CLogThread(IPropertyTree* _cfg , const char* _service, const char* _
     maxLogQueueLength = _cfg->getPropInt(PropMaxLogQueueLength, MaxLogQueueLength);
     signalGrowingQueueAt = _cfg->getPropInt(PropQueueSizeSignal, QueueSizeSignal);
     maxLogRetries = _cfg->getPropInt(PropMaxTriesRS, DefaultMaxTriesRS);
-    ensureFailSafe = _cfg->getPropBool(PropFailSafe);
+    //For decoupled logging, the fail safe is not needed because the logging agent always
+    //picks up the logging requests from tank file.
+    ensureFailSafe = _cfg->getPropBool(PropFailSafe) && !_cfg->getPropBool(PropDisableFailSafe, false);
     if(ensureFailSafe)
+    {
         logFailSafe.setown(createFailSafeLogger(_cfg, _service, _agentName));
+        PROGLOG("FailSafe ensured for %s", agentName.get());
+    }
 
     time_t tNow;
     time(&tNow);

esp/logging/loggingmanager/loggingmanager.cpp

@@ -40,8 +40,9 @@ bool CLoggingManager::init(IPropertyTree* cfg, const char* service)
         return false;
     }
 
+    decoupledLogging = cfg->getPropBool("DecoupledLogging", false);
     oneTankFile = cfg->getPropBool("FailSafe", true);
-    if (oneTankFile)
+    if (oneTankFile || decoupledLogging)
     {
         logFailSafe.setown(createFailSafeLogger(cfg, service, cfg->queryProp("@name")));
         logContentFilter.readAllLogFilters(cfg);
@@ -227,7 +228,7 @@ bool CLoggingManager::updateLog(IEspContext* espContext, IEspUpdateLogRequestWra
         if (espContext)
             espContext->addTraceSummaryTimeStamp(LogMin, "LMgr:startQLog");
 
-        if (oneTankFile)
+        if (oneTankFile || decoupledLogging)
         {
             Owned<CLogRequestInFile> reqInFile = new CLogRequestInFile();
             if (!saveToTankFile(req, reqInFile))
@@ -245,12 +246,15 @@ bool CLoggingManager::updateLog(IEspContext* espContext, IEspUpdateLogRequestWra
             Owned<IEspUpdateLogRequest> logRequest = new CUpdateLogRequest("", "");
             logRequest->setOption(reqInFile->getOption());
             logRequest->setLogContent(logContent);
-            for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
+            if (!decoupledLogging)
             {
-                IUpdateLogThread* loggingThread = loggingAgentThreads[x];
-                if (loggingThread->hasService(LGSTUpdateLOG))
+                for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
                 {
-                    loggingThread->queueLog(logRequest);
+                    IUpdateLogThread* loggingThread = loggingAgentThreads[x];
+                    if (loggingThread->hasService(LGSTUpdateLOG))
+                    {
+                        loggingThread->queueLog(logRequest);
+                    }
                 }
             }
         }

esp/logging/loggingmanager/loggingmanager.hpp

@@ -76,7 +76,7 @@ class CLoggingManager : implements ILoggingManager, public CInterface
 {
     typedef std::vector<IUpdateLogThread*> LOGGING_AGENTTHREADS;
     LOGGING_AGENTTHREADS  loggingAgentThreads;
-    bool oneTankFile = false, initialized;
+    bool oneTankFile = false, decoupledLogging = false, initialized = false;
     Owned<ILogFailSafe> logFailSafe;
     CLogContentFilter logContentFilter;
 

initfiles/componentfiles/configxml/@temp/esp_service_DynamicESDL.xsl

@@ -72,6 +72,9 @@
                 </xsl:if>
 
                 <LoggingManager name="{$managerNode/@name}">
+                    <xsl:if test="string($managerNode/@DecoupledLogging) != ''">
+                        <DecoupledLogging><xsl:value-of select="$managerNode/@DecoupledLogging"/></DecoupledLogging>
+                    </xsl:if>
                     <xsl:if test="string($managerNode/@FailSafe) != ''">
                         <FailSafe><xsl:value-of select="$managerNode/@FailSafe"/></FailSafe>
                     </xsl:if>
@@ -91,6 +94,7 @@
                         <xsl:variable name="agentName" select="@ESPLoggingAgent"/>
                         <xsl:variable name="espLoggingAgentNode" select="/Environment/Software/ESPLoggingAgent[@name=$agentName]"/>
                         <xsl:variable name="wsLogServiceESPAgentNode" select="/Environment/Software/WsLogServiceESPAgent[@name=$agentName]"/>
+                        <xsl:variable name="disableFailSafe" select="$managerNode/@DecoupledLogging"/>
                         <xsl:choose>
                             <xsl:when test="($espLoggingAgentNode)">
                                 <xsl:call-template name="ESPLoggingAgent">
@@ -101,6 +105,7 @@
                             <xsl:when test="($wsLogServiceESPAgentNode)">
                                 <xsl:call-template name="WsLogServiceESPAgent">
                                     <xsl:with-param name="agentName" select="$agentName"/>
+                                    <xsl:with-param name="disableFailSafe" select="$disableFailSafe"/>
                                     <xsl:with-param name="agentNode" select="$wsLogServiceESPAgentNode"/>
                                 </xsl:call-template>
                             </xsl:when>

initfiles/componentfiles/configxml/@temp/wslogserviceespagent.xsl

@@ -27,6 +27,7 @@ xmlns:set="http://exslt.org/sets">
     <xsl:template name="WsLogServiceESPAgent" type="DefaultLoggingAgent">
         <xsl:param name="agentName"/>
         <xsl:param name="agentNode"/>
+        <xsl:param name="disableFailSafe"/>
         <xsl:if test="not($agentNode)">
             <xsl:message terminate="yes">An WsLogService ESP Logging Agent <xsl:value-of select="$agentName"/> is undefined!</xsl:message>
         </xsl:if>
@@ -56,6 +57,9 @@ xmlns:set="http://exslt.org/sets">
             <xsl:call-template name="EspLoggingAgentBasic">
                 <xsl:with-param name="agentNode" select="$agentNode"/>
             </xsl:call-template>
+            <xsl:if test="string($disableFailSafe) != ''">
+                <DisableFailSafe><xsl:value-of select="$disableFailSafe"/></DisableFailSafe>
+            </xsl:if>
             <xsl:if test="string($agentNode/@TransactionSeedType) != ''">
                 <TransactionSeedType><xsl:value-of select="$agentNode/@TransactionSeedType"/></TransactionSeedType>
             </xsl:if>

initfiles/componentfiles/configxml/loggingmanager.xsd

@@ -43,6 +43,13 @@
           </xs:appinfo>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="DecoupledLogging" type="xs:boolean" use="optional" default="false">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Enable Decoupled Logging functionality.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="FailSafe" type="xs:boolean" use="optional" default="true">
         <xs:annotation>
           <xs:appinfo>