HPCC-21219 Config tank file limit for ESP logging

1. Add new safeRolloverReqThreshold to CLogFailSafe which
controls how many logging requests in a tank file. Its
default value is DEFAULT_SAFE_ROLLOVER_THRESHOLD;
2. Add new safeRolloverSizeThreshold to CLogFailSafe which
controls tank file size.
3. Pass setting tree to CLogFailSafe so that >1 settings
(including the tank file limit) may be read.
4. Add SafeRolloverThreshold to the xsd files for esp logging
agents;
5. Move multiple existing settings to new esp_logging_agent_
basic.xsl so that the code may be shared by all esp logging
agents;
6. Read and use SafeRolloverThreshold setting to rollover tank
files.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

esp/logging/logginglib/LogFailSafe.cpp

@@ -35,33 +35,34 @@
 #define SENDING   "_sending_"
 
 const char* const RolloverExt=".old";
-const unsigned long SAFE_ROLLOVER_THRESHOLD = 500000L;
 const unsigned int TRACE_PENDING_LOGS_MIN = 10;
 const unsigned int TRACE_PENDING_LOGS_MAX = 50;
 
-extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(const char* logType, const char* logsdir)
+extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(IPropertyTree* cfg, const char* logType)
 {
-    return new CLogFailSafe(logType, logsdir);
+    return new CLogFailSafe(cfg, logType);
 }
 
-extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(const char* pszService, const char* logType, const char* logsdir)
+extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(IPropertyTree* cfg, const char* pszService, const char* logType)
 {
-    return new CLogFailSafe(pszService, logType, logsdir && *logsdir ? logsdir : "./FailSafeLogs");
+    return new CLogFailSafe(cfg, pszService, logType);
 }
 
 CLogFailSafe::CLogFailSafe()
 {
 }
 
-CLogFailSafe::CLogFailSafe(const char* logType, const char* logsdir) : m_LogType(logType), m_logsdir(logsdir)
+CLogFailSafe::CLogFailSafe(IPropertyTree* cfg, const char* logType) : m_LogType(logType)
 {
+    readCfg(cfg);
     loadFailed(logType);
     createNew(logType);
 }
 
-CLogFailSafe::CLogFailSafe(const char* pszService, const char* logType, const char* logsdir)
-    : m_LogService(pszService), m_LogType(logType), m_logsdir(logsdir)
+CLogFailSafe::CLogFailSafe(IPropertyTree* cfg, const char* pszService, const char* logType)
+    : m_LogService(pszService), m_LogType(logType)
 {
+    readCfg(cfg);
     loadPendingLogReqsFromExistingLogFiles();
 
     StringBuffer send, receive;
@@ -78,6 +79,63 @@ CLogFailSafe::~CLogFailSafe()
     m_Cleared.Close();
 }
 
+void CLogFailSafe::readCfg(IPropertyTree* cfg)
+{
+    StringBuffer safeRolloverThreshold = cfg->queryProp(PropSafeRolloverThreshold);
+    if (!safeRolloverThreshold.isEmpty())
+        readSafeRolloverThresholdCfg(safeRolloverThreshold);
+
+    const char* logsDir = cfg->queryProp(PropFailSafeLogsDir);
+    if (!isEmptyString(logsDir))
+        m_logsdir.set(logsDir);
+    else
+        m_logsdir.set(DefaultFailSafeLogsDir);
+}
+
+void CLogFailSafe::readSafeRolloverThresholdCfg(StringBuffer& safeRolloverThreshold)
+{
+    safeRolloverThreshold.trim();
+
+    const char* ptrStart = safeRolloverThreshold.str();
+    const char* ptrAfterDigit = ptrStart;
+    while (*ptrAfterDigit && isdigit(*ptrAfterDigit))
+        ptrAfterDigit++;
+
+    if (!*ptrAfterDigit)
+    {
+        safeRolloverReqThreshold = atol(safeRolloverThreshold.str());
+        return;
+    }
+
+    const char* ptr = ptrAfterDigit;
+    while (*ptr && (ptr[0] == ' '))
+        ptr++;
+    char c = ptr[0];
+    safeRolloverThreshold.setLength(ptrAfterDigit - ptrStart);
+    safeRolloverSizeThreshold = atol(safeRolloverThreshold.str());
+    switch (c)
+    {
+    case 'k':
+    case 'K':
+        safeRolloverSizeThreshold *= 1000;
+        break;
+    case 'm':
+    case 'M':
+        safeRolloverSizeThreshold *= 1000000;
+        break;
+    case 'g':
+    case 'G':
+        safeRolloverSizeThreshold *= 1000000000;
+        break;
+    case 't':
+    case 'T':
+        safeRolloverSizeThreshold *= 1000000000000;
+        break;
+    default:
+        break;
+    }
+}
+
 bool CLogFailSafe::FindOldLogs()
 {
     if(m_UnsentLogs.ordinality())
@@ -232,10 +290,18 @@ void CLogFailSafe::Add(const char* GUID, const StringBuffer& strContents)
 {
     VStringBuffer dataStr("<cache>%s</cache>", strContents.str());
 
-    unsigned long item_count = m_Added.getItemCount();
-    if (item_count > SAFE_ROLLOVER_THRESHOLD)
-        SafeRollover();
-
+    if (safeRolloverSizeThreshold <= 0)
+    {
+        unsigned long item_count = m_Added.getItemCount();
+        if (item_count > safeRolloverReqThreshold)
+            SafeRollover();
+    }
+    else
+    {
+        unsigned long fileSize = m_Added.getFileSize();
+        if (fileSize > safeRolloverSizeThreshold)
+            SafeRollover();
+    }
     m_Added.Append(GUID, dataStr.str());
 }
 

esp/logging/logginglib/LogFailSafe.hpp

@@ -25,6 +25,11 @@
 #include "loggingcommon.hpp"
 #include "LogSerializer.hpp"
 
+const unsigned long DEFAULT_SAFE_ROLLOVER_REQ_THRESHOLD = 500000L;
+const char* const PropSafeRolloverThreshold = "SafeRolloverThreshold";
+const char* const PropFailSafeLogsDir = "FailSafeLogsDir";
+const char* const DefaultFailSafeLogsDir = "./FailSafeLogs";
+
 interface ILogFailSafe : IInterface
 {
     virtual void Add(const char*, const StringBuffer& strContents)=0;//
@@ -43,8 +48,8 @@ interface ILogFailSafe : IInterface
     virtual bool canRollCurrentLog() = 0;
 };
 
-extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(const char* logType="", const char* logsdir="./logs");
-extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(const char* pszService, const char* logType="", const char* logsdir="./logs");
+extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(IPropertyTree* cfg, const char* logType="");
+extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(IPropertyTree* cfg, const char* pszService, const char* logType="");
 
 class CLogFailSafe : implements ILogFailSafe, public CInterface
 {
@@ -58,8 +63,11 @@ class CLogFailSafe : implements ILogFailSafe, public CInterface
 
     CriticalSection m_critSec;//
     GuidMap m_PendingLogs;//
+    unsigned long safeRolloverReqThreshold = DEFAULT_SAFE_ROLLOVER_REQ_THRESHOLD;
+    unsigned long safeRolloverSizeThreshold = 0;
 
-private:
+    void readCfg(IPropertyTree* cfg);
+    void readSafeRolloverThresholdCfg(StringBuffer& safeRolloverThreshold);
     void createNew(const char* logType);
     void loadFailed(const char* logType);
     StringBuffer& getReceiveFileName(const char* sendFileName, StringBuffer& recieveName);
@@ -70,8 +78,8 @@ private:
 public:
     IMPLEMENT_IINTERFACE;
     CLogFailSafe();
-    CLogFailSafe(const char* logType, const char* logsdir);
-    CLogFailSafe(const char* pszService, const char* logType, const char* logsdir);
+    CLogFailSafe(IPropertyTree* cfg, const char* logType);
+    CLogFailSafe(IPropertyTree* cfg, const char* pszService, const char* logType);
 
     virtual ~CLogFailSafe();
     StringBuffer& GenerateGUID(StringBuffer& GUID,const char* seed="");

esp/logging/logginglib/LogSerializer.cpp

@@ -61,6 +61,7 @@ void CLogSerializer::Init()
     m_ItemCount     = 0;
     m_fileio        = 0;
     m_file          = 0;
+    fileSize = 0;
 }
 
 CLogSerializer::~CLogSerializer()
@@ -84,6 +85,7 @@ void CLogSerializer::Append(const char* GUID, const char* Data)
     //optimize
     CriticalBlock b(crit);
     m_ItemCount++;
+    fileSize += toWrite.length();
     m_bytesWritten += m_fileio->write(m_bytesWritten, toWrite.length(), toWrite.str());
 }
 
@@ -146,6 +148,7 @@ void CLogSerializer::Close()
 
     m_bytesWritten  = 0;//
     m_ItemCount     = 0;//
+    fileSize        = 0;
 }
 
 void CLogSerializer::Rollover(const char* ClosedPrefix)
@@ -281,6 +284,7 @@ void CLogSerializer::loadAckedLogs(GuidSet& ackedLogs)//
 
             finger+=dataLen;
         }
+        fileSize = finger;
         DBGLOG("Total acks loaded %lu", m_ItemCount);
     }
     catch(IException* ex)

esp/logging/logginglib/LogSerializer.hpp

@@ -33,6 +33,7 @@ class CLogSerializer : public CInterface
 {
     __int64 m_bytesWritten;
     unsigned long m_ItemCount;//
+    unsigned long fileSize;//
     CriticalSection crit;
 protected:
     IFile* m_file;
@@ -63,6 +64,7 @@ public:
     void SafeRollover(const char* Directory,const char* NewFileName,const char* Prefix, const char* ClosedPrefix);
 
     unsigned long getItemCount() const { return m_ItemCount; }//
+    unsigned long getFileSize() const { return fileSize; }//
     static StringBuffer& extractFileName(const char* fullName, StringBuffer& fileName);//
     virtual void loadSendLogs(GuidSet& ACKSet, GuidMap& MissedLogs, unsigned long& total_missed);//
     virtual void loadAckedLogs(GuidSet& ReceiveMap);//

esp/logging/logginglib/logthread.cpp

@@ -26,7 +26,6 @@ const char* const PropMaxLogQueueLength = "MaxLogQueueLength";
 const char* const PropQueueSizeSignal = "QueueSizeSignal";
 const char* const PropMaxTriesRS = "MaxTriesRS";
 const char* const PropFailSafe = "FailSafe";
-const char* const PropFailSafeLogsDir = "FailSafeLogsDir";
 
 #define     MaxLogQueueLength   500000 //Write a warning into log when queue length is greater than 500000
 #define     QueueSizeSignal     10000 //Write a warning into log when queue length is increased by 10000
@@ -65,13 +64,8 @@ CLogThread::CLogThread(IPropertyTree* _cfg , const char* _service, const char* _
     maxLogRetries = _cfg->getPropInt(PropMaxTriesRS, DefaultMaxTriesRS);
     ensureFailSafe = _cfg->getPropBool(PropFailSafe);
     if(ensureFailSafe)
-    {
-        const char * logsDir = _cfg->queryProp(PropFailSafeLogsDir);
-        if (!logsDir || !*logsDir)
-            logsDir = "./FailSafeLogs";
+        logFailSafe.setown(createFailSafeLogger(_cfg, _service, _agentName));
 
-        logFailSafe.setown(createFailSafeLogger(_service, _agentName, logsDir));
-    }
     time_t tNow;
     time(&tNow);
     localtime_r(&tNow, &m_startTime);

initfiles/componentfiles/configxml/@temp/CMakeLists.txt

@@ -23,6 +23,7 @@ FOREACH ( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/roxiePlugins.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_DynamicESDL.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_logging_transid.xsl
+    ${CMAKE_CURRENT_SOURCE_DIR}/esp_logging_agent_basic.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_wslogging.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/logging_agent.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/wslogserviceespagent.xsl

initfiles/componentfiles/configxml/@temp/esp_logging_agent_basic.xsl

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xml:space="default"
+xmlns:set="http://exslt.org/sets" exclude-result-prefixes="set">
+
+  <xsl:template name="EspLoggingAgentBasic">
+    <xsl:param name="agentNode"/>
+    <xsl:if test="string($agentNode/@FailSafe) != ''">
+        <FailSafe><xsl:value-of select="$agentNode/@FailSafe"/></FailSafe>
+    </xsl:if>
+    <xsl:if test="string($agentNode/@FailSafeLogsDir) != ''">
+        <FailSafeLogsDir><xsl:value-of select="$agentNode/@FailSafeLogsDir"/></FailSafeLogsDir>
+    </xsl:if>
+    <xsl:if test="string($agentNode/@MaxLogQueueLength) != ''">
+        <MaxLogQueueLength><xsl:value-of select="$agentNode/@MaxLogQueueLength"/></MaxLogQueueLength>
+    </xsl:if>
+    <xsl:if test="string($agentNode/@MaxTriesGTS) != ''">
+        <MaxTriesGTS><xsl:value-of select="$agentNode/@MaxTriesGTS"/></MaxTriesGTS>
+    </xsl:if>
+    <xsl:if test="string($agentNode/@MaxTriesRS) != ''">
+        <MaxTriesRS><xsl:value-of select="$agentNode/@MaxTriesRS"/></MaxTriesRS>
+    </xsl:if>
+    <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
+        <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
+    </xsl:if>
+    <xsl:if test="string($agentNode/@SafeRolloverThreshold) != ''">
+        <SafeRolloverThreshold><xsl:value-of select="$agentNode/@SafeRolloverThreshold"/></SafeRolloverThreshold>
+    </xsl:if>
+  </xsl:template>
+</xsl:stylesheet>

initfiles/componentfiles/configxml/@temp/logging_agent.xsl

@@ -20,6 +20,7 @@
 
 <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xml:space="default"
 xmlns:set="http://exslt.org/sets">
+    <xsl:import href="esp_logging_agent_basic.xsl"/>
     <xsl:import href="esp_logging_transid.xsl"/>
 
     <xsl:template name="LogSourceMap">
@@ -90,28 +91,6 @@ xmlns:set="http://exslt.org/sets">
         </xsl:if>
     </xsl:template>
 
-    <xsl:template name="LogBasic">
-        <xsl:param name="agentNode"/>
-        <xsl:if test="string($agentNode/@FailSafe) != ''">
-            <FailSafe><xsl:value-of select="$agentNode/@FailSafe"/></FailSafe>
-        </xsl:if>
-        <xsl:if test="string($agentNode/@FailSafeLogsDir) != ''">
-            <FailSafeLogsDir><xsl:value-of select="$agentNode/@FailSafeLogsDir"/></FailSafeLogsDir>
-        </xsl:if>
-        <xsl:if test="string($agentNode/@MaxLogQueueLength) != ''">
-            <MaxLogQueueLength><xsl:value-of select="$agentNode/@MaxLogQueueLength"/></MaxLogQueueLength>
-        </xsl:if>
-        <xsl:if test="string($agentNode/@MaxTriesGTS) != ''">
-            <MaxTriesGTS><xsl:value-of select="$agentNode/@MaxTriesGTS"/></MaxTriesGTS>
-        </xsl:if>
-        <xsl:if test="string($agentNode/@MaxTriesRS) != ''">
-            <MaxTriesRS><xsl:value-of select="$agentNode/@MaxTriesRS"/></MaxTriesRS>
-        </xsl:if>
-        <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
-            <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
-        </xsl:if>
-    </xsl:template>
-
     <xsl:template name="CassandraLoggingAgent">
         <xsl:param name="agentName"/>
         <xsl:param name="agentNode"/>
@@ -127,7 +106,7 @@ xmlns:set="http://exslt.org/sets">
         <LogAgent name="{$agentName}" type="LogAgent" services="{$Services}" plugin="cassandralogagent">
             <Cassandra server="{$agentNode/@serverIP}" dbUser="{$agentNode/@userName}" dbPassWord="{$agentNode/@userPassword}" dbName="{$agentNode/@ksName}"/>
 
-            <xsl:call-template name="LogBasic">
+            <xsl:call-template name="EspLoggingAgentBasic">
                 <xsl:with-param name="agentNode" select="$agentNode"/>
             </xsl:call-template>
             <xsl:call-template name="TransactionSeed">
@@ -187,7 +166,7 @@ xmlns:set="http://exslt.org/sets">
                 <xsl:with-param name="agentNode" select="$agentNode"/>
             </xsl:call-template>
 
-            <xsl:call-template name="LogBasic">
+            <xsl:call-template name="EspLoggingAgentBasic">
                 <xsl:with-param name="agentNode" select="$agentNode"/>
             </xsl:call-template>
             <xsl:call-template name="LogSourceMap">

initfiles/componentfiles/configxml/@temp/wslogserviceespagent.xsl

@@ -20,6 +20,7 @@
 
 <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xml:space="default"
 xmlns:set="http://exslt.org/sets">
+    <xsl:import href="esp_logging_agent_basic.xsl"/>
     <xsl:import href="esp_logging_transid.xsl"/>
 
     <xsl:template name="WsLogServiceESPAgent" type="DefaultLoggingAgent">
@@ -48,24 +49,9 @@ xmlns:set="http://exslt.org/sets">
         </xsl:variable>
         <LogAgent name="{$agentName}" type="LogAgent" services="{$Services}" plugin="wslogserviceespagent">
             <LoggingServer url="{$loggingServerUrl}" user="{$loggingServer/@User}" password="{$loggingServer/@Password}"/>
-            <xsl:if test="string($agentNode/@FailSafe) != ''">
-                <FailSafe><xsl:value-of select="$agentNode/@FailSafe"/></FailSafe>
-            </xsl:if>
-            <xsl:if test="string($agentNode/@FailSafeLogsDir) != ''">
-                <FailSafeLogsDir><xsl:value-of select="$agentNode/@FailSafeLogsDir"/></FailSafeLogsDir>
-            </xsl:if>
-            <xsl:if test="string($agentNode/@MaxLogQueueLength) != ''">
-                <MaxLogQueueLength><xsl:value-of select="$agentNode/@MaxLogQueueLength"/></MaxLogQueueLength>
-            </xsl:if>
-            <xsl:if test="string($agentNode/@MaxTriesGTS) != ''">
-                <MaxTriesGTS><xsl:value-of select="$agentNode/@MaxTriesGTS"/></MaxTriesGTS>
-            </xsl:if>
-            <xsl:if test="string($agentNode/@MaxTriesRS) != ''">
-                <MaxTriesRS><xsl:value-of select="$agentNode/@MaxTriesRS"/></MaxTriesRS>
-            </xsl:if>
-            <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
-                <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
-            </xsl:if>
+            <xsl:call-template name="EspLoggingAgentBasic">
+                <xsl:with-param name="agentNode" select="$agentNode"/>
+            </xsl:call-template>
             <xsl:if test="string($agentNode/@TransactionSeedType) != ''">
                 <TransactionSeedType><xsl:value-of select="$agentNode/@TransactionSeedType"/></TransactionSeedType>
             </xsl:if>

initfiles/componentfiles/configxml/cassandraloggingagent.xsd

@@ -112,6 +112,13 @@
           </xs:appinfo>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="SafeRolloverThreshold" type="xs:string" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>The threshold to switch to a new tank file: a number of requests or file size (xBytes, xMBs, etc).</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="logSourcePath" type="xs:string" use="required">
         <xs:annotation>
           <xs:appinfo>

initfiles/componentfiles/configxml/esploggingagent.xsd

@@ -107,6 +107,13 @@
           </xs:appinfo>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="SafeRolloverThreshold" type="xs:string" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>The threshold to switch to a new tank file: a number of requests or file size (xBytes, xMBs, etc).</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="MaxTransIDLength" type="xs:nonNegativeInteger" use="optional">
         <xs:annotation>
           <xs:appinfo>