HPCC-23644 Containerized component logging logic

- Provides simple setupContainerizedLoggingMsgHandler function
- Conditionally removes creation of component log file logic
- Logging to continue on default stderr stream log handler
- Per container customizations are applied to stderr log handler
- Clients and test code continues to log to file
- Provides per component logging configuration support
- Adds log class and log audience string helper methods

Signed-off-by: Rodrigo Pastrana <rodrigo.pastrana@lexisnexisrisk.com>

dali/sasha/saserver.cpp

@@ -311,6 +311,7 @@ int main(int argc, const char* argv[])
         return 1;
     }
 
+#ifndef _CONTAINERIZED
     StringBuffer logname;
     StringBuffer logdir;
     if (!stop)
@@ -322,6 +323,9 @@ int main(int argc, const char* argv[])
         lf->setMaxDetail(TopDetail);
         lf->beginLogging();
     }
+#else
+    setupContainerizedLogMsgHandler();
+#endif
     DBGLOG("Build %s", BUILD_TAG);
 
     bool enableSNMP = false;

dali/server/daserver.cpp

@@ -355,6 +355,8 @@ dali:
   dataPath: "/var/lib/HPCCSystems/dalistore"
   sds:
     environment: "/etc/HPCCSystems/environment.xml"
+  logging:
+    detail: 100
 )!!";
 
 
@@ -405,6 +407,7 @@ int main(int argc, const char* argv[])
             serverConfig.setown(createPTreeFromXMLFile(DALICONF));
 #endif
 
+#ifndef _CONTAINERIZED
         ILogMsgHandler * fileMsgHandler;
         {
             Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(serverConfig, "dali");
@@ -412,7 +415,9 @@ int main(int argc, const char* argv[])
             lf->setName("DaServer");//override default filename
             fileMsgHandler = lf->beginLogging();
         }
-
+#else
+        setupContainerizedLogMsgHandler();
+#endif
         PROGLOG("Build %s", BUILD_TAG);
 
         StringBuffer dataPath;
@@ -605,8 +610,11 @@ int main(int argc, const char* argv[])
         Owned<IMPServer> mpServer = getMPServer();
         Owned<IWhiteListHandler> whiteListHandler = createWhiteListHandler(populateWhiteListFromEnvironment, formatDaliRole);
         mpServer->installWhiteListCallback(whiteListHandler);
-
+#ifndef _CONTAINERIZED
         setMsgLevel(fileMsgHandler, serverConfig->getPropInt("SDS/@msgLevel", 100));
+#else
+        setupContainerizedLogMsgHandler();
+#endif
         startLogMsgChildReceiver(); 
         startLogMsgParentReceiver();
 

ecl/agentexec/agentexec.cpp

@@ -51,10 +51,14 @@ private:
 CEclAgentExecutionServer::CEclAgentExecutionServer(IPropertyTree *_config) : config(_config)
 {
     //Build logfile from component properties settings
+#ifndef _CONTAINERIZED
     Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(config, "eclagent");
     lf->setCreateAliasFile(false);
     lf->beginLogging();
     PROGLOG("Logging to %s",lf->queryLogFileSpec());
+#else
+    setupContainerizedLogMsgHandler();
+#endif
 
     agentName = config->queryProp("@name");
     assertex(agentName);

ecl/eclagent/eclagent.cpp

@@ -3361,12 +3361,15 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
     if (!standAloneExe)
     {
         setStatisticsComponentName(SCThthor, agentTopology->queryProp("@name"), true);
-
+#ifndef _CONTAINERIZED
         Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(agentTopology, "eclagent");
         lf->setCreateAliasFile(false);
         logMsgHandler = lf->beginLogging();
         PROGLOG("Logging to %s", lf->queryLogFileSpec());
         logfilespec.set(lf->queryLogFileSpec());
+#else
+        setupContainerizedLogMsgHandler();
+#endif
     }
     else
     {

ecl/eclcc/eclcc.cpp

@@ -503,6 +503,9 @@ static constexpr const char * defaultYaml = R"!!(
 version: "1.0"
 eclccserver:
     name: eclccserver
+    logging:
+      audiences: "USR+ADT"
+      classes: "ERR"
 )!!";
 
 
@@ -515,9 +518,13 @@ int main(int argc, const char *argv[])
 
     configuration.setown(loadConfiguration(defaultYaml, argv, "eclccserver", "ECLCCSERVER", nullptr, nullptr));
 
+#ifndef _CONTAINERIZED
     // Turn logging down (we turn it back up if -v option seen)
     Owned<ILogMsgFilter> filter = getCategoryLogMsgFilter(MSGAUD_user| MSGAUD_operator, MSGCLS_error);
     queryLogMsgManager()->changeMonitorFilter(queryStderrLogMsgHandler(), filter);
+#else
+    setupContainerizedLogMsgHandler();
+#endif
     unsigned exitCode = doMain(argc, argv);
     stopPerformanceMonitor();
     if (!optReleaseAllMemory)

esp/platform/espp.cpp

@@ -285,11 +285,14 @@ void openEspLogFile(IPropertyTree* envpt, IPropertyTree* procpt)
             procpt->getProp("@logDir", logdir);
     }
 
-
+#ifndef _CONTAINERIZED
     Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(logdir.str(), "esp");
     lf->setName("esp_main");//override default filename
     lf->setAliasName("esp");
     lf->beginLogging();
+#else
+    setupContainerizedLogMsgHandler();
+#endif
 
     if (procpt->getPropBool("@enableSysLog", false))
         UseSysLogForOperatorMessages();

fs/dafilesrv/dafilesrv.cpp

@@ -788,12 +788,16 @@ int main(int argc,char **argv)
             return ret;
 #endif
     }
+#ifndef _CONTAINERIZED
     {
         Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(logDir.str(), "DAFILESRV");
         lf->setCreateAliasFile(false);
         lf->setMaxDetail(TopDetail);
         lf->beginLogging();
     }
+#else
+    setupContainerizedLogMsgHandler();
+#endif
 
     write_pidfile(componentName.str());
     PROGLOG("Dafilesrv starting - Build %s", BUILD_TAG);

roxie/ccd/ccdmain.cpp

@@ -543,6 +543,8 @@ roxie:
     - name: workunit
       port: 0
       numThreads: 0
+  logging:
+      detail: 100
 )!!";
 
 int STARTQUERY_API start_query(int argc, const char *argv[])
@@ -764,6 +766,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         directoryTree.clear();
 
         //Logging stuff
+#ifndef _CONTAINERIZED
         if (topology->getPropBool("@stdlog", traceLevel != 0) || topology->getPropBool("@forceStdLog", false))
             queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time | MSGFIELD_milliTime | MSGFIELD_thread | MSGFIELD_prefix);
         else
@@ -774,6 +777,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             lf->setMaxDetail(TopDetail);
             lf->beginLogging();
             logDirectory.set(lf->queryLogDir());
+
 #ifdef _DEBUG
             unsigned useLogQueue = topology->getPropBool("@useLogQueue", false);
 #else
@@ -789,6 +793,9 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             if (topology->getPropBool("@enableSysLog",true))
                 UseSysLogForOperatorMessages();
         }
+#else
+        setupContainerizedLogMsgHandler();
+#endif
 
         roxieMetrics.setown(createRoxieMetricsManager());
 

roxie/topo/toposerver.cpp

@@ -318,6 +318,7 @@ int main(int argc, const char *argv[])
         Owned<IPropertyTree> topology = loadConfiguration(defaultYaml, argv, "roxie", "ROXIE", nullptr, nullptr);
         traceLevel = topology->getPropInt("@traceLevel", 1);
         topoPort = topology->getPropInt("@topoport", TOPO_SERVER_PORT);
+#ifndef _CONTAINERIZED
         if (topology->getPropBool("@stdlog", traceLevel != 0))
             queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time | MSGFIELD_milliTime | MSGFIELD_thread | MSGFIELD_prefix);
         else
@@ -331,6 +332,9 @@ int main(int argc, const char *argv[])
             queryLogMsgManager()->enterQueueingMode();
             queryLogMsgManager()->setQueueDroppingLimit(512, 32);
         }
+#else
+        setupContainerizedLogMsgHandler();
+#endif
         Owned<ISocket> socket = ISocket::create(topoPort);
         if (traceLevel)
             DBGLOG("Topology server starting");

system/jlib/jlog.cpp

@@ -2204,7 +2204,7 @@ ILogMsgHandler * getLogMsgHandlerFromPTree(IPropertyTree * tree)
         if(isdigit(fstr[0]))
             fields = atoi(fstr);
         else
-            fields = LogMsgFieldsFromAbbrevs(fstr);
+            fields = logMsgFieldsFromAbbrevs(fstr);
     }
     if(strcmp(type.str(), "stderr")==0)
         return getHandleLogMsgHandler(stderr, fields, tree->hasProp("@writeXML"));
@@ -2324,6 +2324,75 @@ MODULE_EXIT()
     thePassAllFilter = NULL;
 }
 
+#ifdef _CONTAINERIZED
+
+static constexpr const char * logFielsdAtt = "@fields";
+static constexpr const char * logMsgDetailAtt = "@detail";
+static constexpr const char * logMsgAudiencesAtt = "@audiences";
+static constexpr const char * logMsgClassesAtt = "@classes";
+static constexpr const char * useLogQueueAtt = "@useLogQueue";
+static constexpr const char * logQueueLenAtt = "@queueLen";
+static constexpr const char * logQueueDropAtt = "@queueDrop";
+static constexpr const char * useSysLogpAtt ="@enableSysLog";
+
+#ifdef _DEBUG
+static constexpr bool useQueueDefault = false;
+#else
+static constexpr bool useQueueDefault = true;
+#endif
+
+static constexpr unsigned queueLenDefault = 512;
+static constexpr unsigned queueDropDefault = 32;
+static constexpr bool useSysLogDefault = false;
+
+void setupContainerizedLogMsgHandler()
+{
+    IPropertyTree * logConfig = queryComponentConfig().queryPropTree("logging");
+    if (logConfig)
+    {
+        if (logConfig->hasProp(logFielsdAtt))
+        {
+            //Supported logging fields: AUD,CLS,DET,MID,TIM,DAT,PID,TID,NOD,JOB,USE,SES,COD,MLT,MCT,NNT,COM,QUO,PFX,ALL,STD
+            const char *logFields = logConfig->queryProp(logFielsdAtt);
+            if (!isEmptyString(logFields))
+                theStderrHandler->setMessageFields(logMsgFieldsFromAbbrevs(logFields));
+        }
+
+        //Only recreate filter if at least one filter attribute configured
+        if (logConfig->hasProp(logMsgDetailAtt) || logConfig->hasProp(logMsgAudiencesAtt) || logConfig->hasProp(logMsgClassesAtt))
+        {
+            LogMsgDetail logDetail = logConfig->getPropInt(logMsgDetailAtt, DefaultDetail);
+
+            unsigned msgClasses = MSGCLS_all;
+            const char *logClasses = logConfig->queryProp(logMsgClassesAtt);
+            if (!isEmptyString(logClasses))
+                msgClasses = logMsgClassesFromAbbrevs(logClasses);
+
+            unsigned msgAudiences = MSGAUD_all;
+            const char *logAudiences = logConfig->queryProp(logMsgAudiencesAtt);
+            if (!isEmptyString(logAudiences))
+                msgAudiences = logMsgAudsFromAbbrevs(logAudiences);
+
+            Owned<ILogMsgFilter> filter = getCategoryLogMsgFilter(msgAudiences, msgClasses, logDetail);
+            theManager->changeMonitorFilter(theStderrHandler, filter);
+        }
+
+        bool useLogQueue = logConfig->getPropBool(useLogQueueAtt, useQueueDefault);
+        if (useLogQueue)
+        {
+            unsigned queueLen = logConfig->getPropInt(logQueueLenAtt, queueLenDefault);
+            unsigned queueDrop = logConfig->getPropInt(logQueueDropAtt, queueDropDefault);
+
+            queryLogMsgManager()->enterQueueingMode();
+            queryLogMsgManager()->setQueueDroppingLimit(queueLen, queueDrop);
+        }
+
+        if (logConfig->getPropBool(useSysLogpAtt, useSysLogDefault))
+            UseSysLogForOperatorMessages();
+    }
+}
+#endif
+
 ILogMsgManager * queryLogMsgManager()
 {
     return theManager;
@@ -2800,7 +2869,7 @@ private:
         flushes = true;
         const char *logFields = queryEnvironmentConf().queryProp("logfields");
         if (!isEmptyString(logFields))
-            msgFields = LogMsgFieldsFromAbbrevs(logFields);
+            msgFields = logMsgFieldsFromAbbrevs(logFields);
         else
             msgFields = MSGFIELD_STANDARD;
         msgAudiences = MSGAUD_all;

system/jlib/jlog.hpp

@@ -35,6 +35,13 @@
 #include "jptree.hpp"
 #include "jsocket.hpp"
 
+typedef enum
+{
+    LogMsgAttType_class          = 0x01,
+    LogMsgAttType_Audience       = 0x02,
+    LogMsgAttType_fields         = 0x04,
+} LogMsgAttType;
+
 /****************************************************************************************/
 /* LOG MESSAGE AUDIENCES:                                                               *
  * MSGAUD_operator - This should be used when the message may be normally monitored by, *
@@ -153,6 +160,20 @@ inline const char * LogMsgAudienceToFixString(LogMsgAudience audience)
         return("UNKNOWN  ");
     }
 }
+inline unsigned LogMsgAudFromAbbrev(char const * abbrev)
+{
+    if(strnicmp(abbrev, "OPR", 3)==0)
+        return MSGAUD_operator;
+    if(strnicmp(abbrev, "USR", 3)==0)
+        return MSGAUD_user;
+    if(strnicmp(abbrev, "PRO", 3)==0)
+        return MSGAUD_programmer;
+    if(strnicmp(abbrev, "ADT", 3)==0)
+        return MSGAUD_audit;
+    if(strnicmp(abbrev, "ALL", 3)==0)
+        return MSGAUD_all;
+    return 0;
+}
 
 inline const char * LogMsgClassToVarString(LogMsgClass msgClass)
 {
@@ -194,6 +215,23 @@ inline const char * LogMsgClassToFixString(LogMsgClass msgClass)
     }
 }
 
+inline unsigned LogMsgClassFromAbbrev(char const * abbrev)
+{
+    if(strnicmp(abbrev, "DIS", 3)==0)
+        return MSGCLS_disaster;
+    if(strnicmp(abbrev, "ERR", 3)==0)
+        return MSGCLS_error;
+    if(strnicmp(abbrev, "WRN", 3)==0)
+        return MSGCLS_warning;
+    if(strnicmp(abbrev, "INF", 3)==0)
+        return MSGCLS_information;
+    if(strnicmp(abbrev, "PRO", 3)==0)
+        return MSGCLS_progress;
+    if(strnicmp(abbrev, "ALL", 3)==0)
+        return MSGCLS_all;
+    return 0;
+}
+
 typedef unsigned LogMsgDetail;
 #define DefaultDetail   100
 #define TopDetail (LogMsgDetail)-1
@@ -347,39 +385,71 @@ inline unsigned LogMsgFieldFromAbbrev(char const * abbrev)
     return 0;
 }
 
-// This function parses strings such as "AUD+CLS+DET+COD" and "STD+MIT-PID", and is used for fields attribute in XML handler descriptions
-
-inline unsigned LogMsgFieldsFromAbbrevs(char const * abbrevs)
+inline unsigned processAbbrevsString(char const * abbrevs, LogMsgAttType type)
 {
-    unsigned fields = 0;
+    unsigned values = 0;
     bool negate = false;
     bool more = true;
     while(more)
     {
-        if(strlen(abbrevs)<3) break;
-        unsigned field = LogMsgFieldFromAbbrev(abbrevs);
-        if(field)
+        if(strlen(abbrevs) < 3)
+            break;
+        unsigned value = 0;
+        switch(type)
+        {
+            case LogMsgAttType_Audience:
+                value = LogMsgAudFromAbbrev(abbrevs);
+                break;
+            case LogMsgAttType_fields:
+                value = LogMsgFieldFromAbbrev(abbrevs);
+                break;
+            case LogMsgAttType_class:
+                value = LogMsgClassFromAbbrev(abbrevs);
+                break;
+            default:
+                throwUnexpected();
+        }
+
+        if(value)
         {
             if(negate)
-                fields &= ~field;
+                values &= ~value;
             else
-                fields |= field;
+                values |= value;
         }
         switch(abbrevs[3])
         {
-        case '+':
-            negate = false;
-            abbrevs += 4;
-            break;
-        case '-':
-            negate = true;
-            abbrevs += 4;
-            break;
-        default:
-            more = false;
+            case '+':
+                negate = false;
+                abbrevs += 4;
+                break;
+            case '-':
+                negate = true;
+                abbrevs += 4;
+                break;
+            default:
+                more = false;
         }
     }
-    return fields;
+    return values;
+}
+
+// This function parses strings such as "ADT+PRO+USR" and "ALL+ADT-PRO"
+inline unsigned logMsgAudsFromAbbrevs(const char * abbrevs)
+{
+    return processAbbrevsString(abbrevs, LogMsgAttType_Audience);
+}
+
+// This function parses strings such as "DIS+ERR+WRN+INF" and "ALL+PRO-INF"
+inline unsigned logMsgClassesFromAbbrevs(const char * abbrevs)
+{
+    return processAbbrevsString(abbrevs, LogMsgAttType_class);
+}
+
+// This function parses strings such as "AUD+CLS+DET+COD" and "STD+MIT-PID", and is used for fields attribute in XML handler descriptions
+inline unsigned logMsgFieldsFromAbbrevs(const char * abbrevs)
+{
+    return processAbbrevsString(abbrevs, LogMsgAttType_fields);
 }
 
 inline char const * msgPrefix(LogMsgClass msgClass)
@@ -756,6 +826,9 @@ extern jlib_decl const LogMsgJobInfo unknownJob;
 
 extern jlib_decl ILogMsgManager * queryLogMsgManager();
 extern jlib_decl ILogMsgHandler * queryStderrLogMsgHandler();
+#ifdef _CONTAINERIZED
+extern jlib_decl void setupContainerizedLogMsgHandler();
+#endif
 extern jlib_decl LogMsgComponentReporter * queryLogMsgComponentReporter(unsigned compo);
 
 extern jlib_decl ILogMsgManager * createLogMsgManager(); // use with care! (needed by mplog listener facility)

thorlcr/master/thmastermain.cpp

@@ -632,16 +632,22 @@ int main( int argc, const char *argv[]  )
     ILogMsgHandler *logHandler;
     try
     {
+#ifndef _CONTAINERIZED
         {
             Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(globals, "thor");
             lf->setName("thormaster");//override default filename
             lf->setCreateAliasFile(false);
             logHandler = lf->beginLogging();
             createUNCFilename(lf->queryLogFileSpec(), logUrl, false);
+
+            LOG(MCdebugProgress, thorJob, "Opened log file %s", logUrl.str());
         }
-        LOG(MCdebugProgress, thorJob, "Opened log file %s", logUrl.str());
+#else
+        setupContainerizedLogMsgHandler();
+        logHandler = queryStderrLogMsgHandler();
+        logUrl.set("stderr");
+#endif
         LOG(MCdebugProgress, thorJob, "Build %s", BUILD_TAG);
-        globals->setProp("@logURL", logUrl.str());
 
         Owned<IGroup> serverGroup = createIGroupRetry(daliServer.str(), DALI_SERVER_PORT);
 

thorlcr/slave/thslavemain.cpp

@@ -266,6 +266,7 @@ public:
 
 void startSlaveLog()
 {
+#ifndef _CONTAINERIZED
     StringBuffer fileName("thorslave");
     Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(globals->queryProp("@logDir"), "thor");
     StringBuffer slaveNumStr;
@@ -274,12 +275,11 @@ void startSlaveLog()
     lf->setName(fileName.str());//override default filename
     lf->beginLogging();
 
-    StringBuffer url;
-    createUNCFilename(lf->queryLogFileSpec(), url);
-
-    LOG(MCdebugProgress, thorJob, "Opened log file %s", url.str());
+    LOG(MCdebugProgress, thorJob, "Opened log file %s", lf->queryLogFileSpec());
+#else
+    setupContainerizedLogMsgHandler();
+#endif
     LOG(MCdebugProgress, thorJob, "Build %s", BUILD_TAG);
-    globals->setProp("@logURL", url.str());
 }
 
 void setSlaveAffinity(unsigned processOnNode)