HPCC-18377 Add the audience field to the logging

Signed-off-by: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -2060,8 +2060,6 @@ void WsWuInfo::readWorkunitLog(IFile* sourceFile, MemoryBuffer& buf, const char*
     VStringBuffer startwuid("Started wuid=%s", wuid.str());
     VStringBuffer endwuid("Finished wuid=%s", wuid.str());
 
-    bool outputThisLine = false;
-    unsigned processID = 0;
     StringBuffer line;
 
     Owned<IFileIOStream> outIOS;
@@ -2072,47 +2070,59 @@ void WsWuInfo::readWorkunitLog(IFile* sourceFile, MemoryBuffer& buf, const char*
     }
 
     Owned<IStreamLineReader> lineReader = createLineReader(ios, true);
-    while (!lineReader->readLine(line.clear()))
+
+    bool eof = lineReader->readLine(line.clear());
+    if (eof)
+        return;
+
+    // Process header for log file format
+    unsigned logfields = getMessageFieldsFromHeader(line);
+    if (logfields==0)   // No header line, so must be in legacy format
+        logfields = MSGFIELD_LEGACY;
+    else
+        eof = lineReader->readLine(line.clear());
+
+    const unsigned positionProcessId = getPositionOfField(logfields, MSGFIELD_process);
+    bool outputThisLine = false;
+    unsigned processID = 0;
+    bool foundEndWUID = false; 
+    while (!eof)
     {
+        if (line.length() > positionProcessId+1)
+        {
+            const char * lineStartingProcessId = line.str()+positionProcessId;
+            if (outputThisLine)
+            {
+                //If the slave is restarted before WU is finished, we cannot find out the "Finished wuid=...".
+                //So, we should check whether the slave is restarting or not.
+                unsigned pID = 0;
+                foundEndWUID = parseLogLine(lineStartingProcessId, endwuid, pID);
+                if ((pID > 0) && (pID != processID))
+                    break;
+            }
+            else if (strstr(lineStartingProcessId, startwuid))
+            {
+                outputThisLine = true;
+                foundEndWUID = false;
+                if (processID == 0)
+                    parseLogLine(lineStartingProcessId, nullptr, processID);
+            }
+        }
         if (outputThisLine)
         {
-            //If the slave is restarted before WU is finished, we cannot find out the "Finished wuid=...".
-            //So, we should check whether the slave is restarting or not.
-            unsigned pID = 0;
-            bool foundEndWUID = parseLogLine(line, endwuid, pID);
-            if ((pID > 0) && (pID != processID))
-                break;
-
             outputALine(line.length(), line.str(), buf, outIOS);
             if (foundEndWUID)
                 outputThisLine = false;
         }
-        else if (strstr(line, startwuid))
-        {
-            outputThisLine = true;
-            outputALine(line.length(), line.str(), buf, outIOS);
-            if (processID == 0)
-                parseLogLine(line, nullptr, processID);
-        }
+        eof = lineReader->readLine(line.clear());
     }
 }
 
-//the expected format for the log line is: 'LineID Date Time ProcessID ThreadID ...'.
 bool WsWuInfo::parseLogLine(const char* line, const char* endWUID, unsigned& processID)
 {
-    //Skip lineID, date and time
     const char* bptr = line;
-    for (unsigned i = 0; i < 3; i++)
-    {
-        bptr = strchr(bptr, ' ');
-        if (!bptr)
-            return false;
-        bptr++;
-    }
-
-    //Read ProcessID
     const char* eptr = bptr + 1;
-    while (*eptr && isdigit(*eptr))
+    while (*eptr && isdigit(*eptr))     //Read ProcessID
         eptr++;
 
     if (*eptr != ' ')

initfiles/etc/DIR_NAME/environment.conf.in

@@ -7,7 +7,7 @@ classpath=${INSTALL_DIR}/classes
 runtime=${RUNTIME_PATH}
 lock=${LOCK_PATH}
 # Supported logging fields: AUD,CLS,DET,MID,TIM,DAT,PID,TID,NOD,JOB,USE,SES,COD,MLT,MCT,NNT,COM,QUO,PFX,ALL,STD
-logfields=TIM+DAT+MLT+MID+PID+TID+COD+QUO+PFX
+logfields=TIM+DAT+MLT+MID+PID+TID+COD+QUO+PFX+AUD
 pid=${PID_PATH}
 log=${LOG_PATH}
 user=${RUNTIME_USER}

system/jlib/jdebug.cpp

@@ -3995,7 +3995,7 @@ public:
     }
 
     // interface ILogMsgHandler
-    virtual void handleMessage(const LogMsg & msg __attribute__((unused))) const {  counter++; }
+    virtual void handleMessage(const LogMsg & msg __attribute__((unused))) { counter++; }
     virtual bool needsPrep() const { return false; }
     virtual void prep() {}
     virtual unsigned queryMessageFields() const { return MSGFIELD_detail; }

system/jlib/jlog.cpp

@@ -86,19 +86,115 @@ void LogMsgSysInfo::deserialize(MemoryBuffer & in)
 #endif
 }
 
+class LoggingFieldColumns
+{
+    const EnumMapping MsgFieldMap[15] =
+    {
+        { MSGFIELD_msgID,     "MsgID    " },
+        { MSGFIELD_audience,  "Audience " },
+        { MSGFIELD_class,     "Class    " },
+        { MSGFIELD_detail,    "Detail     " },
+        { MSGFIELD_date,      "Date       " },
+        { MSGFIELD_microTime, "Time(micro)     " },
+        { MSGFIELD_milliTime, "Time(milli)  " },
+        { MSGFIELD_time,      "Time        " },
+        { MSGFIELD_process,   "PID   " },
+        { MSGFIELD_thread,    "TID   " },
+        { MSGFIELD_session,   "SessionID           " },
+        { MSGFIELD_node,      "Node                " },
+        { MSGFIELD_job,       "JobID  " },
+        { MSGFIELD_user,      "UserID  " },
+        { MSGFIELD_component, "Compo " }
+    };
+    const unsigned sizeMsgFieldMap = arraysize(MsgFieldMap);
+public:
+    unsigned getMaxHeaderSize()
+    {
+        // Note: return length is slightly longer than necessary as only one time field is valid
+        // but the length of all time fields are added
+        unsigned size = 0;
+        for (unsigned i=0; i<sizeMsgFieldMap; ++i)
+            size += strlen(MsgFieldMap[i].str);
+        return size+2; // 2 extra characters for \r\n
+    }
+    unsigned getPositionOfField(unsigned logfields, unsigned positionoffield)
+    {
+        unsigned pos = 0;
+        for (unsigned i=0; i<sizeMsgFieldMap; ++i)
+        {
+            if (MsgFieldMap[i].val==MSGFIELD_time && (logfields & (MSGFIELD_microTime|MSGFIELD_milliTime)) )
+                continue;
+            if (MsgFieldMap[i].val & positionoffield)
+                return pos;
+            if (MsgFieldMap[i].val & logfields)
+                pos += strlen(MsgFieldMap[i].str);
+        }
+        return 0;
+    }
+    unsigned extractMessageFieldsFromHeader(const char *line, bool hashPrefixed)
+    {
+        unsigned fieldHeader = 0;
+
+        if (line && *line=='#')
+        {
+            ++line;
+            const unsigned sizeFieldMap = arraysize(MsgFieldMap);
+            for (unsigned i=0; i<sizeFieldMap; ++i)
+            {
+                const char * linep = line;
+                const char * fieldp = MsgFieldMap[i].str;
+                while( *fieldp && *linep==*fieldp)
+                {
+                    ++linep;
+                    ++fieldp;
+                }
+                if (*fieldp==0) // At the end of the field, so whole field matched
+                {
+                    fieldHeader |= MsgFieldMap[i].val;
+                    if (*linep==0 || *linep=='\n')
+                        break;
+                    line = linep;
+                }
+            }
+        }
+        if (fieldHeader & (MSGFIELD_microTime | MSGFIELD_milliTime))
+            fieldHeader |= MSGFIELD_time;
+        return fieldHeader;
+    }
+    StringBuffer & generateHeaderRow(StringBuffer & out, unsigned fields, bool prefixHash)
+    {
+        if (prefixHash)
+            out.append('#');
+        for (unsigned i=0; i<sizeMsgFieldMap; ++i)
+            if (fields & MsgFieldMap[i].val)
+            {
+                if (MsgFieldMap[i].val==MSGFIELD_time && (fields & (MSGFIELD_microTime|MSGFIELD_milliTime)) )
+                    continue;
+                out.append(MsgFieldMap[i].str);
+            }
+        return out;
+    }
+
+} loggingFieldColumns;
+
+unsigned getPositionOfField(unsigned logfields, unsigned positionoffield)
+{
+    return loggingFieldColumns.getPositionOfField(logfields, positionoffield);
+}
+
 // LogMsg
 
 StringBuffer & LogMsg::toStringPlain(StringBuffer & out, unsigned fields) const
 {
     out.ensureCapacity(LOG_MSG_FORMAT_BUFFER_LENGTH);
+    if(fields & MSGFIELD_msgID)
+        out.appendf("id=%X ", sysInfo.queryMsgID());
     if(fields & MSGFIELD_audience)
         out.append("aud=").append(LogMsgAudienceToVarString(category.queryAudience())).append(' ');
     if(fields & MSGFIELD_class)
         out.append("cls=").append(LogMsgClassToVarString(category.queryClass())).append(' ');
     if(fields & MSGFIELD_detail)
         out.appendf("det=%d ", category.queryDetail());
-    if(fields & MSGFIELD_msgID)
-        out.appendf("id=%X ", sysInfo.queryMsgID());
     if(fields & MSGFIELD_timeDate)
     {
         time_t timeNum = sysInfo.queryTime();
@@ -173,6 +269,8 @@ StringBuffer & LogMsg::toStringXML(StringBuffer & out, unsigned fields) const
 {
     out.ensureCapacity(LOG_MSG_FORMAT_BUFFER_LENGTH);
     out.append("<msg ");
+    if(fields & MSGFIELD_msgID)
+        out.append("MessageID=\"").append(sysInfo.queryMsgID()).append("\" ");
     if(fields & MSGFIELD_audience)
         out.append("Audience=\"").append(LogMsgAudienceToVarString(category.queryAudience())).append("\" ");
     if(fields & MSGFIELD_class)
@@ -182,8 +280,6 @@ StringBuffer & LogMsg::toStringXML(StringBuffer & out, unsigned fields) const
 #ifdef LOG_MSG_NEWLINE
     if(fields & MSGFIELD_allCategory) out.append("\n     ");
 #endif
-    if(fields & MSGFIELD_msgID)
-        out.append("MessageID=\"").append(sysInfo.queryMsgID()).append("\" ");
     if(fields & MSGFIELD_timeDate)
     {
         time_t timeNum = sysInfo.queryTime();
@@ -255,6 +351,8 @@ StringBuffer & LogMsg::toStringXML(StringBuffer & out, unsigned fields) const
 
 StringBuffer & LogMsg::toStringTable(StringBuffer & out, unsigned fields) const
 {
+    if(fields & MSGFIELD_msgID)
+        out.appendf("%8X ", sysInfo.queryMsgID());
     out.ensureCapacity(LOG_MSG_FORMAT_BUFFER_LENGTH);
     if(fields & MSGFIELD_audience)
         out.append(LogMsgAudienceToFixString(category.queryAudience()));
@@ -262,8 +360,6 @@ StringBuffer & LogMsg::toStringTable(StringBuffer & out, unsigned fields) const
         out.append(LogMsgClassToFixString(category.queryClass()));
     if(fields & MSGFIELD_detail)
         out.appendf("%10d ", category.queryDetail());
-    if(fields & MSGFIELD_msgID)
-        out.appendf("%8X ", sysInfo.queryMsgID());
     if(fields & MSGFIELD_timeDate)
     {
         time_t timeNum = sysInfo.queryTime();
@@ -338,46 +434,20 @@ StringBuffer & LogMsg::toStringTable(StringBuffer & out, unsigned fields) const
 
 StringBuffer & LogMsg::toStringTableHead(StringBuffer & out, unsigned fields)
 {
-    if(fields & MSGFIELD_audience)
-        out.append("Audience ");
-    if(fields & MSGFIELD_class)
-        out.append("Class    ");
-    if(fields & MSGFIELD_detail)
-        out.append("    Detail ");
-    if(fields & MSGFIELD_msgID)
-        out.append("   MsgID ");
-    if(fields & MSGFIELD_date)
-        out.append("      Date ");
-    if(fields & (MSGFIELD_microTime | MSGFIELD_milliTime | MSGFIELD_time))
-        out.append("    Time ");
-    if(fields & MSGFIELD_process)
-        out.append("  PID ");
-    if(fields & MSGFIELD_thread)
-        out.append("  TID ");
-    if(fields & MSGFIELD_session)
-        out.append("      SessionID      ");
-    if(fields & MSGFIELD_node)
-        out.append("               Node ");
-    if(fields & MSGFIELD_job)
-        out.append("  JobID ");
-    if(fields & MSGFIELD_user)
-        out.append(" UserID ");
-    if(fields & MSGFIELD_component)
-        out.append(" Compo ");
-    out.append("\n\n");
+    loggingFieldColumns.generateHeaderRow(out, fields, false).append("\n\n");
     return out;
 }
 
 void LogMsg::fprintPlain(FILE * handle, unsigned fields) const
 {
+    if(fields & MSGFIELD_msgID)
+        fprintf(handle, "id=%X ", sysInfo.queryMsgID());
     if(fields & MSGFIELD_audience)
         fprintf(handle, "aud=%s", LogMsgAudienceToVarString(category.queryAudience()));
     if(fields & MSGFIELD_class)
         fprintf(handle, "cls=%s", LogMsgClassToVarString(category.queryClass()));
     if(fields & MSGFIELD_detail)
         fprintf(handle, "det=%d ", category.queryDetail());
-    if(fields & MSGFIELD_msgID)
-        fprintf(handle, "id=%X ", sysInfo.queryMsgID());
     if(fields & MSGFIELD_timeDate)
     {
         time_t timeNum = sysInfo.queryTime();
@@ -448,6 +518,8 @@ void LogMsg::fprintPlain(FILE * handle, unsigned fields) const
 void LogMsg::fprintXML(FILE * handle, unsigned fields) const
 {
     fprintf(handle, "<msg ");
+    if(fields & MSGFIELD_msgID)
+        fprintf(handle, "MessageID=\"%d\" ",sysInfo.queryMsgID());
     if(fields & MSGFIELD_audience)
         fprintf(handle, "Audience=\"%s\" ", LogMsgAudienceToVarString(category.queryAudience()));
     if(fields & MSGFIELD_class)
@@ -457,8 +529,6 @@ void LogMsg::fprintXML(FILE * handle, unsigned fields) const
 #ifdef LOG_MSG_NEWLINE
     if(fields & MSGFIELD_allCategory) fprintf(handle, "\n     ");
 #endif
-    if(fields & MSGFIELD_msgID)
-        fprintf(handle, "MessageID=\"%d\" ",sysInfo.queryMsgID());
     if(fields & MSGFIELD_timeDate)
     {
         time_t timeNum = sysInfo.queryTime();
@@ -530,14 +600,14 @@ void LogMsg::fprintXML(FILE * handle, unsigned fields) const
 
 void LogMsg::fprintTable(FILE * handle, unsigned fields) const
 {
+    if(fields & MSGFIELD_msgID)
+        fprintf(handle, "%08X ", sysInfo.queryMsgID());
     if(fields & MSGFIELD_audience)
         fputs(LogMsgAudienceToFixString(category.queryAudience()), handle);
     if(fields & MSGFIELD_class)
         fputs(LogMsgClassToFixString(category.queryClass()), handle);
     if(fields & MSGFIELD_detail)
         fprintf(handle, "%10d ", category.queryDetail());
-    if(fields & MSGFIELD_msgID)
-        fprintf(handle, "%08X ", sysInfo.queryMsgID());
     if(fields & MSGFIELD_timeDate)
     {
         time_t timeNum = sysInfo.queryTime();
@@ -607,33 +677,38 @@ void LogMsg::fprintTable(FILE * handle, unsigned fields) const
 
 void LogMsg::fprintTableHead(FILE * handle, unsigned fields)
 {
-    if(fields & MSGFIELD_audience)
-        fprintf(handle, "Audience ");
-    if(fields & MSGFIELD_class)
-        fprintf(handle, "Class    ");
-    if(fields & MSGFIELD_detail)
-        fprintf(handle, "    Detail ");
-    if(fields & MSGFIELD_msgID)
-        fprintf(handle, "   MsgID ");
-    if(fields & MSGFIELD_date)
-        fprintf(handle, "      Date ");
-    if(fields & MSGFIELD_time)
-        fprintf(handle, "    Time ");
-    if(fields & MSGFIELD_process)
-        fprintf(handle, "  PID ");
-    if(fields & MSGFIELD_thread)
-        fprintf(handle, "  TID ");
-    if(fields & MSGFIELD_session)
-        fprintf(handle, "      SessionID      ");
-    if(fields & MSGFIELD_node)
-        fprintf(handle, "               Node ");
-    if(fields & MSGFIELD_job)
-        fprintf(handle, "  JobID ");
-    if(fields & MSGFIELD_user)
-        fprintf(handle, " UserID ");
-    if(fields & MSGFIELD_component)
-        fprintf(handle, " Compo ");
-    fprintf(handle, "\n\n");
+    StringBuffer  header;
+    loggingFieldColumns.generateHeaderRow(header, fields, true).append("\n");
+    fputs(header.str(), handle);
+}
+
+unsigned getMessageFieldsFromHeader(FILE *handle)
+{
+    unsigned currentFieldHeader = 0;
+    try
+    {
+        MemoryBuffer mb(loggingFieldColumns.getMaxHeaderSize());
+        fpos_t pos;
+        fgetpos (handle,&pos);
+        rewind (handle);
+
+        mb.reserve(loggingFieldColumns.getMaxHeaderSize());
+        const char * line = fgets (static_cast<char *>(mb.bufferBase()),  loggingFieldColumns.getMaxHeaderSize(), handle );
+        if (line && *line)
+            currentFieldHeader = loggingFieldColumns.extractMessageFieldsFromHeader(line, true);
+        fsetpos (handle, &pos);
+    }
+    catch (...)
+    {
+        currentFieldHeader = 0;
+    }
+    return currentFieldHeader;
+}
+
+unsigned getMessageFieldsFromHeader(const char * line)
+{
+    return loggingFieldColumns.extractMessageFieldsFromHeader(line, true);
+
 }
 
 // Implementations of ILogMsgFilter
@@ -971,7 +1046,7 @@ void RollingFileLogMsgHandler::addToPTree(IPropertyTree * tree) const
 
 #define ROLLOVER_PERIOD 86400
 
-void RollingFileLogMsgHandler::checkRollover() const
+void RollingFileLogMsgHandler::checkRollover()
 {
     time_t tNow;
     time(&tNow);
@@ -984,7 +1059,7 @@ void RollingFileLogMsgHandler::checkRollover() const
     }
 }
 
-void RollingFileLogMsgHandler::doRollover(bool daily, const char *forceName) const
+void RollingFileLogMsgHandler::doRollover(bool daily, const char *forceName)
 {
     CriticalBlock block(crit);
     closeAndDeleteEmpty(filename,handle);
@@ -999,27 +1074,46 @@ void RollingFileLogMsgHandler::doRollover(bool daily, const char *forceName) con
         filename.append(fileextn.get());
     }
     recursiveCreateDirectoryForFile(filename.str());
-    handle = fopen(filename.str(), append ? "a" : "w");
-    if (handle && alias && alias.length())
+    handle = fopen(filename.str(), append ? "a+" : "w");
+    printHeader = true;
+    currentLogFields = 0;
+    if (handle)
     {
-        fclose(handle);
-        handle = 0;
-        remove(alias);
-        try
-        {
-            createHardLink(alias, filename.str());
-        }
-        catch (IException *E)
+        if (append)
         {
-            recursiveCreateDirectoryForFile(filename.str());
-            handle = fopen(filename.str(), append ? "a" : "w");
-            EXCLOG(E);  // Log the fact that we could not create the alias - probably it is locked (tail a bit unfortunate on windows).
-            E->Release();
+            fseek(handle, 0, SEEK_END);
+            long pos = ftell(handle);
+            if (pos > 0 || (pos==-1 && errno==EOVERFLOW)) // If current file is not empty
+            {
+                printHeader = false;
+                unsigned logfields = getMessageFieldsFromHeader(handle);
+                if (logfields == 0) // No header file so write log lines legacy field format
+                    currentLogFields = MSGFIELD_LEGACY;
+                else if (logfields != messageFields) // Different log format from format in current log file
+                    currentLogFields = logfields;
+            }
         }
-        if (!handle)
+        if (alias && alias.length())
         {
-            recursiveCreateDirectoryForFile(filename.str());
-            handle = fopen(filename.str(), append ? "a" : "w");
+            fclose(handle);
+            handle = 0;
+            remove(alias);
+            try
+            {
+                createHardLink(alias, filename.str());
+            }
+            catch (IException *E)
+            {
+                recursiveCreateDirectoryForFile(filename.str());
+                handle = fopen(filename.str(), append ? "a" : "w");
+                EXCLOG(E);  // Log the fact that we could not create the alias - probably it is locked (tail a bit unfortunate on windows).
+                E->Release();
+            }
+            if (!handle)
+            {
+                recursiveCreateDirectoryForFile(filename.str());
+                handle = fopen(filename.str(), append ? "a" : "w");
+            }
         }
     }
     if(!handle) 
@@ -1054,7 +1148,7 @@ BinLogMsgHandler::~BinLogMsgHandler()
     file.clear();
 }
 
-void BinLogMsgHandler::handleMessage(const LogMsg & msg) const
+void BinLogMsgHandler::handleMessage(const LogMsg & msg)
 {
     CriticalBlock block(crit);
     mbuff.clear();
@@ -2501,7 +2595,7 @@ int CSysLogEventLogger::writeDataLog(size32_t datasize, byte const * data)
 
 #endif
 
-void SysLogMsgHandler::handleMessage(const LogMsg & msg) const
+void SysLogMsgHandler::handleMessage(const LogMsg & msg)
 {
     AuditType type = categoryToAuditType(msg.queryCategory());
     StringBuffer text;

system/jlib/jlog.hpp

@@ -252,9 +252,11 @@ typedef enum
 } LogMsgField;
 
 #ifdef _WIN32
-#define MSGFIELD_STANDARD LogMsgField(MSGFIELD_timeDate | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code | MSGFIELD_quote | MSGFIELD_prefix)
+#define MSGFIELD_STANDARD LogMsgField(MSGFIELD_timeDate | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code | MSGFIELD_quote | MSGFIELD_prefix | MSGFIELD_audience)
+#define MSGFIELD_LEGACY LogMsgField(MSGFIELD_timeDate | MSGFIELD_milliTime | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code | MSGFIELD_quote | MSGFIELD_prefix)
 #else
-#define MSGFIELD_STANDARD LogMsgField(MSGFIELD_timeDate | MSGFIELD_milliTime | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code | MSGFIELD_quote | MSGFIELD_prefix)
+#define MSGFIELD_STANDARD LogMsgField(MSGFIELD_timeDate | MSGFIELD_milliTime | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code | MSGFIELD_quote | MSGFIELD_prefix | MSGFIELD_audience)
+#define MSGFIELD_LEGACY LogMsgField(MSGFIELD_timeDate | MSGFIELD_milliTime | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code | MSGFIELD_quote | MSGFIELD_prefix)
 #endif
 
 inline const char * LogMsgFieldToString(LogMsgField field)
@@ -507,6 +509,10 @@ protected:
     bool                      remoteFlag;
 };
 
+unsigned getPositionOfField(unsigned logfields, unsigned positionoffield);
+unsigned getMessageFieldsFromHeader(const char * line);
+unsigned getMessageFieldsFromHeader(FILE *handle);
+
 // INTERFACES
 
 // Filter for log messages --- contains method to accept or reject messages
@@ -530,7 +536,7 @@ interface jlib_decl ILogMsgFilter : public IInterface
 interface jlib_decl ILogMsgHandler : public IInterface
 {
  public:
-    virtual void              handleMessage(const LogMsg & msg) const = 0;
+    virtual void              handleMessage(const LogMsg & msg) = 0;
     virtual bool              needsPrep() const = 0;
     virtual void              prep() = 0;
     virtual unsigned          queryMessageFields() const = 0;

system/jlib/jlog.ipp

@@ -517,7 +517,7 @@ class HandleLogMsgHandlerXML : implements HandleLogMsgHandler, public CInterface
 public:
     HandleLogMsgHandlerXML(FILE * _handle, unsigned _fields) : HandleLogMsgHandler(_handle, _fields) {}
     IMPLEMENT_IINTERFACE;
-    void                      handleMessage(const LogMsg & msg) const { CriticalBlock block(crit); msg.fprintXML(handle, messageFields); }
+    void                      handleMessage(const LogMsg & msg) { CriticalBlock block(crit); msg.fprintXML(handle, messageFields); }
     bool                      needsPrep() const { return false; }
     void                      prep() {}
     void                      addToPTree(IPropertyTree * tree) const;
@@ -528,7 +528,7 @@ class HandleLogMsgHandlerTable : implements HandleLogMsgHandler, public CInterfa
 public:
     HandleLogMsgHandlerTable(FILE * _handle, unsigned _fields) : HandleLogMsgHandler(_handle, _fields), prepped(false) {}
     IMPLEMENT_IINTERFACE;
-    void                      handleMessage(const LogMsg & msg) const { CriticalBlock block(crit); msg.fprintTable(handle, messageFields); }
+    void                      handleMessage(const LogMsg & msg) { CriticalBlock block(crit); msg.fprintTable(handle, messageFields); }
     bool                      needsPrep() const { return !prepped; }
     void                      prep() { CriticalBlock block(crit); LogMsg::fprintTableHead(handle, messageFields); prepped = true; }
     void                      addToPTree(IPropertyTree * tree) const;
@@ -566,7 +566,7 @@ class FileLogMsgHandlerXML : implements FileLogMsgHandler, public CInterface
 public:
     FileLogMsgHandlerXML(const char * _filename, const char * _headerText = 0, unsigned _fields = MSGFIELD_all, bool _append = false, bool _flushes = true) : FileLogMsgHandler(_filename, _headerText, _fields, _append, _flushes) {}
     IMPLEMENT_IINTERFACE;
-    void                      handleMessage(const LogMsg & msg) const { CriticalBlock block(crit); msg.fprintXML(handle, messageFields); if(flushes) fflush(handle); }
+    void                      handleMessage(const LogMsg & msg) { CriticalBlock block(crit); msg.fprintXML(handle, messageFields); if(flushes) fflush(handle); }
     bool                      needsPrep() const { return false; }
     void                      prep() {}
     void                      addToPTree(IPropertyTree * tree) const;
@@ -577,7 +577,7 @@ class FileLogMsgHandlerTable : implements FileLogMsgHandler, public CInterface
 public:
     FileLogMsgHandlerTable(const char * _filename, const char * _headerText = 0, unsigned _fields = MSGFIELD_all, bool _append = false, bool _flushes = true) : FileLogMsgHandler(_filename, _headerText, _fields, _append, _flushes), prepped(false) {}
     IMPLEMENT_IINTERFACE;
-    void                      handleMessage(const LogMsg & msg) const { CriticalBlock block(crit); msg.fprintTable(handle, messageFields); if(flushes) fflush(handle); }
+    void                      handleMessage(const LogMsg & msg) { CriticalBlock block(crit); msg.fprintTable(handle, messageFields); if(flushes) fflush(handle); }
     bool                      needsPrep() const { return !prepped; }
     void                      prep() { CriticalBlock block(crit); LogMsg::fprintTableHead(handle, messageFields); prepped = true; }
     void                      addToPTree(IPropertyTree * tree) const;
@@ -591,11 +591,20 @@ public:
     RollingFileLogMsgHandler(const char * _filebase, const char * _fileextn, unsigned _fields = MSGFIELD_all, bool _append = false, bool _flushes = true, const char *initialName = NULL, const char *alias = NULL, bool daily = false);
     virtual ~RollingFileLogMsgHandler();
     IMPLEMENT_IINTERFACE;
-    void                      handleMessage(const LogMsg & msg) const
+    void                      handleMessage(const LogMsg & msg)
     {
         CriticalBlock block(crit);
         checkRollover();
-        msg.fprintTable(handle, messageFields);
+        if (printHeader)
+        {
+            msg.fprintTableHead(handle, messageFields);
+            printHeader = false;
+        }
+        if (currentLogFields)  // If appending to existing log file, use same format as existing
+            msg.fprintTable(handle, currentLogFields);
+        else
+            msg.fprintTable(handle, messageFields);
+
         if(flushes) fflush(handle);
     }
     bool                      needsPrep() const { return false; }
@@ -609,11 +618,12 @@ public:
     bool                      getLogName(StringBuffer &name) const { CriticalBlock block(crit); name.append(filename); return true; }
     offset_t                  getLogPosition(StringBuffer &name) const { CriticalBlock block(crit); fflush(handle); name.append(filename); return ftell(handle); }
 protected:
-    void                      checkRollover() const;
-    void                      doRollover(bool daily, const char *forceName = NULL) const;
+    void                      checkRollover();
+    void                      doRollover(bool daily, const char *forceName = NULL);
 protected:
     mutable FILE *            handle;
     unsigned                  messageFields;
+    unsigned                  currentLogFields = 0; // When appending to log file, use that log files format (i.e. modifying during the day, doesn't cause immediate change...)
     StringAttr                alias;
     StringAttr                filebase;
     StringAttr                fileextn;
@@ -622,6 +632,7 @@ protected:
     bool                      flushes;
     mutable CriticalSection   crit;
     mutable struct tm         startTime;
+    bool printHeader = true;
 };
 
 // Implementation of handler which writes message to file in binary form
@@ -632,7 +643,7 @@ public:
     BinLogMsgHandler(const char * _filename, bool _append = false);
     virtual ~BinLogMsgHandler();
     IMPLEMENT_IINTERFACE;
-    void                      handleMessage(const LogMsg & msg) const;
+    void                      handleMessage(const LogMsg & msg);
     bool                      needsPrep() const { return false; }
     void                      prep() {}
     void                      addToPTree(IPropertyTree * tree) const;
@@ -661,7 +672,7 @@ class SysLogMsgHandler : implements ILogMsgHandler, public CInterface
 public:
     SysLogMsgHandler(ISysLogEventLogger * _logger, unsigned _fields) : logger(_logger), fields(_fields) {}
     IMPLEMENT_IINTERFACE;
-    void                      handleMessage(const LogMsg & msg) const;
+    void                      handleMessage(const LogMsg & msg);
     bool                      needsPrep() const { return false; }
     void                      prep() {}
     void                      addToPTree(IPropertyTree * tree) const;
@@ -683,7 +694,7 @@ class LogMsgMonitor : public CInterface
 {
 public:
     LogMsgMonitor(ILogMsgFilter * _filter, ILogMsgHandler * _handler) : filter(_filter), handler(_handler) {}
-    void                      processMessage(const LogMsg & msg) const { if(filter->includeMessage(msg)) handler->handleMessage(msg); }
+    void                      processMessage(const LogMsg & msg) { if(filter->includeMessage(msg)) handler->handleMessage(msg); }
     ILogMsgFilter *           queryFilter() const { return filter; }
     ILogMsgFilter *           getFilter() const { return LINK(filter); }
     ILogMsgHandler *          queryHandler() const { return handler; }

system/jlib/jmisc.cpp

@@ -214,7 +214,7 @@ jlib_decl void openLogFile(StringBuffer & resolvedFS, const char *filename, unsi
     lf->setAppend(append);
     lf->setCompleteFilespec(filename);//user specified log filespec
     lf->setMaxDetail(detail ? detail : DefaultDetail);
-    lf->setMsgFields(MSGFIELD_timeDate | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code);
+    lf->setMsgFields(MSGFIELD_STANDARD);
     lf->beginLogging();
     resolvedFS.set(lf->queryLogFileSpec());
 }

system/mp/mplog.cpp

@@ -367,7 +367,7 @@ LinkToParentLogMsgHandler::~LinkToParentLogMsgHandler()
 }
 
 
-void LinkToParentLogMsgHandler::handleMessage(const LogMsg & msg) const
+void LinkToParentLogMsgHandler::handleMessage(const LogMsg & msg)
 {
     CMessageBuffer out;
     msg.serialize(out);

system/mp/mplog.ipp

@@ -138,7 +138,7 @@ public:
     LinkToParentLogMsgHandler(MPLogId _cid, MPLogId _pid, INode * _parentNode, bool _connected) : parentNode(_parentNode), cid(_cid), pid(_pid), receiverThread(new LogMsgFilterReceiverThread(_pid, _parentNode)), connected(_connected) { receiverThread->setHandler(this); }
     ~LinkToParentLogMsgHandler();
     IMPLEMENT_IINTERFACE;
-    void                      handleMessage(LogMsg const & msg) const;
+    void                      handleMessage(LogMsg const & msg);
     bool                      needsPrep() const { return false; }
     void                      prep() {}
     void                      addToPTree(IPropertyTree * tree) const;