HPCC-14702 Add logging service to HPCC repo

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

cmake_modules/commonSetup.cmake

@@ -102,6 +102,8 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       option(USE_TBB "Enable Threading Building Block support" ON)
   endif()
 
+  option(LOGGING_SERVICE "Configure use of logging service" ON)
+
   option(USE_OPTIONAL "Automatically disable requested features with missing dependencies" ON)
 
 

esp/CMakeLists.txt

@@ -28,4 +28,7 @@ HPCC_ADD_SUBDIRECTORY (test "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (tools "CLIENTTOOLS")
 HPCC_ADD_SUBDIRECTORY (xslt)
 HPCC_ADD_SUBDIRECTORY (esdllib)
+if (LOGGING_SERVICE)
+HPCC_ADD_SUBDIRECTORY (logging)
+endif()
 

esp/logging/CMakeLists.txt

@@ -0,0 +1,18 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+HPCC_ADD_SUBDIRECTORY (loggingagent)
+HPCC_ADD_SUBDIRECTORY (loggingmanager)
+HPCC_ADD_SUBDIRECTORY (test)

esp/logging/LogFailSafe.cpp

@@ -0,0 +1,319 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#pragma warning(disable:4786)
+
+/* Review notes: this file is modified based on:
+ * HPCC-Platform/esp/clients/LoggingClient/LogFailSafe.cpp
+
+   3 methods are added to ILogFailSafe: Add(), RolloverAllLogs(), PopPendingLogRecord().
+   1 extern function is added: createFailSafeLogger()
+   3 member variables are added to CLogFailSafe: m_LogService, m_critSec, m_PendingLogs
+   5 methods and one constructor are added to CLogFailSafe: Add(), RolloverAllLogs(), PopPendingLogRecord(),
+       loadPendingLogReqsFromExistingLogFiles(), generateNewFileNames()
+   Minor changes are made in the existing methods.
+ */
+#include "LogFailSafe.hpp"
+#include "jmisc.hpp"
+#include "soapbind.hpp"
+
+#define RECEIVING "_acked_"
+#define SENDING   "_sending_"
+
+const char* const RolloverExt=".old";
+const unsigned long SAFE_ROLLOVER_THRESHOLD = 500000L;
+const unsigned int TRACE_PENDING_LOGS_MIN = 10;
+const unsigned int TRACE_PENDING_LOGS_MAX = 50;
+
+extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(const char* logType, const char* logsdir)
+{
+    return new CLogFailSafe(logType, logsdir);
+}
+
+extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(const char* pszService, const char* logType, const char* logsdir)
+{
+    return new CLogFailSafe(pszService, logType, logsdir && *logsdir ? logsdir : "./FailSafeLogs");
+}
+
+CLogFailSafe::CLogFailSafe()
+{
+}
+
+CLogFailSafe::CLogFailSafe(const char* logType, const char* logsdir) : m_LogType(logType), m_logsdir(logsdir)
+{
+    loadFailed(logType);
+    createNew(logType);
+}
+
+CLogFailSafe::CLogFailSafe(const char* pszService, const char* logType, const char* logsdir)
+    : m_LogService(pszService), m_LogType(logType), m_logsdir(logsdir)
+{
+    loadPendingLogReqsFromExistingLogFiles();
+
+    StringBuffer send, receive;
+    generateNewFileNames(send, receive);
+
+    m_Added.Open(m_logsdir.str(), send.str(), NULL);
+    m_Cleared.Open(m_logsdir.str(), receive.str(), NULL);
+}
+
+CLogFailSafe::~CLogFailSafe()
+{
+    ESPLOG(LogMax, "CLogFailSafe::~CLogFailSafe()");
+    m_Added.Close();
+    m_Cleared.Close();
+}
+
+bool CLogFailSafe::FindOldLogs()
+{
+    if(m_UnsentLogs.ordinality())
+        return true;
+    return false;
+}
+
+void CLogFailSafe::LoadOldLogs(StringArray& oldLogData)
+{
+    ForEachItemIn(aidx, m_UnsentLogs)
+    {
+        oldLogData.append(m_UnsentLogs.item(aidx));
+    }
+}
+
+void CLogFailSafe::loadPendingLogReqsFromExistingLogFiles()
+{
+    VStringBuffer fileName("%s%s%s*.log", m_LogService.str(), m_LogType.str(), SENDING);
+    Owned<IDirectoryIterator> di = createDirectoryIterator(m_logsdir.str(), fileName.str());
+    ForEach (*di)
+    {
+        IFile &file = di->query();
+
+        StringBuffer ackedName;
+        GuidSet ackedSet;
+        getReceiveFileName(file.queryFilename(),ackedName);
+        CLogSerializer ackedLog(ackedName.str());
+        ackedLog.loadAckedLogs(ackedSet);
+
+        CLogSerializer sendLog(file.queryFilename());
+        unsigned long total_missed = 0;
+        {//scope needed for critical block below
+            CriticalBlock b(m_critSec); //since we plan to use m_PendingLogs
+            sendLog.loadSendLogs(ackedSet, m_PendingLogs, total_missed);
+        }
+
+        if (total_missed == 0)
+        {
+            ackedLog.Rollover(RolloverExt);
+            sendLog.Rollover(RolloverExt);
+        }
+    }
+}
+
+void CLogFailSafe::generateNewFileNames(StringBuffer& sendingFile, StringBuffer& receivingFile)
+{
+    StringBuffer GUID;
+    GenerateGUID(GUID);
+
+    StringBuffer tmp;
+    tmp.append(m_LogService).append(m_LogType);
+
+    sendingFile.append(tmp).append(SENDING).append(GUID).append(".log");
+    receivingFile.append(tmp).append(RECEIVING).append(GUID).append(".log");
+}
+
+bool CLogFailSafe::PopPendingLogRecord(StringBuffer& GUID, StringBuffer& cache)
+{
+    CriticalBlock b(m_critSec);
+
+    GuidMap::iterator it = m_PendingLogs.begin();
+    if (it == m_PendingLogs.end())
+        return false;
+
+    GUID.clear().append( (*it).first.c_str() );
+    cache.clear().append( (*it).second.c_str() );
+    m_PendingLogs.erase(it);
+
+    unsigned int nPendingLogs = m_PendingLogs.size();
+    if (nPendingLogs && (nPendingLogs < TRACE_PENDING_LOGS_MIN || (nPendingLogs % TRACE_PENDING_LOGS_MAX == 0)))
+        ESPLOG(LogNormal, "%u logs pending", nPendingLogs);
+
+    return true;
+}
+
+void CLogFailSafe::createNew(const char* logType)
+{
+    StringBuffer UniqueID;
+    GenerateGUID(UniqueID);
+    UniqueID.append(".log");
+
+    StringBuffer send(logType),recieve(logType);
+
+    send.append("_sending");
+    recieve.append("_recieving");
+
+    m_Added.Open(m_logsdir.str(),UniqueID,send.str());
+    m_Cleared.Open(m_logsdir.str(),UniqueID,recieve.str());
+}
+
+void CLogFailSafe::loadFailed(const char* logType)
+{
+    StringBuffer fileName;
+    fileName.appendf("%s_sending*.log",logType);
+
+    Owned<IDirectoryIterator> di = createDirectoryIterator(m_logsdir.str(), fileName.str());
+    ForEach (*di)
+    {
+        IFile &file = di->query();
+
+        StringBuffer recieveName;
+        GuidMap recieve_map;
+        getReceiveFileName(file.queryFilename(),recieveName);
+
+        CRecieveLogSerializer recieveLog(recieveName.str());
+        recieveLog.LoadDataMap(recieve_map);
+
+        CSendLogSerializer sendLog(file.queryFilename());
+        sendLog.LoadDataMap(recieve_map,m_UnsentLogs);
+    }
+}
+
+StringBuffer& CLogFailSafe::getReceiveFileName(const char* sendFileName, StringBuffer& receiveName)
+{
+    if (sendFileName)
+    {
+        receiveName.append(sendFileName);
+        receiveName.replaceString(SENDING, RECEIVING);
+    }
+    return receiveName;
+}
+
+StringBuffer& CLogFailSafe::GenerateGUID(StringBuffer& GUID, const char* seed)
+{
+    GUID.appendf("%u",getRandom());
+    while (GUID.length() < 10)
+        GUID.insert(0,'0');
+
+    addFileTimestamp(GUID);
+    if(seed!=NULL && *seed!='\0')
+        GUID.appendf(".%s",seed);
+    return GUID;
+}
+
+void CLogFailSafe::SplitLogRecord(const char* requestStr,StringBuffer& GUID, StringBuffer& Cache)
+{
+    SplitRecord(requestStr,GUID,Cache);
+}
+
+void CLogFailSafe::Add(const char* GUID,IInterface& pIn)
+{
+    CSoapRequestBinding* reqObj = dynamic_cast<CSoapRequestBinding*>(&pIn);
+    if (reqObj == 0)
+        throw MakeStringException(-1, "Unable to cast interface to SoapBindind");
+
+    StringBuffer dataStr;
+    reqObj->serializeContent(NULL,dataStr,NULL);
+    Add(GUID, dataStr);
+}
+
+void CLogFailSafe::Add(const char* GUID, const StringBuffer& strContents)
+{
+    VStringBuffer dataStr("<cache>%s</cache>", strContents.str());
+
+    unsigned long item_count = m_Added.getItemCount();
+    if (item_count > SAFE_ROLLOVER_THRESHOLD)
+        SafeRollover();
+
+    m_Added.Append(GUID, dataStr.str());
+}
+
+void CLogFailSafe::AddACK(const char* GUID)
+{
+    m_Cleared.Append(GUID, "");
+
+    CriticalBlock b(m_critSec);
+    GuidMap::iterator it = m_PendingLogs.find(GUID);
+    if (it != m_PendingLogs.end())
+        m_PendingLogs.erase(it);
+}
+
+void CLogFailSafe::RollCurrentLog()
+{
+    m_Added.Rollover(RolloverExt);
+    m_Cleared.Rollover(RolloverExt);
+}
+
+void CLogFailSafe::SafeRollover()
+{
+    StringBuffer send, receive;
+    generateNewFileNames(send, receive);
+
+    // Rolling over m_Added first is desirable here beccause requests being written to the new tank file before
+    // m_Cleared finishes rolling all haven't been sent yet (because the sending thread is here busy rolling).
+    m_Added.SafeRollover  (m_logsdir.str(), send.str(), NULL,   RolloverExt);
+    m_Cleared.SafeRollover(m_logsdir.str(), receive.str(), NULL, RolloverExt);
+}
+
+void CLogFailSafe::RollOldLogs()
+{
+    StringBuffer filesToFind;
+    filesToFind.appendf("%s*.log",m_LogType.str());
+
+    Owned<IDirectoryIterator> di = createDirectoryIterator(m_logsdir.str(), filesToFind.str());
+    ForEach (*di)
+    {
+        IFile &file = di->query();
+
+        StringBuffer fileName;
+        ExtractFileName(file.queryFilename(),fileName);
+        if (fileName.length() && strcmp(fileName.str(),m_Added.queryFileName()) != 0 &&  strcmp(fileName.str(),m_Cleared.queryFileName()) != 0 )
+        {
+            fileName.replaceString(".log", RolloverExt);
+            file.rename(fileName.str());
+        }
+    }
+}
+
+//Rename existing .log files (except for current added/cleared log files) to .old files
+void CLogFailSafe::RolloverAllLogs()
+{
+    VStringBuffer filesToFind("%s%s*.log", m_LogService.str(), m_LogType.str());
+    Owned<IDirectoryIterator> di = createDirectoryIterator(m_logsdir.str(), filesToFind.str());
+    ForEach (*di)
+    {
+        IFile &file = di->query();
+
+        StringBuffer fileName;
+        CLogSerializer::extractFileName(file.queryFilename(), fileName);
+        if (fileName.length() && !streq(fileName.str(), m_Added.queryFileName()) &&
+            !streq(fileName.str(), m_Cleared.queryFileName()))
+        {
+            fileName.replaceString(".log", RolloverExt);
+            file.rename(fileName.str());
+        }
+    }
+}
+
+StringBuffer& CLogFailSafe::ExtractFileName(const char* fileName,StringBuffer& FileName)
+{
+    StringBuffer tmp(fileName);
+    for(int i = tmp.length() - 1; i >=0; i--)
+    {
+        if(tmp.charAt(i) == '\\' || tmp.charAt(i) == '/')
+            break;
+        FileName.insert(0,tmp.charAt(i));
+    }
+    return FileName;
+}

esp/logging/LogFailSafe.hpp

@@ -0,0 +1,92 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#pragma warning (disable : 4786)
+#ifndef _LOGFAILSAFE_HPP__
+#define _LOGFAILSAFE_HPP__
+
+#include "jlib.hpp"
+#include "jstring.hpp"
+#include "jutil.hpp" // for StringArray
+#include "loggingcommon.hpp"
+#include "LogSerializer.hpp"
+
+interface ILogFailSafe : IInterface
+{
+    virtual void Add(const char*, const StringBuffer& strContents)=0;//
+    virtual void Add(const char*,IInterface& pIn)=0;
+    virtual StringBuffer& GenerateGUID(StringBuffer& GUID,const char* seed="") = 0;
+    virtual void AddACK(const char* GUID)=0;
+    virtual void RollCurrentLog()=0;
+    virtual void RollOldLogs()=0;
+    virtual bool FindOldLogs() = 0;
+    virtual void LoadOldLogs(StringArray& oldLogData) = 0;
+    virtual void SplitLogRecord(const char* requestStr,StringBuffer& GUID, StringBuffer& Cache)=0;
+    virtual void SafeRollover() = 0;
+
+    virtual void RolloverAllLogs()=0;//
+    virtual bool PopPendingLogRecord(StringBuffer& GUID, StringBuffer& cache) = 0;//
+};
+
+extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(const char* logType="", const char* logsdir="./logs");
+extern LOGGINGCOMMON_API ILogFailSafe* createFailSafeLogger(const char* pszService, const char* logType="", const char* logsdir="./logs");
+
+class CLogFailSafe : public CInterface,  implements ILogFailSafe
+{
+    CLogSerializer m_Added;
+    CLogSerializer m_Cleared;
+    static CLogFailSafe* m_Instance;
+    StringBuffer m_LogType;
+    StringArray m_UnsentLogs;
+    StringBuffer m_logsdir;
+    StringBuffer m_LogService;//
+
+    CriticalSection m_critSec;//
+    GuidMap m_PendingLogs;//
+
+private:
+    void createNew(const char* logType);
+    void loadFailed(const char* logType);
+    StringBuffer& getReceiveFileName(const char* sendFileName, StringBuffer& recieveName);
+    StringBuffer& ExtractFileName(const char* fileName,StringBuffer& FileName);
+
+    void loadPendingLogReqsFromExistingLogFiles();//
+    void generateNewFileNames(StringBuffer& sendingFile, StringBuffer& receivingFile);//
+public:
+    IMPLEMENT_IINTERFACE;
+    CLogFailSafe();
+    CLogFailSafe(const char* logType, const char* logsdir);
+    CLogFailSafe(const char* pszService, const char* logType, const char* logsdir);
+
+    virtual ~CLogFailSafe();
+    StringBuffer& GenerateGUID(StringBuffer& GUID,const char* seed="");
+    virtual void Add(const char*, const StringBuffer& strContents);//
+    virtual void Add(const char*,IInterface& pIn);
+    virtual void AddACK(const char* GUID);
+    virtual void RollCurrentLog();
+    virtual void RollOldLogs();
+    virtual bool FindOldLogs();
+    virtual void LoadOldLogs(StringArray& oldLogData);
+    virtual void SplitLogRecord(const char* requestStr,StringBuffer& GUID, StringBuffer& Cache);
+
+    virtual void SafeRollover();
+
+    virtual void RolloverAllLogs();//
+    virtual bool PopPendingLogRecord(StringBuffer& GUID, StringBuffer& cache);//
+};
+
+#endif // !_LOGFAILSAFE_HPP__

esp/logging/LogSerializer.cpp

@@ -0,0 +1,490 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#pragma warning(disable : 4786)
+#include "LogSerializer.hpp"
+#include "jexcept.hpp"
+#include "jfile.hpp"
+#include "jlog.hpp"
+
+/* Review notes: this file is modified based on:
+ * HPCC-Platform/esp/clients/LoggingClient/LogSerializer.cpp
+
+   5 methods are added to CLogSerializer: splitLogRecord(), getItemCount(), extractFileName(),
+       loadSendLogs(), loadAckedLogs.
+   Minor changes are made in the existing methods.
+ */
+
+#define TRACE_INTERVAL 100
+
+CLogSerializer::CLogSerializer()
+{
+    Init();
+}
+
+CLogSerializer::CLogSerializer(const char* fileName)
+{
+    m_FilePath.append(fileName);
+    Init();
+    extractFileName(m_FilePath, m_FileName);//
+}
+
+StringBuffer& CLogSerializer::extractFileName(const char* fullName,StringBuffer& fileName)
+{
+    StringBuffer tmp(fullName);
+    for(unsigned i = tmp.length(); i-- > 0; )
+    {
+        if(tmp.charAt(i) == '\\' || tmp.charAt(i) == '/')
+            break;
+        fileName.insert(0, tmp.charAt(i));
+    }
+    return fileName;
+}
+
+void CLogSerializer::Init()
+{
+    m_bytesWritten  = 0;
+    m_ItemCount     = 0;
+    m_fileio        = 0;
+    m_file          = 0;
+}
+
+CLogSerializer::~CLogSerializer()
+{
+    DBGLOG("CLogSerializer::~CLogSerializer()");
+    Close();
+}
+
+void CLogSerializer::Append(const char* GUID, const char* Data)
+{
+    StringBuffer toWrite,size;
+
+    toWrite.appendf("%s\t%s\r\n",GUID,Data);
+    size.appendf("%d",toWrite.length());
+    while (size.length() < 8)
+        size.insert(0,'0');
+
+    size.append("\t");
+    toWrite.insert(0,size.str());
+
+    //optimize
+    CriticalBlock b(crit);
+    m_ItemCount++;
+    m_bytesWritten += m_fileio->write(m_bytesWritten, toWrite.length(), toWrite.str());
+}
+
+void CLogSerializer::Remove(const char* GUID)
+{
+}
+
+void CLogSerializer::Open(const char*Directory,const char* NewFileName,const char* Prefix)
+{
+    m_FilePath.clear();
+    m_FilePath.append(Directory);
+    if (!EnsureDirectory(m_FilePath))
+        throw MakeStringException(-1,"Unable to create directory at %s.",m_FilePath.str());
+
+    m_FilePath.append("/");
+
+    m_FileName.clear();
+    if (Prefix && *Prefix)
+        m_FileName.append(Prefix).append("_");
+    m_FileName.append(NewFileName);
+    m_FilePath.append(m_FileName);
+
+    m_file = createIFile(m_FilePath.str());
+    m_fileio  =  m_file->open(IFOcreate);
+    if (m_fileio == 0)
+        throw MakeStringException(-1, "Unable to open logging file %s",m_FilePath.str());
+    else
+        DBGLOG("Tank file %s successfully created", m_FilePath.str());
+}
+
+bool CLogSerializer::EnsureDirectory(StringBuffer& Dir)
+{
+    try
+    {
+        Owned<IFile> pDirectory = createIFile(Dir.str());
+        if(pDirectory->exists() == true)
+            return true;
+        return pDirectory->createDirectory();
+    }
+    catch(IException *ex)
+    {
+        ex->Release();
+    }
+    return false;
+}
+
+void CLogSerializer::Close()
+{
+    if(m_fileio)
+    {
+        m_fileio->Release();
+        m_fileio = 0;
+    }
+
+    if(m_file)
+    {
+        m_file->Release();
+        m_file = 0;
+    }
+
+    m_bytesWritten  = 0;//
+    m_ItemCount     = 0;//
+}
+
+void CLogSerializer::Rollover(const char* ClosedPrefix)
+{
+    Close();
+    Owned<IFile> file = createIFile(m_FilePath.str());
+    if(file.get() && file->exists() == true)
+    {
+        StringBuffer newFileName;
+        GetRolloverFileName(m_FileName,newFileName,ClosedPrefix);
+        file->rename(newFileName.str());
+    }
+}
+
+void CLogSerializer::SafeRollover(const char*Directory,const char* NewFileName,const char* Prefix, const char* ClosedPrefix)
+{
+    CriticalBlock b(crit);
+    Rollover(ClosedPrefix);
+    Init();
+    Open(Directory, NewFileName, Prefix);
+}
+
+void CLogSerializer::splitLogRecord(MemoryBuffer& rawdata, StringBuffer& GUID, StringBuffer& line)//
+{
+    //send log buffer should be in the form of 2635473460.05_01_12_16_13_57\t<cache>...</cache>
+    //parse it into GUID and line (as <cache>...</cache>)
+
+    //receive log buffer should be in the form of 2515777767.12_11_03_08_25_29\t
+    //we want to extract the GUID only
+
+    const char* begin = rawdata.toByteArray(); //no string termination character \0
+    int len = rawdata.length();
+    if (begin && len>0)
+    {
+        const char* p = begin;
+        const char* end = begin + len;
+
+        while (*p && *p != '\t' && p < end)
+            p++;
+
+        GUID.append(p-begin, begin);
+
+        if (++p < end)
+            line.append(end-p, p);
+    }
+}
+
+void CLogSerializer::loadSendLogs(GuidSet& ackSet, GuidMap& missedLogs, unsigned long& total_missed)//
+{
+    try
+    {
+        Close(); //release old file io, if any
+        m_file = createIFile(m_FilePath.str());
+        m_fileio = m_file->open(IFOread);
+        if (m_fileio == 0)
+            throw MakeStringException(-1, "Unable to open logging file %s",m_FilePath.str());
+
+        offset_t finger = 0;
+        total_missed = 0;
+        while(true)
+        {
+            char dataSize[9];
+            memset(dataSize, 0, 9);
+            size32_t bytesRead = m_fileio->read(finger,8,dataSize);
+            if(bytesRead==0)
+                break;
+
+            MemoryBuffer data;
+            int dataLen = atoi(dataSize);
+            finger+=9;
+            bytesRead = m_fileio->read(finger,dataLen,data.reserveTruncate(dataLen));
+            if(bytesRead==0)
+                break;
+
+            StringBuffer GUID,lostlogStr;
+            splitLogRecord(data,GUID,lostlogStr);
+
+            if (ackSet.find(GUID.str())==ackSet.end() && missedLogs.find(GUID.str()) == missedLogs.end())
+            {
+                if(total_missed % TRACE_INTERVAL == 0)
+                    DBGLOG("Miss #%lu GUID: <%s>", total_missed, GUID.str());
+                missedLogs[GUID.str()] = lostlogStr.str();
+                total_missed++;
+            }
+            finger+=dataLen;
+        }
+    }
+    catch(IException* ex)
+    {
+        StringBuffer errorStr;
+        ex->errorMessage(errorStr);
+        ERRLOG("Exception caught within CSendLogSerializer::LoadDataMap: %s",errorStr.str());
+        ex->Release();
+    }
+    catch(...)
+    {
+        DBGLOG("Unknown Exception thrown in CSendLogSerializer::LoadDataMap");
+    }
+    Close();
+}
+
+void CLogSerializer::loadAckedLogs(GuidSet& ackedLogs)//
+{
+    try
+    {
+        Close(); //release old file io, if any
+        m_file = createIFile(m_FilePath.str());
+        m_fileio = m_file->open(IFOread);
+        if (m_fileio == 0)
+            throw MakeStringException(-1, "Unable to open logging file %s",m_FilePath.str());
+
+        offset_t finger = 0;
+        m_ItemCount = 0;
+        while(true)
+        {
+            char dataSize[9];
+            memset(dataSize, 0, 9);
+            size32_t bytesRead = m_fileio->read(finger,8,dataSize);
+            if(bytesRead==0)
+                break;
+
+            MemoryBuffer data;
+            int dataLen = atoi(dataSize);
+            finger+=9;
+            bytesRead = m_fileio->read(finger,dataLen,data.reserveTruncate(dataLen));
+            if(bytesRead==0)
+                break;
+
+            StringBuffer GUID, line;
+            splitLogRecord(data, GUID, line);
+            ackedLogs.insert(GUID.str());
+            m_ItemCount++;
+
+            finger+=dataLen;
+        }
+        DBGLOG("Total acks loaded %lu", m_ItemCount);
+    }
+    catch(IException* ex)
+    {
+        StringBuffer errorStr;
+        ex->errorMessage(errorStr);
+        ERRLOG("Exception caught within CLogSerializer::loadAckedLogs: %s",errorStr.str());
+        ex->Release();
+    }
+    catch(...)
+    {
+        DBGLOG("Unknown Exception thrown in CLogSerializer::loadAckedLogs");
+    }
+    Close();
+}
+
+StringBuffer& CLogSerializer::GetRolloverFileName(StringBuffer& oldFile, StringBuffer& newfile, const char* newExtension)
+{
+    newfile.append(oldFile);
+    newfile.replaceString(".log",newExtension);
+    return newfile;
+}
+
+void CLogSerializer::Remove()
+{
+    Close();
+    Owned<IFile> file = createIFile(m_FilePath.str());
+    if(file.get() && file->exists() == true)
+        file->remove();
+}
+
+__int64 CLogSerializer::WroteBytes()
+{
+    CriticalBlock b(crit);
+    return m_bytesWritten;
+}
+
+void CSendLogSerializer::LoadDataMap(GuidMap& ACKMap,StringArray& MissedLogs)
+{
+    try
+    {
+        m_file = createIFile(m_FilePath.str());
+        m_fileio = m_file->open(IFOread);
+        if (m_fileio == 0)
+            throw MakeStringException(-1, "Unable to open logging file %s",m_FilePath.str());
+        else
+            DBGLOG("File %s successfully opened", m_FilePath.str());
+
+        long finger,bytesRead;
+        finger = bytesRead = 0;
+
+        bool bOk = true;
+        MemoryBuffer dataSize,data;
+        StringBuffer GUID,lostlogStr;
+        int dataLen;
+        unsigned int total = 0;
+        unsigned int total_missed = 0;
+        while(bOk)
+        {
+            bytesRead = m_fileio->read(finger,8,dataSize.reserveTruncate(8));
+            if(bytesRead==0)
+                break;
+
+            finger+=9;
+            dataLen = atoi(dataSize.toByteArray());
+
+            bytesRead = m_fileio->read(finger,dataLen,data.reserveTruncate(dataLen));
+            if(bytesRead==0)
+                break;
+            LoadMap(data,GUID,lostlogStr);
+            if(total % TRACE_INTERVAL == 0)
+            {
+                DBGLOG("Checking log #%u", total);
+                DBGLOG("{%s}", GUID.str());
+            }
+            total++;
+            int i = MissedLogs.find(lostlogStr.str());
+            if (!(*(ACKMap[GUID.str()].c_str())) && MissedLogs.find(lostlogStr.str()) == -1)
+            {
+                if(total_missed % TRACE_INTERVAL == 0)
+                {
+                    DBGLOG("Miss #%u", total_missed);
+                    DBGLOG("<%s>", GUID.str());
+                }
+                MissedLogs.append(lostlogStr.str());
+                total_missed++;
+            }
+            finger+=dataLen;
+            data.clear();
+            dataSize.clear();
+            GUID.clear();
+            lostlogStr.clear();
+        }
+        DBGLOG("Total logs checked %u, total missed %u", total, total_missed);
+    }
+    catch(IException* ex)
+    {
+        StringBuffer errorStr;
+        ex->errorMessage(errorStr);
+        ERRLOG("Exception caught within CSendLogSerializer::LoadDataMap: %s",errorStr.str());
+        ex->Release();
+    }
+    catch(...)
+    {
+        DBGLOG("Unknown Exception thrown in CSendLogSerializer::LoadDataMap");
+    }
+    Close();
+}
+
+void CSendLogSerializer::LoadMap(MemoryBuffer& rawdata,StringBuffer& GUID, StringBuffer& line)
+{
+    line.append(rawdata.length() -1, rawdata.toByteArray());
+    const char* strLine = line.str();
+    while(*strLine && *strLine != '\t' && *strLine != '\0')
+    {
+        GUID.append(*strLine);
+        strLine++;
+    }
+}
+
+void SplitRecord(const char*  strLine, StringBuffer& GUID, StringBuffer& Cache)
+{
+    if(strLine==NULL || *strLine=='\0')
+        return;
+
+    while(*strLine && *strLine != '\t' && *strLine != '\0')
+    {
+        GUID.append(*strLine);
+        strLine++;
+    }
+    strLine++;
+    Cache.appendf("%s",strLine);
+}
+
+void CRecieveLogSerializer::LoadMap(MemoryBuffer& rawdata,GuidMap& GUIDmap, bool printTrace)
+{
+    //buffer chould be in the form of 000000030\t2515777767.12_11_03_08_25_29\r
+    //we want to extract the GUID only....
+    StringBuffer line,GUID;
+    line.append(rawdata.length() -1, rawdata.toByteArray());
+    const char* strLine = line.str();
+    while(*strLine && *strLine != '\t' && *strLine != '\0')
+    {
+        GUID.append(*strLine);
+        strLine++;
+    }
+    if(printTrace)
+        DBGLOG("[%s]", GUID.str());
+    GUIDmap[GUID.str()] = "1";
+}
+
+void CRecieveLogSerializer::LoadDataMap(GuidMap& GUIDmap)
+{
+    try
+    {
+        m_file = createIFile(m_FilePath.str());
+        m_fileio = m_file->open(IFOread);
+        if (m_fileio == 0)
+            throw MakeStringException(-1, "Unable to open logging file %s",m_FilePath.str());
+        else
+            DBGLOG("File %s successfully opened", m_FilePath.str());
+
+        long finger,bytesRead,dataLen;
+        finger = bytesRead = dataLen = 0;
+        MemoryBuffer filecontents,dataSize,data;
+        bool bOk = true;
+        unsigned int total = 0;
+        while(bOk)
+        {
+            bytesRead = m_fileio->read(finger,8,dataSize.reserveTruncate(8));
+            if(bytesRead==0)
+                break;
+
+            finger+=9;
+            dataLen = atoi(dataSize.toByteArray());
+
+            bytesRead = m_fileio->read(finger,dataLen,data.reserveTruncate(dataLen));
+            if(bytesRead==0)
+                break;
+            bool printTrace = false;
+            if(total % TRACE_INTERVAL == 0)
+            {
+                DBGLOG("Loading ack #%u", total);
+                printTrace = true;
+            }
+            LoadMap(data,GUIDmap,printTrace);
+            total++;
+
+            finger+=dataLen;
+            data.clear();
+            dataSize.clear();
+        }
+        DBGLOG("Total acks loaded %u", total);
+    }
+    catch(IException* ex)
+    {
+        StringBuffer errorStr;
+        ex->errorMessage(errorStr);
+        ERRLOG("Exception caught within CRecieveLogSerializer::LoadDataMap: %s",errorStr.str());
+        ex->Release();
+    }
+    catch(...)
+    {
+        DBGLOG("Unknown Exception thrown in CRecieveLogSerializer::LoadDataMap");
+    }
+    Close();
+}

esp/logging/LogSerializer.hpp

@@ -0,0 +1,95 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _LOGSERIALIZER_HPP__
+#define _LOGSERIALIZER_HPP__
+
+#include "jlib.hpp"
+#include "jstring.hpp"
+#include "jutil.hpp" // for StringArray
+#include "jmutex.hpp"
+#include <map>
+#include <string>
+#include <set> //for GuidSet
+
+typedef std::set<std::string> GuidSet;//
+typedef std::map<std::string, std::string> GuidMap;
+
+class CLogSerializer : public CInterface
+{
+    __int64 m_bytesWritten;
+    unsigned long m_ItemCount;//
+    CriticalSection crit;
+protected:
+    IFile* m_file;
+    IFileIO* m_fileio;
+    StringBuffer m_FileName;
+    StringBuffer m_FilePath;
+protected:
+    __int64 WroteBytes();
+    void Init();
+    StringBuffer& GetRolloverFileName(StringBuffer& oldFile, StringBuffer& newfile, const char* newExtension);
+public:
+    bool EnsureDirectory(StringBuffer& Dir);
+public:
+    IMPLEMENT_IINTERFACE;
+    CLogSerializer();
+    CLogSerializer(const char* fileName);
+    virtual ~CLogSerializer();
+    void Open(const char* Directory,const char* NewFileName,const char* Prefix);
+    void Close();
+    void Remove();
+    void Rollover(const char* ClosedPrefix);
+    void Append(const char* GUID, const char* Data);
+    void Remove(const char* GUID);
+    virtual void SplitRecord(StringBuffer& FullStr, StringBuffer& GUID, StringBuffer& Cache){}
+    static void splitLogRecord(MemoryBuffer& rawdata,StringBuffer& GUID, StringBuffer& data);//
+    const char* queryFileName(){return m_FileName.str();};
+    const char* queryFilePath(){return m_FilePath.str();};
+
+    void SafeRollover(const char* Directory,const char* NewFileName,const char* Prefix, const char* ClosedPrefix);
+
+    unsigned long getItemCount() const { return m_ItemCount; }//
+    static StringBuffer& extractFileName(const char* fullName, StringBuffer& fileName);//
+    virtual void loadSendLogs(GuidSet& ACKSet, GuidMap& MissedLogs, unsigned long& total_missed);//
+    virtual void loadAckedLogs(GuidSet& ReceiveMap);//
+};
+
+class CSendLogSerializer : public CLogSerializer
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    CSendLogSerializer();
+    CSendLogSerializer(const char* fileName) : CLogSerializer(fileName) {};
+
+    virtual void LoadDataMap(GuidMap& ACKMap,StringArray& MissedLogs);
+    void LoadMap(MemoryBuffer& rawdata,StringBuffer& GUID, StringBuffer& data);
+};
+
+void SplitRecord(const char* FullStr, StringBuffer& GUID, StringBuffer& Cache);
+
+class CRecieveLogSerializer : public CLogSerializer
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    CRecieveLogSerializer();
+    CRecieveLogSerializer(const char* fileName) : CLogSerializer(fileName){}
+    virtual void LoadDataMap(GuidMap& GUIDmap);
+    void LoadMap(MemoryBuffer& data,GuidMap& GUIDmap, bool printTrace = false);
+};
+
+#endif // !LOGSERIALIZER

esp/logging/LoggingErrors.hpp

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef __LOGGING_ERRORS_HPP__
+#define __LOGGING_ERRORS_HPP__
+
+#include "EspErrors.hpp"
+
+namespace EspLoggingErrors
+{
+    const unsigned int Base = EspCoreErrors::Base+2500;       //3500 -- pls don't change this....the web is coded to expect the following codes
+    const unsigned int ConfigurationFileEntryError  = Base+1;  // Required entry in esp.xml missing or invalid
+    const unsigned int LoadLoggingLibraryError      = Base+2;
+    const unsigned int AddToLoggingQueueFailed      = Base+3;
+    const unsigned int GetTransactionSeedFailed     = Base+4;
+    const unsigned int UpdateLogFailed              = Base+5;
+    const unsigned int WSLoggingAccessDenied        = Base+6;
+}
+
+#endif //__LOGGING_ERRORS_HPP__

esp/logging/loggingagent/CMakeLists.txt

@@ -0,0 +1,16 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+HPCC_ADD_SUBDIRECTORY (espserverloggingagent)

esp/logging/loggingagent/espserverloggingagent/CMakeLists.txt

@@ -0,0 +1,70 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: espserverloggingagent
+#####################################################
+# Description:
+# ------------
+#	 Cmake Input File for espserverloggingagent
+#####################################################
+
+project( espserverloggingagent )
+
+set ( SRCS
+    loggingagent.cpp
+)
+
+include_directories (
+    ${HPCC_SOURCE_DIR}/esp/platform
+    ${HPCC_SOURCE_DIR}/system/include
+    ${HPCC_SOURCE_DIR}/system/jlib
+    ${HPCC_SOURCE_DIR}/system/xmllib
+    ${HPCC_SOURCE_DIR}/system/boost/include
+    ${HPCC_SOURCE_DIR}/system/security/shared
+    ${HPCC_SOURCE_DIR}/esp/bindings
+    ${HPCC_SOURCE_DIR}/esp/bindings/http/client
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/client
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/Platform
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/LoggingService
+    ${HPCC_SOURCE_DIR}/esp/clients
+    ${HPCC_SOURCE_DIR}/esp/clients/espcommonclient
+    ${HPCC_SOURCE_DIR}/esp/esdl
+    ${HPCC_SOURCE_DIR}/esp/esplib
+    ${HPCC_SOURCE_DIR}/esp/logging
+    ./../..
+    ./..
+)
+
+ADD_DEFINITIONS( -D_USRDLL -DESPSERVERLOGGINGAGENT_EXPORTS )
+
+if (RELEASE_BUILD EQUAL 1)
+    ADD_DEFINITIONS( -DISC_NO_MAIN)
+endif (RELEASE_BUILD EQUAL 1)
+
+HPCC_ADD_LIBRARY( espserverloggingagent SHARED ${SRCS} )
+
+add_dependencies( espserverloggingagent espscm )
+
+install ( TARGETS espserverloggingagent DESTINATION ${LIB_DIR} )
+
+target_link_libraries ( espserverloggingagent
+    ${XALAN_LIBRARIES} ${XERCES_LIBRARIES}
+    jlib
+    xmllib
+    esphttp
+)

esp/logging/loggingagent/espserverloggingagent/loggingagent.cpp

@@ -0,0 +1,435 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "httpclient.hpp"
+#include "ws_loggingservice_esp.ipp"
+#include "LoggingErrors.hpp"
+#include "loggingcommon.hpp"
+#include "loggingagentbase.hpp"
+#include "loggingagent.hpp"
+
+const int DefaultMaxTriesGTS = -1;
+const char* const PropESPServer = "ESPServer";
+const char* const PropServerUrl = "@url";
+const char* const PropServerUserID = "@user";
+const char* const PropServerPassword = "@password";
+const char* const PropServerWaitingSeconds = "MaxServerWaitingSeconds";
+const char* const MaxTriesGTS = "MaxTriesGTS";
+
+bool CESPServerLoggingAgent::init(const char * name, const char * type, IPropertyTree * cfg, const char * process)
+{
+    if (!cfg)
+        return false;
+
+    IPropertyTree* espServer = cfg->queryBranch(PropESPServer);
+    if(!espServer)
+        throw MakeStringException(-1,"Unable to find ESPServer settings for log agent %s:%s", name, type);
+
+    const char* url = espServer->queryProp(PropServerUrl);
+    if (url && *url)
+        serverUrl.set(url);
+
+    const char* userID = espServer->queryProp(PropServerUserID);
+    const char* password = espServer->queryProp(PropServerPassword);
+    if (userID && *userID && password && *password)
+    {
+        serverUserID.set(userID);
+        decrypt(serverPassword, password);
+    }
+    maxServerWaitingSeconds = cfg->getPropInt(PropServerWaitingSeconds);
+    maxGTSRetries = cfg->getPropInt(MaxTriesGTS, DefaultMaxTriesGTS);
+
+    readAllLogFilters(cfg);
+    return true;
+}
+
+void CESPServerLoggingAgent::readAllLogFilters(IPropertyTree* cfg)
+{
+    bool groupFilterRead = false;
+    VStringBuffer xpath("Filters/Filter[@type='%s']", espLogContentGroupNames[ESPLCGBackEndResp]);
+    IPropertyTree* filter = cfg->queryBranch(xpath.str());
+    if (filter && filter->hasProp("@value"))
+    {
+        logBackEndResp = filter->getPropBool("@value");
+        groupFilterRead = true;
+    }
+
+    for (unsigned i = 0; i < ESPLCGBackEndResp; i++)
+    {
+        if (readLogFilters(cfg, i))
+            groupFilterRead = true;
+    }
+
+    if (!groupFilterRead)
+    {
+        groupFilters.clear();
+        readLogFilters(cfg, ESPLCGAll);
+    }
+}
+
+bool CESPServerLoggingAgent::readLogFilters(IPropertyTree* cfg, unsigned groupID)
+{
+    Owned<CESPLogContentGroupFilters> espLogContentGroupFilters = new CESPLogContentGroupFilters((ESPLogContentGroup) groupID);
+    StringBuffer xpath;
+    if (groupID != ESPLCGAll)
+        xpath.appendf("Filters/Filter[@type='%s']", espLogContentGroupNames[groupID]);
+    else
+        xpath.append("Filters/Filter");
+    Owned<IPropertyTreeIterator> filters = cfg->getElements(xpath.str());
+    ForEach(*filters)
+    {
+        IPropertyTree &filter = filters->query();
+        StringBuffer value = filter.queryProp("@value");
+        if (!value.length())
+            continue;
+
+        //clean "//"
+        unsigned idx = value.length()-1;
+        while (idx)
+        {
+            if ((value.charAt(idx-1) == '/') && (value.charAt(idx) == '/'))
+                value.remove(idx, 1);
+            idx--;
+        }
+
+        //clean "/*" at the end
+        while ((value.length() > 1) && (value.charAt(value.length()-2) == '/') && (value.charAt(value.length()-1) == '*'))
+            value.setLength(value.length() - 2);
+
+        if (value.length() && !streq(value.str(), "*") && !streq(value.str(), "/") && !streq(value.str(), "*/"))
+        {
+            espLogContentGroupFilters->addFilter(value.str());
+        }
+        else
+        {
+            espLogContentGroupFilters->clearFilters();
+            break;
+        }
+    }
+
+    bool hasFilter = espLogContentGroupFilters->getFilterCount() > 0;
+    if (hasFilter)
+        groupFilters.append(*espLogContentGroupFilters.getClear());
+    return hasFilter;
+}
+
+bool CESPServerLoggingAgent::getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp)
+{
+    bool bRet = false;
+    StringBuffer soapreq(
+        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+        "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
+        " xmlns:SOAP-ENC=\"http://schemas.xmlsoap.org/soap/encoding/\">"
+        " <soap:Body>");
+    soapreq.append("<GetTransactionSeedRequest/>");
+    soapreq.append("</soap:Body></soap:Envelope>");
+
+    unsigned retry = 1;
+    while (1)
+    {
+        try
+        {
+            int statusCode = 0;
+            StringBuffer statusMessage, transactionSeed;
+            if (!getTransactionSeed(soapreq, statusCode, statusMessage, transactionSeed))
+                throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed,"Failed to get TransactionSeed");
+
+            resp.setSeedId(transactionSeed.str());
+            resp.setStatusCode(statusCode);
+            if (statusMessage.length())
+                resp.setStatusMessage(statusMessage.str());
+            bRet = true;
+            break;
+        }
+        catch (IException* e)
+        {
+            StringBuffer errorStr, errorMessage;
+            errorMessage.append("Failed to get TransactionSeed: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
+            ERRLOG("%s -- try %d", errorMessage.str(), retry);
+            e->Release();
+            if (retry < maxGTSRetries)
+            {
+                Sleep(retry*3000);
+                retry++;
+            }
+            else
+            {
+                resp.setStatusCode(-1);
+                resp.setStatusMessage(errorMessage.str());
+                break;
+            }
+        }
+    }
+
+    return bRet;
+}
+
+bool CESPServerLoggingAgent::getTransactionSeed(StringBuffer& soapreq, int& statusCode, StringBuffer& statusMessage, StringBuffer& seedID)
+{
+    StringBuffer status, response;
+    if (!sendHTTPRequest(soapreq, response, status) || !response.length() || !status.length())
+        throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to send Transaction Seed request to %s", serverUrl.str());
+
+    if (!strieq(status, "200 OK"))
+        throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "%s", status.str());
+
+    Owned<IPropertyTree> pTree = createPTreeFromXMLString(response.str());
+    if (!pTree)
+        throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to read response from %s", serverUrl.str());
+
+    statusCode = pTree->getPropInt("soap:Body/GetTransactionSeedResponse/StatusCode");
+    statusMessage.set(pTree->queryProp("soap:Body/GetTransactionSeedResponse/StatusMessage"));
+    seedID.set(pTree->queryProp("soap:Body/GetTransactionSeedResponse/SeedId"));
+
+    if (statusCode || !seedID.length())
+        throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to get Transaction Seed from %s", serverUrl.str());
+    return true;
+}
+
+bool CESPServerLoggingAgent::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp)
+{
+    try
+    {
+        StringBuffer soapreq(
+            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+            "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
+            " xmlns:SOAP-ENC=\"http://schemas.xmlsoap.org/soap/encoding/\">"
+            " <soap:Body>"
+            );
+        soapreq.append(req.getUpdateLogRequest());
+        soapreq.append("</soap:Body></soap:Envelope>");
+
+        StringBuffer status, respStr;
+        if (sendHTTPRequest(soapreq, respStr, status) && status.length() && strieq(status, "200 OK"))
+            resp.setStatusCode(0);
+        else if (status.length() && !strieq(status, "200 OK"))
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "%s", status.str());
+        else if (respStr.length())
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "%s", respStr.str());
+        else
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to send update log request to %s", serverUrl.str());
+    }
+    catch (IException* e)
+    {//retry will be in update log queue.
+        StringBuffer errorStr, errorMessage;
+        errorMessage.append("Failed to update log: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
+        ERRLOG("%s", errorMessage.str());
+        resp.setStatusCode(-1);
+        resp.setStatusMessage(errorMessage.str());
+        e->Release();
+    }
+    return true;
+}
+
+void CESPServerLoggingAgent::addLogContentBranch(StringArray& branchNames, IPropertyTree* contentToLogBranch, IPropertyTree* updateLogRequestTree)
+{
+    IPropertyTree* pTree = updateLogRequestTree;
+    unsigned numOfBranchNames = branchNames.length();
+    if (numOfBranchNames > 0)
+    {
+        unsigned i = 0;
+        while (i < numOfBranchNames)
+        {
+            const char* branchName = branchNames.item(i);
+            if (branchName && *branchName)
+                pTree = ensurePTree(pTree, branchName);
+            i++;
+        }
+    }
+    pTree->addPropTree(contentToLogBranch->queryName(), LINK(contentToLogBranch));
+}
+
+void CESPServerLoggingAgent::filterAndAddLogContentBranch(StringArray& branchNamesInFilter, unsigned idx,
+    StringArray& branchNamesInLogContent, IPropertyTree* originalLogContentBranch, IPropertyTree* updateLogRequestTree, bool& logContentEmpty)
+{
+    Owned<IPropertyTreeIterator> contentItr = originalLogContentBranch->getElements(branchNamesInFilter.item(idx));
+    ForEach(*contentItr)
+    {
+        IPropertyTree& contentToLogBranch = contentItr->query();
+        if (idx == branchNamesInFilter.length() - 1)
+        {
+            addLogContentBranch(branchNamesInLogContent, &contentToLogBranch, updateLogRequestTree);
+            logContentEmpty = false;
+        }
+        else
+        {
+            branchNamesInLogContent.append(contentToLogBranch.queryName());
+            filterAndAddLogContentBranch(branchNamesInFilter, idx+1, branchNamesInLogContent, &contentToLogBranch,
+                updateLogRequestTree, logContentEmpty);
+            branchNamesInLogContent.remove(branchNamesInLogContent.length() - 1);
+        }
+    }
+}
+
+void CESPServerLoggingAgent::filterLogContentTree(StringArray& filters, IPropertyTree* originalContentTree, IPropertyTree* newLogContentTree, bool& logContentEmpty)
+{
+    ForEachItemIn(i, filters)
+    {
+        const char* logContentFilter = filters.item(i);
+        if(!logContentFilter || !*logContentFilter)
+            continue;
+
+        StringArray branchNamesInFilter, branchNamesInLogContent;
+        branchNamesInFilter.appendListUniq(logContentFilter, "/");
+        filterAndAddLogContentBranch(branchNamesInFilter, 0, branchNamesInLogContent, originalContentTree, newLogContentTree, logContentEmpty);
+    }
+}
+
+void CESPServerLoggingAgent::filterLogContent(IEspUpdateLogRequestWrap* req)
+{
+    const char* logContent = req->getUpdateLogRequest();
+    Owned<IPropertyTree> updateLogRequestTree = createPTree("UpdateLogRequest");
+
+    if (groupFilters.length() < 1)
+    {//No filter
+        if (logContent && *logContent)
+        {
+            Owned<IPropertyTree> pTree = createPTreeFromXMLString(logContent);
+            updateLogRequestTree->addPropTree(pTree->queryName(), LINK(pTree));
+        }
+        else
+        {
+            Owned<IPropertyTree> espContext = req->getESPContext();
+            Owned<IPropertyTree> userContext = req->getUserContext();
+            Owned<IPropertyTree> userRequest = req->getUserRequest();
+            const char* userResp = req->getUserResponse();
+            const char* backEndResp = req->getBackEndResponse();
+            if (!espContext && !userContext && !userRequest && (!userResp || !*userResp) && (!backEndResp || !*backEndResp))
+                throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log content");
+
+            StringBuffer espContextXML, userContextXML, userRequestXML;
+            IPropertyTree* logContentTree = ensurePTree(updateLogRequestTree, "LogContent");
+            if (espContext)
+            {
+                logContentTree->addPropTree(espContext->queryName(), LINK(espContext));
+            }
+            if (userContext)
+            {
+                IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGUserContext]);
+                pTree->addPropTree(userContext->queryName(), LINK(userContext));
+            }
+            if (userRequest)
+            {
+                IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGUserReq]);
+                pTree->addPropTree(userRequest->queryName(), LINK(userRequest));
+            }
+            if (userResp && *userResp)
+            {
+                IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGUserResp]);
+                Owned<IPropertyTree> userRespTree = createPTreeFromXMLString(userResp);
+                pTree->addPropTree(userRespTree->queryName(), LINK(userRespTree));
+            }
+            if (backEndResp && *backEndResp)
+                logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndResp], backEndResp);
+        }
+    }
+    else
+    {
+        bool logContentEmpty = true;
+        IPropertyTree* logContentTree = ensurePTree(updateLogRequestTree, "LogContent");
+        if (logContent && *logContent)
+        {
+            Owned<IPropertyTree> originalContentTree = createPTreeFromXMLString(logContent);
+            filterLogContentTree(groupFilters.item(0).getFilters(), originalContentTree, logContentTree, logContentEmpty);
+        }
+        else
+        {
+            for (unsigned group = 0; group < ESPLCGBackEndResp; group++)
+            {
+                Owned<IPropertyTree> originalContentTree;
+                if (group == ESPLCGESPContext)
+                    originalContentTree.setown(req->getESPContext());
+                else if (group == ESPLCGUserContext)
+                    originalContentTree.setown(req->getUserContext());
+                else if (group == ESPLCGUserReq)
+                    originalContentTree.setown(req->getUserRequest());
+                else //group = ESPLCGUserResp
+                {
+                    const char* resp = req->getUserResponse();
+                    if (!resp || !*resp)
+                        continue;
+                    originalContentTree.setown(createPTreeFromXMLString(resp));
+                }
+                if (!originalContentTree)
+                    continue;
+
+                IPropertyTree* newContentTree = ensurePTree(logContentTree, espLogContentGroupNames[group]);
+                bool hasFilters = false;
+                ForEachItemIn(i, groupFilters)
+                {
+                    CESPLogContentGroupFilters& filtersGroup = groupFilters.item(i);
+                    if (filtersGroup.getGroup() == group)
+                    {
+                        if (group != ESPLCGESPContext)//For non ESPLCGESPContext, we want to keep the root of original tree.
+                            newContentTree = ensurePTree(newContentTree, originalContentTree->queryName());
+                        filterLogContentTree(filtersGroup.getFilters(), originalContentTree, newContentTree, logContentEmpty);
+                        hasFilters =  true;
+                        break;
+                    }
+                }
+
+                if (!hasFilters)
+                {
+                    newContentTree->addPropTree(originalContentTree->queryName(), LINK(originalContentTree));
+                    logContentEmpty = false;
+                }
+            }
+            if (logBackEndResp)
+            {
+                const char* resp = req->getBackEndResponse();
+                if (resp && *resp)
+                {
+                    logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndResp], resp);
+                    logContentEmpty = false;
+                }
+            }
+        }
+        if (logContentEmpty)
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log content");
+    }
+    const char* option = req->getOption();
+    if (option && *option)
+        updateLogRequestTree->addProp("Option", option);
+
+    StringBuffer updateLogRequestXML;
+    toXML(updateLogRequestTree, updateLogRequestXML);
+    DBGLOG("filtered content and option: <%s>", updateLogRequestXML.str());
+    req->clearOriginalContent();
+    req->setUpdateLogRequest(updateLogRequestXML.str());
+}
+
+bool CESPServerLoggingAgent::sendHTTPRequest(StringBuffer& req, StringBuffer &resp, StringBuffer &status)
+{
+    Owned<IHttpClientContext> httpctx = getHttpClientContext();
+    Owned <IHttpClient> httpclient = httpctx->createHttpClient(NULL, serverUrl.str());
+    if (serverUserID.length() && serverPassword.length())
+    {
+        httpclient->setUserID(serverUserID.str());
+        httpclient->setPassword(serverPassword.str());
+    }
+    httpclient->setTimeOut(maxServerWaitingSeconds);
+
+    return !httpclient->sendRequest("POST", "text/xml", req, resp, status);
+}
+
+extern "C"
+{
+ESPSERVERLOGGINGAGENT_API IEspLogAgent* newLoggingAgent()
+{
+    return new CESPServerLoggingAgent();
+}
+}

esp/logging/loggingagent/espserverloggingagent/loggingagent.hpp

@@ -0,0 +1,97 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef __ESPSERVERLOGGINGAGENT__HPP__
+#define __ESPSERVERLOGGINGAGENT__HPP__
+
+#include "loggingcommon.hpp"
+
+#ifdef WIN32
+    #ifdef ESPSERVERLOGGINGAGENT_EXPORTS
+        #define ESPSERVERLOGGINGAGENT_API __declspec(dllexport)
+    #else
+        #define ESPSERVERLOGGINGAGENT_API __declspec(dllimport)
+    #endif
+#else
+    #define ESPSERVERLOGGINGAGENT_API
+#endif
+
+enum ESPLogContentGroup
+{
+    ESPLCGESPContext = 0,
+    ESPLCGUserContext = 1,
+    ESPLCGUserReq = 2,
+    ESPLCGUserResp = 3,
+    ESPLCGBackEndResp = 4,
+    ESPLCGAll = 5
+};
+
+static const char * const espLogContentGroupNames[] = { "ESPContext", "UserContext", "UserRequest", "UserResponse", "BackEndResponse", "", NULL };
+
+class CESPLogContentGroupFilters : public CInterface, implements IInterface
+{
+    ESPLogContentGroup group;
+    StringArray filters;
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CESPLogContentGroupFilters(ESPLogContentGroup _group) : group(_group) {};
+    ESPLogContentGroup getGroup() { return group; };
+    StringArray& getFilters() { return filters; };
+    void clearFilters() { filters.clear(); };
+    unsigned getFilterCount() { return filters.length(); };
+    void addFilter(const char* filter)
+    {
+        if (filter && *filter)
+            filters.append(filter);
+    };
+};
+
+class CESPServerLoggingAgent : public CInterface, implements IEspLogAgent
+{
+    StringBuffer serviceName, loggingAgentName;
+    StringBuffer serverUrl, serverUserID, serverPassword;
+    unsigned maxServerWaitingSeconds; //time out value for HTTP connection to logging server
+    unsigned maxGTSRetries;
+    StringArray     logContentFilters;
+    IArrayOf<CESPLogContentGroupFilters> groupFilters;
+    bool logBackEndResp;
+
+    void readAllLogFilters(IPropertyTree* cfg);
+    bool readLogFilters(IPropertyTree* cfg, unsigned groupID);
+    void filterLogContentTree(StringArray& filters, IPropertyTree* originalContentTree, IPropertyTree* newLogContentTree, bool& logContentEmpty);
+    void filterAndAddLogContentBranch(StringArray& branchNamesInFilter, unsigned idx, StringArray& branchNamesInLogContent,
+        IPropertyTree* in, IPropertyTree* updateLogRequestTree, bool& logContentEmpty);
+    void addLogContentBranch(StringArray& branchNames, IPropertyTree* contentToLogBranch, IPropertyTree* updateLogRequestTree);
+    bool sendHTTPRequest(StringBuffer& req, StringBuffer& resp, StringBuffer& status);
+    bool getTransactionSeed(StringBuffer& soapreq, int& statusCode, StringBuffer& statusMessage, StringBuffer& seedID);
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CESPServerLoggingAgent() {};
+    virtual ~CESPServerLoggingAgent() {};
+
+    bool init(const char* name, const char* type, IPropertyTree* cfg, const char* process);
+
+    virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp);
+    virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp);
+    virtual void filterLogContent(IEspUpdateLogRequestWrap* req);
+};
+
+#endif //__ESPSERVERLOGGINGAGENT__HPP__

esp/logging/loggingcommon.hpp

@@ -0,0 +1,42 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#if !defined __LOGGINGCOMMONDEF_HPP__
+#define __LOGGINGCOMMONDEF_HPP__
+
+#pragma warning (disable : 4786)
+
+#ifdef _WIN32
+    #ifdef LOGGINGCOMMON_EXPORTS
+        #define LOGGINGCOMMON_API __declspec(dllexport)
+    #else
+        #define LOGGINGCOMMON_API __declspec(dllimport)
+    #endif
+#else
+    #define LOGGINGCOMMON_API
+#endif
+
+#define MAXLOGSERVICES 32
+
+enum LOGServiceType
+{
+    LGSTterm = 0,
+    LGSTGetTransactionSeed = 1,
+    LGSTUpdateLOG = 2
+};
+
+#endif // !defined __LOGGINGCOMMONDEF_HPP__

esp/logging/loggingmanager/CMakeLists.txt

@@ -0,0 +1,75 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: loggingmanager
+#####################################################
+# Description:
+# ------------
+#   Cmake Input File for loggingmanager
+#####################################################
+
+project( loggingmanager )
+
+# include(${HPCC_SOURCE_DIR}/cmake_modules/shares.cmake)
+include(${HPCC_SOURCE_DIR}/esp/scm/espscm.cmake)
+
+set ( SRCS
+    ${ESPSCM_GENERATED_DIR}/ws_loggingservice_esp.cpp
+    ../LogSerializer.cpp
+    ../LogFailSafe.cpp
+    ../logthread.cpp
+    loggingmanager.cpp
+)
+
+include_directories (
+    ${HPCC_SOURCE_DIR}/esp/platform
+    ${HPCC_SOURCE_DIR}/system/jlib
+    ${HPCC_SOURCE_DIR}/system/boost/include
+    ${HPCC_SOURCE_DIR}/system/security/shared
+    ${HPCC_SOURCE_DIR}/system/xmllib
+    ${HPCC_SOURCE_DIR}/system/include
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/client
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/Platform
+    ${HPCC_SOURCE_DIR}/esp/clients
+    ${HPCC_SOURCE_DIR}/esp/clients/espcommonclient
+    ${HPCC_SOURCE_DIR}/esp/clients/LoggingClient
+    ${HPCC_SOURCE_DIR}/esp/bindings
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/LoggingService
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
+#    ./../../protocols/sidex
+    ${HPCC_SOURCE_DIR}/esp/logging
+    ./..
+)
+
+ADD_DEFINITIONS( -D_USRDLL -DLOGGINGCOMMON_EXPORTS -DLOGGINGMANAGER_EXPORTS )
+
+if (RELEASE_BUILD EQUAL 1)
+    ADD_DEFINITIONS( -DISC_NO_MAIN)
+endif (RELEASE_BUILD EQUAL 1)
+
+HPCC_ADD_LIBRARY( loggingmanager SHARED ${SRCS} )
+
+add_dependencies( loggingmanager espscm )
+
+install ( TARGETS loggingmanager DESTINATION ${LIB_DIR} )
+
+target_link_libraries ( loggingmanager
+    ${XALAN_LIBRARIES} ${XERCES_LIBRARIES}
+    jlib
+    xmllib
+    esphttp
+)

esp/logging/loggingmanager/loggingmanager.cpp

@@ -0,0 +1,257 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "LoggingErrors.hpp"
+#include "loggingcommon.hpp"
+#include "loggingmanager.hpp"
+
+CLoggingManager::~CLoggingManager(void)
+{
+    for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
+    {
+        loggingAgentThreads[x]->stop();
+        loggingAgentThreads[x]->Release();
+    }
+
+    loggingAgentThreads.clear();
+}
+
+//Called when Logging manager is created. Create logging agents based on settings
+bool CLoggingManager::init(IPropertyTree* cfg, const char* service)
+{
+    if (!cfg)
+    {
+        ERRLOG(EspLoggingErrors::ConfigurationFileEntryError, "Logging Manager setting not found for %s", service);
+        return false;
+    }
+
+    Owned<IPTreeIterator> loggingAgentSettings = cfg->getElements("LogAgent");
+    ForEach(*loggingAgentSettings)
+    {
+        IPropertyTree& loggingAgentTree = loggingAgentSettings->query();
+        const char* agentName = loggingAgentTree.queryProp("@name");
+        const char* agentType = loggingAgentTree.queryProp("@type");
+        const char* agentPlugin = loggingAgentTree.queryProp("@plugin");
+        if (!agentName || !*agentName || !agentPlugin || !*agentPlugin)
+            continue;
+
+        IEspLogAgent* loggingAgent = loadLoggingAgent(agentName, agentPlugin, service, cfg);
+        if (!loggingAgent)
+        {
+            ERRLOG(-1, "Failed to create logging agent for %s", agentName);
+            continue;
+        }
+        loggingAgent->init(agentName, agentType, &loggingAgentTree, service);
+        IUpdateLogThread* logThread = createUpdateLogThread(&loggingAgentTree, service, agentName, loggingAgent);
+        if(!logThread)
+            throw MakeStringException(-1, "Failed to create update log thread for %s", agentName);
+        loggingAgentThreads.push_back(logThread);
+    }
+
+    return !loggingAgentThreads.empty();
+}
+
+typedef IEspLogAgent* (*newLoggingAgent_t_)();
+
+IEspLogAgent* CLoggingManager::loadLoggingAgent(const char* name, const char* dll, const char* service, IPropertyTree* cfg)
+{
+    StringBuffer plugin;
+    plugin.append(SharedObjectPrefix).append(dll).append(SharedObjectExtension);
+    HINSTANCE loggingAgentLib = LoadSharedObject(plugin.str(), true, false);
+    if(!loggingAgentLib)
+        throw MakeStringException(EspLoggingErrors::LoadLoggingLibraryError, "can't load library %s", plugin.str());
+
+    newLoggingAgent_t_ xproc = (newLoggingAgent_t_)GetSharedProcedure(loggingAgentLib, "newLoggingAgent");
+    if (!xproc)
+        throw MakeStringException(EspLoggingErrors::LoadLoggingLibraryError, "procedure newLoggingAgent of %s can't be loaded", plugin.str());
+
+    return (IEspLogAgent*) xproc();
+}
+
+bool CLoggingManager::updateLog(const char* option, const char* logContent, StringBuffer& status)
+{
+    bool bRet = false;
+    try
+    {
+        Owned<IEspUpdateLogRequestWrap> req =  new CUpdateLogRequestWrap(NULL, option, logContent);
+        Owned<IEspUpdateLogResponse> resp =  createUpdateLogResponse();
+        bRet = updateLog(*req, *resp, status);
+    }
+    catch (IException* e)
+    {
+        e->errorMessage(status);
+        status.insert(0, "Failed to update log: ");
+        ERRLOG("%s", status.str());
+        e->Release();
+    }
+
+    return bRet;
+}
+
+bool CLoggingManager::updateLog(const char* option, IEspContext& espContext, IPropertyTree* userContext, IPropertyTree* userRequest,
+        const char* backEndResp, const char* userResp, StringBuffer& status)
+{
+    bool bRet = false;
+    try
+    {
+        short port;
+        StringBuffer sourceIP;
+        espContext.getServAddress(sourceIP, port);
+        Owned<IPropertyTree> espContextTree = createPTree("ESPContext");
+        espContextTree->addProp("SourceIP", sourceIP.str());
+        const char* userId = espContext.queryUserId();
+        if (userId && *userId)
+            espContextTree->addProp("UserName", userId);
+
+        Owned<IEspUpdateLogRequestWrap> req =  new CUpdateLogRequestWrap(NULL, option, espContextTree.getClear(), LINK(userContext), LINK(userRequest),
+            backEndResp, userResp);
+        Owned<IEspUpdateLogResponse> resp =  createUpdateLogResponse();
+        bRet = updateLog(*req, *resp, status);
+    }
+    catch (IException* e)
+    {
+        e->errorMessage(status);
+        status.insert(0, "Failed to update log: ");
+        ERRLOG("%s", status.str());
+        e->Release();
+    }
+    return bRet;
+}
+
+bool CLoggingManager::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp, StringBuffer& status)
+{
+    bool bRet = updateLog(req, resp);
+    if (bRet)
+        status.set("Log request has been sent.");
+    else
+    {
+        const char* statusMsg = resp.getStatusMessage();
+        if (statusMsg && *statusMsg)
+            status.setf("Failed to update log: %s", statusMsg);
+        else
+            status.set("Failed to update log");
+    }
+    return bRet;
+}
+
+bool CLoggingManager::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp)
+{
+    bool bRet = false;
+    try
+    {
+        for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
+        {
+            IUpdateLogThread* loggingThread = loggingAgentThreads[x];
+            if (loggingThread->hasService(LGSTUpdateLOG))
+            {
+                loggingThread->queueLog(&req);
+                bRet = true;
+            }
+        }
+    }
+    catch (IException* e)
+    {
+        StringBuffer errorStr;
+        e->errorMessage(errorStr);
+        ERRLOG("Failed to update log: %s",errorStr.str());
+        resp.setStatusCode(-1);
+        resp.setStatusMessage(errorStr.str());
+        e->Release();
+    }
+    return bRet;
+}
+
+bool CLoggingManager::getTransactionSeed(StringBuffer& transactionSeed, StringBuffer& status)
+{
+    bool bRet = false;
+    try
+    {
+        Owned<IEspGetTransactionSeedRequest> req =  createGetTransactionSeedRequest();
+        Owned<IEspGetTransactionSeedResponse> resp =  createGetTransactionSeedResponse();
+        transactionSeed.set("Seed");
+
+        bRet = getTransactionSeed(*req, *resp);
+        if (bRet && !resp->getStatusCode())
+        {
+            const char* seed = resp->getSeedId();
+            if (!seed || !*seed)
+                status.set("Failed to get Transaction Seed");
+            else
+            {
+                transactionSeed.set(seed);
+                status.set("Transaction Seed returned.");
+                bRet = true;
+            }
+        }
+        else
+        {
+            const char* statusMsg = resp->getStatusMessage();
+            if (statusMsg && *statusMsg)
+                status.setf("Failed to get Transaction Seed: %s", statusMsg);
+            else
+                status.set("Failed to get Transaction Seed");
+        }
+    }
+    catch (IException* e)
+    {
+        e->errorMessage(status);
+        status.insert(0, "Failed to get Transaction Seed: ");
+        ERRLOG("%s",status.str());
+        e->Release();
+    }
+
+    return bRet;
+}
+
+
+bool CLoggingManager::getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp)
+{
+    bool bRet = false;
+    try
+    {
+        for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
+        {
+            IUpdateLogThread* loggingThread = loggingAgentThreads[x];
+            if (!loggingThread->hasService(LGSTGetTransactionSeed))
+                continue;
+
+            IEspLogAgent* loggingAgent = loggingThread->getLogAgent();
+            bRet = loggingAgent->getTransactionSeed(req, resp);
+            if (bRet)
+                break;
+        }
+    }
+    catch (IException* e)
+    {
+        StringBuffer errorStr;
+        e->errorMessage(errorStr);
+        ERRLOG("Failed to get Transaction Seed: %s",errorStr.str());
+        resp.setStatusCode(-1);
+        resp.setStatusMessage(errorStr.str());
+        e->Release();
+    }
+
+    return bRet;
+}
+
+extern "C"
+{
+LOGGINGMANAGER_API ILoggingManager* newLoggingManager()
+{
+    return new CLoggingManager();
+}
+}

esp/logging/loggingmanager/loggingmanager.hpp

@@ -0,0 +1,63 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#if !defined(__LOGGINGMANAGER_HPP__)
+#define __LOGGINGMANAGER_HPP__
+
+#include <vector>
+
+#include "loggingcommon.hpp"
+#include "loggingagentbase.hpp"
+#include "logthread.hpp"
+#include "loggingmanager.h"
+
+#ifdef WIN32
+    #ifdef LOGGINGMANAGER_EXPORTS
+        #define LOGGINGMANAGER_API __declspec(dllexport)
+    #else
+        #define LOGGINGMANAGER_API __declspec(dllimport)
+    #endif
+#else
+    #define LOGGINGMANAGER_API
+#endif
+
+
+class CLoggingManager : public CInterface, implements ILoggingManager
+{
+    typedef std::vector<IUpdateLogThread*> LOGGING_AGENTTHREADS;
+    LOGGING_AGENTTHREADS  loggingAgentThreads;
+
+    IEspLogAgent* loadLoggingAgent(const char* name, const char* dll, const char* type, IPropertyTree* cfg);
+    bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp, StringBuffer& status);
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CLoggingManager(void) {};
+    virtual ~CLoggingManager(void);
+
+    virtual bool init(IPropertyTree* cfg, const char* service);
+
+    virtual bool updateLog(const char* option, IEspContext& espContext, IPropertyTree* userContext, IPropertyTree* userRequest,
+        const char* backEndResp, const char* userResp, StringBuffer& status);
+    virtual bool updateLog(const char* option, const char* logContent, StringBuffer& status);
+    virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp);
+    virtual bool getTransactionSeed(StringBuffer& transactionSeed, StringBuffer& status);
+    virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp);
+};
+
+#endif // !defined(__LOGGINGMANAGER_HPP__)

esp/logging/logthread.cpp

@@ -0,0 +1,414 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jmisc.hpp"
+#include "jexcept.hpp"
+#include "jdebug.hpp"
+#include "LoggingErrors.hpp"
+#include "LogSerializer.hpp"
+#include "logthread.hpp"
+
+const char* const PropMaxLogQueueLength = "MaxLogQueueLength";
+const char* const PropQueueSizeSignal = "QueueSizeSignal";
+const char* const PropMaxTriesRS = "MaxTriesRS";
+const char* const PropFailSafe = "FailSafe";
+const char* const PropFailSafeLogsDir = "FailSafeLogsDir";
+
+#define     MaxLogQueueLength   500000 //Write a warning into log when queue length is greater than 500000
+#define     QueueSizeSignal     10000 //Write a warning into log when queue length is increased by 10000
+const int DefaultMaxTriesRS = -1;   // Max. # of attempts to send log message to WsReportService.  Default:  infinite
+
+extern LOGGINGCOMMON_API IUpdateLogThread* createUpdateLogThread(IPropertyTree* _cfg, const char* _service, const char* _agentName, IEspLogAgent* _logAgent)
+{
+    if (!_cfg)
+        return NULL;
+
+    IUpdateLogThread* loggingThread  = new CLogThread(_cfg, _service, _agentName, _logAgent);
+    loggingThread->start();
+
+    return loggingThread;
+}
+
+CLogThread::CLogThread(IPropertyTree* _cfg , const char* _service, const char* _agentName, IEspLogAgent* _logAgent)
+    : stopping(false), agentName(_agentName)
+{
+    if(!_agentName || !*_agentName)
+        throw MakeStringException(-1,"No Logging agent name defined");
+
+    if(!_cfg)
+        throw MakeStringException(-1,"No Logging agent Configuration for %s", _agentName);
+
+    if(!_service || !*_service)
+        throw MakeStringException(-1,"No service name defined for %s", _agentName);
+
+    if(!_logAgent)
+        throw MakeStringException(-1,"No Logging agent interface for %s", _agentName);
+
+    const char* servicesConfig = _cfg->queryProp("@services");
+    if (!servicesConfig || !*servicesConfig)
+        throw MakeStringException(-1,"No Logging Service defined for %s", _agentName);
+
+    StringArray serviceArray;
+    serviceArray.appendListUniq(servicesConfig, ",");
+
+    unsigned i=0;
+    ForEachItemIn(s, serviceArray)
+    {
+        const char* service = serviceArray.item(s);
+        if (service && strieq(service, "UpdateLOG"))
+            services[i++] = LGSTUpdateLOG;
+        else if (service && strieq(service, "GetTransactionSeed"))
+            services[i++] = LGSTGetTransactionSeed;
+    }
+    services[i] = LGSTterm;
+
+    logAgent.setown(_logAgent);
+
+    maxLogQueueLength = _cfg->getPropInt(PropMaxLogQueueLength, MaxLogQueueLength);
+    signalGrowingQueueAt = _cfg->getPropInt(PropQueueSizeSignal, QueueSizeSignal);
+    maxLogRetries = _cfg->getPropInt(PropMaxTriesRS, DefaultMaxTriesRS);
+    failSafeLogging = _cfg->getPropBool(PropFailSafe);
+    if(failSafeLogging)
+    {
+        const char * logsDir = _cfg->queryProp(PropFailSafeLogsDir);
+        if (!logsDir || !*logsDir)
+            logsDir = "./FailSafeLogs";
+
+        logFailSafe.setown(createFailSafeLogger(_service, _agentName, logsDir));
+    }
+}
+
+CLogThread::~CLogThread()
+{
+    ESPLOG(LogMax, "CLogThread::~CLogThread()");
+}
+
+void CLogThread::start()
+{
+    Thread::start();
+}
+
+int CLogThread::run()
+{
+    Link();
+    if(logFailSafe.get())
+        checkPendingLogs(false);
+
+    while(!stopping)
+    {
+        m_sem.wait(UPDATELOGTHREADWAITINGTIME);
+
+        sendLog();
+        if(logFailSafe.get())
+        {
+            checkPendingLogs(true);
+            checkRollOver();
+        }
+    }
+    Release();
+    return 0;
+}
+
+void CLogThread::stop()
+{
+    try
+    {
+        CriticalBlock b(logQueueCrit);
+        if (!logQueue.ordinality() && logFailSafe.get())
+            logFailSafe->RollCurrentLog();
+        //If logQueue is not empty, the log files are rolled over so that queued jobs can be read
+        //when the CLogThread is restarted.
+    }
+    catch(...)
+    {
+        DBGLOG("Exception");
+    }
+    stopping = true;
+    m_sem.signal();
+    join();
+}
+
+bool CLogThread::queueLog(IEspUpdateLogRequest* logRequest)
+{
+    if (!logRequest)
+        return false;
+
+    Owned<IEspUpdateLogRequestWrap> logRequestWrap = new CUpdateLogRequestWrap(NULL, logRequest->getOption(), logRequest->getLogContent());
+    return enqueue(logRequestWrap);
+}
+
+bool CLogThread::queueLog(IEspUpdateLogRequestWrap* logRequest)
+{
+    logAgent->filterLogContent(logRequest);
+    return enqueue(logRequest);
+}
+
+bool CLogThread::enqueue(IEspUpdateLogRequestWrap* logRequest)
+{
+    if (logFailSafe.get())
+    {
+        StringBuffer GUID, reqBuf;
+        logFailSafe->GenerateGUID(GUID, NULL);
+        logRequest->setGUID(GUID.str());
+        if (serializeLogRequestContent(logRequest, reqBuf))
+            logFailSafe->Add(GUID, reqBuf.str());
+    }
+
+    {
+        CriticalBlock b(logQueueCrit);
+        int QueueSize = logQueue.ordinality();
+        if(QueueSize > maxLogQueueLength)
+            ERRLOG("LOGGING QUEUE SIZE %d EXECEEDED MaxLogQueueLength %d, check the logging server.",QueueSize, maxLogQueueLength);
+
+        if(QueueSize!=0 && QueueSize % signalGrowingQueueAt == 0)
+            ERRLOG("Logging Queue at %d records. Check the logging server.",QueueSize);
+
+        logQueue.enqueue(LINK(logRequest));
+    }
+
+    m_sem.signal();
+
+    return true;
+}
+
+void CLogThread::sendLog()
+{
+    try
+    {
+        if(stopping)
+            return;
+
+        int recSend = 0;
+        IEspUpdateLogRequestWrap* logRequest  = 0;
+
+        CriticalBlock b(logQueueCrit);
+
+        ForEachQueueItemIn(i,logQueue)
+        {
+            logRequest  = (IEspUpdateLogRequestWrap*)logQueue.dequeue();
+            if (!logRequest)
+                continue;
+
+            const char* GUID= logRequest->getGUID();
+            if ((!GUID || !*GUID) && failSafeLogging && logFailSafe.get())
+                continue;
+
+            try
+            {
+                Owned<IEspUpdateLogResponse> logResponse = createUpdateLogResponse();
+                logAgent->updateLog(*logRequest, *logResponse);
+                if (!logResponse)
+                    throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "no response");
+                if (logResponse->getStatusCode())
+                {
+                    const char* statusMessage = logResponse->getStatusMessage();
+                    if(statusMessage && *statusMessage)
+                        throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "%s", statusMessage);
+                    else
+                        throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Unknown error");
+                }
+
+                if(failSafeLogging && logFailSafe.get())
+                    logFailSafe->AddACK(GUID);
+                logRequest->Release();//Make sure that no data (such as GUID) is needed before releasing the logRequest.
+            }
+            catch(IException* e)
+            {
+                StringBuffer errorStr, errorMessage;
+                errorMessage.appendf("Failed to update log for %s: error code %d, error message %s", GUID, e->errorCode(), e->errorMessage(errorStr).str());
+                e->Release();
+
+                bool willRetry = false;
+                if (maxLogRetries != 0)
+                {
+                    unsigned retry = logRequest->incrementRetryCount();
+                    if (retry > maxLogRetries)
+                        errorMessage.append(" Max logging retries exceeded.");
+                    else
+                    {
+                        willRetry = true;
+                        logQueue.enqueue(logRequest);
+                        errorMessage.appendf(" Adding back to logging queue for retrying %d.", retry);
+                    }
+                }
+                if (!willRetry)
+                {
+                    if(failSafeLogging && logFailSafe.get())
+                        logFailSafe->AddACK(GUID);
+                    logRequest->Release();
+                }
+                ERRLOG("%s", errorMessage.str());
+            }
+        }
+    }
+    catch(IException* e)
+    {
+        StringBuffer errorStr, errorMessage;
+        errorMessage.append("Exception thrown within update log thread: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
+        ERRLOG("%s", errorMessage.str());
+        e->Release();
+    }
+    catch(...)
+    {
+        ERRLOG("Unknown exception thrown within update log thread");
+    }
+
+    return;
+}
+
+//////////////////////////FailSafe////////////////////////////
+void CLogThread::checkRollOver()
+{
+    try
+    {
+        bool bRollover = false;
+
+        time_t tNow;
+        time(&tNow);
+        struct tm ltNow;
+        localtime_r(&tNow, &ltNow);
+        if ((ltNow.tm_year != m_startTime.tm_year || ltNow.tm_yday != m_startTime.tm_yday))
+        {
+            bRollover = true;
+            localtime_r(&tNow, &m_startTime);  // reset the start time for next rollover check
+        }
+        if (!bRollover)
+            return;
+
+        //Rename .log files to .old files
+        logFailSafe->SafeRollover();
+
+        CriticalBlock b(logQueueCrit);
+
+        //Check and add queued requests to tank(.log) files
+        unsigned numNewArrivals = logQueue.ordinality();
+        if(numNewArrivals <= 0)
+            return;
+
+        ESPLOG(LogMax, "writing %d requests in the queue to the rolled over tank file.", numNewArrivals);
+        for(unsigned i = 0; i < numNewArrivals; i++)
+        {
+            IInterface* pRequest = logQueue.item(i);
+            if (!pRequest)
+                continue;
+
+            IEspUpdateLogRequestWrap* pEspRequest = dynamic_cast<IEspUpdateLogRequestWrap*>(pRequest);
+            if(!pEspRequest)
+                continue;
+
+            StringBuffer reqBuf;
+            const char* GUID = pEspRequest->getGUID();
+            if(GUID && *GUID && serializeLogRequestContent(pEspRequest, reqBuf))
+                logFailSafe->Add(GUID, reqBuf.str());
+        }
+    }
+    catch(IException* Ex)
+    {
+        StringBuffer str;
+        Ex->errorMessage(str);
+        ERRLOG("Exception thrown during tank file rollover: %s",str.str());
+        Ex->Release();
+    }
+    catch(...)
+    {
+        ERRLOG("Unknown exception thrown during tank file rollover.");
+    }
+}
+
+unsigned CLogThread::serializeLogRequestContent(IEspUpdateLogRequestWrap* pRequest, StringBuffer& logData)
+{
+    const char* GUID = pRequest->getGUID();
+    const char* option = pRequest->getOption();
+    const char* logRequest = pRequest->getUpdateLogRequest();
+    if (GUID && *GUID)
+        logData.append("<GUID>").append(GUID).append("</GUID>");
+    if (option && *option)
+        logData.append("<Option>").append(option).append("</Option>");
+    if (logRequest && *logRequest)
+    {
+        StringBuffer buffer;
+        JBASE64_Encode(logRequest, strlen(logRequest), buffer);
+        logData.append("<LogRequest>").append(buffer.str()).append("</LogRequest>");
+    }
+    return logData.length();
+}
+
+void CLogThread::checkPendingLogs(bool bOneRecOnly)
+{
+    try
+    {
+        bool queueLogError = false;
+        bool bFirst = true;
+
+        StringBuffer GUID, logData;
+        while (logFailSafe->PopPendingLogRecord(GUID, logData))
+        {
+            if (bFirst && !bOneRecOnly)
+            {
+                DBGLOG("We have old logs!. Will now try and recover the lost log messages");
+                bFirst = false;
+            }
+
+            Owned<IEspUpdateLogRequestWrap> logRequest = unserializeLogRequestContent(logData.str());
+            if (!logRequest)
+                ERRLOG("checkPendingLogs: failed to unserialize: %s", logData.str());
+            else if (!enqueue(logRequest))
+            {
+                ERRLOG("checkPendingLogs: failed to add a log request to queue");
+                queueLogError=true;
+            }
+
+            if (bOneRecOnly)
+                break;
+        }
+        //if everything went ok then we should be able to rollover the old logs.
+        if (!queueLogError && !bOneRecOnly)
+            logFailSafe->RolloverAllLogs();
+    }
+    catch(IException* ex)
+    {
+        StringBuffer errorStr;
+        ex->errorMessage(errorStr);
+        ERRLOG("CheckPendingLogs: %s:" ,errorStr.str());
+        ex->Release();
+    }
+    catch(...)
+    {
+        ERRLOG("Unknown exception thrown in CheckPendingLogs");
+    }
+}
+
+IEspUpdateLogRequestWrap* CLogThread::unserializeLogRequestContent(const char* logData)
+{
+    if (!logData && *logData)
+        return NULL;
+
+    Owned<IPropertyTree> pLogTree = createPTreeFromXMLString(logData);
+    if (!pLogTree)
+        return NULL;
+
+    const char* guid = pLogTree->queryProp("GUID");
+    const char* opt = pLogTree->queryProp("Option");
+    const char* logRequest = pLogTree->queryProp("LogRequest");
+    if (!logRequest || !*logRequest)
+        return NULL;
+
+    StringBuffer buffer;
+    JBASE64_Decode(logRequest, buffer);
+
+    return new CUpdateLogRequestWrap(guid, opt, buffer.str());
+};

esp/logging/logthread.hpp

@@ -0,0 +1,103 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _LOGTHREAD_HPP__
+#define _LOGTHREAD_HPP__
+
+#include "jthread.hpp"
+#include "jqueue.tpp"
+#include "loggingagentbase.hpp"
+#include "LogFailSafe.hpp"
+
+interface IUpdateLogThread : extends IInterface
+{
+    virtual int run() = 0;
+    virtual void start() = 0;
+    virtual void stop() = 0;
+
+    virtual IEspLogAgent* getLogAgent() = 0;
+
+    virtual bool hasService(LOGServiceType service) = 0;
+    virtual bool queueLog(IEspUpdateLogRequest* logRequest) = 0;
+    virtual bool queueLog(IEspUpdateLogRequestWrap* logRequest) = 0;
+    virtual void sendLog() = 0;
+};
+
+class CLogThread : public Thread , implements IUpdateLogThread
+{
+    bool stopping;
+    StringAttr agentName;
+    int maxLogQueueLength;
+    int signalGrowingQueueAt;
+    unsigned maxLogRetries;   // Max. # of attempts to send log message
+
+    Owned<IEspLogAgent> logAgent;
+    LOGServiceType services[MAXLOGSERVICES];
+    SafeQueueOf<IInterface, false> logQueue;
+    CriticalSection logQueueCrit;
+    Semaphore       m_sem;
+
+    bool failSafeLogging;
+    Owned<ILogFailSafe> logFailSafe;
+    struct tm         m_startTime;
+
+    unsigned serializeLogRequestContent(IEspUpdateLogRequestWrap* request, StringBuffer& logData);
+    IEspUpdateLogRequestWrap* unserializeLogRequestContent(const char* logData);
+    bool enqueue(IEspUpdateLogRequestWrap* logRequest);
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CLogThread();
+    CLogThread(IPropertyTree* _agentConfig, const char* _service, const char* _agentName, IEspLogAgent* _logAgent = NULL);
+    virtual ~CLogThread();
+
+    IEspLogAgent* getLogAgent() {return logAgent;};
+
+    bool hasService(LOGServiceType service)
+    {
+        unsigned int i = 0;
+        while (services[i] != LGSTterm)
+        {
+            if (services[i] == service)
+                return true;
+            i++;
+        }
+        return false;
+    }
+    void addService(LOGServiceType service)
+    {
+        unsigned i=0;
+        while (services[i] != LGSTterm) i++;
+        services[i] = service;
+    };
+
+    int run();
+    void start();
+    void stop();
+
+    bool queueLog(IEspUpdateLogRequest* logRequest);
+    bool queueLog(IEspUpdateLogRequestWrap* logRequest);
+    void sendLog();
+
+    void checkPendingLogs(bool oneRecordOnly=false);
+    void checkRollOver();
+};
+
+extern LOGGINGCOMMON_API IUpdateLogThread* createUpdateLogThread(IPropertyTree* _cfg, const char* _service, const char* _agentName, IEspLogAgent* _logAgent);
+
+#endif // _LOGTHREAD_HPP__

esp/logging/test/CMakeLists.txt

@@ -0,0 +1,62 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: logging_test
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for logging_test
+#####################################################
+
+
+project( logging_test )
+
+set (    SRCS
+         logging_test.cpp
+         main.cpp
+    )
+
+include_directories (
+         ${HPCC_SOURCE_DIR}/system/include
+         ${HPCC_SOURCE_DIR}/system/jlib
+         ${HPCC_SOURCE_DIR}/system/xmllib
+         ${HPCC_SOURCE_DIR}/system/security/shared
+         ${HPCC_SOURCE_DIR}/system/security/securesocket
+         ${HPCC_SOURCE_DIR}/esp/platform
+         ${HPCC_SOURCE_DIR}/esp/bindings
+         ${HPCC_SOURCE_DIR}/esp/bindings/http/platform
+         ${HPCC_SOURCE_DIR}/esp/bindings/http/client
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/client
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/Platform
+         ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/LoggingService
+         ${HPCC_SOURCE_DIR}/esp/clients
+         ${HPCC_SOURCE_DIR}/esp/clients/espcommonclient
+         ${HPCC_SOURCE_DIR}/esp/clients/LoggingClient
+         ${HPCC_SOURCE_DIR}/esp/logging
+    )
+
+ADD_DEFINITIONS( -D_CONSOLE )
+
+HPCC_ADD_EXECUTABLE ( logging_test ${SRCS} )
+
+target_link_libraries ( logging_test
+         jlib
+         securesocket
+         loggingmanager
+    )

esp/logging/test/logging_test.cpp

@@ -0,0 +1,166 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jlib.hpp"
+#include "jfile.hpp"
+#include "jprop.hpp"
+#include "loggingmanager.h"
+
+Owned<IPropertyTree> loggingInput;
+IPropertyTree* loggingConfig = NULL;
+IPropertyTree* testData = NULL;
+Owned<ILoggingManager> loggingManager;
+
+ILoggingManager* loadLogggingManager()
+{
+    StringBuffer realName;
+    realName.append(SharedObjectPrefix).append(LOGGINGMANAGERLIB).append(SharedObjectExtension);
+    HINSTANCE loggingManagerLib = LoadSharedObject(realName.str(), true, false);
+    if(loggingManagerLib == NULL)
+    {
+        printf("can't load library %s\n", realName.str());
+        return NULL;
+    }
+
+    newLoggingManager_t_ xproc = NULL;
+    xproc = (newLoggingManager_t_)GetSharedProcedure(loggingManagerLib, "newLoggingManager");
+
+    if (!xproc)
+    {
+        printf("procedure newLogggingManager of %s can't be loaded\n", realName.str());
+        return NULL;
+    }
+
+    return (ILoggingManager*) xproc();
+}
+
+ILoggingManager* init(const char* inputFileName)
+{
+    if(!inputFileName || !*inputFileName)
+    {
+        printf("Input file not defined.\n");
+        return NULL;
+    }
+
+    printf("Input file: %s\n", inputFileName);
+    loggingInput.setown(createPTreeFromXMLFile(inputFileName, ipt_caseInsensitive));
+    if (!loggingInput)
+    {
+        printf("can't read input file.\n");
+        return NULL;
+    }
+
+    loggingConfig = loggingInput->queryPropTree("config/LoggingManager");
+    testData = loggingInput->queryPropTree("test");
+    if (!loggingConfig || !testData)
+    {
+        printf("can't read input file.\n");
+        return NULL;
+    }
+
+    loggingManager.setown(loadLogggingManager());
+    if (!loggingManager)
+    {
+        printf("No logging manager\n");
+        return NULL;
+    }
+    loggingManager->init(loggingConfig, "test_service");
+
+    return loggingManager;
+}
+
+void sendRequest()
+{
+    if (!loggingManager)
+    {
+        printf("No logging manager.\n");
+        return;
+    }
+
+    StringBuffer action, option, status;
+    testData->getProp("action", action);
+    testData->getProp("option", option);
+    if (action.length() && strieq(action.str(), "getTransactionSeed"))
+    {
+        StringBuffer transactionSeed;
+        loggingManager->getTransactionSeed(transactionSeed, status);
+        if (transactionSeed.length())
+            printf("Got transactionSeed: <%s>\n", transactionSeed.str());
+    }
+    else if (action.length() && strieq(action.str(), "UpdateLog"))
+    {
+        IPropertyTree* logContentTree = testData->queryPropTree("LogContent");
+        if (!logContentTree)
+        {
+            printf("can't read log content.\n");
+            return;
+        }
+        StringBuffer logContentXML;
+        toXML(logContentTree, logContentXML);
+        printf("log content: <%s>.\n", logContentXML.str());
+
+        Owned<IEspContext> espContext =  createEspContext();
+        const char* userName = logContentTree->queryProp("ESPContext/UserName");
+        const char* sourceIP = logContentTree->queryProp("ESPContext/SourceIP");
+        short servPort = logContentTree->getPropInt("ESPContext/Port");
+        espContext->setUserID(userName);
+        espContext->setServAddress(sourceIP, servPort);
+
+        const char* backEndResp = logContentTree->queryProp("BackEndResponse");
+        IPropertyTree* userContextTree = logContentTree->queryPropTree("MyUserContext");
+        IPropertyTree* userRequestTree = logContentTree->queryPropTree("MyUserRequest");
+        IPropertyTree* userRespTree = logContentTree->queryPropTree("MyUserResponseEx");
+        StringBuffer userContextXML, userRequestXML, userRespXML;
+        toXML(userRespTree, userRespXML);
+
+        toXML(userContextTree, userContextXML);
+        toXML(userRequestTree, userRequestXML);
+        printf("userContextXML: <%s>.\n", userContextXML.str());
+        printf("userRequestXML: <%s>.\n", userRequestXML.str());
+        printf("userRespXML: <%s>.\n", userRespXML.str());
+        printf("backEndResp: <%s>.\n", backEndResp);
+
+        //Sleep(5000); //Waiting for loggingManager to start
+        loggingManager->updateLog(option.str(), *espContext, userContextTree, userRequestTree, backEndResp, userRespXML.str(), status);
+    }
+    else if (action.length() && strieq(action.str(), "UpdateLog1"))
+    {
+        IPropertyTree* logContentTree = testData->queryPropTree("LogContent");
+        if (!logContentTree)
+        {
+            printf("can't read log content.\n");
+            return;
+        }
+        StringBuffer logContentXML;
+        toXML(logContentTree, logContentXML);
+        printf("log content: <%s>.\n", logContentXML.str());
+        //Sleep(5000); //Waiting for loggingManager to start
+        loggingManager->updateLog(option.str(), logContentXML.str(), status);
+    }
+    else
+        printf("Invalid action.\n");
+}
+
+void doWork(const char* inputFileName)
+{
+    printf("Enter dowork()\n");
+    init(inputFileName);
+    sendRequest();
+    printf("Waiting for log queue to be read.\n");
+    Sleep(10000); //Waiting for loggingManager to process queue
+    printf("Finish dowork()\n");
+}

esp/logging/test/main.cpp

@@ -0,0 +1,142 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jlib.hpp"
+#include "jfile.hpp"
+#include "jprop.hpp"
+
+void usage()
+{
+    puts("Usage:");
+    puts("   logging_test [optins]");
+    puts("Options:");
+    puts("   -i <filename> : input file name using for both config and request");
+    puts("   -o <filename> : out file name");
+    puts("   -action <soap-action> ");
+    puts("");
+    puts("sample config-file: ");
+    puts("<logging>");
+    puts("  <config>");
+    puts("    <LoggingManager>");
+    puts("      <LogAgent name=\"ESPLoggingAgent\" type=\"LogAgent\" services=\"GetTransactionSeed,UpdateLog\" plugin=\"espserverloggingagent\">");
+    puts("        <url>http://10.176.152.168:7501</url>");
+    puts("        ... ...");
+    puts("        <Filters>");
+    puts("          <Filter value=\"Context\"/>");
+    puts("          <Filter value=\"Request\"/>");
+    puts("        </Filters>");
+    puts("      </LogAgent>");
+    puts("    </LoggingManager>");
+    puts("  </config>");
+    puts("  <test>");
+    puts("    <action>UpdateLog</action>");
+    puts("    <option>SingleInsert</option>");
+    puts("    <LogContent>");
+    puts("      <Context>");
+    puts("        <UserID>myUserID1</UserID>");
+    puts("        <IPAddress>10.10.10.10</IPAddress>");
+    puts("        <FunctionName>Search Movies</FunctionName>");
+    puts("        <NotForLog>abc</NotForLog>");
+    puts("      </Context>");
+    puts("      <Request>");
+    puts("        <Company>ALL Movies</Company>");
+    puts("        <Address>101 1st Ave., LA, CA 12345</Address>");
+    puts("      </Request>");
+    puts("      <Response>");
+    puts("        <RecordCount>2</RecordCount>");
+    puts("        <Dataset name=\"GENERIC_LOG_DATA\">");
+    puts("          <Rec>");
+    puts("            <Data1>data 1</Data1>");
+    puts("            <Data2>2</Data2>");
+    puts("          </Rec>");
+    puts("          <Test2>2</Test2>");
+    puts("        </Dataset>");
+    puts("      </Response>");
+    puts("    </LogContent>");
+    puts("  </test>");
+    puts("</logging>");
+    puts("");
+
+    exit(-1);
+}
+
+void doWork(const char* inputFileName);
+
+int main(int argc, char* argv[])
+{
+    InitModuleObjects();
+
+    StringBuffer in_fname;
+    StringBuffer out_fname;
+
+    int i = 1;
+    while(i<argc)
+    {
+        if (stricmp(argv[i], "-i") == 0)
+        {
+            i++;
+            in_fname.clear().append(argv[i++]);
+        }
+        else if (stricmp(argv[i], "-o") == 0)
+        {
+            i++;
+            out_fname.clear().append(argv[i++]);
+        }
+        else
+        {
+            printf("Error: command format error\n");
+            usage();
+            exit(-1);
+        }
+    }
+
+    try
+    {
+        FILE* ofile = NULL;
+        if(out_fname.length() != 0)
+        {
+            ofile = fopen(out_fname.str(), "a+");
+            if(ofile == NULL)
+            {
+                printf("can't open file %s\n", out_fname.str());
+                exit(-1);
+            }
+        }
+        else
+        {
+            ofile = stdout;
+        }
+
+        doWork(in_fname.str());
+        printf("Finished\n");
+        fclose(ofile);
+    }
+    catch(IException *excpt)
+    {
+        StringBuffer errMsg;
+        printf("Error - %d:%s\n", excpt->errorCode(), excpt->errorMessage(errMsg).str());
+        return -1;
+    }
+    catch(...)
+    {
+        printf("Unknown exception\n");
+        return -1;
+    }
+
+    releaseAtoms();
+    return 0;
+}

esp/services/CMakeLists.txt

@@ -32,4 +32,7 @@ HPCC_ADD_SUBDIRECTORY (WsDeploy "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_packageprocess "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_esdlconfig "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (esdl_svc_engine "PLATFORM")
+if (LOGGING_SERVICE)
+HPCC_ADD_SUBDIRECTORY (ws_loggingservice "PLATFORM")
+endif()
 HPCC_ADD_SUBDIRECTORY (espcontrol "PLATFORM")

esp/services/ws_loggingservice/CMakeLists.txt

@@ -0,0 +1,64 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# Component: ws_loggingservice
+
+project( ws_loggingservice )
+
+# include (${HPCC_SOURCE_DIR}/cmake_modules/shares.cmake)
+include(${HPCC_SOURCE_DIR}/esp/scm/espscm.cmake)
+
+include_directories (
+     ${HPCC_SOURCE_DIR}/system/include
+     ${HPCC_SOURCE_DIR}/system/jlib
+     ${HPCC_SOURCE_DIR}/system/xmllib
+     ${HPCC_SOURCE_DIR}/system/mp
+     ${HPCC_SOURCE_DIR}/system/security/shared
+     ${HPCC_SOURCE_DIR}/system/security/securesocket
+     ${HPCC_SOURCE_DIR}/common/environment
+     ${MYSQL_INCLUDE_DIR}
+     ${HPCC_SOURCE_DIR}/dali/base
+     ${HPCC_SOURCE_DIR}/esp/platform
+     ${HPCC_SOURCE_DIR}/esp/bindings
+     ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
+     ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/Platform
+     ${HPCC_SOURCE_DIR}/esp/esplib
+     ${HPCC_SOURCE_DIR}/esp/clients
+     ${HPCC_SOURCE_DIR}/esp/clients/LoggingClient
+     ${HPCC_SOURCE_DIR}/esp/clients/loggingmanager
+     ${HPCC_SOURCE_DIR}/esp/logging
+     ./..
+)
+
+ADD_DEFINITIONS( -D_USRDLL -DLOGGINGCOMMON_EXPORTS )
+
+set ( SRCS
+    ${ESPSCM_GENERATED_DIR}/ws_loggingservice_esp.cpp
+    ../../logging/LogSerializer.cpp
+    ../../logging/LogFailSafe.cpp
+    ../../logging/logthread.cpp
+    loggingservice.cpp
+    loggingserviceplugin.cpp
+)
+
+HPCC_ADD_LIBRARY( ws_loggingservice SHARED ${SRCS} )
+
+install ( TARGETS ws_loggingservice RUNTIME DESTINATION bin LIBRARY DESTINATION lib )
+add_dependencies (ws_loggingservice espscm)
+target_link_libraries ( ws_loggingservice
+    esphttp
+    jlib
+)

esp/services/ws_loggingservice/loggingservice.cpp

@@ -0,0 +1,141 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#pragma warning (disable : 4786)
+
+#include "LoggingErrors.hpp"
+#include "logthread.hpp"
+#include "loggingservice.hpp"
+
+#define WSLOGGING_ACCESS    "WsLoggingAccess"
+
+typedef IEspLogAgent* (*newLogAgent_t_)();
+
+bool CWsLoggingServiceEx::init(const char* service, const char* type, IPropertyTree* cfg, const char* process)
+{
+    VStringBuffer xpath("Software/EspProcess[@name=\"%s\"]/EspService[@name=\"%s\"]", process, service);
+    Owned<IPropertyTree> pServiceNode = cfg->getPropTree(xpath.str());
+    if (!pServiceNode)
+        throw MakeStringException(-1, "No settings found for service %s", service);
+
+    Owned<IPropertyTreeIterator> logAgents = pServiceNode->getElements("LogAgent");
+    if (!logAgents)
+        throw MakeStringException(-1, "No logAgent is defined for service %s", service);
+
+    ForEach(*logAgents)
+    {
+        IPropertyTree& ptree = logAgents->query();
+        const char* agentName = ptree.queryProp("@name");
+        const char* agentType = ptree.queryProp("@type");
+        const char* agentPlugin = ptree.queryProp("@plugin");
+        if (!agentName || !*agentName || !agentPlugin || !*agentPlugin)
+            continue;
+
+        IEspLogAgent* logAgent = loadLoggingAgent(agentName, agentPlugin);
+        if (!logAgent)
+        {
+            ERRLOG(-1, "Failed to create logging agent for %s", agentName);
+            continue;
+        }
+        logAgent->init(agentName, agentType, &ptree, process);
+        IUpdateLogThread* logThread = createUpdateLogThread(&ptree, service, agentName, logAgent);
+        if(!logThread)
+            throw MakeStringException(-1, "Failed to create update log thread for %s", agentName);
+
+        loggingAgentThreads.push_back(logThread);
+    }
+
+    return true;
+}
+
+IEspLogAgent* CWsLoggingServiceEx::loadLoggingAgent(const char* name, const char* dll)
+{
+    StringBuffer realName;
+    // add suffix and prefix if needed
+    realName.append(SharedObjectPrefix).append(dll).append(SharedObjectExtension);
+    HINSTANCE loggingAgentLib = LoadSharedObject(realName.str(), true, false);
+    if(!loggingAgentLib)
+        throw MakeStringException(EspLoggingErrors::LoadLoggingLibraryError, "can't load library %s", realName.str());
+
+    newLogAgent_t_ xproc = (newLogAgent_t_)GetSharedProcedure(loggingAgentLib, "newLoggingAgent");
+    if (!xproc)
+        throw MakeStringException(EspLoggingErrors::LoadLoggingLibraryError, "procedure newLoggingAgent of %s can't be loaded", realName.str());
+
+    return (IEspLogAgent*) xproc();
+}
+
+bool CWsLoggingServiceEx::onUpdateLog(IEspContext& context, IEspUpdateLogRequest& req, IEspUpdateLogResponse& resp)
+{
+    try
+    {
+        if (!context.validateFeatureAccess(WSLOGGING_ACCESS, SecAccess_Write, false))
+            throw MakeStringException(EspLoggingErrors::WSLoggingAccessDenied, "Failed to update log. Permission denied.");
+
+        for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
+        {
+            IUpdateLogThread* loggingThread = loggingAgentThreads[x];
+            if (!loggingThread->hasService(LGSTUpdateLOG))
+                continue;
+            loggingThread->queueLog(&req);
+        }
+        resp.setStatusCode(0);
+        resp.setStatusMessage("Log will be updated.");
+    }
+    catch (IException* e)
+    {
+        StringBuffer errorStr;
+        e->errorMessage(errorStr);
+        ERRLOG("Failed to update log: cannot add to log queue: %s",errorStr.str());
+        resp.setStatusCode(-1);
+        resp.setStatusMessage(errorStr.str());
+        e->Release();
+    }
+    return true;
+}
+
+bool CWsLoggingServiceEx::onGetTransactionSeed(IEspContext& context, IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp)
+{
+    bool bRet = false;
+    try
+    {
+        if (!context.validateFeatureAccess(WSLOGGING_ACCESS, SecAccess_Write, false))
+            throw MakeStringException(EspLoggingErrors::WSLoggingAccessDenied, "Failed to get transaction  seed. Permission denied.");
+
+        LOGServiceType serviceType = LGSTGetTransactionSeed;
+        for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
+        {
+            IUpdateLogThread* loggingThread = loggingAgentThreads[x];
+            if (!loggingThread->hasService(serviceType))
+                continue;
+
+            IEspLogAgent* loggingAgent = loggingThread->getLogAgent();
+            bRet = loggingAgent->getTransactionSeed(req, resp);
+            break;
+        }
+    }
+    catch (IException* e)
+    {
+        StringBuffer errorStr;
+        e->errorMessage(errorStr);
+        errorStr.insert(0, "Failed to get Transaction Seed: ");
+        ERRLOG("%s", errorStr.str());
+        resp.setStatusCode(-1);
+        resp.setStatusMessage(errorStr.str());
+        e->Release();
+    }
+    return bRet;
+}

esp/services/ws_loggingservice/loggingservice.hpp

@@ -0,0 +1,60 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _WSLOGGINGSERVICE_HPP__
+#define _WSLOGGINGSERVICE_HPP__
+
+#include <vector>
+#include "loggingcommon.hpp"
+#include "logthread.hpp"
+#include "loggingagentbase.hpp"
+
+class CWsLoggingServiceEx : public CWsLoggingService
+{
+    typedef std::vector<IUpdateLogThread*> LOGGING_AGENTTHREADS;
+    LOGGING_AGENTTHREADS  loggingAgentThreads;
+
+    unsigned loggingServiceThreadPoolSize;
+    unsigned loggingServiceThreadWaitingTimeOut;
+    unsigned loggingServiceThreadPoolStackSize;
+    Owned<IThreadPool> loggingServiceThreadPool;
+
+    IEspLogAgent* loadLoggingAgent(const char* name, const char* dll);
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CWsLoggingServiceEx() {};
+    virtual ~CWsLoggingServiceEx()
+    {
+        for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
+        {
+            loggingAgentThreads[x]->stop();
+            loggingAgentThreads[x]->Release();
+        }
+
+        loggingAgentThreads.clear();
+    };
+
+    virtual bool init(const char* name, const char* type, IPropertyTree* cfg, const char* process);
+
+    //interface IEspLoggingService
+    bool onGetTransactionSeed(IEspContext& context, IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp);
+    bool onUpdateLog(IEspContext& context, IEspUpdateLogRequest& req, IEspUpdateLogResponse& resp);
+};
+
+#endif //_WSLOGGINGSERVICE_HPP__

esp/services/ws_loggingservice/loggingserviceplugin.cpp

@@ -0,0 +1,65 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#pragma warning (disable : 4786)
+
+#include "espplugin.hpp"
+#include "http/platform/httpprot.hpp"
+
+//ESP Service
+#include "loggingservice.hpp"
+#include "ws_loggingservice_esp.ipp"
+
+extern "C"
+{
+    //when we aren't loading dynamically
+    // Change the function names when we stick with dynamic loading.
+    ESP_FACTORY IEspService * esp_service_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
+    {
+        if (streq(type, "WsLoggingService"))
+        {
+            CWsLoggingServiceEx* loggingservice = new CWsLoggingServiceEx;
+            loggingservice->init(name, type, cfg, process);
+            return loggingservice;
+        }
+        return NULL;
+    }
+
+    ESP_FACTORY IEspRpcBinding * esp_binding_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
+    {
+        if (streq(type, "loggingservice_binding"))
+            return new CWsLoggingServiceSoapBinding(cfg, name, process, hsl_none);
+
+        return NULL;
+    }
+
+    ESP_FACTORY IEspProtocol * esp_protocol_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
+    {
+        if (streq(type, "http_protocol"))
+            return new CHttpProtocol;
+        else if(streq(type, "secure_http_protocol"))
+        {
+            IPropertyTree *sslSettings;
+            sslSettings = cfg->getPropTree(StringBuffer("Software/EspProcess[@name=\"").append(process).append("\"]").append("/EspProtocol[@name=\"").append(name).append("\"]").str());
+            if(sslSettings != NULL)
+                return new CSecureHttpProtocol(sslSettings);
+        }
+
+        return NULL;
+    }
+
+};

initfiles/componentfiles/configxml/@temp/CMakeLists.txt

@@ -23,6 +23,7 @@ FOREACH ( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_WsSMC.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/roxiePlugins.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_DynamicESDL.xsl
+    ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_wslogging.xsl
 )
     Install ( FILES ${iFILES} DESTINATION componentfiles/configxml/@temp COMPONENT Runtime )
 ENDFOREACH ( iFILES )

initfiles/componentfiles/configxml/@temp/esp_service_wslogging.xsl

@@ -0,0 +1,151 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xml:space="default"
+xmlns:seisint="http://seisint.com"  xmlns:set="http://exslt.org/sets" exclude-result-prefixes="seisint set">
+    <xsl:import href="esp_service.xsl"/>
+
+    <xsl:template match="EspService">
+        <xsl:param name="authNode"/>
+        <xsl:param name="bindingNode"/>
+
+        <xsl:variable name="serviceType" select="'WsLoggingService'"/>
+        <xsl:variable name="bindType" select="'loggingservice_binding'"/>
+        <xsl:variable name="servicePlugin">
+            <xsl:choose>
+                <xsl:when test="$isLinuxInstance">libws_loggingservice.so</xsl:when>
+                <xsl:otherwise>ws_loggingservice.dll</xsl:otherwise>
+            </xsl:choose>
+        </xsl:variable>
+        <xsl:variable name="serviceName" select="concat('WsLoggingService', '_', @name, '_', $process)"/>
+        <xsl:variable name="bindName" select="concat('WsLoggingService', '_', $bindingNode/@name, '_', $process)"/>
+
+        <EspService name="{$serviceName}" type="{$serviceType}" plugin="{$servicePlugin}">
+            <xsl:if test="string(@LoggingManager) != ''">
+                <xsl:variable name="managerName" select="@LoggingManager"/>
+                <xsl:variable name="managerNode" select="/Environment/Software/LoggingManager[@name=$managerName]"/>
+
+                <xsl:if test="not($managerNode)">
+                    <xsl:message terminate="yes">Logging Manager is undefined!</xsl:message>
+                </xsl:if>
+
+                <xsl:if test="not($managerNode/ESPLoggingAgent[1]/@ESPLoggingAgent)">
+                     <xsl:message terminate="yes">ESP Logging Agent <xsl:value-of select="$managerNode/@ESPLoggingAgent[1]"/>is undefined!</xsl:message>
+                </xsl:if>
+
+                <LoggingManager name="{$managerNode/@name}">
+
+                <xsl:for-each select="$managerNode/ESPLoggingAgent">
+                    <xsl:variable name="agentName" select="@ESPLoggingAgent"/>
+                    <xsl:variable name="agentNode" select="/Environment/Software/ESPLoggingAgent[@name=$agentName]"/>
+
+                    <xsl:if test="not($agentNode)">
+                        <xsl:message terminate="yes">An ESP Logging Agent <xsl:value-of select="$agentName"/>  for <xsl:value-of select="$managerNode/@name"/> is undefined!</xsl:message>
+                    </xsl:if>
+
+                    <xsl:if test="string($agentNode/@ESPServer) = ''">
+                        <xsl:message terminate="yes">ESP server is undefined for <xsl:value-of select="$agentName"/> </xsl:message>
+                    </xsl:if>
+
+                    <xsl:variable name="espServer" select="$agentNode/@ESPServer"/>
+                    <xsl:variable name="espNode" select="/Environment/Software/EspProcess[@name=$espServer]"/>
+
+                    <xsl:if test="not($espNode)">
+                        <xsl:message terminate="yes">ESP process is undefined!</xsl:message>
+                    </xsl:if>
+
+                    <xsl:variable name="espPort" select="$espNode/EspBinding[@service='wslogging']/@port"/>
+
+                    <xsl:if test="string($espPort) = ''">
+                        <xsl:message terminate="yes">ESP server port is undefined for <xsl:value-of select="$espServer"/>!</xsl:message>
+                    </xsl:if>
+
+                    <xsl:variable name="espNetAddress" select="$espNode/Instance/@netAddress"/>
+
+                    <xsl:if test="string($espNetAddress) = ''">
+                        <xsl:message terminate="yes">ESP NetAddress is undefined!</xsl:message>
+                    </xsl:if>
+                    <xsl:variable name="wsloggingUrl"><xsl:text>http://</xsl:text><xsl:value-of select="$espNetAddress"/><xsl:text>:</xsl:text><xsl:value-of select="$espPort"/></xsl:variable>
+                        <LogAgent name="{$agentName}" type="LogAgent" services="GetTransactionSeed,UpdateLog" plugin="espserverloggingagent">
+                            <ESPServer url="{$wsloggingUrl}" user="{$agentNode/@User}" password="{$agentNode/@Password}"/>
+                            <xsl:if test="string($agentNode/@MaxServerWaitingSeconds) != ''">
+                                <MaxServerWaitingSeconds><xsl:value-of select="$agentNode/@MaxServerWaitingSeconds"/></MaxServerWaitingSeconds>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@FailSafe) != ''">
+                                <FailSafe><xsl:value-of select="$agentNode/@FailSafe"/></FailSafe>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@FailSafeLogsDir) != ''">
+                                <FailSafeLogsDir><xsl:value-of select="$agentNode/@FailSafeLogsDir"/></FailSafeLogsDir>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@MaxLogQueueLength) != ''">
+                                <MaxLogQueueLength><xsl:value-of select="$agentNode/@MaxLogQueueLength"/></MaxLogQueueLength>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@MaxTriesGTS) != ''">
+                                <MaxTriesGTS><xsl:value-of select="$agentNode/@MaxTriesGTS"/></MaxTriesGTS>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@MaxTriesRS) != ''">
+                                <MaxTriesRS><xsl:value-of select="$agentNode/@MaxTriesRS"/></MaxTriesRS>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
+                                <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
+                            </xsl:if>
+                            <Filters>
+                                <xsl:for-each select="$agentNode/Filter">
+                                    <Filter value="{current()/@filter}" type="{current()/@type}"/>
+                                </xsl:for-each>
+                            </Filters>
+                        </LogAgent>
+                </xsl:for-each>
+
+                </LoggingManager>
+
+            </xsl:if>
+        </EspService>
+
+        <EspBinding name="{$bindName}" service="{$serviceName}" protocol="{$bindingNode/@protocol}" type="{$bindType}" plugin="{$servicePlugin}" netAddress="0.0.0.0" port="{$bindingNode/@port}">
+            <xsl:call-template name="bindAuthentication">
+                <xsl:with-param name="bindingNode" select="$bindingNode"/>
+                <xsl:with-param name="authMethod" select="$authNode/@method"/>
+            </xsl:call-template>
+        </EspBinding>
+    </xsl:template>
+
+    <!-- UTILITY templates-->
+    <xsl:template name="bindAuthentication">
+      <xsl:param name="authMethod"/>
+      <xsl:param name="bindingNode"/>
+      <xsl:choose>
+         <xsl:when test="$authMethod='basic'">
+            <Authenticate type="Basic" method="UserDefined">
+               <xsl:for-each select="$bindingNode/Authenticate[string(@path) != '']">
+                  <Location path="{@path}"/>
+               </xsl:for-each>
+            </Authenticate>
+         </xsl:when>
+         <xsl:when test="$authMethod='local'">
+            <Authenticate method="Local">
+               <xsl:for-each select="$bindingNode/Authenticate[string(@path) != '']">
+                  <Location path="{@path}" resource="{@resource}" required="{@access}" description="{@description}"/>
+               </xsl:for-each>
+            </Authenticate>
+         </xsl:when>
+         <xsl:when test="$authMethod='ldap' or $authMethod='ldaps'">
+            <Authenticate method="LdapSecurity" config="ldapserver">
+            <xsl:copy-of select="$bindingNode/@resourcesBasedn"/> <!--if binding has an ldap resourcebasedn specified then copy it out -->
+
+            <xsl:for-each select="$bindingNode/Authenticate">
+               <Location path="{@path}" resource="{@resource}" access="{@access}"/>
+            </xsl:for-each>
+
+            <xsl:for-each select="$bindingNode/AuthenticateFeature[@authenticate='Yes']">
+               <Feature name="{@name}" path="{@path}" resource="{@resource}" required="{@access}" description="{@description}"/>
+            </xsl:for-each>
+            </Authenticate>
+         </xsl:when>
+        <xsl:when test="$authMethod='htpasswd'">
+          <Authenticate method="htpasswd">
+            <xsl:attribute name="htpasswdFile"> <xsl:value-of select="$bindingNode/../Authentication/@htpasswdFile"/> </xsl:attribute>
+          </Authenticate>
+        </xsl:when>
+      </xsl:choose>
+    </xsl:template>
+
+</xsl:stylesheet>

initfiles/componentfiles/configxml/CMakeLists.txt

@@ -83,6 +83,9 @@ FOREACH( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/eclcc.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/validateAll.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/esdlsvcengine.xsd
+    ${CMAKE_CURRENT_SOURCE_DIR}/loggingmanager.xsd
+    ${CMAKE_CURRENT_SOURCE_DIR}/esploggingagent.xsd
+    ${CMAKE_CURRENT_SOURCE_DIR}/wslogging.xsd
 )
     Install ( FILES ${iFILES} DESTINATION componentfiles/configxml COMPONENT Runtime)
 ENDFOREACH ( iFILES )

initfiles/componentfiles/configxml/buildsetCC.xml.in

@@ -331,6 +331,41 @@
                  plugin="esdl_svc_engine"
                  type="DynamicESDL"/>
     </BuildSet>
+    <BuildSet deployable="no"
+             installSet="deploy_map.xml"
+             name="esploggingagent"
+             path="componentfiles/esploggingagent"
+             processName="ESPLoggingAgent"
+             schema="esploggingagent.xsd"/>
+    <BuildSet deployable="no"
+             installSet="deploy_map.xml"
+             name="loggingmanager"
+             path="componentfiles/loggingmanager"
+             processName="LoggingManager"
+             schema="loggingmanager.xsd"/>
+    <BuildSet deployable="no"
+             installSet="deploy_map.xml"
+             name="wslogging"
+             path="componentfiles/wslogging"
+             processName="EspService"
+             schema="wslogging.xsd">
+      <Properties bindingType="loggingservice_binding"
+                defaultPort="8146"
+                defaultResourcesBasedn="ou=EspServices,ou=ecl"
+                defaultSecurePort="18146"
+                plugin="wslogging"
+                type="wslogging">
+        <Authenticate access="Read"
+                   description="Root access to WS Logging service"
+                   path="/"
+                   required="Read"
+                   resource="WsLoggingAccess"/>
+        <AuthenticateFeature description="Access to WS Logging service"
+                   path="WsLoggingAccess"
+                   resource="WsLoggingAccess"
+                   service="wslogging"/>
+    </Properties>
+   </BuildSet>
   </Build>
 </Programs>
 </Environment>

initfiles/componentfiles/configxml/cgencomplist_linux.xml

@@ -93,6 +93,11 @@
   <Component name="ws_account" processName='EspService' schema='esp_service_account.xsd' deployable='no'>
     <File name="@temp/esp_service.xsl" method="esp_service_module" destName="WsAccount.xml" destPath="@temp"/>
   </Component>
+  <Component name="ESPLoggingAgent" processName='ESPLoggingAgent' schema='esploggingagent.xsd' deployable='no'/>
+  <Component name="LoggingManager" processName='LoggingManager' schema='loggingmanager.xsd' deployable='no'/>
+  <Component name="WsLogging" processName='EspService' schema='wslogging.xsd' deployable='no'>
+    <File name="@temp/esp_service_wslogging.xsl" method="esp_service_module"/>
+  </Component>
   <Component name="ws_accurintAccess" processName='EspService' schema='ws_accurintAccess.xsd' deployable='no'>
     <File name="@temp/esp_service.xsl" method="esp_service_module" destName="WsAccurintAccess.xml" destPath="@temp"/>
   </Component>

initfiles/componentfiles/configxml/cgencomplist_win.xml

@@ -96,6 +96,11 @@
   <Component name="ws_account" processName='EspService' schema='esp_service_account.xsd' deployable='no'>
     <File name="@temp\esp_service.xsl" method="esp_service_module" destName="WsAccount.xml" destPath="@temp"/>
   </Component>
+  <Component name="ESPLoggingAgent" processName='ESPLoggingAgent' schema='esploggingagent.xsd' deployable='no'/>
+  <Component name="LoggingManager" processName='LoggingManager' schema='loggingmanager.xsd' deployable='no'/>
+  <Component name="WsLogging" processName='EspService' schema='wslogging.xsd' deployable='no'>
+    <File name="@temp/esp_service_wslogging.xsl" method="esp_service_module"/>
+  </Component>
   <Component name="ws_accurintAccess" processName='EspService' schema='ws_accurintAccess.xsd' deployable='no'>
     <File name="@temp\esp_service.xsl" method="esp_service_module" destName="WsAccurintAccess.xml" destPath="@temp"/>
   </Component>

initfiles/componentfiles/configxml/esploggingagent.xsd

@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    Copyright (C) 2014 HPCC Systems.
+#
+#    All rights reserved.
+################################################################################
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
+  <xs:include schemaLocation="environment.xsd"/>
+  <xs:element name="ESPLoggingAgent">
+    <xs:complexType>
+      <xs:annotation>
+        <xs:appinfo>
+            <title>ESP Logging Agent</title>
+        </xs:appinfo>
+      </xs:annotation>
+      <xs:attribute name="build" type="buildType" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <viewType>hidden</viewType>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="buildSet" type="buildSetType" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <viewType>hidden</viewType>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="name" type="xs:string" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Name for this agent</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="description" type="xs:string" use="optional" default="ESP Logging Agent">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Description for this process</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="MaxServerWaitingSeconds" type="xs:nonNegativeInteger" use="optional">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>timeout for waiting ESP server response.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="MaxTriesGTS" type="xs:nonNegativeInteger" use="optional" default="3">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>maximum times of resending GetTransactionSeed requests after a failure.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="MaxTriesRS" type="xs:nonNegativeInteger" use="optional" default="3">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>maximum times of resending log update requests after a failure.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="MaxLogQueueLength" type="xs:nonNegativeInteger" use="optional" default="500000">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>maximum queue length for log update requests.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="QueueSizeSignal" type="xs:nonNegativeInteger" use="optional" default="10000">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Add warning to esp log when the queue length of log update requests reaches this value.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="FailSafe" type="xs:boolean" use="optional" default="true">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Enable FailSafe functionality.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="FailSafeLogsDir" type="xs:string" use="optional" default="">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Log directory where Failsafe files are stored.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:sequence>
+        <xs:element name="Filter" minOccurs="0" maxOccurs="unbounded">
+          <xs:annotation>
+            <xs:appinfo>
+              <title>Filters</title>
+            </xs:appinfo>
+          </xs:annotation>
+          <xs:complexType>
+            <xs:attribute name="filter" type="relativePath" use="required">
+              <xs:annotation>
+                <xs:appinfo>
+                  <colIndex>1</colIndex>
+                </xs:appinfo>
+              </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="type" type="xs:string" use="optional">
+              <xs:annotation>
+                <xs:appinfo>
+                  <colIndex>2</colIndex>
+                </xs:appinfo>
+              </xs:annotation>
+            </xs:attribute>
+          </xs:complexType>
+        </xs:element>
+      </xs:sequence>
+      <xs:attributeGroup ref="ESPServer"/>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:attributeGroup name="ESPServer">
+    <xs:attribute name="ESPServer" type="espprocessType" use="required">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Specifies the ESPServer process.</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="User" type="xs:string" use="optional" default="">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Specifies the user name for accessing the ESPServer.</tooltip>
+          <required>true</required>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="Password" type="xs:string" use="optional" default="">
+      <xs:annotation>
+        <xs:appinfo>
+          <viewType>password</viewType>
+          <tooltip>Specifies the user's password for accessing the ESPServer.</tooltip>
+          <required>true</required>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+  </xs:attributeGroup>
+</xs:schema>

initfiles/componentfiles/configxml/loggingmanager.xsd

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    Copyright (C) 2014 HPCC Systems.
+#
+#    All rights reserved.
+################################################################################
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
+  <xs:include schemaLocation="environment.xsd"/>
+  <xs:element name="LoggingManager">
+    <xs:complexType>
+      <xs:annotation>
+        <xs:appinfo>
+            <title>Logging Manager</title>
+        </xs:appinfo>
+      </xs:annotation>
+      <xs:attribute name="build" type="buildType" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <viewType>hidden</viewType>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="buildSet" type="buildSetType" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <viewType>hidden</viewType>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="name" type="xs:string" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Name for this Logging Manager</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="description" type="xs:string" use="optional" default="Logging Manager">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Description for this process</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+
+      <xs:sequence>
+        <xs:element name="ESPLoggingAgent" maxOccurs="unbounded">
+          <xs:annotation>
+            <xs:appinfo>
+              <title>ESP Logging Agents</title>
+            </xs:appinfo>
+          </xs:annotation>
+          <xs:complexType>
+            <xs:attribute name="ESPLoggingAgent" type="esploggingagentType" use="required">
+                <xs:annotation>
+                  <xs:appinfo>
+                    <tooltip>Specifies the ESP Logging Agent.</tooltip>
+                  </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+          </xs:complexType>
+        </xs:element>
+     </xs:sequence>
+
+    </xs:complexType>
+  </xs:element>
+</xs:schema>

initfiles/componentfiles/configxml/wslogging.xsd

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    Copyright (C) 2014 HPCC Systems.
+#
+#    All rights reserved.
+################################################################################
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
+  <xs:include schemaLocation="environment.xsd"/>
+  <xs:element name="EspService">
+    <xs:complexType>
+      <xs:attribute name="build" type="buildType" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>The build name to be deployed</tooltip>
+            <viewType>hidden</viewType>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="buildSet" type="buildSetType" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <viewType>hidden</viewType>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="name" type="xs:string" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Name for this service</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="description" type="xs:string" use="optional" default="ws_logging ESP service">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>ws_logging ESP service</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+
+      <!--Testing code -->
+      <xs:attribute name="LoggingManager" type="loggingmanagerType" use="optional">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Specifies the Logging Manager.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+
+      <!--Add logging agent code here-->
+    </xs:complexType>
+  </xs:element>
+
+</xs:schema>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -276,6 +276,41 @@
              schema="topology.xsd"/>
    <BuildSet deployable="no"
              installSet="deploy_map.xml"
+             name="esploggingagent"
+             path="componentfiles/esploggingagent"
+             processName="ESPLoggingAgent"
+             schema="esploggingagent.xsd"/>
+   <BuildSet deployable="no"
+             installSet="deploy_map.xml"
+             name="loggingmanager"
+             path="componentfiles/loggingmanager"
+             processName="LoggingManager"
+             schema="loggingmanager.xsd"/>
+   <BuildSet deployable="no"
+             installSet="deploy_map.xml"
+             name="wslogging"
+             path="componentfiles/wslogging"
+             processName="EspService"
+             schema="wslogging.xsd">
+     <Properties bindingType="loggingservice_binding"
+                defaultPort="8146"
+                defaultResourcesBasedn="ou=EspServices,ou=ecl"
+                defaultSecurePort="18146"
+                plugin="wslogging"
+                type="wslogging">
+       <Authenticate access="Read"
+                   description="Root access to WS Logging service"
+                   path="/"
+                   required="Read"
+                   resource="WsLoggingAccess"/>
+       <AuthenticateFeature description="Access to WS Logging service"
+                          path="WsLoggingAccess"
+                          resource="WsLoggingAccess"
+                          service="wslogging"/>
+     </Properties>
+   </BuildSet>
+   <BuildSet deployable="no"
+             installSet="deploy_map.xml"
              name="ws_ecl"
              path="componentfiles/ws_ecl"
              processName="EspService"

initfiles/etc/DIR_NAME/genenvrules.conf

@@ -1,7 +1,7 @@
 
 [Algorithm]
 max_comps_per_node=4
-do_not_generate=SiteCertificate,dfuplus,soapplus,eclplus,ldapServer,ws_account,eclserver,ecldirect,DynamicESDL
+do_not_generate=SiteCertificate,dfuplus,soapplus,eclplus,ldapServer,ws_account,eclserver,ecldirect,DynamicESDL,ESPLoggingAgent,LoggingManager,WsLogging
 avoid_combo=dali-eclagent,dali-sasha
 comps_on_all_nodes=dafilesrv,ftslave
 exclude_from_comps_on_all_nodes=ldapServer