HPCC-15481 Add 2 transaction ID functions to logging

Also revised the code to avoid duplicated code in CCassandraLogAgent
and CMySQLLogAgent:
1. Moved ensureInputString(), CLogTable::loadMappings(), and
CLogGroup::loadMappings() to datafieldmap.cpp;
2. Moved the code for reading LogGroup Cfg to readLogGroupCfg();
Moved the code for reading LogSource Cfg to readLogSourceCfg();
3. Created a base class CDBLogAgentBase for CCassandraLogAgent
and CMySQLLogAgent. Moved getTransactionSeed(), updateLog(),
checkLogSource(), buildUpdateLogStatement(), appendFieldInfo(),
addMissingFields(), getLoggingTransactionID(), and filterLog
Content() from CCassandraLogAgent to CDBLogAgentBase. Those
methods can be shared with CMySQLLogAgent.
4. Moved the code for reading DB Cfg to readDBCfg(); Moved the
code for reading Transaction Cfg to readTransactionCfg().

Also: 1. for getTransID(), read method name from trans data array;
2. in cassandra logging agent, set DB server/userID/password every
time before creating a new connection; 3. Change 3 #define's to
static; 4. throw exception if no init() called; 5. add createLocal
TransactionSeed() used for local Transaction ID.

Shared files is moved into logginglib

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

esp/logging/CMakeLists.txt

@@ -13,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
+HPCC_ADD_SUBDIRECTORY (logginglib)
 HPCC_ADD_SUBDIRECTORY (loggingagent)
 HPCC_ADD_SUBDIRECTORY (loggingmanager)
 HPCC_ADD_SUBDIRECTORY (test)

esp/logging/loggingagent/cassandraloggingagent/CMakeLists.txt

@@ -37,7 +37,7 @@ if(USE_CASSANDRA)
      ${HPCC_SOURCE_DIR}/esp/platform
      ${HPCC_SOURCE_DIR}/esp/bindings
      ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
-     ${HPCC_SOURCE_DIR}/esp/logging
+     ${HPCC_SOURCE_DIR}/esp/logging/logginglib
   )
 
   ADD_DEFINITIONS( -D_USRDLL -DCASSANDRALOGAGENT_EXPORTS )
@@ -48,12 +48,13 @@ if(USE_CASSANDRA)
 
   HPCC_ADD_LIBRARY( cassandralogagent SHARED ${SRCS} )
 
-  install ( TARGETS cassandralogagent RUNTIME DESTINATION bin LIBRARY DESTINATION lib )
+  install ( TARGETS cassandralogagent RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
   add_dependencies (cassandralogagent espscm)
   target_link_libraries ( cassandralogagent
     cassandraembed
     eclrtl
     esphttp
     jlib
+    logginglib
   )
 endif(USE_CASSANDRA)

esp/logging/loggingagent/cassandraloggingagent/cassandralogagent.cpp

@@ -19,9 +19,7 @@
 #include "esploggingservice_esp.ipp"
 #include "cassandralogagent.hpp"
 
-const int DefaultMaxTriesGTS = -1;
-const char* const DefaultloggingTransactionApp = "CassandraloggingTransaction";
-const char* const DefaultTransactionApp = "DefauleTransaction";
+static const int defaultMaxTriesGTS = -1;
 
 static void setCassandraLogAgentOption(StringArray& opts, const char* opt, const char* val)
 {
@@ -38,54 +36,6 @@ static const CassValue* getSingleResult(const CassResult* result)
     return row ? cass_row_get_column(row, 0) : NULL;
 }
 
-void ensureInputString(const char* input, bool lowerCase, StringBuffer& inputStr, int code, const char* msg)
-{
-    inputStr.set(input).trim();
-    if (inputStr.isEmpty())
-        throw MakeStringException(code, "%s", msg);
-    if (lowerCase)
-        inputStr.toLowerCase();
-}
-
-void CLogTable::loadMappings(IPropertyTree& fieldList)
-{
-    StringBuffer name, mapTo, fieldType, defaultValue;
-    Owned<IPropertyTreeIterator> itr = fieldList.getElements("Field");
-    ForEach(*itr)
-    {
-        IPropertyTree &map = itr->query();
-
-        ensureInputString(map.queryProp("@name"), false, name, -1, "Field @name required");
-        ensureInputString(map.queryProp("@mapto"), true, mapTo, -1, "Field @mapto required");
-        ensureInputString(map.queryProp("@type"), true, fieldType, -1, "Field @type required");
-        defaultValue = map.queryProp("@default");
-        defaultValue.trim();
-
-        Owned<CLogField> field = new CLogField(name.str(), mapTo.str(), fieldType.str());
-        if (!defaultValue.isEmpty())
-            field->setDefault(defaultValue.str());
-        logFields.append(*field.getClear());
-    }
-}
-
-void CLogGroup::loadMappings(IPropertyTree& fieldList)
-{
-    StringBuffer tableName;
-    Owned<IPropertyTreeIterator> itr = fieldList.getElements("Fieldmap");
-    ForEach(*itr)
-    {
-        ensureInputString(itr->query().queryProp("@table"), true, tableName, -1, "Fieldmap @table required");
-
-        Owned<CLogTable> table = new CLogTable(tableName.str());
-        table->loadMappings(itr->query());
-        CIArrayOf<CLogField>& logFields = table->getLogFields();
-        if (logFields.length() < 1)
-            throw MakeStringException(-1,"No Fieldmap for %s", tableName.str());
-
-        logTables.append(*table.getClear());
-    }
-}
-
 bool CCassandraLogAgent::init(const char* name, const char* type, IPropertyTree* cfg, const char* process)
 {
     if (!name || !*name || !type || !*type)
@@ -98,53 +48,20 @@ bool CCassandraLogAgent::init(const char* name, const char* type, IPropertyTree*
     if(!cassandra)
         throw MakeStringException(-1, "Unable to find Cassandra settings for log agent %s:%s", name, type);
 
-    agentName.set(name);
-    ensureInputString(cassandra->queryProp("@server"), true, dbServer, -1, "Cassandra server required");
-    ensureInputString(cassandra->queryProp("@dbName"), true, defaultDB, -1, "Database name required");
+    readDBCfg(cassandra, dbServer, dbUserID, dbPassword);
 
-    const char* userID = cassandra->queryProp("@dbUser");
-    if (userID && *userID)
-    {
-        const char* encodedPassword = cassandra->queryProp("@dbPassWord");
-        if (!encodedPassword || !*encodedPassword)
-            throw MakeStringException(-1, "Cassandra Database password required");
-
-        decrypt(dbPassword, encodedPassword);
-        dbUserID.set(userID);
-    }
+    //Read information about data mapping for every log groups
+    readLogGroupCfg(cfg, defaultLogGroup, logGroups);
+    if (defaultLogGroup.isEmpty())
+        throw MakeStringException(-1,"LogGroup not defined");
 
-    {
-        //Read information about data mapping
-        StringBuffer groupName;
-        Owned<IPropertyTreeIterator> iter = cfg->getElements("LogGroup");
-        ForEach(*iter)
-        {
-            ensureInputString(iter->query().queryProp("@name"), true, groupName, -1, "LogGroup @name required");
-            Owned<CLogGroup> logGroup = new CLogGroup(groupName.str());
-            logGroup->loadMappings(iter->query());
-            logGroups.setValue(groupName.str(), logGroup);
-            if (defaultLogGroup.isEmpty())
-                defaultLogGroup.set(groupName.str());
-        }
-    }
+    //Read mapping between log sources and log groups
+    readLogSourceCfg(cfg, logSourceCount, logSourcePath, logSources);
 
-    logSourceCount = 0;
-    Owned<IPropertyTreeIterator> iter2 = cfg->getElements("LogSourceMap/LogSource");
-    ForEach(*iter2)
-    {
-        StringBuffer name, groupName, dbName;
-        ensureInputString(iter2->query().queryProp("@name"), false, name, -1, "LogSource @name required");
-        ensureInputString(iter2->query().queryProp("@maptologgroup"), true, groupName, -1, "LogSource @maptologgroup required");
-        ensureInputString(iter2->query().queryProp("@maptodb"), true, dbName, -1, "LogSource @maptodb required");
-        Owned<CLogSource> logSource = new CLogSource(name.str(), groupName.str(), dbName.str());
-        logSources.setValue(name.str(), logSource);
-        logSourceCount++;
-    }
+    //Read transactions settings
+    readTransactionCfg(cfg);
 
-    maxTriesGTS = cfg->getPropInt("MaxTriesGTS", DefaultMaxTriesGTS);
-    loggingTransactionApp.set(cfg->hasProp("loggingTransaction") ? cfg->queryProp("loggingTransaction") : DefaultloggingTransactionApp);
-    defaultTransactionApp.set(cfg->hasProp("defaultTransaction") ? cfg->queryProp("defaultTransaction") : DefaultTransactionApp);
-    loggingTransactionCount = 0;
+    maxTriesGTS = cfg->getPropInt("MaxTriesGTS", defaultMaxTriesGTS);
 
     //Setup Cassandra
     initKeySpace();
@@ -158,27 +75,19 @@ void CCassandraLogAgent::initKeySpace()
     if (!cassSession)
         throw MakeStringException(-1,"Unable to create cassandra cassSession session");
 
-    StringArray opts;
-    setCassandraLogAgentOption(opts, "contact_points", dbServer.str());
-    if (!dbUserID.isEmpty())
-    {
-        setCassandraLogAgentOption(opts, "user", dbUserID.str());
-        setCassandraLogAgentOption(opts, "password", dbPassword.str());
-    }
-
-    cassSession->setOptions(opts);
+    setSessionOptions(NULL);
 
-    //prepare defaultDB
-    ensureKeySpace();
+    //ensure defaultDB
+    ensureDefaultKeySpace();
 
-    //prepare transSeed tables
-    initTransSeedTable();
+    //ensure transSeed tables
+    ensureTransSeedTable();
 
     //Read logging transaction seed
-    queryTransactionSeed(loggingTransactionApp.get(), transactionSeed);
+    queryTransactionSeed(loggingTransactionApp.get(), loggingTransactionSeed);
 }
 
-void CCassandraLogAgent::ensureKeySpace()
+void CCassandraLogAgent::ensureDefaultKeySpace()
 {
     CassandraSession s(cass_session_new());
     CassandraFuture future1(cass_session_connect(s, cassSession->queryCluster()));
@@ -193,33 +102,36 @@ void CCassandraLogAgent::ensureKeySpace()
     s.set(NULL);
 }
 
-void CCassandraLogAgent::initTransSeedTable()
+void CCassandraLogAgent::ensureTransSeedTable()
 {
     //Create transaction seed table as needed
     StringBuffer transSeedTableKeys;
     StringArray transSeedTableColumnNames, transSeedTableColumnTypes;
     transSeedTableColumnNames.append("id");
     transSeedTableColumnTypes.append("int");
-    transSeedTableColumnNames.append("agent_name");
-    transSeedTableColumnTypes.append("varchar");
     transSeedTableColumnNames.append("application");
     transSeedTableColumnTypes.append("varchar");
-    transSeedTableColumnNames.append("update_time");
-    transSeedTableColumnTypes.append("timestamp");
-    transSeedTableKeys.set("application, agent_name"); //primary keys
+    transSeedTableKeys.set("application"); //primary keys
 
-    setKeySpace(defaultDB.str());
+    //The defaultDB has transactions table.
+    setSessionOptions(defaultDB.str());
     cassSession->connect();
-    createTable(defaultDB.str(), "transactions", transSeedTableColumnNames, transSeedTableColumnTypes, transSeedTableKeys.str());
+    createTable(defaultDB.str(), transactionTable.str(), transSeedTableColumnNames, transSeedTableColumnTypes, transSeedTableKeys.str());
 
-    unsigned transactionCount = 0;
-    if (executeSimpleSelectStatement("SELECT COUNT(*) FROM transactions", transactionCount) == 0)
+    unsigned id = 0;
+    VStringBuffer st("SELECT id FROM %s LIMIT 1;", transactionTable.str());
+    if (!executeSimpleSelectStatement(st.str(), id))
     {
-        VStringBuffer st("INSERT INTO transactions (id, agent_name, application, update_time) values ( 10000, '%s', '%s', toUnixTimestamp(now()));", agentName.get(), loggingTransactionApp.get());
+        st.setf("INSERT INTO %s (id, application) values ( 10000, '%s');",
+            transactionTable.str(), loggingTransactionApp.get());
         executeSimpleStatement(st.str());
 
-        st.setf("INSERT INTO transactions (id, agent_name, application, update_time) values ( 10000, '%s', '%s', toUnixTimestamp(now()));", agentName.get(), defaultTransactionApp.get());
-        executeSimpleStatement(st.str());
+        if (!strieq(defaultTransactionApp.get(), loggingTransactionApp.get()))
+        {
+            st.setf("INSERT INTO %s (id, application) values ( 10000, '%s');",
+                transactionTable.str(), defaultTransactionApp.get());
+            executeSimpleStatement(st.str());
+        }
     }
     cassSession->disconnect();
 }
@@ -229,209 +141,42 @@ void CCassandraLogAgent::queryTransactionSeed(const char* appName, StringBuffer&
     CriticalBlock b(transactionSeedCrit);
 
     unsigned seedInt = 0;
-    VStringBuffer st("SELECT id FROM transactions WHERE agent_name ='%s' AND application = '%s'", agentName.get(), appName);
-    setKeySpace(defaultDB.str()); //Switch to defaultDB since it may not be the current keyspace.
+    VStringBuffer st("SELECT id FROM %s WHERE application = '%s'", transactionTable.str(), appName);
+    setSessionOptions(defaultDB.str()); //Switch to defaultDB since it may not be the current keyspace.
     cassSession->connect();
     executeSimpleSelectStatement(st.str(), seedInt);
     seed.setf("%d", seedInt);
 
     //update transactions for the next seed
-    VStringBuffer updateQuery("UPDATE transactions SET id=%d WHERE agent_name ='%s' AND application = '%s'", ++seedInt, agentName.get(), appName);
+    VStringBuffer updateQuery("UPDATE %s SET id=%d WHERE application = '%s'",
+        transactionTable.str(), ++seedInt,  appName);
     executeSimpleStatement(updateQuery.str());
     cassSession->disconnect();
 }
 
-bool CCassandraLogAgent::getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp)
+void CCassandraLogAgent::setSessionOptions(const char *keyspace)
 {
-    unsigned retry = 1;
-    while (1)
-    {
-        try
-        {
-            const char* appName = req.getApplication();
-            if (!appName || !*appName)
-                appName = defaultTransactionApp.get();
-
-            StringBuffer logSeed;
-            queryTransactionSeed(appName, logSeed);
-
-            if (!logSeed.length())
-                throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to get TransactionSeed");
-
-            resp.setSeedId(logSeed.str());
-            resp.setStatusCode(0);
-            return true;
-        }
-        catch (IException* e)
-        {
-            StringBuffer errorStr, errorMessage;
-            errorMessage.append("Failed to get TransactionSeed: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
-            ERRLOG("%s -- try %d", errorMessage.str(), retry);
-            e->Release();
-            if (retry < maxTriesGTS)
-            {
-                Sleep(retry*3000);
-                retry++;
-            }
-            else
-            {
-                resp.setStatusCode(-1);
-                resp.setStatusMessage(errorMessage.str());
-                break;
-            }
-        }
-    }
-    return false;
-}
-
-bool CCassandraLogAgent::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp)
-{
-    try
-    {
-        StringBuffer requestBuf = req.getUpdateLogRequest();
-        if (requestBuf.isEmpty())
-            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log request.");
-
-        StringBuffer logDB, logSource;
-        requestBuf.insert(0, "<LogRequest>");
-        requestBuf.append("</LogRequest>");
-        Owned<IPropertyTree> logRequestTree = createPTreeFromXMLString(requestBuf.length(), requestBuf.str());
-        if (!logRequestTree)
-            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log request.");
-
-        CLogGroup* logGroup = checkLogSource(logRequestTree, logSource, logDB);
-        if (!logGroup)
-            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Log Group %s undefined.", logSource.str());
-
-        StringBuffer logID;
-        getLoggingTransactionID(logID);
-
-        CIArrayOf<CLogTable>& logTables = logGroup->getLogTables();
-        setKeySpace(logDB.str());
-        ForEachItemIn(i, logTables)
-        {
-            CLogTable& table = logTables.item(i);
-
-            StringBuffer cqlStatement;
-            if(!buildUpdateLogStatement(logRequestTree, logDB.str(), table, logID, cqlStatement))
-                throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed in creating SQL statement.");
-
-            if (getEspLogLevel() >= LogMax)
-                DBGLOG("CQL: %s\n", cqlStatement.str());
-
-            cassSession->connect();
-            executeSimpleStatement(cqlStatement);
-            cassSession->disconnect();
-        }
-        resp.setStatusCode(0);
-        return true;
-    }
-    catch (IException* e)
+    StringArray opts;
+    setCassandraLogAgentOption(opts, "contact_points", dbServer.str());
+    if (!dbUserID.isEmpty())
     {
-        StringBuffer errorStr, errorMessage;
-        errorMessage.append("Failed to update log: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
-        ERRLOG("%s", errorMessage.str());
-        e->Release();
-        resp.setStatusCode(-1);
-        resp.setStatusMessage(errorMessage.str());
-    }
-    return false;
-}
-
-CLogGroup* CCassandraLogAgent::checkLogSource(IPropertyTree* logRequest, StringBuffer& source, StringBuffer& logDB)
-{
-    if (logSourceCount == 0)
-    {//if no log source is configured, use default Log Group and DB
-        logDB.set(defaultDB.str());
-        return logGroups.getValue(defaultLogGroup.get());
+        setCassandraLogAgentOption(opts, "user", dbUserID.str());
+        if (!dbPassword.isEmpty())
+            setCassandraLogAgentOption(opts, "password", dbPassword.str());
     }
-    source = logRequest->queryProp("Source");
-    if (source.isEmpty())
-        throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log Source from request.");
-    CLogSource* logSource = logSources.getValue(source.str());
-    if (!logSource)
-        throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Log Source %s undefined.", source.str());
-
-    logDB.set(logSource->getDBName());
-    return logGroups.getValue(logSource->getGroupName());
-}
-
-void CCassandraLogAgent::getLoggingTransactionID(StringBuffer& id)
-{
-    CriticalBlock b(uniqueIDCrit);
-    id.set(transactionSeed.str()).append("-").append(++loggingTransactionCount);
+    if (keyspace && *keyspace)
+        setCassandraLogAgentOption(opts, "keyspace", keyspace);
+    cassSession->setOptions(opts);
 }
 
-bool CCassandraLogAgent::buildUpdateLogStatement(IPropertyTree* logRequest, const char* logDB, CLogTable& table, StringBuffer& logID, StringBuffer& cqlStatement)
+void CCassandraLogAgent::createTable(const char *dbName, const char *tableName, StringArray& columnNames, StringArray& columnTypes, const char* keys)
 {
-    StringBuffer fields, values;
-    BoolHash handledFields;
-    CIArrayOf<CLogField>& logFields = table.getLogFields();
-    ForEachItemIn(i, logFields) //Go through data items to be logged
-    {
-        CLogField& logField = logFields.item(i);
-
-        StringBuffer colName = logField.getMapTo();
-        bool* found = handledFields.getValue(colName.str());
-        if (found && *found)
-            continue;
-
-        StringBuffer path = logField.getName();
-        if (path.charAt(path.length() - 1) == ']')
-        {//Attr filter. Separate the last [] from the path.
-            const char* pTr = path.str();
-            const char* ppTr = strrchr(pTr, '[');
-            if (!ppTr)
-                continue;
-
-            StringBuffer attr;
-            attr.set(ppTr+1);
-            attr.setLength(attr.length() - 1);
-            path.setLength(ppTr - pTr);
-
-            StringBuffer colValue;
-            Owned<IPropertyTreeIterator> itr = logRequest->getElements(path.str());
-            ForEach(*itr)
-            {//Log the first valid match just in case more than one matches.
-                IPropertyTree& ppTree = itr->query();
-                colValue.set(ppTree.queryProp(attr.str()));
-                if (colValue.length())
-                {
-                    addField(logField, colName.str(), colValue, fields, values);
-                    handledFields.setValue(colName.str(), true);
-                    break;
-                }
-            }
-            continue;
-        }
-
-        Owned<IPropertyTreeIterator> itr = logRequest->getElements(path.str());
-        ForEach(*itr)
-        {
-            IPropertyTree& ppTree = itr->query();
-
-            StringBuffer colValue;
-            if (ppTree.hasChildren()) //This is a tree branch.
-                toXML(&ppTree, colValue);
-            else
-                ppTree.getProp(NULL, colValue);
-
-            if (colValue.length())
-            {
-                addField(logField, colName.str(), colValue, fields, values);
-                handledFields.setValue(colName.str(), true);
-                break;
-            }
-        }
-    }
-
-    //add any default fields that may be required but not in request.
-    addMissingFields(logFields, handledFields, fields, values);
-    appendFieldInfo("log_id", logID, fields, values, true);
+    StringBuffer fields;
+    ForEachItemIn(i, columnNames)
+        fields.appendf("%s %s,", columnNames.item(i), columnTypes.item(i));
 
-    cqlStatement.setf("INSERT INTO %s.%s (%s, date_added) values (%s, toUnixTimestamp(now()));",
-        logDB, table.getTableName(), fields.str(), values.str());
-    return true;
+    VStringBuffer createTableSt("CREATE TABLE IF NOT EXISTS %s.%s (%s PRIMARY KEY (%s));", dbName, tableName, fields.str(), keys);
+    executeSimpleStatement(createTableSt.str());
 }
 
 void CCassandraLogAgent::addField(CLogField& logField, const char* name, StringBuffer& value, StringBuffer& fields, StringBuffer& values)
@@ -466,6 +211,8 @@ void CCassandraLogAgent::addField(CLogField& logField, const char* name, StringB
             unsigned char c = str[i];
             if(c == '\t' || c == '\n' || c== '\r')
                 values.append(' ');
+            else if(c == '\'')
+                values.append('"');
             else if(c < 32 || c > 126)
                 values.append('?');
             else
@@ -478,55 +225,10 @@ void CCassandraLogAgent::addField(CLogField& logField, const char* name, StringB
     DBGLOG("Unknown format %s", fieldType);
 }
 
-void CCassandraLogAgent::appendFieldInfo(const char* field, StringBuffer& value, StringBuffer& fields, StringBuffer& values, bool quoted)
-{
-    if(values.length() != 0)
-        values.append(',');
-    if (quoted)
-        values.append('\'').append(value.length(), value.str()).append('\'');
-    else
-        values.append(value.length(), value.str());
-
-    if(fields.length() != 0)
-        fields.append(',');
-    fields.append(field);
-}
-
-void CCassandraLogAgent::addMissingFields(CIArrayOf<CLogField>& logFields, BoolHash& handledFields, StringBuffer& fields, StringBuffer& values)
-{
-    ForEachItemIn(i, logFields) //Go through data items to be logged
-    {
-        CLogField& logField = logFields.item(i);
-        const char* colName = logField.getMapTo();
-        bool* found = handledFields.getValue(colName);
-        if (found && *found)
-            continue;
-        StringBuffer value = logField.getDefault();
-        if (!value.isEmpty())
-            addField(logField, colName, value, fields, values);
-    }
-}
-
-void CCassandraLogAgent::filterLogContent(IEspUpdateLogRequestWrap* req)
-{
-    return; //No filter in CCassandraLogAgent for now
-}
-
-void CCassandraLogAgent::setKeySpace(const char *keyspace)
-{
-    StringArray opts;
-    setCassandraLogAgentOption(opts, "keyspace", keyspace);
-    cassSession->setOptions(opts);
-}
-
-void CCassandraLogAgent::createTable(const char *dbName, const char *tableName, StringArray& columnNames, StringArray& columnTypes, const char* keys)
+void CCassandraLogAgent::setUpdateLogStatement(const char* dbName, const char* tableName,
+        const char* fields, const char* values, StringBuffer& statement)
 {
-    StringBuffer fields;
-    ForEachItemIn(i, columnNames)
-        fields.appendf("%s %s,", columnNames.item(i), columnTypes.item(i));
-
-    VStringBuffer createTableSt("CREATE TABLE IF NOT EXISTS %s.%s (%s PRIMARY KEY (%s));", dbName, tableName, fields.str(), keys);
-    executeSimpleStatement(createTableSt.str());
+    statement.setf("INSERT INTO %s.%s (%s, date_added) values (%s, toUnixTimestamp(now()));", dbName, tableName, fields, values);
 }
 
 void CCassandraLogAgent::executeSimpleStatement(const char* st)
@@ -536,8 +238,9 @@ void CCassandraLogAgent::executeSimpleStatement(const char* st)
     future.wait("execute");
 }
 
-void CCassandraLogAgent::executeSimpleStatement(StringBuffer& st)
+void CCassandraLogAgent::executeUpdateLogStatement(StringBuffer& st)
 {
+    cassSession->connect();
     CassandraFuture futurePrep(cass_session_prepare_n(cassSession->querySession(), st.str(), st.length()));
     futurePrep.wait("prepare statement");
 
@@ -545,16 +248,20 @@ void CCassandraLogAgent::executeSimpleStatement(StringBuffer& st)
     CassandraStatement statement(prepared.getClear());
     CassandraFuture future(cass_session_execute(cassSession->querySession(), statement));
     future.wait("execute");
+    cassSession->disconnect();
 }
 
-unsigned CCassandraLogAgent::executeSimpleSelectStatement(const char* st, unsigned& resultValue)
+bool CCassandraLogAgent::executeSimpleSelectStatement(const char* st, unsigned& resultValue)
 {
     CassandraStatement statement(cassSession->prepareStatement(st, getEspLogLevel()>LogNormal));
     CassandraFuture future(cass_session_execute(cassSession->querySession(), statement));
     future.wait("execute");
     CassandraResult result(cass_future_get_result(future));
+    if (cass_result_row_count(result) == 0)
+        return false;
+
     resultValue = getUnsignedResult(NULL, getSingleResult(result));
-    return resultValue;
+    return true;
 }
 
 extern "C"

esp/logging/loggingagent/cassandraloggingagent/cassandralogagent.hpp

@@ -26,7 +26,7 @@
 #include "cassandraembed.hpp"
 #include "loggingcommon.hpp"
 #include "loggingagentbase.hpp"
-#include "dbfieldmap.hpp"
+#include "datafieldmap.hpp"
 
 using namespace cassandraembed;
 
@@ -40,33 +40,26 @@ using namespace cassandraembed;
     #define CASSANDRALOGAGENT_API
 #endif
 
-class CCassandraLogAgent : public CInterface, implements IEspLogAgent
+class CCassandraLogAgent : public CDBLogAgentBase
 {
-    StringBuffer dbServer, defaultDB, dbUserID, dbPassword, transactionSeed;
-    StringAttr agentName, defaultLogGroup, defaultTransactionApp, loggingTransactionApp;
-    unsigned logSourceCount, loggingTransactionCount, maxTriesGTS;
-    MapStringToMyClass<CLogGroup> logGroups;
-    MapStringToMyClass<CLogSource> logSources;
+    StringBuffer dbServer, dbUserID, dbPassword;
     Owned<CassandraClusterSession> cassSession;
-    CriticalSection uniqueIDCrit, transactionSeedCrit;
-
-    CLogGroup* checkLogSource(IPropertyTree* logRequest, StringBuffer& source, StringBuffer& logDB);
-    void getLoggingTransactionID(StringBuffer& id);
-    bool buildUpdateLogStatement(IPropertyTree* logRequest, const char* logDB, CLogTable& table, StringBuffer& logID, StringBuffer& cqlStatement);
-    void addField(CLogField& logField, const char* name, StringBuffer& value, StringBuffer& fields, StringBuffer& values);
-    void appendFieldInfo(const char* field, StringBuffer& value, StringBuffer& fields, StringBuffer& values, bool quoted);
-    void addMissingFields(CIArrayOf<CLogField>& logFields, BoolHash& HandledFields, StringBuffer& fields, StringBuffer& values);
+    CriticalSection transactionSeedCrit;
 
     void initKeySpace();
-    void ensureKeySpace();
-    void setKeySpace(const char *keyspace);
-    void initTransSeedTable();
-    void queryTransactionSeed(const char* appName, StringBuffer& seed);
+    void ensureDefaultKeySpace();
+    void setSessionOptions(const char *keyspace);
+    void ensureTransSeedTable();
     void createTable(const char *dbName, const char *tableName, StringArray& columnNames, StringArray& columnTypes, const char* keys);
-    //void executeSimpleStatement(CassSession *session, const char *st);
     void executeSimpleStatement(const char *st);
     void executeSimpleStatement(StringBuffer& st);
-    unsigned executeSimpleSelectStatement(const char* st, unsigned& resultValue);
+    bool executeSimpleSelectStatement(const char* st, unsigned& resultValue);
+
+    virtual void queryTransactionSeed(const char* appName, StringBuffer& seed);
+    virtual void addField(CLogField& logField, const char* name, StringBuffer& value, StringBuffer& fields, StringBuffer& values);
+    virtual void setUpdateLogStatement(const char* dbName, const char* tableName,
+        const char* fields, const char* values, StringBuffer& statement);
+    virtual void executeUpdateLogStatement(StringBuffer& statement);
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -75,9 +68,6 @@ public:
     virtual ~CCassandraLogAgent() { logGroups.kill(); logSources.kill(); };
 
     virtual bool init(const char* name, const char* type, IPropertyTree* cfg, const char* process);
-    virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp);
-    virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp);
-    virtual void filterLogContent(IEspUpdateLogRequestWrap* req);
 };
 
 #endif //_CASSANDRALOGAGENT_HPP__

esp/logging/loggingagent/espserverloggingagent/CMakeLists.txt

@@ -44,7 +44,7 @@ include_directories (
     ${HPCC_SOURCE_DIR}/esp/clients/espcommonclient
     ${HPCC_SOURCE_DIR}/esp/esdl
     ${HPCC_SOURCE_DIR}/esp/esplib
-    ${HPCC_SOURCE_DIR}/esp/logging
+    ${HPCC_SOURCE_DIR}/esp/logging/logginglib
     ./../..
     ./..
 )
@@ -59,11 +59,12 @@ HPCC_ADD_LIBRARY( espserverloggingagent SHARED ${SRCS} )
 
 add_dependencies( espserverloggingagent espscm )
 
-install ( TARGETS espserverloggingagent DESTINATION ${LIB_DIR} )
+install ( TARGETS espserverloggingagent RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
 
 target_link_libraries ( espserverloggingagent
     ${XALAN_LIBRARIES} ${XERCES_LIBRARIES}
     jlib
     xmllib
+    logginglib
     esphttp
 )

esp/logging/loggingagent/espserverloggingagent/loggingagent.cpp

@@ -22,13 +22,14 @@
 #include "loggingagentbase.hpp"
 #include "loggingagent.hpp"
 
-const int DefaultMaxTriesGTS = -1;
-const char* const PropESPServer = "ESPServer";
-const char* const PropServerUrl = "@url";
-const char* const PropServerUserID = "@user";
-const char* const PropServerPassword = "@password";
-const char* const PropServerWaitingSeconds = "MaxServerWaitingSeconds";
-const char* const MaxTriesGTS = "MaxTriesGTS";
+static const int DefaultMaxTriesGTS = -1;
+static const char* const PropESPServer = "ESPServer";
+static const char* const PropServerUrl = "@url";
+static const char* const PropServerUserID = "@user";
+static const char* const PropServerPassword = "@password";
+static const char* const PropServerWaitingSeconds = "MaxServerWaitingSeconds";
+static const char* const MaxTriesGTS = "MaxTriesGTS";
+static const char* const appESPServerLoggingAgent = "ESPServerLoggingAgent";
 
 bool CESPServerLoggingAgent::init(const char * name, const char * type, IPropertyTree * cfg, const char * process)
 {
@@ -53,6 +54,38 @@ bool CESPServerLoggingAgent::init(const char * name, const char * type, IPropert
     maxServerWaitingSeconds = cfg->getPropInt(PropServerWaitingSeconds);
     maxGTSRetries = cfg->getPropInt(MaxTriesGTS, DefaultMaxTriesGTS);
 
+    BoolHash uniqueGroupNames;
+    StringBuffer sourceName, groupName, dbName, localTransactionSeed;
+    Owned<IPropertyTreeIterator> iter = cfg->getElements("LogSourceMap/LogSource");
+    ForEach(*iter)
+    {
+        ensureInputString(iter->query().queryProp("@name"), false, sourceName, -1, "LogSource @name required");
+        ensureInputString(iter->query().queryProp("@maptologgroup"), true, groupName, -1, "LogSource @maptologgroup required");
+        ensureInputString(iter->query().queryProp("@maptodb"), true, dbName, -1, "LogSource @maptodb required");
+        Owned<CLogSource> logSource = new CLogSource(sourceName.str(), groupName.str(), dbName.str());
+        logSources.setValue(sourceName.str(), logSource);
+
+        bool* found = uniqueGroupNames.getValue(groupName.str());
+        if (!found || !*found)
+        {
+            uniqueGroupNames.setValue(groupName.str(), true);
+            StringBuffer transactionSeed, statusMessage;
+            getTransactionSeed(groupName.str(), transactionSeed, statusMessage);
+            if (transactionSeed.length() > 0)
+            {
+                Owned<CTransIDBuilder> entry = new CTransIDBuilder(transactionSeed.str(), false);
+                transIDMap.setValue(groupName.str(), entry);
+                if (iter->query().getPropBool("@default", false))
+                    defaultGroup.set(groupName.str());
+            }
+            else
+                PROGLOG("Failed to get TransactionSeed for <%s>", groupName.str());
+        }
+    }
+    createLocalTransactionSeed(localTransactionSeed);
+    Owned<CTransIDBuilder> localTransactionEntry = new CTransIDBuilder(localTransactionSeed.str(), true);
+    transIDMap.setValue(appESPServerLoggingAgent, localTransactionEntry);
+
     readAllLogFilters(cfg);
     return true;
 }
@@ -127,55 +160,66 @@ bool CESPServerLoggingAgent::readLogFilters(IPropertyTree* cfg, unsigned groupID
     return hasFilter;
 }
 
+void CESPServerLoggingAgent::createLocalTransactionSeed(StringBuffer& transactionSeed)
+{
+    unsigned ip = queryHostIP().iphash();
+    unsigned mstick6char = ((unsigned)usTick() & 0xFFFFFF);
+    unsigned processId2char = ((unsigned)GetCurrentProcessId()) & 0xF;
+    unsigned threadId1char = ((unsigned)GetCurrentThreadId()) & 0xF;
+    transactionSeed.setf("%02X%06X%X%X", ip, mstick6char, processId2char, threadId1char);
+}
+
 bool CESPServerLoggingAgent::getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp)
 {
-    bool bRet = false;
+    StringBuffer statusMessage, transactionSeed;
+    int statusCode = getTransactionSeed(req.getApplication(), transactionSeed, statusMessage);
+    resp.setStatusCode(statusCode);
+    resp.setSeedId(transactionSeed.str());
+    if (statusMessage.length())
+        resp.setStatusMessage(statusMessage.str());
+    return (statusCode != -2);
+}
+
+int CESPServerLoggingAgent::getTransactionSeed(const char* appName, StringBuffer& transactionSeed, StringBuffer& statusMessage)
+{
     StringBuffer soapreq(
         "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
         "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
         " xmlns:SOAP-ENC=\"http://schemas.xmlsoap.org/soap/encoding/\">"
         " <soap:Body>");
-    soapreq.append("<GetTransactionSeedRequest/>");
+    if (!appName || !*appName)
+        soapreq.append("<GetTransactionSeedRequest/>");
+    else
+        soapreq.append("<GetTransactionSeedRequest><Application>").append(appName).append("</Application></GetTransactionSeedRequest>");
     soapreq.append("</soap:Body></soap:Envelope>");
 
     unsigned retry = 1;
+    int statusCode = 0;
     while (1)
     {
         try
         {
-            int statusCode = 0;
-            StringBuffer statusMessage, transactionSeed;
             if (!getTransactionSeed(soapreq, statusCode, statusMessage, transactionSeed))
                 throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed,"Failed to get TransactionSeed");
-
-            resp.setSeedId(transactionSeed.str());
-            resp.setStatusCode(statusCode);
-            if (statusMessage.length())
-                resp.setStatusMessage(statusMessage.str());
-            bRet = true;
             break;
         }
         catch (IException* e)
         {
-            StringBuffer errorStr, errorMessage;
-            errorMessage.append("Failed to get TransactionSeed: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
-            ERRLOG("%s -- try %d", errorMessage.str(), retry);
+            StringBuffer errorStr;
+            statusMessage.set("Failed to get TransactionSeed: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
+            ERRLOG("%s -- try %d", statusMessage.str(), retry);
             e->Release();
-            if (retry < maxGTSRetries)
-            {
-                Sleep(retry*3000);
-                retry++;
-            }
-            else
+            if (retry >= maxGTSRetries)
             {
-                resp.setStatusCode(-1);
-                resp.setStatusMessage(errorMessage.str());
+                statusCode = -2;
                 break;
             }
+            Sleep(retry*3000);
+            retry++;
         }
     }
 
-    return bRet;
+    return statusCode;
 }
 
 bool CESPServerLoggingAgent::getTransactionSeed(StringBuffer& soapreq, int& statusCode, StringBuffer& statusMessage, StringBuffer& seedID)
@@ -200,6 +244,27 @@ bool CESPServerLoggingAgent::getTransactionSeed(StringBuffer& soapreq, int& stat
     return true;
 }
 
+void CESPServerLoggingAgent::getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID)
+{
+    CTransIDBuilder* transIDBuilder = NULL;
+    StringAttr* source = transFields->getValue(sTransactionMethod);
+    if (source)
+    {
+        CLogSource* logSource = logSources.getValue(source->get());
+        if (logSource)
+            transIDBuilder = transIDMap.getValue(logSource->getGroupName());
+    }
+    if (!transIDBuilder && (defaultGroup.length() != 0))
+        transIDBuilder = transIDMap.getValue(defaultGroup.str());
+    if (!transIDBuilder)
+        transIDBuilder = transIDMap.getValue(appESPServerLoggingAgent);
+    if (!transIDBuilder) //This should not happen.
+        throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to getTransactionID");
+
+    transIDBuilder->getTransID(transFields, transactionID);
+    return;
+}
+
 bool CESPServerLoggingAgent::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp)
 {
     try

esp/logging/loggingagent/espserverloggingagent/loggingagent.hpp

@@ -19,6 +19,7 @@
 #define __ESPSERVERLOGGINGAGENT__HPP__
 
 #include "loggingcommon.hpp"
+#include "datafieldmap.hpp"
 
 #ifdef WIN32
     #ifdef ESPSERVERLOGGINGAGENT_EXPORTS
@@ -42,12 +43,51 @@ enum ESPLogContentGroup
 
 static const char * const espLogContentGroupNames[] = { "ESPContext", "UserContext", "UserRequest", "UserResponse", "BackEndResponse", "", NULL };
 
-class CESPLogContentGroupFilters : public CInterface
+class CTransIDBuilder : public CInterface, implements IInterface
+{
+    StringAttr seed;
+    bool localSeed;
+    unsigned __int64 seq;
+    void add(StringAttrMapping* transIDFields, const char* key, StringBuffer& id)
+    {
+        StringAttr* value = transIDFields->getValue(key);
+        if (value)
+            id.append(value->get()).append('-');
+        else
+            id.append('??-');
+    }
+
+public:
+    IMPLEMENT_IINTERFACE;
+    CTransIDBuilder(const char* _seed, bool _localSeed) : seed(_seed), localSeed(_localSeed), seq(0) { };
+    virtual ~CTransIDBuilder() {};
+
+    virtual const char* getTransSeed() { return seed.get(); };
+    virtual void getTransID(StringAttrMapping* transIDFields, StringBuffer& id)
+    {
+        id.clear();
+
+        if (transIDFields)
+        {
+            add(transIDFields, sTransactionDateTime, id);
+            add(transIDFields, sTransactionMethod, id);
+            add(transIDFields, sTransactionESPIP, id);
+        }
+        if (localSeed)
+            id.append(seed.get()).append("-X").append(++seq);
+        else
+            id.append(seed.get()).append('-').append(++seq);
+    };
+};
+
+class CESPLogContentGroupFilters : public CInterface, implements IInterface
 {
     ESPLogContentGroup group;
     StringArray filters;
 
 public:
+    IMPLEMENT_IINTERFACE;
+
     CESPLogContentGroupFilters(ESPLogContentGroup _group) : group(_group) {};
     ESPLogContentGroup getGroup() { return group; };
     StringArray& getFilters() { return filters; };
@@ -62,13 +102,15 @@ public:
 
 class CESPServerLoggingAgent : public CInterface, implements IEspLogAgent
 {
-    StringBuffer serviceName, loggingAgentName;
+    StringBuffer serviceName, loggingAgentName, defaultGroup;
     StringBuffer serverUrl, serverUserID, serverPassword;
     unsigned maxServerWaitingSeconds; //time out value for HTTP connection to logging server
     unsigned maxGTSRetries;
     StringArray     logContentFilters;
     CIArrayOf<CESPLogContentGroupFilters> groupFilters;
     bool logBackEndResp;
+    MapStringToMyClass<CTransIDBuilder> transIDMap;
+    MapStringToMyClass<CLogSource> logSources;
 
     void readAllLogFilters(IPropertyTree* cfg);
     bool readLogFilters(IPropertyTree* cfg, unsigned groupID);
@@ -77,8 +119,11 @@ class CESPServerLoggingAgent : public CInterface, implements IEspLogAgent
         IPropertyTree* in, IPropertyTree* updateLogRequestTree, bool& logContentEmpty);
     void addLogContentBranch(StringArray& branchNames, IPropertyTree* contentToLogBranch, IPropertyTree* updateLogRequestTree);
     bool sendHTTPRequest(StringBuffer& req, StringBuffer& resp, StringBuffer& status);
+    int getTransactionSeed(const char* source, StringBuffer& transactionSeed, StringBuffer& statusMessage);
     bool getTransactionSeed(StringBuffer& soapreq, int& statusCode, StringBuffer& statusMessage, StringBuffer& seedID);
 
+    virtual void createLocalTransactionSeed(StringBuffer& transactionSeed);
+
 public:
     IMPLEMENT_IINTERFACE;
 
@@ -88,6 +133,7 @@ public:
     bool init(const char* name, const char* type, IPropertyTree* cfg, const char* process);
 
     virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp);
+    virtual void getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID);
     virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp);
     virtual void filterLogContent(IEspUpdateLogRequestWrap* req);
 };

esp/logging/logginglib/CMakeLists.txt

@@ -0,0 +1,55 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+# Component: logginglib
+
+project( logginglib )
+
+include(${HPCC_SOURCE_DIR}/esp/scm/espscm.cmake)
+
+include_directories (
+    ${HPCC_SOURCE_DIR}/system/include
+    ${HPCC_SOURCE_DIR}/system/jlib
+    ${HPCC_SOURCE_DIR}/system/xmllib
+    ${HPCC_SOURCE_DIR}/system/security/shared       #seclib.hpp in generated/ws_loggingservice_esp.ipp
+    ${HPCC_SOURCE_DIR}/esp/platform                 #EspCoreErrors
+    ${HPCC_SOURCE_DIR}/esp/bindings                 #mime.hpp in generated/ws_loggingservice_esp.ipp
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp        #XmlPullParser.h in generated/ws_loggingservice_esp.ipp
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/Platform   #CSoapRequestBinding
+    ${HPCC_SOURCE_DIR}/esp/clients                  #edwin.h in generated/ws_loggingservice.esp
+)
+
+ADD_DEFINITIONS ( -D_USRDLL -DLOGGINGCOMMON_EXPORTS )
+
+set ( SRCS
+    ${ESPSCM_GENERATED_DIR}/ws_loggingservice_esp.cpp
+    ${HPCC_SOURCE_DIR}/esp/logging/logginglib/datafieldmap.cpp
+    ${HPCC_SOURCE_DIR}/esp/logging/logginglib/loggingagentbase.cpp
+    ${HPCC_SOURCE_DIR}/esp/logging/logginglib/LogSerializer.cpp
+    ${HPCC_SOURCE_DIR}/esp/logging/logginglib/LogFailSafe.cpp
+    ${HPCC_SOURCE_DIR}/esp/logging/logginglib/logthread.cpp
+)
+
+HPCC_ADD_LIBRARY( logginglib SHARED ${SRCS} )
+
+install ( TARGETS logginglib RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
+
+add_dependencies ( logginglib jlib espscm)
+
+target_link_libraries ( logginglib
+    jlib
+    xmllib
+    esphttp
+)

esp/logging/logginglib/datafieldmap.cpp

@@ -0,0 +1,101 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "datafieldmap.hpp"
+static const char* const defaultLogSourcePath = "Source";
+
+void ensureInputString(const char* input, bool lowerCase, StringBuffer& outputStr, int code, const char* msg)
+{
+    outputStr.set(input).trim();
+    if (outputStr.isEmpty())
+        throw MakeStringException(code, "%s", msg);
+    if (lowerCase)
+        outputStr.toLowerCase();
+}
+
+void readLogGroupCfg(IPropertyTree* cfg, StringAttr& defaultLogGroup, MapStringToMyClass<CLogGroup>& logGroups)
+{
+    StringBuffer groupName;
+    Owned<IPropertyTreeIterator> iter = cfg->getElements("LogGroup");
+    ForEach(*iter)
+    {
+        ensureInputString(iter->query().queryProp("@name"), true, groupName, -1, "LogGroup @name required");
+        Owned<CLogGroup> logGroup = new CLogGroup(groupName.str());
+        logGroup->loadMappings(iter->query());
+        logGroups.setValue(groupName.str(), logGroup);
+        if (defaultLogGroup.isEmpty())
+            defaultLogGroup.set(groupName.str());
+    }
+}
+
+void readLogSourceCfg(IPropertyTree* cfg, unsigned& logSourceCount, StringAttr& logSourcePath, MapStringToMyClass<CLogSource>& logSources)
+{
+    logSourceCount = 0;
+    StringBuffer name, groupName, dbName;
+    Owned<IPropertyTreeIterator> iter = cfg->getElements("LogSourceMap/LogSource");
+    ForEach(*iter)
+    {
+        ensureInputString(iter->query().queryProp("@name"), false, name, -1, "LogSource @name required");
+        ensureInputString(iter->query().queryProp("@maptologgroup"), true, groupName, -1, "LogSource @maptologgroup required");
+        ensureInputString(iter->query().queryProp("@maptodb"), true, dbName, -1, "LogSource @maptodb required");
+        Owned<CLogSource> logSource = new CLogSource(name.str(), groupName.str(), dbName.str());
+        logSources.setValue(name.str(), logSource);
+        logSourceCount++;
+    }
+
+    //xpath to read log source from log request
+    logSourcePath.set(cfg->hasProp("LogSourcePath") ? cfg->queryProp("LogSourcePath") : defaultLogSourcePath);
+}
+
+void CLogTable::loadMappings(IPropertyTree& fieldList)
+{
+    StringBuffer name, mapTo, fieldType, defaultValue;
+    Owned<IPropertyTreeIterator> itr = fieldList.getElements("Field");
+    ForEach(*itr)
+    {
+        IPropertyTree &map = itr->query();
+
+        ensureInputString(map.queryProp("@name"), false, name, -1, "Field @name required");
+        ensureInputString(map.queryProp("@mapto"), true, mapTo, -1, "Field @mapto required");
+        ensureInputString(map.queryProp("@type"), true, fieldType, -1, "Field @type required");
+        defaultValue = map.queryProp("@default");
+        defaultValue.trim();
+
+        Owned<CLogField> field = new CLogField(name.str(), mapTo.str(), fieldType.str());
+        if (!defaultValue.isEmpty())
+            field->setDefault(defaultValue.str());
+        logFields.append(*field.getClear());
+    }
+}
+
+void CLogGroup::loadMappings(IPropertyTree& fieldList)
+{
+    StringBuffer tableName;
+    Owned<IPropertyTreeIterator> itr = fieldList.getElements("Fieldmap");
+    ForEach(*itr)
+    {
+        ensureInputString(itr->query().queryProp("@table"), true, tableName, -1, "Fieldmap @table required");
+
+        Owned<CLogTable> table = new CLogTable(tableName.str());
+        table->loadMappings(itr->query());
+        CIArrayOf<CLogField>& logFields = table->getLogFields();
+        if (logFields.length() < 1)
+            throw MakeStringException(-1,"No Fieldmap for %s", tableName.str());
+
+        logTables.append(*table.getClear());
+    }
+}

esp/logging/loggingagent/cassandraloggingagent/dbfieldmap.hpp

@@ -15,14 +15,15 @@
     limitations under the License.
 ############################################################################## */
 
-#ifndef _DBFIELDMAP_HPP__
-#define _DBFIELDMAP_HPP__
+#ifndef _DATAFIELDMAP_HPP__
+#define _DATAFIELDMAP_HPP__
 
 #pragma warning (disable : 4786)
 
 #include "jiface.hpp"
 #include "jstring.hpp"
 #include "jptree.hpp"
+#include "loggingcommon.hpp"
 
 class CLogField : public CInterface
 {
@@ -85,4 +86,9 @@ public:
     const char* getGroupName() { return groupName.get(); };
     const char* getDBName() { return dbName.get(); };
 };
-#endif // !_DBFIELDMAP_HPP__
+
+extern LOGGINGCOMMON_API void ensureInputString(const char* input, bool lowerCase, StringBuffer& outputStr, int code, const char* msg);
+extern LOGGINGCOMMON_API void readLogGroupCfg(IPropertyTree* cfg, StringAttr& defaultLogGroup, MapStringToMyClass<CLogGroup>& logGroups);
+extern LOGGINGCOMMON_API void readLogSourceCfg(IPropertyTree* cfg, unsigned& logSourceCount, StringAttr& logSourcePath, MapStringToMyClass<CLogSource>& logGroups);
+
+#endif // !_DATAFIELDMAP_HPP__

esp/logging/logginglib/loggingagentbase.cpp

@@ -0,0 +1,276 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "LoggingErrors.hpp"
+#include "esploggingservice_esp.ipp"
+#include "loggingagentbase.hpp"
+
+static const char* const defaultTransactionTable = "transactions";
+static const char* const defaultTransactionAppName = "accounting_log";
+static const char* const defaultLoggingTransactionAppName = "logging_transaction";
+
+void CDBLogAgentBase::readDBCfg(IPropertyTree* cfg, StringBuffer& server, StringBuffer& dbUser, StringBuffer& dbPassword)
+{
+    ensureInputString(cfg->queryProp("@server"), true, server, -1, "Database server required");
+    ensureInputString(cfg->queryProp("@dbName"), true, defaultDB, -1, "Database name required");
+
+    transactionTable.set(cfg->hasProp("@dbTableName") ? cfg->queryProp("@dbTableName") : defaultTransactionTable);
+    dbUser.set(cfg->queryProp("@dbUser"));
+    const char* encodedPassword = cfg->queryProp("@dbPassWord");
+    if(encodedPassword && *encodedPassword)
+        decrypt(dbPassword, encodedPassword);
+}
+
+void CDBLogAgentBase::readTransactionCfg(IPropertyTree* cfg)
+{
+    //defaultTransactionApp: if no APP name is given, which APP name (TableName) should be used to get a transaction seed?
+    //loggingTransactionApp: the TableName used to get a transaction seed for this logging agent
+    defaultTransactionApp.set(cfg->hasProp("defaultTransaction") ? cfg->queryProp("defaultTransaction") : defaultTransactionAppName);
+    loggingTransactionApp.set(cfg->hasProp("loggingTransaction") ? cfg->queryProp("loggingTransaction") : defaultLoggingTransactionAppName);
+    loggingTransactionCount = 0;
+}
+
+bool CDBLogAgentBase::getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp)
+{
+    bool bRet = false;
+    StringBuffer appName = req.getApplication();
+    appName.trim();
+    if (appName.length() == 0)
+        appName = defaultTransactionApp.get();
+
+    unsigned retry = 1;
+    while (1)
+    {
+        try
+        {
+            StringBuffer logSeed;
+            queryTransactionSeed(appName, logSeed);
+            if (!logSeed.length())
+                throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to get TransactionSeed");
+
+            resp.setSeedId(logSeed.str());
+            resp.setStatusCode(0);
+            bRet = true;
+            break;
+        }
+        catch (IException* e)
+        {
+            StringBuffer errorStr, errorMessage;
+            errorMessage.append("Failed to get TransactionSeed: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
+            ERRLOG("%s -- try %d", errorMessage.str(), retry);
+            e->Release();
+            if (retry < maxTriesGTS)
+            {
+                Sleep(retry*3000);
+                retry++;
+            }
+            else
+            {
+                resp.setStatusCode(-1);
+                resp.setStatusMessage(errorMessage.str());
+                break;
+            }
+        }
+    }
+    return bRet;
+}
+
+bool CDBLogAgentBase::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp)
+{
+    bool ret = false;
+    try
+    {
+        const char* updateLogReq = req.getUpdateLogRequest();
+        if (!updateLogReq || !*updateLogReq)
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log request.");
+
+        StringBuffer requestBuf, logDB, logSource;
+        requestBuf.append("<LogRequest>").append(updateLogReq).append("</LogRequest>");
+        Owned<IPropertyTree> logRequestTree = createPTreeFromXMLString(requestBuf.length(), requestBuf.str());
+        if (!logRequestTree)
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log request.");
+
+        CLogGroup* logGroup = checkLogSource(logRequestTree, logSource, logDB);
+        if (!logGroup)
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Log Group %s undefined.", logSource.str());
+
+        StringBuffer logID;
+        getLoggingTransactionID(logID);
+
+        CIArrayOf<CLogTable>& logTables = logGroup->getLogTables();
+        ForEachItemIn(i, logTables)
+        {
+            CLogTable& table = logTables.item(i);
+
+            StringBuffer updateDBStatement;
+            if(!buildUpdateLogStatement(logRequestTree, logDB.str(), table, logID, updateDBStatement))
+                throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed in creating SQL statement.");
+
+            if (getEspLogLevel() >= LogMax)
+                DBGLOG("UpdateLog: %s\n", updateDBStatement.str());
+
+            executeUpdateLogStatement(updateDBStatement);
+        }
+        resp.setStatusCode(0);
+        ret = true;
+    }
+    catch (IException* e)
+    {
+        StringBuffer errorStr, errorMessage;
+        errorMessage.append("Failed to update log: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
+        ERRLOG("%s", errorMessage.str());
+        e->Release();
+        resp.setStatusCode(-1);
+        resp.setStatusMessage(errorMessage.str());
+    }
+    return ret;
+}
+
+CLogGroup* CDBLogAgentBase::checkLogSource(IPropertyTree* logRequest, StringBuffer& source, StringBuffer& logDB)
+{
+    if (logSourceCount == 0)
+    {//if no log source is configured, use default Log Group and DB
+        logDB.set(defaultDB.str());
+        source.set(defaultLogGroup.get());
+        return logGroups.getValue(defaultLogGroup.get());
+    }
+    source = logRequest->queryProp(logSourcePath.get());
+    if (source.isEmpty())
+        throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log Source from request.");
+    CLogSource* logSource = logSources.getValue(source.str());
+    if (!logSource)
+        throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Log Source %s undefined.", source.str());
+
+    logDB.set(logSource->getDBName());
+    return logGroups.getValue(logSource->getGroupName());
+}
+
+void CDBLogAgentBase::getLoggingTransactionID(StringBuffer& id)
+{
+    id.set(loggingTransactionSeed.str()).append("-").append(++loggingTransactionCount);
+}
+
+bool CDBLogAgentBase::buildUpdateLogStatement(IPropertyTree* logRequest, const char* logDB,
+    CLogTable& table, StringBuffer& logID, StringBuffer& updateDBStatement)
+{
+    StringBuffer fields, values;
+    BoolHash handledFields;
+    CIArrayOf<CLogField>& logFields = table.getLogFields();
+    ForEachItemIn(i, logFields) //Go through data items to be logged
+    {
+        CLogField& logField = logFields.item(i);
+
+        StringBuffer colName = logField.getMapTo();
+        bool* found = handledFields.getValue(colName.str());
+        if (found && *found)
+            continue;
+
+        StringBuffer path = logField.getName();
+        if (path.charAt(path.length() - 1) == ']')
+        {//Attr filter. Separate the last [] from the path.
+            const char* pTr = path.str();
+            const char* ppTr = strrchr(pTr, '[');
+            if (!ppTr)
+                continue;
+
+            StringBuffer attr;
+            attr.set(ppTr+1);
+            attr.setLength(attr.length() - 1);
+            path.setLength(ppTr - pTr);
+
+            StringBuffer colValue;
+            Owned<IPropertyTreeIterator> itr = logRequest->getElements(path.str());
+            ForEach(*itr)
+            {//Log the first valid match just in case more than one matches.
+                IPropertyTree& ppTree = itr->query();
+                colValue.set(ppTree.queryProp(attr.str()));
+                if (colValue.length())
+                {
+                    addField(logField, colName.str(), colValue, fields, values);
+                    handledFields.setValue(colName.str(), true);
+                    break;
+                }
+            }
+            continue;
+        }
+
+        Owned<IPropertyTreeIterator> itr = logRequest->getElements(path.str());
+        ForEach(*itr)
+        {
+            IPropertyTree& ppTree = itr->query();
+
+            StringBuffer colValue;
+            if (ppTree.hasChildren()) //This is a tree branch.
+                toXML(&ppTree, colValue);
+            else
+                ppTree.getProp(NULL, colValue);
+
+            if (colValue.length())
+            {
+                addField(logField, colName.str(), colValue, fields, values);
+                handledFields.setValue(colName.str(), true);
+                break;
+            }
+        }
+    }
+
+    //add any default fields that may be required but not in request.
+    addMissingFields(logFields, handledFields, fields, values);
+    appendFieldInfo("log_id", logID, fields, values, true);
+
+    setUpdateLogStatement(logDB, table.getTableName(), fields.str(), values.str(), updateDBStatement);
+    return true;
+}
+
+void CDBLogAgentBase::appendFieldInfo(const char* field, StringBuffer& value, StringBuffer& fields, StringBuffer& values, bool quoted)
+{
+    if(values.length() != 0)
+        values.append(',');
+    if (quoted)
+        values.append('\'').append(value.length(), value.str()).append('\'');
+    else
+        values.append(value.length(), value.str());
+
+    if(fields.length() != 0)
+        fields.append(',');
+    fields.append(field);
+}
+
+void CDBLogAgentBase::addMissingFields(CIArrayOf<CLogField>& logFields, BoolHash& handledFields, StringBuffer& fields, StringBuffer& values)
+{
+    ForEachItemIn(i, logFields) //Go through data items to be logged
+    {
+        CLogField& logField = logFields.item(i);
+        const char* colName = logField.getMapTo();
+        bool* found = handledFields.getValue(colName);
+        if (found && *found)
+            continue;
+        StringBuffer value = logField.getDefault();
+        if (!value.isEmpty())
+            addField(logField, colName, value, fields, values);
+    }
+}
+
+void CDBLogAgentBase::getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID)
+{
+    //Not implemented
+}
+
+void CDBLogAgentBase::filterLogContent(IEspUpdateLogRequestWrap* req)
+{
+    //No filter in CDBSQLLogAgent
+}

esp/logging/loggingagentbase.hpp

@@ -22,10 +22,17 @@
 
 #include "jiface.hpp"
 #include "esp.hpp"
+#include "datafieldmap.hpp"
 #include "ws_loggingservice_esp.ipp"
+#include "loggingcommon.hpp"
 
 #define UPDATELOGTHREADWAITINGTIME 3000
 
+
+static const char* sTransactionDateTime = "TransactionDateTime";
+static const char* sTransactionMethod = "TransactionMethod";
+static const char* sTransactionESPIP = "TransactionESPIP";
+
 interface IEspUpdateLogRequestWrap : extends IInterface
 {
     virtual const char* getGUID()=0;
@@ -112,8 +119,43 @@ interface IEspLogAgent : extends IInterface
 {
     virtual bool init(const char * name, const char * type, IPropertyTree * cfg, const char * process) = 0;
     virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp) = 0;
+    virtual void getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID) = 0;
     virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp) = 0;
     virtual void filterLogContent(IEspUpdateLogRequestWrap* req) = 0;
 };
 
+class LOGGINGCOMMON_API CDBLogAgentBase : public CInterface, implements IEspLogAgent
+{
+protected:
+    StringBuffer defaultDB, transactionTable, loggingTransactionSeed;
+    StringAttr defaultLogGroup, defaultTransactionApp, loggingTransactionApp, logSourcePath;
+
+    unsigned logSourceCount, loggingTransactionCount, maxTriesGTS;
+    MapStringToMyClass<CLogGroup> logGroups;
+    MapStringToMyClass<CLogSource> logSources;
+
+    void readDBCfg(IPropertyTree* cfg, StringBuffer& server, StringBuffer& dbUser, StringBuffer& dbPassword);
+    void readTransactionCfg(IPropertyTree* cfg);
+    bool buildUpdateLogStatement(IPropertyTree* logRequest, const char* logDB, CLogTable& table, StringBuffer& logID, StringBuffer& cqlStatement);
+    void appendFieldInfo(const char* field, StringBuffer& value, StringBuffer& fields, StringBuffer& values, bool quoted);
+    void addMissingFields(CIArrayOf<CLogField>& logFields, BoolHash& HandledFields, StringBuffer& fields, StringBuffer& values);
+    CLogGroup* checkLogSource(IPropertyTree* logRequest, StringBuffer& source, StringBuffer& logDB);
+    void getLoggingTransactionID(StringBuffer& id);
+
+    virtual void addField(CLogField& logField, const char* name, StringBuffer& value, StringBuffer& fields, StringBuffer& values) = 0;
+    virtual void queryTransactionSeed(const char* appName, StringBuffer& seed) = 0;
+    virtual void executeUpdateLogStatement(StringBuffer& statement) = 0;
+    virtual void setUpdateLogStatement(const char* dbName, const char* tableName,
+        const char* fields, const char* values, StringBuffer& statement) = 0;
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CDBLogAgentBase() {};
+    virtual ~CDBLogAgentBase() {};
+
+    virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp);
+    virtual void getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID);
+    virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp);
+    virtual void filterLogContent(IEspUpdateLogRequestWrap* req);
+};
 #endif  //_LOGGINGAGENT_HPP__

esp/logging/loggingcommon.hpp

@@ -36,7 +36,8 @@ enum LOGServiceType
 {
     LGSTterm = 0,
     LGSTGetTransactionSeed = 1,
-    LGSTUpdateLOG = 2
+    LGSTUpdateLOG = 2,
+    LGSTGetTransactionID = 3
 };
 
 #endif // !defined __LOGGINGCOMMONDEF_HPP__

esp/logging/logthread.cpp

@@ -73,6 +73,8 @@ CLogThread::CLogThread(IPropertyTree* _cfg , const char* _service, const char* _
             services[i++] = LGSTUpdateLOG;
         else if (service && strieq(service, "GetTransactionSeed"))
             services[i++] = LGSTGetTransactionSeed;
+        else if (service && strieq(service, "GetTransactionID"))
+            services[i++] = LGSTGetTransactionID;
     }
     services[i] = LGSTterm;
 

esp/logging/loggingmanager/CMakeLists.txt

@@ -29,9 +29,6 @@ include(${HPCC_SOURCE_DIR}/esp/scm/espscm.cmake)
 
 set ( SRCS
     ${ESPSCM_GENERATED_DIR}/ws_loggingservice_esp.cpp
-    ../LogSerializer.cpp
-    ../LogFailSafe.cpp
-    ../logthread.cpp
     loggingmanager.cpp
 )
 
@@ -50,7 +47,7 @@ include_directories (
     ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/LoggingService
     ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
 #    ./../../protocols/sidex
-    ${HPCC_SOURCE_DIR}/esp/logging
+    ${HPCC_SOURCE_DIR}/esp/logging/logginglib
     ./..
 )
 
@@ -64,11 +61,12 @@ HPCC_ADD_LIBRARY( loggingmanager SHARED ${SRCS} )
 
 add_dependencies( loggingmanager espscm )
 
-install ( TARGETS loggingmanager DESTINATION ${LIB_DIR} )
+install ( TARGETS loggingmanager RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
 
 target_link_libraries ( loggingmanager
     ${XALAN_LIBRARIES} ${XERCES_LIBRARIES}
     jlib
     xmllib
+    logginglib
     esphttp
 )

esp/logging/loggingmanager/loggingmanager.cpp

@@ -62,6 +62,7 @@ bool CLoggingManager::init(IPropertyTree* cfg, const char* service)
         loggingAgentThreads.push_back(logThread);
     }
 
+    initialized = true;
     return !loggingAgentThreads.empty();
 }
 
@@ -84,6 +85,9 @@ IEspLogAgent* CLoggingManager::loadLoggingAgent(const char* name, const char* dl
 
 bool CLoggingManager::updateLog(const char* option, const char* logContent, StringBuffer& status)
 {
+    if (!initialized)
+        throw MakeStringException(-1,"LoggingManager not initialized");
+
     bool bRet = false;
     try
     {
@@ -105,6 +109,9 @@ bool CLoggingManager::updateLog(const char* option, const char* logContent, Stri
 bool CLoggingManager::updateLog(const char* option, IEspContext& espContext, IPropertyTree* userContext, IPropertyTree* userRequest,
         const char* backEndResp, const char* userResp, StringBuffer& status)
 {
+    if (!initialized)
+        throw MakeStringException(-1,"LoggingManager not initialized");
+
     bool bRet = false;
     try
     {
@@ -150,6 +157,9 @@ bool CLoggingManager::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResp
 
 bool CLoggingManager::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp)
 {
+    if (!initialized)
+        throw MakeStringException(-1,"LoggingManager not initialized");
+
     bool bRet = false;
     try
     {
@@ -177,6 +187,9 @@ bool CLoggingManager::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResp
 
 bool CLoggingManager::getTransactionSeed(StringBuffer& transactionSeed, StringBuffer& status)
 {
+    if (!initialized)
+        throw MakeStringException(-1,"LoggingManager not initialized");
+
     bool bRet = false;
     try
     {
@@ -220,6 +233,9 @@ bool CLoggingManager::getTransactionSeed(StringBuffer& transactionSeed, StringBu
 
 bool CLoggingManager::getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp)
 {
+    if (!initialized)
+        throw MakeStringException(-1,"LoggingManager not initialized");
+
     bool bRet = false;
     try
     {
@@ -248,6 +264,33 @@ bool CLoggingManager::getTransactionSeed(IEspGetTransactionSeedRequest& req, IEs
     return bRet;
 }
 
+bool CLoggingManager::getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID, StringBuffer& status)
+{
+    if (!initialized)
+        throw MakeStringException(-1,"LoggingManager not initialized");
+
+    try
+    {
+        for (unsigned int x = 0; x < loggingAgentThreads.size(); x++)
+        {
+            IUpdateLogThread* loggingThread = loggingAgentThreads[x];
+            if (!loggingThread->hasService(LGSTGetTransactionID))
+                continue;
+
+            IEspLogAgent* loggingAgent = loggingThread->getLogAgent();
+            loggingAgent->getTransactionID(transFields, transactionID);
+            return true;
+        }
+    }
+    catch (IException* e)
+    {
+        e->errorMessage(status);
+        e->Release();
+    }
+
+    return false;
+}
+
 extern "C"
 {
 LOGGINGMANAGER_API ILoggingManager* newLoggingManager()

esp/logging/loggingmanager.h

@@ -34,6 +34,7 @@ interface ILoggingManager : implements IInterface
     virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp) = 0;
     virtual bool getTransactionSeed(StringBuffer& transactionSeed, StringBuffer& status) = 0;
     virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp) = 0;
+    virtual bool getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID, StringBuffer& status) = 0;
 };
 
 typedef ILoggingManager* (*newLoggingManager_t_)();

esp/logging/loggingmanager/loggingmanager.hpp

@@ -40,6 +40,7 @@ class CLoggingManager : implements ILoggingManager, public CInterface
 {
     typedef std::vector<IUpdateLogThread*> LOGGING_AGENTTHREADS;
     LOGGING_AGENTTHREADS  loggingAgentThreads;
+    bool initialized;
 
     IEspLogAgent* loadLoggingAgent(const char* name, const char* dll, const char* type, IPropertyTree* cfg);
     bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp, StringBuffer& status);
@@ -47,7 +48,7 @@ class CLoggingManager : implements ILoggingManager, public CInterface
 public:
     IMPLEMENT_IINTERFACE;
 
-    CLoggingManager(void) {};
+    CLoggingManager(void) { initialized = false; };
     virtual ~CLoggingManager(void);
 
     virtual bool init(IPropertyTree* cfg, const char* service);
@@ -58,6 +59,7 @@ public:
     virtual bool updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp);
     virtual bool getTransactionSeed(StringBuffer& transactionSeed, StringBuffer& status);
     virtual bool getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp);
+    virtual bool getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID, StringBuffer& status);
 };
 
 #endif // !defined(__LOGGINGMANAGER_HPP__)

esp/logging/test/CMakeLists.txt

@@ -48,7 +48,8 @@ include_directories (
          ${HPCC_SOURCE_DIR}/esp/clients
          ${HPCC_SOURCE_DIR}/esp/clients/espcommonclient
          ${HPCC_SOURCE_DIR}/esp/clients/LoggingClient
-         ${HPCC_SOURCE_DIR}/esp/logging
+         ${HPCC_SOURCE_DIR}/esp/logging/logginglib
+         ${HPCC_SOURCE_DIR}/esp/logging/loggingmanager
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )

esp/services/esdl_svc_engine/CMakeLists.txt

@@ -56,7 +56,8 @@ include_directories (
          ${HPCC_SOURCE_DIR}/rtl/eclrtl
          ${HPCC_SOURCE_DIR}/rtl/include
          ${HPCC_SOURCE_DIR}/esp/esdllib
-         ${HPCC_SOURCE_DIR}/esp/logging
+         ${HPCC_SOURCE_DIR}/esp/logging/logginglib
+         ${HPCC_SOURCE_DIR}/esp/logging/loggingmanager
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
     )

esp/services/ws_esdlconfig/CMakeLists.txt

@@ -53,7 +53,8 @@ include_directories (
          ${HPCC_SOURCE_DIR}/esp/smc/SMCLib
          ${HPCC_SOURCE_DIR}/esp/services/esdl_svc_engine
          ${HPCC_SOURCE_DIR}/esp/esdllib
-         ${HPCC_SOURCE_DIR}/esp/logging
+         ${HPCC_SOURCE_DIR}/esp/logging/logginglib
+         ${HPCC_SOURCE_DIR}/esp/logging/loggingmanager
     )
 
 ADD_DEFINITIONS( -D_USRDLL )

esp/services/ws_loggingservice/CMakeLists.txt

@@ -37,8 +37,7 @@ include_directories (
      ${HPCC_SOURCE_DIR}/esp/esplib
      ${HPCC_SOURCE_DIR}/esp/clients
      ${HPCC_SOURCE_DIR}/esp/clients/LoggingClient
-     ${HPCC_SOURCE_DIR}/esp/clients/loggingmanager
-     ${HPCC_SOURCE_DIR}/esp/logging
+     ${HPCC_SOURCE_DIR}/esp/logging/logginglib
      ./..
 )
 
@@ -46,9 +45,6 @@ ADD_DEFINITIONS( -D_USRDLL -DLOGGINGCOMMON_EXPORTS )
 
 set ( SRCS
     ${ESPSCM_GENERATED_DIR}/ws_loggingservice_esp.cpp
-    ../../logging/LogSerializer.cpp
-    ../../logging/LogFailSafe.cpp
-    ../../logging/logthread.cpp
     loggingservice.cpp
     loggingserviceplugin.cpp
 )
@@ -60,4 +56,5 @@ add_dependencies (ws_loggingservice espscm)
 target_link_libraries ( ws_loggingservice
     esphttp
     jlib
+    logginglib
 )