HPCC-24666 Improve filterLogContent in ESDL logging

1. In the existing filterLogContent(), an extra tree layer is
added to the log request even if the log request is already
in a tree structure. That is fixed now. The related code is
moved in the front of the filterLogContent().

2. For the log request not in one tree structure (the log
request with ESPContext, UserContext, etc), only change in
this fix is to move the ScriptValues from the LogContent
branch to the UpdateLogRequest branch.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

esp/logging/logginglib/loggingagentbase.cpp

@@ -154,174 +154,180 @@ void CLogContentFilter::filterLogContentTree(StringArray& filters, IPropertyTree
 
 IEspUpdateLogRequestWrap* CLogContentFilter::filterLogContent(IEspUpdateLogRequestWrap* req)
 {
-    Owned<IPropertyTree> scriptValues = req->getScriptValuesTree();
-    const char* logContent = req->getUpdateLogRequest();
     Owned<IPropertyTree> logRequestTree = req->getLogRequestTree();
-    Owned<IPropertyTree> updateLogRequestTree = createPTree("UpdateLogRequest");
+    const char* logContent = req->getUpdateLogRequest();
+    if (logRequestTree || !isEmptyString(logContent))
+    {
+        if (groupFilters.length() < 1)
+        {//No filter
+            if (!logRequestTree)
+                return LINK(req);
+
+            //The first version of the filterLogContent() always returns the UpdateLogRequest string.
+            StringBuffer updateLogRequestXML;
+            toXML(logRequestTree, updateLogRequestXML);
+            req->setUpdateLogRequest(updateLogRequestXML);
+            return LINK(req);
+        }
 
-    StringBuffer source;
-    if (logBackEndReq && logBackEndResp && groupFilters.length() < 1)
-    {//No filter
+        if (!logRequestTree)
+            logRequestTree.setown(createPTreeFromXMLString(logContent));
+
+        Owned<IPropertyTree> filteredLogRequestTree = createPTree(logRequestTree->queryName());
+        bool logContentEmpty = true;
+        filterLogContentTree(groupFilters.item(0).getFilters(), logRequestTree, filteredLogRequestTree, logContentEmpty);
+        if (logContentEmpty)
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "The filtered content is empty.");
+
+        StringBuffer updateLogRequestXML;
+        toXML(filteredLogRequestTree, updateLogRequestXML);
+        ESPLOG(LogMax, "filtered content and option: <%s>", updateLogRequestXML.str());
+
+        req->setUpdateLogRequest(updateLogRequestXML);
         if (logRequestTree)
+            req->setLogRequestTree(filteredLogRequestTree.getClear());
+        return LINK(req);
+    }
+
+    Owned<IPropertyTree> updateLogRequestTree = createPTree("UpdateLogRequest");
+    IPropertyTree* logContentTree = ensurePTree(updateLogRequestTree, "LogContent");
+    StringBuffer source;
+    if (logBackEndReq && logBackEndResp && (groupFilters.length() < 1))
+    {//No filter. Add all the log items to the LogContent.
+        Owned<IPropertyTree> espContext = req->getESPContext();
+        Owned<IPropertyTree> userContext = req->getUserContext();
+        Owned<IPropertyTree> userRequest = req->getUserRequest();
+        const char* userResp = req->getUserResponse();
+        const char* logDatasets = req->getLogDatasets();
+        const char* backEndReq = req->getBackEndRequest();
+        const char* backEndResp = req->getBackEndResponse();
+        if (!espContext && !userContext && !userRequest && (!userResp || !*userResp) && (!backEndResp || !*backEndResp))
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log content");
+        source = userContext->queryProp("Source");
+
+        StringBuffer espContextXML, userContextXML, userRequestXML;
+        if (espContext)
         {
-            updateLogRequestTree->addPropTree(logRequestTree->queryName(), LINK(logRequestTree));
+            logContentTree->addPropTree(espContext->queryName(), LINK(espContext));
         }
-        else if (logContent && *logContent)
+        if (userContext)
         {
-            Owned<IPropertyTree> pTree = createPTreeFromXMLString(logContent);
-            source = pTree->queryProp("Source");
-            updateLogRequestTree->addPropTree(pTree->queryName(), LINK(pTree));
+            IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGUserContext]);
+            pTree->addPropTree(userContext->queryName(), LINK(userContext));
         }
-        else
+        if (userRequest)
         {
-            Owned<IPropertyTree> espContext = req->getESPContext();
-            Owned<IPropertyTree> userContext = req->getUserContext();
-            Owned<IPropertyTree> userRequest = req->getUserRequest();
-            const char* userResp = req->getUserResponse();
-            const char* logDatasets = req->getLogDatasets();
-            const char* backEndReq = req->getBackEndRequest();
-            const char* backEndResp = req->getBackEndResponse();
-            if (!espContext && !userContext && !userRequest && (!userResp || !*userResp) && (!backEndResp || !*backEndResp))
-                throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log content");
-            source = userContext->queryProp("Source");
-
-            StringBuffer espContextXML, userContextXML, userRequestXML;
-            IPropertyTree* logContentTree = ensurePTree(updateLogRequestTree, "LogContent");
-            if (scriptValues)
-            {
-                logContentTree->addPropTree(scriptValues->queryName(), LINK(scriptValues));
-            }
-            if (espContext)
-            {
-                logContentTree->addPropTree(espContext->queryName(), LINK(espContext));
-            }
-            if (userContext)
-            {
-                IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGUserContext]);
-                pTree->addPropTree(userContext->queryName(), LINK(userContext));
-            }
-            if (userRequest)
-            {
-                IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGUserReq]);
-                pTree->addPropTree(userRequest->queryName(), LINK(userRequest));
-            }
-            if (!isEmptyString(userResp))
-            {
-                IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGUserResp]);
-                Owned<IPropertyTree> userRespTree = createPTreeFromXMLString(userResp);
-                pTree->addPropTree(userRespTree->queryName(), LINK(userRespTree));
-            }
-            if (!isEmptyString(logDatasets))
-            {
-                IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGLogDatasets]);
-                Owned<IPropertyTree> logDatasetTree = createPTreeFromXMLString(logDatasets);
-                pTree->addPropTree(logDatasetTree->queryName(), LINK(logDatasetTree));
-            }
-            if (!isEmptyString(backEndReq))
-                logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndReq], backEndReq);
-            if (!isEmptyString(backEndResp))
-                logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndResp], backEndResp);
+            IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGUserReq]);
+            pTree->addPropTree(userRequest->queryName(), LINK(userRequest));
         }
-    }
-    else
-    {
-        bool logContentEmpty = true;
-        IPropertyTree* logContentTree = ensurePTree(updateLogRequestTree, "LogContent");
-        if (logRequestTree)
+        if (!isEmptyString(userResp))
         {
-            filterLogContentTree(groupFilters.item(0).getFilters(), logRequestTree, logContentTree, logContentEmpty);
+            IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGUserResp]);
+            Owned<IPropertyTree> userRespTree = createPTreeFromXMLString(userResp);
+            pTree->addPropTree(userRespTree->queryName(), LINK(userRespTree));
         }
-        else if (logContent && *logContent)
+        if (!isEmptyString(logDatasets))
         {
-            Owned<IPropertyTree> originalContentTree = createPTreeFromXMLString(logContent);
-            source = originalContentTree->queryProp("Source");
-            filterLogContentTree(groupFilters.item(0).getFilters(), originalContentTree, logContentTree, logContentEmpty);
+            IPropertyTree* pTree = ensurePTree(logContentTree, espLogContentGroupNames[ESPLCGLogDatasets]);
+            Owned<IPropertyTree> logDatasetTree = createPTreeFromXMLString(logDatasets);
+            pTree->addPropTree(logDatasetTree->queryName(), LINK(logDatasetTree));
         }
-        else
+        if (!isEmptyString(backEndReq))
+            logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndReq], backEndReq);
+        if (!isEmptyString(backEndResp))
+            logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndResp], backEndResp);
+    }
+    else
+    {
+        bool logContentEmpty = true;
+
+        //Both ESPLCGBackEndReq and ESPLCGBackEndResp are handled after this loop.
+        for (unsigned group = 0; group < ESPLCGBackEndReq; group++)
         {
-            //Both ESPLCGBackEndReq and ESPLCGBackEndResp are handled after this loop.
-            for (unsigned group = 0; group < ESPLCGBackEndReq; group++)
+            Owned<IPropertyTree> originalContentTree;
+            if (group == ESPLCGESPContext)
+                originalContentTree.setown(req->getESPContext());
+            else if (group == ESPLCGUserContext)
             {
-                Owned<IPropertyTree> originalContentTree;
-                if (group == ESPLCGESPContext)
-                    originalContentTree.setown(req->getESPContext());
-                else if (group == ESPLCGUserContext)
-                {
-                    originalContentTree.setown(req->getUserContext());
-                    source = originalContentTree->queryProp("Source");
-                }
-                else if (group == ESPLCGUserReq)
-                    originalContentTree.setown(req->getUserRequest());
-                else if (group == ESPLCGLogDatasets)
-                {
-                    const char* logDatasets = req->getLogDatasets();
-                    if (logDatasets && *logDatasets)
-                        originalContentTree.setown(createPTreeFromXMLString(logDatasets));
-                }
-                else //group = ESPLCGUserResp
-                {
-                    const char* resp = req->getUserResponse();
-                    if (!resp || !*resp)
-                        continue;
-                    originalContentTree.setown(createPTreeFromXMLString(resp));
-                }
-                if (!originalContentTree)
+                originalContentTree.setown(req->getUserContext());
+                source = originalContentTree->queryProp("Source");
+            }
+            else if (group == ESPLCGUserReq)
+                originalContentTree.setown(req->getUserRequest());
+            else if (group == ESPLCGLogDatasets)
+            {
+                const char* logDatasets = req->getLogDatasets();
+                if (logDatasets && *logDatasets)
+                    originalContentTree.setown(createPTreeFromXMLString(logDatasets));
+            }
+            else //group = ESPLCGUserResp
+            {
+                const char* resp = req->getUserResponse();
+                if (!resp || !*resp)
                     continue;
+                originalContentTree.setown(createPTreeFromXMLString(resp));
+            }
+            if (!originalContentTree)
+                continue;
 
-                bool foundGroupFilters  = false;
-                ForEachItemIn(i, groupFilters)
+            bool foundGroupFilters  = false;
+            ForEachItemIn(i, groupFilters)
+            {
+                CESPLogContentGroupFilters& filtersGroup = groupFilters.item(i);
+                if (filtersGroup.getGroup() == group)
                 {
-                    CESPLogContentGroupFilters& filtersGroup = groupFilters.item(i);
-                    if (filtersGroup.getGroup() == group)
-                    {
-                        IPropertyTree* newContentTree = ensurePTree(logContentTree, espLogContentGroupNames[group]);
-                        if (group != ESPLCGESPContext)//For non ESPLCGESPContext, we want to keep the root of original tree.
-                            newContentTree = ensurePTree(newContentTree, originalContentTree->queryName());
-                        filterLogContentTree(filtersGroup.getFilters(), originalContentTree, newContentTree, logContentEmpty);
-                        foundGroupFilters  =  true;
-                        break;
-                    }
+                    IPropertyTree* newContentTree = ensurePTree(logContentTree, espLogContentGroupNames[group]);
+                    if (group != ESPLCGESPContext)//For non ESPLCGESPContext, we want to keep the root of original tree.
+                        newContentTree = ensurePTree(newContentTree, originalContentTree->queryName());
+                    filterLogContentTree(filtersGroup.getFilters(), originalContentTree, newContentTree, logContentEmpty);
+                    foundGroupFilters  =  true;
+                    break;
                 }
+            }
 
-                if (!foundGroupFilters )
+            if (!foundGroupFilters )
+            {
+                if (group == ESPLCGESPContext)
                 {
-                    if (group == ESPLCGESPContext)
-                    {
-                        //The ESPContext tree itself already has the /ESPContext node
-                        //as the top tree node. We should not add another /ESPContext
-                        //node on the top of the ESPContext tree.
-                        logContentTree->addPropTree(originalContentTree->queryName(), LINK(originalContentTree));
-                    }
-                    else
-                    {
-                        IPropertyTree* newContentTree = ensurePTree(logContentTree, espLogContentGroupNames[group]);
-                        newContentTree->addPropTree(originalContentTree->queryName(), LINK(originalContentTree));
-                    }
-                    logContentEmpty = false;
+                    //The ESPContext tree itself already has the /ESPContext node
+                    //as the top tree node. We should not add another /ESPContext
+                    //node on the top of the ESPContext tree.
+                    logContentTree->addPropTree(originalContentTree->queryName(), LINK(originalContentTree));
                 }
-            }
-            if (logBackEndReq)
-            {
-                const char* request = req->getBackEndRequest();
-                if (!isEmptyString(request))
+                else
                 {
-                    logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndReq], request);
-                    logContentEmpty = false;
+                    IPropertyTree* newContentTree = ensurePTree(logContentTree, espLogContentGroupNames[group]);
+                    newContentTree->addPropTree(originalContentTree->queryName(), LINK(originalContentTree));
                 }
+                logContentEmpty = false;
             }
-            if (logBackEndResp)
+        }
+        if (logBackEndReq)
+        {
+            const char* request = req->getBackEndRequest();
+            if (!isEmptyString(request))
             {
-                const char* resp = req->getBackEndResponse();
-                if (resp && *resp)
-                {
-                    logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndResp], resp);
-                    logContentEmpty = false;
-                }
+                logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndReq], request);
+                logContentEmpty = false;
+            }
+        }
+        if (logBackEndResp)
+        {
+            const char* resp = req->getBackEndResponse();
+            if (resp && *resp)
+            {
+                logContentTree->addProp(espLogContentGroupNames[ESPLCGBackEndResp], resp);
+                logContentEmpty = false;
             }
         }
         if (logContentEmpty)
-            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to read log content");
+            throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "The filtered content is empty.");
     }
+
+    Owned<IPropertyTree> scriptValues = req->getScriptValuesTree();
+    if (scriptValues)
+        updateLogRequestTree->addPropTree(scriptValues->queryName(), LINK(scriptValues));
+
     if (!source.isEmpty())
         updateLogRequestTree->addProp("LogContent/Source", source.str());