Merge pull request #10640 from afishbeck/wseclCaptureQueryLogging

HPCC-18565 Support roxie log capture via SOAP and WSECL

Reviewed-By: Gavin Halliday <gavin.halliday@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/wuwebview/wuwebview.cpp

@@ -75,6 +75,7 @@ public:
     void initResultChildTags()
     {
         resultChildTags.setValue("Dataset", true);
+        resultChildTags.setValue("Tracing", true);
         resultChildTags.setValue("Exception", true);
         resultChildTags.setValue("Warning", true);
         resultChildTags.setValue("Alert", true);

esp/services/common/jsonhelpers.hpp

@@ -322,6 +322,15 @@ namespace JsonHelpers
             }
             else
             {
+                if (flags & REQSF_ROOT)
+                {
+                    bool log = reqTree->getPropBool("@log", false); //not in schema
+                    if (log)
+                        appendJSONValue(out, "@log", true);
+                    int tracelevel = reqTree->getPropInt("@traceLevel", -1);
+                    if (tracelevel>=0)
+                        appendJSONValue(out, "@traceLevel", tracelevel);
+                }
                 int flds = type->getFieldCount();
                 for (int idx=0; idx<flds; idx++)
                 {

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -563,7 +563,16 @@ static void buildReqXml(StringArray& parentTypes, IXmlType* type, StringBuffer&
             else
                 attrval = attr->getSampleValue(s);
             if (attrval)
-                appendXMLAttr(out, attr->queryName(), attrval);
+                appendXMLAttr(out, attr->queryName(), attrval, nullptr, true);
+        }
+        if (flags & REQSF_ROOT)
+        {
+            bool log = reqTree->getPropBool("@log", false);
+            if (log)
+                appendXMLAttr(out, "log", "true", nullptr, true);
+            int tracelevel = reqTree->getPropInt("@traceLevel", -1);
+            if (tracelevel >= 0)
+                out.appendf(" traceLevel=\"%d\"", tracelevel);
         }
         out.append('>');
 
@@ -1469,6 +1478,10 @@ int CWsEclBinding::getGenForm(IEspContext &context, CHttpRequest* request, CHttp
     xform->setParameter("includeSoapTest", "1");
     xform->setParameter("useTextareaForStringArray", "1");
 
+    Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(wuinfo.qsetname);
+    bool isRoxie = clusterInfo && (clusterInfo->getPlatform() == RoxieCluster);
+    xform->setParameter("includeRoxieOptions", isRoxie ? "1" : "0");
+
     // set the prop noDefaultValue param
     IProperties* props = context.queryRequestParameters();
     bool formInitialized = false;

esp/xslt/wsecl3_form.xsl

@@ -19,6 +19,7 @@
     <xsl:param name="formOptionsAccess" select="1"/>
     <xsl:param name="noDefaultValue" select="0"/>
     <xsl:param name="includeSoapTest" select="1"/>
+    <xsl:param name="includeRoxieOptions" select="1"/>
     <xsl:param name="schemaRoot" select="/FormInfo/xsd:schema"/>
     <xsl:param name="esdl_links" select="0"/>
     <xsl:param name="useTextareaForStringArray" select="0"/>
@@ -302,7 +303,16 @@ function switchInputForm()
                      </span>
               </td>
             </tr>
-
+            <xsl:if test="$includeRoxieOptions=1">
+            <tr>
+               <td class='input' align='left'>
+                  <span>
+                     <hr/>
+                     <input type='checkbox' name='@log'/> Capture Log Info.&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Trace Level: <input type='text' name='@traceLevel' size='10'/><br/>
+                  </span>
+               </td>
+            </tr>
+            </xsl:if>
             <tr><td bgcolor="#030303" height="1"></td></tr>
                 <tr><td height="6"></td></tr>
 

roxie/ccd/ccd.hpp

@@ -521,33 +521,30 @@ public:
         }
     }
 
-    void toXML(StringBuffer &out)
-    {
-        out.append("<Log><Category>").append(getCategoryString(category)).append("</Category>");
-        out.append("<Channel>").append(channel).append("</Channel>");
-        out.append("<Time>").append(time/1000).append('.').appendf("%03d", time % 1000).append("</Time>");
-        if (prefix)
-        {
-            out.append("<Prefix>");
-            encodeXML(prefix, out);
-            out.append("</Prefix>");
-        }
-        if (text)
-        {
-            out.append("<Text>");
-            encodeXML(text, out);
-            out.append("</Text>");
-        }
-        out.append("</Log>\n");
-    }
-
     void outputXML(IXmlStreamFlusher &out)
     {
         StringBuffer b;
-        toXML(b);
+        Owned<IXmlWriterExt> writer = createIXmlWriterExt(0, 0, NULL, WTStandard);
+        writeXML(*writer);
+        b.append(writer->str()); //flush can detach pointer
         out.flushXML(b, true);
     }
 
+    void writeXML(IXmlWriter &writer)
+    {
+        writer.outputBeginNested("Log", true);
+        writer.outputCString(getCategoryString(category), "Category");
+        writer.outputUInt(channel, sizeof(channel), "Channel");
+        StringBuffer s;
+        s.append(time/1000).append('.').appendf("%03d", time % 1000);
+        writer.outputCString(s, "Time");
+        if (prefix)
+            writer.outputCString(prefix, "Prefix");
+        if (text)
+            writer.outputCString(text, "Text");
+        writer.outputEndNested("Log");
+    }
+
 };
 
 // MORE - this code probably should be commoned up with some of the new stats code
@@ -592,6 +589,17 @@ public:
         }
     };
 
+    void writeXML(IXmlWriter &writer)
+    {
+        CriticalBlock b(crit);
+        writer.outputBeginArray("Log");
+        ForEachItemIn(idx, log)
+        {
+            log.item(idx).writeXML(writer);
+        }
+        writer.outputEndArray("Log");
+    };
+
     virtual void CTXLOGa(TracingCategory category, const char *prefix, const char *text) const
     {
         if (category == LOG_TRACING)

roxie/ccd/ccdlistener.cpp

@@ -1467,6 +1467,15 @@ public:
         if (logctx)
             logctx->outputXML(out);
     }
+    virtual void writeLogXML(IXmlWriter &writer)
+    {
+        if (logctx)
+        {
+            writer.outputBeginNested("Tracing", true);
+            logctx->writeXML(writer);
+            writer.outputEndNested("Tracing");
+        }
+    }
 
     virtual unsigned getQueryPriority()
     {
@@ -1654,6 +1663,12 @@ public:
         if (!(flags & HPCC_PROTOCOL_NATIVE))
         {
             ctx->process();
+            if (msgctx->getIntercept())
+            {
+                Owned<IXmlWriter> logwriter = protocol->writeAppendContent(nullptr);
+                msgctx->writeLogXML(*logwriter);
+            }
+
             protocol->finalize(idx);
             memused += ctx->getMemoryUsage();
             slavesReplyLen += ctx->getSlavesReplyLen();

roxie/ccd/ccdprotocol.cpp

@@ -831,9 +831,8 @@ public:
                 result->flush(true);
         }
     }
-    virtual void finalize(unsigned seqNo, const char *delim, const char *filter)
+    virtual void finalize(unsigned seqNo, const char *delim, const char *filter, bool *needDelimiter)
     {
-        bool needDelimiter = false;
         ForEachItemIn(seq, resultMap)
         {
             FlushingStringBuffer *result = resultMap.item(seq);
@@ -846,17 +845,17 @@ public:
                     void *payload = result->getPayload(length);
                     if (!length)
                         break;
-                    if (needDelimiter)
+                    if (needDelimiter && *needDelimiter)
                     {
                         StringAttr s(delim); //write() will take ownership of buffer
                         size32_t len = s.length();
                         client->write((void *)s.detach(), len, true);
-                        needDelimiter=false;
+                        *needDelimiter=false;
                     }
                     client->write(payload, length, true);
                 }
-                if (delim)
-                    needDelimiter=true;
+                if (delim && needDelimiter)
+                    *needDelimiter=true;
             }
         }
     }
@@ -1126,6 +1125,8 @@ public:
 
 class CHpccJsonResponse : public CHpccNativeProtocolResponse
 {
+private:
+    bool needDelimiter = false;
 public:
     CHpccJsonResponse(const char *queryname, SafeSocket *_client, unsigned flags, bool _isHttp, const IContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags, const char *_resultFilter, const char *_rootTag) :
         CHpccNativeProtocolResponse(queryname, _client, MarkupFmt_JSON, flags, _isHttp, _logctx, _xmlReadFlags, _resultFilter, _rootTag)
@@ -1172,7 +1173,6 @@ public:
     }
     void outputContent()
     {
-        bool needDelimiter = false;
         ForEachItemIn(seq, contentsMap)
         {
             FlushingStringBuffer *content = contentsMap.item(seq);
@@ -1222,10 +1222,10 @@ public:
             unsigned len = responseHead.length();
             client->write(responseHead.detach(), len, true);
         }
+        if (results)
+            results->finalize(seqNo, ",", resultFilter.ordinality() ? resultFilter.item(0) : NULL, &needDelimiter);
         if (!resultFilter.ordinality())
             outputContent();
-        if (results)
-            results->finalize(seqNo, ",", resultFilter.ordinality() ? resultFilter.item(0) : NULL);
         if (!resultFilter.ordinality() && !(protocolFlags & HPCC_PROTOCOL_CONTROL))
         {
             responseTail.append("}}");
@@ -1288,7 +1288,6 @@ public:
     }
     void outputContent()
     {
-        bool needDelimiter = false;
         ForEachItemIn(seq, contentsMap)
         {
             FlushingStringBuffer *content = contentsMap.item(seq);
@@ -1328,10 +1327,10 @@ public:
             client->write(responseHead.detach(), len, true);
         }
 
+        if (results)
+            results->finalize(seqNo, NULL, resultFilter.ordinality() ? resultFilter.item(0) : NULL, nullptr);
         if (!resultFilter.ordinality())
             outputContent();
-        if (results)
-            results->finalize(seqNo, NULL, resultFilter.ordinality() ? resultFilter.item(0) : NULL);
 
         if (!resultFilter.ordinality() && !(protocolFlags & HPCC_PROTOCOL_CONTROL))
         {
@@ -1935,6 +1934,9 @@ readAnother:
                                 fixedreq->addPropTree(iter->query().queryName(), LINK(&iter->query()));
                             }
                             requestArray.append(*fixedreq);
+
+                            msgctx->setIntercept(queryPT->getPropBool("@log", false));
+                            msgctx->setTraceLevel(queryPT->getPropInt("@traceLevel", logctx.queryTraceLevel()));
                         }
                         if (httpHelper.getTrim())
                             protocolFlags |= HPCC_PROTOCOL_TRIM;

roxie/ccd/hpccprotocol.hpp

@@ -41,6 +41,7 @@ interface IHpccProtocolMsgContext : extends IInterface
     virtual bool getIntercept() = 0;
     virtual void outputLogXML(IXmlStreamFlusher &out) = 0;
     virtual void setTransactionId(const char *id) = 0;
+    virtual void writeLogXML(IXmlWriter &writer) = 0;
 };
 
 interface IHpccProtocolResultsWriter : extends IInterface

system/jlib/jptree.cpp

@@ -7312,6 +7312,13 @@ static void ensureHttpParameter(IPropertyTree *pt, StringBuffer &tag, const char
         idx = (unsigned) atoi(pos.str())+1;
     }
 
+    if ('@'==*tag)
+    {
+        if (path && *path)
+            throw MakeStringException(-1, "'@' not allowed in parent node of parameter path: %s", fullpath);
+        pt->setProp(tag, value);
+        return;
+    }
     if (tag.charAt(tag.length()-1)=='$')
     {
         if (path && *path)

system/jlib/jstring.hpp

@@ -417,14 +417,15 @@ inline StringBuffer &appendXMLTagName(StringBuffer &xml, const char *tag, const
 
 extern jlib_decl StringBuffer & appendXMLOpenTag(StringBuffer &xml, const char *tag, const char *prefix=NULL, bool complete=true, bool close=false, const char *uri=NULL);
 
-inline StringBuffer &appendXMLAttr(StringBuffer &xml, const char *name, const char *value, const char *prefix=NULL)
+inline StringBuffer &appendXMLAttr(StringBuffer &xml, const char *name, const char *value, const char *prefix=NULL, bool useDblQuote=false)
 {
     if (!name || !*name || !value)
         return xml;
     xml.append(' ');
     appendXMLTagName(xml, name, prefix);
-    encodeXML(value, xml.append("='"));
-    xml.append("'");
+    xml.append("=").append(useDblQuote ? '"' : '\'');
+    encodeXML(value, xml);
+    xml.append(useDblQuote ? '"' : '\'');
     return xml;
 }