Merge pull request #1891 from ghalliday/soaplog

Implement LOG(MIN) and LOG(expr) for SOAPCALL

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorsoapcall.cpp

@@ -715,7 +715,8 @@ public:
 
         rowProvider = _rowProvider;
         helper = rowProvider->queryActionHelper();
-        callHelper = rowProvider->queryCallHelper();
+        callHelper = rowProvider->queryCallHelper();  //MORE: This should not be done this way!! Should use extra as below.
+        helperExtra = static_cast<IHThorSoapCallExtra*>(helper->selectInterface(TAIsoapcallextra_1));
         flags = helper->getFlags();
 
         authToken.append(_authToken);
@@ -725,7 +726,10 @@ public:
         else
             maxRetries = helper->numRetries();
 
+        //Allow all of these options to be specified separately.  Possibly useful, and the code is cleaner.
+        logMin = (flags & SOAPFlogmin) != 0;
         logXML = (flags & SOAPFlog) != 0;
+        logUserMsg = (flags & SOAPFlogusermsg) != 0;
 
         timeout = helper->getTimeout();
         if (timeout == (unsigned)-1)
@@ -975,7 +979,19 @@ public:
         return false;
     }
 
+    void addUserLogMsg(const byte * row)
+    {
+        if (logUserMsg)
+        {
+            size32_t lenText;
+            rtlDataAttr text;
+            helperExtra->getLogText(lenText, text.refstr(), row);
+            logctx.CTXLOG("%s: %.*s", wscCallTypeText(), lenText, text.getstr());
+        }
+    }
+
     inline IXmlToRowTransformer * getRowTransformer() { return rowTransformer; }
+    inline const char * wscCallTypeText() const { return wscType == STsoap ? "SOAPCALL" : "HTTPCALL"; }
 
 protected:
     friend class CWSCHelperThread;
@@ -1019,6 +1035,7 @@ protected:
     IWSCRowProvider * rowProvider;
     IHThorWebServiceCallActionArg * helper;
     IHThorWebServiceCallArg *   callHelper;
+    IHThorWebServiceCallExtra * helperExtra;
     Linked<IEngineRowAllocator> outputAllocator;
     Owned<IException> error;
     UrlArray urlArray;
@@ -1030,7 +1047,10 @@ protected:
     unsigned maxRetries;
     unsigned timeout; //seconds
     unsigned timeLimit; //seconds
-    bool logXML, aborted;
+    bool logXML;
+    bool logMin;
+    bool logUserMsg;
+    bool aborted;
     const IContextLogger &logctx;
     unsigned flags;
     StringAttr soapaction;
@@ -1085,6 +1105,8 @@ void CWSCHelperThread::outputXmlRows(CommonXmlWriter &xmlWriter, ConstPointerArr
             xmlWriter.outputQuoted(itemtag);
             xmlWriter.outputQuoted(">");
         }
+
+        master->addUserLogMsg((const byte *)inputRows.item(idx));
     }
 }
 
@@ -1393,7 +1415,10 @@ private:
             request.append("\r\n");//httpcall
 
         if (soapTraceLevel > 6 || master->logXML)
-            master->logctx.CTXLOG("%sCALL: request(%s)", master->wscType == STsoap ? "SOAP" : "HTTP", request.str());
+            master->logctx.CTXLOG("%s: request(%s)", master->wscCallTypeText(), request.str());
+
+        if (master->logMin)
+            master->logctx.CTXLOG("%s: request(%s:%u)", master->wscCallTypeText(), url.host.str(), url.port);
     }
 
     int readHttpResponse(StringBuffer &response, ISocket *socket)

ecl/hql/hqlfold.cpp

@@ -5308,6 +5308,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_serialize:
     case no_typetransfer:
     case no_fromxml:
+    case no_httpcall:
         break;
 
     case no_null:

ecl/hql/hqlgram.y

@@ -3173,6 +3173,15 @@ soapFlag
                             $$.setExpr(createExprAttribute(responseAtom, createAttribute(noTrimAtom)), $1);
                         }
     | commonAttribute
+    | TOK_LOG '(' MIN ')'
+                        {
+                            $$.setExpr(createExprAttribute(logAtom, createAttribute(minAtom)), $1);
+                        }
+    | TOK_LOG '(' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_string, false);
+                            $$.setExpr(createExprAttribute(logAtom, $3.getExpr()), $1);
+                        }
     ;
 
 onFailAction

ecl/hqlcpp/hqlhtcpp.cpp

@@ -16127,6 +16127,24 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
 
     IHqlExpression * namespaceAttr = expr->queryProperty(namespaceAtom);
     IHqlExpression * responseAttr = expr->queryProperty(responseAtom);
+    IHqlExpression * logText = NULL;
+    bool logMin = false;
+    bool logXml = false;
+    ForEachChildFrom(i, expr, 1)
+    {
+        IHqlExpression * cur = expr->queryChild(i);
+        if (cur->isAttribute() && cur->queryName()==logAtom)
+        {
+            IHqlExpression * opt = cur->queryChild(0);
+            if (!opt)
+                logXml = true;
+            else if (!opt->isAttribute())
+                logText = opt;
+            else if (opt->queryName() == minAtom)
+                logMin = true;
+        }
+    }
+
     //virtual unsigned getFlags()
     {
         StringBuffer flags;
@@ -16134,7 +16152,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
             flags.append("|SOAPFgroup");
         if (expr->hasProperty(onFailAtom))
             flags.append("|SOAPFonfail");
-        if (expr->hasProperty(logAtom))
+        if (logXml)
             flags.append("|SOAPFlog");
         if (expr->hasProperty(trimAtom))
             flags.append("|SOAPFtrim");
@@ -16146,6 +16164,10 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
             flags.append("|SOAPFencoding");
         if (responseAttr && responseAttr->hasProperty(noTrimAtom))
             flags.append("|SOAPFpreserveSpace");
+        if (logMin)
+            flags.append("|SOAPFlogmin");
+        if (logText)
+            flags.append("|SOAPFlogusermsg");
 
         if (flags.length())
             doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
@@ -16173,6 +16195,18 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
             doBuildVarStringFunction(instance->startctx, "queryNamespaceVar", namespaceAttr->queryChild(1));
     }
 
+    if (logText)
+    {
+        BuildCtx funcctx(instance->startctx);
+        funcctx.addQuotedCompound("virtual void getLogText(size32_t & __lenResult, char * & __result, const void * _left)");
+        if (dataset)
+        {
+            funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
+            bindTableCursor(funcctx, dataset, "left");
+            bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
+        }
+        doBuildFunctionReturn(funcctx, unknownStringType, logText);
+    }
     if (!isSink)
     {
         //virtual IXmlToRowTransformer * queryTransformer()
@@ -16249,6 +16283,24 @@ ABoundActivity * HqlCppTranslator::doBuildActivityHTTP(BuildCtx & ctx, IHqlExpre
         doBuildVarStringFunction(instance->startctx, "queryOutputIteratorPath", separator->queryChild(0));
 
     IHqlExpression * namespaceAttr = expr->queryProperty(namespaceAtom);
+    IHqlExpression * logText = NULL;
+    bool logMin = false;
+    bool logXml = false;
+    ForEachChildFrom(i, expr, 1)
+    {
+        IHqlExpression * cur = expr->queryChild(i);
+        if (cur->isAttribute() && cur->queryName()==logAtom)
+        {
+            IHqlExpression * opt = cur->queryChild(0);
+            if (!opt)
+                logXml = true;
+            else if (!opt->isAttribute())
+                logText = opt;
+            else if (opt->queryName() == minAtom)
+                logMin = true;
+        }
+    }
+
     //virtual unsigned getFlags()
     {
         StringBuffer flags;
@@ -16256,7 +16308,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityHTTP(BuildCtx & ctx, IHqlExpre
             flags.append("|SOAPFgroup");
         if (expr->hasProperty(onFailAtom))
             flags.append("|SOAPFonfail");
-        if (expr->hasProperty(logAtom))
+        if (logXml)
             flags.append("|SOAPFlog");
         if (expr->hasProperty(trimAtom))
             flags.append("|SOAPFtrim");
@@ -16264,6 +16316,10 @@ ABoundActivity * HqlCppTranslator::doBuildActivityHTTP(BuildCtx & ctx, IHqlExpre
             flags.append("|SOAPFliteral");
         if (namespaceAttr)
             flags.append("|SOAPFnamespace");
+        if (logMin)
+            flags.append("|SOAPFlogmin");
+        if (logText)
+            flags.append("|SOAPFlogusermsg");
 
         if (flags.length())
             doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
@@ -16291,6 +16347,13 @@ ABoundActivity * HqlCppTranslator::doBuildActivityHTTP(BuildCtx & ctx, IHqlExpre
             doBuildVarStringFunction(instance->startctx, "queryNamespaceVar", namespaceAttr->queryChild(1));
     }
 
+    if (logText)
+    {
+        BuildCtx funcctx(instance->startctx);
+        funcctx.addQuotedCompound("virtual void getLogText(size32_t & __lenResult, char * & __result, const void * _left)");
+        doBuildFunctionReturn(funcctx, unknownStringType, logText);
+    }
+
     if (!isSink)
     {
         //virtual IXmlToRowTransformer * queryTransformer()

ecl/regress/soapcall12.ecl

@@ -0,0 +1,73 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+//Test all the different variants of SOAPCALL to ensure they get processed correctly
+
+GetAttributeInRecord := RECORD
+string                ModuleName{xpath('ModuleName')};
+string                AttributeName{xpath('AttributeName')};
+unsigned              Version{xpath('Version')};
+boolean               GetSandbox{xpath('GetSandbox')};
+boolean               GetText{xpath('GetText')};
+                 END;
+
+
+GetAttributeOutRecord := RECORD
+string                ModuleName{xpath('ModuleName')};
+string                text{xpath('Text')};
+                END;
+
+
+GetAttributeInRecord createInRecord := TRANSFORM
+            SELF.ModuleName := 'SearchModule';
+            SELF.AttributeName := 'SearchAttribute';
+            SELF.Version := 0;
+            SELF.GetSandbox := true;
+            SELF.GetText := true;
+        END;
+
+GetAttributeInRecord createInRecord2(string moduleName, string attr) := TRANSFORM
+            SELF.ModuleName := moduleName;
+            SELF.AttributeName := attr;
+            SELF.Version := 0;
+            SELF.GetSandbox := true;
+            SELF.GetText := true;
+        END;
+
+GetAttributeInRecord2 := RECORD
+string                ModuleName{xpath('ModuleName')} := 'MyModule';
+string                AttributeName{xpath('AttributeName')} := 'MyAttribute';
+unsigned              Version{xpath('Version')} := 1;
+boolean               GetSandbox{xpath('GetSandbox')} := true;
+boolean               GetText{xpath('GetText')} := true;
+                 END;
+
+ds := dataset([{'doxie','One'},{'jimbo','Two'},{'yankie','Three'}],{string moduleName, string attr});
+
+//--No argument, record provides values--
+//actions
+SOAPCALL('http://webservices.megacorp.com', 'WsAttributes', GetAttributeInRecord2, LOG(MIN));
+SOAPCALL('http://webservices.megacorp.com', 'WsAttributes', GetAttributeInRecord2, RETRY(100), TIMEOUT(99), LOG('Call abc'));
+
+
+output(SOAPCALL(ds, 'http://webservices.megacorp.com', 'WsAttributesDs', GetAttributeInRecord, createInRecord2(LEFT.moduleName, LEFT.attr), dataset(GetAttributeOutRecord), LOG('megacorp:'+LEFT.moduleName+'.'+LEFT.attr)));
+
+output(SOAPCALL(ds, 'http://webservices.megacorp.com', 'WsAttributesDs', GetAttributeInRecord, createInRecord2(LEFT.moduleName, LEFT.attr), dataset(GetAttributeOutRecord), LOG,LOG(MIN),LOG('megacorp:'+moduleName+'.'+attr)));
+
+
+output(HTTPCALL('http://webservices.megacorp.com', 'WsAttributesDs', 'blah', GetAttributeOutRecord, LOG,LOG(MIN),LOG('megacorp:xxx')));

rtl/include/eclhelper.hpp

@@ -2017,6 +2017,8 @@ enum
     SOAPFnamespace      = 0x0020,
     SOAPFencoding       = 0x0040,
     SOAPFpreserveSpace  = 0x0080,
+    SOAPFlogmin         = 0x0100,
+    SOAPFlogusermsg     = 0x0200,
 };
 
 struct IHThorWebServiceCallActionArg : public IHThorArg
@@ -2053,6 +2055,7 @@ struct IHThorWebServiceCallExtra : public IInterface
     virtual IXmlToRowTransformer * queryInputTransformer() = 0;
     virtual const char * queryInputIteratorPath()       { return NULL; }
     virtual size32_t onFailTransform(ARowBuilder & rowBuilder, const void * left, IException * e) { return 0; }
+    virtual void getLogText(size32_t & lenText, char * & text, const void * left) = 0;  // iff SOAPFlogusermsg set
 };
 typedef IHThorWebServiceCallExtra IHThorSoapCallExtra;
 

rtl/include/eclhelper_base.hpp

@@ -2328,6 +2328,7 @@ class CThorSoapActionArg : public CThorArg, implements IHThorSoapActionArg
     virtual unsigned     getTimeLimit()                 { return 0; }
     virtual const char * queryProxyAddress()            { return NULL; }
     virtual const char * queryAcceptType()              { return NULL; }
+    virtual void getLogText(size32_t & lenText, char * & text, const void * left) { lenText =0; text = NULL; }
 };
 
 class CThorSoapCallArg : public CThorArg, implements IHThorSoapCallArg
@@ -2373,6 +2374,7 @@ class CThorSoapCallArg : public CThorArg, implements IHThorSoapCallArg
     virtual unsigned     getTimeLimit()                 { return 0; }
     virtual const char * queryProxyAddress()            { return NULL; }
     virtual const char * queryAcceptType()              { return NULL; }
+    virtual void getLogText(size32_t & lenText, char * & text, const void * left) { lenText =0; text = NULL; }
 };
 typedef CThorSoapCallArg CThorHttpCallArg;
 

testing/ecl/roxie/soapcall.ecl

@@ -28,7 +28,7 @@ ServiceOutRecord :=
     END;
 
 // simple query->dataset form
-output(SORT(SOAPCALL('http://127.0.0.1:9876','soapbase', { string unkname := 'FRED' }, dataset(ServiceOutRecord)),record));
+output(SORT(SOAPCALL('http://127.0.0.1:9876','soapbase', { string unkname := 'FRED' }, dataset(ServiceOutRecord), LOG('simple')),record));
 
 // double query->dataset form
 output(SORT(SOAPCALL('http://127.0.0.1:9876|http://127.0.0.1:9876','soapbase', { string unkname := 'FRED' }, dataset(ServiceOutRecord)),record));
@@ -53,9 +53,9 @@ END;
 
 // Test some failure cases
 
-output(SORT(SOAPCALL(d, 'http://127.0.0.1:9876|http://127.0.0.1:9875','soapbase', { unkname }, DATASET(ServiceOutRecord), onFail(doError(LEFT)),RETRY(0)), record));
-output(SORT(SOAPCALL('http://127.0.0.1:9876','soapbase', { string unkname := 'FAIL' }, dataset(ServiceOutRecord),onFail(doError2),RETRY(0)),record));
-output(SORT(SOAPCALL(d, 'http://127.0.0.1:9876','soapbaseNOSUCHQUERY', { unkname }, DATASET(ServiceOutRecord), onFail(doError(LEFT)),MERGE(25),RETRY(0)), record));
+output(SORT(SOAPCALL(d, 'http://127.0.0.1:9876|http://127.0.0.1:9875','soapbase', { unkname }, DATASET(ServiceOutRecord), onFail(doError(LEFT)),RETRY(0), log('SOAP: ' + unkname)), record));
+output(SORT(SOAPCALL('http://127.0.0.1:9876','soapbase', { string unkname := 'FAIL' }, dataset(ServiceOutRecord),onFail(doError2),RETRY(0), LOG(MIN)),record));
+output(SORT(SOAPCALL(d, 'http://127.0.0.1:9876','soapbaseNOSUCHQUERY', { unkname }, DATASET(ServiceOutRecord), onFail(doError(LEFT)),MERGE(25),RETRY(0), LOG(MIN)), record));
 
 childRecord := record
 unsigned            id;