HPCC-13162 Allow HTTPCALL/SOAPCALL to set one or more http headers

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/thorhelper/thorsoapcall.cpp

@@ -758,6 +758,9 @@ public:
         else
             timeLimitMS = (unsigned)(dval * 1000);
 
+        if (flags & SOAPFhttpheaders)
+            httpHeaders.set(s.setown(helper->getHttpHeaders()));
+
         if (wscType == STsoap)
         {
             soapaction.set(s.setown(helper->getSoapAction()));
@@ -817,7 +820,7 @@ public:
         {
             service.toUpperCase();  //GET/PUT/POST
             if (strcmp(service.str(), "GET"))
-                throw MakeStringException(0, "HTTPCALL Only 'GET' service supported");
+                throw MakeStringException(0, "HTTPCALL Only 'GET' http method currently supported");
             OwnedRoxieString acceptTypeSupplied(helper->getAcceptType()); // text/html, text/xml, etc
             acceptType.set(acceptTypeSupplied);
             acceptType.trim();
@@ -1045,6 +1048,7 @@ protected:
     StringAttr soapaction;
     StringAttr httpHeaderName;
     StringAttr httpHeaderValue;
+    StringAttr httpHeaders;
     StringAttr inputpath;
     StringBuffer service;
     StringBuffer acceptType;//for httpcall, text/plain, text/html, text/xml, etc
@@ -1273,7 +1277,18 @@ IWSCHelper * createHttpCallHelper(IWSCRowProvider *r, IEngineRowAllocator * outp
 }
 
 //=================================================================================================
-
+bool httpHeaderBlockContainsHeader(const char *httpheaders, const char *header)
+{
+    if (!httpheaders || !*httpheaders)
+        return false;
+    VStringBuffer match("\n%s:", header);
+    const char *matchStart = match.str()+1;
+    if (!strncmp(httpheaders, matchStart, strlen(matchStart)))
+        return true;
+    if (strstr(httpheaders, match))
+        return true;
+    return false;
+}
 class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFor
 {
     class CSocketDataProvider : public CInterface
@@ -1362,30 +1377,43 @@ private:
         else
             request.clear().append(master->service).append(" ").append(url.path).append(" HTTP/1.1\r\n");
 
-        if (url.userPasswordPair.length() > 0)
+        const char *httpheaders = master->httpHeaders.get();
+        if (httpheaders && *httpheaders)
         {
-            StringBuffer authToken;
-            JBASE64_Encode(url.userPasswordPair.str(), url.userPasswordPair.length(), authToken);
-            request.append("Authorization: Basic ").append(authToken).append("\r\n");
+            if (soapTraceLevel > 6 || master->logXML)
+                master->logctx.CTXLOG("%s: Adding HTTP Headers(%s)",  master->wscCallTypeText(), httpheaders);
+            request.append(httpheaders);
         }
-        else if (master->authToken.length() > 0)
+
+        if (!httpHeaderBlockContainsHeader(httpheaders, "Authorization"))
         {
-            request.append("Authorization: Basic ").append(master->authToken).append("\r\n");
+            if (url.userPasswordPair.length() > 0)
+            {
+                StringBuffer authToken;
+                JBASE64_Encode(url.userPasswordPair.str(), url.userPasswordPair.length(), authToken);
+                request.append("Authorization: Basic ").append(authToken).append("\r\n");
+            }
+            else if (master->authToken.length() > 0)
+            {
+                request.append("Authorization: Basic ").append(master->authToken).append("\r\n");
+            }
         }
+
         if (master->wscType == STsoap)
         {
             if (master->soapaction.get())
                 request.append("SOAPAction: ").append(master->soapaction.get()).append("\r\n");
-
-            if (master->httpHeaderName.get() && master->httpHeaderValue.get())
+            if (!master->httpHeaders.length() && master->httpHeaderName.get() && master->httpHeaderValue.get())
             {
+                //backward compatibility
                 StringBuffer hdr = master->httpHeaderName.get();
                 hdr.append(": ").append(master->httpHeaderValue);
                 if (soapTraceLevel > 6 || master->logXML)
                     master->logctx.CTXLOG("SOAPCALL: Adding HTTP Header(%s)", hdr.str());
                 request.append(hdr.append("\r\n"));
             }
-            request.append("Content-Type: text/xml\r\n");
+            if (!httpHeaderBlockContainsHeader(httpheaders, "Content-Type"))
+                request.append("Content-Type: text/xml\r\n");
         }
         else if(master->wscType == SThttp)
             request.append("Accept: ").append(master->acceptType).append("\r\n");

ecl/hqlcpp/hqlcpp.ipp

@@ -1509,6 +1509,8 @@ public:
     ABoundActivity * doBuildActivityWorkunitRead(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityXmlParse(BuildCtx & ctx, IHqlExpression * expr);
 
+    void doBuildHttpHeaderStringFunction(BuildCtx & ctx, IHqlExpression * expr);
+
     void doBuildTempTableFlags(BuildCtx & ctx, IHqlExpression * expr, bool isConstant);
 
     void doBuildXmlEncode(BuildCtx & ctx, const CHqlBoundTarget * tgt, IHqlExpression * expr, CHqlBoundExpr * result);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -17047,6 +17047,32 @@ void HqlCppTranslator::validateExprScope(BuildCtx & ctx, IHqlExpression * datase
         throwError2(HQLERR_OpArgDependsDataset, opName, argName);
 }
 
+void HqlCppTranslator::doBuildHttpHeaderStringFunction(BuildCtx &ctx, IHqlExpression * expr)
+{
+    HqlExprArray headerExprs;
+    gatherAttributes(headerExprs, httpHeaderAtom, expr);
+    if (headerExprs.length())
+    {
+        Owned<ITypeInfo> string2Type = makeStringType(2);
+        OwnedHqlExpr endName = createConstant(createStringValue(": ", LINK(string2Type)));
+        OwnedHqlExpr endLine = createConstant(createStringValue("\r\n", LINK(string2Type)));
+
+        HqlExprArray headerStringExprs;
+        ForEachItemIn(i, headerExprs)
+        {
+            IHqlExpression * httpHeader = &headerExprs.item(i);
+            headerStringExprs.append(*LINK(httpHeader->queryChild(0)));
+            headerStringExprs.append(*LINK(endName));
+            headerStringExprs.append(*LINK(httpHeader->queryChild(1)));
+            headerStringExprs.append(*LINK(endLine));
+        }
+
+        OwnedHqlExpr concatHeaders = createBalanced(no_concat, unknownVarStringType, headerStringExprs);
+        concatHeaders.setown(foldHqlExpression(concatHeaders));
+        doBuildVarStringFunction(ctx, "getHttpHeaders", concatHeaders);
+    }
+
+}
 
 ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpression * expr, bool isSink, bool isRoot)
 {
@@ -17129,9 +17155,12 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
     if (action)
         doBuildVarStringFunction(instance->startctx, "getSoapAction", action->queryChild(0));
 
+    doBuildHttpHeaderStringFunction(instance->startctx, expr);
+
     IHqlExpression * httpHeader = expr->queryAttribute(httpHeaderAtom);
     if (httpHeader)
     {
+        //backward compatible single httpheader support
         doBuildVarStringFunction(instance->startctx, "getHttpHeaderName", httpHeader->queryChild(0));
         doBuildVarStringFunction(instance->startctx, "getHttpHeaderValue", httpHeader->queryChild(1));
     }
@@ -17185,6 +17214,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
             flags.append("|SOAPFlogmin");
         if (logText)
             flags.append("|SOAPFlogusermsg");
+        if (httpHeader)
+            flags.append("|SOAPFhttpheaders");
 
         if (flags.length())
             doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
@@ -17293,6 +17324,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityHTTP(BuildCtx & ctx, IHqlExpre
     //virtual void toXML(const byte * self, StringBuffer & out) = 0;
     buildHTTPtoXml(instance->startctx);
 
+    doBuildHttpHeaderStringFunction(instance->startctx, expr);
+
     //virtual const char * queryOutputIteratorPath()
     IHqlExpression * separator = expr->queryAttribute(separatorAtom);
     if (separator)
@@ -17336,6 +17369,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityHTTP(BuildCtx & ctx, IHqlExpre
             flags.append("|SOAPFlogmin");
         if (logText)
             flags.append("|SOAPFlogusermsg");
+        if (expr->hasAttribute(httpHeaderAtom))
+            flags.append("|SOAPFhttpheaders");
 
         if (flags.length())
             doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);

esp/services/ws_smc/ws_smcService.cpp

@@ -1908,6 +1908,63 @@ bool CWsSMCEx::onBrowseResources(IEspContext &context, IEspBrowseResourcesReques
     return true;
 }
 
+int CWsSMCSoapBindingEx::onHttpEcho(CHttpRequest* request,  CHttpResponse* response)
+{
+    StringBuffer xml;
+    xml.append(
+        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+        "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\">"
+          "<soap:Body>"
+            "<HttpEchoResponse xmlns='urn:hpccsystems:ws:httpecho'>");
+
+    appendXMLTag(xml, "Method", request->queryMethod());
+    appendXMLTag(xml, "UrlPath", request->queryPath());
+    appendXMLTag(xml, "UrlParameters", request->queryParamStr());
+
+    appendXMLOpenTag(xml, "Headers");
+    StringArray &headers = request->queryHeaders();
+    headers.sortAscii(false);
+    ForEachItemIn(i, headers)
+    {
+        const char *h = headers.item(i);
+        if (strnicmp(h, "Authorization", 13))
+            appendXMLTag(xml, "Header", h);
+    }
+    appendXMLCloseTag(xml, "Headers");
+
+    const char *content = request->queryContent();
+    if (content && *content)
+        appendXMLTag(xml, "Content", request->queryContent());
+    xml.append("</HttpEchoResponse></soap:Body></soap:Envelope>");
+
+    response->setContent(xml);
+    response->setContentType("text/xml");
+    response->send();
+    return 0;
+}
+
+
+int CWsSMCSoapBindingEx::onGet(CHttpRequest* request,  CHttpResponse* response)
+{
+    const char *operation = request->queryServiceMethod();
+    if (!operation || !strieq(operation, "HttpEcho"))
+        return CWsSMCSoapBinding::onGet(request, response);
+
+    return onHttpEcho(request, response);
+}
+
+void CWsSMCSoapBindingEx::handleHttpPost(CHttpRequest *request, CHttpResponse *response)
+{
+    sub_service sstype;
+    StringBuffer operation;
+    request->getEspPathInfo(sstype, NULL, NULL, &operation, false);
+    if (!operation || !strieq(operation, "HttpEcho"))
+        CWsSMCSoapBinding::handleHttpPost(request, response);
+    else
+        onHttpEcho(request, response);
+}
+
+
 int CWsSMCSoapBindingEx::onGetForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method)
 {
     try

esp/services/ws_smc/ws_smcService.hpp

@@ -243,6 +243,11 @@ public:
             return NULL;
         return "stub.htm";
     }
+    virtual int onGet(CHttpRequest* request,  CHttpResponse* response);
+    void handleHttpPost(CHttpRequest *request, CHttpResponse *response);
+    int onHttpEcho(CHttpRequest* request,  CHttpResponse* response);
+
+
     virtual int onGetRoot(IEspContext &context, CHttpRequest* request,  CHttpResponse* response)
     {
         return  onGetInstantQuery(context, request, response, "WsSMC", "Activity");

rtl/include/eclhelper.hpp

@@ -39,7 +39,7 @@ if the supplied pointer was not from the roxiemem heap. Usually an OwnedRoxieStr
 
 //Should be incremented whenever the virtuals in the context or a helper are changed, so
 //that a work unit can't be rerun.  Try as hard as possible to retain compatibility.
-#define ACTIVITY_INTERFACE_VERSION      157
+#define ACTIVITY_INTERFACE_VERSION      158
 #define MIN_ACTIVITY_INTERFACE_VERSION  157             //minimum value that is compatible with current interface - without using selectInterface
 
 typedef unsigned char byte;
@@ -2138,6 +2138,7 @@ enum
     SOAPFpreserveSpace  = 0x0080,
     SOAPFlogmin         = 0x0100,
     SOAPFlogusermsg     = 0x0200,
+    SOAPFhttpheaders    = 0x0400
 };
 
 struct IHThorWebServiceCallActionArg : public IHThorArg
@@ -2162,6 +2163,7 @@ struct IHThorWebServiceCallActionArg : public IHThorArg
     virtual const char * getHttpHeaderValue()         { return NULL; }
     virtual const char * getProxyAddress()            { return NULL; }
     virtual const char * getAcceptType()              { return NULL; }
+    virtual const char * getHttpHeaders()             { return NULL; }
 };
 typedef IHThorWebServiceCallActionArg IHThorSoapActionArg ;
 typedef IHThorWebServiceCallActionArg IHThorHttpActionArg ;

rtl/include/eclhelper_base.hpp

@@ -2539,9 +2539,9 @@ class CThorSoapActionArg : public CThorArg, implements IHThorSoapActionArg
     virtual const char * getSoapAction()              { return NULL; }
     virtual const char * getNamespaceName()           { return NULL; }
     virtual const char * getNamespaceVar()            { return NULL; }
-
     virtual const char * getHttpHeaderName()          { return NULL; }
     virtual const char * getHttpHeaderValue()         { return NULL; }
+    virtual const char * getHttpHeaders()             { return NULL; }
     virtual const char * getProxyAddress()            { return NULL; }
     virtual const char * getAcceptType()              { return NULL; }
     virtual void getLogText(size32_t & lenText, char * & text, const void * left) { lenText =0; text = NULL; }
@@ -2584,9 +2584,9 @@ class CThorSoapCallArg : public CThorArg, implements IHThorSoapCallArg
     virtual const char * getSoapAction()              { return NULL; }
     virtual const char * getNamespaceName()           { return NULL; }
     virtual const char * getNamespaceVar()            { return NULL; }
-
     virtual const char * getHttpHeaderName()          { return NULL; }
     virtual const char * getHttpHeaderValue()         { return NULL; }
+    virtual const char * getHttpHeaders()             { return NULL; }
     virtual const char * getProxyAddress()            { return NULL; }
     virtual const char * getAcceptType()              { return NULL; }
     virtual void getLogText(size32_t & lenText, char * & text, const void * left) { lenText =0; text = NULL; }

testing/regress/ecl/httpcall_multiheader.ecl

@@ -0,0 +1,20 @@
+string TargetIP := '.' : stored('TargetIP');
+string storedHeader := 'StoredHeaderDefault' : stored('StoredHeader');
+
+
+httpEchoServiceResponseRecord :=
+    RECORD
+        string method{xpath('Method')};
+        string path{xpath('UrlPath')};
+        string parameters{xpath('UrlParameters')};
+        set of string headers{xpath('Headers/Header')};
+        string content{xpath('Content')};
+    END;
+
+string TargetURL := 'http://' + TargetIP + ':8010/WsSmc/HttpEcho?name=doe,joe&number=1';
+
+string constHeader := 'constHeaderValue';
+
+httpcallResult := HTTPCALL(TargetURL,'GET', 'text/xml', httpEchoServiceResponseRecord, xpath('Envelope/Body/HttpEchoResponse'),httpheader('literalHeader','literalValue'), httpheader('constHeader','constHeaderValue'), httpheader('storedHeader', storedHeader));
+
+output(httpcallResult);

testing/regress/ecl/key/httpcall_multiheader.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Method>GET</Method><UrlPath>/WsSmc/HttpEcho</UrlPath><UrlParameters>name=doe,joe&amp;number=1</UrlParameters><Headers><Header>Accept: text/xml</Header><Header>constHeader: constHeaderValue</Header><Header>literalHeader: literalValue</Header><Header>storedHeader: StoredHeaderDefault</Header></Headers><Content></Content></Row>
+</Dataset>

testing/regress/ecl/soapcall_multihttpheader.ecl

@@ -0,0 +1,27 @@
+string TargetIP := '.' : stored('TargetIP');
+string storedHeader := 'StoredHeaderDefault' : stored('storedHeader');
+
+httpEchoServiceResponseRecord :=
+    RECORD
+        string method{xpath('Method')};
+        string path{xpath('UrlPath')};
+        string parameters{xpath('UrlParameters')};
+        set of string headers{xpath('Headers/Header')};
+        string content{xpath('Content')};
+    END;
+
+string TargetURL := 'http://' + TargetIP + ':8010/WsSmc/HttpEcho?name=doe,joe&number=1';
+
+
+httpEchoServiceRequestRecord :=
+    RECORD
+       string Name{xpath('Name')} := 'Doe, Joe',
+       unsigned id{xpath('ADL')} := 999999,
+       real8 score := 88.88,
+    END;
+
+string constHeader := 'constHeaderValue';
+
+soapcallResult := SOAPCALL(TargetURL, 'HttpEcho', httpEchoServiceRequestRecord, DATASET(httpEchoServiceResponseRecord), LITERAL, xpath('HttpEchoResponse'), httpheader('StoredHeader', storedHeader), httpheader('literalHeader', 'literalHeaderValue'), httpheader('constHeader', constHeader));
+
+output(soapcallResult);