Merge pull request #12705 from mayx/HPCC-18702-gzip

HPCC-18702 Add HTTP Content-Encoding gzip/deflate support to ESP

Reviewed-By: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

esp/bindings/http/client/httpclient.cpp

@@ -376,7 +376,10 @@ HttpClientErrCode CHttpClient::sendRequest(const char* method, const char* conte
 
     httprequest.setown(new CHttpRequest(*m_socket));
     httpresponse.setown(new CHttpResponse(*m_socket));
-    
+
+    httprequest->enableCompression();
+    httpresponse->enableCompression();
+
     httprequest->setMethod(method);
     httprequest->setVersion("HTTP/1.1");
 
@@ -675,6 +678,7 @@ HttpClientErrCode CHttpClient::sendRequest(IProperties *headers, const char* met
 
     httprequest->setContentType(contenttype);
 
+    bool alreadyEncoded = false;
     if (headers)
     {
         Owned<IPropertyIterator> iter = headers->getIterator();
@@ -683,6 +687,8 @@ HttpClientErrCode CHttpClient::sendRequest(IProperties *headers, const char* met
             const char *key = iter->getPropKey();
             if (key && *key)
             {
+                if (strieq(key, HTTP_HEADER_CONTENT_ENCODING) || strieq(key, HTTP_HEADER_TRANSFER_ENCODING))
+                    alreadyEncoded = true;
                 const char *value = headers->queryProp(key);
                 if (value && *value)
                     httprequest->addHeader(key, value);
@@ -690,6 +696,12 @@ HttpClientErrCode CHttpClient::sendRequest(IProperties *headers, const char* met
         }
     }
 
+    if (!alreadyEncoded)
+    {
+        httprequest->enableCompression();
+        httpresponse->enableCompression();
+    }
+
     if(m_userid.length() > 0)
     {
         StringBuffer uidpair;
@@ -851,7 +863,9 @@ HttpClientErrCode CHttpClient::postRequest(ISoapMessage &req, ISoapMessage& resp
     Owned<CHttpRequest> httprequest(new CHttpRequest(*m_socket));
     Owned<CHttpResponse> httpresponse(new CHttpResponse(*m_socket));
 
-    
+    httprequest->enableCompression();
+    httpresponse->enableCompression();
+
     httprequest->setMethod("POST");
     httprequest->setVersion("HTTP/1.1");
 

esp/bindings/http/platform/httpservice.cpp

@@ -186,6 +186,7 @@ int CEspHttpServer::processRequest()
 {
     m_request->setPersistentEnabled(m_apport->queryProtocol()->persistentEnabled() && !shouldClose);
     m_response->setPersistentEnabled(m_apport->queryProtocol()->persistentEnabled() && !shouldClose);
+    m_response->enableCompression();
     try
     {
         if (m_request->receive(NULL)==-1) // MORE - pass in IMultiException if we want to see exceptions (which are not fatal)
@@ -453,7 +454,11 @@ int CEspHttpServer::onGetApplicationFrame(CHttpRequest* request, CHttpResponse*
         response->setContentType("text/html; charset=UTF-8");
         response->setStatus(HTTP_STATUS_OK);
 
-        const char *timestr=ctime(&modtime);
+        char timestr[128];
+        ctime_r(&modtime, timestr);
+        int timelen = strlen(timestr);
+        if (timelen > 0 && timestr[timelen -1] == '\n')
+            timestr[timelen - 1] = '\0';
         response->addHeader("Last-Modified", timestr);
         response->send();
     }

esp/bindings/http/platform/httptransport.cpp

@@ -29,6 +29,9 @@
 //ESP Binidings
 #include "http/platform/httptransport.ipp"
 #include "bindutil.hpp"
+#ifdef _USE_ZLIB
+#include "zcrypt.hpp"
+#endif
 
 bool httpContentFromFile(const char *filepath, StringBuffer &mimetype, MemoryBuffer &fileContents, bool &checkModifiedTime, StringBuffer &lastModified, StringBuffer &etag)
 {
@@ -630,8 +633,18 @@ int CHttpMessage::receive(bool alwaysReadContent, IMultiException *me)
             DBGLOG("length of content read = %d", m_content.length());
     }
 
+    bool decompressed = false;
+    int compressType = 0;
+    if (shouldDecompress(compressType))
+        decompressed = decompressContent(nullptr, compressType);
+
     if (getEspLogRequests() == LogRequestsAlways)
-        logMessage(LOGCONTENT, "HTTP content received:\n");
+    {
+        if (!decompressed)
+            logMessage(LOGCONTENT, "HTTP content received:\n");
+        else
+            logMessage(LOGCONTENT, "Compressed HTTP content received, decompressed content:\n");
+    }
     return 0;
 }
 
@@ -754,18 +767,23 @@ void CHttpMessage::logSOAPMessage(const char* message, const char* prefix)
 
 void CHttpMessage::logMessage(MessageLogFlag messageLogFlag, const char *prefix)
 {
+    logMessage(messageLogFlag, m_content, prefix);
+}
+
+void CHttpMessage::logMessage(MessageLogFlag messageLogFlag, StringBuffer& content, const char *prefix)
+{
     try
     {
         if (((messageLogFlag == LOGHEADERS) || (messageLogFlag == LOGALL)) && (m_header.length() > 0))
             logMessage(m_header.str(), prefix, "Authorization:[~\r\n]*", "Authorization: (hidden)");
 
-        if (((messageLogFlag == LOGCONTENT) || (messageLogFlag == LOGALL)) && (m_content.length() > 0))
+        if (((messageLogFlag == LOGCONTENT) || (messageLogFlag == LOGALL)) && (content.length() > 0))
         {//log content
             if ((m_header.length() > 0) && (startsWith(m_header.str(), "POST /ws_access/AddUser")
                 || startsWith(m_header.str(), "POST /ws_access/UserResetPass") || startsWith(m_header.str(), "POST /ws_account/UpdateUser")))
                 DBGLOG("%s<For security, ESP does not log the content of this request.>", prefix);
             else if (isSoapMessage())
-                logSOAPMessage(m_content.str(), prefix);
+                logSOAPMessage(content.str(), prefix);
             else if(!isTextMessage())
                 DBGLOG("%s<non-text content or content type not specified>", prefix);
             else if ((m_content_type.length() > 0) && (strieq(m_content_type.get(), "text/css") || strieq(m_content_type.get(), "text/javascript")))
@@ -775,20 +793,20 @@ void CHttpMessage::logMessage(MessageLogFlag messageLogFlag, const char *prefix)
                 StringBuffer httpPath;
                 getPath(httpPath);
                 if (!strieq(httpPath.str(), "/esp/login"))
-                    logMessage(m_content.str(), prefix);
+                    logMessage(content.str(), prefix);
                 else
-                    logMessage(m_content.str(), prefix, "password=*", "password=(hidden)");
+                    logMessage(content.str(), prefix, "password=*", "password=(hidden)");
             }
         }
     }
     catch (IException *e)
     {
         StringBuffer msg;
-        IERRLOG("EXCEPTION %s when logging the message: %s", e->errorMessage(msg).str(), m_content.str());
+        IERRLOG("EXCEPTION %s when logging the message: %s", e->errorMessage(msg).str(), content.str());
         if (m_content_type.length() > 0)
-            IERRLOG("EXCEPTION %s when logging the message (m_content_type:%s):%s", e->errorMessage(msg).str(), m_content_type.get(), m_content.str());
+            IERRLOG("EXCEPTION %s when logging the message (m_content_type:%s):%s", e->errorMessage(msg).str(), m_content_type.get(), content.str());
         else
-            IERRLOG("EXCEPTION %s when logging the message: %s", e->errorMessage(msg).str(), m_content.str());
+            IERRLOG("EXCEPTION %s when logging the message: %s", e->errorMessage(msg).str(), content.str());
         e->Release();
     }
     return;
@@ -796,17 +814,29 @@ void CHttpMessage::logMessage(MessageLogFlag messageLogFlag, const char *prefix)
 
 int CHttpMessage::send()
 {
+    bool logMsg = getEspLogResponses();
+    bool compressed = false;
+    StringBuffer originalContent;
+    int compressType = 0;
+    if (shouldCompress(compressType))
+        compressed = compressContent(logMsg?&originalContent:nullptr, compressType);
+
     StringBuffer headers;
     constructHeaderBuffer(headers, true);
     
     int retcode = 0;
 
     // If m_content is empty but m_content_stream is set, the stream will not be logged here.
-    if (getEspLogResponses())
+    if (logMsg)
     {
         logMessage(headers.str(), "Sending out HTTP headers:\n", "Authorization:[~\r\n]*", "Authorization: (hidden)");
         if(m_content_length > 0 && m_content.length() > 0)
-            logMessage(LOGCONTENT, "Sending out HTTP content:\n");
+        {
+            if (!compressed)
+                logMessage(LOGCONTENT, "Sending out HTTP content:\n");
+            else
+                logMessage(LOGCONTENT, originalContent, "Sending out compressed HTTP content, original content:\n");
+        }
     }
 
     try
@@ -1050,6 +1080,46 @@ StringBuffer& CHttpMessage::getHeader(const char* headername, StringBuffer& head
     return headerval;       
 }
 
+bool CHttpMessage::hasHeader(const char* headername)
+{
+    if (!headername || !*headername)
+        return false;
+
+    unsigned headerlen = strlen(headername);
+    ForEachItemIn(x, m_headers)
+    {
+        const char* header = m_headers.item(x);
+        if (header == nullptr)
+            continue;
+        const char* colon = strchr(header, ':');
+        if (colon == nullptr)
+            continue;
+        unsigned len = colon - header;
+        if ((headerlen == len) && (strnicmp(headername, header, len) == 0))
+            return true;
+    }
+    return false;
+}
+
+void CHttpMessage::removeHeader(const char* headername)
+{
+    if (!headername || !*headername)
+        return;
+
+    ForEachItemInRev(x, m_headers)
+    {
+        const char* header = m_headers.item(x);
+        if (header == nullptr)
+            continue;
+        const char* colon = strchr(header, ':');
+        if (colon == nullptr)
+            continue;
+        unsigned len = colon - header;
+        if ((strlen(headername) == len) && (strnicmp(headername, header, len) == 0))
+            m_headers.remove(x);
+    }
+}
+
 bool isSoapContentType(const char* contenttype)
 {
     if(contenttype == NULL)
@@ -1112,6 +1182,14 @@ bool CHttpMessage::isFormSubmission()
         hasContentType(HTTP_TYPE_FORM_ENCODED));
 }
 
+void CHttpMessage::enableCompression()
+{
+    const char* off = getenv("ESP_COMPRESSION_OFF");
+    if (off && streq(off, "1"))
+        return;
+    m_compressionEnabled = true;
+}
+
 /******************************************************************************
               CHttpRequest Implementation
 *******************************************************************************/
@@ -1722,6 +1800,7 @@ void CHttpRequest::updateContext()
 {
     if(m_context)
     {
+        m_context->setRequest(this);
         m_context->setContextPath(m_httpPath.str());
 
         StringBuffer temp;
@@ -1789,6 +1868,9 @@ void CHttpRequest::updateContext()
 
 StringBuffer& CHttpRequest::constructHeaderBuffer(StringBuffer& headerbuf, bool inclLength)
 {
+    if (m_compressionEnabled && !hasHeader(HTTP_HEADER_ACCEPT_ENCODING))
+        addHeader(HTTP_HEADER_ACCEPT_ENCODING, "gzip, deflate");
+
     if(m_httpMethod.length() > 0)
         headerbuf.append(queryMethod()).append(" ");
     else
@@ -2458,9 +2540,20 @@ int CHttpResponse::receive(bool alwaysReadContent, IMultiException *me)
         if (getEspLogLevel()>LogNormal)
             DBGLOG("length of content read = %d", m_content.length());
     }
-    
+
+    bool decompressed = false;
+    int compressType = 0;
+    if (shouldDecompress(compressType))
+        decompressed = decompressContent(nullptr, compressType);
+
     if (getEspLogRequests() == LogRequestsAlways)
-        logMessage(LOGCONTENT, "HTTP response content received:\n");
+    {
+        if (!decompressed)
+            logMessage(LOGCONTENT, "HTTP response content received:\n");
+        else
+            logMessage(LOGCONTENT, "Compressed HTTP response content received, decompressed content:\n");
+    }
+
     return 0;
 
 }
@@ -2563,3 +2656,192 @@ void CHttpResponse::handleExceptions(IXslProcessor *xslp, IMultiException *me, c
     if (handleExceptions(xslp, me, serv, meth, errorXslt) && logHandleExceptions)
         PROGLOG("Exception(s) handled");
 }
+
+inline bool zlibTypeFromHeader(const char* acceptEncodingHeader, int& compressType)
+{
+    if (!acceptEncodingHeader || !*acceptEncodingHeader)
+        return false;
+
+    StringArray encodings;
+    encodings.appendList(acceptEncodingHeader, ",");
+    int gzipq = -1, deflateq = -1, zlibq = -1;
+    ForEachItemIn(x, encodings)
+    {
+        const char* encoding = encodings.item(x);
+        int qval = 1000; //Default q value is 1.0 (1000 after timing 1000)
+        const char* qptr = strstr(encoding, ";q=");
+        if (!qptr)
+            qptr = strstr(encoding, " q=");
+        if (qptr)
+            qval = 1000*atof(qptr+3); //Smallest q value possible is 0.001, so times 1000 to make it integer
+        if (qval == 0)
+            continue;
+        const char* end = encoding;
+        while (*end != '\0' && *end != ';' && *end != ' ')
+            end++;
+        size_t len = end - encoding;
+        if (len == 4 && strncmp(encoding, "gzip", len) == 0)
+            gzipq = qval;
+        else if (len == 7 && strncmp(encoding, "deflate", len) == 0)
+            deflateq = qval;
+        else if (len == 9 && strncmp(encoding, "x-deflate", len) == 0)
+            zlibq = qval;
+    }
+
+    if (gzipq > 0 && gzipq >= deflateq && gzipq >= zlibq) //Use gzip as much as possible due to some reported browser issues with deflate
+        compressType = static_cast<int>(ZlibCompressionType::GZIP);
+    else if (deflateq > 0 && deflateq >= zlibq)
+        compressType = static_cast<int>(ZlibCompressionType::DEFLATE);
+    else if (zlibq > 0)
+        compressType = static_cast<int>(ZlibCompressionType::ZLIB_DEFLATE);
+
+    return gzipq > 0 || deflateq > 0 || zlibq > 0;
+}
+
+inline const char* zlibType2Header(ZlibCompressionType zltype)
+{
+    if (zltype==ZlibCompressionType::GZIP)
+        return "gzip";
+    else if (zltype==ZlibCompressionType::DEFLATE)
+        return "deflate";
+    else if (zltype==ZlibCompressionType::ZLIB_DEFLATE)
+        return "x-deflate";
+    else
+        return "";
+}
+
+bool CHttpResponse::shouldCompress(int& compressType)
+{
+    compressType = 0;
+    if (!m_compressionEnabled)
+        return false;
+    if (m_content_length == 0 || m_content.length() == 0)
+        return false;
+
+#ifndef _USE_ZLIB
+    return false;
+#else
+    if (hasHeader(HTTP_HEADER_CONTENT_ENCODING) || hasHeader(HTTP_HEADER_TRANSFER_ENCODING)) //Already encoded/compressed
+        return false;
+
+    static const int DEFAULT_MIN_COMPRESS_LENGTH = 1000;
+    int min_len = DEFAULT_MIN_COMPRESS_LENGTH;
+    IPropertyTree* proccfg = nullptr;
+    IEspServer* espserver = queryEspServer();
+    if (espserver)
+        proccfg = espserver->queryProcConfig();
+    if (proccfg)
+        min_len = proccfg->getPropInt("@minCompressLength", DEFAULT_MIN_COMPRESS_LENGTH);
+    if (m_content.length() < min_len)
+        return false;
+
+    IEspContext * context = queryContext();
+    if (!context)
+        return false;
+    CHttpRequest* request = dynamic_cast<CHttpRequest*>(context->queryRequest());
+    if (!request)
+        return false;
+    StringBuffer acceptEncoding;
+    request->getHeader(HTTP_HEADER_ACCEPT_ENCODING, acceptEncoding);
+    if (!zlibTypeFromHeader(acceptEncoding.toLowerCase().str(), compressType))
+        return false;
+
+    const char* content_type = m_content_type.get();
+    if (m_content_type.length() == 0 ||
+         !( strnicmp(content_type, "text", 4) == 0
+         || strnicmp(content_type, HTTP_TYPE_APPLICATION_XML, strlen(HTTP_TYPE_APPLICATION_XML)) == 0
+         || strnicmp(content_type, HTTP_TYPE_SOAP, strlen(HTTP_TYPE_SOAP)) == 0
+         || strnicmp(content_type, HTTP_TYPE_SVG_XML, strlen(HTTP_TYPE_SVG_XML)) == 0
+         || strnicmp(content_type, HTTP_TYPE_JSON, strlen(HTTP_TYPE_JSON)) == 0
+         || strnicmp(content_type, HTTP_TYPE_JAVASCRIPT, strlen(HTTP_TYPE_JAVASCRIPT)) == 0
+         || strnicmp(content_type, HTTP_TYPE_JAVASCRIPT2, strlen(HTTP_TYPE_JAVASCRIPT2)) == 0))
+        return false;
+
+    return true;
+#endif
+}
+
+bool CHttpResponse::compressContent(StringBuffer* originalContent, int compressType)
+{
+#ifdef _USE_ZLIB
+    ZlibCompressionType zlibtype = static_cast<ZlibCompressionType>(compressType);
+    MemoryBuffer zippedContent;
+    try
+    {
+        zlib_deflate(zippedContent, m_content.str(), m_content.length(), GZ_DEFAULT_COMPRESSION, zlibtype);
+    }
+    catch(IException* e)
+    {
+        StringBuffer estr;
+        IERRLOG("Exception(%d, %s) compressing response content.", e->errorCode(), e->errorMessage(estr).str());
+        return false;
+    }
+    catch(...)
+    {
+        IERRLOG("Unknown exception compressing response content.");
+        return false;
+    }
+    if (originalContent != nullptr)
+        originalContent->setown(m_content);
+    setContent(zippedContent.length(), zippedContent.toByteArray());
+    addHeader(HTTP_HEADER_CONTENT_ENCODING, zlibType2Header(zlibtype));
+    return true;
+#endif
+}
+
+bool CHttpResponse::shouldDecompress(int& compressType)
+{
+    compressType = 0;
+    if (!m_compressionEnabled)
+        return false;
+    if (m_content_length == 0 || m_content.length() == 0)
+        return false;
+
+#ifdef _USE_ZLIB
+    StringBuffer ceheader;
+    getHeader(HTTP_HEADER_CONTENT_ENCODING, ceheader);
+    ceheader.trim();
+    if (ceheader.length() == 0)
+        return false;
+    if (strieq(ceheader.str(), "gzip"))
+        compressType = static_cast<int>(ZlibCompressionType::GZIP);
+    else if (strieq(ceheader.str(), "deflate"))
+        compressType = static_cast<int>(ZlibCompressionType::DEFLATE);
+    else if (strieq(ceheader.str(), "x-deflate"))
+        compressType = static_cast<int>(ZlibCompressionType::ZLIB_DEFLATE);
+    else
+        return false;
+    return true;
+#else
+    return false;
+#endif
+}
+
+bool CHttpResponse::decompressContent(StringBuffer* originalContent, int compressType)
+{
+#ifdef _USE_ZLIB
+    ZlibCompressionType zlibtype = static_cast<ZlibCompressionType>(compressType);
+    StringBuffer decompressedContent;
+    try
+    {
+        httpInflate((const unsigned char*)m_content.str(), m_content.length(), decompressedContent, zlibtype==ZlibCompressionType::GZIP);
+    }
+    catch(IException* e)
+    {
+        StringBuffer estr;
+        IERRLOG("Exception(%d, %s) decompressing response content.", e->errorCode(), e->errorMessage(estr).str());
+        return false;
+    }
+    catch(...)
+    {
+        IERRLOG("Unknown exception decompressing response content.");
+        return false;
+    }
+    if (originalContent != nullptr)
+        originalContent->setown(m_content);
+    m_content.setown(decompressedContent);
+    m_content_length = m_content.length();
+    removeHeader(HTTP_HEADER_CONTENT_ENCODING);
+    return true;
+#endif
+}

esp/bindings/http/platform/httptransport.hpp

@@ -55,6 +55,7 @@
 #define HTTP_TYPE_FORM_ENCODED                  "application/x-www-form-urlencoded"
 #define HTTP_TYPE_SVG_XML                       "image/svg+xml"
 #define HTTP_TYPE_JAVASCRIPT                    "application/x-javascript"
+#define HTTP_TYPE_JAVASCRIPT2                   "application/javascript"
 
 #define HTTP_TYPE_TEXT_HTML_UTF8                "text/html; charset=UTF-8"
 #define HTTP_TYPE_TEXT_PLAIN_UTF8               "text/plain; charset=UTF-8"

esp/bindings/http/platform/httptransport.ipp

@@ -49,6 +49,10 @@ enum MessageLogFlag
     LOGCONTENT = 2
 };
 
+#define HTTP_HEADER_CONTENT_ENCODING  "Content-Encoding"
+#define HTTP_HEADER_TRANSFER_ENCODING "Transfer-Encoding"
+#define HTTP_HEADER_ACCEPT_ENCODING   "Accept-Encoding"
+
 class esp_http_decl CHttpMessage : implements IHttpMessage, public CInterface
 {
 protected:
@@ -70,6 +74,7 @@ protected:
     bool         m_persistentEligible = false;
     bool         m_persistentEnabled = false;
     bool         m_peerClosed = false;
+    bool         m_compressionEnabled = false;
 
     int m_paramCount;
     int m_attachCount;
@@ -119,6 +124,7 @@ public:
     void logSOAPMessage(const char* message, const char* prefix = NULL);
     void logMessage(const char *message, const char *prefix = NULL, const char *find = NULL, const char *replace = NULL);
     void logMessage(MessageLogFlag logFlag, const char *prefix = NULL);
+    void logMessage(MessageLogFlag logFlag, StringBuffer& content, const char *prefix = NULL);
 
     virtual StringBuffer& getContent(StringBuffer& content);
     virtual void setContent(const char* content);
@@ -152,6 +158,8 @@ public:
     virtual void setHeader(const char* headername, const char* headerval);
     virtual void addHeader(const char* headername, const char* headerval);
     virtual StringBuffer& getHeader(const char* headername, StringBuffer& headerval);
+    virtual bool hasHeader(const char* headername);
+    virtual void removeHeader(const char* headername);
     virtual int getParameterCount(){return m_paramCount;}
     virtual int getAttachmentCount(){return m_attachCount;}
     virtual IProperties *queryParameters();
@@ -259,6 +267,11 @@ public:
     virtual bool getPersistentEligible() { return m_persistentEligible; }
     virtual void setPersistentEnabled(bool enabled) { m_persistentEnabled = enabled; }
     virtual bool getPeerClosed() { return m_peerClosed; }
+    virtual void enableCompression();
+    virtual bool shouldCompress(int& compressType) { return false; }
+    virtual bool compressContent(StringBuffer* originalContent, int compressType) { return false; }
+    virtual bool shouldDecompress(int& compressType) { return false; }
+    virtual bool decompressContent(StringBuffer* originalContent, int compressType) { return false; }
 };
 
 
@@ -402,6 +415,10 @@ public:
     void CheckModifiedHTTPContent(bool modified, const char *lastModified, const char *etag, const char *contenttype, MemoryBuffer &content);
     bool getRespSent() { return m_respSent; }
     void setRespSent(bool sent) { m_respSent = sent; }
+    virtual bool shouldCompress(int& compressType);
+    virtual bool compressContent(StringBuffer* originalContent, int compressType);
+    virtual bool shouldDecompress(int& compressType);
+    virtual bool decompressContent(StringBuffer* originalContent, int compressType);
 };
 
 inline bool canRedirect(CHttpRequest &req)

esp/platform/espcontext.cpp

@@ -87,6 +87,7 @@ private:
     Owned<IEspSecureContext> m_secureContext;
 
     StringAttr   m_transactionID;
+    IHttpMessage* m_request;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -591,6 +592,14 @@ public:
     {
         return m_transactionID.get();
     }
+    virtual void setRequest(IHttpMessage* req)
+    {
+        m_request = req;
+    }
+    virtual IHttpMessage* queryRequest()
+    {
+        return m_request;
+    }
 };
 
 //---------------------------------------------------------

esp/protocols/http/CMakeLists.txt

@@ -64,6 +64,7 @@ include_directories(
     ./../../services/common
     ./../../../system/security/shared
     ./../../../system/security/LdapSecurity
+    ./../../../system/security/zcrypt
     ./../../../system/mp 
     ./../../../dali/base
     ./../../../common/workunit
@@ -96,3 +97,10 @@ endif()
 if(USE_OPENLDAP)
     target_link_libraries(esphttp LdapSecurity)
 endif(USE_OPENLDAP)
+
+IF (USE_ZLIB)
+    target_link_libraries ( esphttp
+        ${ZLIB_LIBRARIES}
+        zcrypt
+    )
+ENDIF()

esp/scm/esp.ecm

@@ -215,6 +215,8 @@ interface IEspContext : extends IInterface
     virtual void setAuthStatus(const char * status)=0;
     virtual const char * getRespMsg()=0;
     virtual void setRespMsg(const char * msg)=0;
+    virtual void setRequest(IHttpMessage* req) = 0;
+    virtual IHttpMessage* queryRequest() = 0;
 };
 
 

esp/tools/soapplus/http.cpp

@@ -1515,6 +1515,8 @@ StringBuffer& HttpClient::generateGetRequest(StringBuffer& request)
     if(m_port != 80)
         request.appendf(":%d", m_port);
     request.append("\r\n");
+    if (m_globals->getPropBool("compress", false))
+        request.append("Accept-Encoding: gzip, deflate\r\n");
     if(!m_globals->getPropBool("isPersist", false))
         request.append("Connection: Close\r\n");
     request.append(m_authheader.str());
@@ -1555,6 +1557,9 @@ StringBuffer& HttpClient::insertSoapHeaders(StringBuffer& request)
     if(m_globals->hasProp("soapaction"))
         headers.append("SOAPAction: ").append(m_globals->queryProp("soapaction")).append("\r\n");
 
+    if (m_globals->getPropBool("compress", false))
+        headers.append("Accept-Encoding: gzip, deflate\r\n");
+
     if(!m_globals->getPropBool("isPersist", false))
         headers.append("Connection: Close\r\n");
 

esp/tools/soapplus/main.cpp

@@ -75,6 +75,7 @@ void usage()
     puts("   -y: use default answers to yes or no prompts.");
     puts("   -wiz: ECL to ESP wizard mode.");
     puts("   -soapaction <url>: specify the soapaction.");
+    puts("   -compress: add header to the request to tell the server to compress the content if possible.");
     puts("   -version: print out soapplus version.");
 }
 
@@ -720,6 +721,11 @@ int main(int argc, char** argv)
             in_fname = argv[i++];
             isEspLogFile = true;
         }
+        else if(stricmp(argv[i], "-compress") == 0)
+        {
+            i++;
+            globals->setProp("compress", "1");
+        }
         else if(stricmp(argv[i], "-version") == 0)
         {
             printf("lexis-nexis soapplus version %s\n", version);

initfiles/componentfiles/configschema/xsd/esp.xsd

@@ -382,6 +382,9 @@
                 <xs:attribute name="maxPersistentRequests" type="xs:integer"
                               hpcc:displayName="Max Persistent Requests" hpcc:presetValue="100"
                               hpcc:tooltip="Maximum number of query requests per persistent http connection. (-1 for unlimited, 0 to disable)"/>
+                <xs:attribute name="minCompressLength" type="xs:integer"
+                              hpcc:displayName="Minimum content length for compression" hpcc:presetValue="1000"
+                              hpcc:tooltip="Minimum content length in bytes for the content to be compressed"/>
                 <xs:attribute name="dafilesrvConnectTimeout" type="xs:nonNegativeInteger"
                               hpcc:displayName="Remote file connection timeout in seconds"
                               hpcc:presentValue="10"

initfiles/componentfiles/configxml/esp.xsd.in

@@ -1061,6 +1061,13 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="minCompressLength" type="xs:integer" use="optional" default="1000">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Minumum content length in bytes for the content to be compressed.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
             <xs:attribute name="namespaceScheme" use="optional" default="basic">
                <xs:annotation>
                  <xs:appinfo>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -265,6 +265,7 @@
               maxBacklogQueueSize="200"
               maxConcurrentThreads="0"
               maxRequestEntityLength="8000000"
+              minCompressLength="1000"
               name="myesp"
               perfReportDelay="60"
               portalurl="${PORTALURL}">