HPCC-16346 Support gzip/deflate compressed I/O in roxie

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/thorhelper/roxiehelper.cpp

@@ -27,6 +27,7 @@
 #include "mpbase.hpp"
 #include "dafdesc.hpp"
 #include "dadfs.hpp"
+#include "zcrypt.hpp"
 
 unsigned traceLevel = 0;
 
@@ -1479,10 +1480,17 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
         unsigned left = 0;
         char *buf;
 
-        if (pHttpHelper != NULL && strncmp((char *)&len, "POST", 4) == 0)
+        if (pHttpHelper)
+        {
+            if (strncmp((char *)&len, "POST", 4) == 0)
+                pHttpHelper->setHttpMethod(HttpMethod::POST);
+            else if (strncmp((char *)&len, "GET", 3) == 0)
+                pHttpHelper->setHttpMethod(HttpMethod::GET);
+        }
+
+        if (pHttpHelper && pHttpHelper->isHttp())
         {
-#define MAX_HTTP_HEADERSIZE 8000
-            pHttpHelper->setHttpMethod(HttpMethod::POST);
+#define MAX_HTTP_HEADERSIZE 16000 //arbitrary per line limit, most web servers are lower, but REST queries can be complex..
             char header[MAX_HTTP_HEADERSIZE + 1]; // allow room for \0
             sock->read(header, 1, MAX_HTTP_HEADERSIZE, bytesRead, timeout);
             header[bytesRead] = 0;
@@ -1491,38 +1499,34 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
             {
                 *payload = 0;
                 payload += 4;
-                char *str;
 
                 pHttpHelper->parseHTTPRequestLine(header);
+                const char *headers = strstr(header, "\r\n");
+                if (headers)
+                    pHttpHelper->parseRequestHeaders(headers+2);
 
-                // capture authentication token
-                if ((str = strstr(header, "Authorization: Basic ")) != NULL)
-                    pHttpHelper->setAuthToken(str+21);
-
-                // capture content type
-                if ((str = strstr(header, "Content-Type: ")) != NULL)
-                    pHttpHelper->setContentType(str+14);
+                if (pHttpHelper->isHttpGet())
+                {
+                    pHttpHelper->checkTarget();
+                    return true;
+                }
 
-                if (strstr(header, "Expect: 100-continue"))
+                const char *val = pHttpHelper->queryRequestHeader("Expect");
+                if (val && streq(val, "100-continue"))
                 {
                     StringBuffer cont("HTTP/1.1 100 Continue\n\n"); //tell client to go ahead and send body
                     sock->write(cont, cont.length());
                 }
 
                 // determine payload length
-                str = strstr(header, "Content-Length: ");
-                if (str)
+                val = pHttpHelper->queryRequestHeader("Content-Length");
+                if (val)
                 {
-                    len = atoi(str + strlen("Content-Length: "));
+                    len = atoi(val);
                     buf = ret.reserveTruncate(len);
                     left = len - (bytesRead - (payload - header));
                     if (len > left)
                         memcpy(buf, payload, len - left);
-                    if (pHttpHelper->isFormPost())
-                    {
-                        pHttpHelper->checkTarget();
-                        pHttpHelper->setFormContent(ret);
-                    }
                 }
                 else
                     left = len = 0;
@@ -1533,28 +1537,6 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
             if (!len)
                 throw MakeStringException(THORHELPER_DATA_ERROR, "Badly formed HTTP header");
         }
-        else if (pHttpHelper != NULL && strncmp((char *)&len, "GET", 3) == 0)
-        {
-#define MAX_HTTP_GET_LINE 16000 //arbitrary per line limit, most web servers are lower, but urls for queries can be complex..
-                pHttpHelper->setHttpMethod(HttpMethod::GET);
-                char headerline[MAX_HTTP_GET_LINE + 1];
-                Owned<IBufferedSocket> linereader = createBufferedSocket(sock);
-
-                int bytesread = readHttpHeaderLine(linereader, headerline, MAX_HTTP_GET_LINE);
-                pHttpHelper->parseHTTPRequestLine(headerline);
-
-                bytesread = readHttpHeaderLine(linereader, headerline, MAX_HTTP_GET_LINE);
-                while(bytesread >= 0 && *headerline && *headerline!='\r')
-                {
-                    // capture authentication token
-                    if (!strnicmp(headerline, "Authorization: Basic ", 21))
-                        pHttpHelper->setAuthToken(headerline+21);
-                    bytesread = readHttpHeaderLine(linereader, headerline, MAX_HTTP_GET_LINE);
-                }
-
-                pHttpHelper->checkTarget();
-                return true;
-        }
         else if (strnicmp((char *)&len, "STAT", 4) == 0)
             isStatus = true;
         else
@@ -1574,10 +1556,28 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
         }
 
         if (left)
-        {
             sock->read(buf + (len - left), left, left, bytesRead, timeout);
-        }
 
+        if (len && pHttpHelper)
+        {
+            if (pHttpHelper->getReqCompression()!=HttpCompression::NONE)
+            {
+        #ifdef _USE_ZLIB
+                StringBuffer decoded;
+                httpInflate((const byte*)ret.str(), len, decoded, pHttpHelper->getReqCompression()==HttpCompression::GZIP);
+                PROGLOG("%s Content decoded from %d bytes to %d bytes", pHttpHelper->queryRequestHeader("Content-Encoding"), len, decoded.length());
+                ret.swapWith(decoded);
+        #else
+                throw MakeStringException(THORHELPER_UNSUPPORTED_ENCODING, "Unsupported Content-Encoding (_USE_ZLIB is required): %s", pHttpHelper->queryRequestHeader("Content-Encoding"));
+        #endif
+            }
+
+            if (pHttpHelper->isFormPost())
+            {
+                pHttpHelper->checkTarget();
+                pHttpHelper->setFormContent(ret);
+            }
+        }
         return len != 0;
     }
     catch (IException *E)
@@ -1599,6 +1599,7 @@ void CSafeSocket::setHttpMode(const char *queryName, bool arrayMode, HttpHelper
     CriticalBlock c(crit); // Should not be needed
     httpMode = true;
     mlResponseFmt = httphelper.queryResponseMlFormat();
+    respCompression = httphelper.getRespCompression();
     heartbeat = false;
     assertex(contentHead.length()==0 && contentTail.length()==0);
     if (mlResponseFmt==MarkupFmt_JSON)
@@ -1741,49 +1742,125 @@ bool CSafeSocket::sendHeartBeat(const IContextLogger &logctx)
         return true;
 };
 
+class HttpResponseHandler
+{
+private:
+    CriticalBlock c; // should not be anyone writing but better to be safe
+    StringBuffer header;
+    StringBuffer content;
+    ISocket *sock = nullptr;
+    HttpCompression compression = HttpCompression::NONE;
+    unsigned int sent = 0;
+public:
+
+    HttpResponseHandler(ISocket *s, CriticalSection &crit) : sock(s), c(crit)
+    {
+    }
+    inline bool compressing()
+    {
+        return compression!=HttpCompression::NONE;
+    }
+    inline const char *traceName()
+    {
+        return compressing() ? "compressor" : "socket";
+    }
+    inline const char *compressTypeName()
+    {
+        return compression==HttpCompression::GZIP ? "gzip" : "deflate";
+    }
+    void init(unsigned length, TextMarkupFormat mlFmt, HttpCompression respCompression)
+    {
+        if (length > 1500)
+            compression = respCompression;
+        header.append("HTTP/1.0 200 OK\r\n");
+        header.append("Content-Type: ").append(mlFmt == MarkupFmt_JSON ? "application/json" : "text/xml").append("\r\n");
+        if (!compressing())
+        {
+            header.append("Content-Length: ").append(length).append("\r\n\r\n");
+            if (traceLevel > 5)
+                DBGLOG("Writing HTTP header length %d to HTTP socket", header.length());
+            sock->write(header.str(), header.length());
+            sent += header.length();
+        }
+        else
+        {
+            header.append("Content-Encoding: ").append(compression==HttpCompression::GZIP ? "gzip" : "deflate").append("\r\n");
+        }
+    }
+    size32_t write(void const* buf, size32_t size)
+    {
+        if (!compressing())
+        {
+            sent += size;
+            return sock->write(buf, size);
+        }
+        content.append(size, (const char *)buf);
+        return size;
+    }
+    size32_t finalize()
+    {
+        if (compressing())
+        {
+            ZlibCompressionType zt = ZlibCompressionType::GZIP;
+            if (compression==HttpCompression::DEFLATE)
+                zt  = ZlibCompressionType::DEFLATE;
+            if (compression==HttpCompression::ZLIB_DEFLATE)
+                zt  = ZlibCompressionType::ZLIB_DEFLATE;
+
+            MemoryBuffer mb;
+            zlib_deflate(mb, content.str(), content.length(), GZ_DEFAULT_COMPRESSION, zt);
+            if (traceLevel > 5)
+                DBGLOG("Compressed content length %u to %u (%s)", content.length(), mb.length(), compressTypeName());
+
+            header.append("Content-Length: ").append(mb.length()).append("\r\n\r\n");
+            if (traceLevel > 5)
+                DBGLOG("Writing HTTP header length %d to HTTP socket (compressed body)", header.length());
+            sock->write(header.str(), header.length());
+            sent += header.length();
+            if (traceLevel > 5)
+                DBGLOG("Writing compressed %s content, length %u, to HTTP socket", compressTypeName(), mb.length());
+
+            sock->write(mb.toByteArray(), mb.length());
+            sent += mb.length();
+        }
+        return sent;
+    }
+
+};
+
 void CSafeSocket::flush()
 {
     if (httpMode)
     {
-        unsigned length = 0;
+        unsigned contentLength = 0;
         if (!adaptiveRoot)
-            length = contentHead.length() + contentTail.length();
+            contentLength = contentHead.length() + contentTail.length();
         ForEachItemIn(idx, lengths)
-            length += lengths.item(idx);
-
-        StringBuffer header;
-        header.append("HTTP/1.0 200 OK\r\n");
-        header.append("Content-Type: ").append(mlResponseFmt == MarkupFmt_JSON ? "application/json" : "text/xml").append("\r\n");
-        header.append("Content-Length: ").append(length).append("\r\n\r\n");
+            contentLength += lengths.item(idx);
 
+        HttpResponseHandler resp(sock, crit);
 
-        CriticalBlock c(crit); // should not be anyone writing but better to be safe
-        if (traceLevel > 5)
-            DBGLOG("Writing HTTP header length %d to HTTP socket", header.length());
-        sock->write(header.str(), header.length());
-        sent += header.length();
+        resp.init(contentLength, mlResponseFmt, respCompression);
         if (!adaptiveRoot || mlResponseFmt != MarkupFmt_JSON)
         {
             if (traceLevel > 5)
-                DBGLOG("Writing content head length %" I64F "u to HTTP socket", static_cast<__uint64>(contentHead.length()));
-            sock->write(contentHead.str(), contentHead.length());
-            sent += contentHead.length();
+                DBGLOG("Writing content head length %" I64F "u to HTTP %s", static_cast<__uint64>(contentHead.length()), resp.traceName());
+            resp.write(contentHead.str(), contentHead.length());
         }
         ForEachItemIn(idx2, queued)
         {
             unsigned length = lengths.item(idx2);
             if (traceLevel > 5)
-                DBGLOG("Writing block length %d to HTTP socket", length);
-            sock->write(queued.item(idx2), length);
-            sent += length;
+                DBGLOG("Writing block length %d to HTTP %s", length, resp.traceName());
+            resp.write(queued.item(idx2), length);
         }
         if (!adaptiveRoot || mlResponseFmt != MarkupFmt_JSON)
         {
             if (traceLevel > 5)
-                DBGLOG("Writing content tail length %" I64F "u to HTTP socket", static_cast<__uint64>(contentTail.length()));
-            sock->write(contentTail.str(), contentTail.length());
-            sent += contentTail.length();
+                DBGLOG("Writing content tail length %" I64F "u to HTTP %s", static_cast<__uint64>(contentTail.length()), resp.traceName());
+            resp.write(contentTail.str(), contentTail.length());
         }
+        sent += resp.finalize();
         if (traceLevel > 5)
             DBGLOG("Total written %d", sent);
     }
@@ -2626,6 +2703,47 @@ void IRoxieContextLogger::CTXLOGae(IException *E, const char *file, unsigned lin
     va_end(args);
 }
 
+void loadHttpHeaders(IProperties *p, const char *finger)
+{
+    while (*finger)
+    {
+        StringBuffer prop, val;
+        while (*finger && *finger != '\r' && *finger != ':')
+            prop.append(*finger++);
+        if (*finger && *finger != '\r')
+        {
+            finger++;
+            while (isspace(*finger) && *finger != '\r')
+                finger++;
+            while (*finger && *finger != '\r')
+                val.append(*finger++);
+            prop.clip();
+            val.clip();
+            if (prop.length())
+                p->setProp(prop.str(), val.str());
+        }
+        if (*finger)
+            finger++;
+        if ('\n'==*finger)
+            finger++;
+    }
+}
+
+void HttpHelper::parseRequestHeaders(const char *headers)
+{
+    if (!reqHeaders)
+        reqHeaders.setown(createProperties());
+    loadHttpHeaders(reqHeaders, headers);
+    const char *val = queryRequestHeader("Content-Type");
+    if (val)
+        setContentType(val); //response type defaults to request type
+
+    val = queryRequestHeader("Authorization");
+    if (val && !strncmp(val, "Basic ", 6))
+        setAuthToken(val+6);
+
+}
+
 void HttpHelper::parseURL()
 {
     const char *start = url.str();

common/thorhelper/roxiehelper.hpp

@@ -31,6 +31,7 @@
 void parseHttpParameterString(IProperties *p, const char *str);
 
 enum class HttpMethod {NONE, GET, POST};
+enum class HttpCompression {NONE, GZIP, DEFLATE, ZLIB_DEFLATE};
 
 class THORHELPER_API HttpHelper : public CInterface
 {
@@ -45,6 +46,7 @@ private:
     StringArray *validTargets;
     Owned<IProperties> parameters;
     Owned<IProperties> form;
+    Owned<IProperties> reqHeaders;
 private:
     inline void setHttpHeaderValue(StringAttr &s, const char *v, bool ignoreExt)
     {
@@ -67,7 +69,39 @@ public:
         const char *control = queryTarget();
         return (control && strieq(control, "control"));
     }
-
+    inline HttpCompression getReqCompression()
+    {
+        const char *encoding = queryRequestHeader("Content-Encoding");
+        if (encoding)
+        {
+            if (strieq(encoding, "gzip") || strieq(encoding, "x-gzip"))
+                return HttpCompression::GZIP;
+            if (strieq(encoding, "deflate") || strieq(encoding, "x-deflate"))
+                return HttpCompression::DEFLATE;
+        }
+        return HttpCompression::NONE;
+    }
+    inline HttpCompression getRespCompression()
+    {
+        const char *encoding = queryRequestHeader("Accept-Encoding");
+        if (encoding)
+        {
+            StringArray encodingList;
+            encodingList.appendList(encoding, ",");
+            if (encodingList.contains("gzip"))
+                return HttpCompression::GZIP;
+            if (encodingList.contains("deflate"))
+                return HttpCompression::DEFLATE;
+            //The reason gzip is preferred is that deflate can mean either of two formats
+            //"x-deflate" isn't any clearer, but since either works either way, we can use the alternate name
+            //to our advantage.  Differentiating here just gives us a way of allowing clients to specify
+            //in case they can't handle one or the other (e.g. SOAPUI can't handle ZLIB_DEFLATE which I think
+            //is the "most proper" choice)
+            if (encodingList.contains("x-deflate"))
+                return HttpCompression::ZLIB_DEFLATE;
+        }
+        return HttpCompression::NONE;
+    }
     bool getUseEnvelope(){return useEnvelope;}
     void setUseEnvelope(bool _useEnvelope){useEnvelope=_useEnvelope;}
     bool getTrim() {return parameters->getPropBool(".trim", true); /*http currently defaults to true, maintain compatibility */}
@@ -89,6 +123,12 @@ public:
         return queryName.str();
     }
 
+    inline const char *queryRequestHeader(const char *header)
+    {
+        if (!reqHeaders)
+            return nullptr;
+        return reqHeaders->queryProp(header);
+    }
     inline void setAuthToken(const char *v)
     {
         setHttpHeaderValue(authToken, v, false);
@@ -107,6 +147,7 @@ public:
             parseURL();
         }
     }
+    void parseRequestHeaders(const char *headers);
     inline bool isFormPost()
     {
         return (strnicmp(queryContentType(), "application/x-www-form-urlencoded", strlen("application/x-www-form-urlencoded"))==0);
@@ -314,6 +355,7 @@ protected:
     bool heartbeat;
     bool adaptiveRoot = false;
     TextMarkupFormat mlResponseFmt = MarkupFmt_Unknown;
+    HttpCompression respCompression = HttpCompression::NONE;
     StringAttr contentHead;
     StringAttr contentTail;
     PointerArray queued;

common/thorhelper/thorherror.h

@@ -24,6 +24,7 @@
 #define THORHELPER_DEBUG_ERROR              (THORHELPER_ERROR_START + 0)
 #define THORHELPER_INTERNAL_ERROR           (THORHELPER_ERROR_START + 1)
 #define THORHELPER_DATA_ERROR               (THORHELPER_ERROR_START + 2)
+#define THORHELPER_UNSUPPORTED_ENCODING     (THORHELPER_ERROR_START + 3)
 
 //Errors with associated text
 #define THORCERR_InvalidXmlFromXml          (THORHELPER_ERROR_START + 50)

common/thorhelper/thorsoapcall.cpp

@@ -1404,6 +1404,10 @@ private:
     {
         if (strieq(encoding, "gzip"))
             return true;
+        if (strieq(encoding, "deflate"))
+            return true;
+        if (strieq(encoding, "x-deflate"))
+            return true;
         return false;
     }
 
@@ -1423,21 +1427,19 @@ private:
 
     void decodeContent(const char* contentEncodingType, StringBuffer& content)
     {
-        StringBuffer contentDecoded;
-        unsigned contentLength = content.length();
-        if (strieq(contentEncodingType, "gzip"))
-        {
 #ifdef _USE_ZLIB
-            gunzip((const byte*)content.str(), contentLength, contentDecoded);
-            PROGLOG("Content decoded from %d bytes to %d bytes", contentLength, contentDecoded.length());
-#else
-            throw MakeStringException(-1, "_USE_ZLIB is required for Content-Encoding:%s", contentEncodingType);
-#endif
-        }
+        unsigned contentLength = content.length();
+        StringBuffer contentDecoded;
 
+        httpInflate((const byte*)content.str(), contentLength, contentDecoded, strieq(contentEncodingType, "gzip"));
+        PROGLOG("Content decoded from %d bytes to %d bytes", contentLength, contentDecoded.length());
         content = contentDecoded;
+
         if (soapTraceLevel > 6 || master->logXML)
             master->logctx.CTXLOG("Content decoded. Original %s %d", CONTENT_LENGTH, contentLength);
+#else
+            throw MakeStringException(-1, "_USE_ZLIB is required for Content-Encoding:%s", contentEncodingType);
+#endif
     }
 
     bool checkContentEncoding(const char* httpheaders, StringBuffer& contentEncodingType)
@@ -1454,19 +1456,25 @@ private:
         return true;
     }
 
-    void encodeContent(const char* contentEncodingType, StringBuffer& content)
+    ZlibCompressionType getEncodeFormat(const char *name)
+    {
+        if (strieq(name, "gzip"))
+            return ZlibCompressionType::GZIP;
+        if (strieq(name, "x-deflate"))
+            return ZlibCompressionType::ZLIB_DEFLATE;
+        if (strieq(name, "deflate"))
+            return ZlibCompressionType::DEFLATE;
+        return ZlibCompressionType::GZIP; //already checked above, shouldn't be here
+    }
+
+    void encodeContent(const char* contentEncodingType, MemoryBuffer& mb)
     {
-        if (strieq(contentEncodingType, "gzip"))
-        {
 #ifdef _USE_ZLIB
-            unsigned outlen;
-            char* outbuf = gzip( xmlWriter.str(), xmlWriter.length(), &outlen, GZ_BEST_SPEED);
-            content.setBuffer(outlen+1, outbuf, outlen);
-            PROGLOG("Content encoded from %d bytes to %d bytes", xmlWriter.length(), outlen);
+        zlib_deflate(mb, xmlWriter.str(), xmlWriter.length(), GZ_BEST_SPEED, getEncodeFormat(contentEncodingType));
+        PROGLOG("Content encoded from %d bytes to %d bytes", xmlWriter.length(), mb.length());
 #else
-            throw MakeStringException(-1, "_USE_ZLIB is required for Content-Encoding:%s", contentEncodingType);
+        throw MakeStringException(-1, "_USE_ZLIB is required for Content-Encoding:%s", contentEncodingType);
 #endif
-        }
     }
 
     void logRequest(bool contentEncoded, StringBuffer& request)
@@ -1540,7 +1548,7 @@ private:
         {
             request.append("Host: ").append(url.host).append(":").append(url.port).append("\r\n");//http 1.1
 
-            StringBuffer contentEncodingType, encodedContentBuf;
+            StringBuffer contentEncodingType;
             if (!checkContentEncoding(httpheaders, contentEncodingType))
             {
                 request.append(CONTENT_LENGTH).append(xmlWriter.length()).append("\r\n\r\n");
@@ -1550,9 +1558,10 @@ private:
             else
             {
                 logRequest(true, request);
+                MemoryBuffer encodedContentBuf;
                 encodeContent(contentEncodingType.str(), encodedContentBuf);
                 request.append(CONTENT_LENGTH).append(encodedContentBuf.length()).append("\r\n\r\n");
-                request.append(encodedContentBuf.length(), encodedContentBuf.str());//add SOAP xml content
+                request.append(encodedContentBuf.length(), encodedContentBuf.toByteArray());
             }
         }
         else

system/security/zcrypt/zcrypt.cpp

@@ -1002,7 +1002,17 @@ ZCRYPT_API void releaseIZ(IZInterface* iz)
 
 }
 
-static void throwGZipException(const char* operation, int errorCode)
+inline const char *getZlibHeaderTypeName(ZlibCompressionType zltype)
+{
+    if (zltype==ZlibCompressionType::GZIP)
+        return "gzip";
+    if (zltype==ZlibCompressionType::ZLIB_DEFLATE)
+        return "zlib_deflate";
+    //DEFLATE, no header
+    return "deflate";
+}
+
+static void throwZlibException(const char* operation, int errorCode, ZlibCompressionType zltype)
 {
     const char* errorMsg;
     switch (errorCode)
@@ -1029,21 +1039,31 @@ static void throwGZipException(const char* operation, int errorCode)
             errorMsg = "Unknown exception";
             break;
     }
-    throw MakeStringException(500, "Exception in gzip %s: %s.", operation, errorMsg);
+    throw MakeStringException(500, "Exception in %s %s: %s.", getZlibHeaderTypeName(zltype), operation, errorMsg);
 }
 
-// Compress a character buffer using zlib in gzip format with given compression level
+inline int getWindowBits(ZlibCompressionType zltype)
+{
+    if (zltype==ZlibCompressionType::GZIP)
+        return 15+16;
+    if (zltype==ZlibCompressionType::ZLIB_DEFLATE)
+        return 15;
+    //DEFLATE, no header
+    return -15;
+}
+
+// Compress a character buffer using zlib in gzip/zlib_deflate format with given compression level
 //
-char* gzip( const char* inputBuffer, unsigned int inputSize, unsigned int* outlen, int compressionLevel)
+void zlib_deflate(MemoryBuffer &mb, const char* inputBuffer, unsigned int inputSize, int compressionLevel, ZlibCompressionType zltype)
 {
     if (inputBuffer == NULL || inputSize == 0)
-        throw MakeStringException(500, "gzip failed: input buffer is empty!");
+        throw MakeStringException(500, "%s failed: input buffer is empty!", gzip ? "gzip" : "zlib_deflate");
 
     /* Before we can begin compressing (aka "deflating") data using the zlib
      functions, we must initialize zlib. Normally this is done by calling the
      deflateInit() function; in this case, however, we'll use deflateInit2() so
-     that the compressed data will have gzip headers. This will make it easy to
-     decompress the data later using a tool like gunzip, WinZip, etc.
+     that the compressed data will have gzip headers if requested. This will make
+     it easy to decompress the data later using a tool like gunzip, WinZip, etc.
      deflateInit2() accepts many parameters, the first of which is a C struct of
      type "z_stream" defined in zlib.h. The properties of this struct are used to
      control how the compression algorithms work. z_stream is also used to
@@ -1078,9 +1098,9 @@ char* gzip( const char* inputBuffer, unsigned int inputSize, unsigned int* outle
                       Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data
                       produced by a filter (or predictor), or Z_HUFFMAN_ONLY to
                       force Huffman encoding only (no string match) */
-    int ret = deflateInit2(&zs, compressionLevel, Z_DEFLATED, (15+16), 8, Z_DEFAULT_STRATEGY);
+    int ret = deflateInit2(&zs, compressionLevel, Z_DEFLATED, getWindowBits(zltype), 8, Z_DEFAULT_STRATEGY);
     if (ret != Z_OK)
-        throwGZipException("initialization", ret);
+        throwZlibException("initialization", ret, zltype);
 
     // set the z_stream's input
     zs.next_in = (Bytef*)inputBuffer;
@@ -1089,7 +1109,7 @@ char* gzip( const char* inputBuffer, unsigned int inputSize, unsigned int* outle
     // Create output memory buffer for compressed data. The zlib documentation states that
     // destination buffer size must be at least 0.1% larger than avail_in plus 12 bytes.
     const unsigned long outsize = (unsigned long)floorf((float)inputSize * 1.01f) + 12;
-    Bytef* outbuf = new Bytef[outsize];
+    Bytef* outbuf = (Bytef*) mb.reserveTruncate(outsize);
 
     do
     {
@@ -1112,35 +1132,38 @@ char* gzip( const char* inputBuffer, unsigned int inputSize, unsigned int* outle
         ret = deflate(&zs, Z_FINISH);
     } while (ret == Z_OK);
 
+    // Free data structures that were dynamically created for the stream.
+    deflateEnd(&zs);
+
     if (ret != Z_STREAM_END)          // an error occurred that was not EOS
     {
-        // Free data structures that were dynamically created for the stream.
-        deflateEnd(&zs);
-        delete[] outbuf;
-        *outlen = 0;
-        throwGZipException("compression", ret);
+        mb.clear();
+        throwZlibException("compression", ret, zltype);
     }
 
-    // Free data structures that were dynamically created for the stream.
-    deflateEnd(&zs);
-    *outlen = zs.total_out;
-    return (char*) outbuf;
+    mb.setLength(zs.total_out);
 }
 
-void gunzip(const byte* compressed, unsigned int comprLen, StringBuffer& sOutput)
+// Compress a character buffer using zlib in gzip format with given compression level
+//
+void gzip(MemoryBuffer &mb, const char* inputBuffer, unsigned int inputSize, int compressionLevel)
+{
+    zlib_deflate(mb, inputBuffer, inputSize, compressionLevel, ZlibCompressionType::GZIP);
+}
+
+bool zlib_inflate(const byte* compressed, unsigned int comprLen, StringBuffer& sOutput, ZlibCompressionType zltype, bool inflateException)
 {
     if (comprLen == 0)
-        return;
+        return true;
 
     const int CHUNK_OUT = 16384;
     z_stream d_stream; // decompression stream
     memset( &d_stream, 0, sizeof(z_stream));
     d_stream.next_in = (byte*) compressed;
     d_stream.avail_in = comprLen;
-
-    int ret = inflateInit2(&d_stream, (15+16));
+    int ret = inflateInit2(&d_stream, getWindowBits(zltype));
     if (ret != Z_OK)
-        throwGZipException("initialization", ret);
+        throwZlibException("initialization", ret, zltype); //don't ignore this
 
     unsigned int outLen = 0;
 
@@ -1163,7 +1186,27 @@ void gunzip(const byte* compressed, unsigned int comprLen, StringBuffer& sOutput
     if (ret != Z_STREAM_END)
     {
         sOutput.clear();
-        throwGZipException("decompression", ret);
+        if (!inflateException)
+            return false;
+        throwZlibException("decompression", ret, zltype);
+    }
+    return true;
+}
+
+void gunzip(const byte* compressed, unsigned int comprLen, StringBuffer& sOutput)
+{
+    zlib_inflate(compressed, comprLen, sOutput, ZlibCompressionType::GZIP, true);
+}
+
+void httpInflate(const byte* compressed, unsigned int comprLen, StringBuffer& sOutput, bool use_gzip)
+{
+    if (use_gzip)
+    {
+        zlib_inflate(compressed, comprLen, sOutput, ZlibCompressionType::GZIP, true);
+    }
+    else if (!zlib_inflate(compressed, comprLen, sOutput, ZlibCompressionType::ZLIB_DEFLATE, false)) //this is why gzip is preferred, deflate can mean 2 things
+    {
+        zlib_inflate(compressed, comprLen, sOutput, ZlibCompressionType::DEFLATE, true);
     }
 }
 

system/security/zcrypt/zcrypt.hpp

@@ -19,6 +19,7 @@
 #define ZCRYPT_HPP__
 
 #include "platform.h"
+#include "jbuff.hpp"
 
 #ifndef ZCRYPT_API
 #ifndef ZCRYPT_EXPORTS
@@ -94,9 +95,12 @@ ZCRYPT_API void releaseIZ(IZInterface* iz);
 class StringBuffer;
 typedef unsigned char byte;
 
-// Compress a character buffer using zlib in gzip format with given compression level
-ZCRYPT_API char* gzip( const char* inputBuffer, unsigned int inputSize,
-    unsigned int* outlen, int compressionLevel=GZ_DEFAULT_COMPRESSION);
+enum class ZlibCompressionType {GZIP, ZLIB_DEFLATE, DEFLATE};
+
+ZCRYPT_API void zlib_deflate(MemoryBuffer &mb, const char* inputBuffer, unsigned int inputSize, int compressionLevel, ZlibCompressionType zltype);
+ZCRYPT_API void gzip(MemoryBuffer &mb, const char* inputBuffer, unsigned int inputSize, int compressionLevel=GZ_DEFAULT_COMPRESSION);
+
+ZCRYPT_API void httpInflate(const byte* compressed, unsigned int comprLen, StringBuffer& sOutput, bool use_gzip);
 ZCRYPT_API void gunzip(const byte* compressed, unsigned int comprLen, StringBuffer& sOutput);
 ZCRYPT_API bool isgzipped(const byte* content, size_t length);
 ZCRYPT_API void removeZipExtension(StringBuffer & target, const char * source);

testing/regress/ecl/roxiegzip.ecl

@@ -0,0 +1,110 @@
+//nothor
+//nohthor
+
+NameRec := RECORD
+  string First;
+  string Last;
+END;
+
+AddressRec := RECORD
+  string City;
+  string State;
+  integer ZipCode;
+END;
+
+PersonRec := RECORD
+  NameRec Name;
+  AddressRec Address;
+END;
+
+peeps_send := DATASET([
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}},
+  {{'Joeseph', 'Johnson'}, {'Fresno', 'CA', 11111}}
+], PersonRec);
+
+roxieEchoTestRequestRecord := RECORD
+  DATASET(PersonRec) Peeps {XPATH('Peeps/Row')} := peeps_send;
+END;
+
+exceptionRec := RECORD
+  string Source {xpath('Source')};
+  integer Code {xpath('Code')};
+  string Message {xpath('Message')};
+END;
+
+roxieEchoTestResponseRecord := RECORD
+  DATASET(PersonRec) Peeps {XPATH('Dataset/Row')} := DATASET([], PersonRec);
+  exceptionRec Exception {XPATH('Exception')};
+END;
+
+roxieEchoTestResponseRecord doFail() := TRANSFORM
+  self.Exception.CODE := IF (FAILCODE=0, 0, ERROR(FAILCODE, FAILMESSAGE));
+  self.Exception.Message := FAILMESSAGE;
+  self.Exception.Source := 'Test';
+END;
+
+string TargetIP := '.' : stored('TargetIP');
+string TargetURL := 'http://' + TargetIP + ':9876';
+
+gzipResult := SOAPCALL(TargetURL, 'roxie_echo', roxieEchoTestRequestRecord,
+    DATASET(roxieEchoTestResponseRecord),
+    LITERAL,
+    XPATH('*/Results/Result'),
+    RESPONSE(NOTRIM),
+    HTTPHEADER('Content-Encoding', 'gzip'),
+    HTTPHEADER('Accept-Encoding', 'gzip'),
+    ONFAIL(doFail()));
+
+OUTPUT(gzipResult, NAMED('gzipResult'));
+
+deflateResult := SOAPCALL(TargetURL, 'roxie_echo', roxieEchoTestRequestRecord,
+    DATASET(roxieEchoTestResponseRecord),
+    LITERAL,
+    XPATH('*/Results/Result'),
+    RESPONSE(NOTRIM),
+    HTTPHEADER('Content-Encoding', 'deflate'),
+    HTTPHEADER('Accept-Encoding', 'deflate'),
+    ONFAIL(doFail()));
+
+OUTPUT(deflateResult, NAMED('deflateResult'));
+
+xdeflateResult := SOAPCALL(TargetURL, 'roxie_echo', roxieEchoTestRequestRecord,
+    DATASET(roxieEchoTestResponseRecord),
+    LITERAL,
+    XPATH('*/Results/Result'),
+    RESPONSE(NOTRIM),
+    HTTPHEADER('Content-Encoding', 'x-deflate'),
+    HTTPHEADER('Accept-Encoding', 'x-deflate'),
+    ONFAIL(doFail()));
+
+OUTPUT(xdeflateResult, NAMED('xdeflatepResult'));
+