libgis: add ZSTD (Zstandard) compression

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@71890 15284696-431f-4ddb-bdfa-cd5b030d7da7

include/defs/gis.h

@@ -191,10 +191,12 @@ void G_remove_commas(char *);
 /* compress.c */
 int G_compressor_number(char *);
 char *G_compressor_name(int);
+int G_default_compressor(void);
 int G_check_compressor(int);
 int G_write_compressed(int, unsigned char *, int, int);
 int G_write_unompressed(int, unsigned char *, int);
 int G_read_compressed(int, int, unsigned char *, int, int);
+int G_compress_bound(int, int);
 int G_compress(unsigned char *, int, unsigned char *, int, int);
 int G_expand(unsigned char *, int, unsigned char *, int, int);
 
@@ -238,6 +240,14 @@ int
 G_bz2_expand(unsigned char *src, int src_sz, unsigned char *dst,
 	      int dst_sz);
 
+/* cmprzstd.c : ZSTD, compression similar to ZLIB's DEFLATE but faster */
+int
+G_zstd_compress(unsigned char *src, int src_sz, unsigned char *dst,
+		int dst_sz);
+int
+G_zstd_expand(unsigned char *src, int src_sz, unsigned char *dst,
+	      int dst_sz);
+
 /* add more compression methods here */
 
 /* copy_dir.c */

lib/gis/cmprbzip.c

@@ -71,11 +71,26 @@
 
 
 int
+G_bz2_compress_bound(int src_sz)
+{
+    /* from the documentation:
+     * To guarantee that the compressed data will fit in its buffer, 
+     * allocate an output buffer of size 1% larger than the uncompressed data, 
+     * plus six hundred extra bytes.
+     * bzip2 does not provide a compressbound fn
+     * and apparently does not have a fast version if destLen is
+     * large enough to hold a worst case result
+     */
+    return src_sz;
+}
+
+int
 G_bz2_compress(unsigned char *src, int src_sz, unsigned char *dst,
 		int dst_sz)
 {
     int err;
-    unsigned int i, nbytes, buf_sz;
+    int i, buf_sz;
+    unsigned int nbytes;
     unsigned char *buf;
 
 #ifndef HAVE_BZLIB_H
@@ -92,11 +107,16 @@ G_bz2_compress(unsigned char *src, int src_sz, unsigned char *dst,
 	return 0;
 
     /* Output buffer has to be 1% + 600 bytes bigger for single pass compression */
-    buf_sz = (unsigned int)((double)dst_sz * 1.01 + (double)600);
-
-    if (NULL == (buf = (unsigned char *)
-		 G_calloc(buf_sz, sizeof(unsigned char))))
-	return -1;
+    buf = dst;
+    buf_sz = G_bz2_compress_bound(src_sz);
+    if (buf_sz > dst_sz) {
+	G_warning("G_bz2_compress(): programmer error, destination is too small");
+	if (NULL == (buf = (unsigned char *)
+		     G_calloc(buf_sz, sizeof(unsigned char))))
+	    return -1;
+    }
+    else
+	buf_sz = dst_sz;
 
     /* Do single pass compression */
     nbytes = buf_sz;
@@ -105,27 +125,28 @@ G_bz2_compress(unsigned char *src, int src_sz, unsigned char *dst,
 				   9,			 /* blockSize100k */ 
 				   0,                    /* verbosity */
 				   100);                 /* workFactor */
+
     if (err != BZ_OK) {
-	G_free(buf);
+	if (buf != dst)
+	    G_free(buf);
 	return -1;
     }
 
     /* updated buf_sz is bytes of compressed data */
     if (nbytes >= (unsigned int)src_sz) {
 	/* compression not possible */
-	G_free(buf);
+	if (buf != dst)
+	    G_free(buf);
 	return -2;
     }
 
-    /* dst too small */
-    if ((unsigned int)dst_sz < nbytes)
-	return -2;
+    if (buf != dst) {
+	/* Copy the data from buf to dst */
+	for (i = 0; i < nbytes; i++)
+	    dst[i] = buf[i];
 
-    /* Copy the data from buf to dst */
-    for (i = 0; i < nbytes; i++)
-	dst[i] = buf[i];
-
-    G_free(buf);
+	G_free(buf);
+    }
 
     return nbytes;
 #endif
@@ -159,11 +180,15 @@ G_bz2_expand(unsigned char *src, int src_sz, unsigned char *dst,
 				     0,                     /* small */
 				     0);                    /* verbosity */
 
+    if (err != BZ_OK) {
+	return -1;
+    }
+
     /* Number of bytes inflated to output stream is
      * updated buffer size
      */
 
-    if (!(err == BZ_OK)) {
+    if (nbytes != dst_sz) {
 	return -1;
     }
 

lib/gis/cmprlz4.c

@@ -26,7 +26,7 @@
  *     int src_sz, dst_sz;                                          *
  *     unsigned char *src, *dst;                                    *
  * ---------------------------------------------------------------- *
- * This function is a wrapper around the LZ4 cimpression function.  *
+ * This function is a wrapper around the LZ4 compression function.  *
  * It uses an all or nothing call.                                  *
  * If you need a continuous compression scheme, you'll have to code *
  * your own.                                                        *
@@ -67,6 +67,14 @@
 
 #include "lz4.h"
 
+int
+G_lz4_compress_bound(int src_sz)
+{
+    /* LZ4 has a fast version if destLen is large enough 
+     * to hold a worst case result
+     */
+    return LZ4_compressBound(src_sz);
+}
 
 int
 G_lz4_compress(unsigned char *src, int src_sz, unsigned char *dst,
@@ -83,32 +91,43 @@ G_lz4_compress(unsigned char *src, int src_sz, unsigned char *dst,
     if (src_sz <= 0 || dst_sz <= 0)
 	return 0;
 
-    /* Output buffer has to be larger for single pass compression */
-    buf_sz = LZ4_compressBound(src_sz);
-    if (NULL == (buf = (unsigned char *)
-		 G_calloc(buf_sz, sizeof(unsigned char))))
-	return -1;
+    /* Output buffer should be large enough for single pass compression */
+    buf = dst;
+    buf_sz = G_lz4_compress_bound(src_sz);
+    if (buf_sz > dst_sz) {
+	G_warning("G_lz4_compress(): programmer error, destination is too small");
+	if (NULL == (buf = (unsigned char *)
+		     G_calloc(buf_sz, sizeof(unsigned char))))
+	    return -1;
+    }
+    else
+	buf_sz = dst_sz;
 
     /* Do single pass compression */
     err = LZ4_compress_default((char *)src, (char *)buf, src_sz, buf_sz);
+
     if (err <= 0) {
-	G_free(buf);
+	if (buf != dst)
+	    G_free(buf);
 	return -1;
     }
     if (err >= src_sz) {
 	/* compression not possible */
-	G_free(buf);
+	if (buf != dst)
+	    G_free(buf);
 	return -2;
     }
     
     /* bytes of compressed data is return value */
     nbytes = err;
 
-    /* Copy the data from buf to dst */
-    for (err = 0; err < nbytes; err++)
-	dst[err] = buf[err];
+    if (buf != dst) {
+	/* Copy the data from buf to dst */
+	for (err = 0; err < nbytes; err++)
+	    dst[err] = buf[err];
 
-    G_free(buf);
+	G_free(buf);
+    }
 
     return nbytes;
 }

lib/gis/cmprrle.c

@@ -63,6 +63,12 @@
 #include <grass/gis.h>
 #include <grass/glocale.h>
 
+/* no fast mode if destination is large enough to hold 
+ * worst case compression */ 
+int G_rle_compress_bound(int src_sz)
+{
+    return ((src_sz >> 1) * 3 + (src_sz & 1));
+}
 
 int
 G_rle_compress(unsigned char *src, int src_sz, unsigned char *dst,
@@ -188,5 +194,4 @@ G_rle_expand(unsigned char *src, int src_sz, unsigned char *dst,
     return nbytes;
 }
 
-
 /* vim: set softtabstop=4 shiftwidth=4 expandtab: */

lib/gis/cmprzlib.c

@@ -74,24 +74,25 @@
 
 #include "G.h"
 
-static void _init_zstruct(z_stream * z)
+
+int
+G_zlib_compress_bound(int src_sz)
 {
-    /* The types are defined in zlib.h, we set to NULL so zlib uses
-     * its default functions.
+    /* from zlib.h:
+     * "when using compress or compress2, 
+     * destLen must be at least the value returned by
+     * compressBound(sourceLen)"
+     * no explanation for the "must be"
      */
-    z->zalloc = (alloc_func) 0;
-    z->zfree = (free_func) 0;
-    z->opaque = (voidpf) 0;
+    return compressBound(src_sz);
 }
 
-
 int
 G_zlib_compress(unsigned char *src, int src_sz, unsigned char *dst,
 		int dst_sz)
 {
-    int err, nbytes, buf_sz;
+    uLong err, nbytes, buf_sz;
     unsigned char *buf;
-    z_stream c_stream;
 
     /* Catch errors early */
     if (src == NULL || dst == NULL)
@@ -103,65 +104,50 @@ G_zlib_compress(unsigned char *src, int src_sz, unsigned char *dst,
 
     /* Output buffer has to be 1% + 12 bytes bigger for single pass deflate */
     /* buf_sz = (int)((double)dst_sz * 1.01 + (double)12); */
-    buf_sz = compressBound(src_sz);
-    if (NULL == (buf = (unsigned char *)
-		 G_calloc(buf_sz, sizeof(unsigned char))))
-	return -1;
 
-    /* Set-up for default zlib memory handling */
-    _init_zstruct(&c_stream);
-
-    /* Set-up the stream */
-    c_stream.avail_in = src_sz;
-    c_stream.next_in = (unsigned char *) src;
-    c_stream.avail_out = buf_sz;
-    c_stream.next_out = buf;
+    /* Output buffer should be large enough for single pass compression */
+    buf = dst;
+    buf_sz = G_zlib_compress_bound(src_sz);
+    if (buf_sz > dst_sz) {
+	G_warning("G_zlib_compress(): programmer error, destination is too small");
+	if (NULL == (buf = (unsigned char *)
+		     G_calloc(buf_sz, sizeof(unsigned char))))
+	    return -1;
+    }
+    else
+	buf_sz = dst_sz;
 
-    /* Initialize */
     /* Valid zlib compression levels -1 - 9 */
     /* zlib default: Z_DEFAULT_COMPRESSION = -1, equivalent to 6 
      * as used here, 1 gives the best compromise between speed and compression */
-    err = deflateInit(&c_stream, G__.compression_level);
-
-    /* If there was an error initializing, return -1 */
-    if (err != Z_OK) {
-	G_free(buf);
-	return -1;
-    }
 
     /* Do single pass compression */
-    err = deflate(&c_stream, Z_FINISH);
-    if (err != Z_STREAM_END) {
-	switch (err) {
-	case Z_OK:		/* Destination too small */
-	    G_free(buf);
-	    deflateEnd(&c_stream);
-	    return -2;
-	    break;
-	default:		/* Give other error */
+    nbytes = buf_sz;
+    err = compress2((Bytef *)buf, &nbytes, 	 /* destination */
+		    (const Bytef *)src, src_sz,  /* source */
+		    G__.compression_level); 	 /* level */
+
+    if (err != Z_OK) {
+	if (buf != dst)
 	    G_free(buf);
-	    deflateEnd(&c_stream);
-	    return -1;
-	    break;
-	}
+	return -1;
     }
 
-    /* avail_out is updated to bytes remaining in buf, so bytes of compressed
-     * data is the original size minus that
-     */
-    nbytes = buf_sz - c_stream.avail_out;
+    /* updated buf_sz is bytes of compressed data */
     if (nbytes >= src_sz) {
 	/* compression not possible */
-	G_free(buf);
-	deflateEnd(&c_stream);
+	if (buf != dst)
+	    G_free(buf);
 	return -2;
     }
-    /* Copy the data from buf to dst */
-    for (err = 0; err < nbytes; err++)
-	dst[err] = buf[err];
 
-    G_free(buf);
-    deflateEnd(&c_stream);
+    if (buf != dst) {
+	/* Copy the data from buf to dst */
+	for (err = 0; err < nbytes; err++)
+	    dst[err] = buf[err];
+
+	G_free(buf);
+    }
 
     return nbytes;
 }				/* G_zlib_compress() */
@@ -171,8 +157,8 @@ int
 G_zlib_expand(unsigned char *src, int src_sz, unsigned char *dst,
 	      int dst_sz)
 {
-    int err, nbytes;
-    z_stream c_stream;
+    int err;
+    uLong ss, nbytes;
 
     /* Catch error condition */
     if (src == NULL || dst == NULL)
@@ -182,45 +168,25 @@ G_zlib_expand(unsigned char *src, int src_sz, unsigned char *dst,
     if (src_sz <= 0 || dst_sz <= 0)
 	return 0;
 
-    /* Set-up default zlib memory handling */
-    _init_zstruct(&c_stream);
-
-    /* Set-up I/O streams */
-    c_stream.avail_in = src_sz;
-    c_stream.next_in = (unsigned char *)src;
-    c_stream.avail_out = dst_sz;
-    c_stream.next_out = dst;
+    ss = src_sz;
 
-    /* Call zlib initialization function */
-    err = inflateInit(&c_stream);
+    /* Do single pass decompression */
+    nbytes = dst_sz;
+    err = uncompress((Bytef *)dst, &nbytes,  /* destination */
+		     (const Bytef *)src, ss);   /* source */
 
     /* If not Z_OK return error -1 */
     if (err != Z_OK)
 	return -1;
 
-    /* Do single pass inflate */
-    err = inflate(&c_stream, Z_FINISH);
-
     /* Number of bytes inflated to output stream is
-     * original bytes available minus what avail_out now says
+     * updated buffer size
      */
-    nbytes = dst_sz - c_stream.avail_out;
 
-    /* Z_STREAM_END means all input was consumed, 
-     * Z_OK means only some was processed (not enough room in dst)
-     */
-    if (!(err == Z_STREAM_END || err == Z_OK)) {
-	if (!(err == Z_BUF_ERROR && nbytes == dst_sz)) {
-	    inflateEnd(&c_stream);
-	    return -1;
-	}
-	/* Else, there was extra input, but requested output size was
-	 * decompressed successfully.
-	 */
+    if (nbytes != dst_sz) {
+	return -1;
     }
 
-    inflateEnd(&c_stream);
-
     return nbytes;
 }				/* G_zlib_expand() */
 

lib/gis/cmprzstd.c

@@ -0,0 +1,184 @@
+/*
+ ****************************************************************************
+ *                     -- GRASS Development Team --
+ *
+ * MODULE:      GRASS gis library
+ * FILENAME:    cmprlz4.c
+ * AUTHOR(S):   Eric G. Miller <egm2@jps.net>
+ *              Markus Metz
+ * PURPOSE:     To provide an interface to lz4 for compressing and 
+ *              decompressing data using LZ$.  It's primary use is in
+ *              the storage and reading of GRASS floating point rasters.
+ *
+ * ALGORITHM:   https://code.google.com/p/lz4/
+ * DATE CREATED: Dec 18 2015
+ * COPYRIGHT:   (C) 2015 by the GRASS Development Team
+ *
+ *              This program is free software under the GNU General Public
+ *              License (version 2 or greater). Read the file COPYING that 
+ *              comes with GRASS for details.
+ *
+ *****************************************************************************/
+
+/********************************************************************
+ * int                                                              *
+ * G_zstd_compress (src, srz_sz, dst, dst_sz)                       *
+ *     int src_sz, dst_sz;                                          *
+ *     unsigned char *src, *dst;                                    *
+ * ---------------------------------------------------------------- *
+ * This function is a wrapper around the Zstd compression function. *
+ * It uses an all or nothing call.                                  *
+ * If you need a continuous compression scheme, you'll have to code *
+ * your own.                                                        *
+ * In order to do a single pass compression, the input src must be  *
+ * copied to a buffer larger than the data.  This may cause         *
+ * performance degradation.                                         *
+ *                                                                  *
+ * The function either returns the number of bytes of compressed    *
+ * data in dst, or an error code.                                   *
+ *                                                                  *
+ * Errors include:                                                  *
+ *        -1 -- Compression failed.                                 *
+ *        -2 -- dst is too small.                                   *
+ *                                                                  *
+ * ================================================================ *
+ * int                                                              *
+ * G_zstd_expand (src, src_sz, dst, dst_sz)                         *
+ *     int src_sz, dst_sz;                                          *
+ *     unsigned char *src, *dst;                                    *
+ * ---------------------------------------------------------------- *
+ * This function is a wrapper around the zstd decompression          *
+ * function.  It uses a single pass call.  If you need a continuous *
+ * expansion scheme, you'll have to code your own.                  *
+ *                                                                  *
+ * The function returns the number of bytes expanded into 'dst' or  *
+ * and error code.                                                  *
+ *                                                                  *
+ * Errors include:                                                  *
+ *        -1 -- Expansion failed.                                   *
+ *                                                                  *
+ ********************************************************************
+ */
+
+#include <grass/config.h>
+
+#ifdef HAVE_ZSTD_H
+#include <zstd.h>
+#endif
+
+#include <grass/gis.h>
+#include <grass/glocale.h>
+
+
+int
+G_zstd_compress_bound(int src_sz)
+{
+    /* ZSTD has a fast version if destLen is large enough 
+     * to hold a worst case result
+     */
+#ifndef HAVE_ZSTD_H
+    G_fatal_error(_("GRASS needs to be compiled with ZSTD for ZSTD compression"));
+    return -1;
+#else
+    return ZSTD_compressBound(src_sz);
+#endif
+}
+
+int
+G_zstd_compress(unsigned char *src, int src_sz, unsigned char *dst,
+		int dst_sz)
+{
+    int err, nbytes, buf_sz;
+    unsigned char *buf;
+
+#ifndef HAVE_ZSTD_H
+    G_fatal_error(_("GRASS needs to be compiled with ZSTD for ZSTD compression"));
+    return -1;
+#else
+
+    /* Catch errors early */
+    if (src == NULL || dst == NULL)
+	return -1;
+
+    /* Don't do anything if either of these are true */
+    if (src_sz <= 0 || dst_sz <= 0)
+	return 0;
+
+    /* Output buffer has to be larger for single pass compression */
+    buf = dst;
+    buf_sz = G_zstd_compress_bound(src_sz);
+    if (buf_sz > dst_sz) {
+	G_warning("G_zstd_compress(): programmer error, destination is too small");
+	if (NULL == (buf = (unsigned char *)
+		     G_calloc(buf_sz, sizeof(unsigned char))))
+	    return -1;
+    }
+    else
+	buf_sz = dst_sz;
+
+    /* Do single pass compression */
+    err = ZSTD_compress((char *)buf, buf_sz, (char *)src, src_sz, 3);
+
+    if (err <= 0 || ZSTD_isError(err)) {
+	if (buf != dst)
+	    G_free(buf);
+	return -1;
+    }
+    if (err >= src_sz) {
+	/* compression not possible */
+	if (buf != dst)
+	    G_free(buf);
+	return -2;
+    }
+    
+    /* bytes of compressed data is return value */
+    nbytes = err;
+
+    if (buf != dst) {
+	/* Copy the data from buf to dst */
+	for (err = 0; err < nbytes; err++)
+	    dst[err] = buf[err];
+
+	G_free(buf);
+    }
+
+    return nbytes;
+#endif
+}
+
+int
+G_zstd_expand(unsigned char *src, int src_sz, unsigned char *dst,
+	      int dst_sz)
+{
+    int err, nbytes;
+
+#ifndef HAVE_ZSTD_H
+    G_fatal_error(_("GRASS needs to be compiled with ZSTD for ZSTD compression"));
+    return -1;
+#else
+
+    /* Catch error condition */
+    if (src == NULL || dst == NULL)
+	return -2;
+
+    /* Don't do anything if either of these are true */
+    if (src_sz <= 0 || dst_sz <= 0)
+	return 0;
+
+    /* Do single pass decompress */
+    err = ZSTD_decompress((char *)dst, dst_sz, (char *)src, src_sz);
+    /* err = LZ4_decompress_fast(src, dst, src_sz); */
+
+    /* Number of bytes inflated to output stream is return value */
+    nbytes = err;
+
+    if (nbytes != dst_sz) {
+	return -1;
+    }
+
+    return nbytes;
+#endif
+}
+
+
+/* vim: set softtabstop=4 shiftwidth=4 expandtab: */

lib/gis/compress.c

@@ -24,6 +24,7 @@
  * 2 : ZLIB's DEFLATE (good speed and compression)                  *
  * 3 : LZ4 (fastest, low compression)                               *
  * 4 : BZIP2 (slowest, high compression)                            *
+ * 5 : ZSTD (faster than ZLIB, higher compression than ZLIB)        *
  *                                                                  *
  * int                                                              *
  * G_read_compressed (fd, rbytes, dst, nbytes, compression_type)    *
@@ -121,6 +122,16 @@ char *G_compressor_name(int number)
     return compressor[number].name;
 }
 
+int G_default_compressor(void)
+{
+#ifdef HAVE_ZSTD_H
+    /* ZSTD */
+    return 5;
+#endif
+    /* ZLIB */
+    return 2;
+}
+
 /* check compressor number
  * return -1 on error
  * return 0 known but not available
@@ -135,6 +146,11 @@ int G_check_compressor(int number)
     return compressor[number].available;
 }
 
+int G_no_compress_bound(int src_sz)
+{
+    return src_sz;
+}
+
 int
 G_no_compress(unsigned char *src, int src_sz, unsigned char *dst,
 		int dst_sz)
@@ -179,6 +195,22 @@ G_no_expand(unsigned char *src, int src_sz, unsigned char *dst,
     return src_sz;
 }
 
+/* G_*_compress_bound() returns an upper bound on the compressed size
+ * which can be larger than the input size
+ * some compressors are a bit faster if the size of the destination
+ * is at least the upper bound (no need to test for buffer overlflow)
+ * read comments on the specific compressor interfaces 
+ */
+int G_compress_bound(int src_sz, int number)
+{
+    if (number < 0 || number >= n_compressors) {
+	G_fatal_error(_("Request for unsupported compressor"));
+	return -1;
+    }
+
+    return compressor[number].bound(src_sz);
+}
+
 /* G_*_compress() returns
  * > 0: number of bytes in dst
  * 0: nothing done
@@ -214,7 +246,7 @@ G_expand(unsigned char *src, int src_sz, unsigned char *dst,
 }
 
 int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes,
-                      int compressor)
+                      int number)
 {
     int bsize, nread, err;
     unsigned char *b;
@@ -262,7 +294,7 @@ int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes,
     /* Just call G_expand() with the buffer we read,
      * Account for first byte being a flag
      */
-    err = G_expand(b + 1, bsize - 1, dst, nbytes, compressor);
+    err = G_expand(b + 1, bsize - 1, dst, nbytes, number);
 
     /* We're done with b */
     G_free(b);
@@ -273,7 +305,7 @@ int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes,
 }				/* G_read_compressed() */
 
 int G_write_compressed(int fd, unsigned char *src, int nbytes,
-                       int compressor)
+                       int number)
 {
     int dst_sz, nwritten, err;
     unsigned char *dst, compressed;
@@ -282,19 +314,20 @@ int G_write_compressed(int fd, unsigned char *src, int nbytes,
     if (src == NULL || nbytes < 0)
 	return -1;
 
-    dst_sz = nbytes;
+    /* get upper bound of compressed size */
+    dst_sz = G_compress_bound(nbytes, number);
     if (NULL == (dst = (unsigned char *)
 		 G_calloc(dst_sz, sizeof(unsigned char))))
 	return -1;
 
     /* Now just call G_compress() */
-    err = G_compress(src, nbytes, dst, dst_sz, compressor);
+    err = G_compress(src, nbytes, dst, dst_sz, number);
 
     /* If compression succeeded write compressed row,
      * otherwise write uncompressed row. Compression will fail
      * if dst is too small (i.e. compressed data is larger)
      */
-    if (err > 0 && err <= dst_sz) {
+    if (err > 0 && err < nbytes) {
 	dst_sz = err;
 	/* Write the compression flag */
 	compressed = G_COMPRESSED_YES;

lib/gis/compress.h

@@ -7,25 +7,36 @@
  * 2: ZLIB's DEFLATE (default)
  * 3: LZ4, fastest but lowest compression ratio
  * 4: BZIP2: slowest but highest compression ratio
+ * 5: ZSTD: faster than ZLIB, higher compression than ZLIB
  */
 
 /* adding a new compressor:
  * add the corresponding functions G_*compress() and G_*_expand()
  * if needed, add checks to configure.in and include/config.in
- * modify compress.h
- * modify G_compress(), G_expand()
+ * modify compress.h (this file)
+ * nothing to change in compress.c
  */
 
+/* upper bounds of the size of the compressed buffer */
+int G_no_compress_bound(int);
+int G_rle_compress_bound(int);
+int G_zlib_compress_bound(int);
+int G_lz4_compress_bound(int);
+int G_bz2_compress_bound(int);
+int G_zstd_compress_bound(int);
+
 typedef int compress_fn(unsigned char *src, int src_sz, unsigned char *dst,
 		int dst_sz);
 typedef int expand_fn(unsigned char *src, int src_sz, unsigned char *dst,
 	      int dst_sz);
+typedef int bound_fn(int src_sz);
 
 struct compressor_list
 {
     int available;
     compress_fn *compress;
     expand_fn *expand;
+    bound_fn *bound;
     char *name;
 };
 
@@ -34,20 +45,27 @@ struct compressor_list
  * 1: RLE
  * 2: ZLIB
  * 3: LZ4
- * 4: BZIP2 */
+ * 4: BZIP2
+ * 5: ZSTD
+ */
  
-static int n_compressors = 5; 
+static int n_compressors = 6; 
 
 struct compressor_list compressor[] = {
-    {1, G_no_compress, G_no_expand, "NONE"},
-    {1, G_rle_compress, G_rle_expand, "RLE"},
-    {1, G_zlib_compress, G_zlib_expand, "ZLIB"},
-    {1, G_lz4_compress, G_lz4_expand, "LZ4"},
+    {1, G_no_compress, G_no_expand, G_no_compress_bound, "NONE"},
+    {1, G_rle_compress, G_rle_expand, G_rle_compress_bound, "RLE"},
+    {1, G_zlib_compress, G_zlib_expand, G_zlib_compress_bound, "ZLIB"},
+    {1, G_lz4_compress, G_lz4_expand, G_lz4_compress_bound, "LZ4"},
 #ifdef HAVE_BZLIB_H
-    {1, G_bz2_compress, G_bz2_expand, "BZIP2"},
+    {1, G_bz2_compress, G_bz2_expand, G_bz2_compress_bound, "BZIP2"},
+#else
+    {0, G_bz2_compress, G_bz2_expand, G_bz2_compress_bound, "BZIP2"},
+#endif
+#ifdef HAVE_ZSTD_H
+    {1, G_zstd_compress, G_zstd_expand, G_zstd_compress_bound, "ZSTD"},
 #else
-    {0, G_bz2_compress, G_bz2_expand, "BZIP2"},
+    {0, G_zstd_compress, G_zstd_expand, G_zstd_compress_bound, "ZSTD"},
 #endif
-    {0, NULL, NULL, NULL}
+    {0, NULL, NULL, NULL, NULL}
 };