Support compressed null bitmap (issue https://trac.osgeo.org/grass/ticket/2349)

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@65272 15284696-431f-4ddb-bdfa-cd5b030d7da7

include/defs/raster.h

@@ -266,7 +266,9 @@ void Rast_abs_log_colors(struct Colors *, struct Colors *, int);
 /* format.c */
 int Rast__check_format(int);
 int Rast__read_row_ptrs(int);
+int Rast__read_null_row_ptrs(int, int);
 int Rast__write_row_ptrs(int);
+int Rast__write_null_row_ptrs(int, int);
 
 /* fpreclass.c */
 void Rast_fpreclass_clear(struct FPReclass *);

lib/raster/R.h

@@ -67,6 +67,7 @@ struct fileinfo			/* Information for opened cell files */
     struct Quant quant;
     struct GDAL_link *gdal;
     int data_fd;		/* Raster data fd               */
+    off_t *null_row_ptr;	/* Null file row addresses      */
 };
 
 struct R__			/*  Structure of library globals */
@@ -77,6 +78,7 @@ struct R__			/*  Structure of library globals */
     int want_histogram;
     int nbytes;
     int compression_type;
+    int compress_nulls;
     int window_set;		/* Flag: window set?                    */
     int split_window;           /* Separate windows for input and output */
     struct Cell_head rd_window;	/* Window used for input        */

lib/raster/close.c

@@ -33,6 +33,7 @@
 #define FORMAT_FILE "f_format"
 #define QUANT_FILE  "f_quant"
 #define NULL_FILE   "null"
+#define NULL2_FILE  "null2"
 
 static int close_old(int);
 static int close_new(int, int);
@@ -278,6 +279,9 @@ static int close_new_gdal(int fd, int ok)
 	G_file_name_misc(path, "cell_misc", NULL_FILE, fcb->name,
 			  G_mapset());
 	remove(path);
+	G_file_name_misc(path, "cell_misc", NULL2_FILE, fcb->name,
+			  G_mapset());
+	remove(path);
 
 	/* write 0-length cell file */
 	G_make_mapset_element("cell");
@@ -364,9 +368,14 @@ static int close_new(int fd, int ok)
 
 	/* create path : full null file name */
 	G__make_mapset_element_misc("cell_misc", fcb->name);
-	G_file_name_misc(path, "cell_misc", NULL_FILE, fcb->name,
-			  G_mapset());
+	G_file_name_misc(path, "cell_misc", NULL_FILE, fcb->name, G_mapset());
 	remove(path);
+	G_file_name_misc(path, "cell_misc", NULL2_FILE, fcb->name, G_mapset());
+	remove(path);
+
+	G_file_name_misc(path, "cell_misc",
+			 fcb->null_row_ptr ? NULL2_FILE : NULL_FILE,
+			 fcb->name, G_mapset());
 
 	if (fcb->null_cur_row > 0) {
 	    /* if temporary NULL file exists, write it into cell_misc/name/null */
@@ -390,6 +399,11 @@ static int close_new(int fd, int ok)
 	    Rast__write_row_ptrs(fd);
 	}
 
+	if (fcb->null_row_ptr) {			/* compressed nulls */
+	    fcb->null_row_ptr[fcb->cellhd.rows] = lseek(fcb->null_fd, 0L, SEEK_CUR);
+	    Rast__write_null_row_ptrs(fd, fcb->null_fd);
+	}
+
 	if (fcb->map_type != CELL_TYPE) {	/* floating point map */
 	    int cell_fd;
 

lib/raster/format.c

@@ -91,10 +91,8 @@ int Rast__check_format(int fd)
     return Rast__read_row_ptrs(fd);
 }
 
-int Rast__read_row_ptrs(int fd)
+static int read_row_ptrs(int nrows, int old, off_t *row_ptr, int fd)
 {
-    struct fileinfo *fcb = &R__.fileinfo[fd];
-    int nrows = fcb->cellhd.rows;
     unsigned char nbytes;
     unsigned char *buf, *b;
     int n;
@@ -105,9 +103,9 @@ int Rast__read_row_ptrs(int fd)
      * (this makes them machine dependent)
      */
 
-    if (fcb->cellhd.compressed < 0) {
+    if (old) {
 	n = (nrows + 1) * sizeof(off_t);
-	if (read(fcb->data_fd, fcb->row_ptr, n) != n)
+	if (read(fd, row_ptr, n) != n)
 	    goto badread;
 	return 1;
     }
@@ -119,14 +117,14 @@ int Rast__read_row_ptrs(int fd)
      *  actual values do not exceed the capability of the off_t)
      */
 
-    if (read(fcb->data_fd, &nbytes, 1) != 1)
+    if (read(fd, &nbytes, 1) != 1)
 	goto badread;
     if (nbytes == 0)
 	goto badread;
 
     n = (nrows + 1) * nbytes;
     buf = G_malloc(n);
-    if (read(fcb->data_fd, buf, n) != n)
+    if (read(fd, buf, n) != n)
 	goto badread;
 
     for (row = 0, b = buf; row <= nrows; row++) {
@@ -143,7 +141,7 @@ int Rast__read_row_ptrs(int fd)
 	    v += c;
 	}
 
-	fcb->row_ptr[row] = v;
+	row_ptr[row] = v;
     }
 
     G_free(buf);
@@ -151,27 +149,52 @@ int Rast__read_row_ptrs(int fd)
     return 1;
 
   badread:
-    G_warning(_("Fail of initial read of compressed file [%s in %s]"),
-	      fcb->name, fcb->mapset);
     return -1;
 }
 
-int Rast__write_row_ptrs(int fd)
+int Rast__read_row_ptrs(int fd)
+{
+    struct fileinfo *fcb = &R__.fileinfo[fd];
+    int nrows = fcb->cellhd.rows;
+    int old = fcb->cellhd.compressed < 0;
+
+    if (read_row_ptrs(nrows, old, fcb->row_ptr, fcb->data_fd) < 0) {
+	G_warning(_("Fail of initial read of compressed file [%s in %s]"),
+		  fcb->name, fcb->mapset);
+	return -1;
+    }
+
+    return 1;
+}
+
+int Rast__read_null_row_ptrs(int fd, int null_fd)
 {
     struct fileinfo *fcb = &R__.fileinfo[fd];
     int nrows = fcb->cellhd.rows;
+
+    if (read_row_ptrs(nrows, 0, fcb->null_row_ptr, null_fd) < 0) {
+	G_warning(_("Fail of initial read of compressed null file [%s in %s]"),
+		  fcb->name, fcb->mapset);
+	return -1;
+    }
+
+    return 1;
+}
+
+static int write_row_ptrs(int nrows, off_t *row_ptr, int fd)
+{
     int nbytes = sizeof(off_t);
     unsigned char *buf, *b;
     int len, row, result;
 
-    lseek(fcb->data_fd, 0L, SEEK_SET);
+    lseek(fd, 0L, SEEK_SET);
 
     len = (nrows + 1) * nbytes + 1;
     b = buf = G_malloc(len);
     *b++ = nbytes;
 
     for (row = 0; row <= nrows; row++) {
-	off_t v = fcb->row_ptr[row];
+	off_t v = row_ptr[row];
 	int i;
 
 	for (i = nbytes - 1; i >= 0; i--) {
@@ -182,8 +205,24 @@ int Rast__write_row_ptrs(int fd)
 	b += nbytes;
     }
 
-    result = (write(fcb->data_fd, buf, len) == len);
+    result = (write(fd, buf, len) == len);
     G_free(buf);
 
     return result;
 }
+
+int Rast__write_row_ptrs(int fd)
+{
+    struct fileinfo *fcb = &R__.fileinfo[fd];
+    int nrows = fcb->cellhd.rows;
+
+    return write_row_ptrs(nrows, fcb->row_ptr, fcb->data_fd);
+}
+
+int Rast__write_null_row_ptrs(int fd, int null_fd)
+{
+    struct fileinfo *fcb = &R__.fileinfo[fd];
+    int nrows = fcb->cellhd.rows;
+
+    return write_row_ptrs(nrows, fcb->null_row_ptr, null_fd);
+}

lib/raster/get_row.c

@@ -804,6 +804,40 @@ void Rast_get_d_row(int fd, DCELL * buf, int row)
     Rast_get_row(fd, buf, row, DCELL_TYPE);
 }
 
+static int read_null_bits_compressed(int null_fd, unsigned char *flags,
+				     int row, size_t size, int fd)
+{
+    struct fileinfo *fcb = &R__.fileinfo[fd];
+    off_t t1 = fcb->null_row_ptr[row];
+    off_t t2 = fcb->null_row_ptr[row + 1];
+    size_t readamount = t2 - t1;
+    unsigned char *compressed_buf;
+
+    if (lseek(null_fd, t1, SEEK_SET) < 0)
+	G_fatal_error(_("Error reading null data for row %d of <%s>"),
+		      row, fcb->name);
+
+    if (readamount == size) {
+	if (read(null_fd, flags, size) != size)
+	    G_fatal_error(_("Error reading null data for row %d of <%s>"),
+			  row, fcb->name);
+    }
+
+    compressed_buf = G_alloca(readamount);
+
+    if (read(null_fd, compressed_buf, readamount) != readamount) {
+	G_freea(compressed_buf);
+	G_fatal_error(_("Error reading null data for row %d of <%s>"),
+		      row, fcb->name);
+    }
+
+    G_zlib_expand(compressed_buf, readamount, flags, size);
+
+    G_freea(compressed_buf);
+
+    return 1;
+}
+
 static int read_null_bits(int fd, int row)
 {
     struct fileinfo *fcb = &R__.fileinfo[fd];
@@ -823,6 +857,10 @@ static int read_null_bits(int fd, int row)
 	return 0;
 
     size = Rast__null_bitstream_size(cols);
+
+    if (fcb->null_row_ptr)
+	return read_null_bits_compressed(null_fd, flags, R, size, fd);
+
     offset = (off_t) size * R;
 
     if (lseek(null_fd, offset, SEEK_SET) < 0)

lib/raster/init.c

@@ -77,7 +77,7 @@ void Rast__error_handler(void *p)
 
 static int init(void)
 {
-    char *zlib;
+    char *zlib, *nulls;
 
     Rast__init_window();
 
@@ -96,6 +96,9 @@ static int init(void)
     zlib = getenv("GRASS_INT_ZLIB");
     R__.compression_type = (!zlib || atoi(zlib)) ? 2 : 1;
 
+    nulls = getenv("GRASS_COMPRESS_NULLS");
+    R__.compress_nulls = (nulls && atoi(nulls)) ? 1 : 0;
+
     G_add_error_handler(Rast__error_handler, NULL);
 
     initialized = 1;

lib/raster/open.c

@@ -28,6 +28,7 @@
 #include "R.h"
 #define FORMAT_FILE "f_format"
 #define NULL_FILE   "null"
+#define NULL2_FILE  "null2"
 
 static int new_fileinfo(void)
 {
@@ -328,6 +329,7 @@ int Rast__open_old(const char *name, const char *mapset)
     fcb->io_error = 0;
     fcb->map_type = MAP_TYPE;
     fcb->nbytes = MAP_NBYTES;
+    fcb->null_row_ptr = NULL;
 
     if (!gdal) {
 	if (!G_find_file2_misc("cell_misc", NULL_FILE, r_name, r_mapset)) {
@@ -335,7 +337,19 @@ int Rast__open_old(const char *name, const char *mapset)
 	    fcb->null_file_exists = 0;
 	}
 	else {
-	    fcb->null_fd = G_open_old_misc("cell_misc", NULL_FILE, r_name, r_mapset);
+	    /* First, check for compressed null file */
+	    fcb->null_fd = G_open_old_misc("cell_misc", NULL2_FILE, r_name, r_mapset);
+	    if (fcb->null_fd >= 0) {
+		fcb->null_row_ptr = G_calloc(fcb->cellhd.rows + 1, sizeof(off_t));
+		if (Rast__read_null_row_ptrs(fd, fcb->null_fd) < 0) {
+		    close(fcb->null_fd);
+		    fcb->null_fd = -1;
+		    G_free(fcb->null_row_ptr);
+		    fcb->null_row_ptr = NULL;
+		}
+	    }
+	    if (fcb->null_fd < 0)
+		fcb->null_fd = G_open_old_misc("cell_misc", NULL_FILE, r_name, r_mapset);
 	    fcb->null_file_exists = fcb->null_fd >= 0;
 	}
     }
@@ -514,6 +528,7 @@ static int open_raster_new_gdal(char *map, char *mapset,
     fcb->null_cur_row = 0;
     fcb->null_bits = NULL;
     fcb->null_fd = -1;
+    fcb->null_row_ptr = NULL;
 
     if (fcb->map_type != CELL_TYPE)
 	Rast_quant_init(&(fcb->quant));
@@ -669,6 +684,12 @@ static int open_raster_new(const char *name, int open_mode,
 
     fcb->null_temp_name = tempname;
 
+    if (R__.compress_nulls) {
+	fcb->null_row_ptr = G_calloc(fcb->cellhd.rows + 1, sizeof(off_t));
+	G_zero(fcb->row_ptr, (fcb->cellhd.rows + 1) * sizeof(off_t));
+	Rast__write_null_row_ptrs(fd, fcb->null_fd);
+    }
+
     /* next row to be written (in order) is zero */
     fcb->null_cur_row = 0;
 

lib/raster/put_row.c

@@ -531,6 +531,33 @@ int Rast__open_null_write(int fd)
     return null_fd;
 }
 
+static void write_null_bits_compressed(int null_fd, const unsigned char *flags,
+				int row, size_t size, int fd)
+{
+    struct fileinfo *fcb = &R__.fileinfo[fd];
+    unsigned char *compressed_buf;
+    ssize_t nwrite;
+
+    fcb->null_row_ptr[row] = lseek(null_fd, 0L, SEEK_CUR);
+
+    compressed_buf = G_alloca(size + 1);
+
+    nwrite = G_zlib_compress(flags, size, compressed_buf, size);
+
+    if (nwrite < size) {
+	if (write(null_fd, compressed_buf, nwrite) != nwrite)
+	    G_fatal_error(_("Error writing compressed null data for row %d of <%s>"),
+			  row, fcb->name);
+    }
+    else {
+	if (write(null_fd, flags, size) != size)
+	    G_fatal_error(_("Error writing compressed null data for row %d of <%s>"),
+			  row, fcb->name);
+    }
+
+    G_freea(compressed_buf);
+}
+
 /*!
    \brief Write null data
 
@@ -550,6 +577,12 @@ void Rast__write_null_bits(int null_fd, const unsigned char *flags, int row,
     size_t size;
 
     size = Rast__null_bitstream_size(cols);
+
+    if (fcb->null_row_ptr) {
+	write_null_bits_compressed(null_fd, flags, row, size, fd);
+	return;
+    }
+
     offset = (off_t) size *row;
 
     if (lseek(null_fd, offset, SEEK_SET) < 0)