segment lib: support for more than 2 billion cells

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@51482 15284696-431f-4ddb-bdfa-cd5b030d7da7

lib/segment/address.c

@@ -14,20 +14,47 @@
 
 #include <grass/segment.h>
 
-int segment_address_fast(const SEGMENT * SEG, int row, int col, int *n,
+#define SEG_N_ROW_NONZERO(SEG, row, col) \
+    (((row) >> (SEG)->srowbits) * (SEG)->spr + ((col) >> (SEG)->scolbits))
+
+#define SEG_INDEX_ROW_NONZERO(SEG, row, col) \
+    ((((row) & ((SEG)->srows - 1)) << (SEG)->scolbits) + ((col) & ((SEG)->scols - 1)))
+
+#define SEG_N_ROW_ZERO(SEG, col)	((col) >> (SEG)->scolbits)
+
+#define SEG_INDEX_ROW_ZERO(SEG, col) ((col) & ((SEG)->scols - 1))
+
+#define INDEX_ADJ(SEG, i) \
+    ((SEG)->fast_seek ? ((i) << (SEG)->lenbits) : ((i) * (SEG)->len))
+
+int segment_address_fast(const SEGMENT * SEG, off_t row, off_t col, int *n,
 			 int *index)
 {
+
+#if 1
+    if (row) {
+	*n = SEG_N_ROW_NONZERO(SEG, row, col);
+	*index = INDEX_ADJ(SEG, SEG_INDEX_ROW_NONZERO(SEG, row, col));
+    }
+    /* for simple arrays */
+    else {
+	*n = SEG_N_ROW_ZERO(SEG, col);
+	*index = INDEX_ADJ(SEG, SEG_INDEX_ROW_ZERO(SEG, col));
+    }
+#else
     if (row) {
-	int seg_r = row >> SEG->srowbits;
-	int seg_c = col >> SEG->scolbits;
+	*n = (row >> SEG->srowbits) * SEG->spr + (col >> SEG->scolbits);
+	*index = ((row & (SEG->srows - 1)) << SEG->scolbits) + (col & (SEG->scols - 1));
+
+	/* slower version for testing */
+	/*
+	off_t seg_r = row >> SEG->srowbits;
+	off_t seg_c = col >> SEG->scolbits;
 
 	*n = seg_r * SEG->spr + seg_c;
+	
 	*index = ((row - (seg_r << SEG->srowbits)) << SEG->scolbits) +
 	         col - (seg_c << SEG->scolbits);
-
-	/*
-	*n = (row >> SEG->srowbits) * SEG->spr + (col >> SEG->scolbits);
-	*index = ((row & (SEG->srows - 1)) << SEG->scolbits) + (col & (SEG->scols - 1));
 	*/
     }
     /* for simple arrays */
@@ -35,20 +62,20 @@ int segment_address_fast(const SEGMENT * SEG, int row, int col, int *n,
 	*n = col >> SEG->scolbits;
 	*index = col - ((*n) << SEG->scolbits);
     }
-    if (!SEG->slow_seek)
-	*index = *index << SEG->lenbits;
-    else
-	*index *= SEG->len;
+
+    *index = SEG->fast_seek ? (*index << SEG->lenbits) : (*index * SEG->len);
+
+#endif
 
     return 0;
 }
 
-int segment_address_slow(const SEGMENT * SEG, int row, int col, int *n,
+int segment_address_slow(const SEGMENT * SEG, off_t row, off_t col, int *n,
 			 int *index)
 {
     if (row) {
-	int seg_r = row / SEG->srows;
-	int seg_c = col / SEG->scols;
+	off_t seg_r = row / SEG->srows;
+	off_t seg_c = col / SEG->scols;
 
 	*n = seg_r * SEG->spr + seg_c;
 	*index = (row - seg_r * SEG->srows) * SEG->scols + col -
@@ -77,7 +104,7 @@ int segment_address_slow(const SEGMENT * SEG, int row, int col, int *n,
  * \return always returns 0
  */
 
-int segment_address(const SEGMENT * SEG, int row, int col, int *n, int *index)
+int segment_address(const SEGMENT * SEG, off_t row, off_t col, int *n, int *index)
 {
     /* old code
      *n = row / SEG->srows * SEG->spr + col / SEG->scols;

lib/segment/format.c

@@ -22,8 +22,9 @@
 #include <grass/segment.h>
 
 
-static int _segment_format(int, int, int, int, int, int, int);
+static int _segment_format(int, off_t, off_t, int, int, int, int);
 static int write_int(int, int);
+static int write_off_t(int, off_t);
 static int zero_fill(int, off_t);
 
 /* fd must be open for write */
@@ -60,7 +61,7 @@ static int zero_fill(int, off_t);
  * \return -3 if illegal parameters are passed
  */
 
-int segment_format(int fd, int nrows, int ncols, int srows, int scols,
+int segment_format(int fd, off_t nrows, off_t ncols, int srows, int scols,
 		   int len)
 {
     return _segment_format(fd, nrows, ncols, srows, scols, len, 1);
@@ -100,7 +101,7 @@ int segment_format(int fd, int nrows, int ncols, int srows, int scols,
  * \return -3 if illegal parameters are passed
  */
 
-int segment_format_nofill(int fd, int nrows, int ncols, int srows, int scols,
+int segment_format_nofill(int fd, off_t nrows, off_t ncols, int srows, int scols,
 			  int len)
 {
     return _segment_format(fd, nrows, ncols, srows, scols, len, 0);
@@ -108,33 +109,35 @@ int segment_format_nofill(int fd, int nrows, int ncols, int srows, int scols,
 
 
 static int _segment_format(int fd,
-			   int nrows, int ncols,
+			   off_t nrows, off_t ncols,
 			   int srows, int scols, int len, int fill)
 {
     off_t nbytes;
     int spr, size;
 
     if (nrows <= 0 || ncols <= 0 || len <= 0 || srows <= 0 || scols <= 0) {
-	G_warning("segment_format(fd,%d,%d,%d,%d,%d): illegal value(s)",
+	G_warning("segment_format(fd,%lld,%lld,%d,%d,%d): illegal value(s)",
 		  nrows, ncols, srows, scols, len);
 	return -3;
     }
 
+    spr = ncols / scols;
+    if (ncols % scols)
+	spr++;
+
+    size = srows * scols * len;
+
     if (sizeof(off_t) == 4 && sizeof(double) >= 8) {
 	double d_size;
 	off_t o_size;
 
-	spr = ncols / scols;
-	if (ncols % scols)
-	    spr++;
-
 	/* calculate total number of segments */
 	d_size = (double) spr * ((nrows + srows - 1) / srows);
 	/* multiply with segment size */
-	d_size *= srows * scols * len;
+	d_size *= size;
 
 	/* add header */
-	d_size += 5 * sizeof(int);
+	d_size += 2 * sizeof(off_t) + 3 * sizeof(int);
 
 	o_size = (off_t) d_size;
 
@@ -152,7 +155,7 @@ static int _segment_format(int fd,
 	return -1;
     }
 
-    if (!write_int(fd, nrows) || !write_int(fd, ncols)
+    if (!write_off_t(fd, nrows) || !write_off_t(fd, ncols)
 	|| !write_int(fd, srows) || !write_int(fd, scols)
 	|| !write_int(fd, len))
 	return -1;
@@ -160,12 +163,6 @@ static int _segment_format(int fd,
     if (!fill)
 	return 1;
 
-    spr = ncols / scols;
-    if (ncols % scols)
-	spr++;
-
-    size = srows * scols * len;
-
     /* calculate total number of segments */
     nbytes = spr * ((nrows + srows - 1) / srows);
     nbytes *= size;
@@ -193,6 +190,15 @@ static int write_int(int fd, int n)
     return 1;
 }
 
+static int write_off_t(int fd, off_t n)
+{
+    if (write(fd, &n, sizeof(off_t)) != sizeof(off_t)) {
+	G_warning("segment_format(): Unable to write (%s)", strerror(errno));
+	return 0;
+    }
+
+    return 1;
+}
 
 static int zero_fill(int fd, off_t nbytes)
 {

lib/segment/get.c

@@ -38,7 +38,7 @@
  * \return -1 if unable to seek or read segment file
  */
 
-int segment_get(SEGMENT * SEG, void *buf, int row, int col)
+int segment_get(SEGMENT * SEG, void *buf, off_t row, off_t col)
 {
     int index, n, i;
 

lib/segment/get_row.c

@@ -40,12 +40,12 @@
  * \return -1 if unable to seek or read segment file
  */
 
-int segment_get_row(const SEGMENT * SEG, void *buf, int row)
+int segment_get_row(const SEGMENT * SEG, void *buf, off_t row)
 {
     int size;
-    int ncols;
+    off_t ncols, col;
     int scols;
-    int n, index, col;
+    int n, index;
 
     ncols = SEG->ncols - SEG->spill;
     scols = SEG->scols;
@@ -53,8 +53,7 @@ int segment_get_row(const SEGMENT * SEG, void *buf, int row)
 
     for (col = 0; col < ncols; col += scols) {
 	SEG->segment_address(SEG, row, col, &n, &index);
-	if (SEG->segment_seek(SEG, n, index) < 0)
-	    return -1;
+	SEG->segment_seek(SEG, n, index);
 
 	if (read(SEG->fd, buf, size) != size) {
 	    G_warning("segment_get_row: %s", strerror(errno));
@@ -70,8 +69,7 @@ int segment_get_row(const SEGMENT * SEG, void *buf, int row)
     }
     if ((size = SEG->spill * SEG->len)) {
 	SEG->segment_address(SEG, row, col, &n, &index);
-	if (SEG->segment_seek(SEG, n, index) < 0)
-	    return -1;
+	SEG->segment_seek(SEG, n, index);
 
 	if (read(SEG->fd, buf, size) != size) {
 	    G_warning("segment_get_row: %s", strerror(errno));

lib/segment/init.c

@@ -26,6 +26,7 @@
 
 
 static int read_int(int, int *);
+static int read_off_t(int, off_t *);
 
 /* fd must be open for read and write */
 
@@ -67,8 +68,8 @@ int segment_init(SEGMENT * SEG, int fd, int nseg)
     }
 
     /* read the header */
-    if (!read_int(fd, &SEG->nrows)
-	|| !read_int(fd, &SEG->ncols)
+    if (!read_off_t(fd, &SEG->nrows)
+	|| !read_off_t(fd, &SEG->ncols)
 	|| !read_int(fd, &SEG->srows)
 	|| !read_int(fd, &SEG->scols)
 	|| !read_int(fd, &SEG->len))
@@ -89,3 +90,15 @@ static int read_int(int fd, int *n)
 
     return bytes_read;
 }
+
+static int read_off_t(int fd, off_t *n)
+{
+    int bytes_read;
+
+    if ((bytes_read = read(fd, n, sizeof(off_t))) == -1)
+	G_warning("read_off_t: %s", strerror(errno));
+
+    bytes_read = (bytes_read == sizeof(off_t));
+
+    return bytes_read;
+}

lib/segment/put.c

@@ -42,7 +42,7 @@
  * \return -1 if unable to seek or write segment file
  */
 
-int segment_put(SEGMENT * SEG, const void *buf, int row, int col)
+int segment_put(SEGMENT * SEG, const void *buf, off_t row, off_t col)
 {
     int index, n, i;
 

lib/segment/put_row.c

@@ -43,13 +43,14 @@
  * \return -1 if unable to seek or write segment file
  */
 
-int segment_put_row(const SEGMENT * SEG, const void *buf, int row)
+int segment_put_row(const SEGMENT * SEG, const void *buf, off_t row)
 {
     int size;
-    int ncols;
+    off_t ncols;
     int scols;
-    int n, index, col;
+    int n, index;
     int result;
+    off_t col;
 
     ncols = SEG->ncols - SEG->spill;
     scols = SEG->scols;
@@ -58,12 +59,7 @@ int segment_put_row(const SEGMENT * SEG, const void *buf, int row)
 
     for (col = 0; col < ncols; col += scols) {
 	SEG->segment_address(SEG, row, col, &n, &index);
-	if (SEG->segment_seek(SEG, n, index) < 0) {
-	    G_warning
-		("Failed seek in segment file for index = %d n = %d at col:row %d:%d",
-		 index, n, col, row);
-	    return -1;
-	}
+	SEG->segment_seek(SEG, n, index);
 
 	if ((result = write(SEG->fd, buf, size)) != size) {
 	    G_warning("segment_put_row write error %s", strerror(errno));
@@ -81,12 +77,8 @@ int segment_put_row(const SEGMENT * SEG, const void *buf, int row)
 
     if ((size = SEG->spill * SEG->len)) {
 	SEG->segment_address(SEG, row, col, &n, &index);
-	if (SEG->segment_seek(SEG, n, index) < 0) {
-	    G_warning
-		("Failed seek in segment file for index = %d n = %d at col:row %d:%d",
-		 index, n, col, row);
-	    return -1;
-	}
+	SEG->segment_seek(SEG, n, index);
+
 	if (write(SEG->fd, buf, size) != size) {
 	    G_warning("segment_put_row final write error: %s",
 		      strerror(errno));

lib/segment/seek.c

@@ -33,13 +33,17 @@
  * \return -1 if unable to seek
  */
 
+#define SEG_SEEK_FAST(SEG, n, index) \
+    ((((off_t) (n)) << (SEG)->sizebits) + (index) + (SEG)->offset)
+
+#define SEG_SEEK_SLOW(SEG, n, index) \
+    ((off_t) (n) * (SEG)->size + (index) + (SEG)->offset)
+
 int segment_seek_fast(const SEGMENT * SEG, int n, int index)
 {
-    off_t offset = (((off_t) n) << SEG->sizebits) + index + SEG->offset;
-
-    if (lseek(SEG->fd, offset, SEEK_SET) == (off_t) - 1) {
-	G_warning("segment_seek: %s", strerror(errno));
-	return -1;
+    if (lseek((SEG)->fd, SEG_SEEK_FAST(SEG, n, index), 
+        SEEK_SET) == (off_t) -1) {
+	G_fatal_error("segment_seek: %s", strerror(errno));
     }
 
     return 0;
@@ -47,11 +51,9 @@ int segment_seek_fast(const SEGMENT * SEG, int n, int index)
 
 int segment_seek_slow(const SEGMENT * SEG, int n, int index)
 {
-    off_t offset = (off_t) n * SEG->size + index + SEG->offset;
-
-    if (lseek(SEG->fd, offset, SEEK_SET) == (off_t) - 1) {
-	G_warning("segment_seek: %s", strerror(errno));
-	return -1;
+    if (lseek((SEG)->fd, SEG_SEEK_SLOW(SEG, n, index), 
+        SEEK_SET) == (off_t) -1) {
+	G_fatal_error("segment_seek: %s", strerror(errno));
     }
 
     return 0;

lib/segment/setup.c

@@ -56,7 +56,7 @@ int segment_setup(SEGMENT * SEG)
 	SEG->spr++;
 
     /* fast address */
-    SEG->slow_adrs = 1;
+    SEG->fast_adrs = 0;
 
     seg_exp = 0;
     while (SEG->scols - (1 << seg_exp) > 0)
@@ -70,32 +70,32 @@ int segment_setup(SEGMENT * SEG)
 	if (SEG->srows - (1 << seg_exp) == 0) {
 	    SEG->srowbits = seg_exp;
 	    SEG->segbits = SEG->srowbits + SEG->scolbits;
-	    SEG->slow_adrs = 0;
+	    SEG->fast_adrs = 1;
 	    G_debug(1, "segment_setup: fast address activated");
 	}
     }
-    if (SEG->slow_adrs)
-	SEG->segment_address = segment_address_slow;
-    else
+    if (SEG->fast_adrs)
 	SEG->segment_address = segment_address_fast;
+    else
+	SEG->segment_address = segment_address_slow;
     
     /* fast seek */
-    SEG->slow_seek = 1;
-    if (SEG->slow_adrs == 0) {
+    SEG->fast_seek = 0;
+    if (SEG->fast_adrs == 1) {
 	seg_exp = 0;
 	while (SEG->len - (1 << seg_exp) > 0)
 	    seg_exp++;
 	if (SEG->len - (1 << seg_exp) == 0) {
 	    SEG->lenbits = seg_exp;
 	    SEG->sizebits = SEG->segbits + SEG->lenbits;
-	    SEG->slow_seek = 0;
+	    SEG->fast_seek = 1;
 	    G_debug(1, "segment_setup: fast seek activated");
 	}
     }
-    if (SEG->slow_seek)
-	SEG->segment_seek = segment_seek_slow;
-    else
+    if (SEG->fast_seek)
 	SEG->segment_seek = segment_seek_fast;
+    else
+	SEG->segment_seek = segment_seek_slow;
 
     /* adjust number of open segments if larger than number of total segments */
     n_total_segs = SEG->spr * ((SEG->nrows + SEG->srows - 1) / SEG->srows);