New tile computation algorithm implemented. New g3d open method implemented. Enhanced r3.info information printing.

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@46673 15284696-431f-4ddb-bdfa-cd5b030d7da7

include/G3d.h

@@ -420,6 +420,7 @@ void *G3d_openNewParam(const char *, int , int, G3D_Region *, int, int, int, int
 void *G3d_openCellOldNoHeader(const char *, const char *);
 void *G3d_openCellOld(const char *, const char *, G3D_Region *, int, int);
 void *G3d_openCellNew(const char *, int, int, G3D_Region *);
+void *G3d_openNewOptTileSize(const char *, int , G3D_Region * , int , int );
 
 /* grass/src/libes/g3d/g3dparam.c */
 void G3d_setStandard3dInputParams(void);
@@ -539,6 +540,7 @@ double G3d_getDoubleRegion(G3D_Map *, int, int, int);
 void G3d_getValueRegion(G3D_Map *, int, int, int, void *, int);
 
 /* grass/src/libes/g3d/tilemath.c */
+void G3d_computeOptimalTileDimension(G3D_Region *, int, int *, int *, int *, int);
 void G3d_tileIndex2tile(G3D_Map *, int, int *, int *, int *);
 int G3d_tile2tileIndex(G3D_Map *, int, int, int);
 void G3d_tileCoordOrigin(G3D_Map *, int, int, int, int *, int *, int *);

lib/g3d/g3ddefaults.c

@@ -11,11 +11,11 @@
 #define G3D_USE_LZW_DEFAULT G3D_NO_LZW
 #define G3D_USE_RLE_DEFAULT G3D_USE_RLE
 #define G3D_PRECISION_DEFAULT G3D_MAX_PRECISION
-#define G3D_CACHE_SIZE_DEFAULT 1000
-#define G3D_CACHE_SIZE_MAX_DEFAULT 2000000
+#define G3D_CACHE_SIZE_DEFAULT 262144
+#define G3D_CACHE_SIZE_MAX_DEFAULT 16777216
 #define G3D_FILE_TYPE_DEFAULT DCELL_TYPE
-#define G3D_TILE_X_DEFAULT 8
-#define G3D_TILE_Y_DEFAULT 8
+#define G3D_TILE_X_DEFAULT 16
+#define G3D_TILE_Y_DEFAULT 16
 #define G3D_TILE_Z_DEFAULT 8
 #define G3D_ERROR_FUN_DEFAULT G3d_skipError
 #define G3D_UNIT_DEFAULT "none"

lib/g3d/g3dmisc.c

@@ -83,9 +83,9 @@ int G3d_length(int t)
 	G3d_fatalError("G3d_length: invalid type");
 
     if (t == FCELL_TYPE)
-	return sizeof(float);
+	return sizeof(FCELL);
     if (t == DCELL_TYPE)
-	return sizeof(double);
+	return sizeof(DCELL);
     return 0;
 }
 

lib/g3d/g3dopen2.c

@@ -3,6 +3,41 @@
 
 /*----------------------------------------------------------------------------*/
 
+/*!
+ * \brief 
+ *
+ * Opens new g3d-file with <em>name</em> in the current mapset. Tiles
+ * are stored in memory with <em>typeIntern</em> which must be one of FCELL_TYPE,
+ * DCELL_TYPE, or G3D_TILE_SAME_AS_FILE. <em>cache</em> specifies the
+ * cache-mode used and must be either G3D_NO_CACHE, G3D_USE_CACHE_DEFAULT,
+ * G3D_USE_CACHE_X, G3D_USE_CACHE_Y, G3D_USE_CACHE_Z,
+ * G3D_USE_CACHE_XY, G3D_USE_CACHE_XZ, G3D_USE_CACHE_YZ,
+ * G3D_USE_CACHE_XYZ, the result of <tt>G3d_cacheSizeEncode ()</tt>
+ * (cf.{g3d:G3d.cacheSizeEncode}), or any positive integer which
+ * specifies the number of tiles buffered in the cache.  <em>region</em> specifies
+ * the 3d region.  
+ * The map is created using the <em>type</em> which must be of FCELL_TYPE or DCELL_TYPE.
+ * The methods for compression can be specified LZW or RLE. The digits of the floating point mantissa
+ * can be specified. In case of FCELL_TYPE 0-23 and 0-52 in case of DCELL_TYPE. 
+ * The number cells in X, Y and Z direction defines the size of each tile.
+ * Returns a pointer to the cell structure ... if successful,
+ * NULL ... otherwise.
+ *
+ *  \param name The name of the map
+ *  \param typeIntern The internal storage type for in memory chached tiles
+ *  \param cache The type of the caching
+ *  \param region The region of the map
+ *  \param type The type of the map (FCELL_TYPE, or DCELL_TYPE)
+ *  \param doLzw Use the LZW compression algorithm  
+ *  \param doRle Use the Run-Length-Encoding algroithm for compression
+ *  \param precision The precision used for the mantissa (0 - 52) or G3D_MAX_PRECISION
+ *  \param tileX The number of cells in X direction of a tile
+ *  \param tileY The number of cells in Y direction of a tile
+ *  \param tileZ The number of cells in Z direction of a tile
+ *  \return void * 
+ */
+
+
 void *G3d_openNewParam(const char *name, int typeIntern, int cache,
 		       G3D_Region * region, int type, int doLzw, int doRle,
 		       int precision, int tileX, int tileY, int tileZ)
@@ -31,3 +66,57 @@ void *G3d_openNewParam(const char *name, int typeIntern, int cache,
 
     return map;
 }
+
+/*----------------------------------------------------------------------------*/
+
+/*!
+ * \brief 
+ *
+ * Opens new g3d-file with <em>name</em> in the current mapset. This method tries to compute 
+ * optimal tile size based on the number of rows, cols and depths and the maximum allowed tile size in KB. 
+ * Tiles are stored in memory using G3D_TILE_SAME_AS_FILE method. <em>cache</em> specifies the
+ * cache-mode used and must be either G3D_NO_CACHE, G3D_USE_CACHE_DEFAULT,
+ * G3D_USE_CACHE_X, G3D_USE_CACHE_Y, G3D_USE_CACHE_Z,
+ * G3D_USE_CACHE_XY, G3D_USE_CACHE_XZ, G3D_USE_CACHE_YZ,
+ * G3D_USE_CACHE_XYZ, the result of <tt>G3d_cacheSizeEncode ()</tt>
+ * (cf.{g3d:G3d.cacheSizeEncode}), or any positive integer which
+ * specifies the number of tiles buffered in the cache.  <em>region</em> specifies
+ * the 3d region.  
+ * The map is created using the <em>type</em> which must be of FCELL_TYPE or DCELL_TYPE.
+ * Returns a pointer to the cell structure ... if successful,
+ * NULL ... otherwise.
+ *
+ *  \param name The name of the map
+ *  \param cache The type of the caching
+ *  \param region The region of the map
+ *  \param type The type of the map (FCELL_TYPE, or DCELL_TYPE)
+ *  \param maxSize The maximum size of a tile in kilo bytes
+ *  \return void * 
+ */
+
+
+void *G3d_openNewOptTileSize(const char *name, int cache, G3D_Region * region, int type, int maxSize)
+{
+    void *map;
+    int oldTileX, oldTileY, oldTileZ, oldType;
+    int tileX, tileY, tileZ;
+
+    G3d_initDefaults();
+
+
+    G3d_getTileDimension(&oldTileX, &oldTileY, &oldTileZ);
+    
+    G3d_computeOptimalTileDimension(region, type, &tileX, &tileY, &tileZ, maxSize);
+
+    G3d_setTileDimension(tileX, tileY, tileZ);
+
+    oldType = G3d_getFileType();
+    G3d_setFileType(type);
+
+    map = G3d_openCellNew(name, G3D_TILE_SAME_AS_FILE, cache, region);
+
+    G3d_setTileDimension(oldTileX, oldTileY, oldTileZ);
+    G3d_setFileType(oldType);
+
+    return map;
+}

lib/g3d/tilemath.c

@@ -368,3 +368,93 @@ G3d_location2coord(G3D_Map * map, double north, double east, double top,
 				       map->region.bottom) *
 	(map->region.depths - 1);
 }
+
+/*---------------------------------------------------------------------------*/
+
+
+/*!
+ * \brief Compute the optimal tile size.
+ *
+ * This function computes tile sizes with an optimal ratio between tile dimensions and
+ * minimized border tile overlapping.
+ * Large dimensions will be reduced mor often than small dimensions to
+ * fit the maxSize criteria.
+ *
+ *  \param region The region of the map
+ *  \param type The type of the map (FCELL_TYPE or DCELL_TYPE)
+ *  \param tileX Pointer of the tile size in x direction for result storage
+ *  \param tileY Pointer of the tile size in y direction for result storage
+ *  \param tileZ Pointer of the tile size in z direction for result storage
+ *  \param maxSize The max size of the tile in kilo bytes
+ *  \return void
+ */
+
+void
+G3d_computeOptimalTileDimension(G3D_Region *region, int type, int *tileX, int *tileY, int *tileZ, int maxSize)
+{
+   int size = 0;
+   int x, y, z;
+   int i = 0;
+   int tileSize;
+   int divx = 2;
+   int divy = 2;
+   int divz = 2;
+
+   if(type == FCELL_TYPE)
+      size = sizeof(FCELL);
+
+   if(type == DCELL_TYPE)
+      size = sizeof(DCELL);
+   
+   x = region->cols;
+   y = region->rows;
+   z = region->depths;
+
+   while(1) {
+       tileSize = size * x * y * z;
+
+       /*
+       printf("Tilesize %i x %i y %i z %i\n", tileSize, x, y, z);
+       */
+
+       if(tileSize <= maxSize * 1024)
+          break;
+
+       /* Compute weighted tile sizes. Take care that the tile size is computed based on
+          the dimension ratio and reduce the border tile overlapping. 
+          In case one dimension is much larger than the other, reduce 
+          the large dimension by a factor till the maxSize is reached or the 
+          the other dimensions are only factor 2 smaller.*/
+       if((y / x) <= 2 && (z / x) <= 2) {
+           if(region->cols % divx != 0)
+               x = region->cols / divx + 1;
+           else
+               x = region->cols / divx;
+           divx += 1;
+       }
+       if((x / y) <= 2 && (z / y) <= 2) {
+           if(region->rows % divy != 0)
+               y = region->rows / divy + 1;
+           else
+               y = region->rows / divy;
+           divy += 1;
+       }
+       if((x / z) <= 2 && (y / z) <= 2) {
+           if(region->depths % divz != 0)
+               z = region->depths / divz + 1;
+           else
+               z = region->depths / divz;
+           divz += 1;
+       }
+
+       i++;
+       if(i > 10000)
+         break;
+   }
+
+   *tileX = x;
+   *tileY = y;
+   *tileZ = z;
+}
+
+

lib/gpde/N_arrays_io.c

@@ -394,7 +394,6 @@ void N_write_array_3d_to_rast3d(N_array_3d * array, char *name, int mask)
     /*get the current region */
     G3d_getWindow(&region);
 
-
     cols = region.cols;
     rows = region.rows;
     depths = region.depths;
@@ -413,11 +412,9 @@ void N_write_array_3d_to_rast3d(N_array_3d * array, char *name, int mask)
 
     /*Open the new map */
     if (type == DCELL_TYPE)
-	map =
-	    G3d_openCellNew(name, DCELL_TYPE, G3D_USE_CACHE_DEFAULT, &region);
+        map = G3d_openNewOptTileSize(name, G3D_USE_CACHE_XY, &region, DCELL_TYPE, 32);
     else if (type == FCELL_TYPE)
-	map =
-	    G3d_openCellNew(name, FCELL_TYPE, G3D_USE_CACHE_DEFAULT, &region);
+        map = G3d_openNewOptTileSize(name, G3D_USE_CACHE_XY, &region, FCELL_TYPE, 32);
 
     if (map == NULL)
 	G3d_fatalError(_("Error opening g3d map <%s>"), name);
@@ -459,6 +456,9 @@ void N_write_array_3d_to_rast3d(N_array_3d * array, char *name, int mask)
 		G3d_maskOff(map);
     }
 
+    /* Flush all tile */
+    if (!G3d_flushAllTiles(map))
+	G3d_fatalError("Error flushing tiles with G3d_flushAllTiles");
     /* Close files and exit */
     if (!G3d_closeCell(map))
 	G3d_fatalError(map, NULL, 0, _("Error closing g3d file"));

raster/r.mapcalc/map3.c

@@ -580,9 +580,7 @@ int open_output_map(const char *name, int res_type)
 
     G3d_setFileType(res_type == FCELL_TYPE ? FCELL_TYPE : DCELL_TYPE);
 
-    handle = G3d_openCellNew((char *)name,
-			     res_type == FCELL_TYPE ? FCELL_TYPE : DCELL_TYPE,
-			     G3D_USE_CACHE_DEFAULT, &current_region3);
+    handle = G3d_openNewOptTileSize((char *)name, G3D_USE_CACHE_XYZ, &current_region3, res_type == FCELL_TYPE ? FCELL_TYPE : DCELL_TYPE, 32);
 
     if (!handle)
 	G_fatal_error(_("Unable to create raster map <%s>"), name);

raster/r.to.rast3/main.c

@@ -27,7 +27,7 @@
 /*- params and global variables -----------------------------------------*/
 typedef struct
 {
-    struct Option *input, *output;
+    struct Option *input, *output, *tilesize;
     struct Flag *mask;
 } paramType;
 
@@ -79,6 +79,14 @@ void set_params()
 
     param.output = G_define_standard_option(G_OPT_R3_OUTPUT);
 
+    param.tilesize = G_define_option();
+    param.tilesize->description = _("The maximum tile size in kilo bytes. Default is 32KB.");
+    param.tilesize->key = "tilesize";
+    param.tilesize->answer = "32";
+    param.tilesize->type = TYPE_INTEGER;
+    param.tilesize->required = NO;
+    param.tilesize->multiple = NO;
+
     param.mask = G_define_flag();
     param.mask->key = 'm';
     param.mask->description = _("Use G3D mask (if exists) with output map");
@@ -185,6 +193,7 @@ int main(int argc, char *argv[])
     char *name;
     int changemask = 0;
     int maptype_tmp, nofile = 0;
+    int maxSize;
 
     /* Initialize GRASS */
     G_gisinit(argv[0]);
@@ -208,6 +217,9 @@ int main(int argc, char *argv[])
     if (param.output->answer == NULL)
 	G3d_fatalError(_("No output map"));
 
+    /* Get the tile size */
+    maxSize = atoi(param.tilesize->answer);
+
     /* Figure out the region from the map */
     G3d_initDefaults();
     G3d_getWindow(&region);
@@ -272,25 +284,13 @@ int main(int argc, char *argv[])
     G_message(_("Creating 3D raster map"));
     map = NULL;
 
-
-    if (globalRastMapType == CELL_TYPE) {
-	map =
-	    G3d_openCellNew(param.output->answer, DCELL_TYPE,
-			    G3D_USE_CACHE_DEFAULT, &region);
+    /* Set the map type depending from the arster maps type */
+    if (globalRastMapType == CELL_TYPE) 
 	globalG3dMapType = DCELL_TYPE;
-    }
-    else if (globalRastMapType == FCELL_TYPE) {
-	map =
-	    G3d_openCellNew(param.output->answer, FCELL_TYPE,
-			    G3D_USE_CACHE_DEFAULT, &region);
+    else 
 	globalG3dMapType = FCELL_TYPE;
-    }
-    else if (globalRastMapType == DCELL_TYPE) {
-	map =
-	    G3d_openCellNew(param.output->answer, DCELL_TYPE,
-			    G3D_USE_CACHE_DEFAULT, &region);
-	globalG3dMapType = DCELL_TYPE;
-    }
+
+    map = G3d_openNewOptTileSize(param.output->answer, G3D_USE_CACHE_XY, &region, globalG3dMapType, maxSize);
 
     if (map == NULL)
 	fatal_error(map, fd, opencells, _("Error opening 3d raster map"));
@@ -323,6 +323,9 @@ int main(int argc, char *argv[])
     if (fd)
 	G_free(fd);
 
+    /* Flush all tile */
+    if (!G3d_flushAllTiles(map))
+	G3d_fatalError("Error flushing tiles with G3d_flushAllTiles");
     /* Close files and exit */
     if (!G3d_closeCell(map))
 	G3d_fatalError(_("Error closing 3d raster map"));

raster/r.to.rast3elev/main.c

@@ -27,7 +27,7 @@
 /*- params and global variables -----------------------------------------*/
 typedef struct
 {
-    struct Option *input, *elev, *output, *upper, *lower;
+    struct Option *input, *elev, *output, *upper, *lower, *tilesize;
     struct Flag *fillup, *filllow, *mask;
 } paramType;
 
@@ -167,6 +167,14 @@ void set_params()
     param.lower->description =
 	_("The value to fill the lower cells, default is null");
 
+    param.tilesize = G_define_option();
+    param.tilesize->description = _("The maximum tile size in kilo bytes. Default is 32KB.");
+    param.tilesize->key = "tilesize";
+    param.tilesize->answer = "32";
+    param.tilesize->type = TYPE_INTEGER;
+    param.tilesize->required = NO;
+    param.tilesize->multiple = NO;
+
     param.fillup = G_define_flag();
     param.fillup->key = 'u';
     param.fillup->description =
@@ -331,6 +339,7 @@ int main(int argc, char *argv[])
     int cols, rows, i;
     char *name = NULL;
     int changemask = 0;
+    double maxSize;
     Database db;
 
     /*Initiate the database structure */
@@ -361,6 +370,9 @@ int main(int argc, char *argv[])
     /*Check if maps exist */
     check_input_maps(&db);
 
+    /* Get the tile size */
+    maxSize = atoi(param.tilesize->answer);
+
     /*Do not use values */
     db.useUpperVal = 0;
     db.useLowerVal = 0;
@@ -422,9 +434,8 @@ int main(int argc, char *argv[])
 
     /*open G3D output map */
     db.map = NULL;
-    db.map =
-	G3d_openCellNew(param.output->answer, DCELL_TYPE,
-			G3D_USE_CACHE_DEFAULT, &region);
+    db.map = G3d_openNewOptTileSize(param.output->answer, G3D_USE_CACHE_XY, &region, DCELL_TYPE, maxSize);
+
     if (db.map == NULL)
 	fatal_error(db, _("Error opening 3d raster map"));
 
@@ -482,6 +493,9 @@ int main(int argc, char *argv[])
 
     G_debug(2, "Close 3d raster map");
 
+    /* Flush all tile */
+    if (!G3d_flushAllTiles(db.map))
+	G3d_fatalError("Error flushing tiles with G3d_flushAllTiles");
     if (!G3d_closeCell(db.map))
 	G3d_fatalError(_("Error closing 3d raster map"));
 

raster3d/r3.gwflow/main.c

@@ -364,7 +364,7 @@ write_result(N_array_3d * status, N_array_3d * phead_start,
     depths = region->depths;
 
     /*Open the new map */
-    map = G3d_openCellNew(name, DCELL_TYPE, G3D_USE_CACHE_DEFAULT, region);
+    map = G3d_openNewOptTileSize(name, G3D_USE_CACHE_XY, region, DCELL_TYPE, 32);
 
     if (map == NULL)
 	G3d_fatalError(_("Error opening g3d map <%s>"), name);
@@ -412,6 +412,9 @@ write_result(N_array_3d * status, N_array_3d * phead_start,
 		G3d_maskOff(map);
     }
 
+    /* Flush all tile */
+    if (!G3d_flushAllTiles(map))
+	G3d_fatalError("Error flushing tiles with G3d_flushAllTiles");
     if (!G3d_closeCell(map))
 	G3d_fatalError(map, NULL, 0, _("Error closing g3d file"));
 

raster3d/r3.info/r3.info.main.c

@@ -214,8 +214,8 @@ int main(int argc, char *argv[])
 	    G_fatal_error(_("Cannot allocate memory for string"));
 
 	if (G_asprintf(&line, "  Data Type:    %s",
-		       (data_type == FCELL_TYPE ? "float" :
-			(data_type == DCELL_TYPE ? "double" : "??"))) > 0)
+		       (data_type == FCELL_TYPE ? "FCELL" :
+			(data_type == DCELL_TYPE ? "DCELL" : "??"))) > 0)
 	    printline(line);
 	else
 	    G_fatal_error(_("Cannot allocate memory for string"));
@@ -244,6 +244,60 @@ int main(int argc, char *argv[])
 	    else
 		G_fatal_error(_("Cannot allocate memory for string"));
 
+            double totalSize = 0;
+            for(i = 0; i < ((G3D_Map* )g3map)->nTiles; i++)
+                totalSize += ((G3D_Map* )g3map)->tileLength[i];
+
+	    if (G_asprintf(&line, "  Total size:           %ld Bytes",
+                (long)(totalSize)) > 0)
+		printline(line);
+	    else
+		G_fatal_error(_("Cannot allocate memory for string"));
+
+	    if (G_asprintf(&line, "  Number of tiles:      %ld",
+                ((G3D_Map* )g3map)->nTiles) > 0)
+		printline(line);
+	    else
+		G_fatal_error(_("Cannot allocate memory for string"));
+
+	    if (G_asprintf(&line, "  Mean tile size:       %ld Bytes",
+                (long)(totalSize/((G3D_Map* )g3map)->nTiles)) > 0)
+		printline(line);
+	    else
+		G_fatal_error(_("Cannot allocate memory for string"));
+
+            int tileSize = 0;
+
+            if(data_type == FCELL_TYPE)
+                tileSize = sizeof(FCELL) * ((G3D_Map* )g3map)->tileX * ((G3D_Map* )g3map)->tileY *
+                        ((G3D_Map* )g3map)->tileZ;
+
+            if(data_type == DCELL_TYPE)
+                tileSize = sizeof(DCELL) * ((G3D_Map* )g3map)->tileX * ((G3D_Map* )g3map)->tileY *
+                        ((G3D_Map* )g3map)->tileZ;
+
+	    if (G_asprintf(&line, "  Tile size in memory:  %ld Bytes",
+                (long)(tileSize)) > 0)
+		printline(line);
+	    else
+		G_fatal_error(_("Cannot allocate memory for string"));
+
+	    if (G_asprintf(&line, "  Number of tiles in x, y and  z:   %d, %d, %d",
+                                  ((G3D_Map* )g3map)->nx, ((G3D_Map* )g3map)->ny,
+                                  ((G3D_Map* )g3map)->nz) > 0)
+		printline(line);
+	    else
+		G_fatal_error(_("Cannot allocate memory for string"));
+
+	    if (G_asprintf(&line, "  Dimension of a tile in x, y, z:   %d, %d, %d",
+                                  ((G3D_Map* )g3map)->tileX, ((G3D_Map* )g3map)->tileY,
+                                  ((G3D_Map* )g3map)->tileZ) > 0)
+		printline(line);
+	    else
+		G_fatal_error(_("Cannot allocate memory for string"));
+
+            printline("");
+
 	    if (G_asprintf
 		(&line, "       Projection: %s (zone %d)",
 		 G_database_projection_name(), G_zone()) > 0)
@@ -379,8 +433,8 @@ int main(int argc, char *argv[])
 	}			/*Datatype */
 	else if (tflag->answer) {
 	    fprintf(out, "datatype=\"%s\"\n",
-		    data_type == FCELL_TYPE ? "float" :
-		    data_type == DCELL_TYPE ? "double" :
+		    data_type == FCELL_TYPE ? "FCELL" :
+		    data_type == DCELL_TYPE ? "DCELL" :
 		    "??");
 
 	}			/*History output */