raster module memory: set new value globally (#922)

* raster module memory: set other value globally

This change allows to globally override the default 300 MB with a user defined value:

r.in.gdal --help
Imports raster data into a GRASS raster map using GDAL library.

Usage:
 r.in.gdal [-ojeflakcrp] input=name output=name
...
Parameters:
           input   Name of raster file to be imported
          output   Name for output raster map
            band   Band(s) to select (default is all bands)
          memory   Maximum memory to be used (in MB)
                   default: 300
          target   Name of GCPs target location
           title   Title for resultant raster map
          offset   Offset to be added to band numbers
                   default: 0

g.gisenv set="MEMORYMB=6000"

r.in.gdal --help
Imports raster data into a GRASS raster map using GDAL library.

Usage:
 r.in.gdal [-ojeflakcrp] input=name output=name
...
Parameters:
           input   Name of raster file to be imported
          output   Name for output raster map
            band   Band(s) to select (default is all bands)
          memory   Maximum memory to be used (in MB)
                   default: 6000
          target   Name of GCPs target location
           title   Title for resultant raster map
          offset   Offset to be added to band numbers
                   default: 0

g.gisenv set="MEMORYMB=300"

Based upon demo patch my @metzm

* fix double description

* code simplification by defining new G_define_standard_option(G_OPT_MEMORYMB)

* update STD_OPT_STRINGS after include/gis.h modification

* Patch added to also set default answer at compile time

* document new GRASS variable MEMORYMB

- fixes the creation of man pages

https://github.com/OSGeo/grass/pull/922#issuecomment-678823688

Contributed by @metzm

general/g.parser/standard_option.c

@@ -12,6 +12,7 @@ static char* STD_OPT_STRINGS[] = {"G_OPT_UNDEFINED",
                                   "G_OPT_DB_KEYCOLUMN",
                                   "G_OPT_I_GROUP",
                                   "G_OPT_I_SUBGROUP",
+                                  "G_OPT_MEMORYMB",
                                   "G_OPT_R_INPUT",
                                   "G_OPT_R_INPUTS",
                                   "G_OPT_R_OUTPUT",

imagery/i.ortho.photo/i.ortho.rectify/main.c

@@ -81,6 +81,7 @@ int main(int argc, char *argv[])
     struct Flag *c, *a, *pan_flag;
     struct GModule *module;
 
+
     G_gisinit(argv[0]);
 
     module = G_define_module();
@@ -107,13 +108,7 @@ int main(int argc, char *argv[])
     tres->required = NO;
     tres->description = _("Target resolution (ignored if -c flag used)");
 
-    mem = G_define_option();
-    mem->key = "memory";
-    mem->type = TYPE_DOUBLE;
-    mem->key_desc = "memory in MB";
-    mem->required = NO;
-    mem->answer = "300";
-    mem->description = _("Amount of memory to use in MB");
+    mem = G_define_standard_option(G_OPT_MEMORYMB);
 
     ipolname = make_ipol_list();
 

imagery/i.rectify/main.c

@@ -77,6 +77,7 @@ int main(int argc, char *argv[])
     struct Flag *c, *a, *t;
     struct GModule *module;
 
+
     G_gisinit(argv[0]);
 
     module = G_define_module();
@@ -113,14 +114,8 @@ int main(int argc, char *argv[])
     tres->required = NO;
     tres->description = _("Target resolution (ignored if -c flag used)");
 
-    mem = G_define_option();
-    mem->key = "memory";
-    mem->type = TYPE_DOUBLE;
-    mem->key_desc = "memory in MB";
-    mem->required = NO;
-    mem->answer = "300";
-    mem->description = _("Amount of memory to use in MB");
-
+    mem = G_define_standard_option(G_OPT_MEMORYMB);
+ 
     ipolname = make_ipol_list();
 
     interpol = G_define_option();

imagery/i.segment/parse_args.c

@@ -20,6 +20,7 @@ int parse_args(int argc, char *argv[], struct globals *globals)
     struct Option *gof, *endt;
     int bands;
 
+
     /* required parameters */
     group = G_define_standard_option(G_OPT_R_INPUTS);
     group->key = "group";
@@ -112,12 +113,7 @@ int parse_args(int argc, char *argv[], struct globals *globals)
     smooth_weight->guisection = _("Settings");
 #endif
 
-    mem = G_define_option();
-    mem->key = "memory";
-    mem->type = TYPE_INTEGER;
-    mem->required = NO;
-    mem->answer = "300";
-    mem->description = _("Memory in MB");
+    mem = G_define_standard_option(G_OPT_MEMORYMB);
 
     /* TODO input for distance function */
 

include/gis.h

@@ -241,6 +241,8 @@ typedef enum
 
     G_OPT_I_GROUP,		/*!< old input imagery group */
     G_OPT_I_SUBGROUP,		/*!< old input imagery subgroup */
+
+    G_OPT_MEMORYMB,		/*!< Maximum memory to be used (in MB): cache size for raster rows */
     G_OPT_R_INPUT,		/*!< old input raster map */
     G_OPT_R_INPUTS,		/*!< old input raster maps */
     G_OPT_R_OUTPUT,		/*!< new output raster map */

lib/gis/parser_standard_options.c

@@ -52,6 +52,7 @@
    - G_OPT_I_SUBGROUP
 
   - raster:
+   - G_OPT_MEMORYMB
    - G_OPT_R_INPUT
    - G_OPT_R_INPUTS
    - G_OPT_R_OUTPUT
@@ -137,6 +138,7 @@
 struct Option *G_define_standard_option(int opt)
 {
     struct Option *Opt;
+    char *memstr;
 
     Opt = G_define_option();
 
@@ -245,6 +247,22 @@ struct Option *G_define_standard_option(int opt)
 	break;
 
 	/* raster maps */
+    case G_OPT_MEMORYMB:
+	Opt->key = "memory";
+	Opt->type = TYPE_INTEGER;
+	Opt->key_desc = "memory in MB";
+	Opt->required = NO;
+	Opt->multiple = NO;
+	Opt->answer = "300";
+	/* start dynamic answer */
+	/* check MEMORYMB in GISRC, set with g.gisenv */
+	memstr = G_store(G_getenv_nofatal("MEMORYMB"));
+	if (memstr && *memstr)
+	    Opt->answer = memstr;
+	/* end dynamic answer */
+	Opt->label = _("Maximum memory to be used (in MB)");
+	Opt->description = _("Cache size for raster rows");
+	break;
     case G_OPT_R_INPUT:
 	Opt->key = "input";
 	Opt->type = TYPE_STRING;

lib/init/variables.html

@@ -531,6 +531,14 @@ g.gisenv set=DEBUG=0
   <dt>MAPSET</dt>
   <dd>initial mapset</dd>
   
+  <dt>MEMORYMB</dt>
+  <dd>[entire GRASS with focus on raster related data processing]<br>
+    sets the maximum memory to be used (in MB), i.e. the cache size for raster rows
+<div class="code"><pre>
+# set to 6 GB (default: 300 MB)
+g.gisenv set="MEMORYMB=6000"
+</pre></div>
+
   <dt>OVERWRITE</dt>
   <dd>[all modules]<br>
     toggles map overwrite.

man/parser_standard_options.py

@@ -50,7 +50,14 @@ def parse_options(lines, startswith='Opt'):
     def parse_glines(glines):
         res = {}
         key = None
+        dynamic_answer = False
         for line in glines:
+            if line.strip() == "/* start dynamic answer */":
+                dynamic_answer = True
+            if line.strip() == "/* end dynamic answer */":
+                dynamic_answer = False
+            if dynamic_answer or line.startswith('/*'):
+                continue
             if line.startswith('/*'):
                 continue
             if line.startswith(startswith) and line.endswith(';'):

raster/r.cost/main.c

@@ -154,6 +154,7 @@ int main(int argc, char *argv[])
     double peak = 0.0;
     int dsize, nearest_size;
     double disk_mb, mem_mb, pq_mb;
+
     int dir_bin;
     DCELL mysolvedir[2], solvedir[2];
 
@@ -249,14 +250,7 @@ int main(int argc, char *argv[])
 	_("Cost assigned to null cells. By default, null cells are excluded");
     opt6->guisection = _("NULL cells");
 
-    opt10 = G_define_option();
-    opt10->key = "memory";
-    opt10->type = TYPE_INTEGER;
-    opt10->key_desc = "value";
-    opt10->required = NO;
-    opt10->multiple = NO;
-    opt10->answer = "300";
-    opt10->description = _("Maximum memory to be used in MB");
+    opt10 = G_define_standard_option(G_OPT_MEMORYMB);
 
     flag2 = G_define_flag();
     flag2->key = 'k';

raster/r.in.gdal/main.c

@@ -116,6 +116,7 @@ int main(int argc, char *argv[])
     struct Flag *flag_o, *flag_e, *flag_k, *flag_f, *flag_l, *flag_c, *flag_p,
         *flag_j, *flag_a, *flag_r;
 
+
     /* -------------------------------------------------------------------- */
     /*      Initialize.                                                     */
     /* -------------------------------------------------------------------- */
@@ -144,16 +145,10 @@ int main(int argc, char *argv[])
     parm.band->description = _("Band(s) to select (default is all bands)");
     parm.band->guisection = _("Bands");
 
-    parm.memory = G_define_option();
-    parm.memory->key = "memory";
-    parm.memory->type = TYPE_INTEGER;
-    parm.memory->required = NO;
+    parm.memory = G_define_standard_option(G_OPT_MEMORYMB);
 #if GDAL_VERSION_NUM < 1800
     parm.memory->options = "0-2047";
 #endif
-    parm.memory->answer = "300";
-    parm.memory->label = _("Maximum memory to be used (in MB)");
-    parm.memory->description = _("Cache size for raster rows");
 
     parm.target = G_define_option();
     parm.target->key = "target";

raster/r.proj/main.c

@@ -186,13 +186,7 @@ int main(int argc, char **argv)
     interpol->guisection = _("Target");
     interpol->descriptions = make_ipol_desc();
 
-    memory = G_define_option();
-    memory->key = "memory";
-    memory->type = TYPE_INTEGER;
-    memory->required = NO;
-    memory->answer = "300";
-    memory->label = _("Maximum memory to be used (in MB)");
-    memory->description = _("Cache size for raster rows");
+    memory = G_define_standard_option(G_OPT_MEMORYMB);
 
     res = G_define_option();
     res->key = "resolution";

raster/r.resamp.bspline/main.c

@@ -50,6 +50,7 @@ int main(int argc, char *argv[])
     int seg_mb, segments_in_memory;
     int have_mask;
 
+
     int inrastfd, outrastfd;
     DCELL *drastbuf, dval;
     struct History history;
@@ -145,13 +146,7 @@ int main(int argc, char *argv[])
     cross_corr_flag->description =
 	_("Find the best Tykhonov regularizing parameter using a \"leave-one-out\" cross validation method");
 
-    memory_opt = G_define_option();
-    memory_opt->key = "memory";
-    memory_opt->type = TYPE_INTEGER;
-    memory_opt->required = NO;
-    memory_opt->answer = "300";
-    memory_opt->label = _("Maximum memory to be used (in MB)");
-    memory_opt->description = _("Cache size for raster rows");
+    memory_opt = G_define_standard_option(G_OPT_MEMORYMB);
 
     /*----------------------------------------------------------------*/
     /* Parsing */

raster/r.stream.extract/main.c

@@ -132,13 +132,7 @@ int main(int argc, char *argv[])
     input.min_stream_length->description =
 	_("Applies only to first-order stream segments (springs/stream heads)");
 
-    input.memory = G_define_option();
-    input.memory->key = "memory";
-    input.memory->type = TYPE_INTEGER;
-    input.memory->required = NO;
-    input.memory->answer = "300";
-    input.memory->label = _("Maximum memory to be used (in MB)");
-    input.memory->description = _("Cache size for raster rows");
+    input.memory = G_define_standard_option(G_OPT_MEMORYMB);
 
     output.stream_rast = G_define_standard_option(G_OPT_R_OUTPUT);
     output.stream_rast->key = "stream_raster";

raster/r.terraflow/main.cpp

@@ -128,14 +128,9 @@ parse_args(int argc, char *argv[]) {
       "SFD (D8) direction (meaningful only for MFD flow). "
       "If no answer is given it defaults to infinity.");
 
-  /* main memory */
+  /* raster cache memory */
   struct Option *mem;
-  mem = G_define_option() ;
-  mem->key         = "memory";
-  mem->type        = TYPE_INTEGER;
-  mem->required    = NO;
-  mem->answer      = (char *) "300";
-  mem->description = _("Maximum memory to be used (in MB)");
+  mem = G_define_standard_option(G_OPT_MEMORYMB);
 
   /* temporary STREAM path */
   struct Option *streamdir;
@@ -143,7 +138,7 @@ parse_args(int argc, char *argv[]) {
   streamdir->key        = "directory";
   streamdir->type       = TYPE_STRING;
   streamdir->required   = NO;
-  //streamdir->answer     = "";
+  /* streamdir->answer     = ""; */
   streamdir->description=
      _("Directory to hold temporary files (they can be large)");
 

raster/r.walk/main.c

@@ -193,6 +193,7 @@ int main(int argc, char *argv[])
     double peak = 0.0;
     int dtm_dsize, cost_dsize, nearest_size;
     double disk_mb, mem_mb, pq_mb;
+
     int dir_bin;
     DCELL mysolvedir[2], solvedir[2];
 
@@ -295,14 +296,7 @@ int main(int argc, char *argv[])
 	_("Cost assigned to null cells. By default, null cells are excluded");
     opt6->guisection = _("NULL cells");
 
-    opt10 = G_define_option();
-    opt10->key = "memory";
-    opt10->type = TYPE_INTEGER;
-    opt10->key_desc = "value";
-    opt10->required = NO;
-    opt10->multiple = NO;
-    opt10->answer = "300";
-    opt10->description = _("Maximum memory to be used in MB");
+    opt10 = G_define_standard_option(G_OPT_MEMORYMB);
 
     opt15 = G_define_option();
     opt15->key = "walk_coeff";

raster/r.watershed/front/main.c

@@ -68,6 +68,7 @@ int main(int argc, char *argv[])
     struct Flag *flag_flat;
     struct GModule *module;
 
+
     G_gisinit(argv[0]);
 
     /* Set description */
@@ -208,12 +209,7 @@ int main(int argc, char *argv[])
     opt15->description =
 	_("1 = most diverging flow, 10 = most converging flow. Recommended: 5");
 
-    opt16 = G_define_option();
-    opt16->key = "memory";
-    opt16->type = TYPE_INTEGER;
-    opt16->required = NO;
-    opt16->answer = "300";	/* 300MB default value, please keep r.terraflow in sync */
-    opt16->description = _("Maximum memory to be used with -m flag (in MB)");
+    opt16 = G_define_standard_option(G_OPT_MEMORYMB);
 
     flag_sfd = G_define_flag();
     flag_sfd->key = 's';

scripts/r.fillnulls/r.fillnulls.py

@@ -97,14 +97,7 @@
 #% answer: 0.01
 #% guisection: Spline options
 #%end
-#%option
-#% key: memory
-#% type: integer
-#% required: no
-#% multiple: no
-#% label: Maximum memory to be used (in MB)
-#% description: Cache size for raster rows
-#% answer: 300
+#%option G_OPT_MEMORYMB
 #%end
 
 

scripts/r.import/r.import.py

@@ -34,14 +34,7 @@
 #% description: Input band(s) to select (default is all bands)
 #% guisection: Input
 #%end
-#%option
-#% key: memory
-#% type: integer
-#% required: no
-#% multiple: no
-#% label: Maximum memory to be used (in MB)
-#% description: Cache size for raster rows
-#% answer: 300
+#%option G_OPT_MEMORYMB
 #%end
 #%option G_OPT_R_OUTPUT
 #% description: Name for output raster map

temporal/t.rast.import/t.rast.import.py

@@ -75,14 +75,7 @@
 #% multiple: no
 #%end
 
-#%option
-#% key: memory
-#% type: integer
-#% description: Cache size for raster rows
-#% label: Maximum memory to be used (in MB)
-#% options: 0-2047
-#% answer: 300
-#% multiple: no
+#%option G_OPT_MEMORYMB
 #%end
 
 #%flag

vector/v.surf.bspline/main.c

@@ -71,6 +71,7 @@ int main(int argc, char *argv[])
 	*memory_opt, *solver, *error, *iter;
     struct Flag *cross_corr_flag, *spline_step_flag;
 
+
     struct Reg_dimens dims;
     struct Cell_head elaboration_reg, original_reg;
     struct bound_box general_box, overlap_box, original_box;
@@ -185,13 +186,7 @@ int main(int argc, char *argv[])
 
     error = N_define_standard_option(N_OPT_ITERATION_ERROR);
 
-    memory_opt = G_define_option();
-    memory_opt->key = "memory";
-    memory_opt->type = TYPE_INTEGER;
-    memory_opt->required = NO;
-    memory_opt->answer = "300";
-    memory_opt->label = _("Maximum memory to be used (in MB)");
-    memory_opt->description = _("Cache size for raster rows");
+    memory_opt = G_define_standard_option(G_OPT_MEMORYMB);
 
     /*----------------------------------------------------------------*/
     /* Parsing */

vector/v.to.rast/main.c

@@ -107,14 +107,7 @@ int main(int argc, char *argv[])
     val_opt->answer = "1";
     val_opt->description = _("Raster value (for use=val)");
     
-    memory = G_define_option();
-    memory->key = "memory";
-    memory->type = TYPE_INTEGER;
-    memory->required = NO;
-    memory->multiple = NO;
-    memory->answer = "300";
-    memory->label = _("Maximum memory to be used (in MB)");
-    memory->description = _("Cache size for raster rows");
+    memory = G_define_standard_option(G_OPT_MEMORYMB);
 
     dense_flag = G_define_flag();
     dense_flag->key = 'd';