temporal modules: Enabled granularity gap sampling. New python tests are implemented.

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@64651 15284696-431f-4ddb-bdfa-cd5b030d7da7

temporal/t.rast.gapfill/t.rast.gapfill.html

@@ -10,14 +10,19 @@ map between them.
 This module uses <a href="r.series.interp.html">r.series.interp</a> to
 perform the interpolation for each gap independently. Hence several
 interpolation processes can be run in parallel.
+<p>
+Each gap is re-sampled by the space time raster dataset granularity. 
+Therefore several time stamped raster map layers will be interpolated 
+if the gap is larger than the STRDS granularity.
 
 <h2>Examples</h2>
 
 In this example we will create 3 raster maps and register them in the
 temporal database an then in the newly created space time raster dataset.
-There are gaps of one day size between the raster maps. The values of
+There are gaps of one and two day size between the raster maps. The values of
 the maps are chosen so that the interpolated values can be estimated.
-We expect two maps with values 2 and 4 after interpolation.
+We expect one map with a value of 2 for the first gap and
+two maps (values 3.666 and 4.333) for the second gap after interpolation.
 
 <div class="code"><pre>
 r.mapcalc expr="map1 = 1" 
@@ -26,7 +31,7 @@ r.mapcalc expr="map3 = 5"
 
 t.register type=raster maps=map1 start=2012-08-20 end=2012-08-21
 t.register type=raster maps=map2 start=2012-08-22 end=2012-08-23
-t.register type=raster maps=map3 start=2012-08-24 end=2012-08-25
+t.register type=raster maps=map3 start=2012-08-25 end=2012-08-26
 
 t.create type=strds temporaltype=absolute \
          output=precipitation_daily \
@@ -40,7 +45,7 @@ t.rast.list input=precipitation_daily columns=name,start_time,min,max
 name|start_time|min|max
 map1|2012-08-20 00:00:00|1.0|1.0
 map2|2012-08-22 00:00:00|3.0|3.0
-map3|2012-08-24 00:00:00|5.0|5.0
+map3|2012-08-25 00:00:00|5.0|5.0
 
 
 t.rast.list input=precipitation_daily method=deltagaps
@@ -58,10 +63,12 @@ t.rast.list input=precipitation_daily columns=name,start_time,min,max
 
 name|start_time|min|max
 map1|2012-08-20 00:00:00|1.0|1.0
-gap_6|2012-08-21 00:00:00|2.0|2.0
+gap_6_1|2012-08-21 00:00:00|2.0|2.0
 map2|2012-08-22 00:00:00|3.0|3.0
-gap_7|2012-08-23 00:00:00|4.0|4.0
-map3|2012-08-24 00:00:00|5.0|5.0
+gap_7_1|2012-08-23 00:00:00|3.666667|3.666667
+gap_7_2|2012-08-24 00:00:00|4.333333|4.333333
+map3|2012-08-25 00:00:00|5.0|5.0
+
 </pre></div>
 
 <h2>SEE ALSO</h2>

temporal/t.rast.gapfill/t.rast.gapfill.py

@@ -51,10 +51,15 @@
 #% description: Assign the space time raster dataset start and end time to the output map
 #%end
 
+import sys
+import copy
 from multiprocessing import Process
 import grass.script as grass
 import grass.temporal as tgis
 
+import grass.pygrass.modules as pymod
+from grass.exceptions import FatalError
+
 ############################################################################
 
 
@@ -81,6 +86,13 @@ def main():
 
     num = len(maps)
 
+    # Configure the r.to.vect module
+    gapfill_module = pymod.Module("r.series.interp",
+                             overwrite=grass.overwrite(), quiet=True, run_=False,
+                             finish_=False,)
+
+    process_queue = pymod.ParallelModuleQueue(int(nprocs))
+
     gap_list = []
     overwrite_flags = {}
 
@@ -91,15 +103,6 @@ def main():
             count += 1
             _id = "%s_%d@%s" % (base, num + count, mapset)
             _map.set_id(_id)
-            overwrite_flags[_id] = False
-            if _map.map_exists() or _map.is_in_db(dbif):
-                if not grass.overwrite:
-                        grass.fatal(_("Map with name <%s> already exists. "
-                                      "Please use another base name." % (_id)))
-                else:
-                    if _map.is_in_db(dbif):
-                        overwrite_flags[_id] = True
-
 
             gap_list.append(_map)
 
@@ -126,42 +129,57 @@ def main():
                             "Using the first found."))
 
     # Interpolate the maps using parallel processing
-    proc_list = []
-    proc_count = 0
-    num = len(gap_list)
+    result_list = []
 
     for _map in gap_list:
         predecessor = _map.get_follows()[0]
         successor = _map.get_precedes()[0]
 
-        # Build the module inputs strings
-        inputs = "%s,%s" % (predecessor.get_map_id(), successor.get_map_id())
-        dpos = "0,1"
-        output = "%s" % (_map.get_name())
-        outpos = "0.5"
-
-        # Start several processes in parallel
-        proc_list.append(Process(
-            target=run_interp, args=(inputs, dpos, output, outpos)))
-        proc_list[proc_count].start()
-        proc_count += 1
+        gran = sp.get_granularity()
+        tmpval,  start = predecessor.get_temporal_extent_as_tuple()
+        end,  tmpval = successor.get_temporal_extent_as_tuple()
+        
+        # Now resample the gap
+        map_matrix = tgis.AbstractSpaceTimeDataset.resample_maplist_by_granularity((_map, ),start, end,  gran)
+        
+        map_names = []
+        map_positions = []
+        
+        increment = 1.0/ (len(map_matrix) + 1.0)
+        position = increment
+        count = 0
+        for intp_list in map_matrix:
+            new_map = intp_list[0]
+            count += 1
+            new_id = "%s_%i@%s"%(_map.get_name(),  count,  tgis.get_current_mapset())
+            new_map.set_id(new_id)
+            
+            overwrite_flags[new_id] = False
+            if new_map.map_exists() or new_map.is_in_db(dbif):
+                if not grass.overwrite:
+                        grass.fatal(_("Map with name <%s> already exists. "
+                                      "Please use another base name." % (_id)))
+                else:
+                    if new_map.is_in_db(dbif):
+                        overwrite_flags[new_id] = True
 
-        if proc_count == nprocs or proc_count == num:
-            proc_count = 0
-            exitcodes = 0
-            for proc in proc_list:
-                proc.join()
-                exitcodes += proc.exitcode
+            map_names.append(new_map.get_name())
+            map_positions.append(position)
+            position += increment
+            
+            result_list.append(new_map)
 
-            if exitcodes != 0:
-                dbif.close()
-                grass.fatal(_("Error while interpolation computation"))
+        mod = copy.deepcopy(gapfill_module)
+        mod(input=(predecessor.get_map_id(), successor.get_map_id()),
+                datapos=(0, 1), output=map_names,  samplingpos=map_positions)
+        sys.stderr.write(mod.get_bash() + "\n")
+        process_queue.put(mod)
 
-            # Empty process list
-            proc_list = []
+    # Wait for unfinished processes
+    process_queue.wait()
 
     # Insert new interpolated maps in temporal database and dataset
-    for _map in gap_list:
+    for _map in result_list:
         id = _map.get_id()
         if overwrite_flags[id] == True:
             if _map.is_time_absolute():

temporal/t.rast.gapfill/test.t.rast.gapfill.sh

@@ -1,37 +0,0 @@
-#!/bin/sh
-# Fill the gaps in a space time raster dataset
-# We need to set a specific region in the
-# @preprocess step of this test. 
-# The region setting should work for UTM and LL test locations
-
-export GRASS_OVERWRITE=1
-
-g.region s=0 n=80 w=0 e=120 b=0 t=50 res=10 res3=10 -p3
-
-r.mapcalc  expr="prec_1 = 100"
-r.mapcalc  expr="prec_2 = 300"
-r.mapcalc  expr="prec_3 = 500"
-
-n1=`g.tempfile pid=1 -d` 
-
-cat > "${n1}" << EOF
-prec_1|2001-01-01|2001-02-01
-prec_2|2001-03-01|2001-04-01
-prec_3|2001-05-01|2001-06-01
-EOF
-
-t.create --v  type=strds temporaltype=absolute output=precip_abs title="A test" descr="A test"
-t.register --v type=raster input=precip_abs file="${n1}"
-
-# @test
-t.rast.gapfill input=precip_abs base="prec" nprocs=2
-t.info precip_abs
-
-t.info type=raster input=prec_6
-t.info type=raster input=prec_7
-
-# @postprocess
-
-t.unregister --v type=raster maps=prec_1,prec_2,prec_3,prec_6,prec_7
-t.remove --v type=strds input=precip_abs
-g.remove -f type=raster name=prec_1,prec_2,prec_3,prec_6,prec_7

temporal/t.rast.gapfill/testsuite/test_gapfill.py

@@ -0,0 +1,186 @@
+"""Test t.rast.to.vect
+
+(C) 2014 by the GRASS Development Team
+This program is free software under the GNU General Public
+License (>=v2). Read the file COPYING that comes with GRASS
+for details.
+
+@author Soeren Gebbert
+"""
+
+import subprocess
+from grass.gunittest.case import TestCase
+from grass.gunittest.gmodules import SimpleModule
+
+class TestRasterToVector(TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        """Initiate the temporal GIS and set the region
+        """
+        cls.use_temp_region()
+        cls.runModule("g.region", s=0, n=80, w=0, e=120, b=0, t=50, res=10, res3=10)
+
+    def setUp(self):
+        self.runModule("r.mapcalc", expression="a_1 = 100", overwrite=True)
+        self.runModule("r.mapcalc", expression="a_2 = 400", overwrite=True)
+        self.runModule("r.mapcalc", expression="a_3 = 1200", overwrite=True)
+
+        self.runModule("t.create", type="strds", temporaltype="absolute",  
+                                 output="A", title="A test", description="A test",  
+                                 overwrite=True)
+        self.runModule("t.register",  flags="i", type="raster", input="A",  
+                                     maps="a_1", start="2001-01-01", 
+                                     increment="1 month", overwrite=True)
+        self.runModule("t.register",  flags="i", type="raster", input="A",  
+                                     maps="a_2", start="2001-04-01", 
+                                     increment="1 months", overwrite=True)
+        self.runModule("t.register",  flags="i", type="raster", input="A",  
+                                     maps="a_3", start="2001-12-01", 
+                                     increment="1 months", overwrite=True)
+
+    @classmethod
+    def tearDownClass(cls):
+        """Remove the temporary region
+        """
+        cls.del_temp_region()
+        
+    def tearDown(self):
+        """Remove generated data"""
+        self.runModule("t.remove", flags="rf", type="strds",  
+                                   inputs="A")
+
+    def test_simple_2procs(self):
+        self.assertModule("t.rast.gapfill", input="A", 
+                                     basename="test", nprocs=2, verbose=True)
+
+        #self.assertModule("t.info",  type="strds", flags="g",  input="A")
+
+        tinfo_string="""start_time=2001-01-01 00:00:00
+                                end_time=2002-01-01 00:00:00
+                                granularity=1 month
+                                map_time=interval
+                                number_of_maps=12
+                                min_min=100.0
+                                min_max=1200.0
+                                max_min=100.0
+                                max_max=1200.0"""
+
+        info = SimpleModule("t.info", flags="g", type="strds", input="A")
+        self.assertModuleKeyValue(module=info, reference=tinfo_string, precision=2, sep="=")
+
+        text="""name|start_time|end_time|min|max
+a_1|2001-01-01 00:00:00|2001-02-01 00:00:00|100.0|100.0
+test_6_1|2001-02-01 00:00:00|2001-03-01 00:00:00|200.0|200.0
+test_6_2|2001-03-01 00:00:00|2001-04-01 00:00:00|300.0|300.0
+a_2|2001-04-01 00:00:00|2001-05-01 00:00:00|400.0|400.0
+test_7_1|2001-05-01 00:00:00|2001-06-01 00:00:00|500.0|500.0
+test_7_2|2001-06-01 00:00:00|2001-07-01 00:00:00|600.0|600.0
+test_7_3|2001-07-01 00:00:00|2001-08-01 00:00:00|700.0|700.0
+test_7_4|2001-08-01 00:00:00|2001-09-01 00:00:00|800.0|800.0
+test_7_5|2001-09-01 00:00:00|2001-10-01 00:00:00|900.0|900.0
+test_7_6|2001-10-01 00:00:00|2001-11-01 00:00:00|1000.0|1000.0
+test_7_7|2001-11-01 00:00:00|2001-12-01 00:00:00|1100.0|1100.0
+a_3|2001-12-01 00:00:00|2002-01-01 00:00:00|1200.0|1200.0
+
+"""
+        rast_list = SimpleModule("t.rast.list", columns=("name","start_time","end_time","min,max"),  input="A")
+        self.assertModule(rast_list)
+        self.assertLooksLike(text,  rast_list.outputs.stdout)
+
+    def test_simple_where(self):
+        self.assertModule("t.rast.gapfill", input="A",  where="start_time >= '2001-03-01'", 
+                          basename="test", nprocs=1, verbose=True)
+
+        #self.assertModule("t.info",  type="strds", flags="g",  input="A")
+
+        tinfo_string="""start_time=2001-01-01 00:00:00
+                                end_time=2002-01-01 00:00:00
+                                granularity=1 month
+                                map_time=interval
+                                number_of_maps=10
+                                min_min=100.0
+                                min_max=1200.0
+                                max_min=100.0
+                                max_max=1200.0"""
+
+        info = SimpleModule("t.info", flags="g", type="strds", input="A")
+        self.assertModuleKeyValue(module=info, reference=tinfo_string, precision=2, sep="=")
+
+        text="""name|start_time|end_time|min|max
+a_1|2001-01-01 00:00:00|2001-02-01 00:00:00|100.0|100.0
+a_2|2001-04-01 00:00:00|2001-05-01 00:00:00|400.0|400.0
+test_4_1|2001-05-01 00:00:00|2001-06-01 00:00:00|500.0|500.0
+test_4_2|2001-06-01 00:00:00|2001-07-01 00:00:00|600.0|600.0
+test_4_3|2001-07-01 00:00:00|2001-08-01 00:00:00|700.0|700.0
+test_4_4|2001-08-01 00:00:00|2001-09-01 00:00:00|800.0|800.0
+test_4_5|2001-09-01 00:00:00|2001-10-01 00:00:00|900.0|900.0
+test_4_6|2001-10-01 00:00:00|2001-11-01 00:00:00|1000.0|1000.0
+test_4_7|2001-11-01 00:00:00|2001-12-01 00:00:00|1100.0|1100.0
+a_3|2001-12-01 00:00:00|2002-01-01 00:00:00|1200.0|1200.0
+"""
+        rast_list = SimpleModule("t.rast.list", columns=("name","start_time","end_time","min,max"),  input="A")
+        self.assertModule(rast_list)
+        self.assertLooksLike(text,  rast_list.outputs.stdout)
+
+    def test_simple_where_2(self):
+        self.assertModule("t.rast.gapfill", input="A",  where="start_time <= '2001-05-01'", 
+                          basename="test", nprocs=1, verbose=True)
+
+        #self.assertModule("t.info",  type="strds", flags="g",  input="A")
+
+        tinfo_string="""start_time=2001-01-01 00:00:00
+                                end_time=2002-01-01 00:00:00
+                                granularity=1 month
+                                map_time=interval
+                                number_of_maps=5
+                                min_min=100.0
+                                min_max=1200.0
+                                max_min=100.0
+                                max_max=1200.0"""
+
+        info = SimpleModule("t.info", flags="g", type="strds", input="A")
+        self.assertModuleKeyValue(module=info, reference=tinfo_string, precision=2, sep="=")
+
+        text="""name|start_time|end_time|min|max
+a_1|2001-01-01 00:00:00|2001-02-01 00:00:00|100.0|100.0
+test_4_1|2001-02-01 00:00:00|2001-03-01 00:00:00|200.0|200.0
+test_4_2|2001-03-01 00:00:00|2001-04-01 00:00:00|300.0|300.0
+a_2|2001-04-01 00:00:00|2001-05-01 00:00:00|400.0|400.0
+a_3|2001-12-01 00:00:00|2002-01-01 00:00:00|1200.0|1200.0
+"""
+        rast_list = SimpleModule("t.rast.list", columns=("name","start_time","end_time","min,max"),  input="A")
+        self.assertModule(rast_list)
+        self.assertLooksLike(text,  rast_list.outputs.stdout)
+
+    def test_simple_empty(self):
+        self.assertModule("t.rast.gapfill", input="A",  where="start_time >= '2001-10-01'", 
+                          basename="test", nprocs=1, verbose=True)
+
+        #self.assertModule("t.info",  type="strds", flags="g",  input="A")
+
+        tinfo_string="""start_time=2001-01-01 00:00:00
+                                end_time=2002-01-01 00:00:00
+                                granularity=1 month
+                                map_time=interval
+                                number_of_maps=3
+                                min_min=100.0
+                                min_max=1200.0
+                                max_min=100.0
+                                max_max=1200.0"""
+
+        info = SimpleModule("t.info", flags="g", type="strds", input="A")
+        self.assertModuleKeyValue(module=info, reference=tinfo_string, precision=2, sep="=")
+
+        text="""name|start_time|end_time|min|max
+a_1|2001-01-01 00:00:00|2001-02-01 00:00:00|100.0|100.0
+a_2|2001-04-01 00:00:00|2001-05-01 00:00:00|400.0|400.0
+a_3|2001-12-01 00:00:00|2002-01-01 00:00:00|1200.0|1200.0
+"""
+        rast_list = SimpleModule("t.rast.list", columns=("name","start_time","end_time","min,max"),  input="A")
+        self.assertModule(rast_list)
+        self.assertLooksLike(text,  rast_list.outputs.stdout)
+
+if __name__ == '__main__':
+    from grass.gunittest.main import test
+    test()