gunittest: use PyGRASS for the whole module run (use finish_ and try-except), few updates in doc

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@61238 15284696-431f-4ddb-bdfa-cd5b030d7da7

lib/python/docs/src/gunittest_testing.rst

@@ -48,7 +48,7 @@ line parameters. This is ensured using `gunittest.test()`.
 
 To run (invoke) all tests in the source tree run::
 
-    python python -m grass.gunittest.main [gisdbase] location test_data_category
+    python -m grass.gunittest.main [gisdbase] location test_data_category
 
 All test files in all ``testsuite`` directories will be executed and
 a report will be created in a newly created ``testreport`` directory.
@@ -81,6 +81,10 @@ documentation.
 Tests of GRASS modules
 ----------------------
 
+This is applicable for both GRASS modules written in C or C++ and
+GRASS modules written in Python since we are testing the whole module
+(which is invoked as a subprocess).
+
 ::
 
     def test_elevation(self):
@@ -104,22 +108,40 @@ in different locations.
 .. todo::
     Add example of assertions of key-value results.
 
-.. todo::
-    Add example with module producing a map.
+Especially if a module module has a lot of different parameters allowed
+in different combinations, you should test the if the wrong ones are really
+disallowed and proper error messages are provided (in addition, you can
+test things such as creation and removal of maps in error states).
 
 ::
 
     from grass.gunittest import SimpleModule
 
-    class TestRInfoInputHandling(TestCase):
+    class TestRInfoParameterHandling(TestCase):
+        """Test r.info handling of wrong input of parameters."""
 
         def test_rinfo_wrong_map(self):
+            """Test input of map which does not exist."""
             map_name = 'does_not_exist'
-            rinfo = SimpleModule('r.info', map=, flags='g')
+            # create a module instance suitable for testing
+            rinfo = SimpleModule('r.info', map=map_name, flags='g')
+            # test that module fails (ends with non-zero return code)
             self.assertModuleFail(rinfo)
+            # test that error output is not empty
             self.assertTrue(rinfo.outputs.stderr)
+            # test that the right map is mentioned in the error message
             self.assertIn(map_name, stderr)
 
+In some cases it might be advantageous to create a module instance
+in `setUp()` method and then modify it in test methods.
+
+.. note:
+    Test should be (natural) language, i.e. locale, independent
+    to allow testing the functionality under different locale settings.
+    So, if you are testing content of messages (which should be usually
+    translated), use `assertIn()` method (regular expression might be
+    applicable in some cases but in most cases `in` is exactly the
+    operation needed).
 
 
 Tests of C and C++ code
@@ -128,13 +150,20 @@ Tests of C and C++ code
 Tests of Python code
 --------------------
 
+Use `gunittest` for this purpose in the same way as `unittest`_ would be used.
+
 
 Testing Python code with doctest
 --------------------------------
 
-In Python, the easiest thing to test are functions which performs some computations
-or string manipulations, i.e. they have sum numbers or strings on the input and
-some others on the output.
+.. note::
+    The primary use of ``doctest`` is to ensure that the documentation
+    for functions and classes is valid. Additionally, it can increase
+    the number of tests when executed together with other tests.
+
+In Python, the easiest thing to test are functions which performs some
+computations or string manipulations, i.e. they have some numbers or strings
+on the input and some other numbers or strings on the output.
 
 At the beginning you can use doctest for this purpose. The syntax is as follows::
 
@@ -155,7 +184,6 @@ this to your script::
             import doctest
             doctest.testmod()
         else:
-           grass.parser()
            main()
 
 No output means that everything was successful. Note that you cannot use all
@@ -163,7 +191,7 @@ the ways of running doctest since doctest will fail don the module file due
 to the dot or dots in the file name. Moreover, it is sometimes required that
 the file is accessible through sys.path which is not true for case of GRASS modules.
 
-Do not use use doctest for tests of edge cases, for tests which require
+However, do not use use doctest for tests of edge cases, for tests which require
 generate complex data first, etc. In these cases use `gunittest`.
 
 

lib/python/gunittest/case.py

@@ -408,6 +408,10 @@ class TestCase(unittest.TestCase):
             finally:
                 call_module('g.remove', rast=diff)
         # general case
+        # TODO: we are using r.info min max and r.univar min max interchangably
+        # but they might be different if region is different from map
+        # not considered as an huge issue since we expect the tested maps
+        # to match with region, however a documentation should containe a notice
         self.assertRastersDifference(actual=actual, reference=reference,
                                      statistics=statistics,
                                      precision=precision, msg=msg)
@@ -446,19 +450,13 @@ class TestCase(unittest.TestCase):
         """
         module = _module_from_parameters(module, **kwargs)
         _check_module_run_parameters(module)
-
-        # do what module.run() with finish_=True would do
-        start = time.time()
-        module.run()
-        stdout, stderr = module.popen.communicate(input=module.stdin)
-        module.outputs['stdout'].value = stdout if stdout else ''
-        module.outputs['stderr'].value = stderr if stderr else ''
-        module.time = time.time() - start
-        if module.popen.poll():
+        try:
+            module.run()
+        except CalledModuleError:
             # here exception raised by run() with finish_=True would be
             # almost enough but we want some additional info to be included
             # in the test report
-            errors = module.outputs['stderr'].value
+            errors = module.outputs.stderr
             # provide diagnostic at least in English locale
             # TODO: standardized error code would be handy here
             import re
@@ -503,18 +501,11 @@ class TestCase(unittest.TestCase):
         """
         module = _module_from_parameters(module, **kwargs)
         _check_module_run_parameters(module)
-
-        # do what module.run() with finish_=True would do
-        start = time.time()
-        module.run()
-        stdout, stderr = module.popen.communicate(input=module.stdin)
-        module.outputs['stdout'].value = stdout if stdout else ''
-        module.outputs['stderr'].value = stderr if stderr else ''
-        module.time = time.time() - start
-        # TODO: these two lines should go to report in some better way
-        print module.outputs['stdout'].value
-        print module.outputs['stderr'].value
-        if module.popen.poll():
+        try:
+            module.run()
+        except CalledModuleError:
+            print module.outputs.stdout
+            print module.outputs.stderr
             # TODO: message format
             # TODO: stderr?
             stdmsg = ('Running <{m.name}> module ended'
@@ -523,10 +514,11 @@ class TestCase(unittest.TestCase):
                       'See the folowing errors:\n'
                       '{errors}'.format(
                           m=module, code=module.get_python(),
-                          errors=module.outputs["stderr"].value
+                          errors=module.outputs.stderr
                       ))
             self.fail(self._formatMessage(msg, stdmsg))
-
+        print module.outputs.stdout
+        print module.outputs.stderr
         # log these to final report
         # TODO: always or only if the calling test method failed?
         # in any case, this must be done before self.fail()
@@ -541,18 +533,16 @@ class TestCase(unittest.TestCase):
         """
         module = _module_from_parameters(module, **kwargs)
         _check_module_run_parameters(module)
-
-        # do what module.run() with finish_=True would do
-        start = time.time()
-        module.run()
-        stdout, stderr = module.popen.communicate(input=module.stdin)
-        module.outputs['stdout'].value = stdout if stdout else ''
-        module.outputs['stderr'].value = stderr if stderr else ''
-        module.time = time.time() - start
-        # TODO: these two lines should go to report in some better way
-        print module.outputs['stdout'].value
-        print module.outputs['stderr'].value
-        if not module.popen.poll():
+        # note that we cannot use finally because we do not leave except
+        try:
+            module.run()
+        except CalledModuleError:
+            print module.outputs.stdout
+            print module.outputs.stderr
+            pass
+        else:
+            print module.outputs.stdout
+            print module.outputs.stderr
             stdmsg = ('Running <%s> ended with zero (successful) return code'
                       ' when expecting module to fail' % module.get_python())
             self.fail(self._formatMessage(msg, stdmsg))
@@ -578,9 +568,9 @@ def _check_module_run_parameters(module):
     # in this case module already run and we would start it again
     if module.run_:
         raise ValueError('Do not run the module manually, set run_=False')
-    if module.finish_:
+    if not module.finish_:
         raise ValueError('This function will always finish module run,'
-                         ' set finish_=None or finish_=False.')
+                         ' set finish_=None or finish_=True.')
     # we expect most of the usages with stdout=PIPE
     # TODO: in any case capture PIPE always?
     if module.stdout_ is None:

lib/python/gunittest/gmodules.py

@@ -50,7 +50,7 @@ class SimpleModule(Module):
                                  ', it would be overriden' % banned)
         kargs['stdout_'] = subprocess.PIPE
         kargs['stderr_'] = subprocess.PIPE
-        kargs['finish_'] = False
+        kargs['finish_'] = True
         kargs['run_'] = False
 
         Module.__init__(self, cmd, *args, **kargs)

lib/python/gunittest/testsuite/test_assertions.py

@@ -85,7 +85,7 @@ mean=240.437
 
 
 class TestAssertCommandKeyValue(grass.gunittest.TestCase):
-    """Test usage of `.assertModuleKeyValue` method."""
+    """Test usage of `assertModuleKeyValue` method."""
     # pylint: disable=R0904
 
     @classmethod
@@ -98,9 +98,9 @@ class TestAssertCommandKeyValue(grass.gunittest.TestCase):
         cls.del_temp_region()
 
     def test_pygrass_module(self):
-        """Test syntax with Module as module"""
+        """Test syntax with Module and required parameters as module"""
         module = Module('r.info', map='elevation', flags='gr',
-                        run_=False, finish_=False)
+                        run_=False, finish_=True)
         self.assertModuleKeyValue(module,
                                   reference=dict(min=55.58, max=156.33),
                                   precision=0.01, sep='=')

lib/python/gunittest/testsuite/test_module_assertions.py

@@ -17,25 +17,25 @@ class TestModuleAssertions(grass.gunittest.TestCase):
     def setUp(self):
         """Create two Module instances one correct and one with wrong map"""
         self.rinfo = Module('r.info', map='elevation', flags='g',
-                            stdout_=subprocess.PIPE, run_=False, finish_=False)
+                            stdout_=subprocess.PIPE, run_=False, finish_=True)
         self.rinfo_wrong = copy.deepcopy(self.rinfo)
         self.wrong_map = 'does_not_exists'
         self.rinfo_wrong.inputs['map'].value = self.wrong_map
 
     def test_runModule(self):
-        """Module used in runModule()"""
+        """Correct and incorrect Module used in runModule()"""
         self.runModule(self.rinfo)
         self.assertTrue(self.rinfo.outputs['stdout'].value)
         self.assertRaises(CalledModuleError, self.runModule, self.rinfo_wrong)
 
     def test_assertModule(self):
-        """Module used in assertModule()"""
+        """Correct and incorrect Module used in assertModule()"""
         self.assertModule(self.rinfo)
         self.assertTrue(self.rinfo.outputs['stdout'].value)
         self.assertRaises(self.failureException, self.assertModule, self.rinfo_wrong)
 
     def test_assertModuleFail(self):
-        """Module used in assertModuleFail()"""
+        """Correct and incorrect Module used in assertModuleFail()"""
         self.assertModuleFail(self.rinfo_wrong)
         stderr = self.rinfo_wrong.outputs['stderr'].value
         self.assertTrue(stderr)
@@ -56,19 +56,19 @@ class TestSimpleModuleAssertions(grass.gunittest.TestCase):
         self.rinfo_wrong.inputs['map'].value = self.wrong_map
 
     def test_runModule(self):
-        """SimpleModule used in runModule()"""
+        """Correct and incorrect SimpleModule used in runModule()"""
         self.runModule(self.rinfo)
         self.assertTrue(self.rinfo.outputs['stdout'].value)
         self.assertRaises(CalledModuleError, self.runModule, self.rinfo_wrong)
 
     def test_assertModule(self):
-        """SimpleModule used in assertModule()"""
+        """Correct and incorrect SimpleModule used in assertModule()"""
         self.assertModule(self.rinfo)
         self.assertTrue(self.rinfo.outputs['stdout'].value)
         self.assertRaises(self.failureException, self.assertModule, self.rinfo_wrong)
 
     def test_assertModuleFail(self):
-        """SimpleModule used in assertModuleFail()"""
+        """Correct and incorrect SimpleModule used in assertModuleFail()"""
         self.assertModuleFail(self.rinfo_wrong)
         stderr = self.rinfo_wrong.outputs['stderr'].value
         self.assertTrue(stderr)
@@ -76,6 +76,5 @@ class TestSimpleModuleAssertions(grass.gunittest.TestCase):
         self.assertRaises(self.failureException, self.assertModuleFail, self.rinfo)
 
 
-
 if __name__ == '__main__':
     grass.gunittest.test()