gunittest: new command line interface for main using argparse, additional info to test run main key-value file, do not fail when test cannot be imported, only try to import grass modules if possible to improve behavior when not running in proper GRASS session

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@61489 15284696-431f-4ddb-bdfa-cd5b030d7da7

lib/python/gunittest/__init__.py

@@ -15,5 +15,21 @@ Initial version of `gunittest` was created during Google Summer of Code 2014
 by Vaclav Petras as a student and Soeren Gebbert as a mentor.
 """
 
-from .case import TestCase
-from .main import test
+from __future__ import print_function
+
+try:
+    from .case import TestCase
+    from .main import test
+except ImportError, e:
+    print('WARNING: Cannot import ({e.message}).\n'
+          'Ignoring the failed import because it does not harm if you need'
+          ' something different'
+          ' from gunittest. Probably the environment is not set properly'
+          ' (e.g. dynamic libraries are not available and ctypes-based modules'
+          ' cannot work).'.format(e=e))
+    # we need to ignore import errors for the cases when we just need
+    # gunittest for reports and ctypes are not available (or the environment
+    # is not set properly)
+    # .main probably does not need to be checked but it imports a lot of
+    # things, so it might be hard to keep track in the future
+    # .case imports PyGRASS which imports ctypes modules in its __init__.py

lib/python/gunittest/checkers.py

@@ -16,7 +16,18 @@ for details.
 import sys
 import re
 import doctest
-import grass.script.core as gcore
+
+try:
+    from grass.script.core import KeyValue
+except (ImportError, AttributeError):
+    # TODO: we are silent about the error and use a object with different
+    # interface, should be replaced by central keyvalue module
+    # this can happen when translations are not available
+    # TODO: grass should survive are give better error when tranlsations are not available
+    # even the lazy loading after firts _ call would be interesting
+    # File "...grass/script/core.py", line 40, in <module>
+    # AttributeError: 'NoneType' object has no attribute 'endswith'
+    KeyValue = dict
 
 # alternative term to check(er(s)) would be compare
 
@@ -153,7 +164,7 @@ def text_to_keyvalue(text, sep=":", val_sep=",", functions=None,
         use ``lambda x: x`` for no conversion
 
     :return: a dictionary representation of text
-    :return type: grass.script.core.KeyValue
+    :return type: grass.script.core.KeyValue or dict
 
     And example of converting text with text, floats, integers and list
     to a dictionary::
@@ -172,7 +183,7 @@ def text_to_keyvalue(text, sep=":", val_sep=",", functions=None,
     # splitting according to universal newlines approach
     # TODO: add also general split with vsep
     text = text.splitlines()
-    kvdict = gcore.KeyValue()
+    kvdict = KeyValue()
     functions = [] if functions is None else functions
 
     for line in text:

lib/python/gunittest/invoker.py

@@ -30,6 +30,10 @@ from .reporters import (GrassTestFilesMultiReporter,
                         NoopFileAnonymizer, keyvalue_to_text)
 from .utils import silent_rmtree, ensure_dir
 
+# needed for write_gisrc
+# TODO: it would be good to find some way of writing rc without the need to
+# have GRASS proprly set (anything from grass.script requires translations to
+# be set, i.e. the GRASS environment properly set)
 import grass.script.setup as gsetup
 
 import collections
@@ -56,6 +60,7 @@ def update_keyval_file(filename, module, returncode):
         keyval['status'] = 'failed' if returncode else 'passed'
     keyval['returncode'] = returncode
     keyval['test_file_authors'] = test_file_authors
+
     with open(filename, 'w') as keyval_file:
         keyval_file.write(keyvalue_to_text(keyval))
     return keyval
@@ -168,7 +173,7 @@ class GrassTestFilesInvoker(object):
         if self.clean_mapsets:
             shutil.rmtree(mapset_dir)
 
-    def run_in_location(self, gisdbase, location, location_shortcut,
+    def run_in_location(self, gisdbase, location, location_type,
                         results_dir):
         """Run tests in a given location"""
         if os.path.abspath(results_dir) == os.path.abspath(self.start_dir):
@@ -179,12 +184,13 @@ class GrassTestFilesInvoker(object):
                 GrassTestFilesTextReporter(stream=sys.stderr),
                 GrassTestFilesHtmlReporter(
                     file_anonymizer=self._file_anonymizer),
-                GrassTestFilesKeyValueReporter()
+                GrassTestFilesKeyValueReporter(
+                    info=dict(location=location, location_type=location_type))
             ])
         self.testsuite_dirs = collections.defaultdict(list)  # reset list of dirs each time
         # TODO: move constants out of loader class or even module
         modules = discover_modules(start_dir=self.start_dir,
-                                   grass_location=location_shortcut,
+                                   grass_location=location_type,
                                    file_pattern=GrassTestLoader.files_in_testsuite,
                                    skip_dirs=GrassTestLoader.skip_dirs,
                                    testsuite_dir=GrassTestLoader.testsuite_dir,

lib/python/gunittest/loader.py

@@ -31,7 +31,7 @@ GrassTestPythonModule = collections.namedtuple('GrassTestPythonModule',
 def discover_modules(start_dir, file_pattern, skip_dirs, testsuite_dir,
                      grass_location,
                      all_locations_value, universal_location_value,
-                     import_modules):
+                     import_modules, add_failed_imports=True):
     """Find all test files (modules) in a directory tree.
 
     The function is designed specifically for GRASS testing framework
@@ -46,7 +46,7 @@ def discover_modules(start_dir, file_pattern, skip_dirs, testsuite_dir,
     :param skip_dirs: directories not to recurse to (e.g. ``.svn``)
     :param testsuite_dir: name of directory where the test files are found,
         the function will not recurse to this directory
-    :param grass_location: string with an accepted location (shortcut)
+    :param grass_location: string with an accepted location type (category, shortcut)
     :param all_locations_value: string used to say that all locations
         should be loaded (grass_location can be set to this value)
     :param universal_location_value: string marking a test as
@@ -86,9 +86,10 @@ def discover_modules(start_dir, file_pattern, skip_dirs, testsuite_dir,
                 # everything was loaded into Python
                 abspath = os.path.abspath(full)
                 sys.path.insert(0, abspath)
+                add = False
                 try:
                     m = importlib.import_module(name)
-                    add = False
+                    # TODO: now we are always importing but also always setting module to None
                     if grass_location == all_locations_value:
                         add = True
                     else:
@@ -101,17 +102,20 @@ def discover_modules(start_dir, file_pattern, skip_dirs, testsuite_dir,
                                 add = True  # cases when it is explicit
                             if grass_location in locations:
                                 add = True  # standard case with given location
-                    if add:
-                        modules.append(GrassTestPythonModule(name=name,
-                                                             module=m,
-                                                             tested_dir=root,
-                                                             file_dir=full,
-                                                             abs_file_path=os.path.join(abspath, name + '.py')))
-                    # in else with some verbose we could tell about skiped test
                 except ImportError as e:
-                    raise ImportError('Cannot import module named %s in %s (%s)' % (name, full, e.message))
-                    # alternative is to create TestClass which will raise
-                    # see unittest.loader
+                    if add_failed_imports:
+                        add = True
+                    else:
+                        raise ImportError('Cannot import module named'
+                                          ' %s in %s (%s)'
+                                          % (name, full, e.message))
+                        # alternative is to create TestClass which will raise
+                        # see unittest.loader
+                if add:
+                    modules.append(GrassTestPythonModule(
+                        name=name, module=None, tested_dir=root, file_dir=full,
+                        abs_file_path=os.path.join(abspath, name + '.py')))
+                # in else with some verbose we could tell about skiped test
     return modules
 
 

lib/python/gunittest/main.py

@@ -13,6 +13,7 @@ for details.
 
 import os
 import sys
+import argparse
 
 from unittest.main import TestProgram, USAGE_AS_MAIN
 TestProgram.USAGE = USAGE_AS_MAIN
@@ -103,11 +104,6 @@ def test():
     sys.exit(not program.result.wasSuccessful())
 
 
-# TODO: test or main? test looks more general
-# unittest has main() but doctest has testmod()
-main = test
-
-
 def discovery():
     """Recursively find all tests in testsuite directories and run them
 
@@ -124,23 +120,41 @@ def discovery():
 
 # TODO: makefile rule should depend on the whole build
 # TODO: create a full interface (using grass parser or argparse)
-if __name__ == '__main__':
-    if len(sys.argv) == 4:
-        gisdbase = sys.argv[1]
-        location = sys.argv[2]
-        location_shortcut = sys.argv[3]
-    elif len(sys.argv) == 3:
-        location = sys.argv[1]
-        location_shortcut = sys.argv[2]
+def main():
+    parser = argparse.ArgumentParser(
+    description='Run test files in all testsuite directories starting'
+                ' from the current one'
+                ' (runs on active GRASS session)')
+    parser.add_argument('--location', dest='location', action='store',
+                        help='Name of location where to perform test', required=True)
+    parser.add_argument('--location-type', dest='location_type', action='store',
+                        default='nc',
+                        help='Type of tests which should be run'
+                             ' (tag corresponding to location)')
+    parser.add_argument('--grassdata', dest='gisdbase', action='store',
+                        default=None,
+                        help='GRASS data(base) (GISDBASE) directory'
+                        ' (current GISDBASE by default)')
+    parser.add_argument('--output', dest='output', action='store',
+                        default='testreport',
+                        help='Output directory')
+    args = parser.parse_args()
+    gisdbase = args.gisdbase
+    if gisdbase is None:
+        # here we already rely on being in GRASS session
         gisdbase = gcore.gisenv()['GISDBASE']
-    else:
-        sys.stderr.write("Usage: %s [gisdbase] location location_shortcut\n" % sys.argv[0])
+    location = args.location
+    location_type = args.location_type
+
+    if not gisdbase:
+        sys.stderr.write("GISDBASE (grassdata directory)"
+                         " cannot be empty string\n" % gisdbase)
         sys.exit(1)
-    assert gisdbase
     if not os.path.exists(gisdbase):
-        sys.stderr.write("GISDBASE <%s> does not exist\n" % gisdbase)
+        sys.stderr.write("GISDBASE (grassdata directory) <%s>"
+                         " does not exist\n" % gisdbase)
         sys.exit(1)
-    results_dir = 'testreport'
+    results_dir = args.output
     silent_rmtree(results_dir)  # TODO: too brute force?
 
     start_dir = '.'
@@ -149,9 +163,14 @@ if __name__ == '__main__':
         start_dir=start_dir,
         file_anonymizer=FileAnonymizer(paths_to_remove=[abs_start_dir]))
     # TODO: remove also results dir from files
+    # as an enhancemnt
     # we can just iterate over all locations available in database
-    # but the we don't know the right location label/shortcut
+    # but the we don't know the right location type (category, label, shortcut)
     invoker.run_in_location(gisdbase=gisdbase,
                             location=location,
-                            location_shortcut=location_shortcut,
+                            location_type=location_type,
                             results_dir=results_dir)
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

lib/python/gunittest/multireport.py

@@ -8,6 +8,8 @@ import datetime
 import operator
 from collections import defaultdict
 
+
+# TODO: we should be able to work without matplotlib
 import matplotlib
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
@@ -22,6 +24,8 @@ class TestResultSummary(object):
     def __init__(self):
         self.timestamp = None
         self.svn_revision = None
+        self.location = None
+        self.location_type = None
 
         self.total = None
         self.successes = None
@@ -142,7 +146,17 @@ def main_page(results, filename, images, captions, title='Test reports',
             )
         for result in results:
             # TODO: include name to summary file
-            name = os.path.basename(result.report)
+            # now using location or test report directory as name
+            if result.location != 'unknown':
+                name = result.location
+            else:
+                name = os.path.basename(result.report)
+                if not name:
+                    # Python basename returns '' for 'abc/'
+                    for d in reversed(os.path.split(result.report)):
+                        if d:
+                            name = d
+                            break
             per_test = success_to_html_percent(
                 total=result.total, successes=result.successes)
             per_file = success_to_html_percent(
@@ -152,7 +166,7 @@ def main_page(results, filename, images, captions, title='Test reports',
                 '<tr>'
                 '<td><a href={report_path}/index.html>{result.timestamp}</a></td>'
                 '<td>{result.svn_revision}</td>'
-                '<td><a href={result.report}/index.html>{name}</a></td>'
+                '<td><a href={report_path}/index.html>{name}</a></td>'
                 '<td>{pfiles}</td><td>{ptests}</td>'
                 '</tr>'
                 .format(result=result, name=name, report_path=report_path,
@@ -216,8 +230,15 @@ def main():
             result.report = report
 
             # let's consider no location as valid state and use 'unknown'
-            location = summary.get('location', 'unknown')
-            results_in_locations[location].append(result)
+            result.location = summary.get('location', 'unknown')
+            result.location_type = summary.get('location_type', 'unknown')
+            # grouping accoring to location types
+            # this can cause that two actual locations tested at the same time
+            # will end up together, this is not ideal but testing with
+            # one location type and different actual locations is not standard
+            # and although it will not break anything it will not give a nice
+            # report
+            results_in_locations[result.location_type].append(result)
 
             all_results.append(result)
             del result
@@ -236,12 +257,17 @@ def main():
         '<tbody>'
         )
 
-    for location, results in results_in_locations.iteritems():
+    for location_type, results in results_in_locations.iteritems():
         results = sorted(results, key=operator.attrgetter('timestamp'))
-        # TODO: document: location shortcut must be a valid dir name
-        directory = os.path.join(output, location)
+        # TODO: document: location type must be a valid dir name
+        directory = os.path.join(output, location_type)
         ensure_dir(directory)
 
+        if location_type == 'unknown':
+            title = 'Test reports'
+        else:
+            title = 'Test reports for ' + location_type + ' location'
+
         x = [date2num(result.timestamp) for result in results]
         xlabels = [result.timestamp.strftime("%Y-%m-%d") + ' (r' + result.svn_revision + ')' for result in results]
         tests_plot(x=x, xlabels=xlabels, results=results,
@@ -255,7 +281,8 @@ def main():
                   images=['tests_plot.png', 'files_plot.png', 'info_plot.png'],
                   captions=['Success of individual tests', 'Success of test files',
                             'Additional information'],
-                  directory=directory)
+                  directory=directory,
+                  title=title)
 
         files_successes = sum(result.files_successes for result in results)
         files_total = sum(result.files_total for result in results)
@@ -270,7 +297,7 @@ def main():
             '<td><a href={location}/index.html>{location}</a></td>'
             '<td>{pfiles}</td><td>{ptests}</td>'
             '</tr>'
-            .format(location=location,
+            .format(location=location_type,
                     pfiles=per_file, ptests=per_test))
     locations_main_page.write('</tbody></table>')
     locations_main_page.write('</body></html>')

lib/python/gunittest/reporters.py

@@ -22,8 +22,6 @@ import collections
 import types
 import re
 
-import grass.script as gscript
-
 from .utils import ensure_dir
 from .checkers import text_to_keyvalue
 
@@ -73,6 +71,7 @@ class NoopFileAnonymizer(object):
         pass
 
 
+# TODO: why not remove GISDBASE by default?
 class FileAnonymizer(object):
     def __init__(self, paths_to_remove, remove_gisbase=True,
                  remove_gisdbase=False):
@@ -81,7 +80,10 @@ class FileAnonymizer(object):
             gisbase = os.environ['GISBASE']
             self._paths_to_remove.append(gisbase)
         if remove_gisdbase:
-            gisdbase = gscript.gis.get['GISDBASE']
+            # import only when really needed to avoid problems with
+            # translations when environment is not set properly
+            import grass.script as gscript
+            gisdbase = gscript.gisenv()['GISDBASE']
             self._paths_to_remove.append(gisdbase)
         if paths_to_remove:
             self._paths_to_remove.extend(paths_to_remove)
@@ -685,7 +687,7 @@ class GrassTestFilesHtmlReporter(GrassTestFilesCountingReporter):
                 modules = [modules]
             file_index.write(
                 '<tr><td>Tested modules</td><td>{}</td></tr>'.format(
-                    ', '.join(modules)))
+                    ', '.join(sorted(set(modules)))))
         file_index.write('<tbody><table>')
 
         # here we would have also links to coverage, profiling, ...
@@ -727,11 +729,14 @@ class GrassTestFilesHtmlReporter(GrassTestFilesCountingReporter):
             # a stream can be added and if not none, we could write
 
 
+# TODO: document info: additional information to be stored type: dict
+# allows to overwrite what was collected
 class GrassTestFilesKeyValueReporter(GrassTestFilesCountingReporter):
 
-    def __init__(self):
+    def __init__(self, info=None):
         super(GrassTestFilesKeyValueReporter, self).__init__()
         self.result_dir = None
+        self._info = info
 
     def start(self, results_dir):
         super(GrassTestFilesKeyValueReporter, self).start(results_dir)
@@ -799,6 +804,10 @@ class GrassTestFilesKeyValueReporter(GrassTestFilesCountingReporter):
         summary['timestamp'] = self.main_start_time.strftime('%Y-%m-%d %H:%M:%S')
         # TODO: add some general metadata here (passed in constructor)
 
+        # add additional information
+        for key, value in self._info.iteritems():
+            summary[key] = value
+
         summary_filename = os.path.join(self.result_dir,
                                         'test_keyvalue_result.txt')
         with open(summary_filename, 'w') as summary_file: