gunittest: Non-zero return code on certian percetage of tests failing (#377)

Consider a certain percentage of failing tests an overall failure
and return non-zero return code (1) from the main runner.
The invoker now returns the counts from the run_in_location() function.

HTML typos fixed in error messages.

lib/python/docs/src/gunittest_running_tests.rst

@@ -67,6 +67,22 @@ scripts. The flag causes execution to stop once some command gives a non-zero
 return code.
 
 
+Setting sensitivity of the test run
+-----------------------------------
+
+Sensitivity, specified by the ``--min-success`` parameter, determined
+how many tests need to fail for the runner to consider it an error
+and return a non-zero return code.
+For example, if at least 60% of test is required to succeed, you can
+use::
+
+    python -m grass.gunittest.main ... --min-success 60
+
+If all tests should succeed, use ``--min-success 100``. If you want
+to run the test and ``grass.gunittest.main`` returning zero return code
+even if some tests fail, use ``--min-success 0``
+
+
 Running tests and creating report
 ---------------------------------
 

lib/python/gunittest/invoker.py

@@ -241,7 +241,13 @@ class GrassTestFilesInvoker(object):
 
     def run_in_location(self, gisdbase, location, location_type,
                         results_dir):
-        """Run tests in a given location"""
+        """Run tests in a given location
+
+        Returns an object with counting attributes of GrassTestFilesCountingReporter,
+        i.e., a file-oriented reporter as opposed to testsuite-oriented one.
+        Use only the attributes related to the summary, such as file_pass_per,
+        not to one file as these will simply contain the last executed file.
+        """
         if os.path.abspath(results_dir) == os.path.abspath(self.start_dir):
             raise RuntimeError("Results root directory should not be the same"
                                " as discovery start directory")
@@ -291,3 +297,4 @@ class GrassTestFilesInvoker(object):
             top_level_testsuite_page_name='testsuite_index.html')
         testsuite_dir_reporter.report_for_dirs(root=results_dir,
                                                directories=self.testsuite_dirs)
+        return self.reporter

lib/python/gunittest/main.py

@@ -135,6 +135,10 @@ def main():
     parser.add_argument('--output', dest='output', action='store',
                         default='testreport',
                         help='Output directory')
+    parser.add_argument('--min-success', dest='min_success', action='store',
+                        default='90', type=int,
+                        help=("Minimum success percentage (lower percentage"
+                              " than this will result in a non-zero return code; values 0-100)"))
     args = parser.parse_args()
     gisdbase = args.gisdbase
     if gisdbase is None:
@@ -168,11 +172,15 @@ def main():
     # as an enhancemnt
     # we can just iterate over all locations available in database
     # but the we don't know the right location type (category, label, shortcut)
-    invoker.run_in_location(gisdbase=gisdbase,
-                            location=location,
-                            location_type=location_type,
-                            results_dir=results_dir)
-    return 0
+    reporter = invoker.run_in_location(
+        gisdbase=gisdbase,
+        location=location,
+        location_type=location_type,
+        results_dir=results_dir
+    )
+    if reporter.file_pass_per >= args.min_success:
+        return 0
+    return 1
 
 if __name__ == '__main__':
     sys.exit(main())

lib/python/gunittest/reporters.py

@@ -307,7 +307,14 @@ def get_html_test_authors_table(directory, tests_authors):
 
 
 class GrassTestFilesMultiReporter(object):
+    """Interface to multiple repoter objects
 
+    For start and finish of the tests and of a test of one file,
+    it calls corresponding methods of all contained reporters.
+    For all other attributes, it returns attribute of a first reporter
+    which has this attribute using the order in which the reporters were
+    provided.
+    """
     def __init__(self, reporters, forgiving=False):
         self.reporters = reporters
         self.forgiving = forgiving
@@ -356,6 +363,14 @@ class GrassTestFilesMultiReporter(object):
                 else:
                     raise
 
+    def __getattr__(self, name):
+        for reporter in self.reporters:
+            try:
+                return getattr(reporter, name)
+            except AttributeError:
+                continue
+        raise AttributeError
+
 
 class GrassTestFilesCountingReporter(object):
     def __init__(self):
@@ -444,10 +459,10 @@ def html_file_preview(filename):
     before = '<pre>'
     after = '</pre>'
     if not os.path.isfile(filename):
-        return '<p style="color: red>File %s does not exist<p>' % filename
+        return '<p style="color: red>File %s does not exist</p>' % filename
     size = os.path.getsize(filename)
     if not size:
-        return '<p style="color: red>File %s is empty<p>' % filename
+        return '<p style="color: red>File %s is empty</p>' % filename
     max_size = 10000
     html = StringIO()
     html.write(before)
@@ -462,7 +477,7 @@ def html_file_preview(filename):
         for line in tail(filename, 50):
             html.write(color_error_line(html_escape(line)))
     else:
-        return '<p style="color: red>File %s is too large to show<p>' % filename
+        return '<p style="color: red>File %s is too large to show</p>' % filename
     html.write(after)
     return html.getvalue()