fix multirunner.py with Py3; fix https://trac.osgeo.org/grass/ticket/3798; more Py3 fixes see https://trac.osgeo.org/grass/ticket/3771

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@74327 15284696-431f-4ddb-bdfa-cd5b030d7da7

lib/python/gunittest/case.py

@@ -29,8 +29,8 @@ from .checkers import (check_text_ellipsis,
 from .utils import safe_repr
 from .gutils import is_map_in_mapset
 
-
-if sys.version_info[0] == 2:
+pyversion = sys.version_info[0]
+if pyversion == 2:
     from StringIO import StringIO
 else:
     from io import StringIO
@@ -169,6 +169,14 @@ class TestCase(unittest.TestCase):
             If you need to test the actual newline characters, use the standard
             string comparison and functions such as ``find()``.
         """
+        # SimpleModule delivers bytes for stderr and stdout
+        # Instead of decoding stdout and stderr in every test, decoding
+        # is done here
+        if pyversion == 3:
+            if isinstance(first, bytes):
+                first = decode(first)
+            if isinstance(second, bytes):
+                second = decode(second)
         if os.linesep != '\n':
             if os.linesep in first:
                 first = first.replace(os.linesep, '\n')
@@ -185,6 +193,12 @@ class TestCase(unittest.TestCase):
 
         See :func:`check_text_ellipsis` for details of behavior.
         """
+        # SimpleModule delivers bytes for stderr and stdout
+        # Instead of decoding stdout and stderr in every test, decoding
+        # is done here
+        if pyversion == 3:
+            if isinstance(actual, bytes):
+                actual = decode(actual)
         self.assertTrue(isinstance(actual, str), (
                         'actual argument is not a string'))
         self.assertTrue(isinstance(reference, str), (
@@ -404,6 +418,12 @@ class TestCase(unittest.TestCase):
         This function does not test geometry itself just the region of the
         vector map and number of features.
         """
+        # SimpleModule delivers bytes for stderr and stdout
+        # Instead of decoding stdout and stderr in every test, decoding
+        # is done here
+        if pyversion == 3:
+            if isinstance(actual, bytes):
+                actual = decode(actual)
         module = SimpleModule('v.info', flags='t', map=reference)
         self.runModule(module)
         ref_topo = text_to_keyvalue(decode(module.outputs.stdout), sep='=')
@@ -1152,8 +1172,8 @@ class TestCase(unittest.TestCase):
             module.run()
             self.grass_modules.append(module.name)
         except CalledModuleError:
-            print(module.outputs.stdout)
-            print(module.outputs.stderr)
+            print(decode(module.outputs.stdout))
+            print(decode(module.outputs.stderr))
             # TODO: message format
             # TODO: stderr?
             stdmsg = ('Running <{m.name}> module ended'
@@ -1165,8 +1185,8 @@ class TestCase(unittest.TestCase):
                           errors=decode(module.outputs.stderr)
                       ))
             self.fail(self._formatMessage(msg, stdmsg))
-        print(module.outputs.stdout)
-        print(module.outputs.stderr)
+        print(decode(module.outputs.stdout))
+        print(decode(module.outputs.stderr))
         # log these to final report
         # TODO: always or only if the calling test method failed?
         # in any case, this must be done before self.fail()
@@ -1186,11 +1206,11 @@ class TestCase(unittest.TestCase):
             module.run()
             self.grass_modules.append(module.name)
         except CalledModuleError:
-            print(module.outputs.stdout)
-            print(module.outputs.stderr)
+            print(decode(module.outputs.stdout))
+            print(decode(module.outputs.stderr))
         else:
-            print(module.outputs.stdout)
-            print(module.outputs.stderr)
+            print(decode(module.outputs.stdout))
+            print(decode(module.outputs.stderr))
             stdmsg = ('Running <%s> ended with zero (successful) return code'
                       ' when expecting module to fail' % module.get_python())
             self.fail(self._formatMessage(msg, stdmsg))

lib/python/gunittest/invoker.py

@@ -24,6 +24,8 @@ from .reporters import (GrassTestFilesMultiReporter,
                         NoopFileAnonymizer, keyvalue_to_text)
 from .utils import silent_rmtree, ensure_dir
 
+from grass.script.utils import decode, encode, _get_encoding
+
 try:
     from string import maketrans
 except ImportError:
@@ -151,8 +153,6 @@ class GrassTestFilesInvoker(object):
 
         stdout_path = os.path.join(cwd, 'stdout.txt')
         stderr_path = os.path.join(cwd, 'stderr.txt')
-        stdout = open(stdout_path, 'w')
-        stderr = open(stderr_path, 'w')
 
         self.reporter.start_file_test(module)
         # TODO: we might clean the directory here before test if non-empty
@@ -166,7 +166,8 @@ class GrassTestFilesInvoker(object):
             else:
                 args = [sys.executable, '-tt', '-3', module.abs_file_path]
             p = subprocess.Popen(args, cwd=cwd, env=env,
-                                 stdout=stdout, stderr=stderr)
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE)
         elif module.file_type == 'sh':
             # ignoring shebang line to pass parameters to shell
             # expecting system to have sh or something compatible
@@ -182,14 +183,40 @@ class GrassTestFilesInvoker(object):
             #                of an '&&' or '||' operator.
             p = subprocess.Popen(['sh', '-e', '-x', module.abs_file_path],
                                  cwd=cwd, env=env,
-                                 stdout=stdout, stderr=stderr)
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE)
         else:
             p = subprocess.Popen([module.abs_file_path],
                                  cwd=cwd, env=env,
-                                 stdout=stdout, stderr=stderr)
-        returncode = p.wait()
-        stdout.close()
-        stderr.close()
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        returncode = p.returncode
+        encodings = [_get_encoding(), 'utf8', 'latin-1', 'ascii']
+        detected = False
+        idx = 0
+        while not detected:
+            try:
+                stdout = decode(stdout, encoding=encodings[idx])
+                detected = True
+            except:
+                idx += 1
+                pass
+
+        detected = False
+        idx = 0
+        while not detected:
+            try:
+                stderr = decode(stderr, encoding=encodings[idx])
+                detected = True
+            except:
+                idx += 1
+                pass
+                    
+        with open(stdout_path, 'w') as stdout_file:
+            stdout_file.write(encode(stdout))
+        with open(stderr_path, 'w') as stderr_file:
+            stderr_file.write(encode(stderr))
         self._file_anonymizer.anonymize([stdout_path, stderr_path])
 
         test_summary = update_keyval_file(