pythonlib: use isinstance to distinguish string types (fixes https://trac.osgeo.org/grass/changeset/65787)

isinstance(bytes) seems to better distinguish if we can/need to decode/encode
a string. hasattr with decode/encode fails in Python 2 because all are
present everywhere (see below).

This enables Python modules to be executed with non-ascii characters
as option values (test which uses UTF-8 included). This also asks for locale
only when needed.

In Python 2:
>>> print(hasattr(b"", "decode"), hasattr("", "decode"), hasattr(u"", "decode"))
(True, True, True)
>>> print(hasattr(b"", "encode"), hasattr("", "encode"), hasattr(u"", "encode"))
(True, True, True)
>>> print(isinstance(b"", bytes), isinstance("", bytes), isinstance(u"", bytes))
(True, True, False)

In Python 3:
>>> print(hasattr(b"", "decode"), hasattr("", "decode"), hasattr(u"", "decode"))
True False False
>>> print(hasattr(b"", "encode"), hasattr("", "encode"), hasattr(u"", "encode"))
False True True
>>> print(isinstance(b"", bytes), isinstance("", bytes), isinstance(u"", bytes))
True False False


git-svn-id: https://svn.osgeo.org/grass/grass/trunk@68927 15284696-431f-4ddb-bdfa-cd5b030d7da7

lib/python/script/testsuite/test_start_command_functions.py

@@ -1,9 +1,11 @@
+# -*- coding: utf-8 -*-
+# the utf-8 is important because we do use the characters
 """Tests of start_command function family (location independent)"""
 
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
 
-from grass.script.core import start_command, PIPE
+from grass.script.core import start_command, PIPE, run_command
 
 
 class TestPythonKeywordsInParameters(TestCase):
@@ -38,5 +40,28 @@ class TestPythonKeywordsInParameters(TestCase):
         self.assertEquals(returncode, 1)
         self.assertIn(b'raster', stderr)
 
+
+class TestPythonModuleWithUnicodeParameters(TestCase):
+    """Tests if unicode works in parameters of Python modules
+
+    This in fact tests also the `parser()` function (original motivation
+    for this tests).
+
+    Using g.search.module because it takes any option values.
+    """
+
+    def test_python_module_ascii(self):
+        """This tests if Python module works"""
+        run_command('g.search.modules', keyword=b'Priserny kun')
+
+    def test_python_module_czech_nonascii(self):
+        """This likely fails on non-UTF-8 systems (i.e. MS Win)"""
+        run_command('g.search.modules', keyword=b'Příšerný kůň')
+
+    def test_python_module_czech_unicode(self):
+        """This likely fails on non-UTF-8 systems (i.e. MS Win)"""
+        run_command('g.search.modules', keyword=u'Příšerný kůň')
+
+
 if __name__ == '__main__':
     test()

lib/python/script/testsuite/test_utils.py

@@ -39,6 +39,10 @@ class TestEncode(TestCase):
     def test_unicode(self):
         self.assertEqual(b'text', utils.encode(u'text'))
 
+    def test_bytes_grabage_in_out(self):
+        """If the input is bytes we should not touch it for encoding"""
+        self.assertEqual(b'Příšerný kůň', utils.encode(b'Příšerný kůň'))
+
 
 class TestDecode(TestCase):
     """Tests function `encode` that convert value to unicode."""

lib/python/script/utils.py

@@ -150,26 +150,31 @@ class KeyValue(dict):
         self[key] = value
 
 
-def decode(bytes):
-    """Decode bytes with default locale
+def decode(bytes_):
+    """Decode bytes with default locale and return (unicode) string
 
-    :param bytes bytes: the bytes to decode
+    No-op if parameter is not bytes (assumed unicode string).
+
+    :param bytes bytes_: the bytes to decode
     """
-    enc = locale.getdefaultlocale()[1]
-    if hasattr(bytes, 'decode'):
-        return bytes.decode(enc) if enc else bytes.decode()
-    return bytes
+    if isinstance(bytes_, bytes):
+        enc = locale.getdefaultlocale()[1]
+        return bytes_.decode(enc) if enc else bytes_.decode()
+    return bytes_
 
 
 def encode(string):
-    """Encode string with default locale -> bytes
+    """Encode string with default locale and return bytes with that encoding
+
+    No-op if parameter is bytes (assumed already encoded).
+    This ensures garbage in, garbage out.
 
     :param str string: the string to encode
     """
+    if isinstance(string, bytes):
+        return string
     enc = locale.getdefaultlocale()[1]
-    if hasattr(string, 'encode'):
-        return string.encode(enc) if enc else string.encode()
-    return string
+    return string.encode(enc) if enc else string.encode()
 
 
 def parse_key_val(s, sep='=', dflt=None, val_type=None, vsep=None):