Fix dbmgr for Python3 (#344)

gui/wxpython/dbmgr/base.py

@@ -34,7 +34,6 @@ import os
 import locale
 import tempfile
 import copy
-import types
 import math
 import functools
 
@@ -58,7 +57,7 @@ from dbmgr.sqlbuilder import SQLBuilderSelect, SQLBuilderUpdate
 from core.gcmd import RunCommand, GException, GError, GMessage, GWarning
 from core.utils import ListOfCatsToRange
 from gui_core.dialogs import CreateNewVector
-from dbmgr.vinfo import VectorDBInfo, GetUnicodeValue, CreateDbInfoDesc
+from dbmgr.vinfo import VectorDBInfo, GetUnicodeValue, CreateDbInfoDesc, GetDbEncoding
 from core.debug import Debug
 from dbmgr.dialogs import ModifyTableRecord, AddColumnDialog
 from core.settings import UserSettings
@@ -269,13 +268,23 @@ class VirtualAttributeList(ListCtrl,
         i = 0
         outFile.seek(0)
 
+        enc = GetDbEncoding()
+        first_wrong_encoding = True
         while True:
             # os.linesep doesn't work here (MSYS)
             # not sure what the replace is for?
             # but we need strip to get rid of the ending newline
             # which on windows leaves \r in a last empty attribute table cell
             # and causes error
-            record = decode(outFile.readline().strip()).replace('\n', '')
+            try:
+                record = decode(outFile.readline(), encoding=enc).strip().replace('\n', '')
+            except UnicodeDecodeError as e:
+                record = outFile.readline().decode(encoding=enc, errors="replace").strip().replace('\n', '')
+                if first_wrong_encoding:
+                    first_wrong_encoding = False
+                    GWarning(parent=self,
+                             message=_("Incorrect encoding {enc} used. Set encoding in GUI Settings"
+                                       " or set GRASS_DB_ENCODING variable.").format(enc=enc))
 
             if not record:
                 break
@@ -425,7 +434,7 @@ class VirtualAttributeList(ListCtrl,
         """Get item text"""
         index = self.itemIndexMap[item]
         s = self.itemDataMap[index][col]
-        return s
+        return str(s)
 
     def OnGetItemAttr(self, item):
         """Get item attributes"""
@@ -480,7 +489,7 @@ class VirtualAttributeList(ListCtrl,
 
         if not self.dbMgrData['editable'] or self.columns[
                 self.GetColumn(self._col).GetText()]['ctype'] not in (
-                types.IntType, types.FloatType):
+                int, float):
             popupMenu.Enable(self.popupId['calculate'], False)
 
         subMenu.Append(self.popupId['area'], text=_("Area size"))
@@ -969,19 +978,12 @@ class DbMgrNotebookBase(FN.FlatNotebook):
         # perform SQL non-select statements (e.g. 'delete from table where
         # cat=1')
         if len(listOfSQLStatements) > 0:
+            enc = GetDbEncoding()
             fd, sqlFilePath = tempfile.mkstemp(text=True)
-            sqlFile = open(sqlFilePath, 'w')
-            for sql in listOfSQLStatements:
-                enc = UserSettings.Get(
-                    group='atm', key='encoding', subkey='value')
-                if not enc and 'GRASS_DB_ENCODING' in os.environ:
-                    enc = os.environ['GRASS_DB_ENCODING']
-                if enc:
-                    sqlFile.write(sql.encode(enc) + ';')
-                else:
-                    sqlFile.write(sql.encode('utf-8') + ';')
-                sqlFile.write(os.linesep)
-            sqlFile.close()
+            with open(sqlFilePath, 'w', encoding=enc) as sqlFile:
+                for sql in listOfSQLStatements:
+                    sqlFile.write(sql + ';')
+                    sqlFile.write(os.linesep)
 
             driver = self.dbMgrData['mapDBInfo'].layers[
                 self.selLayer]["driver"]
@@ -1523,7 +1525,7 @@ class DbMgrBrowsePage(DbMgrNotebookBase):
                             else:
                                 idx = i
 
-                            if column['ctype'] != types.StringType:
+                            if column['ctype'] != str:
                                 tlist.itemDataMap[item][
                                     idx] = column['ctype'](values[i])
                             else:  # -> string
@@ -1533,7 +1535,7 @@ class DbMgrBrowsePage(DbMgrNotebookBase):
                                              {'value': str(values[i]),
                                               'type': column['type']})
 
-                        if column['ctype'] == types.StringType:
+                        if column['ctype'] == str:
                             if "'" in values[i]:  # replace "'" -> "''"
                                 values[i] = values[i].replace("'", "''")
                             updateList.append(

gui/wxpython/dbmgr/dialogs.py

@@ -28,7 +28,7 @@ import wx.lib.scrolledpanel as scrolled
 from core.gcmd import RunCommand, GError
 from core.debug import Debug
 from core.settings import UserSettings
-from dbmgr.vinfo import VectorDBInfo, GetUnicodeValue
+from dbmgr.vinfo import VectorDBInfo, GetUnicodeValue, GetDbEncoding
 from gui_core.widgets import IntegerValidator, FloatValidator
 from gui_core.wrap import SpinCtrl, Button, StaticText, StaticBox, \
     TextCtrl
@@ -333,18 +333,13 @@ class DisplayAttributesDialog(wx.Dialog):
         """Submit records"""
         layer = 1
         close = True
-        enc = UserSettings.Get(group='atm', key='encoding', subkey='value')
-        if not enc and 'GRASS_DB_ENCODING' in os.environ:
-            enc = os.environ['GRASS_DB_ENCODING']
+        enc = GetDbEncoding()
 
         for sql in self.GetSQLString(updateValues=True):
             if not sql:
                 close = False
                 continue
-            if enc:
-                sql = sql.encode(enc)
-            else:
-                sql = sql.encode('utf-8')
+            sql = sql.encode(enc)
 
             driver, database = self.mapDBInfo.GetDbSettings(layer)
             Debug.msg(1, "SQL: %s" % sql)

gui/wxpython/dbmgr/vinfo.py

@@ -15,7 +15,6 @@ This program is free software under the GNU General Public License
 """
 
 import os
-import types
 import sys
 import six
 
@@ -29,24 +28,43 @@ import grass.script as grass
 
 if sys.version_info.major >= 3:
     unicode = str
+    def GetUnicodeValue(value):
+        """Get unicode value
 
-def GetUnicodeValue(value):
-    """Get unicode value
+        :param value: value to be recoded
 
-    :param value: value to be recoded
+        :return: unicode value
+        """
+        if isinstance(value, unicode):
+            return value
+        if isinstance(value, bytes):
+            enc = GetDbEncoding()
+            return str(value, enc, errors='replace')
+        else:
+            return str(value)
+else:
+    def GetUnicodeValue(value):
+        """Get unicode value
+
+        :param value: value to be recoded
+
+        :return: unicode value
+        """
+        if isinstance(value, unicode):
+            return value
+        enc = GetDbEncoding()
+        return unicode(str(value), enc, errors='replace')
 
-    :return: unicode value
-    """
-    if isinstance(value, unicode):
-        return value
 
+def GetDbEncoding():
+    """Checks if user set DB encoding (first user settings,
+    then env variable), if not assumes unicode."""
     enc = UserSettings.Get(group='atm', key='encoding', subkey='value')
     if not enc and 'GRASS_DB_ENCODING' in os.environ:
         enc = os.environ['GRASS_DB_ENCODING']
     else:
         enc = 'utf-8'  # assuming UTF-8
-
-    return unicode(str(value), enc, errors='replace')
+    return enc
 
 
 def CreateDbInfoDesc(panel, mapDBInfo, layer):
@@ -139,7 +157,7 @@ class VectorDBInfo(VectorDBInfoBase):
                 if len(value) < 1:
                     value = None
                 else:
-                    if self.tables[table][key]['ctype'] != types.StringType:
+                    if self.tables[table][key]['ctype'] != str:
                         value = self.tables[table][key]['ctype'](value)
                     else:
                         value = GetUnicodeValue(value)

gui/wxpython/gui_core/goutput.py

@@ -708,6 +708,7 @@ class GStc(stc.StyledTextCtrl):
             try:
                 self.AddText(txt)
             except UnicodeDecodeError:
+                # TODO: this might be dead code for Py3, txt is already unicode?
                 enc = UserSettings.Get(
                     group='atm', key='encoding', subkey='value')
                 if enc: