wxGUI/datacatalog: enable lazy loading of maps (#1434)

* reload mapset when new item is added to not loaded mapset
* suggest turning on lazyloading in infobar if loading took more than 5 s

gui/wxpython/core/settings.py

@@ -159,6 +159,7 @@ class Settings:
             "datacatalog": {
                 # grassdb string
                 "grassdbs": {"listAsString": ""},
+                "lazyLoading": {"enabled": False, "asked": False},
             },
             "manager": {
                 # show opacity level widget

gui/wxpython/datacatalog/catalog.py

@@ -26,9 +26,11 @@ from gui_core.infobar import InfoBar
 from datacatalog.infomanager import DataCatalogInfoManager
 from gui_core.wrap import Menu
 from gui_core.forms import GUI
-from grass.script import gisenv
+from core.settings import UserSettings
 
 from grass.pydispatch.signal import Signal
+from grass.script.utils import clock
+from grass.script import gisenv
 
 from grass.grassdb.manage import split_mapset_path
 from grass.grassdb.checks import (
@@ -55,6 +57,7 @@ class DataCatalog(wx.Panel):
         self.parent = parent
         self.baseTitle = title
         self.giface = giface
+        self._startLoadingTime = 0
         wx.Panel.__init__(self, parent=parent, id=id, **kwargs)
         self.SetName("DataCatalog")
 
@@ -77,6 +80,7 @@ class DataCatalog(wx.Panel):
             infobar=self.infoBar, giface=self.giface
         )
         self.tree.showImportDataInfo.connect(self.showImportDataInfo)
+        self.tree.loadingDone.connect(self._loadingDone)
 
         # some layout
         self._layout()
@@ -126,7 +130,51 @@ class DataCatalog(wx.Panel):
         )
 
     def LoadItems(self):
-        self.tree.ReloadTreeItems()
+        """Reload tree - full or lazy - based on user settings"""
+        self._startLoadingTime = clock()
+        self.tree.ReloadTreeItems(full=False)
+
+    def _loadingDone(self):
+        """If loading took more time, suggest lazy loading"""
+        if clock() - self._startLoadingTime > 5 and not self.tree._useLazyLoading():
+            asked = UserSettings.Get(
+                group="datacatalog", key="lazyLoading", subkey="asked"
+            )
+            if not asked:
+                wx.CallAfter(
+                    self.infoManager.ShowLazyLoadingOn,
+                    setLazyLoadingOnHandler=self._saveLazyLoadingOnSettings,
+                    doNotAskHandler=self._saveDontAskLazyLoadingSettings,
+                )
+
+    def _saveLazyLoadingOnSettings(self, event):
+        """Turn on lazy loading in settings"""
+        UserSettings.Set(
+            group="datacatalog", key="lazyLoading", subkey="enabled", value=True
+        )
+        UserSettings.Set(
+            group="datacatalog", key="lazyLoading", subkey="asked", value=True
+        )
+        self._saveLazyLoadingSettings()
+        event.Skip()
+
+    def _saveDontAskLazyLoadingSettings(self, event):
+        """Save in settings that decision on lazy loading was done to not ask again"""
+        UserSettings.Set(
+            group="datacatalog", key="lazyLoading", subkey="asked", value=True
+        )
+        self._saveLazyLoadingSettings()
+        event.Skip()
+
+    def _saveLazyLoadingSettings(self):
+        dcSettings = {}
+        UserSettings.ReadSettingsFile(settings=dcSettings)
+        if "datacatalog" not in dcSettings:
+            dcSettings["datacatalog"] = UserSettings.Get(group="datacatalog")
+        dcSettings["datacatalog"]["lazyLoading"] = UserSettings.Get(
+            group="datacatalog", key="lazyLoading"
+        )
+        UserSettings.SaveToFile(dcSettings)
 
     def dismissInfobar(self):
         if self.infoBar.IsShown():
@@ -134,7 +182,7 @@ class DataCatalog(wx.Panel):
 
     def OnReloadTree(self, event):
         """Reload whole tree"""
-        self.LoadItems()
+        self.tree.ReloadTreeItems(full=True)
 
     def OnReloadCurrentMapset(self, event):
         """Reload current mapset tree only"""

gui/wxpython/datacatalog/infomanager.py

@@ -64,6 +64,19 @@ class DataCatalogInfoManager:
         ).format(loc=gisenv()["LOCATION_NAME"])
         self.infoBar.ShowMessage(message, wx.ICON_INFORMATION, buttons)
 
+    def ShowLazyLoadingOn(self, setLazyLoadingOnHandler, doNotAskHandler):
+        """Show info about lazy loading"""
+        message = _(
+            "Loading of Data catalog content took rather long. "
+            "To prevent delay, you can enable loading of current mapset only. "
+            "You can change that later in GUI Settings, General tab."
+        )
+        buttons = [
+            (_("Enable loading current mapset only"), setLazyLoadingOnHandler),
+            (_("No change, don't ask me again"), doNotAskHandler),
+        ]
+        self.infoBar.ShowMessage(message, wx.ICON_INFORMATION, buttons)
+
     def ShowFallbackSessionInfo(self, reason_id):
         """Show info when last used mapset is not usable"""
         string = self._text_from_reason_id(reason_id)

gui/wxpython/datacatalog/tree.py

@@ -82,7 +82,7 @@ from grass.exceptions import CalledModuleError
 updateMapset, EVT_UPDATE_MAPSET = NewEvent()
 
 
-def getLocationTree(gisdbase, location, queue, mapsets=None):
+def getLocationTree(gisdbase, location, queue, mapsets=None, lazy=False):
     """Creates dictionary with mapsets, elements, layers for given location.
     Returns tuple with the dictionary and error (or None)"""
     tmp_gisrc_file, env = gscript.create_environment(gisdbase, location, "PERMANENT")
@@ -109,6 +109,9 @@ def getLocationTree(gisdbase, location, queue, mapsets=None):
         Debug.msg(4, "Location <{0}>: {1} mapsets found".format(location, len(mapsets)))
         for each in mapsets:
             maps_dict[each] = []
+    if lazy:
+        queue.put((maps_dict, None))
+        return
     try:
         maplist = gscript.read_command(
             "g.list",
@@ -320,6 +323,7 @@ class DataCatalogTree(TreeView):
 
         self.showNotification = Signal("Tree.showNotification")
         self.showImportDataInfo = Signal("Tree.showImportDataInfo")
+        self.loadingDone = Signal("Tree.loadingDone")
         self.parent = parent
         self.contextMenu.connect(self.OnRightClick)
         self.itemActivated.connect(self.OnDoubleClick)
@@ -397,6 +401,34 @@ class DataCatalogTree(TreeView):
         self.copy_location = None
         self.copy_grassdb = None
 
+    def _useLazyLoading(self):
+        settings = UserSettings.Get(group="datacatalog")
+        # workaround defining new settings in datacatalog group
+        # force writing new settings in the wx.json file during start
+        # can be removed later on
+        if "lazyLoading" not in settings:
+            lazySettings = UserSettings.Get(
+                group="datacatalog", key="lazyLoading", settings_type="default"
+            )
+            # update local settings
+            for subkey, value in lazySettings.items():
+                UserSettings.Append(
+                    UserSettings.userSettings,
+                    group="datacatalog",
+                    key="lazyLoading",
+                    subkey=subkey,
+                    value=value,
+                    overwrite=False,
+                )
+            # update settings file
+            jsonSettings = {}
+            UserSettings.ReadSettingsFile(settings=jsonSettings)
+            jsonSettings["datacatalog"]["lazyLoading"] = lazySettings
+            UserSettings.SaveToFile(jsonSettings)
+        return UserSettings.Get(
+            group="datacatalog", key="lazyLoading", subkey="enabled"
+        )
+
     def _getValidSavedGrassDBs(self):
         """Returns list of GRASS databases from settings.
         Returns only existing directories."""
@@ -484,6 +516,53 @@ class DataCatalogTree(TreeView):
         self._orig_model = copy.deepcopy(self._model)
         return error
 
+    def _lazyReloadGrassDBNode(self, grassdb_node):
+        genv = gisenv()
+        if grassdb_node.children:
+            del grassdb_node.children[:]
+        all_location_nodes = []
+        errors = []
+        current_mapset_node = None
+        locations = GetListOfLocations(grassdb_node.data["name"])
+        for location in locations:
+            loc_node = self._model.AppendNode(
+                parent=grassdb_node, data=dict(type="location", name=location)
+            )
+            all_location_nodes.append(loc_node)
+            q = Queue()
+            getLocationTree(grassdb_node.data["name"], location, q, lazy=True)
+            maps, error = q.get()
+            if error:
+                errors.append(error)
+            for key in sorted(maps.keys()):
+                mapset_path = os.path.join(
+                    loc_node.parent.data["name"], loc_node.data["name"], key
+                )
+                mapset_node = self._model.AppendNode(
+                    parent=loc_node,
+                    data=dict(
+                        type="mapset",
+                        name=key,
+                        lock=is_mapset_locked(mapset_path),
+                        current=False,
+                        is_different_owner=is_different_mapset_owner(mapset_path),
+                        owner=get_mapset_owner(mapset_path),
+                    ),
+                )
+                if (
+                    grassdb_node.data["name"] == genv["GISDBASE"]
+                    and location == genv["LOCATION_NAME"]
+                    and key == genv["MAPSET"]
+                ):
+                    current_mapset_node = mapset_node
+        if current_mapset_node:
+            self._reloadMapsetNode(current_mapset_node)
+        for node in all_location_nodes:
+            self._model.SortChildren(node)
+        self._model.SortChildren(grassdb_node)
+        self._orig_model = copy.deepcopy(self._model)
+        return errors
+
     def _reloadGrassDBNode(self, grassdb_node):
         """Recursively reload the model of a specific grassdb node.
         Runs reloading locations in parallel."""
@@ -574,11 +653,13 @@ class DataCatalogTree(TreeView):
         self._orig_model = copy.deepcopy(self._model)
         return errors
 
-    def _reloadTreeItems(self):
+    def _reloadTreeItems(self, full=False):
         """Updates grass databases, locations, mapsets and layers in the tree.
 
         It runs in thread, so it should not directly interact with GUI.
         In case of any errors it returns the errors as a list of strings, otherwise None.
+
+        Option full=True forces full reload, full=False will behave based on user settings.
         """
         errors = []
         for grassdatabase in self.grassdatabases:
@@ -590,7 +671,10 @@ class DataCatalogTree(TreeView):
                 )
             else:
                 grassdb_node = grassdb_nodes[0]
-            error = self._reloadGrassDBNode(grassdb_node)
+            if full or not self._useLazyLoading():
+                error = self._reloadGrassDBNode(grassdb_node)
+            else:
+                error = self._lazyReloadGrassDBNode(grassdb_node)
             if error:
                 errors += error
 
@@ -712,11 +796,14 @@ class DataCatalogTree(TreeView):
             self.current_mapset_node.data["current"] = True
             self.current_mapset_node.data["lock"] = is_current_mapset_node_locked()
 
-    def ReloadTreeItems(self):
+    def ReloadTreeItems(self, full=False):
         """Reload dbs, locations, mapsets and layers in the tree."""
         self.busy = wx.BusyCursor()
-        self._quickLoading()
-        self.thread.Run(callable=self._reloadTreeItems, ondone=self._loadItemsDone)
+        if full or not self._useLazyLoading():
+            self._quickLoading()
+        self.thread.Run(
+            callable=self._reloadTreeItems, full=full, ondone=self._loadItemsDone
+        )
 
     def _quickLoading(self):
         """Quick loading of locations to show
@@ -745,6 +832,7 @@ class DataCatalogTree(TreeView):
         self.ScheduleWatchCurrentMapset()
         self.RefreshItems()
         self.ExpandCurrentMapset()
+        self.loadingDone.emit()
 
     def ReloadCurrentMapset(self):
         """Reload current mapset tree only."""
@@ -890,6 +978,9 @@ class DataCatalogTree(TreeView):
                     self.DisplayLayer()
                     return
 
+        if node.data["type"] == "mapset" and not node.children:
+            self._reloadMapsetNode(node)
+            self.RefreshNode(node, recursive=True)
         # expand/collapse location/mapset...
         if self.IsNodeExpanded(node):
             self.CollapseNode(node, recursive=False)
@@ -1486,7 +1577,10 @@ class DataCatalogTree(TreeView):
             grassdb_node = self._model.AppendNode(
                 parent=self._model.root, data=dict(type="grassdb", name=name)
             )
-            self._reloadGrassDBNode(grassdb_node)
+            if self._useLazyLoading():
+                self._lazyReloadGrassDBNode(grassdb_node)
+            else:
+                self._reloadGrassDBNode(grassdb_node)
             self.RefreshItems()
 
             # Update user's settings
@@ -1806,8 +1900,12 @@ class DataCatalogTree(TreeView):
                 node = self.GetDbNode(grassdb=grassdb, location=location, mapset=mapset)
                 if node:
                     if map:
+                        # reload entire mapset when using lazy loading
+                        if not node.children and self._useLazyLoading():
+                            self._reloadMapsetNode(node)
+                            self.RefreshNode(node.parent, recursive=True)
                         # check if map already exists
-                        if not self._model.SearchNodes(
+                        elif not self._model.SearchNodes(
                             parent=node, name=map, type=element
                         ):
                             self.InsertLayer(
@@ -1842,6 +1940,8 @@ class DataCatalogTree(TreeView):
     def _updateAfterMapsetChanged(self):
         """Update tree after current mapset has changed"""
         self.UpdateCurrentDbLocationMapsetNode()
+        self._reloadMapsetNode(self.current_mapset_node)
+        self.RefreshNode(self.current_mapset_node, recursive=True)
         self.ExpandCurrentMapset()
         self.RefreshItems()
         self.ScheduleWatchCurrentMapset()
@@ -1918,6 +2018,38 @@ class DataCatalogTree(TreeView):
         if self._model.GetLeafCount(self._model.root) <= 50:
             self.ExpandAll()
 
+    def OnReloadMapset(self, event):
+        """Reload selected mapset"""
+        node = self.selected_mapset[0]
+        self._reloadMapsetNode(node)
+        self.RefreshNode(node, recursive=True)
+        self.ExpandNode(node, recursive=False)
+
+    def OnReloadLocation(self, event):
+        """Reload all mapsets in selected location"""
+        node = self.selected_location[0]
+        self._reloadLocationNode(node)
+        self.UpdateCurrentDbLocationMapsetNode()
+        self.RefreshNode(node, recursive=True)
+        self.ExpandNode(node, recursive=False)
+
+    def OnReloadGrassdb(self, event):
+        """Reload all mapsets in selected grass db"""
+        node = self.selected_grassdb[0]
+        self.busy = wx.BusyCursor()
+        self.thread.Run(
+            callable=self._reloadGrassDBNode,
+            grassdb_node=node,
+            ondone=self._onDoneReloadingGrassdb,
+            userdata={"node": node},
+        )
+
+    def _onDoneReloadingGrassdb(self, event):
+        del self.busy
+        self.UpdateCurrentDbLocationMapsetNode()
+        self.RefreshNode(event.userdata["node"], recursive=True)
+        self.ExpandNode(event.userdata["node"], recursive=False)
+
     def _getNewMapName(self, message, title, value, element, mapset, env):
         """Dialog for simple text entry"""
         dlg = NameEntryDialog(
@@ -2061,6 +2193,10 @@ class DataCatalogTree(TreeView):
         if self._restricted:
             item.Enable(False)
 
+        item = wx.MenuItem(menu, wx.ID_ANY, _("Re&load maps"))
+        menu.AppendItem(item)
+        self.Bind(wx.EVT_MENU, self.OnReloadMapset, item)
+
         self.PopupMenu(menu)
         menu.Destroy()
 
@@ -2084,6 +2220,10 @@ class DataCatalogTree(TreeView):
         if self._restricted:
             item.Enable(False)
 
+        item = wx.MenuItem(menu, wx.ID_ANY, _("Re&load maps"))
+        menu.AppendItem(item)
+        self.Bind(wx.EVT_MENU, self.OnReloadLocation, item)
+
         self.PopupMenu(menu)
         menu.Destroy()
 
@@ -2115,6 +2255,10 @@ class DataCatalogTree(TreeView):
         if self._restricted:
             item.Enable(False)
 
+        item = wx.MenuItem(menu, wx.ID_ANY, _("Re&load maps"))
+        menu.AppendItem(item)
+        self.Bind(wx.EVT_MENU, self.OnReloadGrassdb, item)
+
         self.PopupMenu(menu)
         menu.Destroy()
 

gui/wxpython/gui_core/preferences.py

@@ -371,7 +371,26 @@ class PreferencesDialog(PreferencesBaseDialog):
         gridSizer.AddGrowableCol(0)
         sizer.Add(gridSizer, proportion=1, flag=wx.ALL | wx.EXPAND, border=5)
         border.Add(sizer, proportion=0, flag=wx.ALL | wx.EXPAND, border=3)
-
+        #
+        # Data catalog settings
+        #
+        box = StaticBox(
+            parent=panel, id=wx.ID_ANY, label=" %s " % _("Data Catalog settings")
+        )
+        sizer = wx.StaticBoxSizer(box, wx.VERTICAL)
+        lazyLoadingDataCatalog = wx.CheckBox(
+            parent=panel,
+            label=_("At startup load maps from current mapset only (in the Data tab)"),
+            name="IsChecked",
+        )
+        lazyLoadingDataCatalog.SetValue(
+            self.settings.Get(group="datacatalog", key="lazyLoading", subkey="enabled")
+        )
+        self.winId["datacatalog:lazyLoading:enabled"] = lazyLoadingDataCatalog.GetId()
+        sizer.Add(
+            lazyLoadingDataCatalog, proportion=1, flag=wx.ALL | wx.EXPAND, border=5
+        )
+        border.Add(sizer, proportion=0, flag=wx.ALL | wx.EXPAND, border=3)
         #
         # workspace
         #