g.extension: get branch from version (#1700)

* branch from version

* black

* address review, flake8

* get branch from github API

* handle branch=None

* handle branch=None better

* black/flake8

* cleanup after code review

* update from code review

* try orgs and users

* use repo API

* pass propper API URL

* defaults for get_github_branches, fix l-flag, updates from code review

* abandon github API due to rate limits

* version over default

* use URLError

* use urlparse

* use urlparse

* use get_default_branch for gitlab

* fix keyword issue with urlopen

* global GIT_URL, get_default update

scripts/g.extension/g.extension.py

@@ -77,7 +77,6 @@
 # % description: Specific branch to fetch addon from (only used when fetching from git)
 # % required: no
 # % multiple: no
-# % answer: main
 # %end
 
 # %flag
@@ -169,6 +168,7 @@ from distutils.dir_util import copy_tree
 
 from six.moves.urllib import request as urlrequest
 from six.moves.urllib.error import HTTPError, URLError
+from six.moves.urllib.parse import urlparse
 
 # Get the XML parsing exceptions to catch. The behavior changed with Python 2.7
 # and ElementTree 1.3.
@@ -191,6 +191,7 @@ HEADERS = {
     "User-Agent": "Mozilla/5.0",
 }
 HTTP_STATUS_CODES = list(http.HTTPStatus)
+GIT_URL = "https://github.com/OSGeo/grass-addons"
 
 
 def replace_shebang_win(python_file):
@@ -234,6 +235,78 @@ def urlopen(url, *args, **kwargs):
     return urlrequest.urlopen(request, *args, **kwargs)
 
 
+def get_version_branch(major_version):
+    """Check if version branch for the current GRASS version exists,
+    if not, take branch for the previous version
+    For the official repo we assume that at least one version branch is present"""
+    version_branch = f"grass{major_version}"
+    try:
+        urlrequest.urlopen(f"{GIT_URL}/tree/{version_branch}/src")
+    except URLError:
+        version_branch = "grass{}".format(int(major_version) - 1)
+    return version_branch
+
+
+def get_github_branches(
+    github_api_url="https://api.github.com/repos/OSGeo/grass-addons/branches",
+    version_only=True,
+):
+    """Get ordered list of branch names in repo using github API
+    For the official repo we assume that at least one version branch is present
+    Due to strict rate limits in the github API (60 calls per hour) this function
+    is currently not used."""
+    req = urlrequest.urlopen(github_api_url)
+    content = json.loads(req.read())
+    branches = [repo_branch["name"] for repo_branch in content]
+    if version_only:
+        branches = [
+            version_branch
+            for version_branch in branches
+            if version_branch.startswith("grass")
+        ]
+    branches.sort()
+    return branches
+
+
+def get_default_branch(full_url):
+    """Get default branch for repository in known hosting services
+    (currently only implemented for github, gitlab and bitbucket API)
+    In all other cases "main" is used as default"""
+    # Parse URL
+    url_parts = urlparse(full_url)
+    # Get organization and repository component
+    try:
+        organization, repository = url_parts.path.split("/")[1:3]
+    except URLError:
+        gscript.fatal(
+            _(
+                "Cannot retrieve organization and repository from URL: <{}>.".format(
+                    full_url
+                )
+            )
+        )
+    # Construct API call and retrieve default branch
+    api_calls = {
+        "github.com": f"https://api.github.com/repos/{organization}/{repository}",
+        "gitlab.com": f"https://gitlab.com/api/v4/projects/{organization}%2F{repository}",
+        "bitbucket.org": f"https://api.bitbucket.org/2.0/repositories/{organization}/{repository}/branching-model?",
+    }
+    # Try to get default branch via API. The API call is known to fail a) if the full_url
+    # does not belong to an implemented hosting service or b) if the rate limit of the
+    # API is exceeded
+    try:
+        req = urlrequest.urlopen(api_calls.get(url_parts.netloc))
+        content = json.loads(req.read())
+        # For github and gitlab
+        default_branch = content.get("default_branch")
+        # For bitbucket
+        if not default_branch:
+            default_branch = content.get("development").get("name")
+    except URLError:
+        default_branch = "main"
+    return default_branch
+
+
 def download_addons_paths_file(url, response_format, *args, **kwargs):
     """Generates JSON file containing the download URLs of the official
     Addons
@@ -1403,8 +1476,6 @@ def download_source_code_official_github(url, name, outdev, directory=None):
     """
     if not directory:
         directory = os.path.join(os.getcwd, name)
-    classchar = name.split(".", 1)[0]
-    moduleclass = expand_module_class_name(classchar)
     if grass.call(["svn", "export", url, directory], stdout=outdev) != 0:
         grass.fatal(_("GRASS Addons <%s> not found") % name)
     return directory
@@ -1555,7 +1626,7 @@ def download_source_code(
             response = urlopen(url)
         except URLError:
             # Try download add-on from 'master' branch if default "main" fails
-            if branch == "main":
+            if not branch:
                 try:
                     url = url.replace("main", "master")
                     gscript.message(
@@ -1611,8 +1682,6 @@ def download_source_code(
 def install_extension_std_platforms(name, source, url, branch):
     """Install extension on standard platforms"""
     gisbase = os.getenv("GISBASE")
-    # TODO: workaround, https://github.com/OSGeo/grass-addons/issues/528
-    source_url = "https://github.com/OSGeo/grass-addons/tree/master/grass7/"
 
     # to hide non-error messages from subprocesses
     if grass.verbosity() <= 2:
@@ -1694,7 +1763,7 @@ def install_extension_std_platforms(name, source, url, branch):
         "SCRIPTDIR=%s" % dirs["script"],
         "STRINGDIR=%s" % dirs["string"],
         "ETC=%s" % os.path.join(dirs["etc"]),
-        "SOURCE_URL=%s" % source_url,
+        "SOURCE_URL=%s" % url,
     ]
 
     install_cmd = [
@@ -2140,7 +2209,10 @@ def resolve_xmlurl_prefix(url, source=None):
     gscript.debug("resolve_xmlurl_prefix(url={0}, source={1})".format(url, source))
     if source == "official":
         # use pregenerated modules XML file
-        url = "https://grass.osgeo.org/addons/grass%s/" % version[0]
+        # Define branch to fetch from (latest or current version)
+        version_branch = get_version_branch(version[0])
+
+        url = "https://grass.osgeo.org/addons/{}/".format(version_branch)
     # else try to get extensions XMl from SVN repository (provided URL)
     # the exact action depends on subsequent code (somewhere)
 
@@ -2218,7 +2290,10 @@ def resolve_known_host_service(url, name, branch):
         else:
             actual_start = ""
         if "branch" in match["url_end"]:
-            suffix = match["url_end"].format(name=name, branch=branch)
+            suffix = match["url_end"].format(
+                name=name,
+                branch=branch if branch else get_default_branch(url),
+            )
         else:
             suffix = match["url_end"].format(name=name)
         url = "{prefix}{base}{suffix}".format(
@@ -2299,47 +2374,32 @@ def resolve_source_code(url=None, name=None, branch=None, fork=False):
     >>> resolve_source_code('https://bitbucket.org/joe-user/grass-module') # doctest: +SKIP
     ('remote_zip', 'https://bitbucket.org/joe-user/grass-module/get/default.zip')
     """
-    if not url and name:
-        module_class = get_module_class_name(name)
-        # note: 'trunk' is required to make URL usable for 'svn export' call
-        git_url = (
-            "https://github.com/OSGeo/grass-addons/trunk/"
-            "grass{version}/{module_class}/{module_name}".format(
-                version=version[0], module_class=module_class, module_name=name
-            )
-        )
-        # trac_url = 'https://trac.osgeo.org/grass/browser/grass-addons/' \
-        #            'grass{version}/{module_class}/{module_name}?format=zip' \
-        #            .format(version=version[0],
-        #                    module_class=module_class, module_name=name)
-        # return 'official', trac_url
-        return "official", git_url
-
-    if url and fork:
+    # Handle URL for the offical repo
+    if name and (not url or fork):
         module_class = get_module_class_name(name)
-
         # note: 'trunk' is required to make URL usable for 'svn export' call
-        if branch in ["master", "main"]:
-            svn_reference = "trunk"
+        # and fetches the default branch
+        if not branch:
+            # Fetch from default branch
+            version_branch = get_version_branch(version[0])
+            try:
+                url = url.rstrip("/") if url else GIT_URL
+                urlrequest.urlopen(f"{url}/tree/{version_branch}/src")
+                svn_reference = "branches/{}".format(version_branch)
+            except URLError:
+                svn_reference = "trunk"
         else:
             svn_reference = "branches/{}".format(branch)
 
-        git_url = (
-            "{url}/{branch}/"
-            "grass{version}/{module_class}/{module_name}".format(
-                url=url,
-                version=version[0],
-                module_class=module_class,
-                module_name=name,
-                branch=svn_reference,
-            )
-        )
-        # trac_url = 'https://trac.osgeo.org/grass/browser/grass-addons/' \
-        #            'grass{version}/{module_class}/{module_name}?format=zip' \
-        #            .format(version=version[0],
-        #                    module_class=module_class, module_name=name)
-        # return 'official', trac_url
-        return "official_fork", git_url
+        if not url:
+            # Set URL for the given GRASS version
+            git_url = f"{GIT_URL}/{svn_reference}/src/{module_class}/{name}"
+            return "official", git_url
+        else:
+            # Forks from the official repo should reflect the current structure
+            url = url.rstrip("/")
+            git_url = f"{url}/{svn_reference}/src/{module_class}/{name}"
+            return "official_fork", git_url
 
     # Check if URL can be found
     # Catch corner case if local URL is given starting with file://
@@ -2351,20 +2411,20 @@ def resolve_source_code(url=None, name=None, branch=None, fork=False):
                 open_url = urlopen(url)
                 open_url.close()
                 url_validated = True
-            except:
+            except URLError:
                 pass
         else:
             try:
                 open_url = urlopen("http://" + url)
                 open_url.close()
                 url_validated = True
-            except:
+            except URLError:
                 pass
             try:
                 open_url = urlopen("https://" + url)
                 open_url.close()
                 url_validated = True
-            except:
+            except URLError:
                 pass
 
         if not url_validated:
@@ -2402,10 +2462,9 @@ def get_addons_paths(gg_addons_base_dir):
     and their paths (mkhmtl.py tool)
     """
     get_addons_paths.json_file = "addons_paths.json"
-
-    url = (
-        "https://api.github.com/repos/OSGeo/grass-addons/git/trees/" "main?recursive=1"
-    )
+    # Define branch to fetch from (latest or current version)
+    addons_branch = get_version_branch(version[0])
+    url = f"https://api.github.com/repos/OSGeo/grass-addons/git/trees/{addons_branch}?recursive=1"
 
     response = download_addons_paths_file(
         url=url,
@@ -2505,10 +2564,7 @@ if __name__ == "__main__":
 
     grass_version = grass.version()
     version = grass_version["version"].split(".")
-    # TODO: update temporary workaround of using grass7 subdir of addon-repo, see
-    #       https://github.com/OSGeo/grass-addons/issues/528
-    version[0] = 7
-    version[1] = 9
+
     build_platform = grass_version["build_platform"].split("-", 1)[0]
 
     sys.exit(main())