g.extension: add branch option (#1130)

* add branch option

* use main as default branch

* try download add-on from the master branch if the main branch does not exist

fix install add-on e.g.: g.extension r.example.plus url='https://github.com/wenzeslaus/r.example.plus'

* try download add-ons files paths json file from the master branch if the main branch doesn't exist

e.g. g.extension -j

* add example with branch

* consolidate messages

* fix a flake8 issue

* use format correctly

* improve message

Co-authored-by: Markus Neteler <neteler@gmail.com>

* improve message

Co-authored-by: Markus Neteler <neteler@gmail.com>

Co-authored-by: Tomas Zigo <50632337+tmszi@users.noreply.github.com>
Co-authored-by: Markus Neteler <neteler@gmail.com>

scripts/g.extension/g.extension.html

@@ -185,7 +185,13 @@ g.extension extension=r.stream.distance operation=remove
 Simple URL to GitHub, GitLab, Bitbucket repositories:
 
 <div class="code"><pre>
-g.extension r.example url=github.com/johnsmith/r.example
+g.extension r.example.plus url="https://github.com/wenzeslaus/r.example.plus"
+</pre></div>
+
+Simple URL to GitHub, GitLab, Bitbucket repositories from a specific (e.g. development) branch:
+
+<div class="code"><pre>
+g.extension r.example.plus url="https://github.com/wenzeslaus/r.example.plus" branch=master
 </pre></div>
 
 Simple URL to OSGeo Trac (downloads a ZIP file, requires download to be enabled in Trac):

scripts/g.extension/g.extension.py

@@ -68,6 +68,15 @@
 #% required: no
 #% multiple: yes
 #%end
+#%option
+#% key: branch
+#% type: string
+#% key_desc: branch
+#% description: Specific branch to fetch addon from (only used when fetching from git)
+#% required: no
+#% multiple: no
+#% answer: main
+#%end
 
 #%flag
 #% key: l
@@ -246,14 +255,9 @@ def download_addons_paths_file(
                 ),
             )
         else:
-            gscript.fatal(
-                _(
-                    "Download file from <{url}>, "
-                    "return status code {code}, ".format(
-                        url=url,
-                        code=err,
-                    ),
-                ),
+            return download_addons_paths_file(
+                url=url.replace('main', 'master'),
+                response_format=response_format,
             )
     except URLError:
         gscript.fatal(
@@ -826,7 +830,7 @@ def write_xml_toolboxes(name, tree=None):
     file_.close()
 
 
-def install_extension(source, url, xmlurl):
+def install_extension(source, url, xmlurl, branch):
     """Install extension (e.g. one module) or a toolbox (list of modules)"""
     gisbase = os.getenv('GISBASE')
     if not gisbase:
@@ -867,7 +871,7 @@ def install_extension(source, url, xmlurl):
             ret1, new_modules_ext, new_files_ext = install_extension_win(extension)
         else:
             ret1, new_modules_ext, new_files_ext, tmp_dir = install_extension_std_platforms(extension,
-                                                            source=source, url=url)
+                                                            source=source, url=url, branch=branch)
         if not flags['d'] and not flags['i']:
             edict[extension]['mlist'].extend(new_modules_ext)
             edict[extension]['flist'].extend(new_files_ext)
@@ -1228,7 +1232,8 @@ def install_extension_win(name):
     os.chdir(TMPDIR)  # this is just to not leave something behind
     srcdir = os.path.join(TMPDIR, name)
     download_source_code(source=source, url=url, name=name,
-                         outdev=outdev, directory=srcdir, tmpdir=TMPDIR)
+                         outdev=outdev, directory=srcdir, tmpdir=TMPDIR,
+                         branch=branch)
 
     # collect module names and file names
     module_list = list()
@@ -1440,22 +1445,40 @@ extract_tar.supported_formats = ['tar.gz', 'gz', 'bz2', 'tar', 'gzip', 'targz']
 
 
 def download_source_code(source, url, name, outdev,
-                         directory=None, tmpdir=None):
+                         directory=None, tmpdir=None, branch=None):
     """Get source code to a local directory for compilation"""
-    gscript.verbose("Downloading source code for <{name}> from <{url}>"
-                    " which is identified as '{source}' type of source..."
-                    .format(source=source, url=url, name=name))
+    gscript.verbose(_("Type of source identified as '{source}'.")
+                    .format(source=source))
     if source == 'official':
+        gscript.message(_("Fetching <%s> from "
+                          "GRASS GIS Addons repository (be patient)...") % name)
         download_source_code_official_github(url, name, outdev, directory)
     elif source == 'svn':
+        gscript.message(_("Fetching <{name}> from "
+                          "<{url}> (be patient)...").format(name=name, url=url))
         download_source_code_svn(url, name, outdev, directory)
     elif source in ['remote_zip']:  # , 'official'
+        gscript.message(_("Fetching <{name}> from "
+                          "<{url}> (be patient)...").format(name=name, url=url))
         # we expect that the module.zip file is not by chance in the archive
         zip_name = os.path.join(tmpdir, 'extension.zip')
         try:
             response = urlopen(url)
         except URLError:
-            grass.fatal(_("Extension <%s> not found") % name)
+            # Try download add-on from 'master' branch if default "main" fails
+            if branch == "main":
+                try:
+                    url = url.replace('main', 'master')
+                    gscript.message(_("Expected default branch not found. "
+                                    "Trying again from <{url}>...")
+                                    .format(url=url))
+                    response = urlopen(url)
+                except URLError:
+                    grass.fatal(_("Extension <{name}> not found. Please check "
+                                  "'url' and 'branch' options".format(name=name)))
+            else:
+                grass.fatal(_("Extension <%s> not found") % name)
+
         with open(zip_name, 'wb') as out_file:
             shutil.copyfileobj(response, out_file)
         extract_zip(name=zip_name, directory=directory, tmpdir=tmpdir)
@@ -1485,18 +1508,11 @@ def download_source_code(source, url, name, outdev,
     assert os.path.isdir(directory)
 
 
-def install_extension_std_platforms(name, source, url):
+def install_extension_std_platforms(name, source, url, branch):
     """Install extension on standard platforms"""
     gisbase = os.getenv('GISBASE')
     source_url = 'https://github.com/OSGeo/grass-addons/tree/master/grass7/'
 
-    if source == 'official':
-        gscript.message(_("Fetching <%s> from "
-                          "GRASS GIS Addons repository (be patient)...") % name)
-    else:
-        gscript.message(_("Fetching <{name}> from "
-                          "<{url}> (be patient)...").format(name=name, url=url))
-
     # to hide non-error messages from subprocesses
     if grass.verbosity() <= 2:
         outdev = open(os.devnull, 'w')
@@ -1506,7 +1522,8 @@ def install_extension_std_platforms(name, source, url):
     os.chdir(TMPDIR)  # this is just to not leave something behind
     srcdir = os.path.join(TMPDIR, name)
     download_source_code(source=source, url=url, name=name,
-                         outdev=outdev, directory=srcdir, tmpdir=TMPDIR)
+                         outdev=outdev, directory=srcdir, tmpdir=TMPDIR,
+                         branch=branch)
     os.chdir(srcdir)
 
     # collect module names
@@ -2017,21 +2034,21 @@ KNOWN_HOST_SERVICES_INFO = {
         'ignored_suffixes': ['.zip', '.tar.gz'],
         'possible_starts': ['', 'https://', 'http://'],
         'url_start': 'https://',
-        'url_end': '/archive/master.zip',
+        'url_end': '/archive/{branch}.zip',
     },
     'GitLab': {
         'domain': 'gitlab.com',
         'ignored_suffixes': ['.zip', '.tar.gz', '.tar.bz2', '.tar'],
         'possible_starts': ['', 'https://', 'http://'],
         'url_start': 'https://',
-        'url_end': '/-/archive/master/{name}-master.zip',
+        'url_end': '/-/archive/{branch}/{name}-{branch}.zip',
     },
     'Bitbucket': {
         'domain': 'bitbucket.org',
         'ignored_suffixes': ['.zip', '.tar.gz', '.gz', '.bz2'],
         'possible_starts': ['', 'https://', 'http://'],
         'url_start': 'https://',
-        'url_end': '/get/master.zip',
+        'url_end': '/get/{branch}.zip',
     },
 }
 
@@ -2039,7 +2056,7 @@ KNOWN_HOST_SERVICES_INFO = {
 # https://gitlab.com/user/reponame/repository/archive.zip?ref=b%C3%A9po
 
 
-def resolve_known_host_service(url, name):
+def resolve_known_host_service(url, name, branch):
     """Determine source type and full URL for known hosting service
 
     If the service is not determined from the provided URL, tuple with
@@ -2069,10 +2086,13 @@ def resolve_known_host_service(url, name):
             actual_start = match['url_start']
         else:
             actual_start = ''
+        if 'branch' in  match['url_end']:
+            suffix = match['url_end'].format(name=name, branch=branch)
+        else:
+            suffix = match['url_end'].format(name=name)
         url = '{prefix}{base}{suffix}'.format(prefix=actual_start,
                                               base=url.rstrip('/'),
-                                              suffix=match['url_end'].format(
-                                                  name=name))
+                                              suffix=suffix)
         gscript.verbose(_("Will use the following URL for download: {0}")
                         .format(url))
         return 'remote_zip', url
@@ -2081,7 +2101,7 @@ def resolve_known_host_service(url, name):
 
 
 # TODO: add also option to enforce the source type
-def resolve_source_code(url=None, name=None):
+def resolve_source_code(url=None, name=None, branch=None):
     """Return type and URL or path of the source code
 
     Local paths are not presented as URLs to be usable in standard functions.
@@ -2089,7 +2109,7 @@ def resolve_source_code(url=None, name=None):
     has the unfortunate consequence that the not existing files are evaluated
     as remote URLs. When path is not evaluated, Subversion is assumed for
     backwards compatibility. When GitHub repository is specified, ZIP file
-    link is returned. The ZIP is for master branch, not the default one because
+    link is returned. The ZIP is for {branch} branch, not the default one because
     GitHub does not provide the default branch in the URL (July 2015).
 
     :returns: tuple with type of source and full URL or path
@@ -2202,7 +2222,7 @@ def resolve_source_code(url=None, name=None):
                 return suffix, os.path.abspath(url)
     # Handle remote URLs
     else:
-        source, resolved_url = resolve_known_host_service(url, name)
+        source, resolved_url = resolve_known_host_service(url, name, branch)
         if source:
             return source, resolved_url
         # we allow URL to end with =zip or ?zip and not only .zip
@@ -2225,7 +2245,7 @@ def get_addons_paths(gg_addons_base_dir):
     get_addons_paths.json_file = 'addons_paths.json'
 
     url = 'https://api.github.com/repos/OSGeo/grass-addons/git/trees/'\
-        'master?recursive=1'
+        'main?recursive=1'
 
     response = download_addons_paths_file(
         url=url, response_format='application/json',
@@ -2243,6 +2263,7 @@ def main():
         check_progs()
 
     original_url = options['url']
+    branch = options['branch']
 
     # manage proxies
     global PROXIES
@@ -2268,7 +2289,8 @@ def main():
         # but will work only as long as the function does not check
         # if the URL is actually valid or something
         source, url = resolve_source_code(name='dummy',
-                                          url=original_url)
+                                          url=original_url,
+                                          branch=branch)
         xmlurl = resolve_xmlurl_prefix(original_url, source=source)
         list_available_extensions(xmlurl)
         return 0
@@ -2295,9 +2317,10 @@ def main():
             """
             get_addons_paths(gg_addons_base_dir=options['prefix'])
         source, url = resolve_source_code(name=options['extension'],
-                                          url=original_url)
+                                          url=original_url,
+                                          branch=branch)
         xmlurl = resolve_xmlurl_prefix(original_url, source=source)
-        install_extension(source=source, url=url, xmlurl=xmlurl)
+        install_extension(source=source, url=url, xmlurl=xmlurl, branch=branch)
     else:  # remove
         remove_extension(force=flags['f'])