tools/mkhtml: fix src and src commits url in html addon module manual (#987)

scripts/g.extension/g.extension.py

@@ -130,6 +130,7 @@
 
 from __future__ import print_function
 import fileinput
+import http
 import os
 import sys
 import re
@@ -137,10 +138,12 @@ import atexit
 import shutil
 import zipfile
 import tempfile
+import json
 import xml.etree.ElementTree as etree
 from distutils.dir_util import copy_tree
 
-from six.moves.urllib.request import urlopen, urlretrieve, ProxyHandler, build_opener, install_opener
+from six.moves.urllib.request import ProxyHandler, Request, build_opener, \
+    install_opener, urlopen, urlretrieve
 from six.moves.urllib.error import HTTPError, URLError
 
 # Get the XML parsing exceptions to catch. The behavior changed with Python 2.7
@@ -160,6 +163,70 @@ from grass.script import task as gtask
 REMOVE_TMPDIR = True
 PROXIES = {}
 
+HTTP_STATUS_CODES = list(http.HTTPStatus)
+
+
+def urlopen_(url, response_format, headers={}, *args, **kwargs):
+    """Wrapper around urlopen. Same function as 'urlopen', but with the
+    ability to define headers.
+
+    :param str url: url address
+    :param str response_format: content type
+    :param dict headers: https(s) headers
+
+    :return response: urllib.request.urlopen response object
+    """
+    try:
+        request = Request(url, headers=headers)
+        response = urlopen(request, *args, **kwargs)
+
+        if not response.code == 200:
+            index = HTTP_STATUS_CODES.index(response.code)
+            desc = HTTP_STATUS_CODES[index].description
+            gscript.fatal(
+                _(
+                    "Download file from <{url}>, "
+                    "return status code {code}, "
+                    "{desc}".format(
+                        url=url,
+                        code=response.code,
+                        desc=desc,
+                    ),
+                ),
+            )
+        if response_format not in response.getheader('Content-Type'):
+            gscript.fatal(
+                _(
+                    "Wrong downloaded file format. "
+                    "Check url <{url}>. Allowed file format is "
+                    "{response_format}.".format(
+                        url=url,
+                        response_format=response_format,
+                    ),
+                ),
+            )
+        return response
+
+    except HTTPError as err:
+        gscript.fatal(
+            _(
+                "Download file from <{url}>, "
+                "return status code {code}, ".format(
+                    url=url,
+                    code=err,
+                ),
+            ),
+        )
+    except URLError:
+        gscript.fatal(
+            _(
+                "Download file from <{url}>, "
+                "failed. Check internet connection.".format(
+                    url=url,
+                ),
+            ),
+        )
+
 
 def etree_fromfile(filename):
     """Create XML element tree from a given file name"""
@@ -1383,7 +1450,7 @@ def download_source_code(source, url, name, outdev,
 def install_extension_std_platforms(name, source, url):
     """Install extension on standard platforms"""
     gisbase = os.getenv('GISBASE')
-    source_url = "https://trac.osgeo.org/grass/browser/grass-addons/grass7/"
+    source_url = 'https://github.com/OSGeo/grass-addons/tree/master/grass7/'
 
     if source == 'official':
         gscript.message(_("Fetching <%s> from "
@@ -2108,6 +2175,26 @@ def resolve_source_code(url=None, name=None):
         return 'svn', url
 
 
+def get_addons_paths():
+    """Get and save extensions paths as 'extensions_paths.json' json file
+    in the $GRASS_ADDON_BASE dir. The file serves as a list of all addons,
+    and their paths (mkhmtl.py tool)
+    """
+    url = 'https://api.github.com/repos/OSGeo/grass-addons/git/trees/'\
+        'master?recursive=1'
+    addons_paths = json.loads(
+        gscript.decode(
+            urlopen_(
+                url=url,
+                response_format='application/json',
+            ).read(),
+        )
+    )
+    with open(os.path.join(options['prefix'], 'addons_paths.json'),
+              'w') as f:
+        json.dump(addons_paths, f)
+
+
 def main():
     # check dependencies
     if not flags['a'] and sys.platform != "win32":
@@ -2155,6 +2242,7 @@ def main():
 
     if options['operation'] == 'add':
         check_dirs()
+        get_addons_paths()
         source, url = resolve_source_code(name=options['extension'],
                                           url=original_url)
         xmlurl = resolve_xmlurl_prefix(original_url, source=source)

tools/mkhtml.py

@@ -22,6 +22,7 @@ import string
 import re
 from datetime import datetime
 import locale
+import json
 
 try:
     # Python 2 import
@@ -264,6 +265,29 @@ def update_toc(data):
 
     return '\n'.join(ret_data)
 
+
+def get_addon_path(pgm):
+    """Check if pgm is in addons list and get addon path
+
+    :param pgm str: pgm
+
+    :return tuple: (True, path) if pgm is addon else (None, None)
+    """
+    addon_base = os.getenv('GRASS_ADDON_BASE')
+    if addon_base:
+        """'addons_paths.json' is file created during install extension
+        check get_addons_paths() function in the g.extension.py file
+        """
+        addons_paths = os.path.join(addon_base, 'addons_paths.json')
+        if os.path.exists(addons_paths):
+            with open(addons_paths, 'r') as f:
+                addons_paths = json.load(f)
+            for addon in addons_paths['tree']:
+                if pgm in addon['path']:
+                    return True, addon['path']
+    return None, None
+
+
 # process header
 src_data = read_file(src_file)
 name = re.search('(<!-- meta page name:)(.*)(-->)', src_data, re.IGNORECASE)
@@ -382,11 +406,24 @@ if sys.platform == 'win32':
     url_source = url_source.replace(os.path.sep, '/')
 
 if index_name:
-    sys.stdout.write(sourcecode.substitute(URL_SOURCE=url_source, PGM=pgm,
-                                           URL_LOG=url_source.replace('grass/tree',  'grass/commits')))
+    tree = 'grass/tree'
+    commits = 'grass/commits'
+    is_addon, addon_path = get_addon_path(pgm=pgm)
+    if is_addon:
+        # Fix gui/wxpython addon url path
+        url_source = urlparse.urljoin(
+            os.environ['SOURCE_URL'], addon_path.split('/', 1)[1],
+        )
+        tree = 'grass-addons/tree'
+        commits = 'grass-addons/commits'
+
+    sys.stdout.write(sourcecode.substitute(
+        URL_SOURCE=url_source, PGM=pgm, URL_LOG=url_source.replace(
+            tree,  commits)))
     sys.stdout.write(footer_index.substitute(INDEXNAME=index_name,
                                              INDEXNAMECAP=index_name_cap,
-                                             YEAR=year, GRASS_VERSION=grass_version))
+                                             YEAR=year,
+                                             GRASS_VERSION=grass_version))
 else:
     sys.stdout.write(footer_noindex.substitute(YEAR=year,
                                                GRASS_VERSION=grass_version))