tools/mkhtml.py: get and set git commit and commit date in addon manual page (#2100)

Git commit is getting from local repository if exists (during compilation GRASS GIS core
modules, all addons compilation on our osgeo lxd container server) in others cases getting
commit from remote GitHub API (official addons installed via g.extension module). In case it is not
possible to obtain commit from local git repository or from remote GitHub API server, instead of
commit and commit date, the time when the addon was installed is used (non official addons
installed via g.extension module).

Co-authored-by: Tomas Zigo <tomas.zigo@slovanet.sk>

lib/init/variables.html

@@ -237,7 +237,12 @@ PERMANENT
   <dt>GRASS_PERL</dt>
   <dd>[used during install process for generating man pages]<br>
     set Perl with path.</dd>
-  
+
+  <dt>GRASS_PROXY</dt>
+  <dd>[used during addon install/reinstall process for generating man
+    pages (download commit from GitHub API server and remote modules.xml file)]<br>
+    set the proxy with: <tt>GRASS_PROXY="http=&lt;value&gt;,ftp=&lt;value&gt;"</tt>.</dd>
+
   <dt>GRASS_SKIP_MAPSET_OWNER_CHECK</dt>
   <dd>By default it is not possible to work with MAPSETs that are
     not owned by current user. Setting this variable to any non-empty value

scripts/g.extension/g.extension.py

@@ -355,7 +355,6 @@ def download_addons_paths_file(url, response_format, *args, **kwargs):
                 ),
             )
         return response
-
     except HTTPError as err:
         if err.code == 403 and err.msg == "rate limit exceeded":
             gscript.warning(
@@ -2603,6 +2602,8 @@ def main():
         proxy = urlrequest.ProxyHandler(PROXIES)
         opener = urlrequest.build_opener(proxy)
         urlrequest.install_opener(opener)
+        # Required for mkhtml.py script (get addon git commit from GitHub API server)
+        os.environ["GRASS_PROXY"] = options["proxy"]
 
     # define path
     options["prefix"] = resolve_install_prefix(

tools/mkhtml.py

@@ -16,6 +16,7 @@
 #
 #############################################################################
 
+import http
 import sys
 import os
 import string
@@ -24,6 +25,9 @@ from datetime import datetime
 import locale
 import json
 import pathlib
+import shutil
+import subprocess
+import time
 
 try:
     # Python 2 import
@@ -31,11 +35,26 @@ try:
 except:
     # Python 3 import
     from html.parser import HTMLParser
+
+from six.moves.urllib import request as urlrequest
+from six.moves.urllib.error import HTTPError, URLError
+
 try:
     import urlparse
 except:
     import urllib.parse as urlparse
 
+try:
+    import grass.script as gs
+except ImportError:
+    # During compilation GRASS GIS
+    gs = None
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0",
+}
+HTTP_STATUS_CODES = list(http.HTTPStatus)
+
 if sys.version_info[0] == 2:
     PY2 = True
 else:
@@ -46,6 +65,15 @@ if not PY2:
     unicode = str
 
 
+grass_version = os.getenv("VERSION_NUMBER", "unknown")
+trunk_url = ""
+addons_url = ""
+if grass_version != "unknown":
+    major, minor, patch = grass_version.split(".")
+    trunk_url = "https://github.com/OSGeo/grass/tree/main/"
+    addons_url = f"https://github.com/OSGeo/grass-addons/tree/grass{major}/"
+
+
 def _get_encoding():
     encoding = locale.getdefaultlocale()[1]
     if not encoding:
@@ -68,27 +96,156 @@ def decode(bytes_):
     return unicode(bytes_)
 
 
-html_page_footer_pages_path = os.getenv('HTML_PAGE_FOOTER_PAGES_PATH') if \
-    os.getenv('HTML_PAGE_FOOTER_PAGES_PATH') else ''
+def urlopen(url, *args, **kwargs):
+    """Wrapper around urlopen. Same function as 'urlopen', but with the
+    ability to define headers.
+    """
+    request = urlrequest.Request(url, headers=HEADERS)
+    return urlrequest.urlopen(request, *args, **kwargs)
+
+
+def set_proxy():
+    """Set proxy"""
+    proxy = os.getenv("GRASS_PROXY")
+    if proxy:
+        proxies = {}
+        for ptype, purl in (p.split("=") for p in proxy.split(",")):
+            proxies[ptype] = purl
+        urlrequest.install_opener(
+            urlrequest.build_opener(urlrequest.ProxyHandler(proxies))
+        )
+
+
+set_proxy()
+
+
+def download_git_commit(url, response_format, *args, **kwargs):
+    """Download module/addon last commit from GitHub API
+
+    :param str url: url address
+    :param str response_format: content type
+
+    :return urllib.request.urlopen or None response: response object or
+                                                     None
+    """
+    try:
+        response = urlopen(url, *args, **kwargs)
+        if not response.code == 200:
+            index = HTTP_STATUS_CODES.index(response.code)
+            desc = HTTP_STATUS_CODES[index].description
+            gs.fatal(
+                _(
+                    "Download commit from <{url}>, return status code "
+                    "{code}, {desc}".format(
+                        url=url,
+                        code=response.code,
+                        desc=desc,
+                    ),
+                ),
+            )
+        if response_format not in response.getheader("Content-Type"):
+            gs.fatal(
+                _(
+                    "Wrong downloaded commit file format. "
+                    "Check url <{url}>. Allowed file format is "
+                    "{response_format}.".format(
+                        url=url,
+                        response_format=response_format,
+                    ),
+                ),
+            )
+        return response
+    except HTTPError as err:
+        gs.warning(
+            _(
+                "The download of the commit from the GitHub API "
+                "server wasn't successful, <{}>. Commit and commit "
+                "date will not be included in the <{}> addon html manual "
+                "page.".format(err.msg, pgm)
+            ),
+        )
+    except URLError:
+        gs.warning(
+            _(
+                "Download file from <{url}>, failed. Check internet "
+                "connection. Commit and commit date will not be included "
+                "in the <{pgm}> addon manual page.".format(url=url, pgm=pgm)
+            ),
+        )
+
+
+def get_last_git_commit(src_dir, is_addon, addon_path):
+    """Get last module/addon git commit
+
+    :param str src_dir: module/addon source dir
+    :param bool is_addon: True if it is addon
+    :param str addon_path: addon path
+
+    :return dict git_log: dict with key commit and date, if not
+                          possible download commit from GitHub API server
+                          values of keys have "unknown" string
+    """
+    unknown = "unknown"
+    git_log = {"commit": unknown, "date": unknown}
+    cwd = os.getcwd()
+    datetime_format = "%A %b %d %H:%M:%S %Y"  # e.g. Sun Jan 16 23:09:35 2022
+    grass_modules_url = (
+        "https://api.github.com/repos/osgeo/grass/commits?path={path}"
+        "&page=1&per_page=1&sha=main".format(path=src_dir)
+    )  # sha=git_branch_name
+    grass_addons_url = (
+        "https://api.github.com/repos/osgeo/grass-addons/commits?path={path}"
+        "&page=1&per_page=1&sha=grass{major}".format(
+            path=addon_path,
+            major=major,
+        )
+    )  # sha=git_branch_name
+
+    if shutil.which("git"):
+        if os.path.exists(src_dir):
+            os.chdir(src_dir)
+            git_log["date"] = time.ctime(os.path.getmtime(src_dir))
+        stdout, stderr = subprocess.Popen(
+            args=["git", "log", "-1"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        ).communicate()
+        stdout = decode(stdout)
+        stderr = decode(stderr)
+        os.chdir(cwd)
+
+        if stderr and "fatal: not a git repository" in stderr:
+            response = download_git_commit(
+                url=grass_addons_url if is_addon else grass_modules_url,
+                response_format="application/json",
+            )
+            if response:
+                commit = json.loads(response.read())
+                if commit:
+                    git_log["commit"] = commit[0]["sha"]
+                    git_log["date"] = datetime.strptime(
+                        commit[0]["commit"]["author"]["date"], "%Y-%m-%dT%H:%M:%SZ"
+                    ).strftime(datetime_format)
+        else:
+            if stdout:
+                commit = stdout.splitlines()
+                git_log["commit"] = commit[0].split(" ")[-1]
+                commit_date = commit[2].lstrip("Date:").strip()
+                git_log["date"] = commit_date.rsplit(" ", 1)[0]
+    return git_log
+
+
+html_page_footer_pages_path = (
+    os.getenv("HTML_PAGE_FOOTER_PAGES_PATH")
+    if os.getenv("HTML_PAGE_FOOTER_PAGES_PATH")
+    else ""
+)
 
 pgm = sys.argv[1]
 
 src_file = "%s.html" % pgm
 tmp_file = "%s.tmp.html" % pgm
 
-grass_version = os.getenv("VERSION_NUMBER", "unknown")
-trunk_url = ""
-addons_url = ""
-if grass_version != "unknown":
-    major, minor, patch = grass_version.split(".")
-    base_url = "https://github.com/OSGeo"
-    trunk_url = "{base_url}/grass/tree/releasebranch_{major}_{minor}/".format(
-        base_url=base_url, major=major, minor=minor
-    )
-    addons_url = "{base_url}/grass-addons/tree/grass{major}/".format(
-        base_url=base_url, major=major
-    )
-
 header_base = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 <html>
 <head>
@@ -116,7 +273,14 @@ header_pgm_desc = """<h2>NAME</h2>
 
 sourcecode = string.Template(
 """<h2>SOURCE CODE</h2>
-<p>Available at: <a href="${URL_SOURCE}">${PGM} source code</a> (<a href="${URL_LOG}">history</a>)</p>
+<p>
+  Available at:
+  <a href="${URL_SOURCE}">${PGM} source code</a>
+  (<a href="${URL_LOG}">history</a>)
+</p>
+<p>
+  ${DATE_TAG}
+</p>
 """
 )
 
@@ -428,6 +592,7 @@ else:
     source_url = addons_url
     pgmdir = os.path.sep.join(curdir.split(os.path.sep)[-3:])
 url_source = ""
+addon_path = None
 if os.getenv("SOURCE_URL", ""):
     addon_path = get_addon_path()
     if addon_path:
@@ -458,11 +623,23 @@ if index_name:
     else:
         url_log = url_source.replace(tree, commits)
 
+    git_commit = get_last_git_commit(
+        src_dir=curdir,
+        addon_path=addon_path if addon_path else None,
+        is_addon=True if addon_path else False,
+    )
+    if git_commit["commit"] == "unknown":
+        date_tag = "Accessed: {date}".format(date=git_commit["date"])
+    else:
+        date_tag = "Latest change: {date} in commit: {commit}".format(
+            date=git_commit["date"], commit=git_commit["commit"]
+        )
     sys.stdout.write(
         sourcecode.substitute(
             URL_SOURCE=url_source,
             PGM=pgm,
             URL_LOG=url_log,
+            DATE_TAG=date_tag,
         )
     )
     sys.stdout.write(