libpython: Save and load benchmark results (#1711)

Series of functions to save benchmark results to JSON, load them back, and combine them together.

This needs some command line interface to make it convenient, but I used it in a custom benchmark script.

The JSON save ensures that the saved results are a dictionary not just a list which makes the storage format more extensible as first level can contain additional keys.

python/grass/benchmark/Makefile

@@ -5,7 +5,7 @@ include $(MODULE_TOPDIR)/include/Make/Python.make
 
 DSTDIR = $(ETC)/python/grass/benchmark
 
-MODULES = runners plots
+MODULES = plots results runners
 
 PYFILES := $(patsubst %,$(DSTDIR)/%.py,$(MODULES) __init__)
 PYCFILES := $(patsubst %,$(DSTDIR)/%.pyc,$(MODULES) __init__)

python/grass/benchmark/__init__.py

@@ -6,4 +6,11 @@ file of the subpackage.
 """
 
 from .plots import nprocs_plot, num_cells_plot
+from .results import (
+    join_results,
+    load_results,
+    load_results_from_file,
+    save_results,
+    save_results_to_file,
+)
 from .runners import benchmark_nprocs, benchmark_resolutions

python/grass/benchmark/results.py

@@ -0,0 +1,95 @@
+# MODULE:    grass.benchmark
+#
+# AUTHOR(S): Vaclav Petras <wenzeslaus gmail com>
+#
+# PURPOSE:   Benchmarking for GRASS GIS modules
+#
+# COPYRIGHT: (C) 2021 Vaclav Petras, and by the GRASS Development Team
+#
+#            This program is free software under the GNU General Public
+#            License (>=v2). Read the file COPYING that comes with GRASS
+#            for details.
+
+
+"""Handling of raw results from benchmarking"""
+
+import copy
+import json
+from types import SimpleNamespace
+
+
+class ResultsEncoder(json.JSONEncoder):
+    """Results encoder for JSON which handles SimpleNamespace objects"""
+
+    def default(self, o):
+        """Handle additional types"""
+        if isinstance(o, SimpleNamespace):
+            return o.__dict__
+        return super().default(o)
+
+
+def save_results(data):
+    """Save results structure to JSON.
+
+    If the provided object does not have results attribute,
+    it is assumed that the list which should be results attribute was provided,
+    so the provided object object is saved under new ``results`` key.
+
+    Returns JSON as str.
+    """
+    if not hasattr(data, "results"):
+        data = dict(results=data)
+    return json.dumps(data, cls=ResultsEncoder)
+
+
+def save_results_to_file(results, filename):
+    """Saves results to as file as JSON.
+
+    See :func:`save_results` for details.
+    """
+    text = save_results(results)
+    with open(filename, "w") as file:
+        file.write(text)
+
+
+def load_results(data):
+    """Load results structure from JSON.
+
+    Takes str, returns nested structure with SimpleNamespace instead of the
+    default dictionary object. Use attribute access to access by key
+    (not dict-like syntax).
+    """
+    return json.loads(data, object_hook=lambda d: SimpleNamespace(**d))
+
+
+def load_results_from_file(filename):
+    """Loads results from a JSON file.
+
+    See :func:`load_results` for details.
+    """
+    with open(filename, "r") as file:
+        return load_results(file.read())
+
+
+def join_results(results, prefixes=None):
+    """Join multiple lists of results together
+
+    The *results* argument either needs to be a list of result objects
+    or an object with attribute *results* which is the list of result objects.
+    This allows for results loaded from a file to be combined with a simple list.
+
+    The function always returns just a simple list of result objects.
+    """
+    if not prefixes:
+        prefixes = [None] * len(results)
+    joined = []
+    for result_list, prefix in zip(results, prefixes):
+        if hasattr(result_list, "results"):
+            # This is the actual list in the full results structure.
+            result_list = result_list.results
+        for result in result_list:
+            result = copy.deepcopy(result)
+            if prefix:
+                result.label = f"{prefix}: {result.label}"
+            joined.append(result)
+    return joined

python/grass/benchmark/testsuite/test_benchmark.py

@@ -14,8 +14,17 @@
 
 from pathlib import Path
 from subprocess import DEVNULL
+from types import SimpleNamespace
 
-from grass.benchmark import benchmark_resolutions, num_cells_plot
+from grass.benchmark import (
+    benchmark_resolutions,
+    join_results,
+    load_results,
+    load_results_from_file,
+    num_cells_plot,
+    save_results,
+    save_results_to_file,
+)
 from grass.gunittest.case import TestCase
 from grass.gunittest.main import test
 from grass.pygrass.modules import Module
@@ -52,5 +61,63 @@ class TestBenchmarksRun(TestCase):
         self.assertTrue(Path(plot_file).is_file())
 
 
+class TestBenchmarkResults(TestCase):
+    """Tests that saving results work"""
+
+    def test_save_load(self):
+        """Test that results can be saved and loaded"""
+        resolutions = [300, 200]
+        results = [
+            benchmark_resolutions(
+                module=Module(
+                    "r.univar",
+                    map="elevation",
+                    stdout_=DEVNULL,
+                    stderr_=DEVNULL,
+                    run_=False,
+                ),
+                label="Standard output",
+                resolutions=resolutions,
+            )
+        ]
+        results = load_results(save_results(results))
+        plot_file = "test_res_plot.png"
+        num_cells_plot(results.results, filename=plot_file)
+        self.assertTrue(Path(plot_file).is_file())
+
+    def test_data_file_roundtrip(self):
+        """Test functions can save and load to a file"""
+        original = [SimpleNamespace(nprocs=[1, 2, 3], times=[3, 2, 1], label="Test 1")]
+        filename = "test_res_file.json"
+
+        save_results_to_file(original, filename)
+        self.assertTrue(Path(filename).is_file())
+
+        loaded = load_results_from_file(filename).results
+        self.assertEqual(original, loaded)
+
+    def test_join_results_list(self):
+        """Test that we can join lists"""
+        list_1 = [
+            SimpleNamespace(nprocs=[1, 2, 3], times=[3, 2, 1], label="Test 1"),
+            SimpleNamespace(nprocs=[1, 2, 3], times=[3, 2, 1], label="Test 2"),
+        ]
+        list_2 = [SimpleNamespace(nprocs=[1, 2, 3], times=[3, 2, 1], label="Test 3")]
+        new_results = join_results([list_1, list_2])
+        self.assertEqual(len(new_results), 3)
+
+    def test_join_results_structure(self):
+        """Test that we can join a full results structure"""
+        list_1 = SimpleNamespace(
+            results=[
+                SimpleNamespace(nprocs=[1, 2, 3], times=[3, 2, 1], label="Test 1"),
+                SimpleNamespace(nprocs=[1, 2, 3], times=[3, 2, 1], label="Test 2"),
+            ]
+        )
+        list_2 = [SimpleNamespace(nprocs=[1, 2, 3], times=[3, 2, 1], label="Test 3")]
+        new_results = join_results([list_1, list_2])
+        self.assertEqual(len(new_results), 3)
+
+
 if __name__ == "__main__":
     test()