gunittest: support text files for MD5 sum comparisons in multiplatform way

git-svn-id: https://svn.osgeo.org/grass/grass/trunk@64734 15284696-431f-4ddb-bdfa-cd5b030d7da7

lib/python/gunittest/__init__.py

@@ -12,6 +12,8 @@ Initial version of `gunittest` was created during Google Summer of Code 2014
 by Vaclav Petras as a student and Soeren Gebbert as a mentor.
 """
 
+# TODO: consider removing all from here before the backport or release
+
 from __future__ import print_function
 
 try:

lib/python/gunittest/case.py

@@ -22,7 +22,7 @@ from grass.exceptions import CalledModuleError
 from .gmodules import call_module, SimpleModule
 from .checkers import (check_text_ellipsis,
                        text_to_keyvalue, keyvalue_equals, diff_keyvalue,
-                       file_md5, files_equal_md5)
+                       file_md5, text_file_md5, files_equal_md5)
 from .utils import safe_repr
 from .gutils import is_map_in_mapset
 
@@ -35,6 +35,8 @@ class TestCase(unittest.TestCase):
 
     Always use keyword arguments for all parameters other than first two. For
     the first two, it is recommended to use keyword arguments but not required.
+    Be especially careful and always use keyword argument syntax for *msg*
+    parameter.
     """
     longMessage = True  # to get both standard and custom message
     maxDiff = None  # we can afford long diffs
@@ -543,35 +545,52 @@ class TestCase(unittest.TestCase):
             stdmsg = 'File %s is not accessible for reading' % filename
             self.fail(self._formatMessage(msg, stdmsg))
 
-    def assertFileMd5(self, filename, md5, msg=None):
-        """Test that file MD5 sum is equal to the provided sum.
+    def assertFileMd5(self, filename, md5, text=False, msg=None):
+        r"""Test that file MD5 sum is equal to the provided sum.
+
+        Usually, this function is used to test binary files or large text files
+        which cannot be tested in some other way. Text files can be usually
+        tested by some finer method.
+
+        To test text files with this function, you should always use parameter
+        *text* set to ``True``. Note that function ``checkers.text_file_md5()``
+        offers additional parameters which might be advantageous when testing
+        text files.
 
         The typical workflow is that you create a file in a way you
         trust (that you obtain the right file). Then you compute MD5
         sum of the file. And provide the sum in a test as a string::
 
-            self.assertFileMd5('result.txt', md5='807bba4ffa...')
+            self.assertFileMd5('result.png', md5='807bba4ffa...')
 
         Use `file_md5()` function from this package::
 
-            file_md5('original_result.txt')
+            file_md5('original_result.png')
 
         Or in command line, use ``md5sum`` command if available:
 
         .. code-block:: sh
 
-            md5sum some_file.txt
+            md5sum some_file.png
 
         Finaly, you can use Python ``hashlib`` to obtain MD5::
 
             import hashlib
             hasher = hashlib.md5()
             # expecting the file to fit into memory
-            hasher.update(open('original_result.txt', 'rb').read())
+            hasher.update(open('original_result.png', 'rb').read())
             hasher.hexdigest()
+
+        .. note:
+            For text files, always create MD5 sum using ``\n`` (LF)
+            as newline characters for consistency. Also use newline
+            at the end of file (as for example, Git or PEP8 requires).
         """
         self.assertFileExists(filename, msg=msg)
-        actual = file_md5(filename)
+        if text:
+            actual = text_file_md5(filename)
+        else:
+            actual = file_md5(filename)
         if not actual == md5:
             standardMsg = ('File <{name}> does not have the right MD5 sum.\n'
                            'Expected is <{expected}>,'

lib/python/gunittest/checkers.py

@@ -9,6 +9,7 @@ for details.
 :authors: Vaclav Petras, Soeren Gebbert
 """
 
+import os
 import sys
 import re
 import doctest
@@ -570,17 +571,43 @@ def file_md5(filename):
     return hasher.hexdigest()
 
 
-def text_file_md5(filename, exclude_lines=None,
+def text_file_md5(filename, exclude_lines=None, exclude_re=None,
                   prepend_lines=None, append_lines=None):
     """Get a MD5 (check) sum of a text file.
 
-    Works in the same way as `file_md5()` function but allows to
-    exclude lines from the file as well as prepend or append them.
-
-    .. todo::
-        Implement this function.
+    Works in the same way as `file_md5()` function but ignores newlines
+    characters and excludes lines from the file as well as prepend or
+    append them if requested.
+
+    :param exclude_lines: list of strings to be excluded
+        (newline characters should not be part of the strings)
+    :param exclude_re: regular expression string;
+        lines matching this regular expression will not be considered
+    :param prepend_lines: list of lines to be prepended to the file
+        before computing the sum
+    :param append_lines: list of lines  to be appended to the file
+        before computing the sum
     """
-    raise NotImplementedError("Implement, or use file_md5() function instead")
+    hasher = hashlib.md5()
+    if exclude_re:
+        regexp = re.compile(exclude_re)
+    if prepend_lines:
+        for line in prepend_lines:
+            hasher.update(line)
+    with open(filename, 'r') as f:
+        for line in f:
+            # replace platform newlines by standard newline
+            if os.linesep != '\n':
+                line = line.rstrip(os.linesep) + '\n'
+            if exclude_lines and line in exclude_lines:
+                continue
+            if exclude_re and regexp.match(line):
+                continue
+            hasher.update(line)
+    if append_lines:
+        for line in append_lines:
+            hasher.update(line)
+    return hasher.hexdigest()
 
 
 def files_equal_md5(filename_a, filename_b):

lib/python/gunittest/testsuite/test_checkers.py

@@ -14,11 +14,14 @@ for details.
 """
 
 
-from grass.script.utils import parse_key_val
+from grass.script.utils import parse_key_val, try_remove
 
 import grass.gunittest
-from grass.gunittest.checkers import (values_equal, text_to_keyvalue,
-    keyvalue_equals, proj_info_equals, proj_units_equals)
+from grass.gunittest.checkers import (
+    values_equal, text_to_keyvalue,
+    keyvalue_equals, proj_info_equals, proj_units_equals,
+    file_md5, text_file_md5)
+
 
 
 class TestValuesEqual(grass.gunittest.TestCase):
@@ -308,6 +311,87 @@ class TestRasterMapComparisons(grass.gunittest.TestCase):
                                                           sep='='),
                                          precision=0.001))
 
+CORRECT_LINES = [
+    "null_cells=57995100",
+    "cells=60020100",
+    "min=55.5787925720215",
+    "max=156.329864501953"
+]
+
+INCORRECT_LINES = [
+    "null_cells=579951",
+    "cells=60020100",
+    "min=5.5787925720215",
+    "max=156.329864501953"
+]
+
+
+class TestMd5Sums(grass.gunittest.TestCase):
+    r"""
+
+    To create MD5 which is used for testing use:
+
+    .. code: sh
+    $ cat > test.txt << EOF
+    null_cells=57995100
+    cells=60020100
+    min=55.5787925720215
+    max=156.329864501953
+    EOF
+    $ md5sum test.txt
+    9dd6c4bb9d2cf6051b12f4b5f9d70523  test.txt
+    """
+
+    correct_md5sum = '9dd6c4bb9d2cf6051b12f4b5f9d70523'
+    correct_file_name_platform_nl = 'md5_sum_correct_file_platform_nl'
+    correct_file_name_unix_nl = 'md5_sum_correct_file_unix_nl'
+    wrong_file_name = 'md5_sum_wrong_file'
+
+    @classmethod
+    def setUpClass(cls):
+        with open(cls.correct_file_name_platform_nl, 'w') as f:
+            for line in CORRECT_LINES:
+                # \n should be converted to platform newline
+                f.write(line + '\n')
+        with open(cls.correct_file_name_unix_nl, 'wb') as f:
+            for line in CORRECT_LINES:
+                # binary mode will write pure \n
+                f.write(line + '\n')
+        with open(cls.wrong_file_name, 'w') as f:
+            for line in INCORRECT_LINES:
+                # \n should be converted to platform newline
+                f.write(line + '\n')
+
+    @classmethod
+    def tearDownClass(cls):
+        try_remove(cls.correct_file_name_platform_nl)
+        try_remove(cls.correct_file_name_unix_nl)
+        try_remove(cls.wrong_file_name)
+
+    def test_text_file_binary(self):
+        r"""File with ``\n`` (LF) newlines as binary (MD5 has ``\n``)."""
+        self.assertEquals(file_md5(self.correct_file_name_unix_nl),
+                          self.correct_md5sum,
+                          msg="MD5 sums different")
+
+    def test_text_file_platfrom(self):
+        r"""Text file with platform dependent newlines"""
+        self.assertEquals(text_file_md5(self.correct_file_name_platform_nl),
+                          self.correct_md5sum,
+                          msg="MD5 sums different")
+
+    def test_text_file_unix(self):
+        r"""Text file with ``\n`` (LF) newlines"""
+        self.assertEquals(text_file_md5(self.correct_file_name_unix_nl),
+                          self.correct_md5sum,
+                          msg="MD5 sums different")
+
+    def test_text_file_different(self):
+        r"""Text file with ``\n`` (LF) newlines"""
+        self.assertNotEquals(text_file_md5(self.wrong_file_name),
+                             self.correct_md5sum,
+                             msg="MD5 sums must be different")
+
 
 if __name__ == '__main__':
     grass.gunittest.test()