grass.lib: Update ctypesgen (#2243)

* ctypesgen: update to upstream fd495e5 version

https://github.com/ctypesgen/ctypesgen/commit/fd495e5edb2f69e5407774f8937a1d62cd33fe55

* apply patches

* update makefile

* lib imagery test: HOST_DIRSEP is now str, no need to decode

lib/imagery/testsuite/test_imagery_signature_management.py

@@ -40,19 +40,17 @@ from grass.lib.imagery import (
     I_make_signatures_dir,
 )
 
-H_DIRSEP = HOST_DIRSEP.decode("utf-8")
-
 
 class GetSignaturesElementTestCase(TestCase):
     def test_get_sig(self):
         cdir = ctypes.create_string_buffer(GNAME_MAX)
         I_get_signatures_dir(cdir, I_SIGFILE_TYPE_SIG)
-        self.assertEqual(utils.decode(cdir.value), f"signatures{H_DIRSEP}sig")
+        self.assertEqual(utils.decode(cdir.value), f"signatures{HOST_DIRSEP}sig")
 
     def test_get_sigset(self):
         cdir = ctypes.create_string_buffer(GNAME_MAX)
         I_get_signatures_dir(cdir, I_SIGFILE_TYPE_SIGSET)
-        self.assertEqual(utils.decode(cdir.value), f"signatures{H_DIRSEP}sigset")
+        self.assertEqual(utils.decode(cdir.value), f"signatures{HOST_DIRSEP}sigset")
 
 
 class MakeSignaturesElementTestCase(TestCase):

python/libgrass_interface_generator/ctypesgen/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2007-2008, Ctypesgen Developers
+Copyright (c) 2007-2022, Ctypesgen Developers
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -9,14 +9,11 @@ modification, are permitted provided that the following conditions are met:
 2. Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in the
    documentation and/or other materials provided with the distribution.
-3. Neither the name of the <ORGANIZATION> nor the names of its
-   contributors may be used to endorse or promote products derived from
-   this software without specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS

python/libgrass_interface_generator/Makefile

@@ -56,48 +56,47 @@ proj_INC        = $(PROJINC)
 vector_INC      = $(VECT_INC) $(VECT_CFLAGS)
 vedit_INC       = $(VECT_INC) $(VECT_CFLAGS)
 
-MAC_FLAGS = ""
-ifneq ($(findstring darwin,$(ARCH)),)
-MAC_FLAGS  = "-D_Nullable="
-endif
-
-CTYPESGEN = ./run.py
-CTYPESFLAGS = --cpp "$(CC) -E $(CPPFLAGS) $(LFS_CFLAGS) $(MAC_FLAGS) $(EXTRA_CFLAGS) $(NLS_CFLAGS) $(DEFS) $(EXTRA_INC) $(INC) -D__GLIBC_HAVE_LONG_LONG"
-EXTRA_CLEAN_FILES := $(wildcard ctypesgencore/*.pyc) $(wildcard ctypesgencore/*/*.pyc)
-
 ifneq ($(MINGW),)
 INTLLIB = -lintl-8
 endif
 
+CTYPESGEN = ./run.py
+CTYPESFLAGS = --no-embed-preamble --strip-build-path "$(ARCH_DISTDIR)" \
+	--cpp "$(CC) -E $(CPPFLAGS) $(LFS_CFLAGS) $(EXTRA_CFLAGS) $(NLS_CFLAGS) $(DEFS) $(EXTRA_INC) $(INC) -D__GLIBC_HAVE_LONG_LONG"
+EXTRA_CLEAN_FILES := $(wildcard ctypesgen/*.pyc) $(wildcard ctypesgen/*/*.pyc)
+
 include $(MODULE_TOPDIR)/include/Make/Python.make
 
 PYDIR = $(ETC)/python
 GDIR = $(PYDIR)/grass
 DSTDIR = $(GDIR)/lib
 
-PYFILES  := $(patsubst %,$(DSTDIR)/%.py,$(MODULES) __init__ ctypes_preamble ctypes_loader)
-PYCFILES  := $(patsubst %,$(DSTDIR)/%.pyc,$(MODULES) __init__ ctypes_preamble ctypes_loader)
+PYFILES  := $(patsubst %,$(DSTDIR)/%.py,$(MODULES))
+PYCFILES  := $(patsubst %,$(DSTDIR)/%.pyc,$(MODULES))
 LPYFILES := $(patsubst %,$(OBJDIR)/%.py,$(MODULES))
 
-COPY_FILES = $(DSTDIR)/ctypes_loader.py $(DSTDIR)/ctypes_preamble.py $(DSTDIR)/__init__.py
+COPY_FILES = ctypes_loader ctypes_preamble __init__
+PY_COPY_FILES = $(patsubst %,$(DSTDIR)/%.py,$(COPY_FILES))
+PYC_COPY_FILES = $(patsubst %,$(DSTDIR)/%.pyc,$(COPY_FILES))
 
 ifeq ($(strip $(GRASS_LIBRARY_TYPE)),shlib)
-default: $(COPY_FILES)
+default:
 	$(MAKE) $(DSTDIR)
 	$(MAKE) $(LPYFILES) $(PYFILES) $(PYCFILES)
+	$(MAKE) $(PY_COPY_FILES) $(PYC_COPY_FILES)
 else
 default:
 	@echo "Cannot build Python wrappers for static libraries" >&2
 	exit 1
 endif
 
-$(COPY_FILES): | $(DSTDIR)
-$(DSTDIR)/ctypes_loader.py: ctypesgen/libraryloader.py
-	cp -f $< $@
-$(DSTDIR)/ctypes_preamble.py: ctypesgen/printer_python/preamble/3_2.py
-	cp -f $< $@
+$(PY_COPY_FILES): | $(DSTDIR)
 $(DSTDIR)/__init__.py: libgrass__init__.py
-	cp -f $< $@
+	$(INSTALL_DATA) $< $@
+$(DSTDIR)/ctypes_loader.py: $(OBJDIR)/ctypes_loader.py
+	$(INSTALL_DATA) $< $@
+$(DSTDIR)/ctypes_preamble.py: $(OBJDIR)/ctypes_preamble.py
+	$(INSTALL_DATA) $< $@
 
 $(DSTDIR)/%.py: $(OBJDIR)/%.py | $(DSTDIR)
 	$(INSTALL_DATA) $< $@
@@ -118,4 +117,4 @@ $(GDIR): | $(PYDIR)
 $(DSTDIR): | $(GDIR)
 	$(MKDIR) $@
 
-.SECONDARY: $(patsubst %,%.py,$(MODULES))
+.SECONDARY: $(patsubst %,%.py,$(MODULES)) $(patsubst %,%.py,$(COPY_FILES))

python/libgrass_interface_generator/README.md

@@ -1,7 +1,7 @@
 ## Notes on ctypesgen
 
 Currently installed version:
-https://github.com/ctypesgen/ctypesgen/commit/0681f8ef1742206c171d44b7872c700f34ffe044 (3 March 2020)
+https://github.com/ctypesgen/ctypesgen/commit/fd495e5edb2f69e5407774f8937a1d62cd33fe55 (17 February 2022)
 
 
 ### How to update ctypesgen version
@@ -17,51 +17,6 @@ https://github.com/ctypesgen/ctypesgen/commit/0681f8ef1742206c171d44b7872c700f34
 
 It is highly encouraged to report [upstreams](https://github.com/ctypesgen/ctypesgen) necessary patches for GRASS.
 
-#### POINTER patch
-
-https://trac.osgeo.org/grass/ticket/2748
-https://trac.osgeo.org/grass/ticket/3641
-
-Every generated GRASS library bridge part (gis.py, raster.py etc.) is a standalone
-product. E.g. parts of libgis (include/grass/gis.h) used in libraster
-(etc/python/grass/lib/raster.py) are also defined in raster.py. This way there
-is a definition of `struct_Cell_head` in both gis.py and raster.py -- a situation
-ctypes doesn't approve of. This patch seems to fix related errors in GRASS.
-Manually remove e.g. the gis.py parts in raster.py and make an "import" also did
-the work, but that would be more difficult to implement as part of ctypesgen
-file generation.
-
-```diff
---- ctypesgen/printer_python/preamble/3_2.py.orig
-+++ ctypesgen/printer_python/preamble/3_2.py
-@@ -14,6 +14,24 @@
- del _int_types
- 
- 
-+def POINTER(obj):
-+    p = ctypes.POINTER(obj)
-+
-+    # Convert None to a real NULL pointer to work around bugs
-+    # in how ctypes handles None on 64-bit platforms
-+    if not isinstance(p.from_param, classmethod):
-+
-+        def from_param(cls, x):
-+            if x is None:
-+                return cls()
-+            else:
-+                return x
-+
-+        p.from_param = classmethod(from_param)
-+
-+    return p
-+
-+
- class UserString:
-     def __init__(self, seq):
-         if isinstance(seq, bytes):
-
-```
-
 #### Ctypes "unnamed structure member with 0 bit size"-patch
 
 Using unnamed zero bit sized structure members, e.g.:
@@ -97,7 +52,7 @@ generated files.
 ```diff
 --- ctypesgen/printer_python/printer.py.orig
 +++ ctypesgen/printer_python/printer.py
-@@ -46,6 +46,7 @@
+@@ -22,6 +22,7 @@ class WrapperPrinter:
  
          self.file = open(outpath, "w") if outpath else sys.stdout
          self.options = options
@@ -105,7 +60,7 @@ generated files.
  
          if self.options.strip_build_path and self.options.strip_build_path[-1] != os.path.sep:
              self.options.strip_build_path += os.path.sep
-@@ -82,9 +83,14 @@
+@@ -61,9 +62,14 @@ class WrapperPrinter:
          self.print_group(self.options.inserted_files, "inserted files", self.insert_file)
          self.strip_prefixes()
  
@@ -121,7 +76,7 @@ generated files.
      def print_group(self, list, name, function):
          if list:
              self.file.write("# Begin %s\n" % name)
-@@ -231,6 +237,7 @@
+@@ -254,6 +260,7 @@ class WrapperPrinter:
              mem = list(struct.members[mi])
              if mem[0] is None:
                  while True:
@@ -129,7 +84,7 @@ generated files.
                      name = "%s%i" % (anon_prefix, n)
                      n += 1
                      if name not in names:
-@@ -243,7 +250,10 @@
+@@ -266,7 +273,10 @@ class WrapperPrinter:
  
          self.file.write("%s_%s.__slots__ = [\n" % (struct.variety, struct.tag))
          for name, ctype in struct.members:
@@ -141,7 +96,7 @@ generated files.
          self.file.write("]\n")
  
          if len(unnamed_fields) > 0:
-@@ -255,9 +265,15 @@
+@@ -278,9 +288,15 @@ class WrapperPrinter:
          self.file.write("%s_%s._fields_ = [\n" % (struct.variety, struct.tag))
          for name, ctype in struct.members:
              if isinstance(ctype, CtypesBitfield):
@@ -159,7 +114,7 @@ generated files.
                  )
              else:
                  self.file.write("    ('%s', %s),\n" % (name, ctype.py_string()))
-@@ -458,3 +474,57 @@
+@@ -481,3 +497,57 @@ class WrapperPrinter:
          )
  
          inserted_file.close()
@@ -220,141 +175,16 @@ generated files.
 
 ```
 
-#### Loader and preamble patch
-
-Replaces sed introduced with:
-"Move ctypesgen boilerplate to common module"
-
-https://github.com/OSGeo/grass/commit/59eeff479cd39fd503e276d164977648938cc85b
-
-
-```diff
---- ctypesgen/printer_python/printer.py.orig
-+++ ctypesgen/printer_python/printer.py
-@@ -156,19 +156,22 @@
-         path, v = get_preamble(**m.groupdict())
- 
-         self.file.write("# Begin preamble for Python v{}\n\n".format(v))
--        preamble_file = open(path, "r")
--        self.file.write(preamble_file.read())
--        preamble_file.close()
-+        self.file.write("from .ctypes_preamble import *\n")
-+        self.file.write("from .ctypes_preamble import _variadic_function\n")
-+        # preamble_file = open(path, "r")
-+        # self.file.write(preamble_file.read())
-+        # preamble_file.close()
-         self.file.write("\n# End preamble\n")
- 
-     def print_loader(self):
-         self.file.write("_libs = {}\n")
-         self.file.write("_libdirs = %s\n\n" % self.options.compile_libdirs)
-         self.file.write("# Begin loader\n\n")
--        path = path_to_local_file("libraryloader.py", libraryloader)
--        loader_file = open(path, "r")
--        self.file.write(loader_file.read())
--        loader_file.close()
-+        self.file.write("from .ctypes_loader import *\n")        
-+        # path = path_to_local_file("libraryloader.py", libraryloader)
-+        # loader_file = open(path, "r")
-+        # self.file.write(loader_file.read())
-+        # loader_file.close()
-         self.file.write("\n# End loader\n\n")
-         self.file.write(
-             "add_library_search_dirs([%s])"
-
-```
-
-#### Mac specific patches
-
-Enable Ctypesgen parsing of non-utf8 files on macOS
-https://github.com/OSGeo/grass/pull/385
-
-```diff
---- ctypesgen/parser/preprocessor.py.orig
-+++ ctypesgen/parser/preprocessor.py
-@@ -160,9 +160,32 @@
-         self.cparser.handle_status(cmd)
- 
-         pp = subprocess.Popen(
--            cmd, shell=True, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-+            cmd,
-+            shell=True,
-+            universal_newlines=True,
-+            stdout=subprocess.PIPE,
-+            stderr=subprocess.PIPE,
-         )
--        ppout, pperr = pp.communicate()
-+        try:
-+            ppout, pperr = pp.communicate()
-+        except UnicodeError:
-+            # Fix for https://trac.osgeo.org/grass/ticket/3883,
-+            # handling file(s) encoded with mac_roman
-+            if sys.platform == "darwin":
-+                pp = subprocess.Popen(
-+                    cmd,
-+                    shell=True,
-+                    universal_newlines=False,  # read as binary
-+                    stdout=subprocess.PIPE,
-+                    stderr=subprocess.PIPE,
-+                )
-+                ppout, pperr = pp.communicate()
-+
-+                data = ppout.decode("utf8", errors="replace")
-+                ppout = data.replace("\r\n", "\n").replace("\r", "\n")
-+                pperr = pperr.decode("utf8", errors="replace")
-+            else:
-+                raise UnicodeError
- 
-         for line in pperr.split("\n"):
-             if line:
-
-```
-
-
-macOS: use `@rpath` as dynamic linker
-https://github.com/OSGeo/grass/pull/981
-
-```diff
---- ctypesgen/libraryloader.py.orig
-+++ ctypesgen/libraryloader.py
-@@ -168,6 +168,7 @@
-         dyld_fallback_library_path = _environ_path("DYLD_FALLBACK_LIBRARY_PATH")
-         if not dyld_fallback_library_path:
-             dyld_fallback_library_path = [os.path.expanduser("~/lib"), "/usr/local/lib", "/usr/lib"]
-+        dyld_fallback_library_path.extend(_environ_path('LD_RUN_PATH'))
- 
-         dirs = []
-
-```
-
-
 #### Windows specific patches
 
-The type `__int64` isn't defined in ctypesgen
-https://trac.osgeo.org/grass/ticket/3506
-
-```diff
---- ctypesgen/ctypedescs.py.orig
-+++ ctypesgen/ctypedescs.py
-@@ -41,6 +41,7 @@ ctypes_type_map = {
-     ("int16_t", True, 0): "c_int16",
-     ("int32_t", True, 0): "c_int32",
-     ("int64_t", True, 0): "c_int64",
-+    ("__int64", True, 0): "c_int64",
-     ("uint8_t", True, 0): "c_uint8",
-     ("uint16_t", True, 0): "c_uint16",
-     ("uint32_t", True, 0): "c_uint32",
-
-```
-
 Patch for OSGeo4W packaging, adapted from
 https://github.com/jef-n/OSGeo4W/blob/master/src/grass/osgeo4w/patch
 
 ```diff
 --- ctypesgen/libraryloader.py.orig
 +++ ctypesgen/libraryloader.py
-@@ -321,6 +321,12 @@
- class WindowsLibraryLoader(LibraryLoader):
+@@ -372,6 +372,12 @@ class WindowsLibraryLoader(LibraryLoader):
+ 
      name_formats = ["%s.dll", "lib%s.dll", "%slib.dll", "%s"]
  
 +    def __init__(self):
@@ -364,8 +194,7 @@ https://github.com/jef-n/OSGeo4W/blob/master/src/grass/osgeo4w/patch
 +                os.add_dll_directory(p)
 +
      class Lookup(LibraryLoader.Lookup):
-         def __init__(self, path):
-             super(WindowsLibraryLoader.Lookup, self).__init__(path)
+         """Lookup class for Windows libraries..."""
 
 ```
 
@@ -377,15 +206,15 @@ https://github.com/OSGeo/grass/commit/65eef4767aa416ca55f7e36f62dce7ce083fe450
 ```diff
 --- ctypesgen/parser/preprocessor.py.orig
 +++ ctypesgen/parser/preprocessor.py
-@@ -159,6 +159,9 @@
+@@ -125,6 +125,9 @@ class PreprocessorParser(object):
  
          self.cparser.handle_status(cmd)
  
-+        if sys.platform == "win32":
++        if IS_WINDOWS:
 +            cmd = ["sh.exe", "-c", cmd]
 +
          pp = subprocess.Popen(
-             cmd, shell=True, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-         )
+             cmd,
+             shell=True,
 
 ```

python/libgrass_interface_generator/ctypesgen/.gitignore

@@ -0,0 +1 @@
+VERSION

python/libgrass_interface_generator/ctypesgen/__init__.py

@@ -1,7 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: us-ascii -*-
-# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
-
 """
 Ctypesgencore is the module that contains the main body of ctypesgen - in fact,
 it contains everything but the command-line interface.
@@ -62,15 +58,6 @@ from . import processor
 from . import printer_python
 from . import version
 
-try:
-    from . import printer_json
-except ImportError:
-    pass
-
-__version__ = version.VERSION.partition("-")[-1]
-VERSION = __version__
-
-
 # Modules describing internal format
 from . import descriptions
 from . import ctypedescs
@@ -80,4 +67,12 @@ from . import expressions
 from . import messages
 from . import options
 
+try:
+    from . import printer_json
+except ImportError:
+    pass
+
+__version__ = version.VERSION.partition("-")[-1]
+VERSION = __version__
+
 printer = printer_python  # Default the printer to generating Python

python/libgrass_interface_generator/ctypesgen/ctypedescs.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 ctypesgen.ctypedescs contains classes to represent a C type. All of them
 classes are subclasses of CtypesType.
@@ -19,8 +17,6 @@ representing an array of four integers could be created using:
 str(ctype) would evaluate to "c_int * 4".
 """
 
-import warnings
-
 __docformat__ = "restructuredtext"
 
 ctypes_type_map = {
@@ -38,8 +34,11 @@ ctypes_type_map = {
     ("double", True, 0): "c_double",
     ("double", True, 1): "c_longdouble",
     ("int8_t", True, 0): "c_int8",
+    ("__int8", True, 0): "c_int8",
     ("int16_t", True, 0): "c_int16",
+    ("__int16", True, 0): "c_int16",
     ("int32_t", True, 0): "c_int32",
+    ("__int32", True, 0): "c_int32",
     ("int64_t", True, 0): "c_int64",
     ("__int64", True, 0): "c_int64",
     ("uint8_t", True, 0): "c_uint8",
@@ -62,6 +61,7 @@ ctypes_type_map_python_builtin = {
     ("va_list", True, 0): "c_void_p",
 }
 
+
 # This protocol is used for walking type trees.
 class CtypesTypeVisitor(object):
     def visit_struct(self, struct):
@@ -294,9 +294,11 @@ def anonymous_struct_tagnum():
     last_tagnum += 1
     return last_tagnum
 
+
 def fmt_anonymous_struct_tag(num):
     return "anon_%d" % num
 
+
 def anonymous_struct_tag():
     return fmt_anonymous_struct_tag(anonymous_struct_tagnum())
 
@@ -311,14 +313,14 @@ class CtypesStruct(CtypesType):
 
         if type(self.tag) == int or not self.tag:
             if type(self.tag) == int:
-              self.tag = fmt_anonymous_struct_tag(self.tag)
+                self.tag = fmt_anonymous_struct_tag(self.tag)
             else:
-              self.tag = anonymous_struct_tag()
+                self.tag = anonymous_struct_tag()
             self.anonymous = True
         else:
             self.anonymous = False
 
-        if self.members == None:
+        if self.members is None:
             self.opaque = True
         else:
             self.opaque = False
@@ -368,7 +370,7 @@ class CtypesEnum(CtypesType):
         else:
             self.anonymous = False
 
-        if self.enumerators == None:
+        if self.enumerators is None:
             self.opaque = True
         else:
             self.opaque = False

python/libgrass_interface_generator/ctypesgen/descriptions.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 ctypesgen.descriptions contains classes to represent a description of a
 struct, union, enum, function, constant, variable, or macro. All the

python/libgrass_interface_generator/ctypesgen/expressions.py

@@ -1,16 +1,19 @@
-#!/usr/bin/env python
-
 """
 The expressions module contains classes to represent an expression. The main
 class is ExpressionNode. ExpressionNode's most useful method is py_string(),
 which returns a Python string representing that expression.
 """
 
-import sys
-
-from .ctypedescs import *
+import warnings
 import keyword
 
+from ctypesgen.ctypedescs import (
+    CtypesPointer,
+    CtypesSimple,
+    CtypesStruct,
+    CtypesType,
+)
+
 # Right now, the objects in this module are all oriented toward evaluation.
 # However, they don't have to be, since ctypes objects are mutable. For example,
 # shouldn't it be possible to translate the macro:
@@ -27,17 +30,12 @@ import keyword
 
 
 class EvaluationContext(object):
-    """Interface for evaluating expression nodes.
-    """
+    """Interface for evaluating expression nodes."""
 
     def evaluate_identifier(self, name):
         warnings.warn('Attempt to evaluate identifier "%s" failed' % name)
         return 0
 
-    def evaluate_sizeof(self, type):
-        warnings.warn('Attempt to evaluate sizeof "%s" failed' % str(type))
-        return 0
-
     def evaluate_sizeof(self, object):
         warnings.warn('Attempt to evaluate sizeof object "%s" failed' % str(object))
         return 0
@@ -67,20 +65,21 @@ class ExpressionNode(object):
 
 
 class ConstantExpressionNode(ExpressionNode):
-    def __init__(self, value):
+    def __init__(self, value, is_literal=False):
         ExpressionNode.__init__(self)
         self.value = value
+        self.is_literal = is_literal
 
     def evaluate(self, context):
         return self.value
 
     def py_string(self, can_be_ctype):
-        if sys.platform != "win32" or (sys.platform == "win32" and sys.version_info >= (2, 6)):
-            # Windows python did not get infinity support until 2.6
-            if self.value == float("inf"):
-                return "float('inf')"
-            elif self.value == float("-inf"):
-                return "float('-inf')"
+        if self.is_literal:
+            return self.value
+        if self.value == float("inf"):
+            return "float('inf')"
+        elif self.value == float("-inf"):
+            return "float('-inf')"
         return repr(self.value)
 
 
@@ -308,7 +307,7 @@ class TypeCastExpressionNode(ExpressionNode):
             # c_char can take integer or byte types, but the others can *only*
             # take non-char arguments.
             # ord_if_char must be provided by preambles
-            if isinstance(self.ctype, CtypesSimple) and (self.ctype.name, self.ctype.signed) == (
+            if isinstance(self.ctype, CtypesSimple) and (self.ctype.name, self.ctype.signed,) == (
                 "char",
                 True,
             ):
@@ -321,7 +320,9 @@ class TypeCastExpressionNode(ExpressionNode):
                 ord_if_char = "ord_if_char"
 
             return "({to} ({ord_if_char}({frm}))).value".format(
-                to=self.ctype.py_string(), ord_if_char=ord_if_char, frm=self.base.py_string(False)
+                to=self.ctype.py_string(),
+                ord_if_char=ord_if_char,
+                frm=self.base.py_string(False),
             )
 
 

python/libgrass_interface_generator/ctypesgen/libraryloader.py

@@ -1,3 +1,6 @@
+"""
+Load libraries - appropriately for all our supported platforms
+"""
 # ----------------------------------------------------------------------------
 # Copyright (c) 2008 David James
 # Copyright (c) 2006-2008 Alex Holkner
@@ -32,24 +35,34 @@
 # POSSIBILITY OF SUCH DAMAGE.
 # ----------------------------------------------------------------------------
 
-import os.path, re, sys, glob
-import platform
 import ctypes
 import ctypes.util
+import glob
+import os.path
+import platform
+import re
+import sys
 
 
 def _environ_path(name):
+    """Split an environment variable into a path-like list elements"""
     if name in os.environ:
         return os.environ[name].split(":")
-    else:
-        return []
+    return []
 
 
-class LibraryLoader(object):
+class LibraryLoader:
+    """
+    A base class For loading of libraries ;-)
+    Subclasses load libraries for specific platforms.
+    """
+
     # library names formatted specifically for platforms
     name_formats = ["%s"]
 
-    class Lookup(object):
+    class Lookup:
+        """Looking up calling conventions for a platform"""
+
         mode = ctypes.DEFAULT_MODE
 
         def __init__(self, path):
@@ -57,6 +70,7 @@ class LibraryLoader(object):
             self.access = dict(cdecl=ctypes.CDLL(path, self.mode))
 
         def get(self, name, calling_convention="cdecl"):
+            """Return the given name according to the selected calling convention"""
             if calling_convention not in self.access:
                 raise LookupError(
                     "Unknown calling convention '{}' for function '{}'".format(
@@ -66,6 +80,7 @@ class LibraryLoader(object):
             return getattr(self.access[calling_convention], name)
 
         def has(self, name, calling_convention="cdecl"):
+            """Return True if this given calling convention finds the given 'name'"""
             if calling_convention not in self.access:
                 return False
             return hasattr(self.access[calling_convention], name)
@@ -81,9 +96,10 @@ class LibraryLoader(object):
         paths = self.getpaths(libname)
 
         for path in paths:
+            # noinspection PyBroadException
             try:
                 return self.Lookup(path)
-            except:
+            except Exception:  # pylint: disable=broad-except
                 pass
 
         raise ImportError("Could not load %s." % libname)
@@ -101,9 +117,16 @@ class LibraryLoader(object):
                     # dir_i should be absolute already
                     yield os.path.join(dir_i, fmt % libname)
 
+            # check if this code is even stored in a physical file
+            try:
+                this_file = __file__
+            except NameError:
+                this_file = None
+
             # then we search the directory where the generated python interface is stored
-            for fmt in self.name_formats:
-                yield os.path.abspath(os.path.join(os.path.dirname(__file__), fmt % libname))
+            if this_file is not None:
+                for fmt in self.name_formats:
+                    yield os.path.abspath(os.path.join(os.path.dirname(__file__), fmt % libname))
 
             # now, use the ctypes tools to try to find the library
             for fmt in self.name_formats:
@@ -119,7 +142,8 @@ class LibraryLoader(object):
             for fmt in self.name_formats:
                 yield os.path.abspath(os.path.join(os.path.curdir, fmt % libname))
 
-    def getplatformpaths(self, libname):
+    def getplatformpaths(self, _libname):  # pylint: disable=no-self-use
+        """Return all the library paths available in this platform"""
         return []
 
 
@@ -127,6 +151,8 @@ class LibraryLoader(object):
 
 
 class DarwinLibraryLoader(LibraryLoader):
+    """Library loader for MacOS"""
+
     name_formats = [
         "lib%s.dylib",
         "lib%s.so",
@@ -138,6 +164,10 @@ class DarwinLibraryLoader(LibraryLoader):
     ]
 
     class Lookup(LibraryLoader.Lookup):
+        """
+        Looking up library files for this platform (Darwin aka MacOS)
+        """
+
         # Darwin requires dlopen to be called with mode RTLD_GLOBAL instead
         # of the default RTLD_LOCAL.  Without this, you end up with
         # libraries not being loadable, resulting in "Symbol not found"
@@ -148,13 +178,14 @@ class DarwinLibraryLoader(LibraryLoader):
         if os.path.pathsep in libname:
             names = [libname]
         else:
-            names = [format % libname for format in self.name_formats]
+            names = [fmt % libname for fmt in self.name_formats]
 
-        for dir in self.getdirs(libname):
+        for directory in self.getdirs(libname):
             for name in names:
-                yield os.path.join(dir, name)
+                yield os.path.join(directory, name)
 
-    def getdirs(self, libname):
+    @staticmethod
+    def getdirs(libname):
         """Implements the dylib search as specified in Apple documentation:
 
         http://developer.apple.com/documentation/DeveloperTools/Conceptual/
@@ -167,8 +198,11 @@ class DarwinLibraryLoader(LibraryLoader):
 
         dyld_fallback_library_path = _environ_path("DYLD_FALLBACK_LIBRARY_PATH")
         if not dyld_fallback_library_path:
-            dyld_fallback_library_path = [os.path.expanduser("~/lib"), "/usr/local/lib", "/usr/lib"]
-        dyld_fallback_library_path.extend(_environ_path('LD_RUN_PATH'))
+            dyld_fallback_library_path = [
+                os.path.expanduser("~/lib"),
+                "/usr/local/lib",
+                "/usr/lib",
+            ]
 
         dirs = []
 
@@ -177,8 +211,9 @@ class DarwinLibraryLoader(LibraryLoader):
         else:
             dirs.extend(_environ_path("LD_LIBRARY_PATH"))
             dirs.extend(_environ_path("DYLD_LIBRARY_PATH"))
+            dirs.extend(_environ_path("LD_RUN_PATH"))
 
-        if hasattr(sys, "frozen") and sys.frozen == "macosx_app":
+        if hasattr(sys, "frozen") and getattr(sys, "frozen") == "macosx_app":
             dirs.append(os.path.join(os.environ["RESOURCEPATH"], "..", "Frameworks"))
 
         dirs.extend(dyld_fallback_library_path)
@@ -190,50 +225,60 @@ class DarwinLibraryLoader(LibraryLoader):
 
 
 class PosixLibraryLoader(LibraryLoader):
+    """Library loader for POSIX-like systems (including Linux)"""
+
     _ld_so_cache = None
 
     _include = re.compile(r"^\s*include\s+(?P<pattern>.*)")
 
+    name_formats = ["lib%s.so", "%s.so", "%s"]
+
     class _Directories(dict):
+        """Deal with directories"""
+
         def __init__(self):
+            dict.__init__(self)
             self.order = 0
 
         def add(self, directory):
+            """Add a directory to our current set of directories"""
             if len(directory) > 1:
                 directory = directory.rstrip(os.path.sep)
             # only adds and updates order if exists and not already in set
             if not os.path.exists(directory):
                 return
-            o = self.setdefault(directory, self.order)
-            if o == self.order:
+            order = self.setdefault(directory, self.order)
+            if order == self.order:
                 self.order += 1
 
         def extend(self, directories):
-            for d in directories:
-                self.add(d)
+            """Add a list of directories to our set"""
+            for a_dir in directories:
+                self.add(a_dir)
 
         def ordered(self):
-            return (i[0] for i in sorted(self.items(), key=lambda D: D[1]))
+            """Sort the list of directories"""
+            return (i[0] for i in sorted(self.items(), key=lambda d: d[1]))
 
     def _get_ld_so_conf_dirs(self, conf, dirs):
         """
-        Recursive funtion to help parse all ld.so.conf files, including proper
+        Recursive function to help parse all ld.so.conf files, including proper
         handling of the `include` directive.
         """
 
         try:
-            with open(conf) as f:
-                for D in f:
-                    D = D.strip()
-                    if not D:
+            with open(conf) as fileobj:
+                for dirname in fileobj:
+                    dirname = dirname.strip()
+                    if not dirname:
                         continue
 
-                    m = self._include.match(D)
-                    if not m:
-                        dirs.add(D)
+                    match = self._include.match(dirname)
+                    if not match:
+                        dirs.add(dirname)
                     else:
-                        for D2 in glob.glob(m.group("pattern")):
-                            self._get_ld_so_conf_dirs(D2, dirs)
+                        for dir2 in glob.glob(match.group("pattern")):
+                            self._get_ld_so_conf_dirs(dir2, dirs)
         except IOError:
             pass
 
@@ -248,7 +293,7 @@ class PosixLibraryLoader(LibraryLoader):
         directories = self._Directories()
         for name in (
             "LD_LIBRARY_PATH",
-            "SHLIB_PATH",  # HPUX
+            "SHLIB_PATH",  # HP-UX
             "LIBPATH",  # OS/2, AIX
             "LIBRARY_PATH",  # BE/OS
         ):
@@ -274,8 +319,11 @@ class PosixLibraryLoader(LibraryLoader):
                 # Assume Intel/AMD x86 compat
                 unix_lib_dirs_list += ["/lib/i386-linux-gnu", "/usr/lib/i386-linux-gnu"]
             elif bitage.startswith("64"):
-                # Assume Intel/AMD x86 compat
-                unix_lib_dirs_list += ["/lib/x86_64-linux-gnu", "/usr/lib/x86_64-linux-gnu"]
+                # Assume Intel/AMD x86 compatible
+                unix_lib_dirs_list += [
+                    "/lib/x86_64-linux-gnu",
+                    "/usr/lib/x86_64-linux-gnu",
+                ]
             else:
                 # guess...
                 unix_lib_dirs_list += glob.glob("/lib/*linux-gnu")
@@ -283,10 +331,10 @@ class PosixLibraryLoader(LibraryLoader):
 
         cache = {}
         lib_re = re.compile(r"lib(.*)\.s[ol]")
-        ext_re = re.compile(r"\.s[ol]$")
-        for dir in directories.ordered():
+        # ext_re = re.compile(r"\.s[ol]$")
+        for our_dir in directories.ordered():
             try:
-                for path in glob.glob("%s/*.s[ol]*" % dir):
+                for path in glob.glob("%s/*.s[ol]*" % our_dir):
                     file = os.path.basename(path)
 
                     # Index by filename
@@ -320,6 +368,8 @@ class PosixLibraryLoader(LibraryLoader):
 
 
 class WindowsLibraryLoader(LibraryLoader):
+    """Library loader for Microsoft Windows"""
+
     name_formats = ["%s.dll", "lib%s.dll", "%slib.dll", "%s"]
 
     def __init__(self):
@@ -329,6 +379,8 @@ class WindowsLibraryLoader(LibraryLoader):
                 os.add_dll_directory(p)
 
     class Lookup(LibraryLoader.Lookup):
+        """Lookup class for Windows libraries..."""
+
         def __init__(self, path):
             super(WindowsLibraryLoader.Lookup, self).__init__(path)
             self.access["stdcall"] = ctypes.windll.LoadLibrary(path)
@@ -355,10 +407,10 @@ def add_library_search_dirs(other_dirs):
     If library paths are relative, convert them to absolute with respect to this
     file's directory
     """
-    for F in other_dirs:
-        if not os.path.isabs(F):
-            F = os.path.abspath(F)
-        load_library.other_dirs.append(F)
+    for path in other_dirs:
+        if not os.path.isabs(path):
+            path = os.path.abspath(path)
+        load_library.other_dirs.append(path)
 
 
 del loaderclass

python/libgrass_interface_generator/ctypesgen/main.py

@@ -1,16 +1,19 @@
-# -*- coding: us-ascii -*-
-# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
 """
 Main loop for ctypesgen.
 """
 
-import optparse, sys
+import optparse
+import sys
 
-from . import options as core_options
-from . import parser as core_parser
-from . import printer_python, printer_json, processor
-from . import messages as msgs
-from . import version
+from ctypesgen import (
+    messages as msgs,
+    options as core_options,
+    parser as core_parser,
+    printer_python,
+    printer_json,
+    processor,
+    version,
+)
 
 
 def find_names_in_modules(modules):
@@ -18,7 +21,7 @@ def find_names_in_modules(modules):
     for module in modules:
         try:
             mod = __import__(module)
-        except:
+        except Exception:
             pass
         else:
             names.update(dir(mod))
@@ -131,6 +134,16 @@ def main(givenargs=None):
         default=[],
         help="Add LIBDIR to the run-time library search path.",
     )
+    op.add_option(
+        "",
+        "--no-embed-preamble",
+        action="store_false",
+        dest="embed_preamble",
+        default=True,
+        help="Do not embed preamble and loader in output file. "
+        "Defining --output as a file and --output-language to "
+        "Python is a prerequisite.",
+    )
 
     # Parser options
     op.add_option(
@@ -142,6 +155,17 @@ def main(givenargs=None):
         "necessary options (default: gcc -E)",
     )
     op.add_option(
+        "",
+        "--allow-gnu-c",
+        action="store_true",
+        dest="allow_gnu_c",
+        default=False,
+        help="Specify whether to undefine the '__GNUC__' macro, "
+        "while invoking the C preprocessor.\n"
+        "(default: False. i.e. ctypesgen adds an implicit undefine using '-U __GNUC__'.)\n"
+        "Specify this flag to avoid ctypesgen undefining '__GNUC__' as shown above.",
+    )
+    op.add_option(
         "-D",
         "--define",
         action="append",

python/libgrass_interface_generator/ctypesgen/messages.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 ctypesgen.messages contains functions to display status, error, or warning
 messages to the user. Warning and error messages are also associated
@@ -20,7 +18,6 @@ Warning classes are:
 'other' - catchall.
 """
 
-import sys
 import logging
 
 __all__ = ["error_message", "warning_message", "status_message"]

python/libgrass_interface_generator/ctypesgen/options.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 All of the components of ctypegencore require an argument called "options".
 In command-line usage, this would be an optparser.Values object. However, if
@@ -8,7 +6,8 @@ would be a pain. So this module exists to provide a "default" options object
 for convenience.
 """
 
-import optparse, copy
+import optparse
+import copy
 
 default_values = {
     "other_headers": [],
@@ -17,6 +16,7 @@ default_values = {
     "compile_libdirs": [],
     "runtime_libdirs": [],
     "cpp": "gcc -E",
+    "allow_gnu_c": False,
     "cpp_defines": [],
     "cpp_undefines": [],
     "save_preprocessed_headers": None,
@@ -40,6 +40,7 @@ default_values = {
     "no_python_types": False,
     "debug_level": 0,
     "strip_prefixes": [],
+    "embed_preamble": True,
 }
 
 

python/libgrass_interface_generator/ctypesgen/parser/__init__.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 This package parses C header files and generates lists of functions, typedefs,
 variables, structs, unions, enums, macros, and constants. This package knows

python/libgrass_interface_generator/ctypesgen/parser/cdeclarations.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 This file contains classes that represent C declarations. cparser produces
 declarations in this format, and ctypesparser reformats them into a format that
@@ -24,8 +22,8 @@ class Declaration(object):
         d = {"declarator": self.declarator, "type": self.type}
         if self.storage:
             d["storage"] = self.storage
-        l = ["%s=%r" % (k, v) for k, v in d.items()]
-        return "Declaration(%s)" % ", ".join(l)
+        li = ["%s=%r" % (k, v) for k, v in d.items()]
+        return "Declaration(%s)" % ", ".join(li)
 
 
 class Declarator(object):
@@ -45,7 +43,7 @@ class Declarator(object):
     def __repr__(self):
         s = self.identifier or ""
         if self.bitfield:
-            s += ":%d" % self.bitfield
+            s += f":{self.bitfield.value}"
         if self.array:
             s += repr(self.array)
         if self.initializer:
@@ -98,8 +96,8 @@ class Parameter(object):
             d["declarator"] = self.declarator
         if self.storage:
             d["storage"] = self.storage
-        l = ["%s=%r" % (k, v) for k, v in d.items()]
-        return "Parameter(%s)" % ", ".join(l)
+        li = ["%s=%r" % (k, v) for k, v in d.items()]
+        return "Parameter(%s)" % ", ".join(li)
 
 
 class Type(object):

python/libgrass_interface_generator/ctypesgen/parser/cparser.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 Parse a C source file.
 
@@ -9,17 +7,10 @@ the class with a string to parse.
 
 __docformat__ = "restructuredtext"
 
-import operator
 import os.path
-import re
 import sys
-import time
-import warnings
 
-from . import preprocessor
-from . import yacc
-from . import cgrammar
-from . import cdeclarations
+from ctypesgen.parser import cgrammar, preprocessor, yacc
 
 # --------------------------------------------------------------------------
 # Lexer
@@ -31,6 +22,8 @@ class CLexer(object):
         self.cparser = cparser
         self.type_names = set()
         self.in_define = False
+        self.lineno = -1
+        self.lexpos = -1
 
     def input(self, tokens):
         self.tokens = tokens
@@ -51,12 +44,8 @@ class CLexer(object):
                 self.in_define = False
 
             # Transform PP tokens into C tokens
-            elif t.type == "LPAREN":
-                t.type = "("
-            elif t.type == "PP_NUMBER":
-                t.type = "CONSTANT"
             elif t.type == "IDENTIFIER" and t.value in cgrammar.keywords:
-                t.type = t.value.upper()
+                t.type = cgrammar.keyword_map[t.value]
             elif t.type == "IDENTIFIER" and t.value in self.type_names:
                 if self.pos < 2 or self.tokens[self.pos - 2].type not in (
                     "VOID",
@@ -98,8 +87,7 @@ class CParser(object):
     def __init__(self, options):
         super(CParser, self).__init__()
         self.preprocessor_parser = preprocessor.PreprocessorParser(options, self)
-        self.parser = yacc.Parser()
-        prototype = yacc.yacc(
+        self.parser = yacc.yacc(
             method="LALR",
             debug=False,
             module=cgrammar,
@@ -108,10 +96,7 @@ class CParser(object):
             optimize=True,
         )
 
-        # If yacc is reading tables from a file, then it won't find the error
-        # function... need to set it manually
-        prototype.errorfunc = cgrammar.p_error
-        prototype.init_parser(self.parser)
+        self.parser.errorfunc = cgrammar.p_error
         self.parser.cparser = self
 
         self.lexer = CLexer(self)
@@ -134,7 +119,7 @@ class CParser(object):
         self.preprocessor_parser.parse(filename)
         self.lexer.input(self.preprocessor_parser.output)
         self.handle_status("Parsing %s" % filename)
-        self.parser.parse(lexer=self.lexer, debug=debug)
+        self.parser.parse(lexer=self.lexer, debug=debug, tracking=True)
 
     # ----------------------------------------------------------------------
     # Parser interface.  Override these methods in your subclass.

python/libgrass_interface_generator/ctypesgen/parser/ctypesparser.py

@@ -10,11 +10,32 @@ __docformat__ = "restructuredtext"
 
 __all__ = ["CtypesParser"]
 
-from ..ctypedescs import *
-from ..expressions import *
-
-from .cparser import *
-from .cdeclarations import *
+from ctypesgen.ctypedescs import (
+    CtypesArray,
+    CtypesBitfield,
+    CtypesEnum,
+    CtypesFunction,
+    CtypesPointer,
+    CtypesSimple,
+    CtypesSpecial,
+    CtypesStruct,
+    CtypesTypedef,
+    ctypes_type_map,
+    ctypes_type_map_python_builtin,
+    remove_function_pointer,
+)
+from ctypesgen.expressions import (
+    BinaryExpressionNode,
+    ConstantExpressionNode,
+    IdentifierExpressionNode,
+)
+from ctypesgen.parser.cdeclarations import (
+    Attrib,
+    EnumSpecifier,
+    Pointer,
+    StructTypeSpecifier,
+)
+from ctypesgen.parser.cparser import CParser
 
 
 def make_enum_from_specifier(specifier):

python/libgrass_interface_generator/ctypesgen/parser/datacollectingparser.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 DataCollectingParser subclasses ctypesparser.CtypesParser and builds Description
 objects from the CtypesType objects and other information from CtypesParser.
@@ -7,16 +5,27 @@ After parsing is complete, a DescriptionCollection object can be retrieved by
 calling DataCollectingParser.data().
 """
 
-from . import ctypesparser
-from ..descriptions import *
-from ..ctypedescs import *
-from ..expressions import *
-from ..messages import *
-from tempfile import mkstemp
 import os
+from tempfile import mkstemp
 
-
-class DataCollectingParser(ctypesparser.CtypesParser, ctypesparser.CtypesTypeVisitor):
+from ctypesgen.ctypedescs import CtypesEnum, CtypesType, CtypesTypeVisitor
+from ctypesgen.descriptions import (
+    ConstantDescription,
+    DescriptionCollection,
+    EnumDescription,
+    FunctionDescription,
+    MacroDescription,
+    StructDescription,
+    TypedefDescription,
+    UndefDescription,
+    VariableDescription,
+)
+from ctypesgen.expressions import ConstantExpressionNode
+from ctypesgen.messages import error_message, status_message
+from ctypesgen.parser import ctypesparser
+
+
+class DataCollectingParser(ctypesparser.CtypesParser, CtypesTypeVisitor):
     """Main class for the Parser component. Steps for use:
     p=DataCollectingParser(names_of_header_files,options)
     p.parse()
@@ -113,7 +122,7 @@ class DataCollectingParser(ctypesparser.CtypesParser, ctypesparser.CtypesTypeVis
 
     def handle_ctypes_new_type(self, ctype, filename, lineno):
         # Called by CtypesParser
-        if isinstance(ctype, ctypesparser.CtypesEnum):
+        if isinstance(ctype, CtypesEnum):
             self.handle_enum(ctype, filename, lineno)
         else:
             self.handle_struct(ctype, filename, lineno)
@@ -265,7 +274,7 @@ class DataCollectingParser(ctypesparser.CtypesParser, ctypesparser.CtypesTypeVis
         # Called from within DataCollectingParser
         src = (filename, lineno)
 
-        if expr == None:
+        if expr is None:
             expr = ConstantExpressionNode(True)
             constant = ConstantDescription(name, expr, src)
             self.constants.append(constant)

python/libgrass_interface_generator/ctypesgen/parser/pplexer.py

@@ -1,85 +1,65 @@
-#!/usr/bin/env python
-
 """Preprocess a C source file using gcc and convert the result into
    a token stream
 
-Reference is C99:
-  * http://www.open-std.org/JTC1/SC22/WG14/www/docs/n1124.pdf
-
+Reference is C99 with additions from C11 and C2x:
+* http://www.open-std.org/JTC1/SC22/WG14/www/docs/n1124.pdf
+* http://www.quut.com/c/ANSI-C-grammar-l-2011.html
+* http://www.open-std.org/jtc1/sc22/wg14/www/docs/n2731.pdf
 """
 
 __docformat__ = "restructuredtext"
 
-import os, re, shlex, sys, tokenize, traceback
-import ctypes
-from .lex import TOKEN
-
-tokens = (
-    "HEADER_NAME",
-    "IDENTIFIER",
-    "PP_NUMBER",
-    "CHARACTER_CONSTANT",
-    "STRING_LITERAL",
-    "OTHER",
-    "PTR_OP",
-    "INC_OP",
-    "DEC_OP",
-    "LEFT_OP",
-    "RIGHT_OP",
-    "LE_OP",
-    "GE_OP",
-    "EQ_OP",
-    "NE_OP",
-    "AND_OP",
-    "OR_OP",
-    "MUL_ASSIGN",
-    "DIV_ASSIGN",
-    "MOD_ASSIGN",
-    "ADD_ASSIGN",
-    "SUB_ASSIGN",
-    "LEFT_ASSIGN",
-    "RIGHT_ASSIGN",
-    "AND_ASSIGN",
-    "XOR_ASSIGN",
-    "OR_ASSIGN",
-    "PERIOD",
-    "ELLIPSIS",
-    "LPAREN",
-    "NEWLINE",
-    "PP_DEFINE",
-    "PP_DEFINE_NAME",
-    "PP_DEFINE_MACRO_NAME",
-    "PP_UNDEFINE",
-    "PP_MACRO_PARAM",
-    "PP_STRINGIFY",
-    "PP_IDENTIFIER_PASTE",
-    "PP_END_DEFINE",
-    "PRAGMA",
-    "PRAGMA_PACK",
-    "PRAGMA_END",
-)
+from ctypesgen.parser.lex import TOKEN
+from ctypesgen.parser import cgrammar
 
-states = [("DEFINE", "exclusive"), ("PRAGMA", "exclusive")]
 
-subs = {
-    "D": "[0-9]",
-    "L": "[a-zA-Z_]",
-    "H": "[a-fA-F0-9]",
-    "E": "[Ee][+-]?\s*{D}+",
-    # new float suffixes supported in gcc 7
-    "FS": "([FflL]|D[FDL]|[fF]\d+x?)",
-    "IS": "[uUlL]*",
-}
-# Helper: substitute {foo} with subs[foo] in string (makes regexes more lexy)
-sub_pattern = re.compile("{([^}]*)}")
+tokens = cgrammar.tokens
+keywords = cgrammar.keywords
 
 
-def sub_repl_match(m):
-    return subs[m.groups()[0]]
+states = [("DEFINE", "exclusive"), ("PRAGMA", "exclusive")]
 
 
-def sub(s):
-    return sub_pattern.sub(sub_repl_match, s)
+_B_ = r"[0-1]"
+_O_ = r"[0-7]"
+_D_ = r"[0-9]"
+_NZ_ = r"[1-9]"
+_L_ = r"[a-zA-Z_]"
+_A_ = r"[a-zA-Z_0-9]"
+_H_ = r"[a-fA-F0-9]"
+_HP_ = r"0[xX]"
+_BP_ = r"0[bB]"
+_E_ = r"([Ee][+-]?" + _D_ + r"+)"
+_P_ = r"([Pp][+-]?" + _D_ + r"+)"
+_FS_ = r"(f|F|l|L)"
+_IS_ = r"(((u|U)(ll|LL|l|L)?)|((ll|LL|l|L)(u|U)?))"
+_CP_ = r"(u|U|L)"
+_SP_ = r"(u8|u|U|L)"
+_ES_ = r"(\\([\'\"\?\\abfnrtv]|[0-7]{1,3}|x[a-fA-F0-9]+))"
+_WS_ = r"[ \t\v\n\f]"
+
+I_CONST_HEX = r"(?P<p1>" + _HP_ + _H_ + r"+" + r")" + _IS_ + r"?"
+I_CONST_DEC = r"(?P<p1>" + _NZ_ + _D_ + r"*" + r")" + _IS_ + r"?"
+I_CONST_OCT = r"0" + r"(?P<p1>" + _O_ + r"*)" + _IS_ + r"?"
+I_CONST_BIN = r"(?P<p1>" + _BP_ + _B_ + "*" + r")" + _IS_ + "?"
+
+F_CONST_1 = r"(?P<sig>" + _D_ + r"+" + r")(?P<exp>" + _E_ + r")" + _FS_ + r"?"
+F_CONST_2 = r"(?P<sig>" + _D_ + r"*\." + _D_ + r"+" + r")(?P<exp>" + _E_ + r"?)" + _FS_ + r"?"
+F_CONST_3 = r"(?P<sig>" + _D_ + r"+\." + r")(?P<exp>" + _E_ + r"?)" + _FS_ + r"?"
+F_CONST_4 = r"(?P<hex>" + _HP_ + _H_ + r"+" + _P_ + r")" + _FS_ + r"?"
+F_CONST_5 = r"(?P<hex>" + _HP_ + _H_ + r"*\." + _H_ + r"+" + _P_ + r")" + _FS_ + r"?"
+F_CONST_6 = r"(?P<hex>" + _HP_ + _H_ + r"+\." + _P_ + r")" + _FS_ + r"?"
+
+CHARACTER_CONSTANT = _SP_ + r"?'(?P<p1>\\.|[^\\'])+'"
+IDENTIFIER = _L_ + _A_ + r"*"
+
+escape_sequence_start_in_string = r"""(\\[0-9a-zA-Z._~!=&\^\-\\?'"])"""
+string_char = r"""([^"\\\n]|""" + escape_sequence_start_in_string + ")"
+STRING_LITERAL = '"' + string_char + '*"'
+
+# Process line-number directives from the preprocessor
+# See https://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
+DIRECTIVE = r'\#\s+(?P<lineno>\d+)\s+"(?P<filename>[^"]+)"[ \d]*\n'
 
 
 # --------------------------------------------------------------------------
@@ -102,195 +82,168 @@ class StringLiteral(str):
 # Token declarations
 # --------------------------------------------------------------------------
 
-punctuators = {
-    # value: (regex, type)
-    r"...": (r"\.\.\.", "ELLIPSIS"),
-    r">>=": (r">>=", "RIGHT_ASSIGN"),
-    r"<<=": (r"<<=", "LEFT_ASSIGN"),
-    r"+=": (r"\+=", "ADD_ASSIGN"),
-    r"-=": (r"-=", "SUB_ASSIGN"),
-    r"*=": (r"\*=", "MUL_ASSIGN"),
-    r"/=": (r"/=", "DIV_ASSIGN"),
-    r"%=": (r"%=", "MOD_ASSIGN"),
-    r"&=": (r"&=", "AND_ASSIGN"),
-    r"^=": (r"\^=", "XOR_ASSIGN"),
-    r"|=": (r"\|=", "OR_ASSIGN"),
-    r">>": (r">>", "RIGHT_OP"),
-    r"<<": (r"<<", "LEFT_OP"),
-    r"++": (r"\+\+", "INC_OP"),
-    r"--": (r"--", "DEC_OP"),
-    r"->": (r"->", "PTR_OP"),
-    r"&&": (r"&&", "AND_OP"),
-    r"||": (r"\|\|", "OR_OP"),
-    r"<=": (r"<=", "LE_OP"),
-    r">=": (r">=", "GE_OP"),
-    r"==": (r"==", "EQ_OP"),
-    r"!=": (r"!=", "NE_OP"),
-    r"<:": (r"<:", "["),
-    r":>": (r":>", "]"),
-    r"<%": (r"<%", "{"),
-    r"%>": (r"%>", "}"),
-    r";": (r";", ";"),
-    r"{": (r"{", "{"),
-    r"}": (r"}", "}"),
-    r",": (r",", ","),
-    r":": (r":", ":"),
-    r"=": (r"=", "="),
-    r")": (r"\)", ")"),
-    r"[": (r"\[", "["),
-    r"]": (r"]", "]"),
-    r".": (r"\.", "PERIOD"),
-    r"&": (r"&", "&"),
-    r"!": (r"!", "!"),
-    r"~": (r"~", "~"),
-    r"-": (r"-", "-"),
-    r"+": (r"\+", "+"),
-    r"*": (r"\*", "*"),
-    r"/": (r"/", "/"),
-    r"%": (r"%", "%"),
-    r"<": (r"<", "<"),
-    r">": (r">", ">"),
-    r"^": (r"\^", "^"),
-    r"|": (r"\|", "|"),
-    r"?": (r"\?", "?"),
-}
-
-
-def punctuator_regex(punctuators):
-    punctuator_regexes = [v[0] for v in punctuators.values()]
-    punctuator_regexes.sort(key=len, reverse=True)
-    return "(%s)" % "|".join(punctuator_regexes)
 
-
-# Process line-number directives from the preprocessor
-# See http://docs.freebsd.org/info/cpp/cpp.info.Output.html
-DIRECTIVE = r'\#\s+(\d+)\s+"([^"]+)"[ \d]*\n'
+# Assignment operators
+t_ANY_EQUALS = r"="
+t_ANY_RIGHT_ASSIGN = r">>="
+t_ANY_LEFT_ASSIGN = r"<<="
+t_ANY_ADD_ASSIGN = r"\+="
+t_ANY_SUB_ASSIGN = r"-="
+t_ANY_MUL_ASSIGN = r"\*="
+t_ANY_DIV_ASSIGN = r"/="
+t_ANY_MOD_ASSIGN = r"%="
+t_ANY_AND_ASSIGN = r"&="
+t_ANY_XOR_ASSIGN = r"\^="
+t_ANY_OR_ASSIGN = r"\|="
+
+# Operators
+t_ANY_PLUS = r"\+"
+t_ANY_MINUS = r"-"
+t_ANY_TIMES = r"\*"
+t_ANY_DIVIDE = r"/"
+t_ANY_MOD = r"%"
+t_ANY_AND = r"&"
+t_ANY_OR = r"\|"
+t_ANY_NOT = r"~"
+t_ANY_XOR = r"\^"
+t_ANY_RIGHT_OP = r">>"
+t_ANY_LEFT_OP = r"<<"
+t_ANY_INC_OP = r"\+\+"
+t_ANY_DEC_OP = r"--"
+t_ANY_PTR_OP = r"->"
+t_ANY_AND_OP = r"&&"
+t_ANY_OR_OP = r"\|\|"
+t_ANY_LE_OP = r"<="
+t_ANY_GE_OP = r">="
+t_ANY_EQ_OP = r"=="
+t_ANY_NE_OP = r"!="
+t_ANY_LNOT = r"!"
+t_ANY_LT = r"<"
+t_ANY_GT = r">"
+t_ANY_CONDOP = r"\?"
+
+
+# Delimeters
+t_ANY_PERIOD = r"\."
+t_ANY_LPAREN = r"\("
+t_ANY_RPAREN = r"\)"
+t_ANY_ELLIPSIS = r"\.\.\."
+t_ANY_LBRACKET = r"\["
+t_ANY_RBRACKET = r"\]"
+t_ANY_LBRACE = r"\{"
+t_ANY_RBRACE = r"\}"
+t_ANY_COMMA = r","
+t_ANY_SEMI = r";"
+t_ANY_COLON = r":"
 
 
 @TOKEN(DIRECTIVE)
 def t_ANY_directive(t):
-    t.lexer.filename = t.groups[2]
-    t.lexer.lineno = int(t.groups[1])
+    m = t.lexer.lexmatch
+    t.lexer.filename = m.group("filename")
+    t.lexer.lineno = int(m.group("lineno"))
     return None
 
 
-@TOKEN(punctuator_regex(punctuators))
-def t_ANY_punctuator(t):
-    t.type = punctuators[t.value][1]
+@TOKEN(F_CONST_1)
+def t_ANY_f_const_1(t):
+    t.type = "F_CONST_1"
+    m = t.lexer.lexmatch
+    sig = m.group("sig")
+    exp = m.group("exp")
+    t.value = sig + exp
     return t
 
 
-IDENTIFIER = sub("{L}({L}|{D})*")
-
-
-@TOKEN(IDENTIFIER)
-def t_INITIAL_identifier(t):
-    t.type = "IDENTIFIER"
+@TOKEN(F_CONST_2)
+def t_ANY_f_const_2(t):
+    t.type = "F_CONST_2"
+    m = t.lexer.lexmatch
+    sig = m.group("sig")
+    exp = m.group("exp")
+    t.value = sig + exp
     return t
 
 
-@TOKEN(IDENTIFIER)
-def t_DEFINE_identifier(t):
-    if t.lexer.next_is_define_name:
-        # This identifier is the name of a macro
-        # We need to look ahead and see if this macro takes parameters or not.
-        if (
-            t.lexpos + len(t.value) < t.lexer.lexlen
-            and t.lexer.lexdata[t.lexpos + len(t.value)] == "("
-        ):
+@TOKEN(F_CONST_3)
+def t_ANY_f_const_3(t):
+    t.type = "F_CONST_3"
+    m = t.lexer.lexmatch
+    sig = m.group("sig")
+    exp = m.group("exp")
+    t.value = sig + exp
+    return t
 
-            t.type = "PP_DEFINE_MACRO_NAME"
 
-            # Look ahead and read macro parameter list
-            lexdata = t.lexer.lexdata
-            pos = t.lexpos + len(t.value) + 1
-            while lexdata[pos] not in "\n)":
-                pos += 1
-            params = lexdata[t.lexpos + len(t.value) + 1 : pos]
-            paramlist = [x.strip() for x in params.split(",") if x.strip()]
-            t.lexer.macro_params = paramlist
+@TOKEN(F_CONST_4)
+def t_ANY_f_const_4(t):
+    t.type = "F_CONST_4"
+    m = t.lexer.lexmatch
+    t.value = 'float.fromhex("' + m.group("hex") + '")'
+    return t
 
-        else:
-            t.type = "PP_DEFINE_NAME"
 
-        t.lexer.next_is_define_name = False
-    elif t.value in t.lexer.macro_params:
-        t.type = "PP_MACRO_PARAM"
-    else:
-        t.type = "IDENTIFIER"
+@TOKEN(F_CONST_5)
+def t_ANY_f_const_5(t):
+    t.type = "F_CONST_5"
+    m = t.lexer.lexmatch
+    t.value = 'float.fromhex("' + m.group("hex") + '")'
     return t
 
 
-FLOAT_LITERAL = sub(
-    r"(?P<p1>{D}+)?(?P<dp>[.]?)(?P<p2>(?(p1){D}*|{D}+))"
-    r"(?P<exp>(?:[Ee][+-]?{D}+)?)(?P<suf>{FS}?)(?!\w)"
-)
+@TOKEN(F_CONST_6)
+def t_ANY_f_const_6(t):
+    t.type = "F_CONST_6"
+    m = t.lexer.lexmatch
+    t.value = 'float.fromhex("' + m.group("hex") + '")'
+    return t
 
 
-@TOKEN(FLOAT_LITERAL)
-def t_ANY_float(t):
-    t.type = "PP_NUMBER"
+@TOKEN(I_CONST_BIN)
+def t_ANY_i_const_bin(t):
+    t.type = "I_CONST_BIN"
     m = t.lexer.lexmatch
+    t.value = m.group("p1")
+    return t
 
-    p1 = m.group("p1")
-    dp = m.group("dp")
-    p2 = m.group("p2")
-    exp = m.group("exp")
-    suf = m.group("suf")
-
-    if dp or exp or (suf and re.match(subs["FS"] + "$", suf)):
-        s = m.group(0)
-        if suf:
-            s = s[: -len(suf)]
-        # Attach a prefix so the parser can figure out if should become an
-        # integer, float, or long
-        t.value = "f" + s
-    elif suf and suf in ("Ll"):
-        t.value = "l" + p1
-    else:
-        t.value = "i" + p1
 
+@TOKEN(I_CONST_HEX)
+def t_ANY_i_const_hex(t):
+    t.type = "I_CONST_HEX"
+    m = t.lexer.lexmatch
+    t.value = m.group("p1")
     return t
 
 
-INT_LITERAL = sub(r"(?P<p1>(?:0x{H}+)|(?:{D}+))(?P<suf>{IS})")
+@TOKEN(I_CONST_DEC)
+def t_ANY_i_const_dec(t):
+    t.type = "I_CONST_DEC"
+    m = t.lexer.lexmatch
+    t.value = m.group("p1")
+    return t
 
 
-@TOKEN(INT_LITERAL)
-def t_ANY_int(t):
-    t.type = "PP_NUMBER"
+@TOKEN(I_CONST_OCT)
+def t_ANY_i_const_oct(t):
+    t.type = "I_CONST_OCT"
     m = t.lexer.lexmatch
-
-    if "L" in m.group(3) or "l" in m.group(2):
-        prefix = "l"
+    p1 = m.group("p1")
+    if not p1:
+        t.value = "0"
     else:
-        prefix = "i"
-
-    g1 = m.group(2)
-    if g1.startswith("0x"):
-        # Convert base from hexadecimal
-        g1 = str(int(g1[2:], 16))
-    elif g1[0] == "0":
-        # Convert base from octal
-        g1 = str(int(g1, 8))
-
-    t.value = prefix + g1
-
+        t.value = "0o" + m.group("p1")
     return t
 
 
-CHARACTER_CONSTANT = sub(r"L?'(\\.|[^\\'])+'")
-
-
 @TOKEN(CHARACTER_CONSTANT)
 def t_ANY_character_constant(t):
     t.type = "CHARACTER_CONSTANT"
+    m = t.lexer.lexmatch
+    p1 = m.group("p1")
+    t.value = p1
     return t
 
 
-STRING_LITERAL = sub(r'L?"(\\.|[^\\"])*"')
-
-
 @TOKEN(STRING_LITERAL)
 def t_ANY_string_literal(t):
     t.type = "STRING_LITERAL"
@@ -298,12 +251,41 @@ def t_ANY_string_literal(t):
     return t
 
 
-@TOKEN(r"\(")
-def t_ANY_lparen(t):
-    if t.lexpos == 0 or t.lexer.lexdata[t.lexpos - 1] not in (" \t\f\v\n"):
-        t.type = "LPAREN"
+@TOKEN(IDENTIFIER)
+def t_INITIAL_identifier(t):
+    t.type = "IDENTIFIER"
+    return t
+
+
+@TOKEN(IDENTIFIER)
+def t_DEFINE_identifier(t):
+    if t.lexer.next_is_define_name:
+        # This identifier is the name of a macro
+        # We need to look ahead and see if this macro takes parameters or not.
+        if (
+            t.lexpos + len(t.value) < t.lexer.lexlen
+            and t.lexer.lexdata[t.lexpos + len(t.value)] == "("
+        ):
+
+            t.type = "PP_DEFINE_MACRO_NAME"
+
+            # Look ahead and read macro parameter list
+            lexdata = t.lexer.lexdata
+            pos = t.lexpos + len(t.value) + 1
+            while lexdata[pos] not in "\n)":
+                pos += 1
+            params = lexdata[t.lexpos + len(t.value) + 1 : pos]
+            paramlist = [x.strip() for x in params.split(",") if x.strip()]
+            t.lexer.macro_params = paramlist
+
+        else:
+            t.type = "PP_DEFINE_NAME"
+
+        t.lexer.next_is_define_name = False
+    elif t.value in t.lexer.macro_params:
+        t.type = "PP_MACRO_PARAM"
     else:
-        t.type = "("
+        t.type = "IDENTIFIER"
     return t
 
 

python/libgrass_interface_generator/ctypesgen/parser/preprocessor.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """Preprocess a C source file using gcc and convert the result into
    a token stream
 
@@ -10,11 +8,17 @@ Reference is C99:
 
 __docformat__ = "restructuredtext"
 
-import os, re, shlex, sys, tokenize, traceback, subprocess
-import ctypes
-from . import lex, yacc
-from .lex import TOKEN, LexError
-from . import pplexer
+import os
+import re
+import sys
+import subprocess
+
+from ctypesgen.parser import pplexer, lex
+from ctypesgen.parser.lex import LexError
+
+
+IS_WINDOWS = sys.platform.startswith("win")
+IS_MAC = sys.platform.startswith("darwin")
 
 # --------------------------------------------------------------------------
 # Lexers
@@ -31,28 +35,11 @@ class PreprocessorLexer(lex.Lexer):
         if filename:
             self.filename = filename
         self.lasttoken = None
-        self.input_stack = []
 
         lex.Lexer.input(self, data)
 
-    def push_input(self, data, filename):
-        self.input_stack.append((self.lexdata, self.lexpos, self.filename, self.lineno))
-        self.lexdata = data
-        self.lexpos = 0
-        self.lineno = 1
-        self.filename = filename
-        self.lexlen = len(self.lexdata)
-
-    def pop_input(self):
-        self.lexdata, self.lexpos, self.filename, self.lineno = self.input_stack.pop()
-        self.lexlen = len(self.lexdata)
-
     def token(self):
         result = lex.Lexer.token(self)
-        while result is None and self.input_stack:
-            self.pop_input()
-            result = lex.Lexer.token(self)
-
         if result:
             self.lasttoken = result.type
             result.filename = self.filename
@@ -62,45 +49,6 @@ class PreprocessorLexer(lex.Lexer):
         return result
 
 
-class TokenListLexer(object):
-    def __init__(self, tokens):
-        self.tokens = tokens
-        self.pos = 0
-
-    def token(self):
-        if self.pos < len(self.tokens):
-            t = self.tokens[self.pos]
-            self.pos += 1
-            return t
-        else:
-            return None
-
-
-def symbol_to_token(sym):
-    if isinstance(sym, yacc.YaccSymbol):
-        return sym.value
-    elif isinstance(sym, lex.LexToken):
-        return sym
-    else:
-        assert False, "Not a symbol: %r" % sym
-
-
-def create_token(type, value, production=None):
-    """Create a token of type and value, at the position where 'production'
-    was reduced.  Don't specify production if the token is built-in"""
-    t = lex.LexToken()
-    t.type = type
-    t.value = value
-    t.lexpos = -1
-    if production:
-        t.lineno = production.slice[1].lineno
-        t.filename = production.slice[1].filename
-    else:
-        t.lineno = -1
-        t.filename = "<builtin>"
-    return t
-
-
 # --------------------------------------------------------------------------
 # Grammars
 # --------------------------------------------------------------------------
@@ -118,10 +66,16 @@ class PreprocessorParser(object):
             "CTYPESGEN=1",
         ]
 
-        # On OSX, explicitly add these defines to keep from getting syntax
-        # errors in the OSX standard headers.
-        if sys.platform == "darwin":
-            self.defines += ["__uint16_t=uint16_t", "__uint32_t=uint32_t", "__uint64_t=uint64_t"]
+        # On macOS, explicitly add these defines to keep from getting syntax
+        # errors in the macOS standard headers.
+        if IS_MAC:
+            self.defines += [
+                "__uint16_t=uint16_t",
+                "__uint32_t=uint32_t",
+                "__uint64_t=uint64_t",
+                "_Nullable=",
+                "_Nonnull=",
+            ]
 
         self.matches = []
         self.output = []
@@ -141,14 +95,26 @@ class PreprocessorParser(object):
         """Parse a file and save its output"""
 
         cmd = self.options.cpp
-        cmd += " -U __GNUC__ -dD"
+
+        # Legacy behaviour is to implicitly undefine '__GNUC__'
+        # Continue doing this, unless user explicitly requested to allow it.
+        if self.options.allow_gnu_c:
+            # New behaviour. No implicit override.
+            # (currently NOT enabled by default yet)
+            pass
+        else:
+            # Legacy behaviour. Add an implicit override.
+            # (currently the default)
+            cmd += " -U __GNUC__"
+
+        cmd += " -dD"
 
         for undefine in self.options.cpp_undefines:
             cmd += " -U%s" % undefine
 
         # This fixes Issue #6 where OS X 10.6+ adds a C extension that breaks
         # the parser.  Blocks shouldn't be needed for ctypesgen support anyway.
-        if sys.platform == "darwin":
+        if IS_MAC:
             cmd += " -U __BLOCKS__"
 
         for path in self.options.include_search_paths:
@@ -159,36 +125,30 @@ class PreprocessorParser(object):
 
         self.cparser.handle_status(cmd)
 
-        if sys.platform == "win32":
+        if IS_WINDOWS:
             cmd = ["sh.exe", "-c", cmd]
 
         pp = subprocess.Popen(
             cmd,
             shell=True,
-            universal_newlines=True,
+            universal_newlines=False,  # binary
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
+        ppout_data, pperr_data = pp.communicate()
+
         try:
-            ppout, pperr = pp.communicate()
+            ppout = ppout_data.decode("utf-8")
         except UnicodeError:
-            # Fix for https://trac.osgeo.org/grass/ticket/3883,
-            # handling file(s) encoded with mac_roman
-            if sys.platform == "darwin":
-                pp = subprocess.Popen(
-                    cmd,
-                    shell=True,
-                    universal_newlines=False,  # read as binary
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
-                )
-                ppout, pperr = pp.communicate()
-
-                data = ppout.decode("utf8", errors="replace")
-                ppout = data.replace("\r\n", "\n").replace("\r", "\n")
-                pperr = pperr.decode("utf8", errors="replace")
+            if IS_MAC:
+                ppout = ppout_data.decode("utf-8", errors="replace")
             else:
                 raise UnicodeError
+        pperr = pperr_data.decode("utf-8")
+
+        if IS_WINDOWS:
+            ppout = ppout.replace("\r\n", "\n")
+            pperr = pperr.replace("\r\n", "\n")
 
         for line in pperr.split("\n"):
             if line:

python/libgrass_interface_generator/ctypesgen/printer_json/__init__.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 This module is the backend to ctypesgen; it contains classes to
 produce the final .py output files.

python/libgrass_interface_generator/ctypesgen/printer_json/printer.py

@@ -1,28 +1,21 @@
-#!/usr/bin/env python
+import os
+import sys
+import json
 
-import os, sys, time, json
-from ctypesgen.descriptions import *
-from ctypesgen.ctypedescs import *
-from ctypesgen.messages import *
+from ctypesgen.ctypedescs import CtypesBitfield
+from ctypesgen.messages import status_message
 
-import ctypesgen.libraryloader  # So we can get the path to it
-from . import test  # So we can find the path to local files in the printer package
 
-
-def path_to_local_file(name, known_local_module=test):
-    basedir = os.path.dirname(known_local_module.__file__)
-    return os.path.join(basedir, name)
-
-
-# From http://stackoverflow.com/questions/1036409/recursively-convert-python-object-graph-to-dictionary
+# From:
+# http://stackoverflow.com/questions/1036409/recursively-convert-python-object-graph-to-dictionary
 def todict(obj, classkey="Klass"):
     if isinstance(obj, dict):
         for k in obj.keys():
             obj[k] = todict(obj[k], classkey)
         return obj
     elif isinstance(obj, str) or isinstance(obj, bytes):
-        # must handle strings before __iter__ test, since they now have __iter__
-        # in Python3
+        # must handle strings before __iter__ test, since they now have
+        # __iter__ in Python3
         return obj
     elif hasattr(obj, "__iter__"):
         return [todict(v, classkey) for v in obj]

python/libgrass_interface_generator/ctypesgen/printer_json/test.py

@@ -1,6 +0,0 @@
-"""
-ctypesgen.printer.printer imports this module so that it can find the path
-to defaulttemplate.py and defaultloader.py.
-"""
-
-pass

python/libgrass_interface_generator/ctypesgen/printer_python/__init__.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 This module is the backend to ctypesgen; it contains classes to
 produce the final .py output files.

python/libgrass_interface_generator/ctypesgen/printer_python/preamble/3_2.py

@@ -1,37 +1,21 @@
-import ctypes, os, sys
-from ctypes import *
+import ctypes
+import sys
+from ctypes import *  # noqa: F401, F403
 
-_int_types = (c_int16, c_int32)
+_int_types = (ctypes.c_int16, ctypes.c_int32)
 if hasattr(ctypes, "c_int64"):
-    # Some builds of ctypes apparently do not have c_int64
+    # Some builds of ctypes apparently do not have ctypes.c_int64
     # defined; it's a pretty good bet that these builds do not
     # have 64-bit pointers.
-    _int_types += (c_int64,)
+    _int_types += (ctypes.c_int64,)
 for t in _int_types:
-    if sizeof(t) == sizeof(c_size_t):
+    if ctypes.sizeof(t) == ctypes.sizeof(ctypes.c_size_t):
         c_ptrdiff_t = t
 del t
 del _int_types
 
 
-def POINTER(obj):
-    p = ctypes.POINTER(obj)
-
-    # Convert None to a real NULL pointer to work around bugs
-    # in how ctypes handles None on 64-bit platforms
-    if not isinstance(p.from_param, classmethod):
-
-        def from_param(cls, x):
-            if x is None:
-                return cls()
-            else:
-                return x
-
-        p.from_param = classmethod(from_param)
-
-    return p
-
-
+# ~POINTER~
 class UserString:
     def __init__(self, seq):
         if isinstance(seq, bytes):
@@ -65,12 +49,6 @@ class UserString:
     def __hash__(self):
         return hash(self.data)
 
-    def __cmp__(self, string):
-        if isinstance(string, UserString):
-            return cmp(self.data, string.data)
-        else:
-            return cmp(self.data, string)
-
     def __le__(self, string):
         if isinstance(string, UserString):
             return self.data <= string.data
@@ -342,9 +320,9 @@ class MutableString(UserString):
         return self
 
 
-class String(MutableString, Union):
+class String(MutableString, ctypes.Union):
 
-    _fields_ = [("raw", POINTER(c_char)), ("data", c_char_p)]
+    _fields_ = [("raw", ctypes.POINTER(ctypes.c_char)), ("data", ctypes.c_char_p)]
 
     def __init__(self, obj=b""):
         if isinstance(obj, (bytes, UserString)):
@@ -358,7 +336,7 @@ class String(MutableString, Union):
     def from_param(cls, obj):
         # Convert None or 0
         if obj is None or obj == 0:
-            return cls(POINTER(c_char)())
+            return cls(ctypes.POINTER(ctypes.c_char)())
 
         # Convert from String
         elif isinstance(obj, String):
@@ -373,19 +351,19 @@ class String(MutableString, Union):
             return cls(obj.encode())
 
         # Convert from c_char_p
-        elif isinstance(obj, c_char_p):
+        elif isinstance(obj, ctypes.c_char_p):
             return obj
 
-        # Convert from POINTER(c_char)
-        elif isinstance(obj, POINTER(c_char)):
+        # Convert from POINTER(ctypes.c_char)
+        elif isinstance(obj, ctypes.POINTER(ctypes.c_char)):
             return obj
 
         # Convert from raw pointer
         elif isinstance(obj, int):
-            return cls(cast(obj, POINTER(c_char)))
+            return cls(ctypes.cast(obj, ctypes.POINTER(ctypes.c_char)))
 
-        # Convert from c_char array
-        elif isinstance(obj, c_char * len(obj)):
+        # Convert from ctypes.c_char array
+        elif isinstance(obj, ctypes.c_char * len(obj)):
             return obj
 
         # Convert from object
@@ -405,12 +383,12 @@ def ReturnString(obj, func=None, arguments=None):
 # primitive datatypes.
 #
 # Non-primitive return values wrapped with UNCHECKED won't be
-# typechecked, and will be converted to c_void_p.
+# typechecked, and will be converted to ctypes.c_void_p.
 def UNCHECKED(type):
     if hasattr(type, "_type_") and isinstance(type._type_, str) and type._type_ != "P":
         return type
     else:
-        return c_void_p
+        return ctypes.c_void_p
 
 
 # ctypes doesn't have direct support for variadic functions, so we have to write

python/libgrass_interface_generator/ctypesgen/printer_python/preamble/2_5.py

@@ -1,406 +0,0 @@
-import ctypes, os, sys
-from ctypes import *
-
-_int_types = (c_int16, c_int32)
-if hasattr(ctypes, "c_int64"):
-    # Some builds of ctypes apparently do not have c_int64
-    # defined; it's a pretty good bet that these builds do not
-    # have 64-bit pointers.
-    _int_types += (c_int64,)
-for t in _int_types:
-    if sizeof(t) == sizeof(c_size_t):
-        c_ptrdiff_t = t
-del t
-del _int_types
-
-
-class c_void(Structure):
-    # c_void_p is a buggy return type, converting to int, so
-    # POINTER(None) == c_void_p is actually written as
-    # POINTER(c_void), so it can be treated as a real pointer.
-    _fields_ = [("dummy", c_int)]
-
-
-def POINTER(obj):
-    p = ctypes.POINTER(obj)
-
-    # Convert None to a real NULL pointer to work around bugs
-    # in how ctypes handles None on 64-bit platforms
-    if not isinstance(p.from_param, classmethod):
-
-        def from_param(cls, x):
-            if x is None:
-                return cls()
-            else:
-                return x
-
-        p.from_param = classmethod(from_param)
-
-    return p
-
-
-class UserString:
-    def __init__(self, seq):
-        if isinstance(seq, basestring):
-            self.data = seq
-        elif isinstance(seq, UserString):
-            self.data = seq.data[:]
-        else:
-            self.data = str(seq)
-
-    def __str__(self):
-        return str(self.data)
-
-    def __repr__(self):
-        return repr(self.data)
-
-    def __int__(self):
-        return int(self.data)
-
-    def __long__(self):
-        return long(self.data)
-
-    def __float__(self):
-        return float(self.data)
-
-    def __complex__(self):
-        return complex(self.data)
-
-    def __hash__(self):
-        return hash(self.data)
-
-    def __cmp__(self, string):
-        if isinstance(string, UserString):
-            return cmp(self.data, string.data)
-        else:
-            return cmp(self.data, string)
-
-    def __contains__(self, char):
-        return char in self.data
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, index):
-        return self.__class__(self.data[index])
-
-    def __getslice__(self, start, end):
-        start = max(start, 0)
-        end = max(end, 0)
-        return self.__class__(self.data[start:end])
-
-    def __add__(self, other):
-        if isinstance(other, UserString):
-            return self.__class__(self.data + other.data)
-        elif isinstance(other, basestring):
-            return self.__class__(self.data + other)
-        else:
-            return self.__class__(self.data + str(other))
-
-    def __radd__(self, other):
-        if isinstance(other, basestring):
-            return self.__class__(other + self.data)
-        else:
-            return self.__class__(str(other) + self.data)
-
-    def __mul__(self, n):
-        return self.__class__(self.data * n)
-
-    __rmul__ = __mul__
-
-    def __mod__(self, args):
-        return self.__class__(self.data % args)
-
-    # the following methods are defined in alphabetical order:
-    def capitalize(self):
-        return self.__class__(self.data.capitalize())
-
-    def center(self, width, *args):
-        return self.__class__(self.data.center(width, *args))
-
-    def count(self, sub, start=0, end=sys.maxint):
-        return self.data.count(sub, start, end)
-
-    def decode(self, encoding=None, errors=None):  # XXX improve this?
-        if encoding:
-            if errors:
-                return self.__class__(self.data.decode(encoding, errors))
-            else:
-                return self.__class__(self.data.decode(encoding))
-        else:
-            return self.__class__(self.data.decode())
-
-    def encode(self, encoding=None, errors=None):  # XXX improve this?
-        if encoding:
-            if errors:
-                return self.__class__(self.data.encode(encoding, errors))
-            else:
-                return self.__class__(self.data.encode(encoding))
-        else:
-            return self.__class__(self.data.encode())
-
-    def endswith(self, suffix, start=0, end=sys.maxint):
-        return self.data.endswith(suffix, start, end)
-
-    def expandtabs(self, tabsize=8):
-        return self.__class__(self.data.expandtabs(tabsize))
-
-    def find(self, sub, start=0, end=sys.maxint):
-        return self.data.find(sub, start, end)
-
-    def index(self, sub, start=0, end=sys.maxint):
-        return self.data.index(sub, start, end)
-
-    def isalpha(self):
-        return self.data.isalpha()
-
-    def isalnum(self):
-        return self.data.isalnum()
-
-    def isdecimal(self):
-        return self.data.isdecimal()
-
-    def isdigit(self):
-        return self.data.isdigit()
-
-    def islower(self):
-        return self.data.islower()
-
-    def isnumeric(self):
-        return self.data.isnumeric()
-
-    def isspace(self):
-        return self.data.isspace()
-
-    def istitle(self):
-        return self.data.istitle()
-
-    def isupper(self):
-        return self.data.isupper()
-
-    def join(self, seq):
-        return self.data.join(seq)
-
-    def ljust(self, width, *args):
-        return self.__class__(self.data.ljust(width, *args))
-
-    def lower(self):
-        return self.__class__(self.data.lower())
-
-    def lstrip(self, chars=None):
-        return self.__class__(self.data.lstrip(chars))
-
-    def partition(self, sep):
-        return self.data.partition(sep)
-
-    def replace(self, old, new, maxsplit=-1):
-        return self.__class__(self.data.replace(old, new, maxsplit))
-
-    def rfind(self, sub, start=0, end=sys.maxint):
-        return self.data.rfind(sub, start, end)
-
-    def rindex(self, sub, start=0, end=sys.maxint):
-        return self.data.rindex(sub, start, end)
-
-    def rjust(self, width, *args):
-        return self.__class__(self.data.rjust(width, *args))
-
-    def rpartition(self, sep):
-        return self.data.rpartition(sep)
-
-    def rstrip(self, chars=None):
-        return self.__class__(self.data.rstrip(chars))
-
-    def split(self, sep=None, maxsplit=-1):
-        return self.data.split(sep, maxsplit)
-
-    def rsplit(self, sep=None, maxsplit=-1):
-        return self.data.rsplit(sep, maxsplit)
-
-    def splitlines(self, keepends=0):
-        return self.data.splitlines(keepends)
-
-    def startswith(self, prefix, start=0, end=sys.maxint):
-        return self.data.startswith(prefix, start, end)
-
-    def strip(self, chars=None):
-        return self.__class__(self.data.strip(chars))
-
-    def swapcase(self):
-        return self.__class__(self.data.swapcase())
-
-    def title(self):
-        return self.__class__(self.data.title())
-
-    def translate(self, *args):
-        return self.__class__(self.data.translate(*args))
-
-    def upper(self):
-        return self.__class__(self.data.upper())
-
-    def zfill(self, width):
-        return self.__class__(self.data.zfill(width))
-
-
-class MutableString(UserString):
-    """mutable string objects
-
-    Python strings are immutable objects.  This has the advantage, that
-    strings may be used as dictionary keys.  If this property isn't needed
-    and you insist on changing string values in place instead, you may cheat
-    and use MutableString.
-
-    But the purpose of this class is an educational one: to prevent
-    people from inventing their own mutable string class derived
-    from UserString and than forget thereby to remove (override) the
-    __hash__ method inherited from UserString.  This would lead to
-    errors that would be very hard to track down.
-
-    A faster and better solution is to rewrite your program using lists."""
-
-    def __init__(self, string=""):
-        self.data = string
-
-    def __hash__(self):
-        raise TypeError("unhashable type (it is mutable)")
-
-    def __setitem__(self, index, sub):
-        if index < 0:
-            index += len(self.data)
-        if index < 0 or index >= len(self.data):
-            raise IndexError
-        self.data = self.data[:index] + sub + self.data[index + 1 :]
-
-    def __delitem__(self, index):
-        if index < 0:
-            index += len(self.data)
-        if index < 0 or index >= len(self.data):
-            raise IndexError
-        self.data = self.data[:index] + self.data[index + 1 :]
-
-    def __setslice__(self, start, end, sub):
-        start = max(start, 0)
-        end = max(end, 0)
-        if isinstance(sub, UserString):
-            self.data = self.data[:start] + sub.data + self.data[end:]
-        elif isinstance(sub, basestring):
-            self.data = self.data[:start] + sub + self.data[end:]
-        else:
-            self.data = self.data[:start] + str(sub) + self.data[end:]
-
-    def __delslice__(self, start, end):
-        start = max(start, 0)
-        end = max(end, 0)
-        self.data = self.data[:start] + self.data[end:]
-
-    def immutable(self):
-        return UserString(self.data)
-
-    def __iadd__(self, other):
-        if isinstance(other, UserString):
-            self.data += other.data
-        elif isinstance(other, basestring):
-            self.data += other
-        else:
-            self.data += str(other)
-        return self
-
-    def __imul__(self, n):
-        self.data *= n
-        return self
-
-
-class String(MutableString, Union):
-
-    _fields_ = [("raw", POINTER(c_char)), ("data", c_char_p)]
-
-    def __init__(self, obj=""):
-        if isinstance(obj, (str, unicode, UserString)):
-            self.data = str(obj)
-        else:
-            self.raw = obj
-
-    def __len__(self):
-        return self.data and len(self.data) or 0
-
-    def from_param(cls, obj):
-        # Convert None or 0
-        if obj is None or obj == 0:
-            return cls(POINTER(c_char)())
-
-        # Convert from String
-        elif isinstance(obj, String):
-            return obj
-
-        # Convert from str
-        elif isinstance(obj, str):
-            return cls(obj)
-
-        # Convert from c_char_p
-        elif isinstance(obj, c_char_p):
-            return obj
-
-        # Convert from POINTER(c_char)
-        elif isinstance(obj, POINTER(c_char)):
-            return obj
-
-        # Convert from raw pointer
-        elif isinstance(obj, int):
-            return cls(cast(obj, POINTER(c_char)))
-
-        # Convert from object
-        else:
-            return String.from_param(obj._as_parameter_)
-
-    from_param = classmethod(from_param)
-
-
-def ReturnString(obj, func=None, arguments=None):
-    return String.from_param(obj)
-
-
-# As of ctypes 1.0, ctypes does not support custom error-checking
-# functions on callbacks, nor does it support custom datatypes on
-# callbacks, so we must ensure that all callbacks return
-# primitive datatypes.
-#
-# Non-primitive return values wrapped with UNCHECKED won't be
-# typechecked, and will be converted to c_void_p.
-def UNCHECKED(type):
-    if hasattr(type, "_type_") and isinstance(type._type_, str) and type._type_ != "P":
-        return type
-    else:
-        return c_void_p
-
-
-# ctypes doesn't have direct support for variadic functions, so we have to write
-# our own wrapper class
-class _variadic_function(object):
-    def __init__(self, func, restype, argtypes):
-        self.func = func
-        self.func.restype = restype
-        self.argtypes = argtypes
-
-    def _as_parameter_(self):
-        # So we can pass this variadic function as a function pointer
-        return self.func
-
-    def __call__(self, *args):
-        fixed_args = []
-        i = 0
-        for argtype in self.argtypes:
-            # Typecheck what we can
-            fixed_args.append(argtype.from_param(args[i]))
-            i += 1
-        return self.func(*fixed_args + list(args[i:]))
-
-
-def ord_if_char(value):
-    """
-    Simple helper used for casts to simple builtin types:  if the argument is a
-    string type, it will be converted to it's ordinal value.
-
-    This function will raise an exception if the argument is string with more
-    than one characters.
-    """
-    return ord(value) if isinstance(value, str) else value

python/libgrass_interface_generator/ctypesgen/printer_python/preamble/2_7.py

@@ -1,387 +0,0 @@
-import ctypes, os, sys
-from ctypes import *
-
-_int_types = (c_int16, c_int32)
-if hasattr(ctypes, "c_int64"):
-    # Some builds of ctypes apparently do not have c_int64
-    # defined; it's a pretty good bet that these builds do not
-    # have 64-bit pointers.
-    _int_types += (c_int64,)
-for t in _int_types:
-    if sizeof(t) == sizeof(c_size_t):
-        c_ptrdiff_t = t
-del t
-del _int_types
-
-
-class UserString:
-    def __init__(self, seq):
-        if isinstance(seq, basestring):
-            self.data = seq
-        elif isinstance(seq, UserString):
-            self.data = seq.data[:]
-        else:
-            self.data = str(seq)
-
-    def __str__(self):
-        return str(self.data)
-
-    def __repr__(self):
-        return repr(self.data)
-
-    def __int__(self):
-        return int(self.data)
-
-    def __long__(self):
-        return long(self.data)
-
-    def __float__(self):
-        return float(self.data)
-
-    def __complex__(self):
-        return complex(self.data)
-
-    def __hash__(self):
-        return hash(self.data)
-
-    def __cmp__(self, string):
-        if isinstance(string, UserString):
-            return cmp(self.data, string.data)
-        else:
-            return cmp(self.data, string)
-
-    def __contains__(self, char):
-        return char in self.data
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, index):
-        return self.__class__(self.data[index])
-
-    def __getslice__(self, start, end):
-        start = max(start, 0)
-        end = max(end, 0)
-        return self.__class__(self.data[start:end])
-
-    def __add__(self, other):
-        if isinstance(other, UserString):
-            return self.__class__(self.data + other.data)
-        elif isinstance(other, basestring):
-            return self.__class__(self.data + other)
-        else:
-            return self.__class__(self.data + str(other))
-
-    def __radd__(self, other):
-        if isinstance(other, basestring):
-            return self.__class__(other + self.data)
-        else:
-            return self.__class__(str(other) + self.data)
-
-    def __mul__(self, n):
-        return self.__class__(self.data * n)
-
-    __rmul__ = __mul__
-
-    def __mod__(self, args):
-        return self.__class__(self.data % args)
-
-    # the following methods are defined in alphabetical order:
-    def capitalize(self):
-        return self.__class__(self.data.capitalize())
-
-    def center(self, width, *args):
-        return self.__class__(self.data.center(width, *args))
-
-    def count(self, sub, start=0, end=sys.maxint):
-        return self.data.count(sub, start, end)
-
-    def decode(self, encoding=None, errors=None):  # XXX improve this?
-        if encoding:
-            if errors:
-                return self.__class__(self.data.decode(encoding, errors))
-            else:
-                return self.__class__(self.data.decode(encoding))
-        else:
-            return self.__class__(self.data.decode())
-
-    def encode(self, encoding=None, errors=None):  # XXX improve this?
-        if encoding:
-            if errors:
-                return self.__class__(self.data.encode(encoding, errors))
-            else:
-                return self.__class__(self.data.encode(encoding))
-        else:
-            return self.__class__(self.data.encode())
-
-    def endswith(self, suffix, start=0, end=sys.maxint):
-        return self.data.endswith(suffix, start, end)
-
-    def expandtabs(self, tabsize=8):
-        return self.__class__(self.data.expandtabs(tabsize))
-
-    def find(self, sub, start=0, end=sys.maxint):
-        return self.data.find(sub, start, end)
-
-    def index(self, sub, start=0, end=sys.maxint):
-        return self.data.index(sub, start, end)
-
-    def isalpha(self):
-        return self.data.isalpha()
-
-    def isalnum(self):
-        return self.data.isalnum()
-
-    def isdecimal(self):
-        return self.data.isdecimal()
-
-    def isdigit(self):
-        return self.data.isdigit()
-
-    def islower(self):
-        return self.data.islower()
-
-    def isnumeric(self):
-        return self.data.isnumeric()
-
-    def isspace(self):
-        return self.data.isspace()
-
-    def istitle(self):
-        return self.data.istitle()
-
-    def isupper(self):
-        return self.data.isupper()
-
-    def join(self, seq):
-        return self.data.join(seq)
-
-    def ljust(self, width, *args):
-        return self.__class__(self.data.ljust(width, *args))
-
-    def lower(self):
-        return self.__class__(self.data.lower())
-
-    def lstrip(self, chars=None):
-        return self.__class__(self.data.lstrip(chars))
-
-    def partition(self, sep):
-        return self.data.partition(sep)
-
-    def replace(self, old, new, maxsplit=-1):
-        return self.__class__(self.data.replace(old, new, maxsplit))
-
-    def rfind(self, sub, start=0, end=sys.maxint):
-        return self.data.rfind(sub, start, end)
-
-    def rindex(self, sub, start=0, end=sys.maxint):
-        return self.data.rindex(sub, start, end)
-
-    def rjust(self, width, *args):
-        return self.__class__(self.data.rjust(width, *args))
-
-    def rpartition(self, sep):
-        return self.data.rpartition(sep)
-
-    def rstrip(self, chars=None):
-        return self.__class__(self.data.rstrip(chars))
-
-    def split(self, sep=None, maxsplit=-1):
-        return self.data.split(sep, maxsplit)
-
-    def rsplit(self, sep=None, maxsplit=-1):
-        return self.data.rsplit(sep, maxsplit)
-
-    def splitlines(self, keepends=0):
-        return self.data.splitlines(keepends)
-
-    def startswith(self, prefix, start=0, end=sys.maxint):
-        return self.data.startswith(prefix, start, end)
-
-    def strip(self, chars=None):
-        return self.__class__(self.data.strip(chars))
-
-    def swapcase(self):
-        return self.__class__(self.data.swapcase())
-
-    def title(self):
-        return self.__class__(self.data.title())
-
-    def translate(self, *args):
-        return self.__class__(self.data.translate(*args))
-
-    def upper(self):
-        return self.__class__(self.data.upper())
-
-    def zfill(self, width):
-        return self.__class__(self.data.zfill(width))
-
-
-class MutableString(UserString):
-    """mutable string objects
-
-    Python strings are immutable objects.  This has the advantage, that
-    strings may be used as dictionary keys.  If this property isn't needed
-    and you insist on changing string values in place instead, you may cheat
-    and use MutableString.
-
-    But the purpose of this class is an educational one: to prevent
-    people from inventing their own mutable string class derived
-    from UserString and than forget thereby to remove (override) the
-    __hash__ method inherited from UserString.  This would lead to
-    errors that would be very hard to track down.
-
-    A faster and better solution is to rewrite your program using lists."""
-
-    def __init__(self, string=""):
-        self.data = string
-
-    def __hash__(self):
-        raise TypeError("unhashable type (it is mutable)")
-
-    def __setitem__(self, index, sub):
-        if index < 0:
-            index += len(self.data)
-        if index < 0 or index >= len(self.data):
-            raise IndexError
-        self.data = self.data[:index] + sub + self.data[index + 1 :]
-
-    def __delitem__(self, index):
-        if index < 0:
-            index += len(self.data)
-        if index < 0 or index >= len(self.data):
-            raise IndexError
-        self.data = self.data[:index] + self.data[index + 1 :]
-
-    def __setslice__(self, start, end, sub):
-        start = max(start, 0)
-        end = max(end, 0)
-        if isinstance(sub, UserString):
-            self.data = self.data[:start] + sub.data + self.data[end:]
-        elif isinstance(sub, basestring):
-            self.data = self.data[:start] + sub + self.data[end:]
-        else:
-            self.data = self.data[:start] + str(sub) + self.data[end:]
-
-    def __delslice__(self, start, end):
-        start = max(start, 0)
-        end = max(end, 0)
-        self.data = self.data[:start] + self.data[end:]
-
-    def immutable(self):
-        return UserString(self.data)
-
-    def __iadd__(self, other):
-        if isinstance(other, UserString):
-            self.data += other.data
-        elif isinstance(other, basestring):
-            self.data += other
-        else:
-            self.data += str(other)
-        return self
-
-    def __imul__(self, n):
-        self.data *= n
-        return self
-
-
-class String(MutableString, Union):
-
-    _fields_ = [("raw", POINTER(c_char)), ("data", c_char_p)]
-
-    def __init__(self, obj=""):
-        if isinstance(obj, (str, unicode, UserString)):
-            self.data = str(obj)
-        else:
-            self.raw = obj
-
-    def __len__(self):
-        return self.data and len(self.data) or 0
-
-    def from_param(cls, obj):
-        # Convert None or 0
-        if obj is None or obj == 0:
-            return cls(POINTER(c_char)())
-
-        # Convert from String
-        elif isinstance(obj, String):
-            return obj
-
-        # Convert from str
-        elif isinstance(obj, str):
-            return cls(obj)
-
-        # Convert from c_char_p
-        elif isinstance(obj, c_char_p):
-            return obj
-
-        # Convert from POINTER(c_char)
-        elif isinstance(obj, POINTER(c_char)):
-            return obj
-
-        # Convert from raw pointer
-        elif isinstance(obj, int):
-            return cls(cast(obj, POINTER(c_char)))
-
-        # Convert from c_char array
-        elif isinstance(obj, c_char * len(obj)):
-            return obj
-
-        # Convert from object
-        else:
-            return String.from_param(obj._as_parameter_)
-
-    from_param = classmethod(from_param)
-
-
-def ReturnString(obj, func=None, arguments=None):
-    return String.from_param(obj)
-
-
-# As of ctypes 1.0, ctypes does not support custom error-checking
-# functions on callbacks, nor does it support custom datatypes on
-# callbacks, so we must ensure that all callbacks return
-# primitive datatypes.
-#
-# Non-primitive return values wrapped with UNCHECKED won't be
-# typechecked, and will be converted to c_void_p.
-def UNCHECKED(type):
-    if hasattr(type, "_type_") and isinstance(type._type_, str) and type._type_ != "P":
-        return type
-    else:
-        return c_void_p
-
-
-# ctypes doesn't have direct support for variadic functions, so we have to write
-# our own wrapper class
-class _variadic_function(object):
-    def __init__(self, func, restype, argtypes, errcheck):
-        self.func = func
-        self.func.restype = restype
-        self.argtypes = argtypes
-        if errcheck:
-            self.func.errcheck = errcheck
-
-    def _as_parameter_(self):
-        # So we can pass this variadic function as a function pointer
-        return self.func
-
-    def __call__(self, *args):
-        fixed_args = []
-        i = 0
-        for argtype in self.argtypes:
-            # Typecheck what we can
-            fixed_args.append(argtype.from_param(args[i]))
-            i += 1
-        return self.func(*fixed_args + list(args[i:]))
-
-
-def ord_if_char(value):
-    """
-    Simple helper used for casts to simple builtin types:  if the argument is a
-    string type, it will be converted to it's ordinal value.
-
-    This function will raise an exception if the argument is string with more
-    than one characters.
-    """
-    return ord(value) if isinstance(value, str) else value

python/libgrass_interface_generator/ctypesgen/printer_python/printer.py

@@ -1,43 +1,19 @@
-#!/usr/bin/env python
+import os
+import os.path
+import sys
+import time
+import shutil
 
-import os, sys, time, glob, re
-from ..descriptions import *
-from ..ctypedescs import *
-from ..messages import *
-from .. import expressions
-
-from .. import libraryloader  # So we can get the path to it
-from . import test  # So we can find the path to local files in the printer package
-
-
-def path_to_local_file(name, known_local_module=test):
-    basedir = os.path.dirname(known_local_module.__file__)
-    return os.path.join(basedir, name)
+from ctypesgen.ctypedescs import CtypesBitfield, CtypesStruct
+from ctypesgen.expressions import ExpressionNode
+from ctypesgen.messages import error_message, status_message
 
 
 THIS_DIR = os.path.dirname(__file__)
-PREAMBLE_PATH = os.path.join(THIS_DIR, "preamble", "[0-9]_[0-9].py")
-
-
-def get_preamble(major=None, minor=None):
-    """get the available preambles"""
-    preambles = dict()
-    for fp in glob.glob(PREAMBLE_PATH):
-        m = re.search("(\d)_(\d).py$", fp)
-        if not m:
-            continue
-        preambles[(int(m.group(1)), int(m.group(2)))] = fp
-
-    if None not in (major, minor):
-        v = (int(major), int(minor))
-    else:
-        L = sorted(preambles.keys())
-        v = L[0]
-        for vi in L[1:]:
-            if vi > sys.version_info[:2]:
-                break
-            v = vi
-    return preambles[v], v
+CTYPESGEN_DIR = os.path.join(THIS_DIR, os.path.pardir)
+PREAMBLE_PATH = os.path.join(THIS_DIR, "preamble.py")
+DEFAULTHEADER_PATH = os.path.join(THIS_DIR, "defaultheader.py")
+LIBRARYLOADER_PATH = os.path.join(CTYPESGEN_DIR, "libraryloader.py")
 
 
 class WrapperPrinter:
@@ -51,6 +27,9 @@ class WrapperPrinter:
         if self.options.strip_build_path and self.options.strip_build_path[-1] != os.path.sep:
             self.options.strip_build_path += os.path.sep
 
+        if not self.options.embed_preamble and outpath:
+            self._copy_preamble_loader_files(outpath)
+
         self.print_header()
         self.file.write("\n")
 
@@ -104,7 +83,7 @@ class WrapperPrinter:
         self.file.write("\n")
 
     def srcinfo(self, src):
-        if src == None:
+        if src is None:
             self.file.write("\n")
         else:
             filename, lineno = src
@@ -149,8 +128,7 @@ class WrapperPrinter:
                 )
 
         if not template_file:
-            path = path_to_local_file("defaultheader.py")
-            template_file = open(path, "r")
+            template_file = open(DEFAULTHEADER_PATH, "r")
 
         template_subs = self.template_subs()
         self.file.write(template_file.read() % template_subs)
@@ -158,26 +136,71 @@ class WrapperPrinter:
         template_file.close()
 
     def print_preamble(self):
-        m = re.match("py((?P<major>[0-9])(?P<minor>[0-9]))?", self.options.output_language)
-        path, v = get_preamble(**m.groupdict())
-
-        self.file.write("# Begin preamble for Python v{}\n\n".format(v))
-        self.file.write("from .ctypes_preamble import *\n")
-        self.file.write("from .ctypes_preamble import _variadic_function\n")
-        # preamble_file = open(path, "r")
-        # self.file.write(preamble_file.read())
-        # preamble_file.close()
+        self.file.write("# Begin preamble for Python\n\n")
+        if self.options.embed_preamble:
+            with open(PREAMBLE_PATH, "r") as preamble_file:
+                preamble_file_content = preamble_file.read()
+                filecontent = preamble_file_content.replace("# ~POINTER~", "")
+                self.file.write(filecontent)
+        else:
+            self.file.write("from .ctypes_preamble import *\n")
+            self.file.write("from .ctypes_preamble import _variadic_function\n")
+
         self.file.write("\n# End preamble\n")
 
+    def _copy_preamble_loader_files(self, path):
+        if os.path.isfile(path):
+            abspath = os.path.abspath(path)
+            dst = os.path.dirname(abspath)
+        else:
+            error_message(
+                "Cannot copy preamble and loader files",
+                cls="missing-file",
+            )
+            return
+
+        c_preamblefile = f"{dst}/ctypes_preamble.py"
+        if os.path.isfile(c_preamblefile):
+            return
+
+        pointer = """def POINTER(obj):
+    p = ctypes.POINTER(obj)
+
+    # Convert None to a real NULL pointer to work around bugs
+    # in how ctypes handles None on 64-bit platforms
+    if not isinstance(p.from_param, classmethod):
+
+        def from_param(cls, x):
+            if x is None:
+                return cls()
+            else:
+                return x
+
+        p.from_param = classmethod(from_param)
+
+    return p
+
+"""
+
+        with open(PREAMBLE_PATH) as preamble_file:
+            preamble_file_content = preamble_file.read()
+            filecontent = preamble_file_content.replace("# ~POINTER~", pointer)
+
+        with open(c_preamblefile, "w") as f:
+            f.write(filecontent)
+
+        shutil.copy(LIBRARYLOADER_PATH, f"{dst}")
+        os.rename(f"{dst}/libraryloader.py", f"{dst}/ctypes_loader.py")
+
     def print_loader(self):
         self.file.write("_libs = {}\n")
         self.file.write("_libdirs = %s\n\n" % self.options.compile_libdirs)
         self.file.write("# Begin loader\n\n")
-        self.file.write("from .ctypes_loader import *\n")        
-        # path = path_to_local_file("libraryloader.py", libraryloader)
-        # loader_file = open(path, "r")
-        # self.file.write(loader_file.read())
-        # loader_file.close()
+        if self.options.embed_preamble:
+            with open(LIBRARYLOADER_PATH, "r") as loader_file:
+                self.file.write(loader_file.read())
+        else:
+            self.file.write("from .ctypes_loader import *\n")
         self.file.write("\n# End loader\n\n")
         self.file.write(
             "add_library_search_dirs([%s])"
@@ -223,7 +246,7 @@ class WrapperPrinter:
             aligned = struct.attrib.get("aligned", [1])
             assert len(aligned) == 1, "cgrammar gave more than one arg for aligned attribute"
             aligned = aligned[0]
-            if isinstance(aligned, expressions.ExpressionNode):
+            if isinstance(aligned, ExpressionNode):
                 # TODO: for non-constant expression nodes, this will fail:
                 aligned = aligned.evaluate(None)
             self.file.write("{}_{}._pack_ = {}\n".format(struct.variety, struct.tag, aligned))

python/libgrass_interface_generator/ctypesgen/printer_python/test.py

@@ -1,6 +0,0 @@
-"""
-ctypesgen.printer.printer imports this module so that it can find the path
-to defaulttemplate.py and defaultloader.py.
-"""
-
-pass

python/libgrass_interface_generator/ctypesgen/processor/__init__.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 """
 This module contains functions to operate on the DeclarationCollection produced
 by the parser module and prepare it for output.

python/libgrass_interface_generator/ctypesgen/processor/dependencies.py

@@ -1,20 +1,17 @@
-#!/usr/bin/env python
-
 """
 The dependencies module determines which descriptions depend on which other
 descriptions.
 """
 
-from ..descriptions import *
-from ..ctypedescs import *
-from ..messages import *
+from ctypesgen.descriptions import MacroDescription, UndefDescription
+from ctypesgen.ctypedescs import visit_type_and_collect_info
 
 
 def find_dependencies(data, opts):
     """Visit each description in `data` and figure out which other descriptions
-it depends on, putting the results in desc.requirements. Also find errors in
-ctypedecls or expressions attached to the description and transfer them to the
-description."""
+    it depends on, putting the results in desc.requirements. Also find errors in
+    ctypedecls or expressions attached to the description and transfer them to the
+    description."""
 
     struct_names = {}
     enum_names = {}
@@ -35,7 +32,7 @@ description."""
 
     def depend(desc, nametable, name):
         """Try to add `name` as a requirement for `desc`, looking `name` up in
-`nametable`. Returns True if found."""
+        `nametable`. Returns True if found."""
 
         if name in nametable:
             requirement = nametable[name]
@@ -63,8 +60,8 @@ description."""
 
     def find_dependencies_for(desc, kind):
         """Find all the descriptions that `desc` depends on and add them as
-dependencies for `desc`. Also collect error messages regarding `desc` and
-convert unlocateable descriptions into error messages."""
+        dependencies for `desc`. Also collect error messages regarding `desc` and
+        convert unlocateable descriptions into error messages."""
 
         if kind == "constant":
             roots = [desc.value]
@@ -139,7 +136,7 @@ convert unlocateable descriptions into error messages."""
 
     def add_to_lookup_table(desc, kind):
         """Add `desc` to the lookup table so that other descriptions that use
-it can find it."""
+        it can find it."""
         if kind == "struct":
             if (desc.variety, desc.tag) not in struct_names:
                 struct_names[(desc.variety, desc.tag)] = desc

python/libgrass_interface_generator/ctypesgen/processor/operations.py

@@ -1,15 +1,21 @@
-#!/usr/bin/env python
-
 """
 The operations module contains various functions to process the
 DescriptionCollection and prepare it for output.
 ctypesgen.processor.pipeline calls the operations module.
 """
 
-import ctypes, re, os, sys, keyword
-from ..descriptions import *
-from ..messages import *
-from .. import libraryloader
+import re
+import os
+import keyword
+
+from ctypesgen import libraryloader
+from ctypesgen.descriptions import (
+    EnumDescription,
+    StructDescription,
+    TypedefDescription,
+)
+from ctypesgen.messages import warning_message
+
 
 # Processor functions
 
@@ -31,7 +37,7 @@ def automatically_typedef_structs(data, options):
 
 def remove_NULL(data, options):
     """remove_NULL() removes any NULL definitions from the C headers because
-ctypesgen supplies its own NULL definition."""
+    ctypesgen supplies its own NULL definition."""
 
     for macro in data.macros:
         if macro.name == "NULL":
@@ -45,7 +51,7 @@ def remove_descriptions_in_system_headers(data, opts):
     known_headers = [os.path.basename(x) for x in opts.headers]
 
     for description in data.all:
-        if description.src != None:
+        if description.src is not None:
             if description.src[0] == "<command line>":
                 description.include_rule = "if_needed"
             elif description.src[0] == "<built-in>":
@@ -260,7 +266,7 @@ def find_source_libraries(data, opts):
     for library_name in opts.libraries:
         try:
             library = libraryloader.load_library(library_name)
-        except ImportError as e:
+        except ImportError:
             warning_message(
                 'Could not load library "%s". Okay, I\'ll '
                 "try to load it at runtime instead. " % (library_name),
@@ -268,6 +274,6 @@ def find_source_libraries(data, opts):
             )
             continue
         for symbol in all_symbols:
-            if symbol.source_library == None:
+            if symbol.source_library is None:
                 if hasattr(library, symbol.c_name()):
                     symbol.source_library = library_name

python/libgrass_interface_generator/ctypesgen/processor/pipeline.py

@@ -1,11 +1,3 @@
-#!/usr/bin/env python
-
-import ctypes, re, os
-from ..ctypedescs import *
-from ..messages import *
-from .operations import *
-from .dependencies import find_dependencies
-
 """
 A brief explanation of the processing steps:
 1. The dependencies module builds a dependency graph for the descriptions.
@@ -38,6 +30,24 @@ the errors that print_errors_encountered() has flagged.
 
 """
 
+from ctypesgen.descriptions import MacroDescription
+from ctypesgen.messages import (
+    error_message,
+    status_message,
+    warning_message,
+)
+from ctypesgen.processor.dependencies import find_dependencies
+from ctypesgen.processor.operations import (
+    automatically_typedef_structs,
+    filter_by_regexes_exclude,
+    filter_by_regexes_include,
+    find_source_libraries,
+    fix_conflicting_names,
+    remove_descriptions_in_system_headers,
+    remove_macros,
+    remove_NULL,
+)
+
 
 def process(data, options):
     status_message("Processing description list.")
@@ -61,18 +71,16 @@ def process(data, options):
 
 
 def calculate_final_inclusion(data, opts):
-    """calculate_final_inclusion() calculates which descriptions will be included in the
-    output library.
+    """Calculates which descriptions will be included in the output library.
 
     An object with include_rule="never" is never included.
-    An object with include_rule="yes" is included if its requirements can be
-        included.
-    An object with include_rule="if_needed" is included if an object to be
-        included requires it and if its requirements can be included.
+    An object with include_rule="yes" is included if its requirements can be included.
+    An object with include_rule="if_needed" is included if an object to be included
+        requires it and if its requirements can be included.
     """
 
     def can_include_desc(desc):
-        if desc.can_include == None:
+        if desc.can_include is None:
             if desc.include_rule == "no":
                 desc.can_include = False
             elif desc.include_rule == "yes" or desc.include_rule == "if_needed":

python/libgrass_interface_generator/ctypesgen/test/.gitignore

@@ -1,2 +0,0 @@
-temp.h
-temp.py

python/libgrass_interface_generator/ctypesgen/test/ctypesgentest.py

@@ -1,88 +0,0 @@
-# -*- coding: ascii -*-
-# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
-import os
-import sys
-import io
-import optparse
-import glob
-import json
-
-try:
-    # should succeed for py3
-    from importlib import reload as reload_module
-except:
-    reload_module = reload
-
-# ensure that we can load the ctypesgen library
-PACKAGE_DIR = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
-sys.path.insert(0, PACKAGE_DIR)
-import ctypesgen
-
-"""ctypesgentest is a simple module for testing ctypesgen on various C constructs. It consists of a
-single function, test(). test() takes a string that represents a C header file, along with some
-keyword arguments representing options. It processes the header using ctypesgen and returns a tuple
-containing the resulting module object and the output that ctypesgen produced."""
-
-# set redirect_stdout to False if using console based debugger like pdb
-redirect_stdout = True
-
-
-def test(header, **more_options):
-
-    assert isinstance(header, str)
-    with open("temp.h", "w") as f:
-        f.write(header)
-
-    options = ctypesgen.options.get_default_options()
-    options.headers = ["temp.h"]
-    for opt, val in more_options.items():
-        setattr(options, opt, val)
-
-    if redirect_stdout:
-        # Redirect output
-        sys.stdout = io.StringIO()
-
-    # Step 1: Parse
-    descriptions = ctypesgen.parser.parse(options.headers, options)
-
-    # Step 2: Process
-    ctypesgen.processor.process(descriptions, options)
-
-    # Step 3: Print
-    printer = None
-    if options.output_language.startswith("py"):
-        ctypesgen.printer_python.WrapperPrinter("temp.py", options, descriptions)
-
-        # Load the module we have just produced
-        module = __import__("temp")
-        # import twice, this hack ensure that "temp" is force loaded
-        # (there *must* be a better way to do this)
-        reload_module(module)
-        retval = module
-
-    elif options.output_language == "json":
-        # for ease and consistency with test results, we are going to cheat by
-        # resetting the anonymous tag number
-        ctypesgen.ctypedescs.last_tagnum = 0
-        ctypesgen.printer_json.WrapperPrinter("temp.json", options, descriptions)
-        with open("temp.json") as f:
-            JSON = json.load(f)
-        retval = JSON
-    else:
-        raise RuntimeError("No such output language `" + options.output_language + "'")
-
-    if redirect_stdout:
-        # Un-redirect output
-        output = sys.stdout.getvalue()
-        sys.stdout.close()
-        sys.stdout = sys.__stdout__
-    else:
-        output = ""
-
-    return retval, output
-
-
-def cleanup(filepattern="temp.*"):
-    fnames = glob.glob(filepattern)
-    for fname in fnames:
-        os.unlink(fname)

python/libgrass_interface_generator/ctypesgen/version.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# vim: ts=2:sw=2:tw=80:nowrap
 
 from subprocess import Popen, PIPE
 import os
@@ -19,7 +18,7 @@ def version_tuple(v):
         if len(vs) > 2:
             t += (int(vs[2]),)
         return t
-    except:
+    except Exception:
         return (-1, -1, -1, v)
 
 
@@ -39,11 +38,11 @@ def version():
         if p.returncode:
             raise RuntimeError("no version defined?")
         return out.strip().decode()
-    except:
+    except Exception:
         # failover is to try VERSION_FILE instead
         try:
             return read_file_version()
-        except:
+        except Exception:
             return DEFAULT_PREFIX + "-0.0.0"
 
 
@@ -70,7 +69,7 @@ VERSION_NUMBER = version_number()
 
 
 if __name__ == "__main__":
-    import sys, argparse
+    import argparse
 
     p = argparse.ArgumentParser()
     p.add_argument("--save", action="store_true", help="Store version to " + VERSION_FILE)

python/libgrass_interface_generator/run.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 
-import sys, os
+import sys
+import os
 
 THIS_DIR = os.path.dirname(__file__)
 # ensure that we can load the ctypesgen library
 sys.path.insert(0, THIS_DIR)
 
-import ctypesgen.main
+import ctypesgen.main  # noqa: E402
 
 if __name__ == "__main__":
     ctypesgen.main.main()