CI for MS Windows: Switch to OSGeo4W V2 (#2074)

With this PR merged, CI uses packages from OSGeo4W Version2 (https://github.com/jef-n/OSGeo4W/) to build GRASS GIS using MSYS. Now also shellscript tests are executed. The PR replaces #1873 and parts of #1683. Compilation for tests is now also configured with OpenMP and LAPACK support (amongst others).

.github/workflows/build_osgeo4w.sh

@@ -1,145 +1,22 @@
 #!/bin/sh
 #
-# Usage: build_osgeo4w.sh [-p] [path]
+# Usage: build_osgeo4w.sh [path]
 #
 # By default, the script will look for the source code in the current directory
 # and create bin.x86_64-w64-mingw32\grass$ver.bat (run this batch file to start
 # GRASS GIS) and dist.x86_64-w64-mingw32\etc\env.bat.
 #
-# -p	optionally install GRASS GIS to C:\OSGeo4W64\opt\grass (run
-#	C:\OSGeo4W64\opt\grass\grass$ver.bat) and create an unzippable package
-#	grass$ver-x86_64-w64-mingw32-osgeo4w64-$date.zip
-#
 # path	specify a path to the source code
 #
 
 # stop on errors
 set -e
 
-if [ "$1" = "-p" ]; then
-    package=1
-    shift
-else
-    package=0
-fi
-
 test -d "$1" && cd "$1"
 
-osgeo4w_path=/c/OSGeo4W64
-arch=x86_64-w64-mingw32
-src=$(pwd)
-
-# compile
-
-export PATH=/mingw64/bin:/c/OSGeo4W64/bin:/usr/bin
-export PROJ_LIB=$osgeo4w_path/share/proj
-
-OSGEO4W_ROOT_MSYS=$osgeo4w_path \
-    ./configure \
-    --host=$arch \
-    --with-includes=$osgeo4w_path/include \
-    --with-libs="$osgeo4w_path/lib $osgeo4w_path/bin" \
-    --with-nls \
-    --with-freetype-includes=$osgeo4w_path/include/freetype2 \
-    --with-bzlib \
-    --with-geos=$src/mswindows/osgeo4w/geos-config \
-    --with-netcdf=$osgeo4w_path/bin/nc-config \
-    --with-gdal=$src/mswindows/osgeo4w/gdal-config \
-    --with-liblas=$src/mswindows/osgeo4w/liblas-config \
-    --with-opengl=windows
-
-make
-
-# install
-
-bin=bin.$arch
-dist=dist.$arch
-ver=$(sed -n '/^INST_DIR[ \t]*=/{s/^.*grass//; p}' include/Make/Platform.make)
-
-rm -f $dist/grass$ver.tmp $dist/etc/fontcap
-
-# create batch files
-
-src_esc=$(echo $src | sed 's#^/\([a-z]\)#\1:#; s#/#\\\\\\\\#g')
-dist_esc="$src_esc\\\\$dist"
-
-(
-    sed 's/^\(set GISBASE=\).*/\1'$dist_esc'/' \
-        mswindows/osgeo4w/env.bat.tmpl
-    cat <<EOT
-
-set PATH=C:\\msys64\\mingw64\\bin;%OSGEO4W_ROOT%\\apps\\msys\\bin;%PATH%
-
-if not exist %GISBASE%\etc\fontcap (
-	pushd .
-	%~d0
-	cd %GISBASE%\lib
-	set GISRC=dummy
-	%GISBASE%\bin\g.mkfontcap.exe
-	popd
-)
-EOT
-) >$dist/etc/env.bat
-unix2dos $dist/etc/env.bat
-
-(
-    sed -e 's/^\(call "\)%~dp0\(.*\)$/\1C:\\OSGeo4W64\\bin\2/' \
-        -e 's/^\(call "\).*\(\\etc\\env\.bat"\)$/\1'$dist_esc'\2/' \
-        -e 's/^\(.* "\)%GISBASE%\\etc\(\\grass.*\)$/\1%GISBASE%\\..\\'$bin'\2/' \
-        -e 's/@POSTFIX@/'$ver'/g' \
-        mswindows/osgeo4w/grass.bat.tmpl
-) >$bin/grass$ver.bat
-unix2dos $bin/grass$ver.bat
-
-test $package -eq 0 && exit
-
-# package
-
-opt_path=$osgeo4w_path/opt
-grass_path=$opt_path/grass
-
-mkdir -p $opt_path
-cp -a $dist $grass_path
-# have a versionless and versioned startup script
-cp -a $bin/grass.py $bin/grass$ver.py
-cp -a $bin/grass$ver.py $grass_path/etc
-cp -a $(ldd $dist/lib/*.dll | awk '/mingw64/{print $3}' |
-    sort -u | grep -v 'lib\(crypto\|ssl\)') $grass_path/lib
-
-(
-    sed -e 's/^\(set GISBASE=\).*/\1%OSGEO4W_ROOT%\\opt\\grass/' \
-        mswindows/osgeo4w/env.bat.tmpl
-    cat <<EOT
-
-set PATH=%OSGEO4W_ROOT%\\apps\\msys\\bin;%PATH%
-
-if not exist %GISBASE%\etc\fontcap (
-	pushd .
-	%~d0
-	cd %GISBASE%\lib
-	set GISRC=dummy
-	%GISBASE%\bin\g.mkfontcap.exe
-	popd
-)
-EOT
-) >$grass_path/etc/env.bat
-unix2dos $grass_path/etc/env.bat
-
-(
-    sed -e 's/^\(call "%~dp0\)\(.*\)$/\1\\..\\..\\bin\2/' \
-        -e 's/^\(call "%OSGEO4W_ROOT%\\\).*\(\\etc\\env\.bat"\)$/\1opt\\grass\2/' \
-        -e 's/@POSTFIX@/'$ver'/g' \
-        mswindows/osgeo4w/grass.bat.tmpl
-) >$grass_path/grass$ver.bat
-unix2dos $grass_path/grass$ver.bat
-
-exit
-
-# don't package for GitHub workflow; unnecessary
-
-osgeo4w_basename=$(basename $osgeo4w_path)
-date=$(date +%Y%m%d)
-zip=$src/grass$ver-$arch-osgeo4w64-$date.zip
+export OSGEO4W_ROOT_MSYS=/c/OSGeo4W
+export SRC=$(pwd)
+export UNITTEST=1
 
-cd $osgeo4w_path/..
-zip -r $zip $osgeo4w_basename -x "$osgeo4w_basename/var/*" "*/__pycache__/*"
+# Build according to OSGeo4W recipe
+${SRC}/mswindows/osgeo4w/build_osgeo4W.sh

.github/workflows/test_simple.bat

@@ -1,3 +1,8 @@
 set grass=%1
 
+rem Test execution of binary command
 call %grass% --tmp-location EPSG:4326 --exec g.region res=0.1 -p
+rem Test if batch-wrapper-scripts without extension are found
+call %grass% --tmp-location EPSG:4326 --exec C:/Windows/System32/where.exe t.create
+rem Test if python-scripts can be called
+call %grass% --tmp-location EPSG:4326 --exec t.create --help

.github/workflows/test_simple.sh

@@ -3,4 +3,9 @@
 # fail on non-zero return code from a subprocess
 set -e
 
+# Test execution of binary command
 grass --tmp-location EPSG:4326 --exec g.region res=0.1 -p
+# Test if python modules without extension are found
+grass --tmp-location EPSG:4326 --exec which t.create
+# Test if python modules can be called
+grass --tmp-location EPSG:4326 --exec t.create --help

.github/workflows/test_thorough.bat

@@ -1,5 +1,5 @@
 set grass=%1
-set python=C:\OSGeo4W64\bin\python3
+set python=%2
 
 call %grass% --tmp-location XY --exec g.download.location url=https://grass.osgeo.org/sampledata/north_carolina/nc_spm_full_v2alpha2.tar.gz path=%USERPROFILE%
 call %grass% --tmp-location XY --exec %python% -m grass.gunittest.main --grassdata %USERPROFILE% --location nc_spm_full_v2alpha2 --location-type nc --min-success 60

mswindows/osgeo4w/build_osgeo4w.sh

@@ -0,0 +1,193 @@
+#!/bin/sh
+
+#
+# Usage: build_osgeo4w.sh
+#
+# The following environment variables are supposed to be passed to the build script
+# - SRC: the directory where the grass source code lives
+# - OSGEO4W_ROOT_MSYS: the root directory of OSGeo4W
+# - UNITTEST: If this variable is defined, addition files for unittests are created
+#
+# By default, the script will look for the source code in the current directory
+# and create bin.x86_64-w64-mingw32\grass$ver.bat (run this batch file to start
+# GRASS GIS) and dist.x86_64-w64-mingw32\etc\env.bat.
+#
+# -p	optionally install GRASS GIS to C:\OSGeo4W\opt\grass (run
+#	C:\OSGeo4W64\opt\grass\grass$ver.bat) and create an unzippable package
+#	grass$ver-x86_64-w64-mingw32-osgeo4w64-$date.zip
+#
+# path	specify a path to the source code
+#
+
+# stop on errors
+set -e
+
+
+# compile
+export PATH=${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin
+export C_INCLUDE_PATH=".:${OSGEO4W_ROOT_MSYS}/include:${SRC}/dist.${ARCH}/include:/c/msys64/mingw64/include"
+export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python39
+export ARCH=x86_64-w64-mingw32
+
+./configure \
+    --host=${ARCH} \
+    --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \
+    --with-includes=${OSGEO4W_ROOT_MSYS}/include \
+    --libexecdir=${OSGEO4W_ROOT_MSYS}/bin \
+    --prefix=${OSGEO4W_ROOT_MSYS}/apps/grass \
+    --bindir=${OSGEO4W_ROOT_MSYS}/bin \
+    --includedir=${OSGEO4W_ROOT_MSYS}/include \
+    --without-x \
+    --with-cxx \
+    --enable-shared \
+    --enable-largefile \
+    --with-openmp \
+    --with-fftw \
+    --with-nls \
+    --with-readline \
+    --with-wxwidgets \
+    --with-blas \
+    --with-lapack-includes=/mingw64/include/lapack \
+    --with-freetype \
+    --with-freetype-includes=/mingw64/include/freetype2 \
+    --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \
+    --with-proj-includes=${OSGEO4W_ROOT_MSYS}/include \
+    --with-proj-libs=${OSGEO4W_ROOT_MSYS}/lib \
+    --with-postgres \
+    --with-postgres-includes=${OSGEO4W_ROOT_MSYS}/include \
+    --with-postgres-libs=${OSGEO4W_ROOT_MSYS}/lib \
+    --with-gdal=${SRC}/mswindows/osgeo4w/gdal-config \
+    --with-geos=${SRC}/mswindows/osgeo4w/geos-config \
+    --with-sqlite \
+    --with-sqlite-includes=${OSGEO4W_ROOT_MSYS}/include \
+    --with-sqlite-libs=${OSGEO4W_ROOT_MSYS}/lib \
+    --with-regex \
+    --with-nls \
+    --with-zstd \
+    --with-odbc \
+    --with-cairo \
+    --with-cairo-includes=${SRC}/include \
+    --with-cairo-ldflags="-L${SRC}/mswindows/osgeo4w/lib -lcairo -lfontconfig" \
+    --with-opengl=windows \
+    --with-bzlib \
+    --with-liblas=${SRC}/mswindows/osgeo4w/liblas-config \
+    --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config
+
+make
+
+# install
+
+bin=bin.${ARCH}
+dist=dist.${ARCH}
+ver=$(sed -n '/^INST_DIR[ \t]*=/{s/^.*grass//; p}' include/Make/Platform.make)
+
+rm -f $dist/grass$ver.tmp $dist/etc/fontcap
+
+if [ "$UNITTEST" ]; then
+    # Add executables for bash scripts in unittests
+    bash_bin=bash_bin
+    mkdir "${SRC}/dist.${ARCH}/$bash_bin"
+
+    for f in ${SRC}/dist.${ARCH}/scripts/*.py
+    do
+        bash_exe=$(echo $f | sed -e "s/\/scripts\//\/${bash_bin}\//;s/\.py$//")
+        cp "$f" "$bash_exe"
+        # dos2unix "$bash_exe"
+        chmod ugo+x "$bash_exe"
+    done
+    bash_exe_path=$(echo "${SRC}/dist.${ARCH}/$bash_bin" | sed -e "s/^\/c/\;c:/;s/^\/d/d:/")
+    export PATH="$PATH:${SRC}/dist.${ARCH}/$bash_bin"
+fi
+
+# create batch files
+src_esc=$(echo ${SRC} | sed 's#^/\([a-z]\)#\1:#; s#/#\\\\\\\\#g')
+dist_esc="$src_esc\\\\$dist"
+
+(
+    sed 's/^\(set GISBASE=\).*/\1'$dist_esc'/' \
+        mswindows/osgeo4w/env.bat.tmpl
+    cat <<EOT
+
+
+set PATH=%PATH%;C:\\msys64\\mingw64\\bin;C:\\msys64\\usr\\bin
+
+if not exist %GISBASE%\etc\fontcap (
+	pushd .
+	%~d0
+	cd %GISBASE%\lib
+	set GISRC=dummy
+	%GISBASE%\bin\g.mkfontcap.exe
+	popd
+)
+EOT
+) >$dist/etc/env.bat
+unix2dos $dist/etc/env.bat
+
+(
+    sed -e 's/^\(call "\)%~dp0\(.*\)$/\1C:\\OSGeo4W\\bin\2/' \
+        -e 's/^\(call "\).*\(\\etc\\env\.bat"\)$/\1'$dist_esc'\2/' \
+        -e 's/^\(.* "\)%GISBASE%\\etc\(\\grass.*\)$/\1%GISBASE%\\..\\'$bin'\2/' \
+        -e 's/@POSTFIX@/'$ver'/g' \
+        mswindows/osgeo4w/grass.bat.tmpl
+) >$bin/grass$ver.bat
+unix2dos $bin/grass$ver.bat
+
+opt_path=${OSGEO4W_ROOT_MSYS}/opt
+grass_path=$opt_path/grass
+
+if [ "$UNITTEST" ]; then
+    msys_path=";C:/msys64/usr/bin;C:/msys64/mingw64/bin"
+fi
+
+mkdir -p $opt_path
+cp -a $dist $grass_path
+# have a versionless and versioned startup script
+cp -a $bin/grass.py $bin/grass$ver.py
+cp -a $bin/grass$ver.py $grass_path/etc
+cp -a $(ldd $dist/lib/*.dll | awk '/mingw64/{print $3}' |
+    sort -u | grep -v 'lib\(crypto\|ssl\)') $grass_path/lib
+
+(
+    sed -e 's/^\(set GISBASE=\).*/\1%OSGEO4W_ROOT%\\opt\\grass/' \
+        mswindows/osgeo4w/env.bat.tmpl
+    cat <<EOT
+
+set PATH=%OSGEO4W_ROOT%\\bin${msys_path};%PATH%;$bash_exe_path
+
+if not exist %GISBASE%\etc\fontcap (
+	pushd .
+	%~d0
+	cd %GISBASE%\lib
+	set GISRC=dummy
+	%GISBASE%\bin\g.mkfontcap.exe
+	popd
+)
+EOT
+) >$grass_path/etc/env.bat
+unix2dos $grass_path/etc/env.bat
+
+cat $grass_path/etc/env.bat
+
+(
+    sed -e 's/^\(call "%~dp0\)\(.*\)$/\1\\..\\..\\bin\2/' \
+        -e 's/^\(call "%OSGEO4W_ROOT%\\\).*\(\\etc\\env\.bat"\)$/\1opt\\grass\2/' \
+        -e 's/@POSTFIX@/'$ver'/g' \
+        mswindows/osgeo4w/grass.bat.tmpl
+) >$grass_path/grass$ver.bat
+unix2dos $grass_path/grass$ver.bat
+
+if [ "$UNITTEST" ]; then
+    cp $grass_path/grass$ver.bat $grass_path/grass.bat
+    cp -a $bin/grass.py ${SRC}/dist.${ARCH}/$bash_bin/grass
+    dos2unix ${SRC}/dist.${ARCH}/$bash_bin/grass
+    chmod ugo+x ${SRC}/dist.${ARCH}/$bash_bin/grass
+    # Set path for bash if not called from OSGeo shell
+    arch_esc=$(sed 's/[\/\*\.]/\\&/g' <<<"${ARCH}")
+    src_esc=$(sed 's/[\/\*\.]/\\&/g' <<<"${SRC}")
+    osgeo4w_esc=$(sed 's/[\/\*\.]/\\&/g' <<<"${OSGEO4W_ROOT_MSYS}")
+    pythonhome_esc=$(sed 's/[\/\*\.]/\\&/g' <<<"${PYTHONHOME}")
+    sed -i "5s/^/export PATH=\"${osgeo4w_esc}\/bin:\/usr\/bin:\/mingw64\/bin:$src_esc\/dist\.$arch_esc\/bin:$src_esc\/dist\.$arch_esc\/$bash_bin\"\nexport PYTHONHOME=\"${pythonhome_esc}\"/" ${SRC}/.github/workflows/test_simple.sh
+    printf "export PATH=\"${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${SRC}/dist.${ARCH}/$bash_bin\"\nexport PYTHONHOME=\"${PYTHONHOME}\"\nexport PYTHONUTF8=1" > $HOME/.bash_profile
+    printf "export PATH=\"${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${SRC}/dist.${ARCH}/$bash_bin\"\nexport PYTHONHOME=\"${PYTHONHOME}\"\nexport PYTHONUTF8=1" > $HOME/.profile
+    printf "export PATH=\"${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${SRC}/dist.${ARCH}/$bash_bin\"\nexport PYTHONHOME=\"${PYTHONHOME}\"\nexport PYTHONUTF8=1" > $HOME/.bashrc
+fi