Merge pull request #15729 from GordonSmith/HPCC-27124-USE_VCPKG

HPCC-27124 Add Ubuntu and OSX support for vcpkg build

Reviewed-By: Gavin Halliday <gavin.halliday@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

.github/workflows/build-vcpkg.yml

@@ -0,0 +1,136 @@
+name: build-vcpkg
+
+env:
+  VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite"
+
+on:
+  pull_request:
+    branches:
+      - "master"
+      - "candidate-*"
+      - "!candidate-7.10.*"
+      - "!candidate-7.8.*"
+      - "!candidate-7.6.*"
+      - "!candidate-7.4.*"
+      - "!candidate-7.2.*"
+      - "!candidate-7.0.*"
+      - "!candidate-6.*"
+
+jobs:
+  pre_job:
+    # continue-on-error: true # Uncomment once integration is finished
+    runs-on: ubuntu-20.04
+    # Map a step output to a job output
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: hpcc-systems/skip-duplicate-actions@master
+        with:
+          github_token: ${{ github.token }}
+          paths_ignore: '["docs/**", "esp/src/**/*", "dockerfiles/**", "helm/**", "ecllibrary/**", "testing/**", "initfiles/**" ]'
+
+  build:
+    strategy:
+      matrix:
+        os: ["ubuntu-20.04", "windows-2019", "macos-10.15"]
+        include:
+          - os: "ubuntu-20.04"
+            triplet: "x64-linux"
+            mono: "mono"
+            sudo: "sudo"
+            cmake_config_options: "-DCMAKE_BUILD_TYPE=Debug"
+            cmake_build_options: "-- -j$(nproc)"
+          - os: "windows-2019"
+            triplet: "x64-windows"
+            mono: ""
+            sudo: ""
+            cmake_config_options: '-G "Visual Studio 16 2019" -T host=x64 -A x64'
+            cmake_build_options: "--config Debug -- -m"
+          - os: "macos-10.15"
+            triplet: "x64-osx"
+            mono: "mono"
+            sudo: "sudo"
+            cmake_config_options: "-DCMAKE_BUILD_TYPE=Debug"
+            cmake_build_options: "-- -j$(nproc)"
+      fail-fast: false
+
+    name: "Build with vcpkg"
+    needs: pre_job
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Display build summary
+        working-directory: .
+        run: |
+          echo ${{ matrix.os }} ${{ matrix.triplet }}
+          echo "Checkout to $Env:GITHUB_WORKSPACE"
+
+      - name: Ubuntu Dependencies
+        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        shell: "bash"
+        run: |
+          sudo apt-get -yq install bison flex build-essential binutils-dev libldap2-dev libcppunit-dev libicu-dev libxslt1-dev \
+            zlib1g-dev libboost-regex-dev libarchive-dev libv8-dev default-jdk libapr1-dev libaprutil1-dev libiberty-dev \
+            libhiredis-dev libtbb-dev libxalan-c-dev libnuma-dev libevent-dev libatlas-base-dev libblas-dev python3-dev \
+            default-libmysqlclient-dev libsqlite3-dev libmemcached-dev libcurl4-openssl-dev pkg-config libtool autotools-dev automake \
+            libssl-dev xmlstarlet
+
+          curl https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo apt-key add -
+          sudo apt-add-repository "deb https://deb.nodesource.com/node_16.x $(lsb_release -sc) main"
+          sudo apt-get update -y
+          sudo apt-get install -y nodejs
+
+      - name: OSX Dependencies
+        if: ${{ matrix.os == 'macos-10.15' }}
+        shell: "bash"
+        run: |
+          brew install bison flex
+          export CMAKE_INCLUDE_PATH="/usr/local/opt/flex/include"
+          export CMAKE_LIBRARY_PATH="/usr/local/opt/flex/lib;/usr/local/opt/bison/lib"
+          export PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH"
+
+          brew install pkg-config automake libtool cppunit boost mysql-client openldap
+          brew tap AdoptOpenJDK/openjdk
+          brew install --cask adoptopenjdk13
+
+      - name: "Remove builtin vcpkg"
+        working-directory: .
+        shell: "bash"
+        run: |
+          ${{ matrix.sudo }} rm -rf "$VCPKG_INSTALLATION_ROOT"
+
+      - name: Check out source code
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: "Bootstrap vcpkg"
+        working-directory: .
+        shell: "bash"
+        run: |
+          ./vcpkg/bootstrap-vcpkg.sh
+
+      - name: "Setup NuGet credentials"
+        working-directory: ./vcpkg
+        shell: "bash"
+        run: |
+          ${{ matrix.mono }} `./vcpkg fetch nuget | tail -n 1` \
+            sources add \
+            -source "https://nuget.pkg.github.com/hpcc-systems/index.json" \
+            -storepasswordincleartext \
+            -name "GitHub" \
+            -username "hpcc-systems" \
+            -password "${{ secrets.GITHUB_TOKEN }}"
+          ${{ matrix.mono }} `./vcpkg fetch nuget | tail -n 1` \
+            setapikey "${{ secrets.GITHUB_TOKEN }}" \
+            -source "https://nuget.pkg.github.com/hpcc-systems/index.json"
+
+      - name: "Build HPCC-Platform"
+        working-directory: .
+        shell: "bash"
+        run: |
+          mkdir build
+          cd build
+          cmake .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake ${{ matrix.cmake_config_options }}
+          cmake --build . ${{ matrix.cmake_build_options }}

.gitignore

@@ -1,6 +1,6 @@
 build/
 node_modules/
-vcpkg/
+vcpkg_installed/
 externals/
 .project
 .svn/

cmake_modules/FindOPENSSL.cmake

@@ -1,78 +0,0 @@
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-
-# - Try to find the OPENSSL library
-# Once done this will define
-#
-#  OPENSSL_FOUND - system has the OPENSSL library
-#  OPENSSL_INCLUDE_DIR - the OPENSSL include directory
-#  OPENSSL_LIBRARIES - The libraries needed to use OPENSSL
-
-IF (NOT OPENSSL_FOUND)
-  IF (WIN32)
-    SET (ssl_lib "libssl.lib")
-  ELSEIF (APPLE)
-    SET (ssl_lib "libssl.dylib")
-  ELSE()
-    SET (ssl_lib "libssl.so")
-  ENDIF()
-  IF (NOT "${EXTERNALS_DIRECTORY}" STREQUAL "")
-    IF (UNIX)
-      IF (${ARCH64BIT} EQUAL 1)
-        SET (osdir "linux64_gcc4.1.1")
-      ELSE()
-        SET (osdir "linux32_gcc4.1.1")
-      ENDIF()
-    ELSEIF(WIN32)
-      IF (${ARCH64BIT} EQUAL 1)
-        SET (osdir "win64")
-      ELSE()
-        SET (osdir "win32")
-      ENDIF()
-    ELSE()
-      SET (osdir "unknown")
-    ENDIF()
-    IF (NOT ("${osdir}" STREQUAL "unknown"))
-      FIND_PATH (OPENSSL_INCLUDE_DIR NAMES openssl/ssl.h PATHS "${EXTERNALS_DIRECTORY}/openssl/1.1/${osdir}/include" NO_DEFAULT_PATH)
-      FIND_LIBRARY (OPENSSL_LIBRARIES NAMES ${ssl_lib} PATHS "${EXTERNALS_DIRECTORY}/openssl/1.1/${osdir}/lib" NO_DEFAULT_PATH)
-    ENDIF()
-  ENDIF()
-
-  if (USE_NATIVE_LIBRARIES)
-    if (APPLE)
-      FIND_PATH (OPENSSL_INCLUDE_DIR NAMES openssl/ssl.h PATHS /opt/local/include /usr/local/include NO_DEFAULT_PATH)
-      FIND_LIBRARY (OPENSSL_LIBRARIES NAMES ${ssl_lib} PATHS /opt/local/lib /usr/local/lib NO_DEFAULT_PATH)
-    else()
-	  # if we didn't find in externals, look in system include path
-	  FIND_PATH (OPENSSL_INCLUDE_DIR NAMES openssl/ssl.h)
-	  FIND_LIBRARY (OPENSSL_LIBRARIES NAMES ${ssl_lib})
-    endif()
-  endif()
-
-  include(FindPackageHandleStandardArgs)
-  find_package_handle_standard_args(OPENSSL DEFAULT_MSG
-    OPENSSL_LIBRARIES 
-    OPENSSL_INCLUDE_DIR
-  )
-  IF (OPENSSL_FOUND)
-    STRING(REPLACE "/${ssl_lib}" "" OPENSSL_LIBRARY_DIR "${OPENSSL_LIBRARIES}")
-    STRING(REPLACE "libssl" "libcrypto" OPENSSL_EXTRA "${OPENSSL_LIBRARIES}")
-    set (OPENSSL_LIBRARIES ${OPENSSL_LIBRARIES} ${OPENSSL_EXTRA} )
-  ENDIF()
-
-  MARK_AS_ADVANCED(OPENSSL_INCLUDE_DIR OPENSSL_LIBRARIES)
-ENDIF()

cmake_modules/FindZLIB.cmake

@@ -1,70 +0,0 @@
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-    
-# - Try to find the ZLib compression library
-# Once done this will define
-#
-#  ZLIB_FOUND - system has the ZLib library
-#  ZLIB_INCLUDE_DIR - the ZLib include directory
-#  ZLIB_LIBRARIES - The libraries needed to use ZLib
-
-IF (NOT ZLIB_FOUND)
-  IF (WIN32)
-    SET (zlib_lib "zlib")
-  ELSE()
-    SET (zlib_lib "z")
-  ENDIF()
-
-  IF (NOT "${EXTERNALS_DIRECTORY}" STREQUAL "")
-    IF (UNIX)
-      IF (${ARCH64BIT} EQUAL 1)
-        SET (osdir "linux64_gcc4.1.1")
-      ELSE()
-        SET (osdir "linux32_gcc4.1.1")
-      ENDIF()
-      SET (zlibver "1.2.5")
-    ELSEIF(WIN32)
-      IF (${ARCH64BIT} EQUAL 1)
-        SET (osdir "Win64")
-      ELSE()
-        SET (osdir "Win32")
-      ENDIF()  
-      SET (zlibver "1.2.8")
-    ELSE()
-      SET (osdir "unknown")
-      SET (zlibver "unknown")
-    ENDIF()
-    IF (NOT ("${osdir}" STREQUAL "unknown"))
-      FIND_PATH (ZLIB_INCLUDE_DIR NAMES zlib.h PATHS "${EXTERNALS_DIRECTORY}/zlib/${zlibver}/include" NO_DEFAULT_PATH)
-      FIND_LIBRARY (ZLIB_LIBRARIES NAMES ${zlib_lib} PATHS "${EXTERNALS_DIRECTORY}/zlib/${zlibver}/lib/${osdir}" NO_DEFAULT_PATH)
-    ENDIF()
-  ENDIF()
-
-  if (USE_NATIVE_LIBRARIES)
-    # if we didn't find in externals, look in system include path
-    FIND_PATH (ZLIB_INCLUDE_DIR NAMES zlib.h)
-    FIND_LIBRARY (ZLIB_LIBRARIES NAMES ${zlib_lib})
-  endif()
-
-  include(FindPackageHandleStandardArgs)
-  find_package_handle_standard_args(ZLIB DEFAULT_MSG
-    ZLIB_LIBRARIES 
-    ZLIB_INCLUDE_DIR
-  )
-
-  MARK_AS_ADVANCED(ZLIB_INCLUDE_DIR ZLIB_LIBRARIES)
-ENDIF()

cmake_modules/commonSetup.cmake

@@ -133,6 +133,17 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(INCLUDE_TREEVIEW "Build legacy treeview" OFF)
   option(INCLUDE_CONFIG_MANAGER "Build config manager" ON)
   option(USE_ELASTICSTACK_CLIENT "Configure use of Elastic Stack client" ON)
+if (WIN32)
+  option(USE_JWT "Enable JSON Web Tokens" OFF)
+else ()
+  option(USE_JWT "Enable JSON Web Tokens" ON)
+endif ()
+#########################################################
+
+  if (VCPKG_APPLOCAL_DEPS)
+    include(${HPCC_SOURCE_DIR}/cmake_modules/vcpkgSetup.cmake)
+  endif ()
+
   set(CUSTOM_PACKAGE_SUFFIX "" CACHE STRING "Custom package suffix to differentiate development builds")
 
      MACRO(SET_PLUGIN_PACKAGE plugin)
@@ -677,12 +688,37 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     endif()
 
     IF ( NOT MAKE_DOCS_ONLY )
+      # On macOS, search Homebrew for keg-only versions of Bison and Flex. Xcode does
+      # not provide new enough versions for us to use.
+      if (CMAKE_HOST_SYSTEM_NAME MATCHES "Darwin")
+        execute_process(
+            COMMAND brew --prefix bison
+            RESULT_VARIABLE BREW_BISON
+            OUTPUT_VARIABLE BREW_BISON_PREFIX
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+        )
+        if (BREW_BISON EQUAL 0 AND EXISTS "${BREW_BISON_PREFIX}")
+            message(STATUS "Found Bison keg installed by Homebrew at ${BREW_BISON_PREFIX}")
+            set(BISON_EXECUTABLE "${BREW_BISON_PREFIX}/bin/bison")
+        endif()
+
+        execute_process(
+            COMMAND brew --prefix flex
+            RESULT_VARIABLE BREW_FLEX
+            OUTPUT_VARIABLE BREW_FLEX_PREFIX
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+        )
+        if (BREW_FLEX EQUAL 0 AND EXISTS "${BREW_FLEX_PREFIX}")
+          message(STATUS "Found Flex keg installed by Homebrew at ${BREW_FLEX_PREFIX}")
+          set(FLEX_EXECUTABLE "${BREW_FLEX_PREFIX}/bin/flex")
+        endif ()
+      endif ()
       FIND_PACKAGE(BISON)
       FIND_PACKAGE(FLEX)
       IF ( BISON_FOUND AND FLEX_FOUND )
         SET(BISON_exename ${BISON_EXECUTABLE})
         SET(FLEX_exename ${FLEX_EXECUTABLE})
-        IF (WIN32)
+        IF (WIN32 OR APPLE)
           SET(bisoncmd ${BISON_exename})
           SET(flexcmd ${FLEX_exename})
         ELSE()
@@ -958,7 +994,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       endif(USE_BOOST_REGEX)
 
       if(USE_OPENSSL)
-        find_package(OPENSSL)
+        find_package(OpenSSL)
         if (OPENSSL_FOUND)
           add_definitions (-D_USE_OPENSSL)
           include_directories(${OPENSSL_INCLUDE_DIR})

cmake_modules/vcpkgSetup.cmake

@@ -0,0 +1,32 @@
+set (USE_OPENLDAP OFF)
+set (USE_ICU ON)
+set (USE_LIBXSLT ON)
+set (USE_OPENSSL ON)
+find_package(OpenSSL REQUIRED)
+
+set (USE_CBLAS OFF)
+set (USE_ZLIB ON)
+find_package(ZLIB REQUIRED)
+
+set (USE_LIBARCHIVE OFF)
+set (USE_BOOST_REGEX OFF)
+
+set (USE_APR OFF)
+set (USE_AZURE OFF)
+set (USE_AWS OFF)
+set (WSSQL_SERVICE OFF)
+set (USE_PYTHON3 OFF)
+set (USE_PYTHON2 OFF)
+set (USE_CASSANDRA OFF)
+set (USE_TBB OFF)
+set (USE_JAVA OFF)
+set (USE_NATIVE_LIBRARIES ON)
+set (INCLUDE_CONFIG_MANAGER OFF)
+set (INCLUDE_TREEVIEW OFF)
+set (CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
+
+# Additional
+option(BUILD_TESTS "Enable libgit2 tests (override libgit2 option)" ON)
+set (BUILD_TESTS OFF)
+set (USE_JWT OFF)
+

esp/platform/espcontext.cpp

@@ -459,12 +459,14 @@ public:
 
     virtual void ensureSuperUser(unsigned excCode, const char* excMsg)
     {
+#ifdef _USE_OPENLDAP
         CLdapSecManager* secmgr = dynamic_cast<CLdapSecManager*>(m_secmgr.get());
         if (secmgr && !secmgr->isSuperUser(m_user.get()))
         {
             setAuthStatus(AUTH_STATUS_NOACCESS);
             throw makeStringException(excCode, excMsg);
         }
+#endif
     }
 
     void AuditMessage(AuditType type, const char *filterType, const char *title, const char *parms, ...) __attribute__((format(printf, 5, 6)));

esp/protocols/http/CMakeLists.txt

@@ -71,7 +71,9 @@ include_directories(
     )
 
 add_definitions(-DESPHTTP_EXPORTS -DESP_TIMING -D_USRDLL -DESP_PLUGIN -DTXSUMMARY_EXPORTS)
-
+if(USE_OPENLDAP)
+    add_definitions(-D_USE_OPENLDAP)
+endif()
 HPCC_ADD_LIBRARY(esphttp SHARED ${SRCS})
 add_dependencies(esphttp  espscm)
 install(TARGETS esphttp RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR})

esp/services/ws_access/CMakeLists.txt

@@ -51,8 +51,10 @@ include_directories (
          ${HPCC_SOURCE_DIR}/common/thorhelper
     )
 
-# NOTE - this should not be needed, it's the result of poor encapsulation and using CLdapSecManager directly 
-include_directories ( ${OPENLDAP_INCLUDE_DIR} )
+if(USE_OPENLDAP)
+    # NOTE - this should not be needed, it's the result of poor encapsulation and using CLdapSecManager directly 
+    include_directories ( ${OPENLDAP_INCLUDE_DIR} )
+endif()
 
 ADD_DEFINITIONS( -D_USRDLL )
 

system/security/CMakeLists.txt

@@ -28,6 +28,6 @@ IF (USE_APR)
   HPCC_ADD_SUBDIRECTORY (plugins/htpasswdSecurity)
 ENDIF(USE_APR)
 HPCC_ADD_SUBDIRECTORY (plugins/singleuserSecurity)
-IF (NOT WIN32)
+IF (USE_JWT)
   HPCC_ADD_SUBDIRECTORY (plugins/jwtSecurity)
 ENDIF()

system/security/cryptohelper/digisign.cpp

@@ -206,7 +206,9 @@ public:
 
 
 static IDigitalSignatureManager * dsm = nullptr;
+#if defined(_USE_OPENSSL) && !defined(_WIN32)
 static std::once_flag dsmInitFlag;
+#endif
 
 MODULE_INIT(INIT_PRIORITY_STANDARD)
 {

vcpkg

@@ -1 +1 @@
-Subproject commit 6114d10957ab676b795a595468fc779f50aba573
+Subproject commit 892d5a293a7911cd9dd47b52ae5bf44d88e52d2a

vcpkg.json

@@ -0,0 +1,27 @@
+{
+    "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg/master/scripts/vcpkg.schema.json",
+    "name": "hpcc-platform",
+    "version": "8.6.0",
+    "dependencies": [
+        {
+            "name": "icu",
+            "platform": "windows"
+        },
+        {
+            "name": "zlib",
+            "platform": "windows | osx"
+        },
+        {
+            "name": "openssl",
+            "platform": "windows | osx"
+        },
+        {
+            "name": "libxslt",
+            "platform": "windows"
+        },
+        {
+            "name": "winflexbison",
+            "platform": "windows"
+        }
+    ]
+}