Merge pull request #8644 from jamienoss/issue12635-plugin-embed-example

HPCC-12635 Add example plugin

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

CMakeLists.txt

@@ -127,6 +127,7 @@ if ( PLUGIN )
     HPCC_ADD_SUBDIRECTORY (plugins/kafka "KAFKA")
     HPCC_ADD_SUBDIRECTORY (plugins/sqlite3 "SQLITE3EMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/mysql "MYSQLEMBED")
+    HPCC_ADD_SUBDIRECTORY (plugins/exampleplugin "EXAMPLEPLUGIN")
 elseif ( NOT MAKE_DOCS_ONLY )
     HPCC_ADD_SUBDIRECTORY (initfiles)
     HPCC_ADD_SUBDIRECTORY (tools)

cmake_modules/FindEXAMPLE_PLUGIN_DEP.cmake

@@ -0,0 +1,91 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# - Try to find any DEPendencies required for the exampleplugin
+# Once done this will define
+#
+#  EXAMPLE_PLUGIN_DEP_FOUND - system has the dependencies
+#  EXAMPLE_PLUGIN_DEP_INCLUDE_DIR - the dependency include directory
+#  EXAMPLE_PLUGIN_DEP_LIBRARIES - The dependency libraries needed.
+
+IF (NOT EXAMPLE_PLUGIN_DEP_FOUND)
+  #The following two libraries are required dependencies for this plugin: libhpcc-example-plugin-deps_1 and libhpcc-example-plugin-deps_2.
+  IF (WIN32)
+    SET (libdep_1 "libhpcc-example-plugin-deps_1")
+    SET (libdep_1 "libhpcc-example-plugin-deps_2")
+  ELSE()
+    SET (libdep_1 "hpcc-example-plugin-deps_1")
+    SET (libdep_2 "hpcc-example-plugin-deps_2")
+  ENDIF()
+
+  #Find the path to any required include file
+  FIND_PATH (EXAMPLE_PLUGIN_DEP_INCLUDE_DIR hpcc-example-plugin-deps PATHS /usr/include /usr/share/include /usr/local/include PATH_SUFFIXES dep)
+  #Find the path to any required libraries, in this example there are two.
+  FIND_LIBRARY (EXAMPLE_PLUGIN_DEP_LIBRARY_1 NAMES ${libdep_1} PATHS /usr/lib /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64)
+  FIND_LIBRARY (EXAMPLE_PLUGIN_DEP_LIBRARY_2 NAMES ${libdep_2} PATHS /usr/lib /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64)
+  SET (EXAMPLE_PLUGIN_DEP_LIBRARIES ${EXAMPLE_PLUGIN_DEP_LIBRARY_1} ${EXAMPLE_PLUGIN_DEP_LIBRARY_2})
+
+  #The required include file may contain certain definitions for the major, minor, and patch versions,
+  #in which case this can be extracted such that a minimum version check can be conducted at cmake
+  #configuration time. In this example the following is being extracted from a C header file:
+  #"#define EXAMPLE_PLUGIN_DEP_MAJOR 5"
+  #"#define EXAMPLE_PLUGIN_DEP_MINOR 4"
+  #"#define EXAMPLE_PLUGIN_DEP_PATCH 2"
+  #The minimum version to requirement is made visible to this cmake file from
+  #HPCC/plugins/exampleplugin/CMakeLists.txt:ADD_PLUGIN(EXAMPLEPLUGIN PACKAGES EXAMPLE_PLUGIN_DEP MINVERSION 4.6.2)
+  IF  (EXAMPLE_PLUGIN_DEP_INCLUDE_DIR)
+    #MAJOR
+    FILE (STRINGS "${EXAMPLE_PLUGIN_DEP_INCLUDE_DIR}/hpcc-example-plugin-deps.h" major REGEX "#define EXAMPLE_PLUGIN_DEP_MAJOR")
+    STRING (REGEX REPLACE "#define EXAMPLE_PLUGIN_DEP_MAJOR " "" major "${major}")
+    STRING (REGEX REPLACE "\"" "" major "${major}")
+    #MINOR
+    FILE (STRINGS "${EXAMPLE_PLUGIN_DEP_INCLUDE_DIR}/deps.h" minor REGEX "#define EXAMPLE_PLUGIN_DEP_MINOR")
+    STRING (REGEX REPLACE "#define EXAMPLE_PLUGIN_DEP_MINOR " "" minor "${minor}")
+    STRING (REGEX REPLACE "\"" "" minor "${minor}")
+    #PATCH
+    FILE (STRINGS "${EXAMPLE_PLUGIN_DEP_INCLUDE_DIR}/deps.h" patch REGEX "#define EXAMPLE_PLUGIN_DEP_PATCH")
+    STRING (REGEX REPLACE "#define EXAMPLE_PLUGIN_DEP_PATCH " "" patch "${patch}")
+    STRING (REGEX REPLACE "\"" "" patch "${patch}")
+
+    SET (EXAMPLE_PLUGIN_DEP_VERSION_STRING "${major}.${minor}.${patch}")
+    IF ("${EXAMPLE_PLUGIN_DEP_VERSION_STRING}" VERSION_LESS "${EXAMPLE_PLUGIN_DEP_FIND_VERSION}")
+      SET(MSG "libhpcc-example-plugin-deps version '${EXAMPLE_PLUGIN_DEP_VERSION_STRING}' incompatible with min version>=${EXAMPLE_PLUGIN_DEP_FIND_VERSION}")
+    ELSE()
+      SET (EXAMPLE_PLUGIN_DEP_VERSION_OK 1)
+      SET (MSG "${DEFAULT_MSG}")
+    ENDIF()
+  ENDIF()
+
+  #The following three lines are used for building this example as part of the HPCC regression suite.
+  #They are not intended to be present as part of this example itself and should be removed.
+  SET (EXAMPLE_PLUGIN_DEP_LIBRARIES "jlib")
+  SET (EXAMPLE_PLUGIN_DEP_INCLUDE_DIR "./")
+  SET (EXAMPLE_PLUGIN_DEP_VERSION_OK 1)
+
+  #If the following three variables (EXAMPLE_PLUGIN_DEP_LIBRARIES, EXAMPLE_PLUGIN_DEP_INCLUDE_DIR, &
+  #EXAMPLE_PLUGIN_DEP_VERSION_OK) have not been set, MAKE_EXAMPLEPLUGIN will be set false when
+  #returning to HPCC/plugins/exampleplugin/CMakeLists.txt and thus this plugin will not be built
+  #and a message will be given explaining which of three where missing or give the above version
+  #incompatible message.
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(example_plugin_dep ${MSG}
+    EXAMPLE_PLUGIN_DEP_LIBRARIES
+    EXAMPLE_PLUGIN_DEP_INCLUDE_DIR
+    EXAMPLE_PLUGIN_DEP_VERSION_OK
+  )
+
+  MARK_AS_ADVANCED(EXAMPLE_PLUGIN_DEP_INCLUDE_DIR EXAMPLE_PLUGIN_DEP_LIBRARIES)
+ENDIF()

cmake_modules/commonSetup.cmake

@@ -109,6 +109,8 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(JAVAEMBED "Create a package with ONLY the javaembed plugin" OFF)
   option(SQLITE3EMBED "Create a package with ONLY the sqlite3embed plugin" OFF)
   option(KAFKA "Create a package with ONLY the kafkaembed plugin" OFF)
+  #"cmake -DEXAMPLEPLUGIN=ON <path-to/HPCC-Platform/>" will configure the plugin makefiles to be built with "make".
+  option(EXAMPLEPLUGIN "Create a package with ONLY the exampleplugin plugin" OFF)
 
   if (APPLE OR WIN32)
       option(USE_TBB "Enable Threading Building Block support" OFF)
@@ -123,7 +125,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
 
 
     if(REMBED OR V8EMBED OR MEMCACHED OR PYEMBED OR REDIS OR JAVAEMBED OR MYSQLEMBED
-        OR SQLITE3EMBED OR KAFKA)
+        OR SQLITE3EMBED OR KAFKA OR EXAMPLEPLUGIN)
         set(PLUGIN ON)
         set(CLIENTTOOLS OFF)
         set(PLATFORM OFF)
@@ -194,6 +196,13 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
             set(pluginname "kafka")
         endif()
     endif()
+    if(EXAMPLEPLUGIN)
+        if(DEFINED pluginname)
+            message(FATAL_ERROR "Cannot enable exampleplugin, already declared ${pluginname}")
+        else()
+           set(pluginname "exampleplugin")
+        endif()
+    endif()
 
   if ( USE_XALAN AND USE_LIBXSLT )
       set(USE_LIBXSLT OFF)
@@ -231,6 +240,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
         set(JAVAEMBED ON)
         set(SQLITE3EMBED ON)
         set(KAFKA ON)
+        set(EXAMPLEPLUGIN ON)
     endif()
 
   option(PORTALURL "Set url to hpccsystems portal download page")

plugins/CMakeLists.txt

@@ -34,3 +34,4 @@ add_subdirectory (cassandra)
 add_subdirectory (memcached)
 add_subdirectory (redis)
 add_subdirectory (kafka)
+add_subdirectory (exampleplugin)

plugins/exampleplugin/CMakeLists.txt

@@ -0,0 +1,84 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# Component: exampleplugin
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for exampleplugin
+#####################################################
+
+project(exampleplugin)
+
+if(EXAMPLEPLUGIN)
+    #ADD_PLUGIN(...) executes HPCC/cmake_modules/FindEXAMPLE_PLUGIN_DEP.cmake to find any required DEPendencies such
+    #as include files and libraries to link against. MAKE_EXAMPLEPLUGIN is set true if all the required dependencies
+    #set in HPCC/cmake_modules/FindEXAMPLE_PLUGIN_DEP.cmake are met.
+    #Both EXAMPLE_PLUGIN_DEP_INCLUDE_DIR and EXAMPLE_PLUGIN_DEP_LIBRARIES are also found and set in
+    #HPCC/cmake_modules/FindEXAMPLE_PLUGIN_DEP.cmake.
+    ADD_PLUGIN(exampleplugin PACKAGES EXAMPLE_PLUGIN_DEP MINVERSION 0.1.0)
+
+    if(MAKE_EXAMPLEPLUGIN)
+        set(
+            SRCS
+            #Add all sources to be compiled here.
+            exampleplugin.hpp
+            exampleplugin.cpp)
+
+        include_directories(
+            #Add any required HPCC include dirs here. EXAMPLE_PLUGIN_DEP_INCLUDE_DIR contains any and all external (non HPCC) paths.
+            ./../../system/include
+            ./../../rtl/eclrtl
+            ./../../rtl/include
+            ./../../common/deftype
+            ./../../system/jlib
+            ${EXAMPLE_PLUGIN_DEP_INCLUDE_DIR})
+
+        add_definitions(-D_USRDLL -DECL_EXMAPLE_PLUGIN_EXPORTS)
+
+        HPCC_ADD_LIBRARY(exampleplugin SHARED ${SRCS})
+        if(${CMAKE_VERSION} VERSION_LESS "2.8.9")
+            message(WARNING "Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+        elseif(NOT APPLE)
+            set_target_properties(exampleplugin PROPERTIES NO_SONAME 1)
+        endif()
+
+        install(
+            TARGETS exampleplugin
+            DESTINATION plugins)
+
+        target_link_libraries(
+            exampleplugin
+            #Add any required HPCC libraries to be linked against here. EXAMPLE_PLUGIN_DEP_LIBRARIES contains any and all external (non HPCC) libraries.
+            eclrtl
+            jlib
+            ${EXAMPLE_PLUGIN_DEP_LIBRARIES})
+    endif()
+endif()
+
+#Here the ECL plugin/service definitions contained with *.ecllib are set to be installed with the platform regardless
+#of whether the plugin itself was configured to be built. This is because the plugin libraries are dynamically linked
+#with the platform such that ECL queries can be compiled without the plugin needing to be installed locally, e.g. as
+#it the case for eclserver. Adequate tests must be conducted to ensure against undefined references. In addition,
+#this means that any changes to the plugin/service definitions must be correlated with the correct platform build
+#version when installing.
+if(PLATFORM)
+    install(
+        FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib_exampleplugin.ecllib
+        DESTINATION plugins
+        COMPONENT Runtime)
+endif()

plugins/exampleplugin/README.md

@@ -0,0 +1,53 @@
+ECL Example Plugin
+================
+
+This is the ECL plugin to utilize the Blah Blah Blah.
+It utilizes the API Blah Blah Blah.
+
+Installation and Dependencies
+----------------------------
+
+To build the <plugin name> plugin with the HPCC-Platform, <dependency> is required.
+```
+sudo apt-get install <dependency>
+```
+
+*Note:* Add notes such as min versions etc.
+
+
+Getting started
+---------------
+
+Blah Blah Blah.
+
+The Actual Plugin
+-----------------
+
+Blah Blah Blah.
+
+###Sub Section
+
+Blah Blah Blah.
+
+###An ECL Example
+```c
+IMPORT example-plugin FROM lib_example-plugin;
+IMPORT Wrapper FROM lib_example-plugin;
+
+myWrapper := Wrapper('param1', 'param2');
+
+UNSIGNED4 param3 := 7;
+example1 := myWrapper.func1(param3);
+example2 := myWrapper.func2('MaKEALLlOWeRCASe');
+
+OUTPUT(example1);
+OUTPUT(example2);
+```
+Yields the results '8' and 'makealllowercase'
+
+etc etc.
+
+Behaviour and Implementation Details
+------------------------------------
+
+Blah Blah Blah.

plugins/exampleplugin/exampleplugin.cpp

@@ -0,0 +1,67 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "eclrtl.hpp"
+#include "jstring.hpp"
+#include "exampleplugin.hpp"
+
+#define EXAMPLE_PLUGIN_VERSION "example-plugin plugin 1.0.0"
+ECL_EXAMPLE_PLUGIN_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    /*  Warning:    This function may be called without the plugin being loaded fully.
+     *              It should not make any library calls or assume that dependent modules
+     *              have been loaded or that it has been initialised.
+     *
+     *              Specifically:  "The system does not call DllMain for process and thread
+     *              initialization and termination.  Also, the system does not load
+     *              additional executable modules that are referenced by the specified module."
+     */
+
+    if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = EXAMPLE_PLUGIN_VERSION;
+    pb->moduleName = "lib_redis";
+    pb->ECL = NULL;
+    pb->flags = PLUGIN_IMPLICIT_MODULE;
+    pb->description = "ECL plugin library for BLAH BLAH BLAH\n";
+    return true;
+}
+
+namespace ExamplePlugin {
+
+
+//--------------------------------------------------------------------------------
+//                           ECL SERVICE ENTRYPOINTS
+//--------------------------------------------------------------------------------
+ECL_EXAMPLE_PLUGIN_API unsigned ECL_EXAMPLE_PLUGIN_CALL func1(ICodeContext * ctx, const char * param1, const char * param2, unsigned param3)
+{
+    return param3 + 1;
+}
+
+ECL_EXAMPLE_PLUGIN_API void ECL_EXAMPLE_PLUGIN_CALL func2 (ICodeContext * _ctx, size32_t & returnLength, char * & returnValue, const char * param1, const char * param2, size32_t param3ValueLength, const char * param3Value)
+{
+    StringBuffer buffer(param3Value);
+    buffer.toLowerCase();
+    returnLength = buffer.length();
+    returnValue = buffer.detach();
+    return;
+}
+
+}//close namespace

plugins/exampleplugin/exampleplugin.hpp

@@ -0,0 +1,50 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef ECL_EXAMPLE_PLUGIN_SYNC_INCL
+#define ECL_EXAMPLE_PLUGIN_SYNC_INCL
+
+#ifdef _WIN32
+#define ECL_EXAMPLE_PLUGIN_CALL _cdecl
+#ifdef ECL_EXAMPLE_PLUGIN_EXPORTS
+#define ECL_EXAMPLE_PLUGIN_API __declspec(dllexport)
+#else
+#define ECL_EXAMPLE_PLUGIN_API __declspec(dllimport)
+#endif
+#else
+#define ECL_EXAMPLE_PLUGIN_CALL
+#define ECL_EXAMPLE_PLUGIN_API
+#endif
+
+#include "hqlplugins.hpp"
+#include "eclhelper.hpp"
+
+extern "C"
+{
+    ECL_EXAMPLE_PLUGIN_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
+    ECL_EXAMPLE_PLUGIN_API void setPluginContext(IPluginContext * _ctx);
+}
+
+extern "C++"
+{
+namespace ExamplePlugin {
+    //--------------------------SET----------------------------------------
+    ECL_EXAMPLE_PLUGIN_API unsigned ECL_EXAMPLE_PLUGIN_CALL func1  (ICodeContext * _ctx, const char * param1, const char * param2, unsigned param3);
+    ECL_EXAMPLE_PLUGIN_API void ECL_EXAMPLE_PLUGIN_CALL func2 (ICodeContext * _ctx, size32_t & returnLength, char * & returnValue, const char * param1, const char * param2, size32_t param3ValueLength, const char * param3Value);
+}
+}
+#endif

plugins/exampleplugin/lib_exampleplugin.ecllib

@@ -0,0 +1,27 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the License);
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an AS IS BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+EXPORT exampleplugin := SERVICE : plugin('exampleplugin'), namespace('ExamplePlugin')
+  UNSIGNED4 func1(CONST VARSTRING param1, CONST VARSTRING param2, UNSIGNED4 param3) : cpp,action,context,entrypoint='func1';
+  STRING func2(CONST VARSTRING param1, CONST VARSTRING param2, CONST STRING param3) : cpp,action,context,entrypoint='func2';
+END;
+
+EXPORT Wrapper(VARSTRING param1, VARSTRING  param2 = '') := MODULE
+  EXPORT func1(UNSIGNED4 param3) := exampleplugin.func1(param1, param2, param3);
+  EXPORT func2(STRING param3)    := exampleplugin.func2(param1, param2, param3);
+END;

testing/regress/ecl/exampleplugin.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=embedded
+//class=3rdparty
+
+IMPORT * FROM lib_exampleplugin;
+
+OUTPUT(lib_exampleplugin.exampleplugin.func1('not used', 'also not used', 6));
+
+myWrapper := lib_exampleplugin.Wrapper('not used');
+OUTPUT(myWrapper.func1(3));
+
+OUTPUT(myWrapper.func2('alL TO LoWer CaSE'));

testing/regress/ecl/key/exampleplugin.xml

@@ -0,0 +1,9 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>7</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>4</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>all to lower case</Result_3></Row>
+</Dataset>