HPCC-17199 The python2 and python3 plugins should not be loaded together

Because the libpython exported symbols are the same in both libraries, loading
both at the same time can be problematical (it may work in some cases on some
distros, but not all).

However we want to include python support by default in the platform making it
hard to say "just install one".

Therefore I have introduced a mechanism whereby plugins can be
enabled/disabled via the environment.conf file. By default the python3 plugin
will be present but disabled, and the python2 plugin will be present and
enabled. But people wanting to use Python3 can modify their environment.conf
files accordingly.

By removing the .so's from the pluginpath, eclcc/eclserver should generate
link errors if anyone tries to use a disabled plugin.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

CMakeLists.txt

@@ -150,7 +150,7 @@ if ( PLUGIN )
     HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/memcached "MEMCACHED")
-    HPCC_ADD_SUBDIRECTORY (plugins/pyembed "PYEMBED")
+    HPCC_ADD_SUBDIRECTORY (plugins/pyembed "PY2EMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/py3embed "PY3EMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/redis "REDIS")
     HPCC_ADD_SUBDIRECTORY (plugins/javaembed "JAVAEMBED")

cmake_modules/commonSetup.cmake

@@ -141,7 +141,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     REMBED
     V8EMBED
     MEMCACHED
-    PYEMBED
+    PY2EMBED
     PY3EMBED
     REDIS
     MYSQLEMBED

common/dllserver/CMakeLists.txt

@@ -41,6 +41,8 @@ include_directories (
          ./../../system/jlib 
          ./../../common/environment
          ./../../system/security/shared
+         ${CMAKE_BINARY_DIR}
+         ${CMAKE_BINARY_DIR}/oss
     )
 
 IF (NOT WIN32)

common/dllserver/thorplugin.cpp

@@ -21,8 +21,10 @@
 #include "jsocket.hpp"
 #include "jprop.hpp"
 #include "jdebug.hpp"
+#include "jregexp.hpp"
 #include "jlzw.hpp"
 #include "eclrtl.hpp"
+#include "build-config.h"
 #if defined(__APPLE__)
 #include <mach-o/getsect.h>
 #include <sys/mman.h>
@@ -612,11 +614,35 @@ extern DLLSERVER_API bool getManifestXMLFromFile(const char *filename, StringBuf
     return getResourceXMLFromFile(filename, "MANIFEST", 1000, xml);
 }
 
-
 //-------------------------------------------------------------------------------------------------------------------
 
-
-//-------------------------------------------------------------------------------------------------------------------
+extern DLLSERVER_API void getAdditionalPluginsPath(StringBuffer &pluginsPath, const char *_base)
+{
+    // We only add the additional plugins if the plugins path already includes the default plugins location
+    StringBuffer base(_base);
+    removeTrailingPathSepChar(base);
+    StringBuffer defaultLocation(base);
+    defaultLocation.append(PATHSEPSTR "plugins");
+    StringArray paths;
+    paths.appendList(pluginsPath, ENVSEPSTR);
+    if (paths.contains(defaultLocation))
+    {
+        const char *additional = queryEnvironmentConf().queryProp("additionalPlugins");
+        if (additional)
+        {
+            StringArray additionalPaths;
+            additionalPaths.appendList(additional, ENVSEPSTR);
+            ForEachItemIn(idx, additionalPaths)
+            {
+                const char *additionalPath = additionalPaths.item(idx);
+                pluginsPath.append(ENVSEPCHAR);
+                if (!isAbsolutePath(additionalPath))
+                    pluginsPath.append(base).append(PATHSEPSTR "versioned" PATHSEPSTR);
+                pluginsPath.append(additionalPath);
+            }
+        }
+    }
+}
 
 bool SafePluginMap::addPlugin(const char *path, const char *dllname)
 {

common/dllserver/thorplugin.hpp

@@ -48,6 +48,14 @@ extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, Mem
 extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data);
 extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress);
 
+/**
+ * Check with environment.conf to see whether to add any of the versioned plugin directories
+ *
+ * @param pluginsPath   The additional plugin locations are appended to this string
+ * @param base          The home directory - relative paths are assumed to be relative to base/versioned
+ */
+extern DLLSERVER_API void getAdditionalPluginsPath(StringBuffer &pluginsPath, const char *base);
+
 class DLLSERVER_API SimplePluginCtx : implements IPluginContextEx
 {
 public:

ecl/eclcc/CMakeLists.txt

@@ -35,6 +35,7 @@ include_directories (
          ${CMAKE_BINARY_DIR}/oss
          ./../../ecl/hqlcpp 
          ./../../common/workunit 
+         ./../../common/dllserver 
          ./../../common/deftype 
          ./../../system/include 
          ./../../ecl/hql
@@ -67,6 +68,7 @@ target_link_libraries ( eclcc
          thorhelper 
          hql
          hqlcpp
+         dllserver
     )
 
 IF (USE_ZLIB)

ecl/eclcc/eclcc.cpp

@@ -24,7 +24,7 @@
 
 #include "build-config.h"
 #include "workunit.hpp"
-
+#include "thorplugin.hpp"
 #ifndef _WIN32
 #include <pwd.h>
 #endif
@@ -587,10 +587,14 @@ void EclCC::loadOptions()
 #else
         extractOption(compilerPath, globals, "CL_PATH", "compilerPath", "/usr", NULL);
 #endif
-        if (!extractOption(libraryPath, globals, "ECLCC_LIBRARY_PATH", "libraryPath", syspath, "lib"))
-            libraryPath.append(ENVSEPCHAR).append(syspath).append("plugins");
         extractOption(cppIncludePath, globals, "ECLCC_INCLUDE_PATH", "includePath", syspath, "componentfiles" PATHSEPSTR "cl" PATHSEPSTR "include");
         extractOption(pluginsPath, globals, "ECLCC_PLUGIN_PATH", "plugins", syspath, "plugins");
+        getAdditionalPluginsPath(pluginsPath, syspath);
+        if (!extractOption(libraryPath, globals, "ECLCC_LIBRARY_PATH", "libraryPath", syspath, "lib"))
+        {
+            libraryPath.append(ENVSEPCHAR).append(syspath).append("plugins");
+            getAdditionalPluginsPath(libraryPath, syspath);
+        }
         extractOption(hooksPath, globals, "HPCC_FILEHOOKS_PATH", "filehooks", syspath, "filehooks");
         extractOption(templatePath, globals, "ECLCC_TPL_PATH", "templatePath", syspath, "componentfiles");
         extractOption(eclLibraryPath, globals, "ECLCC_ECLLIBRARY_PATH", "eclLibrariesPath", syspath, "share" PATHSEPSTR "ecllibrary" PATHSEPSTR);

ecl/hql/hqlfold.cpp

@@ -685,32 +685,11 @@ void *loadExternalEntryPoint(IHqlExpression* expr, unsigned foldOptions, ITempla
     IHqlExpression * funcdef = expr->queryExternalDefinition();
     IHqlExpression *body = funcdef->queryChild(0);
     // Get the handle to the library and procedure.
-#ifdef __APPLE__
+#ifndef _WIN32
     StringBuffer fullLibraryPath;
-    // OSX is not good at finding eclrtl. This hack is a workaround
-    if (streq(library, "libeclrtl.dylib"))
-    {
-        Dl_info info;
-        if (dladdr((const void *) rtlStrToUInt4, &info))  // Any function in eclrtl would do...
-        {
-            fullLibraryPath.set(info.dli_fname);
-            library = fullLibraryPath.str();
-        }
-    }
-#ifdef _DEBUG
-    if (streq(library, "libpy3embed.dylib") || streq(library, "libpyembed.dylib") || streq(library, "libv8embed.dylib") || streq(library, "libjavaembed.dylib"))
-    {
-        Dl_info info;
-        if (dladdr((const void *) rtlStrToUInt4, &info))  // Any function in eclrtl would do...
-        {
-            fullLibraryPath.set(info.dli_fname);
-            fullLibraryPath.replaceString("libeclrtl.dylib", library);
-            library = fullLibraryPath.str();
-        }
-    }
-#endif
+    if (findLoadedModule(fullLibraryPath, library))
+        library = fullLibraryPath.str();
 #endif
-
     hDLL=LoadSharedObject(library, false, false);
     if (!LoadSucceeded(hDLL))
     {

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -457,9 +457,9 @@ void EsdlServiceImpl::configureTargets(IPropertyTree *cfg, const char *service)
             m_pServiceMethodTargets->addPropTree("Target", createPTreeFromIPT(&itns->query()));
 
         StringBuffer classPath;
-        Owned<IProperties> envConf = createProperties(CONFIG_DIR PATHSEPSTR "environment.conf", true);
-        if (envConf && envConf->hasProp("classpath"))
-            envConf->getProp("classpath", classPath);
+        const IProperties &envConf = queryEnvironmentConf();
+        if (envConf.hasProp("classpath"))
+            envConf.getProp("classpath", classPath);
         else
             classPath.append(INSTALL_DIR).append(PATHSEPCHAR).append("classes");
 

initfiles/etc/DIR_NAME/environment.conf.in

@@ -34,3 +34,14 @@ mpTraceLevel=0
 #dfsSSLPrivateKeyFile=/keyfilepath/keyfile
 jvmoptions=-XX:-UsePerfData
 #JNI_PATH=/absolute/path/to/alternative/libjvm.so
+
+# Although HPCC platform includes plugins for both Python2 and Python3, only one may be safely enabled at a time
+# as the Python libraries export the same symbols for both versions. Enabling both may lead to unpredicatable results
+# including segfaults or undefined symbol errors.
+#
+# If you would prefer to use python 3 and disable python2, change the line below to read
+#  additionalPlugins=python3
+#
+# Multiple paths can be specified (separate with :, or ; on Windows).
+# Relative paths are assumed to be relative to ${INSTALL_DIR}/versioned
+additionalPlugins=python2

plugins/javaembed/javaembed.cpp

@@ -98,12 +98,10 @@ public:
         {
             newPath.append(origPath).append(ENVSEPCHAR);
         }
-        StringBuffer envConf;
-        envConf.append(CONFIG_DIR).append(PATHSEPSTR).append("environment.conf");
-        Owned<IProperties> conf = createProperties(envConf.str(), true);
-        if (conf && conf->hasProp("classpath"))
+        const IProperties &conf = queryEnvironmentConf();
+        if (conf.hasProp("classpath"))
         {
-            conf->getProp("classpath", newPath);
+            conf.getProp("classpath", newPath);
             newPath.append(ENVSEPCHAR);
         }
         else
@@ -113,21 +111,21 @@ public:
         newPath.append(".");
         optionStrings.append(newPath);
 
-        if (conf && conf->hasProp("jvmlibpath"))
+        if (conf.hasProp("jvmlibpath"))
         {
             StringBuffer libPath;
             libPath.append("-Djava.library.path=");
-            conf->getProp("jvmlibpath", libPath);
+            conf.getProp("jvmlibpath", libPath);
             optionStrings.append(libPath);
         }
 
         // Options we should set (but allow for override with jvmoptions below)
         optionStrings.append("-XX:-UseLargePages");
 
-        if (conf && conf->hasProp("jvmoptions"))
+        if (conf.hasProp("jvmoptions"))
         {
             // Use space as field sep as ':' and ';' are valid
-            optionStrings.appendList(conf->queryProp("jvmoptions"), " ");
+            optionStrings.appendList(conf.queryProp("jvmoptions"), " ");
         }
 
         // Options we know we always want set

plugins/py3embed/CMakeLists.txt

@@ -71,7 +71,7 @@ if(PY3EMBED)
 
         install(
             TARGETS py3embed
-            DESTINATION plugins)
+            DESTINATION versioned/python3)
 
         target_link_libraries(py3embed ${PYTHON3_LIBRARY})
 

plugins/pyembed/CMakeLists.txt

@@ -15,20 +15,20 @@
 ################################################################################
 
 
-# Component: pyembed
+# Component: py2embed
 
 #####################################################
 # Description:
 # ------------
-#    Cmake Input File for pyembed
+#    Cmake Input File for py2embed
 #####################################################
 
 set(debug_python OFF)   # A lot slower but can assist in debugging...
 set(DEBUG_PYTHON_LIBRARY "/usr/lib/libpython2.7_d.so")
 
-project(pyembed)
+project(py2embed)
 
-if(PYEMBED)
+if(PY2EMBED)
     unset(PYTHONLIBS_FOUND CACHE)
     unset(PYTHON_LIBRARY CACHE)
     unset(PYTHON_LIBRARIES CACHE)
@@ -36,7 +36,7 @@ if(PYEMBED)
     unset(PYTHON_INCLUDE_DIR CACHE)
     unset(PYTHON_DEBUG_LIBRARIES CACHE)
     unset(PYTHONLIBS_VERSION_STRING CACHE)
-    ADD_PLUGIN(pyembed PACKAGES PythonLibs MINVERSION 2.6 MAXVERSION 2.7)
+    ADD_PLUGIN(py2embed PACKAGES PythonLibs MINVERSION 2.6 MAXVERSION 2.7)
     set(PYTHON2LIBS_FOUND ${PYTHONLIBS_FOUND})
     set(PYTHON2_LIBRARY ${PYTHON_LIBRARY})
     set(PYTHON2_LIBRARIES ${PYTHON_LIBRARIES})
@@ -45,7 +45,7 @@ if(PYEMBED)
     set(PYTHON2_DEBUG_LIBRARIES ${PYTHON_DEBUG_LIBRARIES})
     set(PYTHON2LIBS_VERSION_STRING ${PYTHONLIBS_VERSION_STRING})
     message("Python2 library is ${PYTHON2_LIBRARY}")
-    if(MAKE_PYEMBED)
+    if(MAKE_PY2EMBED)
         set(
             SRCS
             pyembed.cpp)
@@ -63,29 +63,29 @@ if(PYEMBED)
             ./../../roxie/roxiemem
             ./../../system/jlib)
 
-        add_definitions(-D_USRDLL -DPYEMBED_EXPORTS)
+        add_definitions(-D_USRDLL -DPY2EMBED_EXPORTS)
         if(debug_python)
             add_definitions(-DPy_DEBUG)
         endif()
 
-        HPCC_ADD_LIBRARY(pyembed SHARED ${SRCS})
+        HPCC_ADD_LIBRARY(py2embed SHARED ${SRCS})
         if(${CMAKE_VERSION} VERSION_LESS "2.8.9")
             message(WARNING "Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
         elseif(NOT APPLE)
-            set_target_properties(pyembed PROPERTIES NO_SONAME 1)
+            set_target_properties(py2embed PROPERTIES NO_SONAME 1)
         endif()
 
         install(
-            TARGETS pyembed
-            DESTINATION plugins)
+            TARGETS py2embed
+            DESTINATION versioned/python2)
         if(debug_python)
-            target_link_libraries(pyembed ${DEBUG_PYTHON2_LIBRARY})
+            target_link_libraries(py2embed ${DEBUG_PYTHON2_LIBRARY})
         else()
-            target_link_libraries(pyembed ${PYTHON2_LIBRARY})
+            target_link_libraries(py2embed ${PYTHON2_LIBRARY})
         endif()
 
         target_link_libraries(
-            pyembed
+            py2embed
             eclrtl
             roxiemem
             jlib)

plugins/pyembed/pyembed.cpp

@@ -55,7 +55,7 @@ extern "C" DECL_EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
     return true;
 }
 
-namespace pyembed {
+namespace py2embed {
 
 // Use class OwnedPyObject for any objects that are not 'borrowed references'
 // so that the appropriate Py_DECREF call is made when the OwnedPyObject goes
@@ -234,38 +234,6 @@ static void releaseContext()
     }
 }
 
-#ifndef _WIN32
-static bool findLoadedModule(StringBuffer &ret,  const char *match)
-{
-    bool found = false;
-    FILE *diskfp = fopen("/proc/self/maps", "r");
-    if (diskfp)
-    {
-        char ln[_MAX_PATH];
-        while (fgets(ln, sizeof(ln), diskfp))
-        {
-            if (strstr(ln, match))
-            {
-                const char *fullName = strchr(ln, '/');
-                if (fullName)
-                {
-                    char * lf = (char *) strchr(fullName, '\n');
-                    if (lf)
-                    {
-                        *lf = 0;
-                        ret.set(fullName);
-                        found = true;
-                        break;
-                    }
-                }
-            }
-        }
-        fclose(diskfp);
-    }
-    return found;
-}
-#endif
-
 // Use a global object to ensure that the Python interpreter is initialized on main thread
 
 static class Python27GlobalState
@@ -483,16 +451,17 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
 {
     // Make sure we are never unloaded (as Python may crash if we are)
     // we do this by doing a dynamic load of the pyembed library
+    // This also allows eclcc to be able to use the library for constant folding
 #ifdef _WIN32
     ::GetModuleFileName((HINSTANCE)&__ImageBase, helperLibraryName, _MAX_PATH);
-    if (strstr(path, "pyembed"))
+    if (strstr(path, "py2embed"))
     {
         HINSTANCE h = LoadSharedObject(helperLibraryName, false, false);
         DBGLOG("LoadSharedObject returned %p", h);
     }
 #else
     StringBuffer modname;
-    if (findLoadedModule(modname, "libpyembed"))
+    if (findLoadedModule(modname, "libpy2embed"))
     {
         HINSTANCE h = LoadSharedObject(modname, false, false);
         // Deliberately leak this handle
@@ -664,23 +633,23 @@ static void getSetResult(PyObject *obj, bool & isAllResult, size32_t & resultByt
         switch ((type_t) elemType)
         {
         case type_int:
-            rtlWriteInt(outData, pyembed::getSignedResult(NULL, elem), elemSize);
+            rtlWriteInt(outData, py2embed::getSignedResult(NULL, elem), elemSize);
             break;
         case type_unsigned:
-            rtlWriteInt(outData, pyembed::getUnsignedResult(NULL, elem), elemSize);
+            rtlWriteInt(outData, py2embed::getUnsignedResult(NULL, elem), elemSize);
             break;
         case type_real:
             if (elemSize == sizeof(double))
-                * (double *) outData = (double) pyembed::getRealResult(NULL, elem);
+                * (double *) outData = (double) py2embed::getRealResult(NULL, elem);
             else
             {
                 assertex(elemSize == sizeof(float));
-                * (float *) outData = (float) pyembed::getRealResult(NULL, elem);
+                * (float *) outData = (float) py2embed::getRealResult(NULL, elem);
             }
             break;
         case type_boolean:
             assertex(elemSize == sizeof(bool));
-            * (bool *) outData = pyembed::getBooleanResult(NULL, elem);
+            * (bool *) outData = py2embed::getBooleanResult(NULL, elem);
             break;
         case type_string:
         case type_varstring:
@@ -815,47 +784,47 @@ public:
     virtual bool getBooleanResult(const RtlFieldInfo *field)
     {
         nextField(field);
-        return pyembed::getBooleanResult(field, elem);
+        return py2embed::getBooleanResult(field, elem);
     }
     virtual void getDataResult(const RtlFieldInfo *field, size32_t &len, void * &result)
     {
         nextField(field);
-        pyembed::getDataResult(field, elem, len, result);
+        py2embed::getDataResult(field, elem, len, result);
     }
     virtual double getRealResult(const RtlFieldInfo *field)
     {
         nextField(field);
-        return pyembed::getRealResult(field, elem);
+        return py2embed::getRealResult(field, elem);
     }
     virtual __int64 getSignedResult(const RtlFieldInfo *field)
     {
         nextField(field);
-        return pyembed::getSignedResult(field, elem);
+        return py2embed::getSignedResult(field, elem);
     }
     virtual unsigned __int64 getUnsignedResult(const RtlFieldInfo *field)
     {
         nextField(field);
-        return pyembed::getUnsignedResult(field, elem);
+        return py2embed::getUnsignedResult(field, elem);
     }
     virtual void getStringResult(const RtlFieldInfo *field, size32_t &chars, char * &result)
     {
         nextField(field);
-        pyembed::getStringResult(field, elem, chars, result);
+        py2embed::getStringResult(field, elem, chars, result);
     }
     virtual void getUTF8Result(const RtlFieldInfo *field, size32_t &chars, char * &result)
     {
         nextField(field);
-        pyembed::getUTF8Result(field, elem, chars, result);
+        py2embed::getUTF8Result(field, elem, chars, result);
     }
     virtual void getUnicodeResult(const RtlFieldInfo *field, size32_t &chars, UChar * &result)
     {
         nextField(field);
-        pyembed::getUnicodeResult(field, elem, chars, result);
+        py2embed::getUnicodeResult(field, elem, chars, result);
     }
     virtual void getDecimalResult(const RtlFieldInfo *field, Decimal &value)
     {
         nextField(field);
-        double ret = pyembed::getRealResult(field, elem);
+        double ret = py2embed::getRealResult(field, elem);
         value.setReal(ret);
     }
 
@@ -1274,7 +1243,7 @@ public:
         if (!row)
             return NULL;
         RtlDynamicRowBuilder rowBuilder(resultAllocator);
-        size32_t len = pyembed::getRowResult(row, rowBuilder);
+        size32_t len = py2embed::getRowResult(row, rowBuilder);
         return rowBuilder.finalizeRowClear(len);
     }
     virtual void stop()
@@ -1379,39 +1348,39 @@ public:
 
     virtual bool getBooleanResult()
     {
-        return pyembed::getBooleanResult(NULL, result);
+        return py2embed::getBooleanResult(NULL, result);
     }
     virtual void getDataResult(size32_t &__chars, void * &__result)
     {
-        pyembed::getDataResult(NULL, result, __chars, __result);
+        py2embed::getDataResult(NULL, result, __chars, __result);
     }
     virtual double getRealResult()
     {
-        return pyembed::getRealResult(NULL, result);
+        return py2embed::getRealResult(NULL, result);
     }
     virtual __int64 getSignedResult()
     {
-        return pyembed::getSignedResult(NULL, result);
+        return py2embed::getSignedResult(NULL, result);
     }
     virtual unsigned __int64 getUnsignedResult()
     {
-        return pyembed::getUnsignedResult(NULL, result);
+        return py2embed::getUnsignedResult(NULL, result);
     }
     virtual void getStringResult(size32_t &__chars, char * &__result)
     {
-        pyembed::getStringResult(NULL, result, __chars, __result);
+        py2embed::getStringResult(NULL, result, __chars, __result);
     }
     virtual void getUTF8Result(size32_t &__chars, char * &__result)
     {
-        pyembed::getUTF8Result(NULL, result, __chars, __result);
+        py2embed::getUTF8Result(NULL, result, __chars, __result);
     }
     virtual void getUnicodeResult(size32_t &__chars, UChar * &__result)
     {
-        pyembed::getUnicodeResult(NULL, result, __chars, __result);
+        py2embed::getUnicodeResult(NULL, result, __chars, __result);
     }
     virtual void getSetResult(bool & __isAllResult, size32_t & __resultBytes, void * & __result, int elemType, size32_t elemSize)
     {
-        pyembed::getSetResult(result, __isAllResult, __resultBytes, __result, elemType, elemSize);
+        py2embed::getSetResult(result, __isAllResult, __resultBytes, __result, elemType, elemSize);
     }
     virtual IRowStream *getDatasetResult(IEngineRowAllocator * _resultAllocator)
     {
@@ -1420,12 +1389,12 @@ public:
     virtual byte * getRowResult(IEngineRowAllocator * _resultAllocator)
     {
         RtlDynamicRowBuilder rowBuilder(_resultAllocator);
-        size32_t len = pyembed::getRowResult(result, rowBuilder);
+        size32_t len = py2embed::getRowResult(result, rowBuilder);
         return (byte *) rowBuilder.finalizeRowClear(len);
     }
     virtual size32_t getTransformResult(ARowBuilder & builder)
     {
-        return pyembed::getRowResult(result, builder);
+        return py2embed::getRowResult(result, builder);
     }
     virtual void bindBooleanParam(const char *name, bool val)
     {

plugins/pyembed/python.ecllib

@@ -15,9 +15,9 @@
     limitations under the License.
 ############################################################################## */
 
-EXPORT Language := SERVICE : plugin('pyembed')
-  integer getEmbedContext():cpp,pure,fold,namespace='pyembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';
-  boolean syntaxCheck(const varstring src):cpp,pure,namespace='pyembed',entrypoint='syntaxCheck';
+EXPORT Language := SERVICE : plugin('py2embed')
+  integer getEmbedContext():cpp,pure,fold,namespace='py2embed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';
+  boolean syntaxCheck(const varstring src):cpp,pure,namespace='py2embed',entrypoint='syntaxCheck';
 END;
 EXPORT getEmbedContext := Language.getEmbedContext;
 EXPORT syntaxCheck := Language.syntaxCheck;

roxie/ccd/ccdmain.cpp

@@ -894,7 +894,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         topology->getProp("@pluginDirectory", pluginDirectory);
         if (pluginDirectory.length() == 0)
             pluginDirectory.append(codeDirectory).append("plugins");
-
+        getAdditionalPluginsPath(pluginDirectory, codeDirectory);
         if (queryDirectory.length() == 0)
         {
             topology->getProp("@queryDir", queryDirectory);

roxie/ccd/ccdstate.cpp

@@ -2835,10 +2835,10 @@ IRoxieQueryPackageManagerSet *globalPackageSetManager = NULL;
 
 extern void loadPlugins()
 {
-    if (pluginDirectory.length() && isDirectory(pluginDirectory.str()))
+    if (pluginDirectory.length())
     {
         plugins = new SafePluginMap(&PluginCtx, traceLevel >= 1);
-        plugins->loadFromDirectory(pluginDirectory);
+        plugins->loadFromList(pluginDirectory);
     }
 }
 

system/jlib/jdebug.cpp

@@ -2360,8 +2360,7 @@ public:
         term = false;
         latestCPU = 0;
         // UDP stats reported unless explicitly disabled
-        Owned<IProperties> conf = createProperties(CONFIG_DIR PATHSEPSTR "environment.conf", true);
-        if (conf->getPropBool("udp_stats", true))
+        if (queryEnvironmentConf().getPropBool("udp_stats", true))
             traceMode |= PerfMonUDP;
 #ifdef _WIN32
         memset(&liOldIdleTime,0,sizeof(liOldIdleTime));

system/jlib/jfile.cpp

@@ -2784,8 +2784,7 @@ static inline bool isPCFlushAllowed()
     CriticalBlock block(flushsect);
     if (gbl_flush_allowed == FLUSH_INIT)
     {
-        Owned<IProperties> conf = createProperties(CONFIG_DIR PATHSEPSTR "environment.conf", true);
-        if (conf->getPropBool("allow_pgcache_flush", true))
+        if (queryEnvironmentConf().getPropBool("allow_pgcache_flush", true))
             gbl_flush_allowed = FLUSH_ALLOWED;
         else
             gbl_flush_allowed = FLUSH_DISALLOWED;

system/jlib/jlog.cpp

@@ -2661,17 +2661,15 @@ private:
         rolling = true;
         append = true;
         flushes = true;
-        Owned<IProperties> conf = createProperties(CONFIG_DIR PATHSEPSTR "environment.conf", true);
-        StringBuffer logFields;
-        conf->getProp("logfields", logFields);
-        if (logFields.length())
+        const char *logFields = queryEnvironmentConf().queryProp("logfields");
+        if (!isEmptyString(logFields))
             msgFields = LogMsgFieldsFromAbbrevs(logFields);
         else
             msgFields = MSGFIELD_STANDARD;
         msgAudiences = MSGAUD_all;
         msgClasses = MSGCLS_all;
         maxDetail = DefaultDetail;
-        name.set(component);//logfile defaults to component name. Change via setName(), setPrefix() and setPostfix()
+        name.set(component); //logfile defaults to component name. Change via setName(), setPrefix() and setPostfix()
         extension.set(".log");
         local = false;
         createAlias = true;

system/jlib/jprop.cpp

@@ -43,12 +43,12 @@ class PropertyIteratorOf : implements PITER, public CInterface
 {
 protected:
     HashIterator *piter;
-    HashTable &properties;
+    const HashTable &properties;
 
 public:
     IMPLEMENT_IINTERFACE; 
 
-    PropertyIteratorOf(HashTable &_properties) : properties(_properties)
+    PropertyIteratorOf(const HashTable &_properties) : properties(_properties)
     {
         properties.Link();
         piter = new HashIterator(properties);
@@ -77,7 +77,7 @@ typedef IPropertyIterator char_ptrIPropertyIterator;
 class char_ptrPropertyIterator : public PropertyIteratorOf<const char *, char_ptrIPropertyIterator>
 {
 public:
-    char_ptrPropertyIterator(HashTable &_properties) : PropertyIteratorOf<const char *, char_ptrIPropertyIterator>(_properties) { }
+    char_ptrPropertyIterator(const HashTable &_properties) : PropertyIteratorOf<const char *, char_ptrIPropertyIterator>(_properties) { }
     virtual const char *getPropKey()
     {
         IMapping &cur = piter->query();
@@ -214,10 +214,10 @@ public:
                 finger++;
         }
     }
-    virtual PTYPE toPType(const char *p) = 0;
-    virtual const char *fromPType(PTYPE p) = 0;
-    virtual PTYPE toKeyVal(const void *) = 0;
-    virtual int getPropInt(PTYPE propname, int dft)
+    virtual PTYPE toPType(const char *p) const = 0;
+    virtual const char *fromPType(PTYPE p) const = 0;
+    virtual PTYPE toKeyVal(const void *) const = 0;
+    virtual int getPropInt(PTYPE propname, int dft) const override
     {
         if (propname)
         {
@@ -227,7 +227,7 @@ public:
         }
         return dft;
     }
-    virtual bool getPropBool(PTYPE propname, bool dft) 
+    virtual bool getPropBool(PTYPE propname, bool dft) const override
     {
         if (propname)
         {
@@ -237,7 +237,7 @@ public:
         }
         return dft;
     }
-    virtual bool getProp(PTYPE propname, StringBuffer &ret)
+    virtual bool getProp(PTYPE propname, StringBuffer &ret) const override
     {
         if (propname)
         {
@@ -250,7 +250,7 @@ public:
         }
         return false;
     }
-    virtual const char *queryProp(PTYPE propname)
+    virtual const char *queryProp(PTYPE propname) const override
     {
         if (propname)
         {
@@ -260,7 +260,7 @@ public:
         }
         return NULL;
     }
-    virtual void saveFile(const char *filename)
+    virtual void saveFile(const char *filename) const override
     {
         FILE *outFile = fopen(filename, "w" TEXT_TRANS);
         if (outFile)
@@ -324,7 +324,7 @@ public:
             return properties.remove(propname);
         return false;
     }
-    virtual bool hasProp(PTYPE propname)
+    virtual bool hasProp(PTYPE propname) const override
     {
         if (propname)
         {
@@ -371,7 +371,7 @@ public:
             setProp(ptype, value);
         }
     }
-    virtual IPROPITER *getIterator()
+    virtual IPROPITER *getIterator() const override
     {
         return new PROPITER(properties);
     }
@@ -383,9 +383,9 @@ class PCLASS : public CPropertiesBase<PTYPE, MAPPING, PTYPE##IProperties, PTYPE#
 public:                                                                                                 \
     PCLASS(const char *filename, bool nocase) : CPropertiesBase<PTYPE, MAPPING, PTYPE##IProperties, PTYPE##IPropertyIterator, PTYPE##PropertyIterator>(nocase) { loadFile(filename); } \
     PCLASS(bool nocase) : CPropertiesBase<PTYPE, MAPPING, PTYPE##IProperties, PTYPE##IPropertyIterator, PTYPE##PropertyIterator>(nocase)    { }         \
-    virtual PTYPE toPType(const char *p) { return conv2##PTYPE(p); }                                    \
-    virtual const char *fromPType(PTYPE p) { return conv##PTYPE##2(p); }                                \
-    virtual PTYPE toKeyVal(const void *p) { return tokv##PTYPE(p); }                                \
+    virtual PTYPE toPType(const char *p) const override { return conv2##PTYPE(p); }                                    \
+    virtual const char *fromPType(PTYPE p) const override { return conv##PTYPE##2(p); }                                \
+    virtual PTYPE toKeyVal(const void *p) const override { return tokv##PTYPE(p); }                                \
 };
 typedef IProperties char_ptrIProperties;
 MAKECPropertyOf(Atom, char_ptr, StringAttrMapping, CProperties);

system/jlib/jprop.hpp

@@ -38,23 +38,23 @@ interface jlib_decl IPropertyIteratorOf : extends IInterface
 template <class PTYPE, class PITER>
 interface jlib_decl IPropertiesOf : extends serializable
 {
-    virtual int getPropInt(PTYPE propname, int dft=0) = 0;
-    virtual bool getProp(PTYPE propname, StringBuffer &ret) = 0;
-    virtual const char *queryProp(PTYPE propname) = 0;
+    virtual int getPropInt(PTYPE propname, int dft=0) const = 0;
+    virtual bool getProp(PTYPE propname, StringBuffer &ret) const = 0;
+    virtual const char *queryProp(PTYPE propname) const = 0;
     virtual void setProp(PTYPE propname, int val) = 0;
     virtual void setProp(PTYPE propname, const char *val) = 0;
     virtual void appendProp(PTYPE propname, const char *val) = 0;
-    virtual bool hasProp(PTYPE propname) = 0;
-    virtual PITER *getIterator() = 0;
+    virtual bool hasProp(PTYPE propname) const = 0;
+    virtual PITER *getIterator() const = 0;
     virtual void loadFile(const char *filename) = 0;
     virtual void loadProps(const char *text) = 0;
     virtual void loadProp(const char *text) = 0;
     virtual void loadProp(const char *text, int dft) = 0;
     virtual void loadProp(const char *text, bool dft) = 0;
     virtual void loadProp(const char *text, const char * dft) = 0;
-    virtual void saveFile(const char *filename) = 0;
+    virtual void saveFile(const char *filename) const = 0;
     virtual bool removeProp(PTYPE propname) = 0;
-    virtual bool getPropBool(PTYPE propname, bool dft=false) = 0;
+    virtual bool getPropBool(PTYPE propname, bool dft=false) const = 0;
 };
 
 #ifdef _MSC_VER

system/jlib/jsocket.cpp

@@ -2866,7 +2866,6 @@ static StringAttr cachehostname;
 static IpAddress cachehostip;
 static IpAddress localhostip;
 static CriticalSection hostnamesect;
-static StringBuffer EnvConfPath;
 
 const char * GetCachedHostName()
 {
@@ -2875,13 +2874,8 @@ const char * GetCachedHostName()
     {
 #ifndef _WIN32
         IpAddress ip;
-        if (EnvConfPath.length() == 0)
-            EnvConfPath.append(CONFIG_DIR).append(PATHSEPSTR).append("environment.conf");
-        Owned<IProperties> conf = createProperties(EnvConfPath.str(), true);
-
-        StringBuffer ifs;
-        conf->getProp("interface", ifs);
-        if (getInterfaceIp(ip, ifs.str()))
+        const char *ifs = queryEnvironmentConf().queryProp("interface");
+        if (getInterfaceIp(ip, ifs))
         {
             StringBuffer ips;
             ip.getIpText(ips);
@@ -5057,14 +5051,13 @@ ISocketSelectHandler *createSocketSelectHandler(const char *trc)
     {
         CriticalBlock block(epollsect);
         // DBGLOG("createSocketSelectHandler(): epoll_method = %d",epoll_method);
-        if (epoll_method == EPOLL_INIT) {
-            Owned<IProperties> conf = createProperties(CONFIG_DIR PATHSEPSTR "environment.conf", true);
-            if (conf->getPropBool("use_epoll", true)) {
+        if (epoll_method == EPOLL_INIT)
+        {
+            if (queryEnvironmentConf().getPropBool("use_epoll", true))
                 epoll_method = EPOLL_ENABLED;
-            } else {
+            else
                 epoll_method = EPOLL_DISABLED;
-            }
-            // DBGLOG("createSocketSelectHandler(): after reading conf file, epoll_method = %d",epoll_method);
+        // DBGLOG("createSocketSelectHandler(): after reading conf file, epoll_method = %d",epoll_method);
         }
     }
     if (epoll_method == EPOLL_ENABLED)

system/jlib/jutil.cpp

@@ -346,6 +346,58 @@ static bool isCorruptDll(const char *errorMessage)
 
 }
 //-----------------------------------------------------------------------
+#ifdef __APPLE__
+bool findLoadedModule(StringBuffer &ret,  const char *match)
+{
+    bool found = false;
+    unsigned count = _dyld_image_count();
+    for (unsigned i = 0; i<count; i++)
+    {
+        const char *ln = _dyld_get_image_name(i);
+        if (ln)
+        {
+            if (strstr(ln, match))
+            {
+                ret.set(ln);
+                found = true;
+                break;
+            }
+        }
+    }
+    return found;
+}
+#elif !defined(WIN32)
+bool findLoadedModule(StringBuffer &ret,  const char *match)
+{
+    bool found = false;
+    FILE *diskfp = fopen("/proc/self/maps", "r");
+    if (diskfp)
+    {
+        char ln[_MAX_PATH];
+        while (fgets(ln, sizeof(ln), diskfp))
+        {
+            if (strstr(ln, match))
+            {
+                const char *fullName = strchr(ln, '/');
+                if (fullName)
+                {
+                    char * lf = (char *) strchr(fullName, '\n');
+                    if (lf)
+                    {
+                        *lf = 0;
+                        ret.set(fullName);
+                        found = true;
+                        break;
+                    }
+                }
+            }
+        }
+        fclose(diskfp);
+    }
+    return found;
+}
+#endif
+
 HINSTANCE LoadSharedObject(const char *name, bool isGlobal, bool raiseOnError)
 {
 #if defined(_WIN32)
@@ -2300,39 +2352,37 @@ StringBuffer & fillConfigurationDirectoryEntry(const char *dir,const char *name,
     return dirout;
 }
 
-IPropertyTree *getHPCCEnvironment(const char *configFileName)
+IPropertyTree *getHPCCEnvironment()
 {
-    StringBuffer configFileSpec(configFileName);
-    if (!configFileSpec.length())
-#ifdef _WIN32 
-        return NULL;
-#else
-        configFileSpec.set(CONFIG_DIR).append(PATHSEPSTR).append("environment.conf");
-#endif  
-    Owned<IProperties> props = createProperties(configFileSpec.str());
-    if (props)
+    StringBuffer envfile;
+    if (queryEnvironmentConf().getProp("environment",envfile) && envfile.length())
     {
-        StringBuffer envfile;
-        if (props->getProp("environment",envfile)&&envfile.length())
+        if (!isAbsolutePath(envfile.str()))
         {
-            if (!isAbsolutePath(envfile.str()))
-            {
-                StringBuffer tail(envfile);
-                splitDirTail(configFileSpec.str(),envfile.clear());
-                addPathSepChar(envfile).append(tail);
-            }
-            Owned<IFile> file = createIFile(envfile.str());
-            if (file)
-            {
-                Owned<IFileIO> fileio = file->open(IFOread);
-                if (fileio)
-                    return createPTree(*fileio);
-            }
+            envfile.insert(0, CONFIG_DIR PATHSEPSTR);
+        }
+        Owned<IFile> file = createIFile(envfile.str());
+        if (file)
+        {
+            Owned<IFileIO> fileio = file->open(IFOread);
+            if (fileio)
+                return createPTree(*fileio);
         }
     }
     return NULL;
 }
 
+static Owned<IProperties> envConfFile;
+static CriticalSection envConfCrit;
+
+jlib_decl const IProperties &queryEnvironmentConf()
+{
+    CriticalBlock b(envConfCrit);
+    if (!envConfFile)
+        envConfFile.setown(createProperties(CONFIG_DIR PATHSEPSTR ENV_CONF_FILE, true));
+    return *envConfFile;
+}
+
 static CriticalSection securitySettingsCrit;
 static bool useSSL = false;
 static StringAttr certificate;

system/jlib/jutil.hpp

@@ -33,6 +33,7 @@ extern mach_timebase_info_data_t timebase_info;   // Calibration for nanosecond
 //#define NAMEDCOUNTS
 
 interface IPropertyTree;
+interface IProperties;
 
 void jlib_decl MilliSleep(unsigned milli);
 long jlib_decl atolong_l(const char * s,int l);
@@ -62,6 +63,17 @@ int jlib_decl numtostr(char *dst, unsigned int value);
 int jlib_decl numtostr(char *dst, unsigned long value);
 int jlib_decl numtostr(char *dst, unsigned __int64 _value);
 
+#ifndef _WIN32
+/**
+ * Return full path name of a currently loaded dll that matches the supplied tail
+ *
+ * @param ret    StringBuffer to receive full path name
+ * @param match  Partial name to be located
+ * @return       True if a matching loaded dll was found
+ */
+extern jlib_decl bool findLoadedModule(StringBuffer &ret,  const char *match);
+#endif
+
 extern jlib_decl HINSTANCE LoadSharedObject(const char *name, bool isGlobal, bool raiseOnError);
 extern jlib_decl void FreeSharedObject(HINSTANCE h);
 
@@ -326,7 +338,22 @@ public:
 
 extern jlib_decl StringBuffer passwordInput(const char* prompt, StringBuffer& passwd);
 
-extern jlib_decl IPropertyTree *getHPCCEnvironment(const char *configFileName=NULL);
+/**
+ * Return a reference to a shared IProperties object representing the environment.conf settings.
+ * The object is loaded when first needed, and freed at program termination. This function is threadsafe.
+ *
+ * @return    The environment.conf properties
+ *
+ */
+extern jlib_decl const IProperties &queryEnvironmentConf();
+
+/**
+ * Return an owned copy of the local environment.xml file
+ *
+ * @return    The environment.xml property tree
+ *
+ */
+extern jlib_decl IPropertyTree *getHPCCEnvironment();
 extern jlib_decl bool getConfigurationDirectory(const IPropertyTree *dirtree, // NULL to use HPCC config
                                                 const char *category, 
                                                 const char *component,

testing/regress/ecl/embedpy3-catch.ecl

@@ -16,7 +16,7 @@
 ############################################################################## */
 
 //class=embedded
-
+//class=python3
 //nothor
 
 //Thor doesn't handle CATCH properly, see HPCC-9059

testing/regress/ecl/embedpy3-fold.ecl

@@ -16,7 +16,7 @@
 ############################################################################## */
 
 //class=embedded
-
+//class=python3
 //nothor
 
 IMPORT Python3;

testing/regress/ecl/embedpy3.ecl

@@ -16,7 +16,7 @@
 ############################################################################## */
 
 //class=embedded
-
+//class=python3
 //nothor
 
 IMPORT Python3;

testing/regress/ecl/embedpy3a.ecl

@@ -16,6 +16,7 @@
 ############################################################################## */
 
 //class=embedded
+//class=python3
 
 import python3;
 

testing/regress/ecl/py3import.ecl

@@ -16,6 +16,7 @@
 ############################################################################## */
 
 //class=embedded
+//class=python3
 
 import python3;
 string pcat(string a, string b) := IMPORT(Python3, '/opt/HPCCSystems/examples/embed/python_cat.cat':time);

testing/regress/ecl/py3streame.ecl

@@ -16,6 +16,7 @@
 ############################################################################## */
 
 //class=embedded
+//class=python3
 
 IMPORT Python3;
 

testing/regress/ecl/py3streame2.ecl

@@ -16,6 +16,7 @@
 ############################################################################## */
 
 //class=embedded
+//class=python3
 
 IMPORT Python3;
 

testing/regress/ecl/py3streame3.ecl

@@ -16,6 +16,7 @@
 ############################################################################## */
 
 //class=embedded
+//class=python3
 
 IMPORT Python3;