HPCC-19150 Core at closedown unloading python plugin

Roxie and eclagent may core at closedown when unloading the python plugin.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

initfiles/etc/DIR_NAME/environment.conf.in

@@ -45,6 +45,10 @@ jvmoptions=-XX:-UsePerfData
 #jvmoptions=-XX:-UsePerfData -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=2000
 #JNI_PATH=/absolute/path/to/alternative/libjvm.so
 
+# Python plugins can call python cleanup code on exit, but this seems to cause lockups in some Tensorflow examples
+# In most cases, skipping the cleanup is harmless and avoids these lockups
+skipPythonCleanup=true
+
 # Although HPCC platform includes plugins for both Python2 and Python3, only one may be safely enabled at a time
 # as the Python libraries export the same symbols for both versions. Enabling both may lead to unpredicatable results
 # including segfaults or undefined symbol errors.

plugins/py3embed/py3embed.cpp

@@ -288,17 +288,15 @@ public:
         PyEval_InitThreads();
         preservedScopes.setown(PyDict_New());
         tstate = PyEval_SaveThread();
+        skipPythonCleanup = queryEnvironmentConf().getPropBool("skipPythonCleanup", true);
         initialized = true;
     }
     ~Python3xGlobalState()
     {
-        if (keepLoadedHandle)
-            FreeSharedObject(keepLoadedHandle);  // Must be process termination - ok to free now (and helps stop lockups at closedown in some Python libraries eg Tensorflow).
-
         if (threadContext)
             delete threadContext;   // The one on the main thread won't get picked up by the thread hook mechanism
         threadContext = NULL;
-        if (initialized)
+        if (initialized && !skipPythonCleanup)
         {
             PyEval_RestoreThread(tstate);
             // Finish the Python Interpreter
@@ -307,9 +305,9 @@ public:
             compiledScripts.clear();
             preservedScopes.clear();
             Py_Finalize();
+            if (pythonLibrary)
+                FreeSharedObject(pythonLibrary);
         }
-        if (pythonLibrary)
-            FreeSharedObject(pythonLibrary);
     }
     bool isInitialized()
     {
@@ -492,6 +490,7 @@ protected:
     }
     PyThreadState *tstate = nullptr;
     bool initialized = false;
+    bool skipPythonCleanup = true; // Tensorflow seems to often lockup in the python cleanup process.
     HINSTANCE pythonLibrary = 0;
     OwnedPyObject namedtuple;      // collections.namedtuple
     OwnedPyObject namedtupleTypes; // dictionary of return values from namedtuple()

plugins/pyembed/pyembed.cpp

@@ -281,17 +281,15 @@ public:
         PyEval_InitThreads();
         preservedScopes.setown(PyDict_New());
         tstate = PyEval_SaveThread();
+        skipPythonCleanup = queryEnvironmentConf().getPropBool("skipPythonCleanup", true);
         initialized = true;
     }
     ~Python27GlobalState()
     {
-        if (keepLoadedHandle)
-            FreeSharedObject(keepLoadedHandle);  // Must be process termination - ok to free now (and helps stop lockups at closedown in some Python libraries eg Tensorflow).
-
         if (threadContext)
             delete threadContext;   // The one on the main thread won't get picked up by the thread hook mechanism
         threadContext = NULL;
-        if (initialized)
+        if (initialized && !skipPythonCleanup)
         {
             PyEval_RestoreThread(tstate);
             // Finish the Python Interpreter
@@ -300,9 +298,9 @@ public:
             compiledScripts.clear();
             preservedScopes.clear();
             Py_Finalize();
+            if (pythonLibrary)
+                FreeSharedObject(pythonLibrary);
         }
-        if (pythonLibrary)
-            FreeSharedObject(pythonLibrary);
     }
     bool isInitialized()
     {
@@ -489,6 +487,7 @@ protected:
     }
     PyThreadState *tstate = nullptr;
     bool initialized = false;
+    bool skipPythonCleanup = true; // Tensorflow seems to often lockup in the python cleanup process.
     HINSTANCE pythonLibrary = 0;
     OwnedPyObject namedtuple;      // collections.namedtuple
     OwnedPyObject namedtupleTypes; // dictionary of return values from namedtuple()