Browse Source

HPCC-12981 Thor may core on second job that uses python

Unloading then reloading the pyembed plugin can lead to cores - particularly
if imported python modules such as numpy have cached data.

We hit similar issues in other embedded languages, and 'solved' them by
preventing the relevant embed plugin from unloading

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>
Richard Chapman 10 years ago
parent
commit
950fd61ec6
1 changed files with 39 additions and 0 deletions
  1. 39 0
      plugins/pyembed/pyembed.cpp

+ 39 - 0
plugins/pyembed/pyembed.cpp

@@ -364,6 +364,45 @@ protected:
     CriticalSection lock;
 } globalState;
 
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    // Make sure we are never unloaded (as Python may crash if we are)
+    // we do this by doing a dynamic load of the pyembed library
+#ifdef _WIN32
+    ::GetModuleFileName((HINSTANCE)&__ImageBase, helperLibraryName, _MAX_PATH);
+    if (strstr(path, "pyembed"))
+    {
+        HINSTANCE h = LoadSharedObject(helperLibraryName, false, false);
+        DBGLOG("LoadSharedObject returned %p", h);
+    }
+#else
+    FILE *diskfp = fopen("/proc/self/maps", "r");
+    if (diskfp)
+    {
+        char ln[_MAX_PATH];
+        while (fgets(ln, sizeof(ln), diskfp))
+        {
+            if (strstr(ln, "libpyembed"))
+            {
+                const char *fullName = strchr(ln, '/');
+                if (fullName)
+                {
+                    char *tail = (char *) strstr(fullName, SharedObjectExtension);
+                    if (tail)
+                    {
+                        tail[strlen(SharedObjectExtension)] = 0;
+                        HINSTANCE h = LoadSharedObject(fullName, false, false);
+                        break;
+                    }
+                }
+            }
+        }
+        fclose(diskfp);
+    }
+#endif
+    return true;
+}
+
 PyObject *PythonThreadContext::getNamedTupleType(const RtlTypeInfo *type)
 {
     if (!lru || (type!=lrutype))