HPCC-15130 Reduce jlib dependencies

Move NUMA dependency to thorhelper

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -50,7 +50,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(CLIENTTOOLS_ONLY "Enable the building of Client Tools only." OFF)
   option(INCLUDE_PLUGINS "Enable the building of platform and all plugins for testing purposes" OFF)
   option(PLUGIN "Enable building of a plugin" OFF)
-  option(USE_SHLIBDEPS "Enable the use of dpkg-shlibdeps on ubuntu packagin" OFF)
+  option(USE_SHLIBDEPS "Enable the use of dpkg-shlibdeps on ubuntu packaging" OFF)
 
   if (APPLE OR WIN32)
     option(USE_BINUTILS "Enable use of binutils to embed workunit info into shared objects" OFF)
@@ -282,14 +282,6 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     message(FATAL_ERROR "No threading support found")
   ENDIF()
 
-  if (USE_NUMA)
-    find_package(NUMA)
-    add_definitions (-D_USE_NUMA)
-    if (NOT NUMA_FOUND)
-      message(FATAL_ERROR "Support for numa not found")
-    endif()
-  endif()
-
   if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
    set (CMAKE_COMPILER_IS_CLANGXX 1)
   endif()
@@ -844,6 +836,14 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
         add_definitions (-D_NO_APR)
       endif(USE_APR)
 
+      if (USE_NUMA)
+        find_package(NUMA)
+        add_definitions (-D_USE_NUMA)
+        if (NOT NUMA_FOUND)
+          message(FATAL_ERROR "NUMA requested but package not found")
+        endif()
+      endif()
+
       if(USE_TBB)
         find_package(TBB)
         if (TBB_FOUND)

common/thorhelper/CMakeLists.txt

@@ -103,6 +103,10 @@ target_link_libraries ( thorhelper
          dalibase 
     )
 
+if (USE_NUMA)
+ target_link_libraries ( thorhelper numa )
+endif ()
+
 
 IF (USE_OPENSSL)
     target_link_libraries ( thorhelper 

common/thorhelper/thorcommon.cpp

@@ -28,6 +28,9 @@
 #include "eclrtl.hpp"
 #include "rtlread_imp.hpp"
 #include <algorithm>
+#ifdef _USE_NUMA
+#include <numa.h>
+#endif
 
 #include "thorstep.hpp"
 
@@ -1717,5 +1720,112 @@ void ActivityTimeAccumulator::merge(const ActivityTimeAccumulator & other)
     }
 }
 
+//---------------------------------------------------------------------------------------------------------------------
 
+//MORE: Not currently implemented for windows.
+#ifdef CPU_SETSIZE
+static unsigned getCpuId(const char * text, char * * next)
+{
+    unsigned cpu = (unsigned)strtoul(text, next, 10);
+    if (*next == text)
+        throw makeStringExceptionV(1, "Invalid CPU: %s", text);
+    else if (cpu >= CPU_SETSIZE)
+        throw makeStringExceptionV(1, "CPU %u is out of range 0..%u", cpu, CPU_SETSIZE);
+    return cpu;
+}
+#endif
 
+void setProcessAffinity(const char * cpuList)
+{
+    assertex(cpuList);
+#ifdef CPU_ZERO
+    cpu_set_t cpus;
+    CPU_ZERO(&cpus);
+
+    const char * cur = cpuList;
+    loop
+    {
+        char * next;
+        unsigned cpu1 = getCpuId(cur, &next);
+        if (*next == '-')
+        {
+            const char * range = next+1;
+            unsigned cpu2 = getCpuId(range, &next);
+            for (unsigned cpu= cpu1; cpu <= cpu2; cpu++)
+                CPU_SET(cpu, &cpus);
+        }
+        else
+            CPU_SET(cpu1, &cpus);
+
+        if (*next == '\0')
+            break;
+
+        if (*next != ',')
+            throw makeStringExceptionV(1, "Invalid cpu affinity list %s", cur);
+
+        cur = next+1;
+    }
+
+    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpus))
+        throw makeStringException(errno, "Failed to set affinity");
+    DBGLOG("Process affinity set to %s", cpuList);
+#endif
+    clearAffinityCache();
+}
+
+void setAutoAffinity(unsigned curProcess, unsigned processPerMachine, const char * optNodes)
+{
+#if defined(CPU_ZERO) && defined(_USE_NUMA)
+    if (processPerMachine <= 1)
+        return;
+
+    if (numa_available() == -1)
+    {
+        DBGLOG("Numa functions not available");
+        return;
+    }
+
+    if (optNodes)
+        throw makeStringException(1, "Numa node list not yet supported");
+
+    unsigned numNumaNodes = numa_max_node()+1;
+    if (numNumaNodes <= 1)
+        return;
+
+    //MORE: If processPerMachine < numNumaNodes we may want to associate with > 1 node.
+    unsigned curNode = curProcess % numNumaNodes;
+
+#if defined(LIBNUMA_API_VERSION) && (LIBNUMA_API_VERSION>=2)
+    struct bitmask * cpus = numa_allocate_cpumask();
+    numa_node_to_cpus(curNode, cpus);
+    bool ok = (numa_sched_setaffinity(0, cpus) == 0);
+    numa_bitmask_free(cpus);
+#else
+    cpu_set_t cpus;
+    CPU_ZERO(&cpus);
+    numa_node_to_cpus(curNode, (unsigned long *) &cpus, sizeof (cpus));
+    bool ok = sched_setaffinity (0, sizeof(cpus), &cpus) != 0;
+#endif
+
+    if (!ok)
+        throw makeStringExceptionV(1, "Failed to set affinity for node %u", curNode);
+
+    DBGLOG("Process bound to numa node %u of %u", curNode, numNumaNodes);
+#endif
+    clearAffinityCache();
+}
+
+void bindMemoryToLocalNodes()
+{
+#if defined(LIBNUMA_API_VERSION) && (LIBNUMA_API_VERSION>=2)
+    numa_set_bind_policy(1);
+
+    unsigned numNumaNodes = numa_max_node() + 1;
+    if (numNumaNodes <= 1)
+        return;
+    struct bitmask *nodes = numa_get_run_node_mask();
+    numa_set_membind(nodes);
+    DBGLOG("Process memory bound to numa nodemask 0x%x (of %u nodes total)", (unsigned)(*(nodes->maskp)), numNumaNodes);
+    numa_bitmask_free(nodes);
+#endif
+}

common/thorhelper/thorcommon.hpp

@@ -577,5 +577,8 @@ extern THORHELPER_API bool isActivitySink(ThorActivityKind kind);
 extern THORHELPER_API bool isActivitySource(ThorActivityKind kind);
 extern THORHELPER_API const char * getActivityText(ThorActivityKind kind);
 
+extern THORHELPER_API void setProcessAffinity(const char * cpus);
+extern THORHELPER_API void setAutoAffinity(unsigned curProcess, unsigned processPerNode, const char * optNodes);
+extern THORHELPER_API void bindMemoryToLocalNodes();
 
 #endif // THORHELPER_HPP

system/jlib/CMakeLists.txt

@@ -209,10 +209,6 @@ else ()
  target_link_libraries ( jlib rt )
 endif ()
 
-if (USE_NUMA)
- target_link_libraries ( jlib numa )
-endif ()
-
 if (NOT PLUGIN)
     if (WIN32)
         FILE(TO_NATIVE_PATH "${CMAKE_CURRENT_BINARY_DIR}" WINDOWS_CMAKE_CURRENT_BINARY_DIR)

system/jlib/jthread.cpp

@@ -33,9 +33,6 @@
 #include <sys/syscall.h>
 #include <sys/types.h>
 #include <sys/resource.h>
-#ifdef _USE_NUMA
-#include <numa.h>
-#endif
 #endif
 
 #if defined(_DEBUG) && defined(_WIN32) && !defined(USING_MPATROL)
@@ -2370,113 +2367,3 @@ unsigned threadLogID()  // for use in logging
 #endif
     return (unsigned)(memsize_t) GetCurrentThreadId(); // truncated in 64bit
 }
-
-//---------------------------------------------------------------------------------------------------------------------
-
-//MORE: Not currently implemented for windows.
-#ifdef CPU_SETSIZE
-static unsigned getCpuId(const char * text, char * * next)
-{
-    unsigned cpu = (unsigned)strtoul(text, next, 10);
-    if (*next == text)
-        throw makeStringExceptionV(1, "Invalid CPU: %s", text);
-    else if (cpu >= CPU_SETSIZE)
-        throw makeStringExceptionV(1, "CPU %u is out of range 0..%u", cpu, CPU_SETSIZE);
-    return cpu;
-}
-#endif
-
-void setProcessAffinity(const char * cpuList)
-{
-    assertex(cpuList);
-#ifdef CPU_ZERO
-    cpu_set_t cpus;
-    CPU_ZERO(&cpus);
-
-    const char * cur = cpuList;
-    loop
-    {
-        char * next;
-        unsigned cpu1 = getCpuId(cur, &next);
-        if (*next == '-')
-        {
-            const char * range = next+1;
-            unsigned cpu2 = getCpuId(range, &next);
-            for (unsigned cpu= cpu1; cpu <= cpu2; cpu++)
-                CPU_SET(cpu, &cpus);
-        }
-        else
-            CPU_SET(cpu1, &cpus);
-
-        if (*next == '\0')
-            break;
-
-        if (*next != ',')
-            throw makeStringExceptionV(1, "Invalid cpu affinity list %s", cur);
-
-        cur = next+1;
-    }
-
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpus))
-        throw makeStringException(errno, "Failed to set affinity");
-    DBGLOG("Process affinity set to %s", cpuList);
-#endif
-    clearAffinityCache();
-}
-
-void setAutoAffinity(unsigned curProcess, unsigned processPerMachine, const char * optNodes)
-{
-#if defined(CPU_ZERO) && defined(_USE_NUMA)
-    if (processPerMachine <= 1)
-        return;
-
-    if (numa_available() == -1)
-    {
-        DBGLOG("Numa functions not available");
-        return;
-    }
-
-    if (optNodes)
-        throw makeStringException(1, "Numa node list not yet supported");
-
-    unsigned numNumaNodes = numa_max_node()+1;
-    if (numNumaNodes <= 1)
-        return;
-
-    //MORE: If processPerMachine < numNumaNodes we may want to associate with > 1 node.
-    unsigned curNode = curProcess % numNumaNodes;
-
-#if defined(LIBNUMA_API_VERSION) && (LIBNUMA_API_VERSION>=2)
-    struct bitmask * cpus = numa_allocate_cpumask();
-    numa_node_to_cpus(curNode, cpus);
-    bool ok = (numa_sched_setaffinity(0, cpus) == 0);
-    numa_bitmask_free(cpus);
-#else
-    cpu_set_t cpus;
-    CPU_ZERO(&cpus);
-    numa_node_to_cpus(curNode, (unsigned long *) &cpus, sizeof (cpus));
-    bool ok = sched_setaffinity (0, sizeof(cpus), &cpus) != 0;
-#endif
-
-    if (!ok)
-        throw makeStringExceptionV(1, "Failed to set affinity for node %u", curNode);
-
-    DBGLOG("Process bound to numa node %u of %u", curNode, numNumaNodes);
-#endif
-    clearAffinityCache();
-}
-
-void bindMemoryToLocalNodes()
-{
-#if defined(LIBNUMA_API_VERSION) && (LIBNUMA_API_VERSION>=2)
-    numa_set_bind_policy(1);
-
-    unsigned numNumaNodes = numa_max_node() + 1;
-    if (numNumaNodes <= 1)
-        return;
-    struct bitmask *nodes = numa_get_run_node_mask();
-    numa_set_membind(nodes);
-    DBGLOG("Process memory bound to numa nodemask 0x%x (of %u nodes total)", (unsigned)(*(nodes->maskp)), numNumaNodes);
-    numa_bitmask_free(nodes);
-#endif
-}

system/jlib/jthread.hpp

@@ -309,9 +309,4 @@ interface IWorkQueueThread: extends IInterface
 // internally thread persists for specified time waiting before self destroying
 extern jlib_decl IWorkQueueThread *createWorkQueueThread(unsigned persisttime=1000*60);
 
-extern jlib_decl void setProcessAffinity(const char * cpus);
-extern jlib_decl void setAutoAffinity(unsigned curProcess, unsigned processPerNode, const char * optNodes);
-extern jlib_decl void bindMemoryToLocalNodes();
-
-
 #endif