HPCC-7880 - Additional thor memory tracing

Trace # maxCores setting per job
Trace roxiemem and heap usage at end of each job.

Signed-off-by: Jake Smith <jake.smith@lexisnexis.com>

system/jlib/jdebug.cpp

@@ -785,6 +785,11 @@ typedef LONG (WINAPI *PROCNTQSI)(UINT,PVOID,ULONG,PULONG);
 typedef LONG (WINAPI *PROCNTQIP)(HANDLE,UINT,PVOID,ULONG,PULONG);
 typedef LONG (WINAPI *PROCNTGST)(LARGE_INTEGER*, LARGE_INTEGER*, LARGE_INTEGER*);
 
+memsize_t getMapInfo(const char *type)
+{
+    return 0; // TODO/UNKNOWN
+}
+
 void getCpuInfo(unsigned &numCPUs, unsigned &CPUSpeed)
 {
     // MORE: Might be a better way to get CPU speed (actual) than the one stored in Registry
@@ -842,6 +847,31 @@ unsigned getAffinityCpus()
 
 #else // linux
 
+memsize_t getMapInfo(const char *type)
+{
+    memsize_t ret = 0;
+    VStringBuffer procMaps("/proc/%d/maps", GetCurrentProcessId());
+    VStringBuffer typeStr("[%s]", type);
+    FILE *diskfp = fopen(procMaps.str(), "r");
+    if (!diskfp)
+        return false;
+    char ln[256];
+    while (fgets(ln, sizeof(ln), diskfp))
+    {
+        if (strstr(ln, typeStr.str()))
+        {
+            unsigned __int64 addrLow, addrHigh;
+            if (2 == sscanf(ln, "%16Lx-%16Lx", &addrLow, &addrHigh))
+            {
+                ret = (memsize_t)(addrHigh-addrLow);
+                break;
+            }
+        }
+    }
+    fclose(diskfp);
+    return ret;
+}
+
 void getCpuInfo(unsigned &numCPUs, unsigned &CPUSpeed)
 {
     int cpufd = open("/proc/cpuinfo",O_RDONLY);

system/jlib/jdebug.hpp

@@ -253,6 +253,7 @@ unsigned jlib_decl setAllocHook(bool on);  // bwd compat returns unsigned
 #endif
 
 extern jlib_decl void getHardwareInfo(HardwareInfo &hdwInfo, const char *primDiskPath = NULL, const char *secDiskPath = NULL);
+extern jlib_decl memsize_t getMapInfo(const char *type);
 extern jlib_decl void getCpuInfo(unsigned &numCPUs, unsigned &CPUSpeed);
 extern jlib_decl unsigned getAffinityCpus();
 extern jlib_decl void printProcMap(const char *fn, bool printbody, bool printsummary, StringBuffer *lnout, MemoryBuffer *mb, bool useprintf);

thorlcr/graph/thgraph.cpp

@@ -2354,6 +2354,12 @@ void CJobBase::init()
     if (gmemSize && largeMemSize >= gmemSize)
         throw MakeStringException(0, "largeMemSize(%d) can not exceed globalMemorySize(%d)", largeMemSize, gmemSize);
     PROGLOG("Global memory size = %d MB, large mem size = %d MB", gmemSize, largeMemSize);
+    StringBuffer tracing("maxActivityCores = ");
+    if (maxActivityCores)
+        tracing.append(maxActivityCores);
+    else
+        tracing.append("[unbound]");
+    PROGLOG("%s", tracing.str());
     setLargeMemSize(largeMemSize);
     graphExecutor.setown(new CGraphExecutor(*this));
 }
@@ -2368,6 +2374,13 @@ CJobBase::~CJobBase()
     ::Release(userDesc);
     timeReporter->Release();
     delete pluginMap;
+
+    StringBuffer memStatsStr;
+    roxiemem::memstats(memStatsStr);
+    PROGLOG("Roxiemem stats: %s", memStatsStr.str());
+    memsize_t heapUsage = getMapInfo("heap");
+    if (heapUsage) // if 0, assumed to be unavailable
+        PROGLOG("Heap usage : %"I64F"d bytes", (unsigned __int64)heapUsage);
 }
 
 bool CJobBase::queryForceLogging(graph_id graphId, bool def) const