Merge branch 'candidate-7.6.0' into candidate-7.6.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/FindLIBMEMCACHED.cmake

@@ -75,7 +75,7 @@ if((LIBMEMCACHEDCORE_LIBRARY STREQUAL "LIBMEMCACHEDCORE_LIBRARY-NOTFOUND"
             DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/downloads
             SOURCE_DIR ${CMAKE_BINARY_DIR}/downloads/libmemcached
             CONFIGURE_COMMAND "${CMAKE_BINARY_DIR}/downloads/libmemcached/configure" --prefix=${INSTALL_DIR} LDFLAGS=-L${LIB_PATH}
-            BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} LDFLAGS=-Wl,-rpath-link,${LIB_PATH}
+            BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} CXXFLAGS='-fpermissive' LDFLAGS='-Wl,-rpath-link,${LIB_PATH}'
             BINARY_DIR ${CMAKE_BINARY_DIR}/build-libmemcached
             INSTALL_COMMAND "")
         add_library(libmemcached SHARED IMPORTED GLOBAL)

cmake_modules/buildBOOST_REGEX.cmake

@@ -0,0 +1,32 @@
+
+include(ExternalProject)
+ExternalProject_Add(
+  generate-boost-regex
+  URL https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz
+  URL_HASH SHA256=96b34f7468f26a141f6020efb813f1a2f3dfb9797ecf76a7d7cbd843cc95f5bd
+  TIMEOUT 15
+  DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/downloads
+  SOURCE_DIR ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0
+  CONFIGURE_COMMAND ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0/bootstrap.sh
+  BUILD_COMMAND ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0/b2 --prefix=${INSTALL_DIR} --exec-prefix=${INSTALL_DIR} --with-regex --with-headers
+  BUILD_IN_SOURCE TRUE
+  INSTALL_COMMAND ""
+  )
+
+add_library(boost-regex SHARED IMPORTED GLOBAL)
+set_property(TARGET boost-regex
+  PROPERTY IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0/stage/lib/libboost_regex.so.1.71.0)
+add_dependencies(boost-regex generate-boost-regex)
+
+if(PLATFORM OR CLIENTTOOLS_ONLY)
+  install(CODE "set(ENV{LD_LIBRARY_PATH} \"\$ENV{LD_LIBRARY_PATH}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/downloads/boost_1_71_0/stage/lib\")")
+  install(PROGRAMS
+    ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0/stage/lib/libboost_regex.so.1.71.0
+    ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0/stage/lib/libboost_regex.so
+    DESTINATION ${LIB_DIR})
+endif()
+
+set(BOOST_REGEX_INCLUDE_DIR ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0)
+set(BOOST_REGEX_LIBRARIES ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0/stage/lib/libboost_regex.so.1.17.0)
+message(STATUS "----------------------- ${BOOST_REGEX_LIBRARIES}")
+mark_as_advanced(BOOST_REGEX_LIBRARIES BOOST_REGEX_INCLUDE_DIR)

cmake_modules/commonSetup.cmake

@@ -58,6 +58,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(USE_OPENLDAP "Enable OpenLDAP support (requires OpenLDAP)" ON)
   option(USE_ICU "Enable unicode support (requires ICU)" ON)
   option(USE_BOOST_REGEX "Configure use of boost regex" ON)
+  option(CENTOS_6_BOOST "Supply regex library on CentOS 6" OFF)
   # USE_C11_REGEX is only checked if USE_BOOST_REGEX is OFF
   # to disable REGEX altogether turn both off
   option(USE_C11_REGEX "Configure use of c++11 std::regex" ON)
@@ -878,12 +879,16 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       endif(USE_URIPARSER)
 
       if(USE_BOOST_REGEX)
-        find_package(BOOST_REGEX)
-        if (BOOST_REGEX_FOUND)
-          message(STATUS "BOOST_REGEX enabled")
-          add_definitions (-D_USE_BOOST_REGEX)
+        if(CENTOS_6_BOOST)
+          include(${CMAKE_MODULE_PATH}/buildBOOST_REGEX.cmake)
         else()
-          message(FATAL_ERROR "BOOST_REGEX requested but package not found")
+          find_package(BOOST_REGEX)
+          if (BOOST_REGEX_FOUND)
+            message(STATUS "BOOST_REGEX enabled")
+            add_definitions (-D_USE_BOOST_REGEX)
+          else()
+            message(FATAL_ERROR "BOOST_REGEX requested but package not found")
+          endif()
         endif()
       else(USE_BOOST_REGEX)
         if (USE_C11_REGEX)

configuration/configmgr/configmgrlib/CMakeLists.txt

@@ -68,6 +68,9 @@ INCLUDE_DIRECTORIES(
 
 ADD_DEFINITIONS( -D_USRDLL -DCFGMGRLIB_EXPORTS)
 HPCC_ADD_LIBRARY( configmgr SHARED ${SRCS} )
+if(CENTOS_6_BOOST)
+  add_dependencies(configmgr generate-boost-regex)
+endif()
 TARGET_LINK_LIBRARIES( configmgr jlib)
 
 INSTALL ( TARGETS configmgr RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )

docs/BuildTools/fo.xsl

@@ -156,7 +156,7 @@
 
 <xsl:attribute-set name="section.title.level1.properties">
  <xsl:attribute name="font-size">
-    <xsl:value-of select="$body.font.master * 2.525"></xsl:value-of>
+    <xsl:value-of select="$body.font.master * 2.425"></xsl:value-of>
     <xsl:text>pt</xsl:text>
   </xsl:attribute>
     <xsl:attribute name="color">#A91919</xsl:attribute>

docs/BuildTools/fo.xsl.in

@@ -156,7 +156,7 @@
 
 <xsl:attribute-set name="section.title.level1.properties">
  <xsl:attribute name="font-size">
-    <xsl:value-of select="$body.font.master * 2.525"></xsl:value-of>
+    <xsl:value-of select="$body.font.master * 2.425"></xsl:value-of>
     <xsl:text>pt</xsl:text>
   </xsl:attribute>
     <xsl:attribute name="color">#A91919</xsl:attribute>

esp/src/eclwatch/templates/GraphTree7Widget.html

@@ -43,7 +43,7 @@
                                 <div data-dojo-type="dijit.Fieldset">
                                     <legend>${i18n.Edges}</legend>
                                     <div data-dojo-type="hpcc.TableContainer">
-                                        <input title="${i18n.Label}:" name="elabel" value="%Label%\n%NumRowsProcessed%" style="width: 95%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.TextBox" />
+                                        <input title="${i18n.Label}:" name="elabel" value="%Label%\n%NumRowsProcessed%\n%SkewMinRowsProcessed% - %SkewMaxRowsProcessed%" style="width: 95%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.TextBox" />
                                     </div>
                                 </div>
                                 <div class="dijitDialogPaneActionBar">
@@ -54,7 +54,7 @@
                         </div>
                     </div>
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
-                    <span id="${id}GraphStatus" ></span>
+                    <span id="${id}GraphStatus"></span>
                     <div class="right" data-dojo-attach-event="onChange:_onMaximizeGraph" data-dojo-props="iconClass:'iconMaximize', showLabel:false" checked=false data-dojo-type="dijit.form.ToggleButton">${i18n.MaximizeRestore}</div>
                 </div>
             </div>
@@ -88,4 +88,4 @@
             </div>
         </div>
     </div>
-</div>
+</div>

esp/src/src/WUScopeController.ts

@@ -2,8 +2,11 @@ import { Icon, Palette } from "@hpcc-js/common";
 import { BaseScope, ScopeEdge, ScopeGraph, ScopeSubgraph, ScopeVertex } from "@hpcc-js/comms";
 import { Edge, IGraphData, Lineage, Subgraph, Vertex } from "@hpcc-js/graph";
 import { Edge as UtilEdge, Subgraph as UtilSubgraph, Vertex as UtilVertex } from "@hpcc-js/util";
+import { format as d3Format } from "d3-format";
 import { decodeHtml } from "./Utility";
 
+const formatNum = d3Format(",");
+
 export type VertexType = Vertex | Icon;
 
 export interface WUGraphLegendData {
@@ -139,7 +142,7 @@ export class WUScopeController {
         return this;
     }
 
-    _edgeLabelTpl = "%Label%\n%NumRowsProcessed%";
+    _edgeLabelTpl = "%Label%\n%NumRowsProcessed%\n%SkewMinRowsProcessed% - %SkewMaxRowsProcessed%";
     edgeLabelTpl(): string;
     edgeLabelTpl(_: string): this;
     edgeLabelTpl(_?: string): string | this {
@@ -270,11 +273,25 @@ export class WUScopeController {
         return retVal;
     }
 
-    format(labelTpl, obj) {
-        labelTpl = labelTpl.split("\\n").join("\n");
+    formatNum(str): string {
+        if (isNaN(str)) {
+            return str;
+        }
+        return formatNum(str);
+    }
+
+    formatNums(obj) {
+        for (const key in obj) {
+            obj[key] = this.formatNum(obj[key]);
+        }
+        return obj;
+    }
+
+    formatLine(labelTpl, obj): string {
         let retVal = "";
         let lpos = labelTpl.indexOf("%");
         let rpos = -1;
+        let replacementFound = lpos >= 0 ? false : true;  //  If a line has no symbols always include it, otherwise only include that line IF a replacement was found  ---
         while (lpos >= 0) {
             retVal += labelTpl.substring(rpos + 1, lpos);
             rpos = labelTpl.indexOf("%", lpos + 1);
@@ -283,11 +300,23 @@ export class WUScopeController {
                 break;
             }
             const key = labelTpl.substring(lpos + 1, rpos);
+            replacementFound = replacementFound || !!obj[labelTpl.substring(lpos + 1, rpos)];
             retVal += !key ? "%" : (obj[labelTpl.substring(lpos + 1, rpos)] || "");
             lpos = labelTpl.indexOf("%", rpos + 1);
         }
         retVal += labelTpl.substring(rpos + 1, labelTpl.length);
-        return retVal.split("\n").filter(d => d.trim().length > 0).map(d => decodeHtml(d)).join("\n");
+        return replacementFound ? retVal : "";
+    }
+
+    format(labelTpl, obj) {
+        labelTpl = labelTpl.split("\\n").join("\n");
+        return labelTpl
+            .split("\n")
+            .map(line => this.formatLine(line, obj))
+            .filter(d => d.trim().length > 0)
+            .map(decodeHtml)
+            .join("\n")
+            ;
     }
 
     createSubgraph(subgraph: ScopeSubgraph): Subgraph {
@@ -306,10 +335,11 @@ export class WUScopeController {
     }
 
     createVertex(vertex: ScopeVertex): VertexType {
-        const attrs = vertex._.rawAttrs();
-        attrs["ID"] = vertex._.Id;
-        attrs["Parent ID"] = vertex.parent && vertex.parent._.Id;
-        attrs["Scope"] = vertex._.ScopeName;
+        const rawAttrs = vertex._.rawAttrs();
+        const formattedAttrs = this.formatNums(vertex._.formattedAttrs());
+        formattedAttrs["ID"] = vertex._.Id;
+        formattedAttrs["Parent ID"] = vertex.parent && vertex.parent._.Id;
+        formattedAttrs["Scope"] = vertex._.ScopeName;
         let v = this.verticesMap[vertex._.Id];
         if (!v) {
             if (vertex._.ScopeType === "dummy") {
@@ -329,7 +359,7 @@ export class WUScopeController {
                     .icon_shape_colorStroke(UNKNOWN_STROKE)
                     .icon_shape_colorFill(UNKNOWN_STROKE)
                     .icon_image_colorFill(Palette.textColor(UNKNOWN_STROKE))
-                    .faChar(faCharFactory(attrs["Kind"]))
+                    .faChar(faCharFactory(rawAttrs["Kind"]))
                     .textbox_shape_colorStroke(UNKNOWN_STROKE)
                     .textbox_shape_colorFill(UNKNOWN_FILL)
                     .textbox_text_colorFill(Palette.textColor(UNKNOWN_FILL))
@@ -359,7 +389,7 @@ export class WUScopeController {
             this.rVerticesMap[v.id()] = vertex;
         }
         if (v instanceof Vertex) {
-            const label = this.format(this.vertexLabelTpl(), attrs);
+            const label = this.format(this.vertexLabelTpl(), formattedAttrs);
             v
                 .icon_diameter(this.showIcon() ? 24 : 0)
                 .text(label)
@@ -379,10 +409,11 @@ export class WUScopeController {
     }
 
     createEdge(edge: ScopeEdge): Edge | undefined {
-        const attrs = edge._.rawAttrs();
-        attrs["ID"] = edge._.Id;
-        attrs["Parent ID"] = edge.parent && edge.parent._.Id;
-        attrs["Scope"] = edge._.ScopeName;
+        const rawAttrs = edge._.rawAttrs();
+        const formattedAttrs = this.formatNums(edge._.formattedAttrs());
+        formattedAttrs["ID"] = edge._.Id;
+        formattedAttrs["Parent ID"] = edge.parent && edge.parent._.Id;
+        formattedAttrs["Scope"] = edge._.ScopeName;
         let e = this.edgesMap[edge._.Id];
         if (!e) {
             const sourceV = this.verticesMap[edge.source._.Id];
@@ -393,10 +424,10 @@ export class WUScopeController {
 
                 let strokeDasharray = null;
                 let weight = 100;
-                if (attrs["IsDependency"]) {
+                if (rawAttrs["IsDependency"]) {
                     weight = 10;
                     strokeDasharray = "1,2";
-                } else if (attrs["_childGraph"]) {
+                } else if (rawAttrs["_childGraph"]) {
                     strokeDasharray = "5,5";
                 } else if (isSpill) {
                     weight = 25;
@@ -418,7 +449,7 @@ export class WUScopeController {
             }
         }
         if (e instanceof Edge) {
-            const label = this.format(this.edgeLabelTpl(), attrs);
+            const label = this.format(this.edgeLabelTpl(), formattedAttrs);
             e.text(label);
         }
         return e;
@@ -706,12 +737,12 @@ export class WUScopeController {
             //  TODO Move into BaseScope  ---
             retVal[key] = decodeHtml(retVal[key]);
         }
-        retVal.__formatted = item._.formattedAttrs();
+        retVal.__formatted = this.formatNums(item._.formattedAttrs());
         return retVal;
     }
 
     formatRow(item: ScopeEdge | ScopeSubgraph | ScopeVertex, columns, row) {
-        const attrs = item._.formattedAttrs();
+        const attrs = this.formatNums(item._.formattedAttrs());
         for (const key in attrs) {
             const idx = columns.indexOf(key);
             if (idx === -1) {
@@ -801,7 +832,7 @@ export class WUScopeController {
             rows.push(`<tr><td class="key">Parent ID:</td><td class="value">${highlightText("Parent ID", parentScope.Id)}</td></tr>`);
         }
         rows.push(`<tr><td class="key">Scope:</td><td class="value">${highlightText("Scope", scope.ScopeName)}</td></tr>`);
-        const attrs = scope.formattedAttrs();
+        const attrs = this.formatNums(scope.formattedAttrs());
         for (const key in attrs) {
             if (key === "Label") {
                 label = attrs[key];

roxie/topo/toposerver.cpp

@@ -69,7 +69,7 @@ unsigned lastTopologyReport = 0;
 const unsigned timeoutCheckInterval = 1000;
 const unsigned heartbeatInterval = 5000;
 const unsigned timeoutHeartbeatInterval = 10000;
-const unsigned topologyReportInterval = 10000;
+const unsigned topologyReportInterval = 60000;
 bool aborted = false;
 Semaphore stopping;
 StringBuffer topologyFile;
@@ -92,6 +92,8 @@ extern "C" void caughtSIGALRM(int sig)
 extern "C" void caughtSIGTERM(int sig)
 {
     DBGLOG("Caught sigterm %d", sig);
+    stopping.signal();
+    aborted = true;
 }
 
 extern "C" void caughtSIGABRT(int sig)
@@ -177,9 +179,8 @@ void regenerateResponse()
     }
 }
 
-void doServer()
+void doServer(ISocket *socket)
 {
-    Owned<ISocket> socket = ISocket::create(topoPort);
     std::thread pinger([]()
     {
         // Force a periodic refresh so we see missing heartbeats even when none are coming in, and we can interrupt the socket on closedown
@@ -284,81 +285,93 @@ int main(int argc, const char *argv[])
         E->Release();
         return EXIT_FAILURE;
     }
-    Owned<IProperties> globals = createProperties(true);
-    for (unsigned i=0; i<(unsigned)argc; i++)
+    Owned<IFile> sentinelFile = createSentinelTarget();
+    removeSentinelFile(sentinelFile);
+    try
     {
-        if (stricmp(argv[i], "--help")==0 ||
-            stricmp(argv[i], "-h")==0)
+        Owned<IProperties> globals = createProperties(true);
+        for (unsigned i=0; i<(unsigned)argc; i++)
         {
-            topo_server_usage();
-            return EXIT_SUCCESS;
-        }
-        else if (streq(argv[i],"--daemon") || streq(argv[i],"-d")) {
-            if (daemon(1,0) || write_pidfile(argv[++i])) {
-                perror("Failed to daemonize");
-                return EXIT_FAILURE;
+            if (stricmp(argv[i], "--help")==0 ||
+                stricmp(argv[i], "-h")==0)
+            {
+                topo_server_usage();
+                return EXIT_SUCCESS;
             }
+            else if (streq(argv[i],"--daemon") || streq(argv[i],"-d")) {
+                if (daemon(1,0) || write_pidfile(argv[++i])) {
+                    perror("Failed to daemonize");
+                    return EXIT_FAILURE;
+                }
+            }
+            else
+                globals->loadProp(argv[i], true);  // We ignore unrecognized options for now
+        }
+
+        // locate settings xml file in runtime dir
+        char currentDirectory[_MAX_DIR];
+        if (!getcwd(currentDirectory, sizeof(currentDirectory)))
+        {
+            perror("getcwd failure");
+            return EXIT_FAILURE;
         }
+        topologyFile.set(currentDirectory);
+        addNonEmptyPathSepChar(topologyFile);
+        topologyFile.append(PATHSEPCHAR).append("toposerver.xml");
+        IPropertyTree *topology;
+        if (checkFileExists(topologyFile.str()))
+            topology = createPTreeFromXMLFile(topologyFile.str(), ipt_lowmem);
         else
-            globals->loadProp(argv[i], true);  // We ignore unrecognized options for now
-    }
+            topology = createPTreeFromXMLString(
+                "<TopoServerProcess port='9004' traceLevel='1'>"
+                "</TopoServerProcess>"
+                , ipt_lowmem
+                );
+        if (globals->hasProp("--traceLevel"))
+            topology->setProp("@traceLevel", globals->queryProp("--traceLevel"));
+        if (globals->hasProp("--port"))
+            topology->setProp("@port", globals->queryProp("--port"));
+        if (globals->hasProp("--stdlog"))
+            topology->setProp("@stdlog", globals->queryProp("--stdlog"));
+        if (globals->hasProp("--logdir"))
+            topology->setProp("@logdir", globals->queryProp("--logdir"));
+
+        traceLevel = topology->getPropInt("@traceLevel", 1);
+        topoPort = topology->getPropInt("@port", TOPO_SERVER_PORT);
+        if (topology->getPropBool("@stdlog", traceLevel != 0))
+            queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time | MSGFIELD_milliTime | MSGFIELD_thread | MSGFIELD_prefix);
+        else
+            removeLog();
+        const char *logdir = topology->queryProp("@logdir");
+        if (logdir)
+        {
+            Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(logdir, "toposerver");
+            lf->setMaxDetail(TopDetail);
+            lf->beginLogging();
+            queryLogMsgManager()->enterQueueingMode();
+            queryLogMsgManager()->setQueueDroppingLimit(512, 32);
+        }
+        Owned<ISocket> socket = ISocket::create(topoPort);
+        if (traceLevel)
+            DBGLOG("Topology server starting");
 
-    // locate settings xml file in runtime dir
-    char currentDirectory[_MAX_DIR];
-    if (!getcwd(currentDirectory, sizeof(currentDirectory)))
+        writeSentinelFile(sentinelFile);
+        doServer(socket);
+        if (traceLevel)
+            DBGLOG("Topology server stopping");
+        removeSentinelFile(sentinelFile);
+    }
+    catch (IException *E)
     {
-        perror("getcwd failure");
-        return EXIT_FAILURE;
+        EXCLOG(E);
+        E->Release();
     }
-    topologyFile.set(currentDirectory);
-    addNonEmptyPathSepChar(topologyFile);
-    topologyFile.append(PATHSEPCHAR).append("toposerver.xml");
-    IPropertyTree *topology;
-    if (checkFileExists(topologyFile.str()))
-        topology = createPTreeFromXMLFile(topologyFile.str(), ipt_lowmem);
-    else
-        topology = createPTreeFromXMLString(
-            "<TopoServerProcess port='9004' traceLevel='1'>"
-            "</TopoServerProcess>"
-            , ipt_lowmem
-            );
-    if (globals->hasProp("--traceLevel"))
-        topology->setProp("@traceLevel", globals->queryProp("--traceLevel"));
-    if (globals->hasProp("--port"))
-        topology->setProp("@port", globals->queryProp("--port"));
-    if (globals->hasProp("--stdlog"))
-        topology->setProp("@stdlog", globals->queryProp("--stdlog"));
-    if (globals->hasProp("--logdir"))
-        topology->setProp("@logdir", globals->queryProp("--logdir"));
-
-
-    // take ownership of sentinel file
-    Owned<IFile> sentinelFile = createSentinelTarget();
-    removeSentinelFile(sentinelFile);
-
-    traceLevel = topology->getPropInt("@traceLevel", 1);
-    topoPort = topology->getPropInt("@port", TOPO_SERVER_PORT);
-    if (topology->getPropBool("@stdlog", traceLevel != 0))
-        queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time | MSGFIELD_milliTime | MSGFIELD_thread | MSGFIELD_prefix);
-    else
-        removeLog();
-    const char *logdir = topology->queryProp("@logdir");
-    if (logdir)
+    catch (...)
     {
-        Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(logdir, "toposerver");
-        lf->setMaxDetail(TopDetail);
-        lf->beginLogging();
-        queryLogMsgManager()->enterQueueingMode();
-        queryLogMsgManager()->setQueueDroppingLimit(512, 32);
+        Owned<IException> E = makeStringException(MSGAUD_programmer, 999, "Unexpected exception");
+        EXCLOG(E);
+        E->Release();
     }
-    if (traceLevel)
-        DBGLOG("Topology server starting");
-    
-    writeSentinelFile(sentinelFile);
-    doServer();
-    if (traceLevel)
-        DBGLOG("Topology server stopping");
-    removeSentinelFile(sentinelFile);
     ExitModuleObjects();
     return 0;
 }

rtl/eclrtl/CMakeLists.txt

@@ -92,8 +92,15 @@ if (USE_BOOST_REGEX)
 endif ()
 
 HPCC_ADD_LIBRARY( eclrtl SHARED ${SRCS} )
+
+if(CENTOS_6_BOOST)
+  add_dependencies(eclrtl boost-regex)
+  target_link_libraries( eclrtl boost-regex)
+else()
+  target_link_libraries( eclrtl ${BOOST_REGEX_LIBRARIES} )
+endif()
+
 target_link_libraries ( eclrtl 
-      ${BOOST_REGEX_LIBRARIES}
       jlib
       nbcd
       roxiemem

system/CMakeLists.txt

@@ -28,6 +28,7 @@ if (NOT JLIB_ONLY)
    
    if (USE_AERON)
        project (aeron_include)
+         set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lrt")
          remove_definitions(-fvisibility=hidden)
          HPCC_ADD_SUBDIRECTORY (aeron)
    endif()

thorlcr/thorutil/thmem.hpp

@@ -224,6 +224,7 @@ graph_decl StringBuffer &getRecordString(const void *key, IOutputRowSerializer *
 #define SPILL_PRIORITY_HASHJOIN SPILL_PRIORITY_HIGH
 #define SPILL_PRIORITY_LARGESORT SPILL_PRIORITY_HIGH
 #define SPILL_PRIORITY_LOOKUPJOIN SPILL_PRIORITY_HIGH
+#define SPILL_PRIORITY_KEYEDJOIN SPILL_PRIORITY_LOW+900
 
 
 enum StableSortFlag { stableSort_none, stableSort_earlyAlloc, stableSort_lateAlloc };

thorlcr/thorutil/thormisc.hpp

@@ -83,7 +83,10 @@
 #define THOROPT_KEYLOOKUP_MAX_FETCH_THREADS "maxFetchThreads"   // Maximum number of threads performing keyed lookups                            (default = 10)
 #define THOROPT_KEYLOOKUP_MAX_PROCESS_THREADS "keyLookupMaxProcessThreads" // Maximum number of threads performing keyed lookups                 (default = 10)
 #define THOROPT_KEYLOOKUP_MAX_QUEUED  "keyLookupMaxQueued"      // Total maximum number of rows (across all parts/threads) to queue              (default = 10000)
+#define THOROPT_KEYLOOKUP_MIN_MB      "keyLookupMinJoinGroupMB" // Min(MB) for groups (across all parts/threads) to queue)                       (default = 50)
 #define THOROPT_KEYLOOKUP_MAX_DONE    "keyLookupMaxDone"        // Maximum number of done items pending to be ready by next activity             (default = 10000)
+#define THOROPT_KEYLOOKUP_PROCESS_BATCHLIMIT "keyLookupProcessBatchLimit" // Maximum number of key lookups on queue before passing to a processor (default = 1000)
+#define THOROPT_FETCHLOOKUP_PROCESS_BATCHLIMIT "fetchLookupProcessBatchLimit" // Maximum number of fetch lookups on queue before passing to a processor (default = 10000)
 #define THOROPT_REMOTE_KEYED_LOOKUP   "remoteKeyedLookup"       // Send key request to remote node unless part is local                          (default = true)
 #define THOROPT_REMOTE_KEYED_FETCH    "remoteKeyedFetch"        // Send fetch request to remote node unless part is local                        (default = true)
 #define THOROPT_FORCE_REMOTE_KEYED_LOOKUP "forceRemoteKeyedLookup" // force all keyed lookups, even where part local to be sent as if remote     (default = false)