Merge branch 'candidate-6.0.2'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -412,6 +412,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=logical-op-parentheses -Werror=bool-conversions -Werror=return-type -Werror=comment")
       if (APPLE)
         SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+        SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
       endif ()
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Werror=bitwise-op-parentheses -Werror=tautological-compare")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Wno-switch-enum -Wno-format-zero-length -Wno-switch")

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -2753,20 +2753,20 @@ sudo /etc/init.d/hpcc-init -c esp start
         <para>The second set has the same output to STDOUT/STDERROR as
         directly invoking /etc/init.d/&lt;hpcc-init|dafilesrv&gt;</para>
 
-        <para>Use either: </para>
+        <para>Use either:</para>
 
         <programlisting>sudo service &lt;service&gt; &lt;start|stop|restart&gt;</programlisting>
 
-        <para>or </para>
+        <para>or</para>
 
         <programlisting>sudo systemctl &lt;start|stop|restart&gt; &lt;full_service_name&gt; </programlisting>
 
-        <para>The systemd displays the service status in it's own format.
-        </para>
+        <para>The systemd displays the service status in it's own
+        format.</para>
 
         <programlisting>sudo service &lt;service&gt; status </programlisting>
 
-        <para>or </para>
+        <para>or</para>
 
         <programlisting>sudo systemctl status &lt;full_service_name&gt; </programlisting>
 
@@ -2784,7 +2784,7 @@ sudo /etc/init.d/hpcc-init -c esp start
         <para>HPCC uninstall will automatically remove HPCC services from
         active list and /etc/systemd/system/ directory.</para>
 
-        <sect3>
+        <sect3 id="hpcc-init_sysd_svc_usage">
           <title id="hpcc_systemd_usage">hpcc-init systemd service
           usage</title>
 

system/jlib/jthread.cpp

@@ -1906,6 +1906,13 @@ public:
         if (haserror)
             if (::pipe(errpipe)==-1)
                 throw makeOsException(errno);
+
+        /* NB: Important to call splitargs (which calls malloc) before the fork()
+         * and not in the child process. Because performing malloc in the child
+         * process, which then calls exec() can cause problems for TBB malloc proxy.
+         */
+        unsigned argc;
+        char **argv=splitargs(prog,argc);
         loop
         {
             pipeProcess = (HANDLE)fork();
@@ -1948,8 +1955,6 @@ public:
                 close(errpipe[1]);
             }
 
-            unsigned argc;
-            char **argv=splitargs(prog,argc);
             if (dir.get()) {
                 if (chdir(dir) == -1)
                     throw MakeStringException(-1, "CLinuxPipeProcess::run: could not change dir to %s", dir.get());
@@ -1961,6 +1966,7 @@ public:
             execvp(argv[0],argv);
             _exit(START_FAILURE);    // must be _exit!!     
         }
+        free(argv);
         if (hasinput) 
             close(inpipe[0]);
         if (hasoutput) 

testing/regress/ecl/key/smartjoin.xml

@@ -62,33 +62,48 @@
  <Row><i>2</i><idl>Y</idl><idr>Y</idr></Row>
 </Dataset>
 <Dataset name='Result 6'>
+ <Row><i>1</i><idl>A</idl><idr>a</idr></Row>
+ <Row><i>1</i><idl>A</idl><idr>b</idr></Row>
+ <Row><i>1</i><idl>A</idl><idr>c</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>a</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>b</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>c</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>a</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>b</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>c</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>x</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>y</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>x</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>y</idr></Row>
+</Dataset>
+<Dataset name='Result 7'>
  <Row><i>1</i><idl>A</idl><idr>abc</idr></Row>
  <Row><i>1</i><idl>B</idl><idr>abc</idr></Row>
  <Row><i>1</i><idl>C</idl><idr>abc</idr></Row>
  <Row><i>2</i><idl>X</idl><idr>xy</idr></Row>
  <Row><i>2</i><idl>Y</idl><idr>xy</idr></Row>
 </Dataset>
-<Dataset name='Result 7'>
+<Dataset name='Result 8'>
  <Row><cnt>5</cnt></Row>
 </Dataset>
-<Dataset name='Result 8'>
+<Dataset name='Result 9'>
  <Row><i>1</i><idl>A</idl><idr>ABC</idr></Row>
  <Row><i>1</i><idl>B</idl><idr>ABC</idr></Row>
  <Row><i>1</i><idl>C</idl><idr>ABC</idr></Row>
  <Row><i>2</i><idl>X</idl><idr>XY</idr></Row>
  <Row><i>2</i><idl>Y</idl><idr>XY</idr></Row>
 </Dataset>
-<Dataset name='Result 9'>
+<Dataset name='Result 10'>
  <Row><i>1</i><idl>A</idl><idr>abc</idr></Row>
  <Row><i>1</i><idl>B</idl><idr>abc</idr></Row>
  <Row><i>1</i><idl>C</idl><idr>abc</idr></Row>
  <Row><i>2</i><idl>X</idl><idr>xy</idr></Row>
  <Row><i>2</i><idl>Y</idl><idr>xy</idr></Row>
 </Dataset>
-<Dataset name='Result 10'>
+<Dataset name='Result 11'>
  <Row><cnt>5</cnt></Row>
 </Dataset>
-<Dataset name='Result 11'>
+<Dataset name='Result 12'>
  <Row><i>1</i><idl>A</idl><idr>ABC</idr></Row>
  <Row><i>1</i><idl>B</idl><idr>ABC</idr></Row>
  <Row><i>1</i><idl>C</idl><idr>ABC</idr></Row>

testing/regress/ecl/smartjoin.ecl

@@ -43,6 +43,7 @@ j2 := JOIN(gr1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), SMART);
 j3 := JOIN(gr1, gr1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), SMART);
 j4 := JOIN(gr1, gr1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), SMART, HINT(lkjoin_localfailover));
 j5 := JOIN(gr1, gr1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), SMART, HINT(lkjoin_hashjoinfailover));
+j6 := JOIN(DISTRIBUTE(ds1, i), DISTRIBUTE(ds2, i), LEFT.i = RIGHT.i, trans(LEFT, RIGHT), SMART, LOCAL);
 
 p1 := PROJECT(ds1, createOut(LEFT.i, LEFT.id, ''));
 gp1 := GROUP(p1, i, idL);
@@ -79,6 +80,7 @@ sequential(
     output(SORT(j3, i)),
     output(SORT(j4, i)),
     output(SORT(j5, i)),
+    output(SORT(j6, i)),
     output(SORT(d1, i)),
     output(SORT(TABLE(d2, { cnt := COUNT(GROUP) }), cnt)),  // check output is not grouped
     output(SORT(d3, i)),

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -1939,7 +1939,7 @@ protected:
     {
         try
         {
-            if (!marker.init(rhs.ordinality(), queryRowManager()))
+            if (!marker.init(rightCollector.numRows(), queryRowManager()))
                 return false;
         }
         catch (IException *e)
@@ -1948,7 +1948,6 @@ protected:
             e->Release();
             return false;
         }
-        // Either was already sorted, or rowLoader->load() sorted on transfer out to rhs
 
         rowidx_t uniqueKeys = 0;
         {
@@ -2132,7 +2131,11 @@ protected:
         {
             const void *next = right->nextRow();
             if (!next)
-                break;
+            {
+                next = right->nextRow();
+                if (!next)
+                    break;
+            }
             writer->putRow(next);
         }
         return channelCollector.getClear();
@@ -2416,9 +2419,9 @@ protected:
                 }
                 ICompare *cmp = helper->isRightAlreadyLocallySorted() ? NULL : compareRight;
                 rightCollector.setown(handleLocalRHS(right, cmp));
-                rightStream.setown(rightCollector->getStream(&rhs));
-                if (rightStream)
+                if (rightCollector->hasSpilt())
                 {
+                    rightStream.setown(rightCollector->getStream());
                     ActPrintLog("Local SMART JOIN spilt to disk. Failing over to regular local join");
                     setFailoverToStandard(true);
                 }
@@ -2435,7 +2438,7 @@ protected:
                         marker.reset();
                     if (!prepareLocalHT(marker, *rightCollector)) // can cause others to spill, but must not be allowed to spill channel rows I'm working on.
                         ActPrintLog("Out of memory trying to prepare [LOCAL] hashtable for a SMART join (%" RIPF "d rows), will now failover to a std hash join", rhs.ordinality());
-                    rightStream.setown(rightCollector->getStream(&rhs));
+                    rightStream.setown(rightCollector->getStream(false, &rhs));
                 }
             }
             if (rightStream)