Merge branch 'candidate-5.4.2'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/dfuplus/dfuplus.cpp

@@ -343,6 +343,9 @@ bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char*
     if(globals->hasProp("transferbuffersize"))
         req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
 
+    if(globals->hasProp("failIfNoSourceFile"))
+        req->setFailIfNoSourceFile(globals->getPropBool("failIfNoSourceFile",false));
+
     if(srcxml == NULL)
         info("\nFixed spraying from %s on %s to %s\n", srcfile, srcip, dstname);
     else

dali/ft/filecopy.cpp

@@ -2902,6 +2902,9 @@ bool FileSprayer::isSameSizeHeaderFooter()
     headerSize = 0;
     footerSize = 0;
 
+    if (sources.ordinality() == 0)
+        return retVal;
+
     ForEachItemIn(idx, partition)
     {
         PartitionPoint & cur = partition.item(idx);

docs/ECLLanguageReference/ECLR_mods/BltInFunc-IMPORT.xml

@@ -64,7 +64,7 @@
   This may be used to call Java or Python code, but is not usable with
   Javascript or R code (use the EMBED structure instead). Java code must be
   placed in a .java file and compiled using the javac compiler in the usual
-  way.</para>
+  way. All Java classes used must be thread safe.</para>
 
   <para><emphasis role="bold">WARNING: This feature could create memory
   corruption and/or security issues, so great care and forethought are

docs/ECLLanguageReference/ECLR_mods/BltInFunc-LOOP.xml

@@ -36,9 +36,8 @@
   role="bold">)</emphasis><emphasis role="bold"></emphasis></para>
 
   <para><emphasis role="bold">LOOP(</emphasis><emphasis>
-  dataset,</emphasis><emphasis role="bold">
-  </emphasis><emphasis>loopcondition, rowfilter, loopbody </emphasis><emphasis
-  role="bold">)</emphasis></para>
+  dataset,</emphasis><emphasis role="bold"> </emphasis><emphasis>rowfilter,
+  loopcondition, loopbody </emphasis><emphasis role="bold">)</emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <tgroup cols="2">

docs/ECLLanguageReference/ECLR_mods/ExtrSvcs-ExternalServicesImpl.xml

@@ -14,7 +14,8 @@
     </indexterm><indexterm>
       <primary>Shared Object</primary>
     </indexterm>. An ECL system service .SO can contain one or more services
-  and (possibly) a single .SO initialization routine.</para>
+  and (possibly) a single .SO initialization routine. All system service
+  libraries must be thread safe.</para>
 
   <para>All exported functions in the .SO (hereafter referred to as "entry
   points") must adhere to certain calling and naming conventions. First, entry
@@ -171,7 +172,8 @@
               <entry><emphasis>CONTEXT</emphasis></entry>
 
               <entry>Internal use, only. Indicates an extra internal context
-              parameter (ICodeContext *) is passed to the function. This must be the first function parameter.</entry>
+              parameter (ICodeContext *) is passed to the function. This must
+              be the first function parameter.</entry>
             </row>
 
             <row>
@@ -358,13 +360,14 @@
     <para>The function must fill the passed structure with correct information
     for the features of the plug-in. The structure is defined as
     follows:</para>
-    
-    <para><emphasis role="bold">Warning:</emphasis> This function may be called 
-    without the plugin being loaded fully.  It should not make any library 
-    calls or assume that dependent modules have been loaded or that it has been 
-    initialised.  Specifically:  "The system does not call DllMain for process 
-    and thread initialization and termination.  Also, the system does not load 
-    additional executable modules that are referenced by the specified module."</para>
+
+    <para><emphasis role="bold">Warning:</emphasis> This function may be
+    called without the plugin being loaded fully. It should not make any
+    library calls or assume that dependent modules have been loaded or that it
+    has been initialised. Specifically: "The system does not call DllMain for
+    process and thread initialization and termination. Also, the system does
+    not load additional executable modules that are referenced by the
+    specified module."</para>
 
     <programlisting>Struct ECLPluginDefinitionBlock
   {

docs/ECLLanguageReference/ECLR_mods/SpecStruc-BeginC++.xml

@@ -60,6 +60,10 @@
   plugin would be a better solution (see the <emphasis role="bold">External
   Service Implementation</emphasis> discussion).</para>
 
+  <para>The implementation must be written to be thread safe and any calls to
+  external libraries must be made to thread safe versions of those
+  libraries.</para>
+
   <para><emphasis role="bold">WARNING: This feature could create memory
   corruption and/or security issues, so great care and forethought are
   advised—consult with Technical Support before using.</emphasis></para>
@@ -165,7 +169,7 @@ STREAMED DATASET(r)
     <para>A function that takes a string parameter should also have the type
     prefixed by <emphasis role="bold">const</emphasis> in the ECL code so that
     modern compilers don't report errors when constant strings are passed to
-    the function. </para>
+    the function.</para>
 
     <programlisting>BOOLEAN isUpper(const string mystring) := BEGINC++
   size_t i=0;

docs/ECLLanguageReference/ECLR_mods/Templ-OPTION.xml

@@ -1261,6 +1261,16 @@
               <entry>Number of activities in each c++ file (requires
               spanMultipleCpp)</entry>
             </row>
+
+            <row>
+              <entry><emphasis>obfuscateOutput</emphasis></entry>
+
+              <entry>Default false</entry>
+
+              <entry>If true, details are removed from the generated workunit,
+              including ECL code, estimates of record size, and number of
+              records.</entry>
+            </row>
           </tbody>
         </tgroup>
       </informaltable></para>

docs/HPCCDataHandling/DH-Mods/DH-Mod1.xml

@@ -470,9 +470,9 @@
 
           <para><informaltable colsep="0" frame="none" rowsep="0">
               <tgroup cols="2">
-                <colspec colwidth="122.40pt" />
+                <colspec colwidth="1*" />
 
-                <colspec colwidth="333.00pt" />
+                <colspec colwidth="3*" />
 
                 <tbody>
                   <row>
@@ -536,6 +536,13 @@
                   </row>
 
                   <row>
+                    <entry><emphasis role="bold">Escape</emphasis></entry>
+
+                    <entry>A null-terminated string containing the CSV escape
+                    characters.</entry>
+                  </row>
+
+                  <row>
                     <entry><emphasis role="bold">Line
                     Terminators</emphasis></entry>
 
@@ -598,6 +605,14 @@
                     it allows quicker partitioning of the file (avoiding a
                     complete file scan).</entry>
                   </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Record Structure
+                    Present</emphasis></entry>
+
+                    <entry>Flag indicating whether to derive the record
+                    structure from the header of the file.</entry>
+                  </row>
                 </tbody>
               </tgroup>
             </informaltable></para>

docs/IMDB/IMDB.xml

@@ -454,15 +454,13 @@ Blankline</programlisting></para>
           <listitem>
             <?dbfo keep-together="always"?>
 
-            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> and
-            <emphasis role="bold">Replicate</emphasis><emphasis role="bold">
-            </emphasis>boxes are checked.</para>
+            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> box
+            is checked. </para>
 
-            <para><emphasis role="bold">Note:</emphasis> The Replicate option
-            is only available on systems where replication has been
-            enabled.</para>
-
-            <para></para>
+            <para>If available, make sure the <emphasis
+            role="bold">Replicate</emphasis> box is checked. (The Replicate
+            option is only available on systems where replication has been
+            enabled.)</para>
           </listitem>
 
           <listitem>

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -563,8 +563,7 @@
               <para>A Workunit is a record of a task submitted to an HPCC. It
               contains an identifier--workunit ID, the ECL code, results, and
               other information about the job.</para>
-            </footnote> ID, status, and result are returned to the command
-          line.</para>
+            </footnote> result is returned to the command line.</para>
 
           <para>View the full details of the workunit using the ECL Watch
           interface for your HPCC at this location
@@ -1646,13 +1645,13 @@ OUTPUT(L);</programlisting></para>
           <listitem>
             <?dbfo keep-together="always"?>
 
-            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> and
-            <emphasis role="bold">Replicate</emphasis><emphasis role="bold">
-            </emphasis>boxes are checked.</para>
+            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> box
+            is checked.</para>
 
-            <para><emphasis role="bold">Note:</emphasis> The <emphasis
-            role="bold">Replicate</emphasis> option is only available on
-            systems where replication has been enabled.</para>
+            <para>If available, make sure the <emphasis
+            role="bold">Replicate</emphasis> box is checked. (The Replicate
+            option is only available on systems where replication has been
+            enabled.)</para>
           </listitem>
 
           <listitem>

docs/InstantCloud/InstantCloud.xml

@@ -1236,13 +1236,13 @@ OUTPUT(L);</programlisting></para>
           <listitem>
             <?dbfo keep-together="always"?>
 
-            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> and
-            <emphasis role="bold">Replicate</emphasis><emphasis role="bold">
-            </emphasis>boxes are checked.</para>
+            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> box
+            is checked.</para>
 
-            <para><emphasis role="bold">Note:</emphasis> The <emphasis
-            role="bold">Replicate</emphasis> option is only available on
-            systems where replication has been enabled.</para>
+            <para>If available, make sure the <emphasis
+            role="bold">Replicate</emphasis> box is checked. (The Replicate
+            option is only available on systems where replication has been
+            enabled.)</para>
           </listitem>
 
           <listitem>

docs/One-Click_HPCCinAWS/One-Click_RuningHPCCinAmazonWebServicesEC2.xml

@@ -29,7 +29,7 @@
       message.</para>
 
       <para>LexisNexis and the Knowledge Burst logo are registered trademarks
-      of Reed Elsevier Properties Inc., used under license. </para>
+      of Reed Elsevier Properties Inc., used under license.</para>
 
       <para>HPCC Systems is a registered trademark of LexisNexis Risk Data
       Management Inc.</para>
@@ -40,7 +40,7 @@
       countries.</para>
 
       <para>Other products, logos, and services may be trademarks or
-      registered trademarks of their respective companies. </para>
+      registered trademarks of their respective companies.</para>
 
       <para>All names and example data used in this manual are fictitious. Any
       similarity to actual persons, living or dead, is purely
@@ -1236,9 +1236,11 @@ OUTPUT(L);</programlisting></para>
           </listitem>
 
           <listitem>
-            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> and
-            <emphasis role="bold">Replicate</emphasis><emphasis role="bold">
-            </emphasis>boxes are checked.</para>
+            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> box
+            is checked. If available, make sure the <emphasis
+            role="bold">Replicate</emphasis> box is checked. (The Replicate
+            option is only available on systems where replication has been
+            enabled.)</para>
 
             <para><figure>
                 <title>Spray the File</title>

docs/RuningHPCCinAmazonWebServicesEC2/RuningHPCCinAmazonWebServicesEC2.xml

@@ -2698,9 +2698,11 @@ OUTPUT(L);</programlisting></para>
           </listitem>
 
           <listitem>
-            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> and
-            <emphasis role="bold">Replicate</emphasis><emphasis role="bold">
-            </emphasis>boxes are checked.</para>
+            <para>Make sure the <emphasis role="bold">Overwrite</emphasis> box
+            is checked. If available, make sure the <emphasis
+            role="bold">Replicate</emphasis> box is checked. (The Replicate
+            option is only available on systems where replication has been
+            enabled.)</para>
 
             <para><figure>
                 <title>Spray the File</title>

docs/RunningHPCCinaVirtualMachine/RunningHPCCinaVirtualMachine.xml

@@ -1478,12 +1478,12 @@ OUTPUT(L);</programlisting></para>
               <?dbfo keep-together="always"?>
 
               <para>Make sure the <emphasis role="bold">Overwrite</emphasis>
-              and <emphasis role="bold">Replicate</emphasis><emphasis
-              role="bold"> </emphasis>boxes are checked.</para>
+              box is checked.</para>
 
-              <para><emphasis role="bold">Note:</emphasis> The <emphasis
-              role="bold">Replicate</emphasis> option is only available on
-              systems where replication has been enabled.</para>
+              <para>If available, make sure the <emphasis
+              role="bold">Replicate</emphasis> box is checked. (The Replicate
+              option is only available on systems where replication has been
+              enabled.)</para>
             </listitem>
 
             <listitem>

ecl/hql/hqlgram2.cpp

@@ -2908,7 +2908,7 @@ void HqlGram::processForwardModuleDefinition(const attribute & errpos)
         switch (next)
         {
         case ASSIGN:
-            if ((sharedSymbolKind != 0) && prevId && (endNesting == 0))
+            if ((sharedSymbolKind != 0) && !sharedSymbolName && prevId && (endNesting == 0))
             {
                 sharedSymbolName = prevId;
             }

ecl/hqlcpp/hqlcppcase.cpp

@@ -1054,7 +1054,7 @@ bool HqlCppCaseInfo::canBuildArrayLookup(const CHqlBoundExpr & test)
 
 bool HqlCppCaseInfo::queryBuildArrayLookup(BuildCtx & ctx, const CHqlBoundTarget & target, const CHqlBoundExpr & test)
 {
-    if (canBuildArrayLookup(test) && canBuildStaticList(resultType))
+    if (canBuildArrayLookup(test) && canBuildStaticList(resultType) && defaultValue->isConstant())
     {
         //MORE: Also support this for high density tables that don't start at 0... - checking upper and lower bounds
         ITypeInfo * condType = test.queryType()->queryPromotedType();

ecl/hqlcpp/hqlinline.cpp

@@ -1480,6 +1480,11 @@ bool EvalContext::evaluateInParent(BuildCtx & ctx, IHqlExpression * expr, bool h
     case no_loopcounter:
         return !translator.isCurrentActiveGraph(ctx, expr->queryChild(0));
     case no_getgraphresult:
+        if (expr->hasAttribute(externalAtom))
+        {
+            IHqlExpression * resultInstance = queryAttributeChild(expr, externalAtom, 0);
+            return !ctx.queryMatchExpr(resultInstance);
+        }
         return !translator.isCurrentActiveGraph(ctx, expr->queryChild(1));
     case no_getresult:
     case no_workunit_dataset:

ecl/hqlcpp/hqlresource.cpp

@@ -2740,7 +2740,8 @@ IHqlExpression * SpillerInfo::createSpilledWrite(IHqlExpression * transformed, b
 
 void SpillerInfo::setPotentialSpillFile(IHqlExpression * expr)
 {
-    if (!expr || canUseResultInChildQuery(expr) || !isUsedFromChild())
+    if (!expr ||
+        ((canUseResultInChildQuery(expr) || !isUsedFromChild()) && (expr->getOperator() != no_setgraphresult)))
         outputToUseForSpill = expr;
 }
 

ecl/regress/issue14086.ecl

@@ -0,0 +1,32 @@
+l2n(string1 letter) := case(letter,
+                            'A' => 0,       'K' => 10,       'U' => 20,
+                            'B' => 1,       'L' => 11,       'V' => 21,
+                            'C' => 2,       'M' => 12,       'W' => 22,
+                            'D' => 3,       'N' => 13,       'X' => 23,
+                            'E' => 4,       'O' => 14,       'Y' => 24,
+                            'F' => 5,       'P' => 15,       'Z' => 25,
+                            'G' => 6,       'Q' => 16,
+                            'H' => 7,       'R' => 17,
+                            'I' => 8,       'S' => 18,
+                            'J' => 9,       'T' => 19,
+                            error('Valid letter not provided: [' + letter + ']'));
+                            // -1);       // works if not using error
+
+sampleExternals :=
+  dataset([
+    {'B000'}
+  ],{string4 db_ext});
+
+// ds0 := sampleExternals;     // works if read inline
+ds0 := dataset('~jira::sample',{string4 db_ext},thor);
+
+ds := project(ds0,
+              transform({string4 orig, unsigned4 db_int},
+                        self.orig := left.db_ext,
+                        self.db_int := l2n(left.db_ext[1]),
+                        ));
+
+sequential (
+  output (sampleExternals,,'~jira::sample',overwrite),
+  output(ds,all)
+);

ecl/regress/issue14161.ecl

@@ -0,0 +1,22 @@
+testforward := MODULE, FORWARD
+
+Ins001_Structure:=RECORD
+ INTEGER2 ordernumber;
+ STRING orderdate;
+ STRING requireddate;
+ STRING shippeddate;
+END;
+;
+
+EXPORT Ins001_dsOutput := DATASET('~birt::orders',Ins001_Structure,THOR);
+
+EXPORT Ins003_dsOutput:=PROJECT(Ins001_dsOutput,TRANSFORM({RECORDOF(Ins001_dsOutput),STRING year,STRING shippedyear},
+SELF.year:=LEFT.orderdate[1..4];
+SELF.shippedyear:=LEFT.shippeddate[1..4];
+SELF:=LEFT;));
+
+  EXPORT x := Ins003_dsOutput;
+
+END;
+
+output(testforward.x);

esp/services/ws_machine/ws_machineService.cpp

@@ -1121,15 +1121,18 @@ void Cws_machineEx::buildPreflightCommand(IEspContext& context, CMachineInfoThre
         if (!process.getName() || !*process.getName())
             continue;
 
-        if (idx < 1)
-            preflightCommand.appendf(" -n=%s", process.getName());
+        StringBuffer procName;
+        if (streq(process.getType(), eqThorSlaveProcess))
+            procName.appendf("thorslave_%s_%d,%s_slave_%d", process.getName(), process.getProcessNumber(), process.getName(), process.getProcessNumber());
+        else if (streq(process.getType(), eqThorMasterProcess))
+            procName.appendf("%s,%s_master", process.getName(), process.getName());
         else
-            preflightCommand.appendf(",%s", process.getName());
+            procName.append(process.getName());
 
-        if (process.getType() && streq(process.getType(), eqThorMasterProcess))
-            preflightCommand.append("_master");
-        else if (process.getType() && streq(process.getType(), eqThorSlaveProcess))
-            preflightCommand.appendf("_slave_%d", process.getProcessNumber());
+        if (idx < 1)
+            preflightCommand.appendf(" -n=%s", procName.str());
+        else
+            preflightCommand.appendf(",%s", procName.str());
 
         if (!process.getDependencies().empty())
             checkDependency = true;
@@ -1428,6 +1431,14 @@ bool Cws_machineEx::readStorageSpace(const char *line, StringBuffer& title, __in
     return true;
 }
 
+void Cws_machineEx::buildProcessPath(StringBuffer &processPath, const char * processName, CMachineInfoThreadParam * pParam)
+{
+    if (environmentConfData.m_pidPath.charAt(environmentConfData.m_pidPath.length() - 1) != pParam->m_machineData.getPathSep())
+        processPath.setf("%s%c%s:", environmentConfData.m_pidPath.str(), pParam->m_machineData.getPathSep(), processName);
+    else
+        processPath.setf("%s%s:", environmentConfData.m_pidPath.str(), processName);
+}
+
 void Cws_machineEx::readProcessData(const char* response, CMachineInfoThreadParam* pParam)
 {
     if (!response || !*response)
@@ -1440,19 +1451,44 @@ void Cws_machineEx::readProcessData(const char* response, CMachineInfoThreadPara
         if (!process.getName() || !*process.getName())
             continue;
 
-        StringBuffer processData, processPath;
-        if (environmentConfData.m_pidPath.charAt(environmentConfData.m_pidPath.length() - 1) != pParam->m_machineData.getPathSep())
-            processPath.appendf("%s%c%s", environmentConfData.m_pidPath.str(), pParam->m_machineData.getPathSep(), process.getName());
+        StringBuffer procName, catError, processPath, processData;
+        if (streq(process.getType(), eqThorSlaveProcess))
+        {
+            procName.appendf("thorslave_%s_%d", process.getName(), process.getProcessNumber());
+            buildProcessPath(processPath,procName.str(),pParam);
+            catError.setf("cat: %s",processPath.str());
+            catError.insert(catError.length()-1,".pid");
+            if (!strstr(response,catError.str()))
+                readALineFromResult(response, processPath.str(), processData, true);
+            else
+            {
+                procName.setf("%s_slave_%d", process.getName(), process.getProcessNumber());
+                buildProcessPath(processPath,procName.str(),pParam);
+                readALineFromResult(response, processPath.str(), processData, true);
+            }
+        }
+        else if (streq(process.getType(), eqThorMasterProcess))
+        {
+            procName.appendf("%s", process.getName());
+            buildProcessPath(processPath,procName.str(),pParam);
+            catError.setf("cat: %s",processPath.str());
+            catError.insert(catError.length()-1,".pid");
+            if (!strstr(response,catError.str()))
+                readALineFromResult(response, processPath.str(), processData, true);
+            else
+            {
+                procName.setf("%s_master", process.getName());
+                buildProcessPath(processPath,procName.str(),pParam);
+                readALineFromResult(response, processPath.str(), processData, true);
+            }
+        }
         else
-            processPath.appendf("%s%s", environmentConfData.m_pidPath.str(), process.getName());
-
-        if (process.getType() && streq(process.getType(), eqThorMasterProcess))
-            processPath.append("_master");
-        else if (process.getType() && streq(process.getType(), eqThorSlaveProcess))
-            processPath.appendf("_slave_%d", process.getProcessNumber());
-        processPath.append(":");
+        {
+            procName.append(process.getName());
+            buildProcessPath(processPath,procName.str(),pParam);
+            readALineFromResult(response, processPath.str(), processData, true);
+        }
 
-        readALineFromResult(response, processPath.str(), processData, true);
         if (processData.length() < 1)
         {
             DBGLOG("Information for process %s not found", processPath.str());

esp/services/ws_machine/ws_machineService.hpp

@@ -783,6 +783,7 @@ private:
     void parseAddresses(const char *address, StringBuffer& address1, StringBuffer& address2);
     void readPreflightResponse(IEspContext& context, CMachineInfoThreadParam* pParam, const char *response,int error);
     void readStorageData(const char* response, CMachineInfoThreadParam* pParam);
+    void buildProcessPath(StringBuffer &buf, const char * procName,CMachineInfoThreadParam * pParam);
     void readProcessData(const char* response, CMachineInfoThreadParam* pParam);
     void readRunningProcesses(const char* response, CMachineInfoThreadParam* pParam);
     bool readStorageSpace(const char *line, StringBuffer& title, __int64& free, __int64& total, int& percentAvail);

esp/src/eclwatch/LZBrowseWidget.js

@@ -78,7 +78,6 @@ define([
         templateString: template,
         baseClass: "LZBrowseWidget",
         i18n: nlsHPCC,
-        firstLoad: true,
 
         postCreate: function (args) {
             this.inherited(arguments);
@@ -557,12 +556,6 @@ define([
                     displayName: tree({
                         label: this.i18n.Name,
                         sortable: false,
-                        shouldExpand: function (row, level, previouslyExpanded) {
-                            if (context.firstLoad === true) {
-                                return true;
-                            }
-                            return previouslyExpanded;
-                        },
                         formatter: function (_name, row) {
                             var img = "";
                             var name = _name;
@@ -694,7 +687,6 @@ define([
         },
 
         refreshGrid: function (clearSelection) {
-            this.firstLoad = false;
             this.landingZonesGrid.set("query", {
                 id: "*"
             });

roxie/ccd/ccdfile.cpp

@@ -794,6 +794,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
             IException *E = MakeStringException(ROXIE_DISKSPACE_ERROR, "%s", err.str());
             EXCLOG(MCoperatorError, E);
             E->Release();
+            f->setCopying(false);
         }
         else
         {

roxie/ccd/ccdmain.cpp

@@ -965,6 +965,8 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         }
         if (!numChannels)
             throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numChannels calculated at 0");
+        if (numChannels > 1 && localSlave)
+            throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - localSlave requires single channel (%d channels specified)", numChannels);
         // Now we know all the channels, we can open and subscribe the multicast channels
         if (!localSlave)
             openMulticastSocket();

roxie/ccd/ccdqueue.cpp

@@ -992,7 +992,7 @@ public:
                 header.toString(s);
                 logctx.logOperatorException(E, NULL, 0, "%s", s.str());
                 header.setException();
-                if (!header.allChannelsFailed())
+                if (!header.allChannelsFailed() && !localSlave)
                 {
                     if (logctx.queryTraceLevel() > 1) 
                         logctx.CTXLOG("resending packet from slave in case others want to try it");

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -952,12 +952,10 @@ public:
                     }
                 }
                 else
-                {
                     totalCount += (rowcount_t)klManager->getCount();
-                    if ((totalCount > choosenLimit))
-                        break;
-                }
                 callback.clearManager();
+                if ((totalCount > choosenLimit))
+                    break;
             }
             if (!container.queryLocalOrGrouped())
             {

thorlcr/activities/msort/thgroupsortslave.cpp

@@ -114,8 +114,6 @@ class CSortedSlaveActivity : public CSlaveActivity, public CThorDataLink, public
     IThorDataLink *input;
     IHThorSortedArg *helper;
     ICompare *icompare;
-
-    IRangeCompare *stepCompare;
     OwnedConstThorRow prev; 
 
 public:
@@ -126,7 +124,6 @@ public:
     {
         helper = (IHThorSortedArg *)queryHelper();
         icompare = helper->queryCompare();
-        stepCompare = NULL;
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
@@ -140,9 +137,6 @@ public:
         dataLinkStart();
         input = inputs.item(0);
         startInput(input);
-        IInputSteppingMeta *stepMeta = input->querySteppingMeta();
-        if (stepMeta)
-            stepCompare = stepMeta->queryCompare();
     }
     void stop()
     {

thorlcr/graph/thgraph.cpp

@@ -373,7 +373,7 @@ CGraphElementBase::CGraphElementBase(CGraphBase &_owner, IPropertyTree &_xgmml)
     whichBranch = (unsigned)-1;
     whichBranchBitSet.setown(createThreadSafeBitSet());
     newWhichBranch = false;
-    isEof = false;
+    hasNullInput = false;
     log = true;
     sentActInitData.setown(createThreadSafeBitSet());
 }
@@ -476,7 +476,7 @@ void CGraphElementBase::doconnect()
     {
         CIOConnection *io = connectedInputs.item(i);
         if (io)
-            io->connect(i, activity);
+            io->connect(i, queryActivity());
     }
 }
 
@@ -619,7 +619,7 @@ bool CGraphElementBase::prepareContext(size32_t parentExtractSz, const byte *par
                 return false;
         }
         whichBranch = (unsigned)-1;
-        isEof = false;
+        hasNullInput = false;
         alreadyUpdated = false;
         switch (getKind())
         {
@@ -688,16 +688,16 @@ bool CGraphElementBase::prepareContext(size32_t parentExtractSz, const byte *par
                 switch (getKind())
                 {
                     case TAKfilter:
-                        isEof = !((IHThorFilterArg *)baseHelper.get())->canMatchAny();
+                        hasNullInput = !((IHThorFilterArg *)baseHelper.get())->canMatchAny();
                         break;
                     case TAKfiltergroup:
-                        isEof = !((IHThorFilterGroupArg *)baseHelper.get())->canMatchAny();
+                        hasNullInput = !((IHThorFilterGroupArg *)baseHelper.get())->canMatchAny();
                         break;
                     case TAKfilterproject:
-                        isEof = !((IHThorFilterProjectArg *)baseHelper.get())->canMatchAny();
+                        hasNullInput = !((IHThorFilterProjectArg *)baseHelper.get())->canMatchAny();
                         break;
                 }
-                if (isEof)
+                if (hasNullInput)
                     return true;
                 break;
             }
@@ -831,45 +831,45 @@ void CGraphElementBase::createActivity(size32_t parentExtractSz, const byte *par
                 break;
             }
             default:
-                if (!isEof)
+                if (!hasNullInput)
                 {
                     ForEachItemIn(i, inputs)
                     {
                         CGraphElementBase *input = inputs.item(i)->activity;
                         input->createActivity(parentExtractSz, parentExtract);
                     }
-                }
-                onCreate();
-                if (isDiskInput(getKind()))
-                    onStart(parentExtractSz, parentExtract);
-                ForEachItemIn(i2, inputs)
-                {
-                    CIOConnection *inputIO = inputs.item(i2);
-                    loop
+                    onCreate();
+                    if (isDiskInput(getKind()))
+                        onStart(parentExtractSz, parentExtract);
+                    ForEachItemIn(i2, inputs)
                     {
-                        CGraphElementBase *input = inputIO->activity;
-                        switch (input->getKind())
+                        CIOConnection *inputIO = inputs.item(i2);
+                        loop
                         {
-                            case TAKif:
-                            case TAKcase:
+                            CGraphElementBase *input = inputIO->activity;
+                            switch (input->getKind())
                             {
-                                if (input->whichBranch >= input->getInputs()) // if, will have TAKnull activity, made at create time.
+                                case TAKif:
+                                case TAKcase:
                                 {
-                                    input = NULL;
+                                    if (input->whichBranch >= input->getInputs()) // if, will have TAKnull activity, made at create time.
+                                    {
+                                        input = NULL;
+                                        break;
+                                    }
+                                    inputIO = input->inputs.item(input->whichBranch);
+                                    assertex(inputIO);
                                     break;
                                 }
-                                inputIO = input->inputs.item(input->whichBranch);
-                                assertex(inputIO);
-                                break;
+                                default:
+                                    input = NULL;
+                                    break;
                             }
-                            default:
-                                input = NULL;
+                            if (!input)
                                 break;
                         }
-                        if (!input)
-                            break;
+                        connectInput(i2, inputIO->activity, inputIO->index);
                     }
-                    connectInput(i2, inputIO->activity, inputIO->index);
                 }
                 initActivity();
                 break;
@@ -1591,14 +1591,14 @@ public:
     CGraphTraverseConnectedIterator(CGraphBase &graph) : CGraphTraverseIteratorBase(graph) { }
     virtual bool next()
     {
-        if (cur->isEof)
+        if (cur->hasNullInput)
         {
             do
             {
                 if (!popNext())
                     return false;
             }
-            while (cur->isEof);
+            while (cur->hasNullInput);
         }
         else
             setNext(cur->connectedInputs);

thorlcr/graph/thgraph.hpp

@@ -260,7 +260,7 @@ public:
 
     const void *queryFindParam() const { return &queryId(); } // for SimpleHashTableOf
 
-    bool alreadyUpdated, isEof, newWhichBranch;
+    bool alreadyUpdated, hasNullInput, newWhichBranch;
     EclHelperFactory helperFactory;
 
     CIOConnectionArray inputs, outputs, connectedInputs, connectedOutputs;
@@ -330,7 +330,6 @@ public:
     IHThorArg *queryHelper() const { return baseHelper; }
 
     IPropertyTree &queryXGMML() const { return *xgmml; }
-    CActivityBase *queryActivity() const { return activity; }
     const activity_id &queryOwnerId() const { return ownerId; }
     void createActivity(size32_t parentExtractSz, const byte *parentExtract);
 //
@@ -343,6 +342,7 @@ public:
     }
     virtual bool prepareContext(size32_t parentExtractSz, const byte *parentExtract, bool checkDependencies, bool shortCircuit, bool async);
 //
+    virtual CActivityBase *queryActivity() { return activity; }
     virtual void initActivity();
     virtual CActivityBase *factory(ThorActivityKind kind) { assertex(false); return NULL; }
     virtual CActivityBase *factory() { return factory(getKind()); }

thorlcr/graph/thgraphmaster.cpp

@@ -585,10 +585,10 @@ bool CMasterGraphElement::checkUpdate()
 void CMasterGraphElement::initActivity()
 {
     CriticalBlock b(crit);
-    bool first = (NULL == activity);
+    bool first = (NULL == queryActivity());
     CGraphElementBase::initActivity();
-    if (first || activity->needReInit())
-        ((CMasterActivity *)activity.get())->init();
+    if (first || queryActivity()->needReInit())
+        ((CMasterActivity *)queryActivity())->init();
 }
 
 void CMasterGraphElement::doCreateActivity(size32_t parentExtractSz, const byte *parentExtract)
@@ -627,7 +627,7 @@ void CMasterGraphElement::doCreateActivity(size32_t parentExtractSz, const byte
 
 void CMasterGraphElement::slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
 {
-    ((CMasterActivity *)activity.get())->slaveDone(slaveIdx, mb);
+    ((CMasterActivity *)queryActivity())->slaveDone(slaveIdx, mb);
 }
 
 

thorlcr/slave/slave.cpp

@@ -264,6 +264,8 @@ class CGenericSlaveGraphElement : public CSlaveGraphElement
 {
     bool wuidread2diskread; // master decides after interrogating result and sneaks in info before slave creates
     StringAttr wuidreadFilename;
+    Owned<CActivityBase> nullActivity;
+    CriticalSection nullActivityCs;
 public:
     CGenericSlaveGraphElement(CGraphBase &_owner, IPropertyTree &xgmml) : CSlaveGraphElement(_owner, xgmml)
     {
@@ -288,6 +290,18 @@ public:
         }
         haveCreateCtx = true;
     }
+    virtual CActivityBase *queryActivity()
+    {
+        if (hasNullInput)
+        {
+            CriticalBlock b(nullActivityCs);
+            if (!nullActivity)
+                nullActivity.setown(createNullSlave(this));
+            return nullActivity;
+        }
+        else
+            return activity;
+    }
     virtual CActivityBase *factory(ThorActivityKind kind)
     {
         CActivityBase *ret = NULL;