Merge branch 'candidate-7.2.x' into candidate-7.4.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -398,7 +398,7 @@ public:
     }
     virtual const char * queryScope() const
     {
-        return scope;
+        return scope ? scope : "";
     }
     virtual IStringVal & getFormattedValue(IStringVal & str) const
     {

plugins/workunitservices/workunitservices.cpp

@@ -735,6 +735,7 @@ public:
         extra.getCreator(creator);
         StatisticScopeType scopeType = extra.getScopeType();
         const char * scope = extra.queryScope();
+        if (!scope) scope = "";
         extra.getDescription(description, true);
         StatisticMeasure measure = extra.getMeasure();
 

roxie/ccd/ccdmain.cpp

@@ -539,6 +539,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             topology->setPropInt("RoxieFarmProcess/@port", port);
             topology->setProp("@daliServers", globals->queryProp("--daliServers"));
             topology->setProp("@traceLevel", globals->queryProp("--traceLevel"));
+            topology->setPropBool("@traceStartStop", globals->getPropInt("--traceStartStop", 0));
             topology->setPropInt("@allFilesDynamic", globals->getPropInt("--allFilesDynamic", 1));
             topology->setProp("@memTraceLevel", globals->queryProp("--memTraceLevel"));
             topology->setPropInt64("@totalMemoryLimit", globals->getPropInt("--totalMemoryLimitMb", 0) * (memsize_t) 0x100000);

roxie/ccd/ccdserver.cpp

@@ -5277,6 +5277,26 @@ IRoxieServerActivityFactory *createRoxieServerNullActivityFactory(unsigned _id,
 
 //=================================================================================
 
+class CRoxieServerNullSinkActivity : public CRoxieServerInternalSinkActivity
+{
+public:
+    CRoxieServerNullSinkActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
+        : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, 0)
+    {
+    }
+
+    virtual void onExecute() override
+    {
+    }
+};
+
+IRoxieServerActivity * createRoxieServerNullSinkActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
+{
+    return new CRoxieServerNullSinkActivity(_ctx, _factory, _probeManager);
+}
+
+//=================================================================================
+
 class CRoxieServerPassThroughActivity : public CRoxieServerActivity
 {
 public:
@@ -12095,13 +12115,18 @@ public:
         case 0:
             switch (kind)
             {
-            case TAKdiskwrite: return new CRoxieServerDiskWriteActivity(_ctx, this, _probeManager);
-            case TAKcsvwrite: return new CRoxieServerCsvWriteActivity(_ctx, this, _probeManager);
+            case TAKdiskwrite:
+                return new CRoxieServerDiskWriteActivity(_ctx, this, _probeManager);
+            case TAKcsvwrite:
+                return new CRoxieServerCsvWriteActivity(_ctx, this, _probeManager);
             case TAKxmlwrite:
             case TAKjsonwrite:
                 return new CRoxieServerXmlWriteActivity(_ctx, this, _probeManager, kind);
+            case TAKspillwrite:
+                return new CRoxieServerNullSinkActivity(_ctx, this, _probeManager);
             };
             throwUnexpected();
+
         case 1:
             return new CRoxieServerSpillWriteActivity(_ctx, this, _probeManager);
         default:
@@ -12111,7 +12136,7 @@ public:
 
     virtual bool isSink() const
     {
-        return numOutputs == 0 && !isTemp; // MORE - check with Gavin if this is right if not a temp but reread in  same job...
+        return numOutputs == 0 && (kind==TAKspillwrite || !isTemp); // MORE - check with Gavin if this is right if not a temp but reread in  same job...
     }
 
 };
@@ -21675,6 +21700,11 @@ public:
         return new CRoxieServerRemoteResultActivity(_ctx, this, _probeManager, usageCount);
     }
 
+    virtual bool isSink() const override
+    {
+        return CRoxieServerInternalSinkFactory::isSink() || dependentCount == 0;  // Codegen normally optimizes these away, but if it doesn't we need to treat as a sink rather than a dependency or upstream activities are not stopped properly
+    }
+
 };
 
 IRoxieServerActivityFactory *createRoxieServerRemoteResultActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)

system/jlib/jstats.cpp

@@ -122,16 +122,16 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
 
 extern jlib_decl int compareScopeName(const char * left, const char * right)
 {
-    if (!*left)
+    if (!left || !*left)
     {
-        if (!*right)
+        if (!right || !*right)
             return 0;
         else
             return -1;
     }
     else
     {
-        if (!*right)
+        if (!right || !*right)
             return +1;
     }
 

testing/regress/ecl/issue23168.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import dbglog from Std.System.log;
+
+namesRecord :=
+            RECORD
+unsigned        id;
+string          name;
+            END;
+
+names2 := DEDUP(NOCOMBINE(DATASET([{1,'Gavin'},{2,'Bill'},{3,'John'},{4,'Gerald'},{5,'Fluffy'}], namesRecord)),id);
+names1 := DEDUP(NOCOMBINE(DATASET([{1,'Oscar'},{2,'Charles'},{3,'Freddie'},{4,'Winifred'},{5,'Bouncer'}], namesRecord)), id);
+
+s := nofold(sort(names2, name));
+
+j := join(names1, s, left.id = right.id, transform(namesRecord, self.id := left.id + s[2].id; self.name := right.name), left outer, keep(1));
+
+output(names2);
+dbglog('Hello ' + (string)names2[3].name + ' and ' + (string)count(j) + 'again');

testing/regress/ecl/key/issue23168.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><id>1</id><name>Gavin</name></Row>
+ <Row><id>2</id><name>Bill</name></Row>
+ <Row><id>3</id><name>John</name></Row>
+ <Row><id>4</id><name>Gerald</name></Row>
+ <Row><id>5</id><name>Fluffy</name></Row>
+</Dataset>