Merge branch 'candidate-7.8.x' into candidate-7.10.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -251,7 +251,6 @@ public:
         cclogFilename.append("cc.").append((unsigned)GetCurrentProcessId()).append(".log");
         defaultAllowed[false] = true;  // May want to change that?
         defaultAllowed[true] = true;
-        optMetaLocation.set(".eclcc/metacache");
     }
     ~EclCC()
     {

ecl/hqlcpp/hqlcpp.cpp

@@ -1850,6 +1850,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.addDefaultBloom,"addDefaultBloom", true),
         DebugOption(options.newDiskReadMapping, "newDiskReadMapping", true),
         DebugOption(options.transformNestedSequential, "transformNestedSequential", true),
+        DebugOption(options.preserveWhenSequential, "preserveWhenSequential", false),
         DebugOption(options.forceAllProjectedDiskSerialized, "internalForceAllProjectedDiskSerialized", false),  // Delete in 8.0 once new code has been proved in anger
         DebugOption(options.newIndexReadMapping, "newIndexReadMapping", false), // Not yet enabled due to problems with merging mapped fields and roxie/thor integration
         DebugOption(options.checkDuplicateThreshold, "checkDuplicateThreshold", 0), // If non zero, create a warning if duplicates > this percentage increase

ecl/hqlcpp/hqlcpp.ipp

@@ -827,6 +827,7 @@ struct HqlCppOptions
     bool                addDefaultBloom;
     bool                newDiskReadMapping;
     bool                transformNestedSequential;
+    bool                preserveWhenSequential;
     bool                forceAllProjectedDiskSerialized;
     bool                newIndexReadMapping;
     bool                diskReadsAreSimple;

ecl/hqlcpp/hqlttcpp.cpp

@@ -6156,7 +6156,10 @@ IHqlExpression * WorkflowTransformer::extractWorkflow(IHqlExpression * untransfo
         }
     }
 
-    IHqlExpression * value = expr->queryChild(0);
+    LinkedHqlExpr value = expr->queryChild(0);
+    if (translator.queryOptions().preserveWhenSequential)
+        value.setown(transformSequentialEtc(value));
+
     GlobalAttributeInfo info("jobtemp::wf", "wf", value);
     info.sequence.setown(getLocalSequenceNumber());
     OwnedHqlExpr scheduleActions;

testing/regress/ecl/key/workflow_9c.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='B2'>
+ <Row><B2>2</B2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='B4'>
+ <Row><B4>4</B4></Row>
+</Dataset>

testing/regress/ecl/workflow_9c.ecl

@@ -0,0 +1,61 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the 'License');
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an 'AS IS' BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+//version parallel=false
+//version parallel=true,nothor
+
+#option ('preserveWhenSequential', true);
+
+import ^ as root;
+optParallel := #IFDEFINED(root.parallel, false);
+
+#option ('parallelWorkflow', optParallel);
+#option('numWorkflowThreads', 5);
+
+
+import Std.File AS FileServices;
+import $.setup;
+prefix := setup.Files(false, false).QueryFilePrefix;
+filename := prefix + 'workflow_9bFile';
+
+display(String8 thisString) := FUNCTION
+  ds := dataset([thisString], {String8 text});
+  RETURN Output(ds, NAMED('logging'), EXTEND);
+END;
+Import sleep from std.System.Debug;
+
+//This makes sure that item B is executed twice because it is **not** independent
+names := RECORD
+  STRING8 firstname;
+  STRING8 surname;
+END;
+
+ds := DATASET([{'nathan', 'halliday'}, {'margaret', 'thatcher'}], names);
+
+A := OUTPUT(ds,,filename, OVERwrite) : independent;
+A2 := OUTPUT(ds+ds,,filename, OVERwrite) : independent;
+
+MyFile := DATASET(filename, names, thor);
+B := COUNT(myFile);
+
+conditionalDelete(string lfn) := FUNCTION
+  RETURN IF(FileServices.FileExists(lfn), FileServices.DeleteLogicalFile(lfn));
+END;
+SEQUENTIAL(
+  conditionalDelete(filename),
+  SEQUENTIAL(A,output(B,named('B2')),A2,output(B,named('B4'))),
+  FileServices.DeleteLogicalFile(filename)
+) : WHEN(CRON('* * * * *'), count(1));