Merge pull request #7178 from ghalliday/issue13275

HPCC-13275 Split the child dependency spotting from the resourcer

Reviewed By: Jamie Noss <james.noss@lexisnexis.com>
Reviewed By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -189,6 +189,7 @@ IAtom * gctxmethodAtom;
 IAtom * getAtom;
 IAtom * globalAtom;
 IAtom * graphAtom;
+IAtom * _graphLocal_Atom;
 IAtom * groupAtom;
 IAtom * groupedAtom;
 IAtom * hashAtom;
@@ -226,6 +227,7 @@ IAtom * keyedAtom;
 IAtom * labeledAtom;
 IAtom * languageAtom;
 IAtom * lastAtom;
+IAtom * _lazy_Atom;
 IAtom * leftAtom;
 IAtom * leftonlyAtom;
 IAtom * leftouterAtom;
@@ -622,6 +624,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(global);
     MAKEATOM(globalContext);
     MAKEATOM(graph);
+    MAKESYSATOM(graphLocal);
     MAKEATOM(group);
     MAKEATOM(grouped);
     MAKEATOM(hash);
@@ -660,6 +663,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(labeled);
     MAKEATOM(language);
     MAKEATOM(last);
+    MAKESYSATOM(lazy);
     MAKEATOM(left);
     leftonlyAtom = createLowerCaseAtom("left only");
     leftouterAtom = createLowerCaseAtom("left outer");

ecl/hql/hqlatoms.hpp

@@ -229,6 +229,7 @@ extern HQL_API IAtom * keyedAtom;
 extern HQL_API IAtom * labeledAtom;
 extern HQL_API IAtom * languageAtom;
 extern HQL_API IAtom * lastAtom;
+extern HQL_API IAtom * _lazy_Atom;
 extern HQL_API IAtom * leftAtom;
 extern HQL_API IAtom * leftonlyAtom;
 extern HQL_API IAtom * leftouterAtom;

ecl/hql/hqlutil.cpp

@@ -619,6 +619,16 @@ static IHqlExpression * findCommonExpression(IHqlExpression * lower, IHqlExpress
 
 //---------------------------------------------------------------------------------------------------------------------
 
+bool isFileOutput(IHqlExpression * expr)
+{
+    return (expr->getOperator() == no_output) && (queryRealChild(expr, 1) != NULL);
+}
+
+bool isWorkunitOutput(IHqlExpression * expr)
+{
+    return (expr->getOperator() == no_output) && (queryRealChild(expr, 1) == NULL);
+}
+
 bool isCommonSubstringRange(IHqlExpression * expr)
 {
     if (expr->getOperator() != no_substring)

ecl/hql/hqlutil.hpp

@@ -681,6 +681,8 @@ extern HQL_API IHqlExpression * queryTransformAssign(IHqlExpression * transform,
 extern HQL_API IHqlExpression * queryTransformAssignValue(IHqlExpression * transform, IHqlExpression * searchField);
 
 extern HQL_API bool isCommonSubstringRange(IHqlExpression * expr);
+extern HQL_API bool isFileOutput(IHqlExpression * expr);
+extern HQL_API bool isWorkunitOutput(IHqlExpression * expr);
 
 class HQL_API AtmostLimit
 {

ecl/hqlcpp/hqlcatom.cpp

@@ -93,7 +93,6 @@ IAtom * _spill_Atom;
 IAtom * _spillReason_Atom;
 IAtom * _steppedMeta_Atom;
 IAtom * subgraphAtom;
-IAtom * _tempCount_Atom;
 IAtom * _translated_Atom;
 IAtom * utf8Atom;
 IAtom * wrapperAtom;
@@ -1488,7 +1487,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKESYSATOM(spill);
     MAKESYSATOM(spillReason);
     MAKESYSATOM(steppedMeta);
-    MAKESYSATOM(tempCount);
     MAKESYSATOM(translated);
     return true;
 }

ecl/hqlcpp/hqlcatom.hpp

@@ -93,7 +93,6 @@ extern IAtom * _spill_Atom;
 extern IAtom * _spillReason_Atom;
 extern IAtom * _steppedMeta_Atom;
 extern IAtom * subgraphAtom;
-extern IAtom * _tempCount_Atom;
 extern IAtom * _translated_Atom;
 extern IAtom * utf8Atom;
 extern IAtom * wrapperAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -1761,6 +1761,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.showEclInGraph,"showEclInGraph",true),
         DebugOption(options.optimizeSortAllFields,"optimizeSortAllFields",true),
         DebugOption(options.optimizeSortAllFieldsStrict,"optimizeSortAllFieldsStrict",false),
+        DebugOption(options.alwaysReuseGlobalSpills,"alwaysReuseGlobalSpills",true),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -748,6 +748,7 @@ struct HqlCppOptions
     bool                showEclInGraph;
     bool                optimizeSortAllFields;
     bool                optimizeSortAllFieldsStrict;
+    bool                alwaysReuseGlobalSpills;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class

ecl/hqlcpp/hqlhtcpp.cpp

@@ -7712,7 +7712,7 @@ static bool isFilePersist(IHqlExpression * expr)
             expr = expr->queryChild(1);
             break;
         case no_output:
-            return (queryRealChild(expr, 1) != NULL);
+            return isFileOutput(expr);
         case no_actionlist:
         case no_orderedactionlist:
             expr = expr->queryChild(expr->numChildren()-1);

ecl/hqlcpp/hqlresource.ipp

@@ -38,44 +38,21 @@ enum ResourceType {
 class CResourceOptions
 {
 public:
-    CResourceOptions(UniqueSequenceCounter & _spillSequence)
-    : spillSequence(_spillSequence)
-    {
-        filteredSpillThreshold = 0;
-        minimizeSpillSize = 0;
-        allowThroughSpill = false;
-        allowThroughResult = false;
-        cloneFilteredIndex = false;
-        spillSharedConditionals = false;
-        shareDontExpand = false;
-        useGraphResults = false;
-        noConditionalLinks = false;
-        minimiseSpills = false;
-        hoistResourced = false;
-        isChildQuery = false;
-        groupedChildIterators = false;
-        allowSplitBetweenSubGraphs = false;
-        preventKeyedSplit = false;
-        preventSteppedSplit = false;
-        minimizeSkewBeforeSpill = false;
-        expandSingleConstRow = false;
-        createSpillAsDataset = false;
-        optimizeSharedInputs = false;
-        combineSiblings = false;
-        actionLinkInNewGraph = false;
-        convertCompoundToExecuteWhen = false;
-        useResultsForChildSpills = false;
-        alwaysUseGraphResults = false;
-        newBalancedSpotter = false;
-        graphIdExpr = NULL;
-        nextResult = 0;
-        clusterSize = 0;
-        targetClusterType = ThorLCRCluster;
-        state.updateSequence = 0;
-    }
+    CResourceOptions(ClusterType _targetClusterType, unsigned _clusterSize, const HqlCppOptions & _translatorOptions, UniqueSequenceCounter & _spillSequence);
+
+    IHqlExpression * createResultSpillName();
+    IHqlExpression * createDiskSpillName();
+    IHqlExpression * createGlobalSpillName();
 
-    IHqlExpression * createSpillName(bool isGraphResult);
     void noteGraphsChanged() { state.updateSequence++; }
+    void setChildQuery(bool value);
+    void setNewChildQuery(IHqlExpression * graphIdExpr, unsigned numResults);
+    void setUseGraphResults(bool _useGraphResults)
+    {
+        useGraphResults = _useGraphResults;
+    }
+    bool useGraphResult(bool linkedFromChild);
+    bool useGlobalResult(bool linkedFromChild);
 
 public:
     UniqueSequenceCounter & spillSequence;
@@ -105,6 +82,9 @@ public:
     bool     useResultsForChildSpills;
     bool     alwaysUseGraphResults;
     bool     newBalancedSpotter;
+    bool     spillMultiCondition;
+    bool     spotThroughAggregate;
+    bool     alwaysReuseGlobalSpills;
 
     IHqlExpression * graphIdExpr;
     unsigned nextResult;
@@ -160,7 +140,8 @@ public:
     virtual void changeSourceGraph(ResourceGraphInfo * newGraph);
     virtual void changeSinkGraph(ResourceGraphInfo * newGraph);
     virtual bool isRedundantLink();
-    virtual bool isDependency() { return false; }
+    virtual bool isDependency() const { return false; }
+    virtual IHqlExpression * queryDependency() const { return NULL; }
 
 protected:
     void trace(const char * name);
@@ -176,12 +157,16 @@ public:
 class ResourceGraphDependencyLink : public ResourceGraphLink
 {
 public:
-    ResourceGraphDependencyLink(ResourceGraphInfo * _sourceGraph, IHqlExpression * _sourceNode, ResourceGraphInfo * _sinkGraph, IHqlExpression * _sinkNode);
+    ResourceGraphDependencyLink(ResourceGraphInfo * _sourceGraph, IHqlExpression * _sourceNode, ResourceGraphInfo * _sinkGraph, IHqlExpression * _sinkNode, IHqlExpression * _dependency);
 
     virtual void changeSourceGraph(ResourceGraphInfo * newGraph);
     virtual void changeSinkGraph(ResourceGraphInfo * newGraph);
     virtual bool isRedundantLink()          { return false; }
-    virtual bool isDependency() { return true; }
+    virtual bool isDependency() const { return true; }
+    virtual IHqlExpression * queryDependency() const { return dependency; }
+
+protected:
+    LinkedHqlExpr dependency;
 };
 
 typedef CIArrayOf<ResourceGraphInfo> ResourceGraphArray;
@@ -204,7 +189,7 @@ public:
     bool isDependentOn(ResourceGraphInfo & other, bool allowDirect);
     bool isVeryCheap();
     bool mergeInSibling(ResourceGraphInfo & other, const CResources & limit);
-    bool mergeInSource(ResourceGraphInfo & other, const CResources & limit);
+    bool mergeInSource(ResourceGraphInfo & other, const CResources & limit, bool ignoreConditional);
     void removeResources(const CResources & value);
 
     bool isSharedInput(IHqlExpression * expr);
@@ -219,7 +204,7 @@ protected:
 public:
     OwnedHqlExpr createdGraph;
     CResourceOptions * options;
-    GraphLinkArray dependsOn;           // NB: These do no link....
+    GraphLinkArray dependsOn;           // NB: These do not link....
     GraphLinkArray sources;
     GraphLinkArray sinks;
     HqlExprArray conditions;
@@ -327,14 +312,40 @@ public:
     bool ignoreExternalDependencies;
 };
 
-class ResourcerInfo : public CInterfaceOf<IInterface>
+class SpillerInfo : public NewTransformInfo
+{
+public:
+    SpillerInfo(IHqlExpression * _original, CResourceOptions * _options);
+
+    IHqlExpression * createSpilledRead(IHqlExpression * spillReason);
+    IHqlExpression * createSpilledWrite(IHqlExpression * transformed, bool lazy);
+    bool isUsedFromChild() const { return linkedFromChild; }
+    IHqlExpression * queryOutputSpillFile() const { return outputToUseForSpill; }
+    void setPotentialSpillFile(IHqlExpression * expr);
+
+
+protected:
+    void addSpillFlags(HqlExprArray & args, bool isRead);
+    IHqlExpression * createSpillName();
+    bool useGraphResult();
+    bool useGlobalResult();
+    IHqlExpression * wrapRowOwn(IHqlExpression * expr);
+
+protected:
+    CResourceOptions * options;
+    HqlExprAttr spillName;
+    HqlExprAttr spilledDataset;
+    IHqlExpression * outputToUseForSpill;
+    bool linkedFromChild; // could reuse a spare byte in the parent class
+};
+
+class ResourcerInfo : public SpillerInfo
 {
 public:
     enum { PathUnknown, PathConditional, PathUnconditional };
 
     ResourcerInfo(IHqlExpression * _original, CResourceOptions * _options);
 
-    IHqlExpression * createSpilledRead(IHqlExpression * spillReason);
     IHqlExpression * createTransformedExpr(IHqlExpression * expr);
 
     bool addCondition(IHqlExpression * condition);
@@ -356,8 +367,12 @@ public:
     void resetBalanced();
     void setConditionSource(IHqlExpression * condition, bool isFirst);
 
-    //hthor - don't merge anything to a global result because we don't allow splitters.
-    inline bool preventMerge()          { return !options->canSplit() && useGlobalResult(); }
+    inline bool preventMerge()
+    {
+        //If we need to create a spill global result, but engine can't split then don't merge
+        //Only required because hthor doesn't support splitters (or through-workunit results).
+        return !options->canSplit() && options->useGlobalResult(linkedFromChild);
+    }
     inline bool isUnconditional()       { return (pathToExpr == ResourcerInfo::PathUnconditional); }
     inline bool isConditionExpr()
     {
@@ -393,34 +408,20 @@ public:
 
 protected:
     bool spillSharesSplitter();
-    bool useGraphResult();
-    bool useGlobalResult();
     IHqlExpression * createAggregation(IHqlExpression * expr);
-    IHqlExpression * createSpilledWrite(IHqlExpression * transformed);
     IHqlExpression * createSpiller(IHqlExpression * transformed, bool reuseSplitter);
     IHqlExpression * createSplitter(IHqlExpression * transformed);
 
-protected:
-    void addSpillFlags(HqlExprArray & args, bool isRead);
-    IHqlExpression * createSpillName();
-    IHqlExpression * wrapRowOwn(IHqlExpression * expr);
-
 public:
-    HqlExprAttr original;
     Owned<ResourceGraphInfo> graph;
-    HqlExprAttr spillName;
-    IHqlExpression * transformed;
-    IHqlExpression * outputToUseForSpill;
-    CResourceOptions * options;
     HqlExprAttr pathToSplitter;
     HqlExprArray aggregates;
     HqlExprArray conditions;
-    ChildDependentArray childDependents;
-    HqlExprAttr spilledDataset;
     HqlExprAttr splitterOutput;
     HqlExprArray projected;
     HqlExprAttr projectedExpr;
     CIArrayOf<CSplitterLink> balancedLinks;
+    GraphLinkArray dependsOn;           // NB: These do not link....
 
     unsigned numUses;
     unsigned numExternalUses;
@@ -440,7 +441,6 @@ public:
     bool isSpillPoint:1;
     bool balanced:1;
     bool isAlreadyInScope:1;
-    bool linkedFromChild:1;
     bool forceHoist:1;
     bool neverSplit:1;
     bool isConditionalFilter:1;
@@ -449,32 +449,19 @@ public:
     bool balancedVisiting:1;
 };
 
-struct DependencySourceInfo
-{
-    HqlExprArray                    search;
-    CIArrayOf<ResourceGraphInfo>    graphs;
-    HqlExprArray                    exprs;
-};
-
-
+class EclResourceDependencyGatherer;
 class EclResourcer
 {
     friend class SelectHoistTransformer;
     friend class CSplitterInfo;
 public:
-    EclResourcer(IErrorReceiver & _errors, IConstWorkUnit * _wu, ClusterType _targetClusterType, unsigned _clusterSize, const HqlCppOptions & _translatorOptions, UniqueSequenceCounter & _spillSequence);
+    EclResourcer(IErrorReceiver & _errors, IConstWorkUnit * _wu, const HqlCppOptions & _translatorOptions, CResourceOptions & _options);
     ~EclResourcer();
 
     void resourceGraph(IHqlExpression * expr, HqlExprArray & transformed);
     void resourceRemoteGraph(IHqlExpression * expr, HqlExprArray & transformed);
-    void setChildQuery(bool value);
-    void setNewChildQuery(IHqlExpression * graphIdExpr, unsigned numResults);
     void setSequential(bool _sequential) { sequential = _sequential; }
-    void setUseGraphResults(bool _useGraphResults) 
-    { 
-        options.useGraphResults = _useGraphResults; 
-    }
-    void tagActiveCursors(HqlExprCopyArray & activeRows);
+    void tagActiveCursors(HqlExprCopyArray * activeRows);
     inline unsigned numGraphResults() { return options.nextResult; }
 
 protected:
@@ -487,15 +474,11 @@ protected:
     void replaceGraphReferences(ResourceGraphInfo * oldGraph, ResourceGraphInfo * newGraph);
 
 //Pass 1
-    void gatherChildSplitPoints(IHqlExpression * expr, ResourcerInfo * info, unsigned first, unsigned last);
     bool findSplitPoints(IHqlExpression * expr, bool isProjected);
     void findSplitPoints(HqlExprArray & exprs);
     void noteConditionalChildren(BoolArray & alwaysHoistChild);
     void deriveUsageCounts(IHqlExpression * expr);
     void deriveUsageCounts(const HqlExprArray & exprs);
-    void extendSplitPoints();
-    void projectChildDependents();
-    IHqlExpression * projectChildDependent(IHqlExpression * expr);
 
 //Pass 2
     void createInitialGraph(IHqlExpression * expr, IHqlExpression * owner, ResourceGraphInfo * ownerGraph, LinkKind linkKind, bool forceNewGraph);
@@ -520,13 +503,7 @@ protected:
     void resourceSubGraphs(HqlExprArray & exprs);
 
 //Pass 5
-    void addDependencySource(IHqlExpression * search, ResourceGraphInfo * curGraph, IHqlExpression * expr);
-    void addDependencyUse(IHqlExpression * search, ResourceGraphInfo * curGraph, IHqlExpression * expr);
-    bool addExprDependency(IHqlExpression * expr, ResourceGraphInfo * curGraph, IHqlExpression * activityExpr);
-    void addRefExprDependency(IHqlExpression * expr, ResourceGraphInfo * curGraph, IHqlExpression * activityExpr);
-    void doAddChildDependencies(IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr);
-    void addChildDependencies(IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr);
-    void addDependencies(IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr);
+    void addDependencies(EclResourceDependencyGatherer & gatherer, IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr);
     void addDependencies(HqlExprArray & exprs);
 
 //Pass 6
@@ -560,7 +537,6 @@ protected:
     void moveExternalSpillPoints();
 
 //Pass 9
-    IHqlExpression * replaceResourcedReferences(ResourcerInfo * info, IHqlExpression * expr);
     IHqlExpression * doCreateResourced(IHqlExpression * expr, ResourceGraphInfo * graph, bool expandInParent, bool defineSideEffect);
     IHqlExpression * createResourced(IHqlExpression * expr, ResourceGraphInfo * graph, bool expandInParent, bool defineSideEffect);
     void createResourced(HqlExprArray & transformed);
@@ -582,21 +558,16 @@ protected:
     CIArrayOf<ResourceGraphInfo> graphs;
     CIArrayOf<ResourceGraphLink> links;
     ClusterType targetClusterType;
-    DependencySourceInfo dependencySource;                  
-    unsigned clusterSize;
     CResources * resourceLimit;
     IErrorReceiver * errors;
     unsigned thisPass;
     bool spilled;
-    bool spillMultiCondition;
-    bool spotThroughAggregate;
     bool insideNeverSplit;
     bool insideSteppedNeverSplit;
     bool sequential;
-    CResourceOptions options;
+    CResourceOptions & options;
     HqlExprArray rootConditions;
     HqlExprCopyArray activeSelectors;
-    ChildDependentArray allChildDependents;
 };
 
 #endif

testing/regress/ecl/childds1.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//This is needed to prevent cntBad being hoisted before the resourcing, and becoming unconditional
+//The tests are part of the work aiming to remove this code.
+#option ('workunitTemporaries', false);
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+//Case 1 - a child query where the filter is always correct
+cnt := COUNT(ds(assertTrue(seq < 10, 'seq < 10'))) + NOFOLD(100000);
+output(ds(seq != cnt));
+
+
+//Case 2 - a condition that is always false is used from a branch that should never be executed
+cntBad := COUNT(ds(assertTrue(seq > 10, 'seq > 10'))) + NOFOLD(100000);
+
+//NOFOLD is required to stop code generator converting this to a filter (trueValue && ...) which means that cntBad is evaluated always
+//See childds1err.ecl
+cond := IF(trueValue, ds, NOFOLD(ds)(seq != cntBad));
+output(cond);
+

testing/regress/ecl/childds1err.ecl

@@ -0,0 +1,62 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//fail
+//NOTE: This example shouldn't really fail, it illustates a problem converting
+//IF(c, ds, ds(f1)) to ds(c OR f1)
+//which causes dependencies of f1 to be evaluated when they wouldn't be otherwise.
+
+//This is needed to prevent cntBad being hoisted before the resourcing, and becoming unconditional
+//The tests are part of the work aiming to remove this code.
+#option ('workunitTemporaries', false);
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+//Case 1 - a child query where the filter is always correct
+cnt := COUNT(ds(assertTrue(seq < 10, 'seq < 10'))) + NOFOLD(100000);
+output(ds(seq != cnt));
+
+
+//Case 2 - a condition that is always false is used from a branch that should never be executed
+cntBad := COUNT(ds(assertTrue(seq > 10, 'seq > 10'))) + NOFOLD(100000);
+
+//Problem1: Converting this to a filtered disk read means that cntBad is evaluated always
+cond := IF(trueValue, ds, ds(seq != cntBad));
+output(cond);
+

testing/regress/ecl/childds2.ecl

@@ -0,0 +1,53 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//This is needed to prevent cntBad being hoisted before the resourcing, and becoming unconditional
+//The tests are part of the work aiming to remove this code.
+#option ('workunitTemporaries', false);
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+//Case 2 - a condition that is always false is used from a multiple branches that should never be executed
+cntBad := COUNT(ds(assertTrue(seq > 10, 'seq > 10'))) + NOFOLD(100000);
+
+//Problem1: Converting this to a filtered disk read means that cntBad is evaluated always
+cond1 := IF(trueValue, ds, NOFOLD(ds)(seq != cntBad));
+cond2 := IF(falseValue, NOFOLD(ds)(seq != cntBad), ds);
+output(cond1+cond2);
+

testing/regress/ecl/childds3.ecl

@@ -0,0 +1,55 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    sortedIds := nofold(sort(l.ids, id));
+    cntGood := COUNT(sortedIds(assertTrue(id < 10, 'seq < 10')));
+    RETURN count(sortedIds(id != cntGood)) = 3;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));
+
+output(ds(evalFilter(ds)));

testing/regress/ecl/childds4.ecl

@@ -0,0 +1,57 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    sortedIds := nofold(sort(l.ids, id));
+    cntBad := COUNT(sortedIds(assertTrue(id > 10, 'seq > 10')));
+    
+    f := IF(trueValue, sortedIds, NOFOLD(sortedIds(id != cntBad)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));
+
+output(ds(evalFilter(ds)));

testing/regress/ecl/childds5.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    sortedIds := nofold(sort(l.ids, id));
+    cntBad1 := COUNT(sortedIds(assertTrue(id > 10, 'seq > 10')));
+    cntBad2 := COUNT(sortedIds(assertTrue(id > 11, 'seq > 11')));
+    
+    f := IF(trueValue, sortedIds, NOFOLD(sortedIds(id != cntBad1 * cntBad2)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));
+
+output(ds(evalFilter(ds)));

testing/regress/ecl/childds6.ecl

@@ -0,0 +1,59 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    sortedIds := nofold(sort(l.ids, id));
+    bad := sortedIds(assertTrue(id > 10, 'seq > 10'));
+    cntBad1 := COUNT(bad);
+    cntBad2 := SUM(bad, id);
+    
+    f := IF(trueValue, sortedIds, NOFOLD(sortedIds(id != cntBad1 * cntBad2)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));
+
+output(ds(evalFilter(ds)));

testing/regress/ecl/childds7.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    dedupIds := DEDUP(l.ids, id);
+    sortedIds := nofold(sort(l.ids, id));
+    bad := sortedIds(assertTrue(id > 10, 'seq > 10'));
+    cntGood := COUNT(sortedIds(id != 10000));
+    cntBad1 := COUNT(bad);
+    
+    f := IF(trueValue, dedupIds(id != cntGood), NOFOLD(dedupIds(id != cntBad1)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));

testing/regress/ecl/childds7b.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    dedupIds := l.ids;
+    sortedIds := nofold(sort(l.ids, id));
+    bad := sortedIds(assertTrue(id > 10, 'seq > 10'));
+    cntGood := COUNT(sortedIds(id != 10000));
+    cntBad1 := COUNT(bad);
+    
+    f := IF(trueValue, dedupIds(id != cntGood), NOFOLD(dedupIds(id != cntBad1)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));

testing/regress/ecl/key/childds1.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds1err.xml

@@ -0,0 +1 @@
+<Exception><Source>eclagent</Source><Message>System error: 0: Graph[4], filter[7]: SLAVE #1 [192.168.0.200:20100]: seq &gt; 10, </Message></Exception>

testing/regress/ecl/key/childds2.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds3.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds4.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds5.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds6.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds7.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds7b.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>