Merge pull request #4650 from ghalliday/issue9707

HPCC-9707 Ensure scalar and set datasets are always hoisted

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlresource.cpp

@@ -1003,6 +1003,7 @@ ResourcerInfo::ResourcerInfo(IHqlExpression * _original, CResourceOptions * _opt
     balanced = true;
     currentSource = 0;
     linkedFromChild = false;
+    forceHoist = false;
     neverSplit = false;
     isConditionalFilter = false;
     projectResult = true; // projected must also be non empty to actually project this dataset
@@ -1353,6 +1354,19 @@ bool ResourcerInfo::okToSpillThrough()
     return (options->allowThroughSpill && !options->createSpillAsDataset);
 }
 
+void ResourcerInfo::noteUsedFromChild(bool _forceHoist)
+{
+    linkedFromChild = true;
+    outputToUseForSpill = NULL;
+    if (_forceHoist)
+        forceHoist = true;
+}
+
+unsigned ResourcerInfo::numInternalUses()
+{
+    return numUses - numExternalUses - aggregates.ordinality();
+}
+
 bool ResourcerInfo::spillSharesSplitter()
 {
     if (outputToUseForSpill || useGraphResult() || useGlobalResult())
@@ -1480,6 +1494,8 @@ bool ResourcerInfo::expandRatherThanSpill(bool noteOtherSpills)
         ResourcerInfo * info = queryResourceInfo(expr);
         if (info && info->neverSplit)
             return true;
+        if (info && info->forceHoist)
+            return false;
 
         node_operator op = expr->getOperator();
         switch (op)
@@ -1996,7 +2012,7 @@ public:
             if (!hoisted)
                 hoisted = expr;
 
-            CChildDependent & depend = * new CChildDependent(expr, hoisted, alwaysHoist, alwaysSingle);
+            CChildDependent & depend = * new CChildDependent(expr, hoisted, alwaysHoist, alwaysSingle, false);
             matched.append(depend);
         }
         else
@@ -2061,7 +2077,7 @@ public:
                 hoisted.setown(createRow(no_createrow, LINK(transform)));
             }
 
-            CChildDependent & depend = * new CChildDependent(expr, hoisted, true, true);
+            CChildDependent & depend = * new CChildDependent(expr, hoisted, true, true, true);
             depend.projected = projected;
             matched.append(depend);
         }
@@ -3809,7 +3825,7 @@ void EclResourcer::addDependencies(IHqlExpression * expr, ResourceGraphInfo * gr
 
             ResourcerInfo * sourceInfo = queryResourceInfo(cur.projectedHoisted);
             if (cur.isSingleNode)
-                sourceInfo->noteUsedFromChild();
+                sourceInfo->noteUsedFromChild(cur.forceHoist);
             ResourceGraphLink * link = new ResourceGraphDependencyLink(sourceInfo->graph, cur.projectedHoisted, graph, expr);
             graph->dependsOn.append(*link);
             links.append(*link);

ecl/hqlcpp/hqlresource.ipp

@@ -206,8 +206,8 @@ public:
 class CChildDependent : public CInterface
 {
 public:
-    CChildDependent(IHqlExpression * _original, IHqlExpression * _hoisted, bool _alwaysHoist, bool _isSingleNode)
-    : original(_original), hoisted(_hoisted), alwaysHoist(_alwaysHoist), isSingleNode(_isSingleNode)
+    CChildDependent(IHqlExpression * _original, IHqlExpression * _hoisted, bool _alwaysHoist, bool _isSingleNode, bool _forceHoist)
+    : original(_original), hoisted(_hoisted), alwaysHoist(_alwaysHoist), isSingleNode(_isSingleNode), forceHoist(_forceHoist)
     {
         projectedHoisted.set(hoisted);
         projected = NULL;
@@ -217,6 +217,7 @@ public:
     IHqlExpression * original;
     LinkedHqlExpr hoisted;
     LinkedHqlExpr projectedHoisted;
+    bool forceHoist;
     bool alwaysHoist;
     bool isSingleNode;
     IHqlExpression * projected;
@@ -251,8 +252,8 @@ public:
     bool isSplit();
     bool isSpilledWrite();
     bool okToSpillThrough();
-    void noteUsedFromChild()            { linkedFromChild = true; outputToUseForSpill = NULL; }
-    unsigned numInternalUses()          { return numUses - numExternalUses - aggregates.ordinality(); }
+    void noteUsedFromChild(bool _forceHoist);
+    unsigned numInternalUses();
     unsigned numSplitPaths();
     void setConditionSource(IHqlExpression * condition, bool isFirst);
 
@@ -323,6 +324,7 @@ public:
     bool balanced;
     bool isAlreadyInScope;
     bool linkedFromChild;
+    bool forceHoist;
     bool neverSplit;
     byte pathToExpr;
     bool isConditionalFilter;