HPCC-9707 Ensure scalar and set datasets are always hoisted

Has quite a big effect on a few complex archived examples.

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hqlcpp/hqlresource.cpp

@@ -1003,6 +1003,7 @@ ResourcerInfo::ResourcerInfo(IHqlExpression * _original, CResourceOptions * _opt
     balanced = true;
     currentSource = 0;
     linkedFromChild = false;
+    forceHoist = false;
     neverSplit = false;
     isConditionalFilter = false;
     projectResult = true; // projected must also be non empty to actually project this dataset
@@ -1480,6 +1481,8 @@ bool ResourcerInfo::expandRatherThanSpill(bool noteOtherSpills)
         ResourcerInfo * info = queryResourceInfo(expr);
         if (info && info->neverSplit)
             return true;
+        if (info && info->forceHoist)
+            return false;
 
         node_operator op = expr->getOperator();
         switch (op)
@@ -1996,7 +1999,7 @@ public:
             if (!hoisted)
                 hoisted = expr;
 
-            CChildDependent & depend = * new CChildDependent(expr, hoisted, alwaysHoist, alwaysSingle);
+            CChildDependent & depend = * new CChildDependent(expr, hoisted, alwaysHoist, alwaysSingle, false);
             matched.append(depend);
         }
         else
@@ -2061,7 +2064,7 @@ public:
                 hoisted.setown(createRow(no_createrow, LINK(transform)));
             }
 
-            CChildDependent & depend = * new CChildDependent(expr, hoisted, true, true);
+            CChildDependent & depend = * new CChildDependent(expr, hoisted, true, true, true);
             depend.projected = projected;
             matched.append(depend);
         }
@@ -3807,7 +3810,7 @@ void EclResourcer::addDependencies(IHqlExpression * expr, ResourceGraphInfo * gr
 
             ResourcerInfo * sourceInfo = queryResourceInfo(cur.projectedHoisted);
             if (cur.isSingleNode)
-                sourceInfo->noteUsedFromChild();
+                sourceInfo->noteUsedFromChild(cur.forceHoist);
             ResourceGraphLink * link = new ResourceGraphDependencyLink(sourceInfo->graph, cur.projectedHoisted, graph, expr);
             graph->dependsOn.append(*link);
             links.append(*link);

ecl/hqlcpp/hqlresource.ipp

@@ -206,8 +206,8 @@ public:
 class CChildDependent : public CInterface
 {
 public:
-    CChildDependent(IHqlExpression * _original, IHqlExpression * _hoisted, bool _alwaysHoist, bool _isSingleNode)
-    : original(_original), hoisted(_hoisted), alwaysHoist(_alwaysHoist), isSingleNode(_isSingleNode)
+    CChildDependent(IHqlExpression * _original, IHqlExpression * _hoisted, bool _alwaysHoist, bool _isSingleNode, bool _forceHoist)
+    : original(_original), hoisted(_hoisted), alwaysHoist(_alwaysHoist), isSingleNode(_isSingleNode), forceHoist(_forceHoist)
     {
         projectedHoisted.set(hoisted);
         projected = NULL;
@@ -217,6 +217,7 @@ public:
     IHqlExpression * original;
     LinkedHqlExpr hoisted;
     LinkedHqlExpr projectedHoisted;
+    bool forceHoist;
     bool alwaysHoist;
     bool isSingleNode;
     IHqlExpression * projected;
@@ -251,7 +252,7 @@ public:
     bool isSplit();
     bool isSpilledWrite();
     bool okToSpillThrough();
-    void noteUsedFromChild()            { linkedFromChild = true; outputToUseForSpill = NULL; }
+    void noteUsedFromChild(bool _forceHoist)            { linkedFromChild = true; outputToUseForSpill = NULL; if (_forceHoist) forceHoist = true; }
     unsigned numInternalUses()          { return numUses - numExternalUses - aggregates.ordinality(); }
     unsigned numSplitPaths();
     void setConditionSource(IHqlExpression * condition, bool isFirst);
@@ -323,6 +324,7 @@ public:
     bool balanced;
     bool isAlreadyInScope;
     bool linkedFromChild;
+    bool forceHoist;
     bool neverSplit;
     byte pathToExpr;
     bool isConditionalFilter;