HPCC-23135 Fix problems reading spilled files that are never written

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hqlcpp/hqlhtcpp.cpp

@@ -19366,6 +19366,7 @@ IHqlExpression * HqlCppTranslator::extractGlobalCSE(IHqlExpression * expr)
     HqlExprArray exprs;
     unwindCommaCompound(exprs, expr);
     transformer.analyseArray(exprs, 0);
+    transformer.analyseArray(exprs, 1);
     if (!transformer.worthTransforming())
         return LINK(expr);
 
@@ -19432,6 +19433,7 @@ void HqlCppTranslator::spotGlobalCSE(HqlExprArray & exprs)
     AutoScopeMigrateTransformer transformer(wu(), *this);
 
     transformer.analyseArray(exprs, 0);
+    transformer.analyseArray(exprs, 1);
     if (transformer.worthTransforming())
     {
         transformer.transformRoot(exprs, results);

ecl/hqlcpp/hqlresource.cpp

@@ -3905,7 +3905,18 @@ void EclResourcer::deriveUsageCounts(IHqlExpression * expr)
     if (info->numUses)
     {
         if (insideNeverSplit || insideSteppedNeverSplit)
+        {
             info->neverSplit = true;
+            //If this expression should never be split, ensure no input datasets are split, otherwise
+            //a input expression that has already been visited on another path may expect the result to be
+            //spilled, but no spill write will be generated.
+            IHqlExpression * cur = expr;
+            while (getNumActivityArguments(cur) == 1)
+            {
+                cur = cur->queryChild(0);
+                queryResourceInfo(cur)->neverSplit = true;
+            }
+        }
 
         if (info->isAlreadyInScope || info->isActivity || !info->containsActivity)
         {

ecl/hqlcpp/hqlttcpp.cpp

@@ -8504,7 +8504,7 @@ bool AutoScopeMigrateInfo::addGraph(unsigned graph)
 
 bool AutoScopeMigrateInfo::doAutoHoist(IHqlExpression * transformed, bool minimizeWorkunitTemporaries)
 {
-    if (useCount == 0)
+    if ((useCount == 0) || neverHoist)
         return false;
 
     node_operator op = original->getOperator();
@@ -8564,7 +8564,48 @@ AutoScopeMigrateTransformer::AutoScopeMigrateTransformer(IWorkUnit * _wu, HqlCpp
     globalTarget = NULL;
 }
 
-void AutoScopeMigrateTransformer::analyseExpr(IHqlExpression * expr)
+//Ensure all input activities are marked as never hoisting, but child activities are unaffected
+void AutoScopeMigrateTransformer::ensureNeverHoisted(IHqlExpression * expr)
+{
+    for (;;)
+    {
+        AutoScopeMigrateInfo * extra = queryBodyExtra(expr);
+        if (extra->neverHoist)
+            return;
+        extra->neverHoist = true;
+        if (getNumActivityArguments(expr) != 1)
+            return;
+        expr = expr->queryChild(0);
+    }
+}
+
+//If an expression must be part of a compound source operation, ensure that the input
+//dataset is never hoisted, otherwise it will cause a later cannot key error.
+void AutoScopeMigrateTransformer::analysePass0(IHqlExpression * expr)
+{
+    switch (expr->getOperator())
+    {
+    case no_keyedlimit:
+        ensureNeverHoisted(expr);
+        break;
+    case no_hqlproject:
+    case no_newusertable:
+    case no_aggregate:
+    case no_newaggregate:
+        if (expr->hasAttribute(keyedAtom))
+            ensureNeverHoisted(expr);
+        break;
+    case no_filter:
+        if (filterIsKeyed(expr))
+            ensureNeverHoisted(expr);
+        break;
+    }
+
+    if (!alreadyVisited(expr))
+        NewHqlTransformer::analyseExpr(expr);
+}
+
+void AutoScopeMigrateTransformer::analysePass1(IHqlExpression * expr)
 {
     AutoScopeMigrateInfo * extra = queryBodyExtra(expr);
     if (isConditional)
@@ -8587,6 +8628,19 @@ void AutoScopeMigrateTransformer::analyseExpr(IHqlExpression * expr)
     activityDepth = savedDepth;
 }
 
+void AutoScopeMigrateTransformer::analyseExpr(IHqlExpression * expr)
+{
+    switch (pass)
+    {
+    case 0:
+        analysePass0(expr);
+        break;
+    case 1:
+        analysePass1(expr);
+        break;
+    }
+}
+
 void AutoScopeMigrateTransformer::doAnalyseConditionalExpr(IHqlExpression * expr, unsigned firstConditional)
 {
     bool wasConditional = isConditional;

ecl/hqlcpp/hqlttcpp.ipp

@@ -709,19 +709,20 @@ private:
 class AutoScopeMigrateInfo : public NewTransformInfo
 {
 public:
-    AutoScopeMigrateInfo(IHqlExpression * _original) : NewTransformInfo(_original) { useCount = 0; condUseCount = 0; manyGraphs = false; firstUseIsConditional = false; firstUseIsSequential = false; globalInsideChild = false; lastGraph = 0; }
+    AutoScopeMigrateInfo(IHqlExpression * _original) : NewTransformInfo(_original) { }
 
     bool addGraph(unsigned graph);
     bool doAutoHoist(IHqlExpression * transformed, bool minimizeWorkunitTemporaries);
 
 public:
-    unsigned    useCount;
-    unsigned    condUseCount;
-    unsigned    lastGraph;
-    bool manyGraphs;
-    bool firstUseIsConditional;
-    bool firstUseIsSequential;
-    bool globalInsideChild;
+    unsigned    useCount = 0;
+    unsigned    condUseCount = 0;
+    unsigned    lastGraph = 0;
+    bool manyGraphs = false;
+    bool firstUseIsConditional = false;
+    bool firstUseIsSequential = false;
+    bool globalInsideChild = false;
+    bool neverHoist = false;
 };
 
 class AutoScopeMigrateTransformer : public NewHqlTransformer
@@ -738,6 +739,10 @@ protected:
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
     virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr) { return CREATE_NEWTRANSFORMINFO(AutoScopeMigrateInfo, expr); }
 
+    void analysePass0(IHqlExpression * expr);
+    void analysePass1(IHqlExpression * expr);
+
+    void ensureNeverHoisted(IHqlExpression * expr);
     IHqlExpression * hoist(IHqlExpression * expr, IHqlExpression * hoisted);
     IHqlExpression * transformCond(IHqlExpression * expr);
     void doAnalyseExpr(IHqlExpression * expr);