Merge pull request #9868 from ghalliday/issue17436

HPCC-17436 Remove unused fields before optimizing the graph

Reviewed-By: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlthql.cpp

@@ -967,6 +967,34 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                 break;
             }
         }
+        else if (expandProcessed && no == no_alias)
+        {
+            if (!isNamedSymbol)
+            {
+                if (!expr->queryTransformExtra())
+                {
+                    bool wasInsideNewTransform = insideNewTransform;
+                    insideNewTransform = false;
+
+                    StringBuffer temp;
+                    scope.append(NULL);
+                    temp.appendf("alias%p ", expr);
+                    temp.append(":= ");
+
+                    toECL(expr, temp, false, false, 0, true);
+                    temp.append(";").newline();
+                    addExport(temp);
+
+                    scope.pop();
+                    insideNewTransform = wasInsideNewTransform;
+                    expr->setTransformExtra(expr);
+                }
+                s.appendf("alias%p", expr);
+                if (paren)
+                    s.append(')');
+                return;
+            }
+        }
 
         if (expandProcessed && !isNamedSymbol && no == no_record)
         {

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9421,6 +9421,12 @@ IHqlExpression * HqlCppTranslator::getResourcedGraph(IHqlExpression * expr, IHql
     if (true)
         resourced.setown(optimizeCompoundSource(resourced, CSFpreload|csfFlags));
 
+    //Check to see if fields can be removed - this helps LOOP bodies, but also seems to help situations where hoisting
+    //expressions prevents child expressions from preventing fields from being removed.
+    //Perform before the optimizeHqlExpression() so decisions about reducing row sizes are accurate
+    traceExpression("BeforeImplicitProjectGraph", resourced);
+    resourced.setown(insertImplicitProjects(*this, resourced, false));
+
     // Call optimizer before resourcing so items get moved over conditions, and remove other items
     // which would otherwise cause extra spills.
     traceExpression("BeforeOptimize", resourced);