Merge pull request #4434 from ghalliday/issue8316

HPCC-8316 Clean up the code to remove lots of workflow loops

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlcpp.ipp

@@ -1793,19 +1793,18 @@ protected:
     void doReportWarning(IHqlExpression * location, unsigned id, const char * msg);
 
     void optimizePersists(HqlExprArray & exprs);
-    void optimizePersists(WorkflowArray & workflow);
     void allocateSequenceNumbers(HqlExprArray & exprs);
-    void convertLogicalToActivities(WorkflowArray & array);
+    void convertLogicalToActivities(WorkflowItem & curWorkflow);
     void flattenDatasets(WorkflowArray & array);
 
-    void spotGlobalCSE(WorkflowArray & array);
+    void spotGlobalCSE(WorkflowItem & curWorkflow);
     IHqlExpression * spotGlobalCSE(IHqlExpression * _expr);
     void spotGlobalCSE(HqlExprArray & exprs);
     IHqlExpression * extractGlobalCSE(IHqlExpression * expr);
     void processCppBodyDirectives(IHqlExpression * expr);
 
 
-    void markThorBoundaries(WorkflowArray & array);
+    void markThorBoundaries(WorkflowItem & curWorkflow);
     bool transformGraphForGeneration(HqlQueryContext & query, WorkflowArray & exprs);
     void processEmbeddedLibraries(HqlExprArray & exprs, HqlExprArray & internalLibraries, bool isLibrary);
     void pickBestEngine(WorkflowArray & array);
@@ -1856,10 +1855,12 @@ public:
 public:
     void traceExpression(const char * title, IHqlExpression * expr, unsigned level=500);
     void traceExpressions(const char * title, HqlExprArray & exprs, unsigned level=500);
+    void traceExpressions(const char * title, WorkflowItem & workflow, unsigned level=500) { traceExpressions(title, workflow.queryExprs(), level); };
     void traceExpressions(const char * title, WorkflowArray & exprs);
 
     void checkNormalized(IHqlExpression * expr);
     void checkNormalized(WorkflowArray & array);
+    void checkNormalized(WorkflowItem & workflow) { checkNormalized(workflow.queryExprs()); }
     void checkNormalized(HqlExprArray & exprs);
     void checkNormalized(BuildCtx & ctx, IHqlExpression * expr);
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -17183,13 +17183,13 @@ void HqlCppTranslator::buildWorkflow(WorkflowArray & workflow)
     BuildCtx switchctx(performctx);
     IHqlStmt * switchStmt = switchctx.addSwitch(function);
 
-    optimizePersists(workflow);
     ForEachItemIn(idx, workflow)
     {
         WorkflowItem & action = workflow.item(idx);
         HqlExprArray & exprs = action.queryExprs();
         unsigned wfid = action.queryWfid();
 
+        optimizePersists(exprs);
         bool isEmpty = exprs.ordinality() == 0;
         if (exprs.ordinality() == 1 && (exprs.item(0).getOperator() == no_workflow_action))
             isEmpty = true;
@@ -17924,12 +17924,6 @@ void HqlCppTranslator::optimizePersists(HqlExprArray & exprs)
     createCompoundEnsure(exprs, 0, max-1);
 }
 
-void HqlCppTranslator::optimizePersists(WorkflowArray & workflow)
-{
-    ForEachItemIn(idx, workflow)
-        optimizePersists(workflow.item(idx).queryExprs());
-}
-
 IHqlExpression * HqlCppTranslator::extractGlobalCSE(IHqlExpression * expr)
 {
     AutoScopeMigrateTransformer transformer(wu(), *this);
@@ -18017,13 +18011,12 @@ void HqlCppTranslator::spotGlobalCSE(HqlExprArray & exprs)
     }
 }
 
-void HqlCppTranslator::spotGlobalCSE(WorkflowArray & array)
+void HqlCppTranslator::spotGlobalCSE(WorkflowItem & curWorkflow)
 {
     if (!insideLibrary() && options.globalAutoHoist)
     {
         unsigned startTime = msTick();
-        ForEachItemIn(idx, array)
-            spotGlobalCSE(array.item(idx).queryExprs());
+        spotGlobalCSE(curWorkflow.queryExprs());
         DEBUG_TIMER("EclServer: tree transform: spot global cse", msTick()-startTime);
     }
 }
@@ -18303,14 +18296,14 @@ void HqlCppTranslator::pickBestEngine(HqlExprArray & exprs)
 }
 
 
-void HqlCppTranslator::pickBestEngine(WorkflowArray & array)
+void HqlCppTranslator::pickBestEngine(WorkflowArray & workflow)
 {
     if (targetThor())
     {
         unsigned time = msTick();
-        ForEachItemIn(idx2, array)
+        ForEachItemIn(idx2, workflow)
         {
-            HqlExprArray & exprs = array.item(idx2).queryExprs();
+            HqlExprArray & exprs = workflow.item(idx2).queryExprs();
             ForEachItemIn(idx, exprs)
             {
                 if (needsRealThor(&exprs.item(idx)))

ecl/hqlcpp/hqlttcpp.cpp

@@ -981,18 +981,14 @@ void HqlThorBoundaryTransformer::transformRoot(const HqlExprArray & in, HqlExprA
 }
 
 
-void HqlCppTranslator::markThorBoundaries(WorkflowArray & array)
+void HqlCppTranslator::markThorBoundaries(WorkflowItem & curWorkflow)
 {
-    HqlThorBoundaryTransformer thorTransformer(wu(), targetRoxie(), options.maxRootMaybeThorActions, options.resourceConditionalActions, options.resourceSequential);
-    ForEachItemIn(idx, array)
-    {
-        WorkflowItem & cur = array.item(idx);
-        HqlExprArray & exprs = cur.queryExprs();
-        HqlExprArray bounded;
+    HqlExprArray & exprs = curWorkflow.queryExprs();
+    HqlExprArray bounded;
 
-        thorTransformer.transformRoot(exprs, bounded);
-        replaceArray(exprs, bounded);
-    }
+    HqlThorBoundaryTransformer thorTransformer(wu(), targetRoxie(), options.maxRootMaybeThorActions, options.resourceConditionalActions, options.resourceSequential);
+    thorTransformer.transformRoot(exprs, bounded);
+    replaceArray(exprs, bounded);
 }
 
 //---------------------------------------------------------------------------
@@ -1188,27 +1184,20 @@ IHqlExpression * ThorScalarTransformer::createTransformed(IHqlExpression * expr)
 
 */
 
-static void normalizeResultFormat(WorkflowArray & workflow, const HqlCppOptions & options)
+static void normalizeResultFormat(WorkflowItem & curWorkflow, const HqlCppOptions & options)
 {
-    ForEachItemIn(idx, workflow)
-    {
-        WorkflowItem & cur = workflow.item(idx);
-        HqlExprArray & exprs = cur.queryExprs();
+    //Until thor has a way of calling a graph and returning a result we need to call this transformer, so that
+    //scalars that need to be evaluated in thor are correctly hoisted.
+    ThorScalarTransformer transformer(options);
+    HqlExprArray & exprs = curWorkflow.queryExprs();
 
-        //Until thor has a way of calling a graph and returning a result we need to call this transformer, so that
-        //scalars that need to be evaluated in thor are correctly hoisted.
-        {
-            ThorScalarTransformer transformer(options);
-
-            transformer.analyseArray(exprs, 0);
+    transformer.analyseArray(exprs, 0);
 
-            if (transformer.needToTransform())
-            {
-                HqlExprArray transformed;
-                transformer.transformRoot(exprs, transformed);
-                replaceArray(exprs, transformed);
-            }
-        }
+    if (transformer.needToTransform())
+    {
+        HqlExprArray transformed;
+        transformer.transformRoot(exprs, transformed);
+        replaceArray(exprs, transformed);
     }
 }
 
@@ -3512,25 +3501,23 @@ IHqlExpression * ThorHqlTransformer::normalizeTableGrouping(IHqlExpression * exp
 }
 
 
-void HqlCppTranslator::convertLogicalToActivities(WorkflowArray & workflow)
+void HqlCppTranslator::convertLogicalToActivities(WorkflowItem & curWorkflow)
 {
     {
         unsigned time = msTick();
         ThorHqlTransformer transformer(*this, targetClusterType, wu());
-        ForEachItemIn(idx, workflow)
-        {
-            HqlExprArray & exprs = workflow.item(idx).queryExprs();
-            HqlExprArray transformed;
 
-            transformer.transformRoot(exprs, transformed);
+        HqlExprArray & exprs = curWorkflow.queryExprs();
+        HqlExprArray transformed;
 
-            replaceArray(exprs, transformed);
-        }
+        transformer.transformRoot(exprs, transformed);
+
+        replaceArray(exprs, transformed);
         DEBUG_TIMER("EclServer: tree transform: convert logical", msTick()-time);
     }
 
     if (queryOptions().normalizeLocations)
-        normalizeAnnotations(*this, workflow);
+        normalizeAnnotations(*this, curWorkflow.queryExprs());
 }
 
 //------------------------------------------------------------------------
@@ -4742,13 +4729,6 @@ IHqlExpression * optimizeActivities(IHqlExpression * expr, bool optimizeCountCom
     return transformer.transformRoot(expr);
 }
 
-void optimizeActivities(WorkflowArray & array, bool optimizeCountCompare, bool optimizeNonEmpty)
-{
-    ForEachItemIn(idx, array)
-        optimizeActivities(array.item(idx).queryExprs(), optimizeCountCompare, optimizeNonEmpty);
-}
-
-
 IHqlExpression * GlobalAttributeInfo::queryAlias(IHqlExpression * value)
 {
     if (!aliasName)
@@ -6778,9 +6758,9 @@ bool moveUnconditionalEarlier(HqlExprArray & in)
 
 //------------------------------------------------------------------------
 
-void mergeThorGraphs(HqlExprArray & exprs, bool resourceConditionalActions, bool resourceSequential);
+static void mergeThorGraphs(HqlExprArray & exprs, bool resourceConditionalActions, bool resourceSequential);
 
-IHqlExpression * mergeThorGraphs(IHqlExpression * expr, bool resourceConditionalActions, bool resourceSequential)
+static IHqlExpression * mergeThorGraphs(IHqlExpression * expr, bool resourceConditionalActions, bool resourceSequential)
 {
     HqlExprArray args;
     expr->unwindList(args, no_actionlist);
@@ -6789,7 +6769,7 @@ IHqlExpression * mergeThorGraphs(IHqlExpression * expr, bool resourceConditional
 }
 
 
-void mergeThorGraphs(HqlExprArray & exprs, bool resourceConditionalActions, bool resourceSequential)
+static void mergeThorGraphs(HqlExprArray & exprs, bool resourceConditionalActions, bool resourceSequential)
 {
     HqlExprArray thorActions;
     HqlExprArray combined;
@@ -6923,13 +6903,11 @@ void mergeThorGraphs(HqlExprArray & exprs, bool resourceConditionalActions, bool
     replaceArray(exprs, combined);
 }
 
-void mergeThorGraphs(WorkflowArray & array, bool resourceConditionalActions, bool resourceSequential)
+void mergeThorGraphs(WorkflowItem & workflow, bool resourceConditionalActions, bool resourceSequential)
 {
-    ForEachItemIn(idx4, array)
-        groupThorGraphs(array.item(idx4).queryExprs());
+    groupThorGraphs(workflow.queryExprs());
 
-    ForEachItemIn(idx2, array)
-        mergeThorGraphs(array.item(idx2).queryExprs(), resourceConditionalActions, resourceSequential);
+    mergeThorGraphs(workflow.queryExprs(), resourceConditionalActions, resourceSequential);
 }
 
 //------------------------------------------------------------------------
@@ -7458,58 +7436,54 @@ IHqlExpression * NewScopeMigrateTransformer::createTransformed(IHqlExpression *
 
 
 
-void migrateExprToNaturalLevel(WorkflowArray & array, IWorkUnit * wu, HqlCppTranslator & translator)
+void migrateExprToNaturalLevel(WorkflowItem & cur, IWorkUnit * wu, HqlCppTranslator & translator)
 {
     const HqlCppOptions & options = translator.queryOptions();
-    ForEachItemIn(idx, array)
+    HqlExprArray & exprs = cur.queryExprs();
+    if (translator.queryOptions().moveUnconditionalActions)
+        moveUnconditionalEarlier(exprs);
+    translator.checkNormalized(exprs);
+
+    if (options.hoistSimpleGlobal)
     {
-        WorkflowItem & cur = array.item(idx);
-        HqlExprArray & exprs = cur.queryExprs();
-        if (translator.queryOptions().moveUnconditionalActions)
-            moveUnconditionalEarlier(exprs);
+        ScalarGlobalTransformer transformer(translator);
+        HqlExprArray results;
+        transformer.analyseArray(exprs, 0);
+        transformer.transformRoot(exprs, results);
+        replaceArray(exprs, results);
         translator.checkNormalized(exprs);
+    }
 
-        if (options.hoistSimpleGlobal)
-        {
-            ScalarGlobalTransformer transformer(translator);
-            HqlExprArray results;
-            transformer.analyseArray(exprs, 0);
-            transformer.transformRoot(exprs, results);
-            replaceArray(exprs, results);
-            translator.checkNormalized(exprs);
-        }
-
-        translator.traceExpressions("m0", exprs);
-
-        if (options.workunitTemporaries)
-        {
-            ExplicitGlobalTransformer transformer(wu, translator);
+    translator.traceExpressions("m0", exprs);
 
-            transformer.analyseArray(exprs, 0);
-            if (transformer.needToTransform())
-            {
-                HqlExprArray results;
-                transformer.transformRoot(exprs, results);
-                replaceArray(exprs, results);
-            }
-            translator.checkNormalized(exprs);
-        }
-
-        translator.traceExpressions("m1", exprs);
+    if (options.workunitTemporaries)
+    {
+        ExplicitGlobalTransformer transformer(wu, translator);
 
-        if (options.allowScopeMigrate) // && !options.minimizeWorkunitTemporaries)
+        transformer.analyseArray(exprs, 0);
+        if (transformer.needToTransform())
         {
-            NewScopeMigrateTransformer transformer(wu, translator);
             HqlExprArray results;
-
-            transformer.analyseArray(exprs, 0);
             transformer.transformRoot(exprs, results);
             replaceArray(exprs, results);
-            translator.checkNormalized(exprs);
         }
+        translator.checkNormalized(exprs);
+    }
 
-        translator.traceExpressions("m2", exprs);
+    translator.traceExpressions("m1", exprs);
+
+    if (options.allowScopeMigrate) // && !options.minimizeWorkunitTemporaries)
+    {
+        NewScopeMigrateTransformer transformer(wu, translator);
+        HqlExprArray results;
+
+        transformer.analyseArray(exprs, 0);
+        transformer.transformRoot(exprs, results);
+        replaceArray(exprs, results);
+        translator.checkNormalized(exprs);
     }
+
+    translator.traceExpressions("m2", exprs);
 }
 
 void expandGlobalDatasets(WorkflowArray & array, IWorkUnit * wu, HqlCppTranslator & translator)
@@ -7810,20 +7784,16 @@ bool TrivialGraphRemover::isTrivialGraph(IHqlExpression * expr)
 
 }
 
-void removeTrivialGraphs(WorkflowArray & workflow)
+void removeTrivialGraphs(WorkflowItem & curWorkflow)
 {
-    ForEachItemIn(idx, workflow)
+    HqlExprArray & exprs = curWorkflow.queryExprs();
+    TrivialGraphRemover transformer;
+    transformer.analyseArray(exprs, 0);
+    if (transformer.worthTransforming())
     {
-        WorkflowItem & cur = workflow.item(idx);
-        HqlExprArray & exprs = cur.queryExprs();
-        TrivialGraphRemover transformer;
-        transformer.analyseArray(exprs, 0);
-        if (transformer.worthTransforming())
-        {
-            HqlExprArray simplified;
-            transformer.transformRoot(exprs, simplified);
-            replaceArray(exprs, simplified);
-        }
+        HqlExprArray simplified;
+        transformer.transformRoot(exprs, simplified);
+        replaceArray(exprs, simplified);
     }
 }
 
@@ -9892,12 +9862,6 @@ void normalizeAnnotations(HqlCppTranslator & translator, HqlExprArray & exprs)
     DEBUG_TIMERX(translator.queryTimeReporter(), "EclServer: tree transform: normalize.annotations", msTick()-time);
 }
 
-void normalizeAnnotations(HqlCppTranslator & translator, WorkflowArray & workflow)
-{
-    ForEachItemIn(i, workflow)
-        normalizeAnnotations(translator, workflow.item(i).queryExprs());
-}
-
 //---------------------------------------------------------------------------
 
 static HqlTransformerInfo containsCompoundTransformerInfo("ContainsCompoundTransformer");
@@ -12589,36 +12553,12 @@ bool HqlCppTranslator::transformGraphForGeneration(HqlQueryContext & query, Work
 
     unsigned time4 = msTick();
     ::extractWorkflow(*this, exprs, workflow);
+
+
     traceExpressions("workflow", workflow);
     checkNormalized(workflow);
     DEBUG_TIMER("EclServer: tree transform: stored results", msTick()-time4);
 
-#ifdef USE_SELSEQ_UID
-    if (options.normalizeSelectorSequence)
-    {
-        unsigned time = msTick();
-        ForEachItemIn(i, workflow)
-        {
-            LeftRightTransformer normalizer;
-            normalizer.process(workflow.item(i).queryExprs());
-        }
-        DEBUG_TIMERX(queryTimeReporter(), "EclServer: tree transform: left right", msTick()-time);
-        //traceExpressions("after implicit alias", workflow);
-    }
-#endif
-
-    if (queryOptions().createImplicitAliases)
-    {
-        unsigned time = msTick();
-        ForEachItemIn(i, workflow)
-        {
-            ImplicitAliasTransformer normalizer;
-            normalizer.process(workflow.item(i).queryExprs());
-        }
-        DEBUG_TIMERX(queryTimeReporter(), "EclServer: tree transform: implicit alias", msTick()-time);
-        //traceExpressions("after implicit alias", workflow);
-    }
-
     if (outputLibrary && workflow.ordinality() > 1)
     {
         unsigned cnt = 0;
@@ -12635,93 +12575,117 @@ bool HqlCppTranslator::transformGraphForGeneration(HqlQueryContext & query, Work
         }
     }
 
+    ForEachItemIn(i, workflow)
     {
-        unsigned startTime = msTick();
-        hoistNestedCompound(*this, workflow);
-        DEBUG_TIMER("EclServer: tree transform: hoist nested compound", msTick()-startTime);
-    }
+        WorkflowItem & curWorkflow = workflow.item(i);
 
-    if (options.optimizeNestedConditional)
-    {
-        cycle_t time = msTick();
-        ForEachItemIn(idx, workflow)
-            optimizeNestedConditional(workflow.item(idx).queryExprs());
-        DEBUG_TIMER("EclServer: optimize nested conditional", msTick()-time);
-        traceExpressions("nested", workflow);
-        checkNormalized(workflow);
-    }
+#ifdef USE_SELSEQ_UID
+        if (options.normalizeSelectorSequence)
+        {
+            unsigned time = msTick();
+            LeftRightTransformer normalizer;
+            normalizer.process(curWorkflow.queryExprs());
+            DEBUG_TIMERX(queryTimeReporter(), "EclServer: tree transform: left right", msTick()-time);
+            //traceExpressions("after implicit alias", workflow);
+        }
+#endif
 
-    checkNormalized(workflow);
-    //sort(x)[n] -> topn(x, n)[]n, count(x)>n -> count(choosen(x,n+1)) > n and possibly others
-    {
-        unsigned startTime = msTick();
-        optimizeActivities(workflow, !targetThor(), options.optimizeNonEmpty);
-        DEBUG_TIMER("EclServer: tree transform: optimize activities", msTick()-startTime);
-    }
-    checkNormalized(workflow);
+        if (queryOptions().createImplicitAliases)
+        {
+            unsigned time = msTick();
+            ImplicitAliasTransformer normalizer;
+            normalizer.process(curWorkflow.queryExprs());
+            DEBUG_TIMERX(queryTimeReporter(), "EclServer: tree transform: implicit alias", msTick()-time);
+            //traceExpressions("after implicit alias", workflow);
+        }
 
-    unsigned time5 = msTick();
-    migrateExprToNaturalLevel(workflow, wu(), *this);       // Ensure expressions are evaluated at the best level - e.g., counts moved to most appropriate level.
-    DEBUG_TIMER("EclServer: tree transform: migrate", msTick()-time5);
-    //transformToAliases(exprs);
-    traceExpressions("migrate", workflow);
-    checkNormalized(workflow);
+        {
+            unsigned startTime = msTick();
+            hoistNestedCompound(*this, curWorkflow.queryExprs());
+            DEBUG_TIMER("EclServer: tree transform: hoist nested compound", msTick()-startTime);
+        }
 
-    unsigned time2 = msTick();
-    markThorBoundaries(workflow);                                               // work out which engine is going to perform which operation.
-    DEBUG_TIMER("EclServer: tree transform: thor hole", msTick()-time2);
-    traceExpressions("boundary", workflow);
-    checkNormalized(workflow);
+        if (options.optimizeNestedConditional)
+        {
+            cycle_t time = msTick();
+            optimizeNestedConditional(curWorkflow.queryExprs());
+            DEBUG_TIMER("EclServer: optimize nested conditional", msTick()-time);
+            traceExpressions("nested", curWorkflow);
+            checkNormalized(curWorkflow);
+        }
 
-    if (options.optimizeGlobalProjects)
-    {
-        cycle_t time = msTick();
-        ForEachItemIn(idx, workflow)
-            insertImplicitProjects(*this, workflow.item(idx).queryExprs());
-        DEBUG_TIMER("EclServer: global implicit projects", msTick()-time);
-        traceExpressions("implicit", workflow);
-        checkNormalized(workflow);
-    }
+        checkNormalized(curWorkflow);
+        //sort(x)[n] -> topn(x, n)[]n, count(x)>n -> count(choosen(x,n+1)) > n and possibly others
+        {
+            unsigned startTime = msTick();
+            optimizeActivities(curWorkflow.queryExprs(), !targetThor(), options.optimizeNonEmpty);
+            DEBUG_TIMER("EclServer: tree transform: optimize activities", msTick()-startTime);
+        }
+        checkNormalized(curWorkflow);
 
-    unsigned time3 = msTick();
-    normalizeResultFormat(workflow, options);
-    DEBUG_TIMER("EclServer: tree transform: normalize result", msTick()-time3);
-    traceExpressions("results", workflow);
-    checkNormalized(workflow);
+        unsigned time5 = msTick();
+        migrateExprToNaturalLevel(curWorkflow, wu(), *this);       // Ensure expressions are evaluated at the best level - e.g., counts moved to most appropriate level.
+        DEBUG_TIMER("EclServer: tree transform: migrate", msTick()-time5);
+        //transformToAliases(exprs);
+        traceExpressions("migrate", curWorkflow);
+        checkNormalized(curWorkflow);
 
-    optimizePersists(workflow);
+        unsigned time2 = msTick();
+        markThorBoundaries(curWorkflow);                                               // work out which engine is going to perform which operation.
+        DEBUG_TIMER("EclServer: tree transform: thor hole", msTick()-time2);
+        traceExpressions("boundary", curWorkflow);
+        checkNormalized(curWorkflow);
 
-    traceExpressions("per", workflow);
-    checkNormalized(workflow);
-//  flattenDatasets(workflow);
-//  traceExpressions("flatten", workflow);
+        if (options.optimizeGlobalProjects)
+        {
+            cycle_t time = msTick();
+            insertImplicitProjects(*this, curWorkflow.queryExprs());
+            DEBUG_TIMER("EclServer: global implicit projects", msTick()-time);
+            traceExpressions("implicit", curWorkflow);
+            checkNormalized(curWorkflow);
+        }
 
-    {
-        unsigned startTime = msTick();
-        mergeThorGraphs(workflow, options.resourceConditionalActions, options.resourceSequential);          // reduces number of graphs sent to thor
-        DEBUG_TIMER("EclServer: tree transform: merge thor", msTick()-startTime);
-    }
+        unsigned time3 = msTick();
+        normalizeResultFormat(curWorkflow, options);
+        DEBUG_TIMER("EclServer: tree transform: normalize result", msTick()-time3);
+        traceExpressions("results", curWorkflow);
+        checkNormalized(curWorkflow);
 
-    traceExpressions("merged", workflow);
-    checkNormalized(workflow);
+        optimizePersists(curWorkflow.queryExprs());
 
-    if (queryOptions().normalizeLocations)
-        normalizeAnnotations(*this, workflow);
+        traceExpressions("per", curWorkflow);
+        checkNormalized(curWorkflow);
+    //  flattenDatasets(workflow);
+    //  traceExpressions("flatten", workflow);
 
-    spotGlobalCSE(workflow);                                                        // spot CSE within those graphs, and create some more
-    checkNormalized(workflow);
+        {
+            unsigned startTime = msTick();
+            mergeThorGraphs(curWorkflow, options.resourceConditionalActions, options.resourceSequential);          // reduces number of graphs sent to thor
+            DEBUG_TIMER("EclServer: tree transform: merge thor", msTick()-startTime);
+        }
 
-    //expandGlobalDatasets(workflow, wu(), *this);
+        traceExpressions("merged", curWorkflow);
+        checkNormalized(curWorkflow);
 
-    {
-        unsigned startTime = msTick();
-        mergeThorGraphs(workflow, options.resourceConditionalActions, options.resourceSequential);
-        DEBUG_TIMER("EclServer: tree transform: merge thor", msTick()-startTime);
+        if (queryOptions().normalizeLocations)
+            normalizeAnnotations(*this, curWorkflow.queryExprs());
+
+        spotGlobalCSE(curWorkflow);                                                        // spot CSE within those graphs, and create some more
+        checkNormalized(curWorkflow);
+
+        //expandGlobalDatasets(workflow, wu(), *this);
+
+        {
+            unsigned startTime = msTick();
+            mergeThorGraphs(curWorkflow, options.resourceConditionalActions, options.resourceSequential);
+            DEBUG_TIMER("EclServer: tree transform: merge thor", msTick()-startTime);
+        }
+        checkNormalized(curWorkflow);
+
+        removeTrivialGraphs(curWorkflow);
+        checkNormalized(curWorkflow);
     }
-    checkNormalized(workflow);
 
-    removeTrivialGraphs(workflow);
-    checkNormalized(workflow);
 
 #ifndef PICK_ENGINE_EARLY
     if (options.pickBestEngine)
@@ -12729,22 +12693,25 @@ bool HqlCppTranslator::transformGraphForGeneration(HqlQueryContext & query, Work
 #endif
     updateClusterType();
 
-    traceExpressions("before convert to logical", workflow);
+    ForEachItemIn(i2, workflow)
+    {
+        WorkflowItem & curWorkflow = workflow.item(i2);
+        traceExpressions("before convert to logical", curWorkflow);
 
-    convertLogicalToActivities(workflow);                                           // e.g., merge disk reads, transform group, all to sort etc.
+        convertLogicalToActivities(curWorkflow);                                           // e.g., merge disk reads, transform group, all to sort etc.
 
-#ifndef _DEBUG
-    if (options.regressionTest)
-#endif
-    {
-        unsigned startTime = msTick();
-        ForEachItemIn(icheck, workflow)
-            checkDependencyConsistency(workflow.item(icheck).queryExprs());
-        DEBUG_TIMER("EclServer: tree transform: check dependency", msTick()-startTime);
-    }
+    #ifndef _DEBUG
+        if (options.regressionTest)
+    #endif
+        {
+            unsigned startTime = msTick();
+            checkDependencyConsistency(curWorkflow.queryExprs());
+            DEBUG_TIMER("EclServer: tree transform: check dependency", msTick()-startTime);
+        }
 
-    traceExpressions("end transformGraphForGeneration", workflow);
-    checkNormalized(workflow);
+        traceExpressions("end transformGraphForGeneration", curWorkflow);
+        checkNormalized(curWorkflow);
+    }
     return true;
 }
 

ecl/hqlcpp/hqlttcpp.ipp

@@ -1085,7 +1085,6 @@ protected:
 };
 
 void normalizeAnnotations(HqlCppTranslator & translator, HqlExprArray & exprs);
-void normalizeAnnotations(HqlCppTranslator & translator, WorkflowArray & workflow);
 
 //---------------------------------------------------------------------------
 
@@ -1216,13 +1215,12 @@ void hoistNestedCompound(HqlCppTranslator & _translator, HqlExprArray & exprs);
 void hoistNestedCompound(HqlCppTranslator & _translator, WorkflowArray & workflow);
 
 //---------------------------------------------------------------------------
-void expandGlobalDatasets(WorkflowArray & array, IWorkUnit * wu, HqlCppTranslator & translator);
-void mergeThorGraphs(WorkflowArray & array, bool resourceConditionalActions, bool resourceSequential);
-void migrateExprToNaturalLevel(WorkflowArray & array, IWorkUnit * wu, HqlCppTranslator & translator);
-void removeTrivialGraphs(WorkflowArray & workflow);
+void expandGlobalDatasets(WorkflowItem & curWorkflow, IWorkUnit * wu, HqlCppTranslator & translator);
+void mergeThorGraphs(WorkflowItem & curWorkflow, bool resourceConditionalActions, bool resourceSequential);
+void migrateExprToNaturalLevel(WorkflowItem & curWorkflow, IWorkUnit * wu, HqlCppTranslator & translator);
+void removeTrivialGraphs(WorkflowItem & curWorkflow);
 void extractWorkflow(HqlCppTranslator & translator, HqlExprArray & exprs, WorkflowArray & out);
 void optimizeActivities(HqlExprArray & exprs, bool optimizeCountCompare, bool optimizeNonEmpty);
-void optimizeActivities(WorkflowArray & array, bool optimizeCountCompare, bool optimizeNonEmpty);
 IHqlExpression * optimizeActivities(IHqlExpression * expr, bool optimizeCountCompare, bool optimizeNonEmpty);
 IHqlExpression * insertImplicitProjects(HqlCppTranslator & translator, IHqlExpression * expr, bool optimizeSpills);
 void insertImplicitProjects(HqlCppTranslator & translator, HqlExprArray & exprs);