HPCC-17370 Common up uses of CLUSTERSIZE within a workflow item

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlexpr.cpp

@@ -4222,8 +4222,13 @@ void CHqlRealExpression::updateFlagsAfterOperands()
             break;
         }
     case no_clustersize:
-        //wrong, but improves the generated code
-        infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
+        //pure is added with the wfid as a parameter which guarantees that it can be commoned up.
+        if (!hasAttribute(pureAtom))
+            infoFlags |= (HEFnoduplicate);
+
+        //Wrong, but improves the generated code (preventing it being serialized).
+        //Even better would be to evaluate once, but not serialize...
+        infoFlags |= (HEFcontextDependentException);
         break;
     case no_type:
         {

ecl/hql/hqlthql.cpp

@@ -2080,6 +2080,12 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             else
                 toECL(child0, s, false, inType);
             break;
+        case no_clustersize:
+            if (expandProcessed)
+                defaultToECL(expr, s, inType);
+            else
+                s.append(getEclOpString(no));
+            break;
         case no_map:
         {
             unsigned kids = expr->numChildren();

ecl/hql/hqltrans.cpp

@@ -1784,6 +1784,13 @@ void NewHqlTransformer::transformRoot(const HqlExprArray & in, HqlExprArray & ou
     }
 }
 
+void NewHqlTransformer::transformRoot(HqlExprArray & exprs)
+{
+    HqlExprArray temp;
+    transformRoot(exprs, temp);
+    exprs.swapWith(temp);
+}
+
 
 IHqlExpression * NewHqlTransformer::doTransformRootExpr(IHqlExpression * expr)
 {

ecl/hql/hqltrans.ipp

@@ -441,6 +441,7 @@ public:
     void analyseArray(const HqlExprArray & exprs, unsigned pass);
 
     void transformRoot(const HqlExprArray & in, HqlExprArray & out);
+    void transformRoot(HqlExprArray & out);
     IHqlExpression * transformRoot(IHqlExpression * expr) { return doTransformRootExpr(expr); }
 
 protected:

ecl/hqlcpp/hqlttcpp.cpp

@@ -4817,8 +4817,8 @@ IHqlExpression * CompoundActivityTransformer::createTransformed(IHqlExpression *
 //------------------------------------------------------------------------
 
 static HqlTransformerInfo optimizeActivityTransformerInfo("OptimizeActivityTransformer");
-OptimizeActivityTransformer::OptimizeActivityTransformer(bool _optimizeCountCompare, bool _optimizeNonEmpty)
-: NewHqlTransformer(optimizeActivityTransformerInfo)
+OptimizeActivityTransformer::OptimizeActivityTransformer(unsigned _wfid, bool _optimizeCountCompare, bool _optimizeNonEmpty)
+: NewHqlTransformer(optimizeActivityTransformerInfo), wfid(_wfid)
 {
     optimizeCountCompare = _optimizeCountCompare; optimizeNonEmpty = _optimizeNonEmpty;
 }
@@ -5161,7 +5161,14 @@ IHqlExpression * OptimizeActivityTransformer::doCreateTransformed(IHqlExpression
             unwindChildren(args, expr, 2);
             return expr->clone(args);
         }
-
+    case no_clustersize:
+        if (wfid && !expr->hasAttribute(pureAtom))
+        {
+            OwnedHqlExpr attribute = createExprAttribute(pureAtom, getSizetConstant(wfid));
+            OwnedHqlExpr modified = appendOwnedOperand(expr, attribute.getClear());
+            return expr->cloneAllAnnotations(modified);
+        }
+        break;
     case no_eq:
     case no_ne:
     case no_le:
@@ -5187,21 +5194,11 @@ IHqlExpression * OptimizeActivityTransformer::doCreateTransformed(IHqlExpression
 }
 
 
-void optimizeActivities(HqlExprArray & exprs, bool optimizeCountCompare, bool optimizeNonEmpty)
+void optimizeActivities(unsigned wfid, HqlExprArray & exprs, bool optimizeCountCompare, bool optimizeNonEmpty)
 {
-    OptimizeActivityTransformer transformer(optimizeCountCompare, optimizeNonEmpty);
-    HqlExprArray results;
+    OptimizeActivityTransformer transformer(wfid, optimizeCountCompare, optimizeNonEmpty);
     transformer.analyseArray(exprs, 0);
-    transformer.transformRoot(exprs, results);
-    replaceArray(exprs, results);
-}
-
-IHqlExpression * optimizeActivities(IHqlExpression * expr, bool optimizeCountCompare, bool optimizeNonEmpty)
-{
-    OptimizeActivityTransformer transformer(optimizeCountCompare, optimizeNonEmpty);
-    HqlExprArray results;
-    transformer.analyse(expr, 0);
-    return transformer.transformRoot(expr);
+    transformer.transformRoot(exprs);
 }
 
 IHqlExpression * GlobalAttributeInfo::queryAlias(IHqlExpression * value)
@@ -7701,6 +7698,8 @@ bool ScalarGlobalTransformer::isComplex(IHqlExpression * expr, bool checkGlobal)
     case no_constant:
     case no_globalscope:
     case no_libraryinput:
+    case no_clustersize:
+    case no_nothor:
         return false;
     case no_cast:
     case no_implicitcast:
@@ -10780,6 +10779,7 @@ void LeftRightTransformer::process(HqlExprArray & exprs)
     transformRoot(exprs, transformed);
     replaceArray(exprs, transformed);
 }
+
 //---------------------------------------------------------------------------------------------------------------------
 
 /*
@@ -13355,9 +13355,7 @@ void normalizeHqlTree(HqlCppTranslator & translator, HqlExprArray & exprs)
     {
         cycle_t startCycles = get_cycles_now();
         HqlScopeTagger normalizer(translator.queryErrorProcessor(), translator.queryLocalOnWarningMapper());
-        HqlExprArray transformed;
-        normalizer.transformRoot(exprs, transformed);
-        replaceArray(exprs, transformed);
+        normalizer.transformRoot(exprs);
         translator.noteFinishedTiming("compile:tree transform: normalize.scope", startCycles);
     }
 
@@ -13730,7 +13728,7 @@ void HqlCppTranslator::transformWorkflowItem(WorkflowItem & curWorkflow)
     //sort(x)[n] -> topn(x, n)[]n, count(x)>n -> count(choosen(x,n+1)) > n and possibly others
     {
         cycle_t startCycles = get_cycles_now();
-        optimizeActivities(curWorkflow.queryExprs(), !targetThor(), options.optimizeNonEmpty);
+        optimizeActivities(curWorkflow.queryWfid(), curWorkflow.queryExprs(), !targetThor(), options.optimizeNonEmpty);
         noteFinishedTiming("compile:tree transform: optimize activities", startCycles);
     }
     checkNormalized(curWorkflow);

ecl/hqlcpp/hqlttcpp.ipp

@@ -363,7 +363,7 @@ private:
 class OptimizeActivityTransformer : public NewHqlTransformer
 {
 public:
-    OptimizeActivityTransformer(bool _optimizeCountCompare, bool _optimizeNonEmpty);
+    OptimizeActivityTransformer(unsigned _wfid, bool _optimizeCountCompare, bool _optimizeNonEmpty);
 
     virtual void analyseExpr(IHqlExpression * expr);
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
@@ -379,6 +379,7 @@ protected:
     inline bool isShared(IHqlExpression * expr)                 { return queryBodyExtra(expr)->isShared(); }
 
 protected:
+    unsigned wfid;
     bool optimizeCountCompare;
     bool optimizeNonEmpty;
 };
@@ -1299,8 +1300,7 @@ void mergeThorGraphs(WorkflowItem & curWorkflow, bool resourceConditionalActions
 void migrateExprToNaturalLevel(WorkflowItem & curWorkflow, IWorkUnit * wu, HqlCppTranslator & translator);
 void removeTrivialGraphs(WorkflowItem & curWorkflow);
 void extractWorkflow(HqlCppTranslator & translator, HqlExprArray & exprs, WorkflowArray & out);
-void optimizeActivities(HqlExprArray & exprs, bool optimizeCountCompare, bool optimizeNonEmpty);
-IHqlExpression * optimizeActivities(IHqlExpression * expr, bool optimizeCountCompare, bool optimizeNonEmpty);
+void optimizeActivities(unsigned wfid, HqlExprArray & exprs, bool optimizeCountCompare, bool optimizeNonEmpty);
 IHqlExpression * insertImplicitProjects(HqlCppTranslator & translator, IHqlExpression * expr, bool optimizeSpills);
 void insertImplicitProjects(HqlCppTranslator & translator, HqlExprArray & exprs);