Reimplement the way child queries are commoned up.

This is a reasonably substantial change.  It is partially complete, but
sufficiently good to be released and used as a building block for
further work.

The main change is in the code that spots multiple dataset operations
that will require a child query to be generated to execute them, and
combine them into a single child query so that any shared code is
only evalauted once.

The new transformer has the following stages
a) track which expressions are conditional/unconditional
b) work out which expressions are candidates for commoning up
c) calculate where those commoned up expressions should be evaluated
d) what if() conditions are needed to protect the canidate expressions
if they are executed all together?
e) What are the final conclusions on which candidates to combine.

One difference is that the transformer now creates a graph node to
represent the child query (no_childquery) instead of directly building
the child query.
This helps prepares the way for a long term goal of performing all tree
transformations up front before we start to generate any code.  There
are still other things that prevent that, but this was a key issue.

Associated changes include refactoring of the transformer that
works out whether something is conditional.  Disassociating the unique
id that represents a graph from the activity id of the graph,
supporting no_compound in more places (although not actually needed for
these changes).

In future I would hope to also combine some conditional child graphs
but that will require problems in other areas to be fixed first.

ecl/hql/hqlatoms.cpp

@@ -288,6 +288,7 @@ _ATOM rightAtom;
 _ATOM rightonlyAtom;
 _ATOM rightouterAtom;
 _ATOM rollbackAtom;
+_ATOM _root_Atom;
 _ATOM rowAtom;
 _ATOM _rowsid_Atom;
 _ATOM rowLimitAtom;
@@ -668,6 +669,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     rightonlyAtom = createLowerCaseAtom("right only");
     rightouterAtom = createLowerCaseAtom("right outer");
     MAKEATOM(rollback);
+    MAKESYSATOM(root);
     MAKEATOM(row);
     MAKESYSATOM(rowsid);
     MAKEATOM(rowLimit);

ecl/hql/hqlatoms.hpp

@@ -292,6 +292,7 @@ extern HQL_API _ATOM rightAtom;
 extern HQL_API _ATOM rightonlyAtom;
 extern HQL_API _ATOM rightouterAtom;
 extern HQL_API _ATOM rollbackAtom;
+extern HQL_API _ATOM _root_Atom;
 extern HQL_API _ATOM rowAtom;
 extern HQL_API _ATOM _rowsid_Atom;
 extern HQL_API _ATOM rowLimitAtom;

ecl/hql/hqlattr.cpp

@@ -435,6 +435,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_reference:
     case no_assign_addfiles:
     case no_nullptr:
+    case no_childquery:
 
 //Workflow
     case no_stored:
@@ -606,7 +607,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_persist_check:
     case no_dataset_from_transform:
 
-    case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:

ecl/hql/hqlexpr.cpp

@@ -1431,8 +1431,9 @@ const char *getOpString(node_operator op)
     case no_assign_addfiles: return "+=";
     case no_debug_option_value: return "__DEBUG__";
     case no_dataset_alias: return "TABLE";
+    case no_childquery: return "no_childquery";
 
-    case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
@@ -3769,27 +3770,6 @@ CHqlExpression::~CHqlExpression()
 {
 //  DBGLOG("%lx: Destroy", (unsigned)(IHqlExpression *)this);
     ::Release(type);
-//  RELEASE_TRANSFORM_EXTRA(transformDepth, transformExtra);
-#ifdef _DEBUG       // Not strictly correct, but good enough for me in debug mode...
-    if (!hashcode)
-    {
-
-        switch(op)
-        {
-        case no_mergedscope:
-        case no_remotescope:
-        case no_privatescope:
-        case no_service:
-        case no_none:
-            break;
-        default:
-
-            PrintLog("%s did not hash", getOpString(op));
-            // assertex(false);
-            break;
-        }
-    }
-#endif
 }
 
 IHqlScope * CHqlExpression::queryScope() 

ecl/hql/hqlexpr.hpp

@@ -353,7 +353,7 @@ enum _node_operator {
     no_unused23,
     no_unused24,
         no_dataset_from_transform,
-    no_unused2,
+        no_childquery,
         no_unknown,
     no_unused3,
     no_unused4,

ecl/hql/hqltrans.cpp

@@ -1102,9 +1102,8 @@ NewHqlTransformer::NewHqlTransformer(HqlTransformerInfo & _info) : ANewHqlTransf
 }
 
 
-bool NewHqlTransformer::alreadyVisited(IHqlExpression * expr)
+bool NewHqlTransformer::alreadyVisited(ANewTransformInfo * extra)
 {
-    ANewTransformInfo * extra = queryTransformExtra(expr);
     if (extra->lastPass == pass)
         return true;
 
@@ -1117,6 +1116,12 @@ bool NewHqlTransformer::alreadyVisited(IHqlExpression * expr)
 }
 
 
+bool NewHqlTransformer::alreadyVisited(IHqlExpression * expr)
+{
+    return alreadyVisited(queryTransformExtra(expr));
+}
+
+
 void NewHqlTransformer::analyse(IHqlExpression * expr, unsigned _pass)
 {
     pass = _pass;
@@ -2093,80 +2098,42 @@ NestedHqlMapTransformer::NestedHqlMapTransformer() : NestedHqlTransformer(nested
 
 //---------------------------------------------------------------------------
 
-HoistingHqlTransformer::HoistingHqlTransformer(HqlTransformerInfo & _info, unsigned _flags) : NewHqlTransformer(_info)
+ConditionalHqlTransformer::ConditionalHqlTransformer(HqlTransformerInfo & _info, unsigned _flags) : NewHqlTransformer(_info)
 { 
     flags = _flags; 
-    target = NULL; 
     conditionDepth = 0; 
     containsUnknownIndependentContents = false;
 }
 
-void HoistingHqlTransformer::setParent(const HoistingHqlTransformer * parent)
-{
-    assertex(parent->independentCache);
-    independentCache.set(parent->independentCache);
-}
-
-
 
-void HoistingHqlTransformer::transformRoot(const HqlExprArray & in, HqlExprArray & out)
+ANewTransformInfo * ConditionalHqlTransformer::createTransformInfo(IHqlExpression * expr)
 {
-    HqlExprArray * savedTarget = target;
-    target = &out;
-
-    //If there is a single ensureresult, then transform it specially, so that any hoisted values are
-    //only inside the ensure result.  If multiple assume we want to create cses globally (e.g., shared stored)
-    if ((in.ordinality() == 1) && (in.item(0).getOperator() == no_ensureresult))
-        out.append(*transformEnsureResult(&in.item(0)));
-    else
-        NewHqlTransformer::transformRoot(in, out);
-    target = savedTarget;
-}
-
-
-void HoistingHqlTransformer::appendToTarget(IHqlExpression & cur)
-{
-    target->append(cur);
+    return CREATE_NEWTRANSFORMINFO(ConditionalTransformInfo, expr);
 }
 
-ANewTransformInfo * HoistingHqlTransformer::createTransformInfo(IHqlExpression * expr)
+bool ConditionalHqlTransformer::analyseThis(IHqlExpression * expr)
 {
-    return CREATE_NEWTRANSFORMINFO(HoistingTransformInfo, expr);
-}
-
-void HoistingHqlTransformer::transformArray(const HqlExprArray & in, HqlExprArray & out)
-{
-    HqlExprArray * savedTarget = target;
-    target = &out;
-    ForEachItemIn(idx, in)
-        out.append(*transform(&in.item(idx)));
-    target = savedTarget;
-}
-
-IHqlExpression * HoistingHqlTransformer::transformRoot(IHqlExpression * expr)
-{
-    HqlExprArray args, transformed;
-    unwindCommaCompound(args, expr);
-    transformRoot(args, transformed);
-    return createActionList(transformed);
-}
-
-
-bool HoistingHqlTransformer::analyseThis(IHqlExpression * expr)
-{
-    HoistingTransformInfo * extra = queryBodyExtra(expr);
-    if (alreadyVisited(expr->queryBody()))
+    if (pass == 0)
     {
-        if ((pass != 0) || extra->isUnconditional() || (conditionDepth > 0))
-            return false;
+        ConditionalTransformInfo * extra = queryBodyExtra(expr);
+        if (alreadyVisited(extra))
+        {
+            if (extra->isUnconditional() || (conditionDepth > 0))
+                return false;
+            extra->setUnconditional();
+        }
+        else
+        {
+            if (conditionDepth == 0)
+                extra->setFirstUnconditional();
+        }
+        return true;
     }
-    if (conditionDepth == 0)
-        extra->setUnconditional();
-    return true;
+    return !alreadyVisited(expr);
 }
 
 
-void HoistingHqlTransformer::analyseExpr(IHqlExpression * expr)
+void ConditionalHqlTransformer::analyseExpr(IHqlExpression * expr)
 {
     if (!analyseThis(expr))
         return;
@@ -2175,7 +2142,7 @@ void HoistingHqlTransformer::analyseExpr(IHqlExpression * expr)
 }
 
 
-void HoistingHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
+void ConditionalHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
 {
     node_operator op = expr->getOperator();
     switch (op)
@@ -2190,7 +2157,7 @@ void HoistingHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
         return;
     case no_allnodes:
     case no_thisnode:
-        if (!(flags & HTFtraverseallnodes))
+        if (!(flags & CTFtraverseallnodes))
             return;
         break;
     case no_globalscope:
@@ -2204,13 +2171,31 @@ void HoistingHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
         }
         break;
     case no_if:
-        if (checkConditional(expr))
+    case no_and:
+    case no_or:
+    case no_mapto:
+    case no_map:
+    case no_which:
+    case no_rejected:
+    case no_choose:
+        if (treatAsConditional(expr))
         {
             analyseExpr(expr->queryChild(0));
             conditionDepth++;
+            ForEachChildFrom(idx, expr, 1)
+                analyseExpr(expr->queryChild(idx));
+            conditionDepth--;
+            return;
+        }
+        break;
+    case no_case:
+        if (treatAsConditional(expr))
+        {
+            analyseExpr(expr->queryChild(0));
             analyseExpr(expr->queryChild(1));
-            if (expr->queryChild(2))
-                analyseExpr(expr->queryChild(2));
+            conditionDepth++;
+            ForEachChildFrom(idx, expr, 2)
+                analyseExpr(expr->queryChild(idx));
             conditionDepth--;
             return;
         }
@@ -2222,6 +2207,83 @@ void HoistingHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
     NewHqlTransformer::analyseExpr(expr);
 }
 
+bool ConditionalHqlTransformer::treatAsConditional(IHqlExpression * expr)
+{
+    switch (expr->getOperator())
+    {
+    case no_if:
+        return ((flags & CTFnoteifall) ||
+            ((flags & CTFnoteifactions) && expr->isAction()) ||
+            ((flags & CTFnoteifdatasets) && expr->isDataset()) ||
+            ((flags & CTFnoteifdatarows) && expr->isDatarow()));
+    case no_or:
+        return (flags & CTFnoteor) != 0;
+    case no_and:
+        return (flags & CTFnoteor) != 0;
+    case no_case:
+    case no_map:
+    case no_mapto:
+        return (flags & CTFnotemap) != 0;
+    case no_which:
+    case no_rejected:
+    case no_choose:
+        return (flags & CTFnotewhich) != 0;
+    }
+    return false;
+}
+
+
+//---------------------------------------------------------------------------
+
+HoistingHqlTransformer::HoistingHqlTransformer(HqlTransformerInfo & _info, unsigned _flags) : ConditionalHqlTransformer(_info, _flags)
+{
+    target = NULL;
+}
+
+void HoistingHqlTransformer::setParent(const HoistingHqlTransformer * parent)
+{
+    assertex(parent->independentCache);
+    independentCache.set(parent->independentCache);
+}
+
+void HoistingHqlTransformer::transformRoot(const HqlExprArray & in, HqlExprArray & out)
+{
+    HqlExprArray * savedTarget = target;
+    target = &out;
+
+    //If there is a single ensureresult, then transform it specially, so that any hoisted values are
+    //only inside the ensure result.  If multiple assume we want to create cses globally (e.g., shared stored)
+    if ((in.ordinality() == 1) && (in.item(0).getOperator() == no_ensureresult))
+        out.append(*transformEnsureResult(&in.item(0)));
+    else
+        NewHqlTransformer::transformRoot(in, out);
+    target = savedTarget;
+}
+
+
+void HoistingHqlTransformer::appendToTarget(IHqlExpression & cur)
+{
+    target->append(cur);
+}
+
+void HoistingHqlTransformer::transformArray(const HqlExprArray & in, HqlExprArray & out)
+{
+    HqlExprArray * savedTarget = target;
+    target = &out;
+    ForEachItemIn(idx, in)
+        out.append(*transform(&in.item(idx)));
+    target = savedTarget;
+}
+
+IHqlExpression * HoistingHqlTransformer::transformRoot(IHqlExpression * expr)
+{
+    HqlExprArray args, transformed;
+    unwindCommaCompound(args, expr);
+    transformRoot(args, transformed);
+    return createActionList(transformed);
+}
+
+
 //A good idea, but I need to get my head around roxie/thor differences and see if we can execute graphs more dynamically.
 IHqlExpression * HoistingHqlTransformer::createTransformed(IHqlExpression * expr)
 {
@@ -2235,7 +2297,7 @@ IHqlExpression * HoistingHqlTransformer::createTransformed(IHqlExpression * expr
     case no_allnodes:               // MORE: This needs really needs to recurse, and substitute within no_thisnode - not quite sure how that would happen
     case no_thisnode:
         //I'm not sure this is a good solution - really contents of no_thisnode should be hoisted in common, would require dependence on insideAllNodes
-        if (!(flags & HTFtraverseallnodes))
+        if (!(flags & CTFtraverseallnodes))
             return LINK(expr);
         break;
     case no_cluster:
@@ -2276,7 +2338,7 @@ IHqlExpression * HoistingHqlTransformer::createTransformed(IHqlExpression * expr
             return completeTransform(expr, transformedArgs);
         }
     case no_if:
-        if (checkConditional(expr))
+        if (treatAsConditional(expr))
         {
             HqlExprArray args;
             args.append(*transform(expr->queryChild(0)));

ecl/hql/hqltrans.ipp

@@ -444,6 +444,7 @@ public:
     IHqlExpression * transformRoot(IHqlExpression * expr) { return doTransformRootExpr(expr); }
 
 protected:
+            bool alreadyVisited(ANewTransformInfo * extra);
             bool alreadyVisited(IHqlExpression * pass);
     virtual void analyseExpr(IHqlExpression * expr);
     virtual void analyseSelector(IHqlExpression * expr);
@@ -585,13 +586,19 @@ This provides a mechanism for handling the code correctly, but also allowing the
 Flags provided to the constructor indicate which options for transforming are used.
 */
 
-class HoistingTransformInfo : public NewTransformInfo
+//---------------------------------------------------------------------------------------------------------------------
+
+class ConditionalTransformInfo : public NewTransformInfo
 {
+    enum { CTFunconditional = 1, CTFfirstunconditional = 2 };
 public:
-    HoistingTransformInfo(IHqlExpression * _original) : NewTransformInfo(_original) { spareByte1 = false; }
+    ConditionalTransformInfo(IHqlExpression * _original) : NewTransformInfo(_original) { spareByte1 = 0; }
     
-    inline bool isUnconditional() { return spareByte1 != 0; }
-    inline void setUnconditional() { spareByte1 = true; }
+    inline bool isUnconditional() const { return (spareByte1 & CTFunconditional) != 0; }
+    inline bool isFirstUseUnconditional() const { return (spareByte1 & CTFfirstunconditional) != 0; }
+
+    inline void setUnconditional() { spareByte1 |= CTFunconditional; }
+    inline void setFirstUnconditional() { spareByte1 |= (CTFfirstunconditional|CTFunconditional); }
 
 private:
     using NewTransformInfo::spareByte1;             //prevent derived classes from also using this spare byte
@@ -600,7 +607,51 @@ private:
 //This allows expressions to be evaluated in a nested context, so that once that nested context is finished
 //all the mapped expressions relating to that nested context are removed, and not commoned up.
 //Useful for processing 
-class HQL_API HoistingHqlTransformer : public NewHqlTransformer
+class HQL_API ConditionalHqlTransformer : public NewHqlTransformer
+{
+public:
+    enum { CTFnoteifactions     = 0x0001,
+           CTFnoteifdatasets    = 0x0002,
+           CTFnoteifdatarows    = 0x0004,
+           CTFnoteifall         = 0x0008,
+           CTFnoteor            = 0x0010,
+           CTFnoteand           = 0x0020,
+           CTFnotemap           = 0x0040,
+           CTFnotewhich         = 0x0080,
+           CTFnoteall           = 0xFFFF,
+           CTFtraverseallnodes  = 0x10000,
+    };
+    ConditionalHqlTransformer(HqlTransformerInfo & _info, unsigned _flags);
+
+    void setFlags(unsigned _flags) { flags = _flags; }
+
+protected:
+    bool analyseThis(IHqlExpression * expr);
+
+    virtual void doAnalyseExpr(IHqlExpression * expr);
+
+    virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr);
+    inline ConditionalTransformInfo * queryBodyExtra(IHqlExpression * expr)     { return static_cast<ConditionalTransformInfo *>(queryTransformExtra(expr->queryBody())); }
+    inline bool isUsedUnconditionally(IHqlExpression * expr)                { return queryBodyExtra(expr)->isUnconditional(); }
+
+    virtual void analyseExpr(IHqlExpression * expr);
+
+    inline bool treatAsConditional(IHqlExpression * expr);
+
+protected:
+    unsigned conditionDepth;
+    unsigned flags;
+    bool containsUnknownIndependentContents;
+};
+
+
+//---------------------------------------------------------------------------------------------------------------------
+
+typedef ConditionalTransformInfo HoistingTransformInfo;
+
+//This allows expressions to be evaluated in a nested context, so that once that nested context is finished
+//all the mapped expressions relating to that nested context are removed, and not commoned up.
+class HQL_API HoistingHqlTransformer : public ConditionalHqlTransformer
 {
     class IndependentTransformMap : public CInterface
     {
@@ -611,30 +662,13 @@ class HQL_API HoistingHqlTransformer : public NewHqlTransformer
         HqlExprArray cache;
     };
 public:
-    enum { HTFnoteconditionalactions = 0x01, 
-           HTFnoteconditionaldatasets= 0x02,
-           HTFnoteconditionaldatarows= 0x04,
-           HTFtraverseallnodes = 0x08,
-           HTFnoteallconditionals = 0x10,
-    };
     HoistingHqlTransformer(HqlTransformerInfo & _info, unsigned _flags);
 
     void appendToTarget(IHqlExpression & curOwned);
-    void setFlags(unsigned _flags) { flags = _flags; }
-    void setParent(const HoistingHqlTransformer * parent);
     void transformRoot(const HqlExprArray & in, HqlExprArray & out);
     IHqlExpression * transformRoot(IHqlExpression * expr);
 
 protected:
-    bool analyseThis(IHqlExpression * expr);
-
-    virtual void doAnalyseExpr(IHqlExpression * expr);
-
-    virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr);
-    inline HoistingTransformInfo * queryBodyExtra(IHqlExpression * expr)        { return static_cast<HoistingTransformInfo *>(queryTransformExtra(expr->queryBody())); }
-    inline bool isUsedUnconditionally(IHqlExpression * expr)                { return queryBodyExtra(expr)->isUnconditional(); }
-
-    virtual void analyseExpr(IHqlExpression * expr);
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
     IHqlExpression * transformIndependent(IHqlExpression * expr);
     virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr) = 0;
@@ -642,22 +676,11 @@ protected:
     void transformArray(const HqlExprArray & in, HqlExprArray & out);
     IHqlExpression * transformEnsureResult(IHqlExpression * expr);
 
-    inline bool checkConditional(IHqlExpression * expr)
-    {
-        return ((flags & HTFnoteallconditionals) ||
-            ((flags & HTFnoteconditionalactions) && expr->isAction()) ||
-            ((flags & HTFnoteconditionaldatasets) && expr->isDataset()) ||
-            ((flags & HTFnoteconditionaldatarows) && expr->isDatarow()));
-    }
+    void setParent(const HoistingHqlTransformer * parent);
 
 private:
     HqlExprArray *      target;
-    unsigned flags;
     Owned<IndependentTransformMap> independentCache;
-
-protected:
-    unsigned conditionDepth;
-    bool containsUnknownIndependentContents;
 };
 
 

ecl/hql/hqlutil.cpp

@@ -1158,6 +1158,7 @@ IHqlExpression * replaceChild(IHqlExpression * expr, unsigned childIndex, IHqlEx
 
 IHqlExpression * createIf(IHqlExpression * cond, IHqlExpression * left, IHqlExpression * right)
 {
+    assertex(right);
     if (left->isDataset() || right->isDataset())
         return createDataset(no_if, cond, createComma(left, right));
 
@@ -4165,6 +4166,17 @@ IHqlExpression * removeLocalAttribute(IHqlExpression * expr)
     return removeProperty(expr, localAtom);
 }
 
+bool hasOperand(IHqlExpression * expr, IHqlExpression * child)
+{
+    expr = expr->queryBody();
+    ForEachChild(i, expr)
+    {
+        if (expr->queryChild(i) == child)
+            return true;
+    }
+    return false;
+}
+
 //-------------------------------------------------------------------------------------------------------
 
 class HQL_API SplitDatasetAttributeTransformer : public NewHqlTransformer

ecl/hql/hqlutil.hpp

@@ -151,6 +151,8 @@ extern HQL_API IHqlExpression * appendOwnedOperand(IHqlExpression * expr, IHqlEx
 extern HQL_API IHqlExpression * replaceOwnedProperty(IHqlExpression * expr, IHqlExpression * ownedProeprty);
 extern HQL_API IHqlExpression * appendOwnedOperandsF(IHqlExpression * expr, ...);
 extern HQL_API IHqlExpression * inheritAttribute(IHqlExpression * expr, IHqlExpression * donor, _ATOM name);
+extern HQL_API bool hasOperand(IHqlExpression * expr, IHqlExpression * child);
+
 extern HQL_API unsigned numRealChildren(IHqlExpression * expr);
 
 extern HQL_API IHqlExpression * createEvaluateOutputModule(HqlLookupContext & ctx, IHqlExpression * scopeExpr, IHqlExpression * ifaceExpr, bool expandCallsWhenBound, node_operator outputOp);

ecl/hqlcpp/CMakeLists.txt

@@ -41,6 +41,7 @@ set (    SRCS
          hqlcset.cpp 
          hqlecl.cpp 
          hqlgraph.cpp 
+         hqlhoist.cpp
          hqlhtcpp.cpp 
          hqlinline.cpp 
          hqliproj.cpp 
@@ -82,6 +83,7 @@ set (    SRCS
          hqlcset.ipp
          hqlfunc.ipp
          hqlgraph.ipp
+         hqlhoist.hpp
          hqlhtcpp.ipp
          hqliproj.ipp
          hqliter.ipp

ecl/hqlcpp/hqlcatom.cpp

@@ -501,6 +501,7 @@ _ATOM reportFieldOverflowAtom;
 _ATOM reportRowOverflowAtom;
 _ATOM responseinfoAtom;
 _ATOM restoreClusterAtom;
+_ATOM resultsAtom;
 _ATOM returnAtom;
 _ATOM returnPersistVersionAtom;
 _ATOM reverseIntAtom[9][2];
@@ -1218,6 +1219,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(reportRowOverflow);
     MAKEATOM(responseinfo);
     MAKEATOM(restoreCluster);
+    MAKEATOM(results);
     MAKEATOM(return);
     MAKEATOM(returnPersistVersion);
 

ecl/hqlcpp/hqlcatom.hpp

@@ -501,6 +501,7 @@ extern _ATOM reportFieldOverflowAtom;
 extern _ATOM reportRowOverflowAtom;
 extern _ATOM responseinfoAtom;
 extern _ATOM restoreClusterAtom;
+extern _ATOM resultsAtom;
 extern _ATOM returnAtom;
 extern _ATOM returnPersistVersionAtom;
 extern _ATOM reverseIntAtom[9][2];

ecl/hqlcpp/hqlcpp.cpp

@@ -873,10 +873,11 @@ ExpressionFormat queryNaturalFormat(ITypeInfo * type)
 
 //===========================================================================
 
-SubGraphInfo::SubGraphInfo(IPropertyTree * _tree, unsigned _id, IHqlExpression * _graphTag, SubGraphType _type) 
-    : HqlExprAssociation(subGraphMarker), tree(_tree) 
+SubGraphInfo::SubGraphInfo(IPropertyTree * _tree, unsigned _id, unsigned _graphId, IHqlExpression * _graphTag, SubGraphType _type)
+    : HqlExprAssociation(subGraphMarker), tree(_tree)
 { 
-    id = _id; 
+    id = _id;
+    graphId = _graphId;
     type = _type; 
     graphTag.set(_graphTag);
 }
@@ -1486,7 +1487,6 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.checkRowOverflow,"checkRowOverflow", true),
         DebugOption(options.freezePersists,"freezePersists", false),
         DebugOption(options.maxRecordSize, "defaultMaxLengthRecord", MAX_RECORD_SIZE),
-        DebugOption(options.maxInlineDepth, "maxInlineDepth", 999),         // a debugging setting...
 
         DebugOption(options.checkRoxieRestrictions,"checkRoxieRestrictions", true),     // a debug aid for running regression suite
         DebugOption(options.checkThorRestrictions,"checkThorRestrictions", true),       // a debug aid for running regression suite
@@ -2721,6 +2721,13 @@ void HqlCppTranslator::buildFilter(BuildCtx & ctx, IHqlExpression * expr)
             buildFilter(ctx, expr->queryChild(0));
             return;
         }
+    case no_compound:
+        {
+            buildStmt(ctx, expr->queryChild(0));
+            buildFilter(ctx, expr->queryChild(1));
+            break;
+        }
+
     }
     buildFilterViaExpr(ctx, expr);
 }
@@ -3295,6 +3302,8 @@ bool HqlCppTranslator::specialCaseBoolReturn(BuildCtx & ctx, IHqlExpression * ex
         return false;
     if (expr->getOperator() == no_alias_scope)
         expr = expr->queryChild(0);
+    if (expr->getOperator() == no_compound)
+        expr = expr->queryChild(1);
     if ((expr->getOperator() == no_and) || (expr->getOperator() == no_or))
         return true;
     return false;
@@ -3575,11 +3584,14 @@ void HqlCppTranslator::buildStmt(BuildCtx & _ctx, IHqlExpression * expr)
     case no_persist_check:
         buildWorkflowPersistCheck(ctx, expr);
         return;
+    case no_childquery:
+        buildChildGraph(ctx, expr);
+        return;
     case no_evaluate_stmt:
         expr = expr->queryChild(0);
         if (expr->queryValue())
             return;
-        // fall through to default behaviour.
+        break; // evaluate default behaviour.
     }
     CHqlBoundExpr tgt;
     buildAnyExpr(ctx, expr, tgt);
@@ -4371,7 +4383,8 @@ void HqlCppTranslator::doBuildExprAlias(BuildCtx & ctx, IHqlExpression * expr, C
 {
     //MORE These will be declared in a different context later.
     IHqlExpression * value = expr->queryChild(0);
-    assertex(value->getOperator() != no_alias);
+    while (value->getOperator() == no_alias)
+        value = value->queryChild(0);
 
     //The second half of this test could cause aliases to be duplicated, but has the significant effect of reducing the amount of data that is serialised.
     //so far on my examples it does the latter, but doesn't seem to cause the former

ecl/hqlcpp/hqlcpp.ipp

@@ -485,13 +485,14 @@ enum SubGraphType { SubGraphRoot, SubGraphRemote, SubGraphChild };
 struct SubGraphInfo : public HqlExprAssociation
 {
 public:
-    SubGraphInfo(IPropertyTree * _tree, unsigned _id, IHqlExpression * graphTag, SubGraphType _type);
+    SubGraphInfo(IPropertyTree * _tree, unsigned _id, unsigned _graphId, IHqlExpression * graphTag, SubGraphType _type);
 
     virtual AssocKind getKind() { return AssocSubGraph; }
 
 public:
     Linked<IPropertyTree> tree;
     unsigned id;
+    unsigned graphId;
     LinkedHqlExpr graphTag;
     SubGraphType type;
 };
@@ -524,7 +525,6 @@ struct HqlCppOptions
     unsigned            optimizeDiskFlag;
     unsigned            activitiesPerCpp;
     unsigned            maxRecordSize;
-    unsigned            maxInlineDepth;
     unsigned            inlineStringThreshold;
     unsigned            maxRootMaybeThorActions;
     unsigned            maxStaticRowSize;
@@ -1216,8 +1216,9 @@ public:
     void doBuildReturnCompare(BuildCtx & ctx, IHqlExpression * expr, node_operator op, bool isBoolEquality);
     void buildReturnOrder(BuildCtx & ctx, IHqlExpression *sortList, const DatasetReference & dataset);
 
-    unique_id_t buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, unique_id_t containerId, bool multiInstance);
-    unique_id_t buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset, unique_id_t containerId);
+    IHqlExpression * createLoopSubquery(IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * filter, IHqlExpression * again, IHqlExpression * counter, bool multiInstance, unsigned & loopAgainResult);
+    unique_id_t buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, bool multiInstance);
+    unique_id_t buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset);
         
     void doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt, IHqlExpression * expr, CHqlBoundExpr * result);
     IHqlExpression * doBuildInternalFunction(IHqlExpression * funcdef);
@@ -1319,6 +1320,8 @@ public:
     void doBuildStmtUpdate(BuildCtx & ctx, IHqlExpression * expr);
     void doBuildStmtWait(BuildCtx & ctx, IHqlExpression * expr);
 
+    void optimizeBuildActionList(BuildCtx & ctx, IHqlExpression * exprs);
+
     bool buildNWayInputs(CIArrayOf<ABoundActivity> & inputs, BuildCtx & ctx, IHqlExpression * input);
 
 //Activities.   
@@ -1633,6 +1636,7 @@ public:
     bool insideChildQuery(BuildCtx & ctx);
     bool insideRemoteGraph(BuildCtx & ctx);
     bool isCurrentActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag);
+    void buildChildGraph(BuildCtx & ctx, IHqlExpression * expr);
 
     void buildXmlReadChildrenIterator(BuildCtx & subctx, const char * iterTag, IHqlExpression * rowName, SharedHqlExpr & subRowExpr);
     void buildXmlReadTransform(IHqlExpression * dataset, StringBuffer & className, bool & usesContents);
@@ -1782,7 +1786,7 @@ protected:
 
     void logGraphEdge(IPropertyTree * subGraph, unsigned __int64 source, unsigned __int64 target, unsigned sourceIndex, unsigned targetIndex, const char * label, bool nWay);
 
-    void beginGraph(BuildCtx & ctx, const char * graphName = NULL);
+    void beginGraph(const char * graphName = NULL);
     void clearGraph();
     void endGraph();
 
@@ -1907,39 +1911,61 @@ protected:
     unique_id_t         instance;
 };
 
+//---------------------------------------------------------------------------------------------------------------------
+
+class ChildGraphExprBuilder : public CInterface
+{
+public:
+    ChildGraphExprBuilder(unsigned _numInputs);
+
+    IHqlExpression * addDataset(IHqlExpression * expr);
+    void addAction(IHqlExpression * expr);
+    unsigned addInput();
+    IHqlExpression * getGraph();
+
+    inline IHqlExpression * queryRepresents() const { return represents; }
+    inline unsigned numResults() const { return numInputs + numOutputs; }
+
+public:
+    HqlExprArray results;
+    OwnedHqlExpr represents;
+    OwnedHqlExpr resultsExpr;
+    unsigned numInputs;
+    unsigned numOutputs;
+};
+
 
 //===========================================================================
 
 class ChildGraphBuilder : public CInterface
 {
 public:
-    ChildGraphBuilder(HqlCppTranslator & _translator);
+    ChildGraphBuilder(HqlCppTranslator & _translator, IHqlExpression * subgraph);
 
-    IHqlExpression * addDataset(IHqlExpression * expr);
-    void buildStmt(BuildCtx & ctx, IHqlExpression * expr);
-    unique_id_t buildGraphLoopBody(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, unique_id_t containerId, bool multiInstance);
-    unique_id_t buildLoopBody(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * filter, IHqlExpression * again, IHqlExpression * counter, unique_id_t containerId, bool multiInstance);
-    unique_id_t buildRemoteGraph(BuildCtx & ctx, IHqlExpression * ds, unique_id_t containerId);
+    unique_id_t buildGraphLoopBody(BuildCtx & ctx, bool multiInstance);
+    unique_id_t buildLoopBody(BuildCtx & ctx, bool multiInstance);
+    unique_id_t buildRemoteGraph(BuildCtx & ctx);
     void generateGraph(BuildCtx & ctx);
-    void generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * retGraphExpr, OwnedHqlExpr * retResultsExpr);
-    bool isDatasetPresent(IHqlExpression * expr);
-    unsigned queryLoopConditionResult() const { return loopAgainResult; }
+    void generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * retGraphExpr);
+
 protected:
     void createBuilderAlias(BuildCtx & ctx, ParentExtract * extractBuilder);
 
 protected:
     HqlCppTranslator & translator;
     unsigned id;
-    OwnedHqlExpr idExpr;
     StringBuffer instanceName;
     OwnedHqlExpr instanceExpr;
     OwnedHqlExpr resultInstanceExpr;
     OwnedHqlExpr represents;
-    unsigned loopAgainResult;
+    OwnedHqlExpr resultsExpr;
     HqlExprArray results;
     unsigned numResults;
 };
 
+//move to a better header file
+IHqlExpression * createCounterAsResult(IHqlExpression * counter, IHqlExpression * represents, unsigned seq);
+
 //===========================================================================
 
 class CompoundBuilder
@@ -2019,9 +2045,12 @@ extern bool isGraphIndependent(IHqlExpression * expr, IHqlExpression * graph);
 extern IHqlExpression * adjustBoundIntegerValues(IHqlExpression * left, IHqlExpression * right, bool subtract);
 extern bool isNullAssign(const CHqlBoundTarget & target, IHqlExpression * expr);
 
+SubGraphInfo * matchActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag);
+bool isActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag);
 inline SubGraphInfo * queryActiveSubGraph(BuildCtx & ctx) 
 { 
     return static_cast<SubGraphInfo *>(ctx.queryFirstAssociation(AssocSubGraph));
 }
+void addGraphIdAttribute(ActivityInstance * instance, BuildCtx & ctx, IHqlExpression * graphId);
 
 #endif

ecl/hqlcpp/hqlcppds.cpp

@@ -53,6 +53,13 @@
 #define MAX_FIXED_SIZE_RAW 1024
 #define INLINE_TABLE_EXPAND_LIMIT 4
 
+void addGraphIdAttribute(ActivityInstance * instance, BuildCtx & ctx, IHqlExpression * graphId)
+{
+    SubGraphInfo * match = matchActiveGraph(ctx, graphId);
+    assertex(match);
+    instance->addAttributeInt("_graphId", match->graphId);
+}
+
 //===========================================================================
 
 
@@ -1145,54 +1152,92 @@ bool isGraphIndependent(IHqlExpression * expr, IHqlExpression * graph)
     return checker.isIndependent();
 }
 
-//---------------------------------------------------------------------------
-// Child dataset processing
+///--------------------------------------------------------------------------------------------------------------------
 
-static IHqlExpression * convertScalarToResult(IHqlExpression * value, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq)
+ChildGraphExprBuilder::ChildGraphExprBuilder(unsigned _numInputs)
+: numInputs(_numInputs)
 {
-    OwnedHqlExpr row = convertScalarToRow(value, fieldType);
-    OwnedHqlExpr ds = createDatasetFromRow(LINK(row));
-    HqlExprArray args;
-    args.append(*LINK(ds));
-    args.append(*LINK(represents));
-    args.append(*getSizetConstant(seq));
-    args.append(*createAttribute(rowAtom));
-    return createValue(no_setgraphresult, makeVoidType(), args);
+    numOutputs=0;
+    represents.setown(createAttribute(graphAtom, createUniqueId()));
+    resultsExpr.setown(createAttribute(resultsAtom, LINK(represents)));
 }
 
-static IHqlExpression * createScalarFromResult(ITypeInfo * scalarType, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq)
+IHqlExpression * ChildGraphExprBuilder::addDataset(IHqlExpression * expr)
 {
-    OwnedHqlExpr counterField = createField(unnamedAtom, LINK(fieldType), NULL, NULL);
-    OwnedHqlExpr counterRecord = createRecord(counterField);
+    OwnedHqlExpr resultNumExpr;
+    ForEachItemIn(i, results)
+    {
+        IHqlExpression & curSetResult = results.item(i);
+        if (expr->queryBody() == curSetResult.queryChild(0)->queryBody())
+        {
+            resultNumExpr.set(curSetResult.queryChild(2));
+            break;
+        }
+    }
+
+    if (!resultNumExpr)
+    {
+        resultNumExpr.setown(getSizetConstant(numResults()));
+        results.append(*createValue(no_setgraphresult, makeVoidType(), LINK(expr), LINK(represents), LINK(resultNumExpr)));
+        numOutputs++;
+    }
+
     HqlExprArray args;
-    args.append(*LINK(counterRecord));
+    args.append(*LINK(expr->queryRecord()));
     args.append(*LINK(represents));
-    args.append(*getSizetConstant(seq));
-    args.append(*createAttribute(rowAtom));
-    OwnedHqlExpr counterResult = createDataset(no_getgraphresult, args);
-    OwnedHqlExpr select = createNewSelectExpr(createRow(no_selectnth, LINK(counterResult), getSizetConstant(1)), LINK(counterField));
-    OwnedHqlExpr cast = ensureExprType(select, scalarType);
-    return createAlias(cast, internalAttrExpr);
+    args.append(*LINK(resultNumExpr));
+    if (isGrouped(expr))
+        args.append(*createAttribute(groupedAtom));
+    if (!expr->isDataset())
+        args.append(*createAttribute(rowAtom));
+    args.append(*createAttribute(externalAtom, LINK(resultsExpr)));
+    args.append(*createAttribute(_original_Atom, LINK(expr)));
+    OwnedHqlExpr ret = createDataset(no_getgraphresult, args);
+    if (expr->isDatarow())
+        ret.setown(createRow(no_selectnth, LINK(ret), createComma(getSizetConstant(1), createAttribute(noBoundCheckAtom))));
+    return ret.getClear();
 }
 
-static IHqlExpression * createCounterAsResult(IHqlExpression * counter, IHqlExpression * represents, unsigned seq)
+void ChildGraphExprBuilder::addAction(IHqlExpression * expr)
 {
-    return createScalarFromResult(counter->queryType(), unsignedType, represents, seq);
+    results.append(*LINK(expr));
 }
 
-ChildGraphBuilder::ChildGraphBuilder(HqlCppTranslator & _translator) 
+unsigned ChildGraphExprBuilder::addInput()
+{
+    unsigned id = numResults();
+    numInputs++;
+    return id;
+}
+
+IHqlExpression * ChildGraphExprBuilder::getGraph()
+{
+    HqlExprArray args;
+    args.append(*LINK(represents));
+    args.append(*getSizetConstant(numResults()));
+    args.append(*createActionList(results));
+    return createValue(no_childquery, makeVoidType(), args);
+}
+
+//---------------------------------------------------------------------------
+// Child dataset processing
+
+ChildGraphBuilder::ChildGraphBuilder(HqlCppTranslator & _translator, IHqlExpression * subgraph)
 : translator(_translator)
 {
+    represents.set(subgraph->queryChild(0));
     id = translator.nextActivityId();
-    idExpr.setown(createConstant((__int64)id));
+
     appendUniqueId(instanceName.append("child"), id);
     instanceExpr.setown(createQuoted(instanceName, makeBoolType()));
-    represents.setown(createAttribute(graphAtom, LINK(idExpr)));
-    numResults = 0;
-    loopAgainResult = 0;
+    resultsExpr.setown(createAttribute(resultsAtom, LINK(represents)));
 
     StringBuffer s;
     resultInstanceExpr.setown(createQuoted(appendUniqueId(s.append("res"), id), makeBoolType()));
+    numResults = (unsigned)getIntValue(subgraph->queryChild(1));
+
+    IHqlExpression * actions = subgraph->queryChild(2);
+    actions->unwindList(results, no_actionlist);
 }
 
 void ChildGraphBuilder::generateGraph(BuildCtx & ctx)
@@ -1241,10 +1286,10 @@ void ChildGraphBuilder::generateGraph(BuildCtx & ctx)
     graphctx.addQuoted(s);
 
     translator.endExtract(graphctx, extractBuilder);
-    ctx.associateExpr(resultInstanceExpr, resultInstanceExpr);
+    ctx.associateExpr(resultsExpr, resultInstanceExpr);
 }
 
-void ChildGraphBuilder::generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * retGraphExpr, OwnedHqlExpr * retResultsExpr)
+void ChildGraphBuilder::generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * retGraphExpr)
 {
     BuildCtx ctx(_ctx);
     ctx.addGroup();
@@ -1268,59 +1313,6 @@ void ChildGraphBuilder::generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * re
     assertex(retInstanceExpr == instanceExpr);
 
     retGraphExpr->setown(retInstanceExpr.getClear());
-    retResultsExpr->set(resultInstanceExpr);
-}
-
-void ChildGraphBuilder::buildStmt(BuildCtx & ctx, IHqlExpression * expr)
-{
-    results.append(*LINK(expr));
-}
-
-bool ChildGraphBuilder::isDatasetPresent(IHqlExpression * expr)
-{
-    ForEachItemIn(i, results)
-    {
-        IHqlExpression & curSetResult = results.item(i);
-        if (expr->queryBody() == curSetResult.queryChild(0)->queryBody())
-            return true;
-    }
-    return false;
-}
-
-
-IHqlExpression * ChildGraphBuilder::addDataset(IHqlExpression * expr)
-{
-    OwnedHqlExpr resultNumExpr;
-    ForEachItemIn(i, results)
-    {
-        IHqlExpression & curSetResult = results.item(i);
-        if (expr->queryBody() == curSetResult.queryChild(0)->queryBody())
-        {
-            resultNumExpr.set(curSetResult.queryChild(2));
-            break;
-        }
-    }
-
-    if (!resultNumExpr)
-    {
-        resultNumExpr.setown(getSizetConstant(numResults++));
-        results.append(*createValue(no_setgraphresult, makeVoidType(), LINK(expr), LINK(represents), LINK(resultNumExpr)));
-    }
-
-    HqlExprArray args;
-    args.append(*LINK(expr->queryRecord()));
-    args.append(*LINK(represents));
-    args.append(*LINK(resultNumExpr));
-    if (isGrouped(expr))
-        args.append(*createAttribute(groupedAtom));
-    if (!expr->isDataset())
-        args.append(*createAttribute(rowAtom));
-    args.append(*createAttribute(externalAtom, LINK(resultInstanceExpr)));
-    args.append(*createAttribute(_original_Atom, LINK(expr)));          
-    OwnedHqlExpr ret = createDataset(no_getgraphresult, args);
-    if (expr->isDatarow())
-        ret.setown(createRow(no_selectnth, LINK(ret), createComma(getSizetConstant(1), createAttribute(noBoundCheckAtom))));
-    return ret.getClear();
 }
 
 void ChildGraphBuilder::createBuilderAlias(BuildCtx & ctx, ParentExtract * extractBuilder)
@@ -1332,70 +1324,13 @@ void ChildGraphBuilder::createBuilderAlias(BuildCtx & ctx, ParentExtract * extra
     ctx.addQuoted(s);
 }
 
-unique_id_t ChildGraphBuilder::buildLoopBody(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * filter, IHqlExpression * again, IHqlExpression * counter, unique_id_t containerId, bool multiInstance)
+unique_id_t ChildGraphBuilder::buildLoopBody(BuildCtx & ctx, bool multiInstance)
 {
-    LinkedHqlExpr transformedBody = body;
-    LinkedHqlExpr transformedAgain = again;
-    numResults = 2;
-
-    //Result 1 is the input dataset.
-    HqlExprArray args;
-    args.append(*LINK(dataset->queryRecord()));
-    args.append(*LINK(represents));
-    args.append(*getSizetConstant(1));
-    if (isGrouped(dataset))
-        args.append(*createAttribute(groupedAtom));
-    args.append(*createAttribute(_loop_Atom));
-    if (multiInstance)
-        args.append(*createAttribute(_streaming_Atom));
-    if (translator.targetThor())    // MORE: && !isChildQuery(ctx)..
-        args.append(*createAttribute(_distributed_Atom));
-    OwnedHqlExpr inputResult= createDataset(no_getgraphresult, args);
-
-    //Result 2 is the counter - if present
-    OwnedHqlExpr counterResult;
-    if (counter)
-    {
-        OwnedHqlExpr select = createCounterAsResult(counter, represents, 2);
-        transformedBody.setown(replaceExpression(transformedBody, counter, select));
-        if (transformedAgain)
-        {
-            //The COUNTER for the global termination condition is whether to execute iteration COUNTER, 1=1st iter
-            //Since we're evaluating the condition in the previous iteration it needs to be increased by 1.
-            OwnedHqlExpr nextCounter = adjustValue(select, 1);
-            transformedAgain.setown(replaceExpression(transformedAgain, counter, nextCounter));
-        }
-        numResults = 3;
-    }
-
-    //first create the result...
-    //Need to replace ROWS(LEFT) with the result1
-    OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
-    OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(left), LINK(rowsid));
-    transformedBody.setown(replaceExpression(transformedBody, rowsExpr, inputResult));
-
-    OwnedHqlExpr result = createValue(no_setgraphresult, makeVoidType(), LINK(transformedBody), LINK(represents), getSizetConstant(0), createAttribute(_loop_Atom));
-
-    if (transformedAgain)
-    {
-        LinkedHqlExpr nextLoopDataset = transformedBody;
-        if (filter)
-        {
-            //If there is a loop filter then the global condition is applied to dataset filtered by that.
-            OwnedHqlExpr mappedFilter = replaceSelector(filter, left, nextLoopDataset);
-            nextLoopDataset.setown(createDataset(no_filter, nextLoopDataset.getClear(), LINK(mappedFilter)));
-        }
-        transformedAgain.setown(replaceExpression(transformedAgain, rowsExpr, nextLoopDataset));
-        OwnedHqlExpr againResult = convertScalarToResult(transformedAgain, queryBoolType(), represents, 3);
-        result.setown(createCompound(result.getClear(), againResult.getClear()));
-        loopAgainResult = 3;
-        numResults = 4;
-    }
-
     BuildCtx subctx(ctx);
     subctx.addGroup();
 
-    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, result, represents, numResults, no_loop);
+    OwnedHqlExpr query = createActionList(results);
+    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_loop);
     //Add a flag to indicate multi instance
     if (multiInstance)
         resourced.setown(appendOwnedOperand(resourced, createAttribute(multiInstanceAtom)));
@@ -1491,27 +1426,14 @@ protected:
 
 
 
-unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, unique_id_t containerId, bool isParallel)
+unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, bool isParallel)
 {
-    OwnedHqlExpr transformedBody;
-    OwnedHqlExpr counterResult;
-    {
-        GraphLoopReplacer replacer(rowsid, represents, counter, isParallel);
-        transformedBody.setown(replacer.transformRoot(body));
-        counterResult.set(replacer.queryCounterResult());
-    }
-
-    numResults = 0;
-    if (counterResult)
-        numResults++;
-
-    OwnedHqlExpr result = createValue(no_setgraphloopresult, makeVoidType(), LINK(transformedBody), LINK(represents));
-
     BuildCtx subctx(ctx);
     subctx.addGroup();
 
-    translator.traceExpression("Before Loop resource", result);
-    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, result, represents, numResults, no_loop);
+    OwnedHqlExpr query = createActionList(results);
+    translator.traceExpression("Before Loop resource", query);
+    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_loop);
     translator.traceExpression("After Loop resource", resourced);
 
     //Add a flag to indicate multi instance
@@ -1537,19 +1459,12 @@ unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, IHqlExpression
     return id;
 }
 
-unique_id_t ChildGraphBuilder::buildRemoteGraph(BuildCtx & ctx, IHqlExpression * ds, unique_id_t containerId)
+unique_id_t ChildGraphBuilder::buildRemoteGraph(BuildCtx & ctx)
 {
     BuildCtx subctx(ctx);
     subctx.addGroup();
 
-    OwnedHqlExpr query;
-    if (!ds->isAction())
-    {
-        OwnedHqlExpr result = addDataset(ds);
-        query.setown(createActionList(results));
-    }
-    else
-        query.set(ds);
+    OwnedHqlExpr query = createActionList(results);
     OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_allnodes);
 
     Owned<ParentExtract> extractBuilder = translator.createExtractBuilder(ctx, PETremote, represents, GraphRemote, false);
@@ -1564,6 +1479,18 @@ unique_id_t ChildGraphBuilder::buildRemoteGraph(BuildCtx & ctx, IHqlExpression *
 }
 
 
+void HqlCppTranslator::buildChildGraph(BuildCtx & ctx, IHqlExpression * expr)
+{
+    IHqlExpression * represents= expr->queryChild(0);
+    OwnedHqlExpr resultsExpr = createAttribute(resultsAtom, LINK(represents));
+    //Shouldn't really happen, but if this graph has already benn called just use the results
+    if (ctx.queryMatchExpr(resultsExpr))
+        return;
+
+    ChildGraphBuilder graphBuilder(*this, expr);
+    graphBuilder.generateGraph(ctx);
+}
+
 void HqlCppTranslator::beginExtract(BuildCtx & ctx, ParentExtract * extractBuilder)
 {
     ctx.associate(*extractBuilder);
@@ -1588,9 +1515,11 @@ void HqlCppTranslator::buildAssignChildDataset(BuildCtx & ctx, const CHqlBoundTa
 
     OwnedHqlExpr call;
     {
-        ChildGraphBuilder builder(*this);
+        ChildGraphExprBuilder builder(0);
         call.setown(builder.addDataset(expr));
-        builder.generateGraph(ctx);
+
+        OwnedHqlExpr subquery = builder.getGraph();
+        buildStmt(ctx, subquery);
     }
 
     buildExprAssign(ctx, target, call);
@@ -1679,6 +1608,7 @@ IHqlExpression * HqlCppTranslator::getResourcedChildGraph(BuildCtx & ctx, IHqlEx
     HqlExprArray children;
     resourced->unwindList(children, no_actionlist);
     children.append(*createAttribute(numResultsAtom, getSizetConstant(numResults)));
+    children.append(*LINK(graphIdExpr));
     resourced.setown(createValue(no_subgraph, makeVoidType(), children));
 
     if (options.paranoidCheckNormalized || options.paranoidCheckDependencies)
@@ -1709,16 +1639,46 @@ void HqlCppTranslator::buildChildDataset(BuildCtx & ctx, IHqlExpression * expr,
 }
 
 
-unique_id_t HqlCppTranslator::buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, unique_id_t containerId, bool multiInstance)
+unique_id_t HqlCppTranslator::buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, bool multiInstance)
 {
-    ChildGraphBuilder builder(*this);
-    return builder.buildGraphLoopBody(ctx, dataset, selSeq, rowsid, body, counter, containerId, multiInstance);
+    ChildGraphExprBuilder graphBuilder(0);
+
+    OwnedHqlExpr transformedBody;
+    OwnedHqlExpr counterResult;
+    IHqlExpression * graphid = graphBuilder.queryRepresents();
+    {
+        const bool isParallel = multiInstance;
+        GraphLoopReplacer replacer(rowsid, graphid, counter, isParallel);
+        transformedBody.setown(replacer.transformRoot(body));
+        counterResult.set(replacer.queryCounterResult());
+    }
+
+    if (counterResult)
+        graphBuilder.addInput();
+
+    OwnedHqlExpr result = createValue(no_setgraphloopresult, makeVoidType(), LINK(transformedBody), LINK(graphid));
+    graphBuilder.addAction(result);
+    OwnedHqlExpr subquery = graphBuilder.getGraph();
+
+    ChildGraphBuilder builder(*this, subquery);
+    return builder.buildGraphLoopBody(ctx, multiInstance);
 }
 
-unique_id_t HqlCppTranslator::buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset, unique_id_t containerId)
+unique_id_t HqlCppTranslator::buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset)
 {
-    ChildGraphBuilder builder(*this);
-    return builder.buildRemoteGraph(ctx, dataset, containerId);
+    ChildGraphExprBuilder graphBuilder(0);
+    if (dataset->isAction())
+    {
+        graphBuilder.addAction(dataset);
+    }
+    else
+    {
+        OwnedHqlExpr ignoredResult = graphBuilder.addDataset(dataset);
+    }
+
+    OwnedHqlExpr subquery = graphBuilder.getGraph();
+    ChildGraphBuilder builder(*this, subquery);
+    return builder.buildRemoteGraph(ctx);
 }
 
 //---------------------------------------------------------------------------
@@ -1756,7 +1716,7 @@ bool HqlCppTranslator::isInlineOk()
 {
     if (!activeGraphCtx)
         return true;
-    return (options.maxInlineDepth != 0);
+    return true;
 }
 
 IHqlExpression * HqlCppTranslator::buildSpillChildDataset(BuildCtx & ctx, IHqlExpression * expr)
@@ -1768,14 +1728,25 @@ IHqlExpression * HqlCppTranslator::buildSpillChildDataset(BuildCtx & ctx, IHqlEx
 
 IHqlExpression * HqlCppTranslator::forceInlineAssignDataset(BuildCtx & ctx, IHqlExpression * expr)
 {
-    CHqlBoundExpr bound;
-    if (expr->isPure() && ctx.getMatchExpr(expr, bound))
-        return bound.getTranslatedExpr();
+    loop
+    {
+        CHqlBoundExpr bound;
+        if (expr->isPure() && ctx.getMatchExpr(expr, bound))
+            return bound.getTranslatedExpr();
 
-    if (canProcessInline(&ctx, expr) || (expr->getOperator() == no_translated))
-        return LINK(expr);
+        if (canProcessInline(&ctx, expr) || (expr->getOperator() == no_translated))
+            return LINK(expr);
 
-    return buildSpillChildDataset(ctx, expr);
+        switch (expr->getOperator())
+        {
+        case no_compound:
+            buildStmt(ctx, expr->queryChild(0));
+            expr = expr->queryChild(1);
+            break;
+        default:
+            return buildSpillChildDataset(ctx, expr);
+        }
+    }
 }
 
 //---------------------------------------------------------------------------
@@ -2049,6 +2020,12 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
         if (doBuildDatasetInlineTable(ctx, expr, tgt, format))
             return;
         break;
+    case no_compound:
+        {
+            buildStmt(ctx, expr->queryChild(0));
+            buildDataset(ctx, expr->queryChild(1), tgt, format);
+            return;
+        }
     }
 
     bool singleRow = hasSingleRow(expr);
@@ -2219,6 +2196,12 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
             }
             break;
         }
+    case no_compound:
+        {
+            buildStmt(ctx, expr->queryChild(0));
+            buildDatasetAssign(ctx, target, expr->queryChild(1));
+            return;
+        }
     case no_compound_childread:
     case no_compound_childnormalize:
     case no_compound_childaggregate:
@@ -3736,6 +3719,7 @@ void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, IReferenceSelector * targe
                 return;
             }
         }
+        break;
         /*
     case no_externalcall:
         //MORE: Should assign directly to the target, but may not be very easy....
@@ -3748,6 +3732,11 @@ void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, IReferenceSelector * targe
         }
         break;
         */
+    case no_comma:
+    case no_compound:
+        buildStmt(ctx, expr->queryChild(0));
+        buildRowAssign(ctx, target, expr->queryChild(1));
+        return;
     }
 
     Owned<IReferenceSelector> src = buildNewRow(ctx, expr);
@@ -4235,7 +4224,8 @@ IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpre
     if (expr->hasProperty(externalAtom))
     {
         IHqlExpression * resultInstance = queryPropertyChild(expr, externalAtom, 0);
-        if (!ctx.queryMatchExpr(resultInstance))
+        HqlExprAssociation * matchedResults = ctx.queryMatchExpr(resultInstance);
+        if (!matchedResults)
         {
             //Very unusual - a result is required from a child query, but that child query is actually in
             //the parent/grandparent.  We need to evaluate in the parent instead.
@@ -4246,7 +4236,7 @@ IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpre
         }
 
         HqlExprArray args;
-        args.append(*LINK(resultInstance));
+        args.append(*LINK(matchedResults->queryExpr()));
         args.append(*LINK(resultNum));
         if (preferLinkedRows)
             return bindFunctionCall(getChildQueryLinkedResultAtom, args, exprType);
@@ -4255,7 +4245,10 @@ IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpre
 
     assertex(activeActivities.ordinality());
     queryAddResultDependancy(activeActivities.tos(), graphId, resultNum);
-    unique_id_t id = getIntValue(graphId->queryChild(0), -1);
+
+    SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
+    assertex(activeSubgraph && graphId == activeSubgraph->graphTag);
+    unique_id_t id = activeSubgraph->graphId;
 
     EvalContext * instance = queryEvalContext(ctx);
     OwnedHqlExpr retInstanceExpr;
@@ -4358,7 +4351,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGetGraphResult(BuildCtx & ctx,
     else
         instance->addConstructorParameter(resultNum);
 
-    instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+    addGraphIdAttribute(instance, ctx, graphId);
     buildInstanceSuffix(instance);
 
     queryAddResultDependancy(*instance->queryBoundActivity(), graphId, resultNum);
@@ -4423,7 +4416,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySetGraphResult(BuildCtx & ctx,
 
     instance->addAttributeBool("_isSpill", isSpill);
     if (targetRoxie())
-        instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+        addGraphIdAttribute(instance, ctx, graphId);
 
     buildInstanceSuffix(instance);
 
@@ -4498,7 +4491,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGetGraphLoopResult(BuildCtx &
 
     buildInstancePrefix(instance);
     doBuildUnsignedFunction(instance->startctx, "querySequence", resultNum);
-    instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+    addGraphIdAttribute(instance, ctx, graphId);
     buildInstanceSuffix(instance);
 
     return instance->getBoundActivity();
@@ -4525,7 +4518,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySetGraphLoopResult(BuildCtx &
     if (parentActivity && !insideRemoteGraph(ctx) && !isSpill)
         addDependency(ctx, instance->queryBoundActivity(), parentActivity, childAtom, "Body");
     if (targetRoxie())
-        instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+        addGraphIdAttribute(instance, ctx, graphId);
 
     buildInstanceSuffix(instance);
 
@@ -4608,4 +4601,3 @@ void HqlCppTranslator::doBuildStmtApply(BuildCtx & ctx, IHqlExpression * expr)
     if (end)
         buildStmt(ctx, end->queryChild(0));
 }
-

ecl/hqlcpp/hqlhoist.cpp

@@ -0,0 +1,796 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+#include "platform.h"
+#include "jlib.hpp"
+
+#include "hqlexpr.hpp"
+#include "hqlfold.hpp"
+#include "hqlhoist.hpp"
+#include "hqlutil.hpp"
+#include "hqlcpputil.hpp"
+
+bool canSurroundWithAlias(IHqlExpression * expr)
+{
+    switch (expr->getOperator())
+    {
+    case no_range:
+    case no_rangefrom:
+    case no_rangeto:
+    case no_rangecommon:
+    case no_mapto:
+    case no_recordlist:
+    case no_transformlist:
+    case no_rowvalue:
+    case no_sortlist:
+    case no_attr_expr:
+    case no_transform:
+    case no_newtransform:
+        return false;
+    case no_alias_scope:
+        return canSurroundWithAlias(expr->queryChild(0));
+    default:
+        return true;
+    }
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+static HqlTransformerInfo externalToInternalResultMapperInfo("ExternalToInternalResultMapper");
+class ExternalToInternalResultMapper : public NewHqlTransformer
+{
+public:
+    ExternalToInternalResultMapper(IHqlExpression * _graph) : NewHqlTransformer(externalToInternalResultMapperInfo), graph(_graph)
+    {
+    }
+
+protected:
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr)
+    {
+        OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
+        if (transformed->getOperator() == no_getgraphresult)
+        {
+            if (hasOperand(transformed, graph))
+                return removeProperty(transformed, externalAtom);
+        }
+        return transformed.getClear();
+    }
+
+protected:
+    IHqlExpression * graph;
+};
+
+IHqlExpression * mapExternalToInternalResults(IHqlExpression * expr, IHqlExpression * graph)
+{
+    ExternalToInternalResultMapper mapper(graph);
+    return mapper.transformRoot(expr);
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+void CHqlExprMultiGuard::addGuarded(IHqlExpression * original)
+{
+    addGuarded(queryBoolExpr(true), original, false);
+}
+
+void CHqlExprMultiGuard::addGuarded(IHqlExpression * cond, IHqlExpression * original, bool guardContainsCandidate)
+{
+    guarded.append(*new CHqlExprGuard(cond, original, guardContainsCandidate));
+}
+
+void CHqlExprMultiGuard::combine(CHqlExprMultiGuard & other)
+{
+    //Potentially O(N^2).  If there are vast numbers of candidates this could become problematic.
+    ForEachItemIn(i, other.guarded)
+    {
+        CHqlExprGuard & cur = other.guarded.item(i);
+        combine(cur);
+    }
+}
+
+void CHqlExprMultiGuard::combine(CHqlExprGuard & other)
+{
+    ForEachItemIn(i, guarded)
+    {
+        CHqlExprGuard & cur = guarded.item(i);
+        if (cur.original == other.original)
+        {
+            //condition is now (a || b)
+            OwnedHqlExpr newCond;
+            if (matchesBoolean(cur.guard, true) || matchesBoolean(other.guard, true))
+                newCond.set(queryBoolExpr(true));
+            else
+                newCond.setown(createBoolExpr(no_or, LINK(cur.guard), LINK(other.guard)));
+
+            //MORE: Could sometimes reuse the existing guard if this was created by this node
+            Owned<CHqlExprGuard> newGuard = new CHqlExprGuard(newCond, cur.original, cur.guardContainsCandidate||other.guardContainsCandidate);
+            guarded.replace(*newGuard.getClear(), i);
+            return;
+        }
+    }
+    guarded.append(OLINK(other));
+}
+
+void CHqlExprMultiGuard::gatherCandidates(HqlExprCopyArray & candidates) const
+{
+    ForEachItemIn(i, guarded)
+    {
+        CHqlExprGuard & cur = guarded.item(i);
+        if (!candidates.contains(*cur.original))
+            candidates.append(*cur.original);
+    }
+}
+
+IHqlExpression * CHqlExprMultiGuard::queryGuardCondition(IHqlExpression * original) const
+{
+    ForEachItemIn(i, guarded)
+    {
+        CHqlExprGuard & cur = guarded.item(i);
+        if (cur.original == original)
+            return cur.guard;
+    }
+    return NULL;
+}
+
+bool CHqlExprMultiGuard::guardContainsCandidate(IHqlExpression * original) const
+{
+    ForEachItemIn(i, guarded)
+    {
+        CHqlExprGuard & cur = guarded.item(i);
+        if (cur.original == original)
+            return cur.guardContainsCandidate;
+    }
+    return false;
+}
+
+void gatherCandidates(HqlExprCopyArray & candidates, CHqlExprMultiGuard * guards)
+{
+    if (guards)
+        guards->gatherCandidates(candidates);
+}
+
+
+IHqlExpression * queryGuardCondition(CHqlExprMultiGuard * guards, IHqlExpression * original)
+{
+    if (guards)
+    {
+        IHqlExpression * condition = guards->queryGuardCondition(original);
+        if (condition)
+            return condition;
+    }
+    return queryBoolExpr(false);
+}
+
+
+
+//---------------------------------------------------------------------------------------------------------------------
+
+void ConditionalContextInfo::calcInheritedGuards()
+{
+    if (guards && definitions.ordinality() != 0)
+    {
+        Owned<CHqlExprMultiGuard> newGuards = new CHqlExprMultiGuard;
+        ForEachItemIn(i, guards->guarded)
+        {
+            CHqlExprGuard & cur = guards->guarded.item(i);
+            if (!definitions.contains(*cur.original))
+                newGuards->guarded.append(OLINK(cur));
+        }
+        if (newGuards->guarded.ordinality())
+            inheritedGuards.setown(newGuards.getClear());
+    }
+    else
+        inheritedGuards.set(guards);
+}
+
+
+bool ConditionalContextInfo::isCandidateThatMoves() const
+{
+    if (!isCandidateExpr)
+        return false;
+
+    if (!changesLocation())
+        return false;
+
+    return true;
+}
+
+bool ConditionalContextInfo::usedOnMultiplePaths() const
+{
+    if (extraParents.ordinality() != 0)
+        return true;
+    if (firstParent)
+        return firstParent->usedOnMultiplePaths();
+    return false;
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+
+ConditionalContextTransformer::ConditionalContextTransformer(HqlTransformerInfo & info, bool _alwaysEvaluateGuardedTogether)
+: ConditionalHqlTransformer(info, CTFnoteor|CTFnoteand|CTFnotemap|CTFnoteifall),
+    alwaysEvaluateGuardedTogether(_alwaysEvaluateGuardedTogether)
+{
+    seq = 0;
+    hasConditionalCandidate = false;
+    noteManyUnconditionalParents = false;//true;
+    createRootGraph = false;
+    rootExpr.setown(createValue(no_null, makeVoidType(), createAttribute(_root_Atom)));
+    activeParent = NULL;  // cannot call queryBodyExtra(rootExpr) since in constructor
+}
+
+
+ANewTransformInfo * ConditionalContextTransformer::createTransformInfo(IHqlExpression * expr)
+{
+    return CREATE_NEWTRANSFORMINFO(ConditionalContextInfo, expr);
+}
+
+
+void ConditionalContextTransformer::analyseExpr(IHqlExpression * expr)
+{
+    switch (pass)
+    {
+    case PassFindConditions:
+        ConditionalHqlTransformer::analyseExpr(expr);
+        return;
+    case PassFindParents:
+        analyseConditionalParents(expr);
+        return;
+    case PassGatherGuards:
+        analyseGatherGuards(expr);
+        break;
+    default:
+        ConditionalHqlTransformer::analyseExpr(expr);
+        break;
+    }
+}
+
+bool ConditionalContextTransformer::hasSingleConditionalCandidate() const
+{
+    if (candidates.ordinality() != 1)
+        return false;
+    ConditionalContextInfo & candidate = candidates.item(0);
+    if (candidate.isUnconditional())
+        return false;
+    return !candidate.usedOnMultiplePaths();
+}
+
+
+// ---- pass 1 --------
+
+void ConditionalContextTransformer::noteCandidate(ConditionalContextInfo * extra)
+{
+    extra->isCandidateExpr = true;
+    //MORE: If is used unconditionally enough???? then mark as such.  (don't mark as unconditional)
+    if (!extra->isUnconditional())
+        hasConditionalCandidate = true;
+    candidates.append(*LINK(extra));
+}
+
+
+void ConditionalContextTransformer::analyseConditionalParents(IHqlExpression * expr)
+{
+    ConditionalContextInfo * extra = queryBodyExtra(expr);
+    if (extra->seq)
+    {
+        extra->addExtraParent(activeParent, noteManyUnconditionalParents);
+        return;
+    }
+
+    if (!extra->firstAnnotatedExpr && (expr != expr->queryBody()))
+        extra->firstAnnotatedExpr = expr;
+
+    extra->setFirstParent(activeParent);
+    extra->seq = ++seq;
+
+    {
+        ConditionalContextInfo * savedParent = activeParent;
+        activeParent = extra;
+        ConditionalHqlTransformer::analyseExpr(expr);
+        activeParent = savedParent;
+    }
+}
+
+// ---- pass 1b --------
+
+void ConditionalContextTransformer::addDefinition(ConditionalContextInfo * location, ConditionalContextInfo * candidate)
+{
+    location->definitions.append(*LINK(candidate->original));
+    candidate->moveTo = location;
+
+    if (!insertLocations.contains(*location))
+        insertLocations.append(*location);
+}
+
+void ConditionalContextTransformer::removeDefinition(ConditionalContextInfo * location, ConditionalContextInfo * candidate)
+{
+    location->definitions.zap(*LINK(candidate->original));
+    candidate->moveTo = NULL;
+}
+
+bool ConditionalContextTransformer::findCommonLocations()
+{
+    bool changed = false;
+    ForEachItemIn(idx, candidates)
+    {
+        ConditionalContextInfo& cur = candidates.item(idx);
+        ConditionalContextInfo * candidateLocation = findCandidateLocation(&cur);
+        //A null alias location forces an expression to stay where it is
+        if (candidateLocation)
+        {
+            addDefinition(candidateLocation, &cur);
+            changed = true;
+        }
+    }
+    return changed;
+}
+
+//Find a single location to evaluate all the child items in
+bool ConditionalContextTransformer::findSingleCommonLocation()
+{
+    ConditionalContextInfo * bestLocation = NULL;
+    PointerArrayOf<ConditionalContextInfo> matched;
+    ForEachItemIn(idx, candidates)
+    {
+        ConditionalContextInfo& cur = candidates.item(idx);
+        ConditionalContextInfo * candidateLocation = findCandidateLocation(&cur);
+        if (candidateLocation)
+        {
+            matched.append(&cur);
+            if (!bestLocation)
+                bestLocation = candidateLocation;
+            else
+                bestLocation = findCommonPath(bestLocation, candidateLocation);
+        }
+    }
+
+    while (bestLocation)
+    {
+        if (canSurroundWithAlias(bestLocation->original))
+            break;
+        bestLocation = selectParent(bestLocation);
+    }
+
+    //If best location is NULL then all expressions are marked as staying where they are.
+    if (!bestLocation)
+        return false;
+
+    ForEachItemIn(i, matched)
+        addDefinition(bestLocation, matched.item(i));
+    return true;
+}
+
+
+//Find a single location to evaluate all the child items in
+bool ConditionalContextTransformer::associateCandidatesWithRoot()
+{
+    ConditionalContextInfo * rootLocation = queryBodyExtra(rootExpr);
+    PointerArrayOf<ConditionalContextInfo> matched;
+    ForEachItemIn(idx, candidates)
+    {
+        ConditionalContextInfo& cur = candidates.item(idx);
+        ConditionalContextInfo * candidateLocation = findCandidateLocation(&cur);
+        if (candidateLocation)
+            addDefinition(rootLocation, &cur);
+    }
+
+    return rootLocation->hasDefinitions();
+}
+
+
+//What location in the tree should extra be evaluated?
+//If it is unconditional, then if first use is unconditional, eval here, else in the root.
+//If this expression is conditional and has a single use,
+//   then evaluate here or globally - if always done globally.
+//if conditional, and multiple uses, then find the common evaluation condition of the parents.
+ConditionalContextInfo * ConditionalContextTransformer::calcCommonLocation(ConditionalContextInfo * extra)
+{
+    if (extra->calcedCommonLocation)
+        return extra->commonLocation;
+
+    if (!extra->firstParent)
+    {
+        dbgassertex(extra == queryBodyExtra(rootExpr));
+        return extra;
+    }
+
+    ConditionalContextInfo * commonLocation = extra;
+    if (extra->isUnconditional() && !alwaysEvaluateGuardedTogether)
+    {
+        if (!extra->isFirstUseUnconditional())
+            commonLocation = queryBodyExtra(rootExpr);
+    }
+    else
+    {
+        commonLocation = calcCommonLocation(extra->firstParent);
+        if (extra->hasSharedParent)
+        {
+            ForEachItemIn(i, extra->extraParents)
+            {
+                ConditionalContextInfo * curParent = extra->extraParents.item(i);
+                ConditionalContextInfo * nextExtra = calcCommonLocation(curParent);
+
+                //MORE: What should be done if some expressions can be guarded, and others are marked as unmovable?
+                commonLocation = findCommonPath(commonLocation, nextExtra);
+                if (!commonLocation)
+                    break;
+            }
+        }
+        else
+        {
+            if (commonLocation == extra->firstParent)
+                commonLocation = extra;
+        }
+
+        //MORE commonLocation == NULL if you never want to move this - e.g., if parent is a choose/which/rejected/....
+    }
+    extra->calcedCommonLocation = true;
+    extra->commonLocation = commonLocation;
+    return commonLocation;
+}
+
+ConditionalContextInfo * ConditionalContextTransformer::findCandidateLocation(ConditionalContextInfo * extra)
+{
+    ConditionalContextInfo * best = calcCommonLocation(extra);
+    loop
+    {
+        if (!best)
+            return NULL;
+        IHqlExpression * bestLocation = best->original;
+        if (canSurroundWithAlias(bestLocation))
+            return best;
+        best = selectParent(best);
+    }
+}
+
+
+ConditionalContextInfo * ConditionalContextTransformer::selectParent(ConditionalContextInfo * info)
+{
+    if (info->hasSharedParent)
+        return calcCommonLocation(info);
+    return info->firstParent;
+}
+
+ConditionalContextInfo * ConditionalContextTransformer::findCommonPath(ConditionalContextInfo * left, ConditionalContextInfo * right)
+{
+    loop
+    {
+        if (!left || !right)
+            return NULL;
+        if (left == right)
+            return left;
+
+        //Ensure if there is a child and a parent that the child's parent is selected
+        if (left->seq > right->seq)
+            left = selectParent(left);
+        else
+            right = selectParent(right);
+    }
+}
+
+
+// ---- pass 2 --------
+
+//Recursively walk the tree calculating guards, and gathering a list of guards for child expressions
+//Must be called after the common locations have been calculated
+void ConditionalContextTransformer::analyseGatherGuards(IHqlExpression * expr)
+{
+    ConditionalContextInfo * extra = queryBodyExtra(expr);
+    if (!alreadyVisited(extra))
+    {
+        unsigned prevGuards = childGuards.ordinality();
+
+        doAnalyseExpr(expr);
+
+        extra->guards.setown(calcGuard(extra, prevGuards));
+        extra->calcInheritedGuards();
+        childGuards.trunc(prevGuards);
+    }
+
+    if (extra->inheritedGuards)
+        childGuards.append(*extra->inheritedGuards);
+}
+
+
+//Calculate guards for all candidates - even those that will be defined here.
+CHqlExprMultiGuard * ConditionalContextTransformer::calcGuard(ConditionalContextInfo * cur, unsigned firstGuard)
+{
+    bool needToAddCandidate = cur->isCandidateThatMoves();
+    unsigned maxGuard = childGuards.ordinality();
+    if ((firstGuard == maxGuard) && !needToAddCandidate)
+        return NULL;
+
+    IHqlExpression * original = cur->original;
+    switch (original->getOperator())
+    {
+    case no_if:
+        return createIfGuard(cur);
+    case no_or:
+    case no_and:
+        return createAndOrGuard(cur);
+    case no_map:
+    case no_case:
+    case no_which:
+    case no_rejected:
+    case no_choose:
+        //MORE!!!
+        break;
+    }
+
+    //Get the union of all guards present for the child expressions
+    if ((firstGuard == maxGuard-1) && !needToAddCandidate)
+        return LINK(&childGuards.item(firstGuard));
+
+    Owned<CHqlExprMultiGuard> newGuard = new CHqlExprMultiGuard;
+    for (unsigned i=firstGuard; i < maxGuard; i++)
+        newGuard->combine(childGuards.item(i));
+    if (needToAddCandidate)
+        newGuard->addGuarded(original);
+    return newGuard.getClear();
+}
+
+CHqlExprMultiGuard * ConditionalContextTransformer::createIfGuard(ConditionalContextInfo * cur)
+{
+    IHqlExpression * original = cur->original;
+    IHqlExpression * ifCond = original->queryChild(0);
+    CHqlExprMultiGuard * condGuard = queryGuards(ifCond);
+    CHqlExprMultiGuard * trueGuard = queryGuards(original->queryChild(1));
+    CHqlExprMultiGuard * falseGuard = queryGuards(original->queryChild(2));
+    if (!trueGuard && !falseGuard && !cur->isCandidateThatMoves())
+        return LINK(condGuard);
+
+    //If you want to common up the conditions between the child query and the parent code you might acheive
+    //if with forcing the condition into an alias.  E.g.,
+    //queryBodyExtra(ifCond)->createAlias = true;
+
+    Owned<CHqlExprMultiGuard> newGuards = new CHqlExprMultiGuard;
+    HqlExprCopyArray candidates;
+    gatherCandidates(candidates, trueGuard);
+    gatherCandidates(candidates, falseGuard);
+    ForEachItemIn(i, candidates)
+    {
+        IHqlExpression * candidate = &candidates.item(i);
+        IHqlExpression * trueCond = queryGuardCondition(trueGuard, candidate);
+        IHqlExpression * falseCond = queryGuardCondition(falseGuard, candidate);
+        OwnedHqlExpr newCond;
+        if (trueCond != falseCond)
+            newCond.setown(createValue(no_if, makeBoolType(), LINK(ifCond), LINK(trueCond), LINK(falseCond)));
+        else //MORE - check if is inverse, and map to true if so.
+            newCond.set(trueCond);
+        newGuards->addGuarded(newCond, candidate, condGuard != NULL);
+    }
+
+    if (condGuard)
+        newGuards->combine(*condGuard);
+    if (cur->isCandidateThatMoves())
+        newGuards->addGuarded(cur->original);
+    return newGuards.getClear();
+}
+
+CHqlExprMultiGuard * ConditionalContextTransformer::createAndOrGuard(ConditionalContextInfo * cur)
+{
+    IHqlExpression * original = cur->original;
+    IHqlExpression * left = original->queryChild(0);
+    CHqlExprMultiGuard * leftGuard = queryGuards(left);
+    CHqlExprMultiGuard * rightGuard = queryGuards(original->queryChild(1));
+    if (!rightGuard && !cur->isCandidateThatMoves())
+        return LINK(leftGuard);
+
+    node_operator op = original->getOperator();
+    Owned<CHqlExprMultiGuard> newGuards = new CHqlExprMultiGuard;
+    ForEachItemIn(i, rightGuard->guarded)
+    {
+        CHqlExprGuard & cur = rightGuard->guarded.item(i);
+        OwnedHqlExpr cond;
+        //(a || b):  b is evaluated if a is false.  => guard'(b,x) = !a && guard(b,x)
+        //(a && b):  b is evaluated if a is true.   => guard'(b,x) = a && guard(b,x)
+        if (op == no_or)
+            cond.setown(createBoolExpr(no_and, getInverse(left), LINK(cur.guard)));
+        else
+            cond.setown(createBoolExpr(no_and, LINK(left), LINK(cur.guard)));
+        newGuards->addGuarded(cond, cur.original, leftGuard != NULL);
+    }
+
+    //MORE: Is the reversal of the order going to matter?  E.g., instead of guard(a,x) || (a && guard(b,x))?
+    if (leftGuard)
+        newGuards->combine(*leftGuard);
+    if (cur->isCandidateThatMoves())
+        newGuards->addGuarded(cur->original);
+    return newGuards.getClear();
+}
+
+CHqlExprMultiGuard * ConditionalContextTransformer::queryGuards(IHqlExpression * expr)
+{
+    if (!expr)
+        return NULL;
+    return queryBodyExtra(expr)->guards;
+}
+
+// ---- pass 3 --------
+
+
+// ---- summary --------
+
+bool ConditionalContextTransformer::hasUnconditionalCandidate() const
+{
+    ForEachItemIn(i, candidates)
+    {
+        if (candidates.item(i).isUnconditional())
+            return true;
+    }
+    return false;
+}
+
+bool ConditionalContextTransformer::analyseNeedsTransform(const HqlExprArray & exprs)
+{
+    ConditionalContextInfo * rootInfo = queryBodyExtra(rootExpr);
+
+    activeParent = rootInfo;
+    rootInfo->setFirstUnconditional();
+
+    // Spot conditional and unconditional expressions
+    analyseArray(exprs, PassFindConditions);
+    analyseArray(exprs, PassFindCandidates);
+    if (candidates.ordinality() == 0)
+        return false;
+
+    // Tag expressions with their parent locations, and gather a list of candidates.
+    analyseArray(exprs, PassFindParents);
+
+    // Work out the locations conditional expressions need to be evaluated in
+    if (createRootGraph)
+        associateCandidatesWithRoot();
+    else if (!alwaysEvaluateGuardedTogether)
+        findCommonLocations();
+    else
+        findSingleCommonLocation();
+
+    if (hasConditionalCandidate)
+    {
+        // Calculate the guard conditions for each condition
+        analyseArray(exprs, PassGatherGuards);
+        rootInfo->guards.setown(calcGuard(rootInfo, 0));
+        childGuards.kill();
+
+        // Any guard conditions that are always true should become unconditional if context is unconditional
+        analyseArray(exprs, 3);
+    }
+
+    return hasConditionalCandidate || alwaysEvaluateGuardedTogether;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+static IHqlExpression * createConditionalExpr(IHqlExpression * condition, IHqlExpression * expr, IHqlExpression * elseExpr)
+{
+    if (matchesBoolean(condition, true))
+        return LINK(expr);
+    if (matchesBoolean(condition, false))
+        return LINK(elseExpr);
+
+    //MORE: This code is currently disabled.
+    //The intent is to ensure that in the condition IF (a || b) that b is only evaluated if a is false.
+    //However there are complications and problems with nested guard conditions which need independently solving.
+    if (false)
+    {
+        switch (condition->getOperator())
+        {
+        case no_and:
+            if (expr->queryRecord())
+            {
+                //Use dataset if operators to ensure that the conditions are sortcircuited.
+                // if (a && b, c, -) => if (a, if (b, c, -), -)
+                OwnedHqlExpr second = createConditionalExpr(condition->queryChild(1), expr, elseExpr);
+                return createConditionalExpr(condition->queryChild(0), second, elseExpr);
+            }
+            break;
+        case no_or:
+            if (expr->queryRecord())
+            {
+                // if (a || b, c, -) => if (a, c, if (b, c, -))
+                OwnedHqlExpr second = createConditionalExpr(condition->queryChild(1), expr, elseExpr);
+                return createConditionalExpr(condition->queryChild(0), expr, second);
+            }
+            break;
+        case no_if:
+            {
+                OwnedHqlExpr trueExpr = createConditionalExpr(condition->queryChild(1), expr, elseExpr);
+                OwnedHqlExpr falseExpr = createConditionalExpr(condition->queryChild(2), expr, elseExpr);
+                return createIf(LINK(condition), trueExpr.getClear(), falseExpr.getClear());
+            }
+        }
+    }
+
+    return createIf(LINK(condition), LINK(expr), LINK(elseExpr));
+}
+
+IHqlExpression * ConditionalContextTransformer::getGuardCondition(ConditionalContextInfo * extra, IHqlExpression * expr)
+{
+    CHqlExprMultiGuard * guards = extra->guards;
+    if (guards)
+    {
+        if (expr != extra->original)
+            return foldHqlExpression(guards->queryGuardCondition(expr));
+    }
+    return LINK(queryBoolExpr(true));
+}
+
+IHqlExpression * ConditionalContextTransformer::createGuardedDefinition(ConditionalContextInfo * extra, IHqlExpression * expr)
+{
+    OwnedHqlExpr guard = getGuardCondition(extra, expr);
+    assertex(guard);
+    return createGuardedDefinition(extra, expr, guard);
+}
+
+IHqlExpression * ConditionalContextTransformer::createGuardedDefinition(ConditionalContextInfo * extra, IHqlExpression * expr, IHqlExpression * condition)
+{
+    if (matchesBoolean(condition, true))
+        return LINK(expr);
+
+    OwnedHqlExpr guard = transform(condition);
+    OwnedHqlExpr defaultExpr = createNullExpr(expr);
+    return createConditionalExpr(guard, expr, defaultExpr);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+IHqlExpression * ConditionalContextTransformer::createTransformed(IHqlExpression * expr)
+{
+    OwnedHqlExpr transformed = ConditionalHqlTransformer::createTransformed(expr);
+    updateOrphanedSelectors(transformed, expr);
+    switch (transformed->getOperator())
+    {
+    case no_compound_indexread:
+    case no_compound_indexcount:
+    case no_compound_indexaggregate:
+    case no_compound_indexgroupaggregate:
+    case no_compound_indexnormalize:
+        if (!queryPhysicalRootTable(transformed))
+            return LINK(transformed->queryChild(0));
+        break;
+    }
+
+    if (queryBodyExtra(expr)->createAlias)
+    {
+        if (expr == expr->queryBody(true))
+            transformed.setown(createAlias(transformed, NULL));
+    }
+    return transformed.getClear();
+}
+
+void ConditionalContextTransformer::transformAllCandidates()
+{
+    //Process definitions in the order they were found - since this is a toplogical dependency ordering
+    ForEachItemIn(i, candidates)
+        transformCandidate(&candidates.item(i));
+}
+
+void ConditionalContextTransformer::transformAll(HqlExprArray & exprs)
+{
+    transformAllCandidates();
+    OwnedHqlExpr rootDefinitions = createDefinitions(queryBodyExtra(rootExpr));
+    HqlExprArray transformed;
+    if (rootDefinitions)
+        transformed.append(*LINK(rootDefinitions));
+    ForEachItemIn(i, exprs)
+    {
+        IHqlExpression * cur = &exprs.item(i);
+        OwnedHqlExpr mapped = transformRoot(cur);
+        transformed.append(*mapped.getClear());
+    }
+    exprs.swapWith(transformed);
+}

ecl/hqlcpp/hqlhoist.hpp

@@ -0,0 +1,189 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+#ifndef __HQLHOIST_HPP_
+#define __HQLHOIST_HPP_
+
+#include "hqltrans.ipp"
+
+bool canSurroundWithAlias(IHqlExpression * expr);
+
+//This class represents the guard/protection condition for a single candiate expression
+class CHqlExprGuard : public CInterface
+{
+public:
+    CHqlExprGuard(IHqlExpression * _guard, IHqlExpression * _original, bool _guardContainsCandidate)
+        : guard(_guard), original(_original), guardContainsCandidate(_guardContainsCandidate)
+    {
+    }
+
+    bool guardContainsCandidate;
+    LinkedHqlExpr guard;
+    LinkedHqlExpr original;
+};
+
+//This class represents a list of the guards for each of the guarded expressions used within the owning expression.
+class CHqlExprMultiGuard : public CInterface
+{
+public:
+    void addGuarded(IHqlExpression * original);
+    void addGuarded(IHqlExpression * cond, IHqlExpression * original, bool guardContainsCandidate);
+
+    void combine(CHqlExprMultiGuard & other);
+    void combine(CHqlExprGuard & other);
+    void gatherCandidates(HqlExprCopyArray & candidiates) const;
+    bool guardContainsCandidate(IHqlExpression * original) const;
+    IHqlExpression * queryGuardCondition(IHqlExpression * original) const;
+
+public:
+    CIArrayOf<CHqlExprGuard> guarded;
+};
+
+//---------------------------------------------------------------------------------------------------------------------
+
+class ConditionalContextInfo : public ConditionalTransformInfo
+{
+public:
+    ConditionalContextInfo(IHqlExpression * expr) : ConditionalTransformInfo(expr)
+    {
+        firstParent = NULL;
+        commonLocation = NULL;
+        moveTo = NULL;
+        firstAnnotatedExpr = NULL;
+        seq = 0;
+        hasSharedParent = false;
+        calcedCommonLocation = false;
+        isCandidateExpr = false;
+        createAlias = false;
+    }
+
+    void addExtraParent(ConditionalContextInfo * nextParent, bool noteManyUnconditionalParents)
+    {
+        hasSharedParent = true;
+        //We only care about parents of conditional expressions
+        if (!isUnconditional() || noteManyUnconditionalParents || canSurroundWithAlias(original))
+        {
+            extraParents.append(nextParent);
+        }
+    }
+
+    void calcInheritedGuards();
+    bool isCandidateThatMoves() const;
+    bool usedOnMultiplePaths() const;
+
+    inline bool hasDefinitions() const { return definitions.ordinality() != 0; }
+    inline bool changesLocation() const { return moveTo != this; }
+    inline void setFirstParent(ConditionalContextInfo * parent) { firstParent = parent; }
+
+public:
+    // use a pointer for the first match to avoid reallocation for unshared items
+    ConditionalContextInfo * firstParent;  // NULL for the root item
+    // if this expression is conditional, then which expression contains all uses of it?
+    ConditionalContextInfo * commonLocation;
+    // where will the definition be moved to.  NULL if it will not be moved.
+    ConditionalContextInfo * moveTo;
+    // First annotated expression seen (if any)
+    IHqlExpression * firstAnnotatedExpr;
+    //Which guarded expressions need to be created at this point?
+    HqlExprArray definitions;
+    //If conditional, a list of all the other places it is used.
+    PointerArrayOf<ConditionalContextInfo> extraParents;
+    // guards for all expressions that this expression contains
+    Owned<CHqlExprMultiGuard> guards;
+    //All guards that need to be passed on the container/parent expressions [excludes definitions]
+    Owned<CHqlExprMultiGuard> inheritedGuards;
+    //A sequence number use to ensure it is efficient to find a common parent
+    unsigned seq;
+    bool hasSharedParent;
+    bool calcedCommonLocation;
+    //Is this a candidate for evaluating in a single location with guards?
+    bool isCandidateExpr;
+    bool createAlias;
+};
+
+//---------------------------------------------------------------------------------------------------------------------
+
+class ConditionalContextTransformer : public ConditionalHqlTransformer
+{
+public:
+    enum { PassFindConditions,
+           PassFindCandidates,
+           PassFindParents,
+           PassGatherGuards,
+    };
+
+public:
+    virtual void analyseExpr(IHqlExpression * expr);
+            bool findSingleCommonLocation();
+            bool findCommonLocations();
+            bool associateCandidatesWithRoot();
+
+    bool analyseNeedsTransform(const HqlExprArray & exprs);
+    bool hasSingleConditionalCandidate() const;
+    void transformAll(HqlExprArray & exprs);
+
+protected:
+    virtual void transformCandidate(ConditionalContextInfo * candidate) = 0;
+    ConditionalContextTransformer(HqlTransformerInfo & info, bool _alwaysEvaluateGuardedTogether);
+    virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr);
+
+    bool hasUnconditionalCandidate() const;
+
+    void analyseConditionalParents(IHqlExpression * expr);
+    void analyseGatherGuards(IHqlExpression * expr);
+
+    ConditionalContextInfo * calcCommonLocation(ConditionalContextInfo * extra);
+    ConditionalContextInfo * findCandidateLocation(ConditionalContextInfo * extra);
+    ConditionalContextInfo * findCommonPath(ConditionalContextInfo * left, ConditionalContextInfo * right);
+    ConditionalContextInfo * selectParent(ConditionalContextInfo * info);
+
+    CHqlExprMultiGuard * calcGuard(ConditionalContextInfo * cur, unsigned firstGuard);
+    CHqlExprMultiGuard * createIfGuard(ConditionalContextInfo * cur);
+    CHqlExprMultiGuard * createAndOrGuard(ConditionalContextInfo * cur);
+    CHqlExprMultiGuard * queryGuards(IHqlExpression * expr);
+
+    void addDefinition(ConditionalContextInfo * location, ConditionalContextInfo * candidate);
+    void removeDefinition(ConditionalContextInfo * location, ConditionalContextInfo * candidate);
+    virtual IHqlExpression * createDefinitions(ConditionalContextInfo * extra) = 0;
+
+    inline ConditionalContextInfo * queryBodyExtra(IHqlExpression * expr) { return (ConditionalContextInfo*)NewHqlTransformer::queryTransformExtra(expr->queryBody()); }
+    void noteCandidate(ConditionalContextInfo * extra);
+    inline void noteCandidate(IHqlExpression * expr) { noteCandidate(queryBodyExtra(expr)); }
+
+    IHqlExpression * getGuardCondition(ConditionalContextInfo * extra, IHqlExpression * expr);
+    IHqlExpression * createGuardedDefinition(ConditionalContextInfo * extra, IHqlExpression * expr);
+    IHqlExpression * createGuardedDefinition(ConditionalContextInfo * extra, IHqlExpression * expr, IHqlExpression * condition);
+
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr);
+    void transformAllCandidates();
+
+protected:
+    CIArrayOf<ConditionalContextInfo> candidates;
+    ConditionalContextInfo * activeParent;
+    OwnedHqlExpr rootExpr;
+    CopyCIArrayOf<CHqlExprMultiGuard> childGuards;
+    CopyCIArrayOf<ConditionalContextInfo> insertLocations;
+    unsigned seq;
+    bool alwaysEvaluateGuardedTogether;
+    bool noteManyUnconditionalParents;
+    bool hasConditionalCandidate;
+    bool createRootGraph;
+};
+
+IHqlExpression * mapExternalToInternalResults(IHqlExpression * expr, IHqlExpression * graph);
+
+#endif

ecl/hqlcpp/hqlhtcpp.cpp

@@ -55,6 +55,7 @@
 #include "deffield.hpp"
 #include "hqlinline.hpp"
 #include "hqlusage.hpp"
+#include "hqlhoist.hpp"
 
 //The following are include to ensure they call compile...
 #include "eclhelper.hpp"
@@ -134,6 +135,25 @@ static void markSubGraphAsRoot(IPropertyTree * tree)
         addGraphAttributeBool(tree, "rootGraph", true);
 }
 
+SubGraphInfo * matchActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag)
+{
+    FilteredAssociationIterator iter(ctx, AssocSubGraph);
+    ForEach(iter)
+    {
+        SubGraphInfo & cur = static_cast<SubGraphInfo &>(iter.get());
+        if (graphTag == cur.graphTag)
+            return &cur;
+    }
+    return NULL;
+}
+
+
+bool isActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag)
+{
+    return matchActiveGraph(ctx, graphTag) != NULL;
+}
+
+
 //---------------------------------------------------------------------------
 
 class InternalResultTracker : public CInterface
@@ -338,81 +358,35 @@ MemberFunction::~MemberFunction()
 
 //---------------------------------------------------------------------------
 
-class ChildSpotterInfo : public HoistingTransformInfo
-{
-public:
-    ChildSpotterInfo(IHqlExpression * _original) : HoistingTransformInfo(_original) { spareByte2 = false; }
-    
-    inline bool getHoist() { return spareByte2 != 0; }
-    inline void setHoist() { spareByte2 = true; }
-
-private:
-    using HoistingTransformInfo::spareByte2;                //prevent derived classes from also using this spare byte
-};
-
-
 static HqlTransformerInfo childDatasetSpotterInfo("ChildDatasetSpotter");
-class ChildDatasetSpotter : public HoistingHqlTransformer
+class NewChildDatasetSpotter : public ConditionalContextTransformer
 {
 public:
-    ChildDatasetSpotter(HqlCppTranslator & _translator, BuildCtx & _ctx) 
-        : HoistingHqlTransformer(childDatasetSpotterInfo, HTFnoteallconditionals), translator(_translator), ctx(_ctx) 
-    { 
-        candidate = false; 
-    }
-
-    virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr)
+    NewChildDatasetSpotter(HqlCppTranslator & _translator, BuildCtx & _ctx, bool _forceRoot)
+        : ConditionalContextTransformer(childDatasetSpotterInfo, true), translator(_translator), ctx(_ctx), forceRoot(_forceRoot)
     {
-        return CREATE_NEWTRANSFORMINFO(ChildSpotterInfo, expr);
+        //The child dataset code could create a single no_childquery at any place in the graph, or multiple subgraphs
+        //however adding a no_compound(no_childquery, ...) into the tree can cause problems for subsequent optimizations
+        //if any of them caused the no_getgraphresult to be hoisted before the no_childquery it would create an
+        //invalid expression.
+        //Instead, add the no_childquery before the first item that changes, and repeat for children of if actions.
+        createRootGraph = true;//_forceRoot;
     }
-    inline ChildSpotterInfo * queryBodyExtra(IHqlExpression * expr)     { return static_cast<ChildSpotterInfo *>(queryTransformExtra(expr->queryBody())); }
 
     virtual void analyseExpr(IHqlExpression * expr)
     {
-        if (pass == 0)
-        {
-            if (!analyseThis(expr))
-                return;
-            switch (expr->getOperator())
-            {
-            case no_if:
-            case no_and:
-            case no_which:
-            case no_rejected:
-            case no_or:
-            case no_map:
-                {
-                    analyseExpr(expr->queryChild(0));
-                    conditionDepth++;
-                    ForEachChildFrom(i, expr, 1)
-                        analyseExpr(expr->queryChild(i));
-                    conditionDepth--;
-                    break;
-                }
-            case no_choose:
-            case no_case:
-                {
-                    analyseExpr(expr->queryChild(0));
-                    analyseExpr(expr->queryChild(1));
-                    conditionDepth++;
-                    ForEachChildFrom(i, expr, 2)
-                        analyseExpr(expr->queryChild(i));
-                    conditionDepth--;
-                    break;
-                }
-            default:
-                doAnalyseExpr(expr);
-                break;
-            }
-        }
-        else
+        switch (pass)
         {
+        case PassFindCandidates:
             if (!alreadyVisited(expr->queryBody()))
                 markHoistPoints(expr);
+            break;
+        default:
+            ConditionalContextTransformer::analyseExpr(expr);
+            break;
         }
     }
 
-
     //MORE: This is a bit of a hack, and should be improved (share code with resource child hoist?)
     inline bool walkFurtherDownTree(IHqlExpression * expr)
     {
@@ -437,27 +411,13 @@ public:
     void markHoistPoints(IHqlExpression * expr)
     {
         node_operator op = expr->getOperator();
-        switch (op)
-        {
-        case no_transform:
-        case no_assign:
-        case no_assignall:
-            break;
-        default:
-            if (!containsNonActiveDataset(expr))
-                return;
-            break;
-        }
-
         if (expr->isDataset() || (expr->isDatarow() && (op != no_select)))
         {
-            if (isUsedUnconditionallyEnough(expr) && !translator.canAssignInline(&ctx, expr))
+            if (!translator.canAssignInline(&ctx, expr))
             {
-                candidate = true;
-                queryBodyExtra(expr)->setHoist();
+                noteCandidate(expr);
                 return;
             }
-
             if (!walkFurtherDownTree(expr))
                 return;
         }
@@ -465,6 +425,97 @@ public:
         doAnalyseExpr(expr);
     }
 
+    IHqlExpression * createTransformed(IHqlExpression * expr)
+    {
+        IHqlExpression * body = expr->queryBody(true);
+        if (expr != body)
+            return createTransformedAnnotation(expr);
+
+        ConditionalContextInfo * extra = queryBodyExtra(expr);
+
+        //The following must preceed transforming the children
+        OwnedHqlExpr subgraph = createDefinitions(extra);
+
+        OwnedHqlExpr transformed = ConditionalContextTransformer::createTransformed(expr);
+        updateOrphanedSelectors(transformed, expr);
+
+        assertex(!extra->moveTo);
+
+        if (subgraph)
+            return createCompound(subgraph.getClear(), transformed.getClear());
+        return transformed.getClear();
+    }
+
+    virtual void transformCandidate(ConditionalContextInfo * candidate)
+    {
+        while (builders.ordinality() < insertLocations.ordinality())
+            builders.append(* new ChildGraphExprBuilder(0));
+
+        IHqlExpression * expr = candidate->original;
+        ConditionalContextInfo * moveTo = candidate->moveTo;
+        OwnedHqlExpr guard;
+        if (moveTo)
+        {
+            guard.setown(getGuardCondition(moveTo, expr));
+
+            //Expressions which are very simple functions of unconditional expressions are treated as if they
+            //are unconditional
+            if (moveTo->isUnconditional() && isUsedUnconditionallyEnough(expr))
+                guard.set(queryBoolExpr(true));
+
+            //version 1: don't guard any child queries.
+            if (!matchesBoolean(guard, true))
+            {
+                assertex(moveTo->guards);
+                //MORE: For the moment disable any expressions that are only used conditionally.
+                //Often including conditions improves the code, but sometimes the duplicate evaluation of the
+                //guard conditions in the parent and the child causes excessive code generation.
+                //And forcing it into an alias doesn't help becuase that isn't currently executed in the parent.
+                //Uncomment: if (moveTo->guards->guardContainsCandidate(expr))
+                {
+                    removeDefinition(moveTo, candidate);
+                    moveTo = NULL;
+                }
+            }
+        }
+
+        //A candidate inside a condition that prevents it being moved just creates a definition where it is.
+        if (!moveTo)
+            return;
+
+        ChildGraphExprBuilder & builder = queryBuilder(moveTo);
+        IHqlExpression * annotated = candidate->firstAnnotatedExpr ? candidate->firstAnnotatedExpr : expr;
+        OwnedHqlExpr guarded = createGuardedDefinition(moveTo, annotated, guard);
+        OwnedHqlExpr transformed = builder.addDataset(guarded);
+
+        if (moveTo == candidate)
+        {
+            OwnedHqlExpr subgraph = createDefinitions(moveTo);
+            transformed.setown(createCompound(subgraph.getClear(), transformed.getClear()));
+        }
+
+        setTransformed(expr, transformed);
+    }
+
+    virtual IHqlExpression * createDefinitions(ConditionalContextInfo * extra)
+    {
+        if (!extra->hasDefinitions())
+            return NULL;
+
+        ChildGraphExprBuilder & builder = queryBuilder(extra);
+        OwnedHqlExpr graph = builder.getGraph();
+        OwnedHqlExpr cleanedGraph = mapExternalToInternalResults(graph, builder.queryRepresents());
+        return cleanedGraph.getClear();
+    }
+
+
+    ChildGraphExprBuilder & queryBuilder(ConditionalContextInfo * extra)
+    {
+        unsigned match = insertLocations.find(*extra);
+        assertex(match != NotFound);
+        return builders.item(match);
+    }
+
     inline bool isUsedUnconditionallyEnough(IHqlExpression * expr)
     {
         IHqlExpression * search = expr;
@@ -489,47 +540,11 @@ public:
         }
     }
 
-
-    IHqlExpression * createTransformed(IHqlExpression * expr)
-    {
-        OwnedHqlExpr transformed = HoistingHqlTransformer::createTransformed(expr);
-        updateOrphanedSelectors(transformed, expr);
-
-        if ((expr->isDataset() || expr->isDatarow()) && (expr->getOperator() != no_select))
-        {
-            if (queryBodyExtra(expr)->getHoist() && !translator.canAssignInline(&ctx, transformed))
-            {
-                if (!builder)
-                    builder.setown(new ChildGraphBuilder(translator));
-                return builder->addDataset(expr);
-            }
-        }
-
-        //Very occasionally an index read being hoisted means that an index aggregate is now an aggregate
-        //on a hoisted result.
-        switch (transformed->getOperator())
-        {
-            case no_compound_indexcount:
-            case no_compound_indexaggregate:
-                if (!queryPhysicalRootTable(transformed))
-                    transformed.set(transformed->queryChild(0));
-                break;
-        }
-
-        return transformed.getClear();
-    }
-
-    virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr) { return LINK(expr); }
-
-    bool hasCandidate() const { return candidate; }
-    ChildGraphBuilder * getBuilder() { return builder.getClear(); };
-
-
 protected:
+    CIArrayOf<ChildGraphExprBuilder> builders;
     HqlCppTranslator & translator;
-    Owned<ChildGraphBuilder> builder;
     BuildCtx & ctx;
-    bool candidate;
+    bool forceRoot;
 };
 
 
@@ -667,27 +682,35 @@ public:
         pending.append(*LINK(stmt));
     }
 
+    void processStmts(IHqlExpression * expr)
+    {
+        expr->unwindList(pending, no_actionlist);
+    }
+
     void clear()
     {
-        builder.clear();
         pending.kill();
         processed = false;
     }
 
-    bool hasGraphPending()
+    IHqlExpression * getPrefetchGraph()
     {
-        spotChildDatasets();
-        return builder != NULL;
+        spotChildDatasets(true);
+        if (pending.ordinality() == 0)
+            return NULL;
+        IHqlExpression & subquery = pending.item(0);
+        if (subquery.getOperator() == no_childquery)
+        {
+            pending.remove(0, true);
+            return &subquery;
+        }
+
+        return NULL;
     }
 
     void flush(BuildCtx & ctx)
     {
-        spotChildDatasets();
-        if (builder)
-        {
-            builder->generateGraph(ctx);
-            builder.clear();
-        }
+        spotChildDatasets(false);
         combineConditions();
         optimizeAssigns();
         ForEachItemIn(i, pending)
@@ -695,10 +718,18 @@ public:
         pending.kill();
     }
 
-    void generatePrefetch(BuildCtx & ctx, OwnedHqlExpr * retGraphExpr, OwnedHqlExpr * retResultsExpr)
+    void optimize()
     {
-        builder->generatePrefetchGraph(ctx, retGraphExpr, retResultsExpr);
-        builder.clear();
+        spotChildDatasets(false);
+        combineConditions();
+        optimizeAssigns();
+    }
+
+    IHqlExpression * getActionList()
+    {
+        OwnedHqlExpr ret = createActionList(pending);
+        pending.kill();
+        return ret.getClear();
     }
 
 protected:
@@ -708,39 +739,47 @@ protected:
         pending.combineConditions();
     }
 
-    void spotChildDatasets()
+    void spotChildDatasets(bool forceRoot)
     {
         if (!processed && translator.queryCommonUpChildGraphs())
         {
-            ChildDatasetSpotter spotter(translator, buildctx);
-            for (unsigned pass=0; pass < 2; pass++)
+            HqlExprArray analyseExprs;
+            ForEachItemIn(i, pending)
             {
-                ForEachItemIn(i, pending)
+                IHqlExpression & cur = pending.item(i);
+                IHqlExpression * value = &cur;
+                switch (value->getOperator())
                 {
-                    IHqlExpression & cur = pending.item(i);
-                    IHqlExpression * value = &cur;
-                    switch (value->getOperator())
-                    {
-                    case no_assign:
-                        value = value->queryChild(1);
-                        break;
-                    case no_alias:
-                    case no_skip:
-                        value = value->queryChild(0);
-                        break;
-                    }
-                    if (value)
-                        spotter.analyse(value, pass);
+                case no_assign:
+                    value = value->queryChild(1);
+                    break;
+                case no_alias:
+                case no_skip:
+                    value = value->queryChild(0);
+                    break;
                 }
+                if (value)
+                    analyseExprs.append(*LINK(value));
             }
 
-            if (spotter.hasCandidate())
+            NewChildDatasetSpotter spotter(translator, buildctx, forceRoot);
+            if (spotter.analyseNeedsTransform(analyseExprs))
             {
-                HqlExprArray replacement;
-                ForEachItemIn(i, pending)
-                    replacement.append(*spotter.transformRoot(&pending.item(i)));
-                replaceArray(pending, replacement);
-                builder.setown(spotter.getBuilder());
+                //This could be conditional on whether or not there is an unconditional candidate, but that would stop
+                //the same expression being commoned up between two conditional branches.
+                //So, only avoid if 1 conditional candidate used in a single location.
+                bool worthHoisting = true;
+                if (!forceRoot)
+                {
+                    if (spotter.hasSingleConditionalCandidate())
+                        worthHoisting = false;
+                }
+
+                if (worthHoisting)
+                {
+                    spotter.transformAll(pending);
+                    translator.traceExpressions("spotted child", pending);
+                }
             }
             processed = true;
         }
@@ -753,10 +792,18 @@ protected:
     HqlCppTranslator & translator;
     BuildCtx buildctx;
     StatementCollection pending;
-    Owned<ChildGraphBuilder> builder;
     bool processed;
 };
 
+void HqlCppTranslator::optimizeBuildActionList(BuildCtx & ctx, IHqlExpression * exprs)
+{
+    DelayedStatementExecutor delayed(*this, ctx);
+
+    delayed.processStmts(exprs);
+    delayed.flush(ctx);
+}
+
+
 //---------------------------------------------------------------------------
 
 static IHqlExpression * getExtractMatchingAssign(HqlExprArray & assigns, IHqlExpression * search, unsigned & expectedIndex, IHqlExpression * selfSelector)
@@ -2592,6 +2639,12 @@ AColumnInfo * DatasetSelector::resolveField(IHqlExpression * search) const
 
 void DatasetSelector::set(BuildCtx & ctx, IHqlExpression * expr)
 {
+    while (expr->getOperator() == no_compound)
+    {
+        translator.buildStmt(ctx, expr->queryChild(0));
+        expr = expr->queryChild(1);
+    }
+
     assertex(row->isModifyable());
     ITypeInfo * type = column->queryType();
     if (!hasReferenceModifier(type))
@@ -7890,7 +7943,40 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySpill(BuildCtx & ctx, IHqlExpr
 }
 
 
-//---------------------------------------------------------------------------
+//---------------------------------------------------------------------------------------------------------------------
+
+static IHqlExpression * convertScalarToResult(IHqlExpression * value, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq)
+{
+    OwnedHqlExpr row = convertScalarToRow(value, fieldType);
+    OwnedHqlExpr ds = createDatasetFromRow(LINK(row));
+    HqlExprArray args;
+    args.append(*LINK(ds));
+    args.append(*LINK(represents));
+    args.append(*getSizetConstant(seq));
+    args.append(*createAttribute(rowAtom));
+    return createValue(no_setgraphresult, makeVoidType(), args);
+}
+
+static IHqlExpression * createScalarFromResult(ITypeInfo * scalarType, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq)
+{
+    OwnedHqlExpr counterField = createField(unnamedAtom, LINK(fieldType), NULL, NULL);
+    OwnedHqlExpr counterRecord = createRecord(counterField);
+    HqlExprArray args;
+    args.append(*LINK(counterRecord));
+    args.append(*LINK(represents));
+    args.append(*getSizetConstant(seq));
+    args.append(*createAttribute(rowAtom));
+    OwnedHqlExpr counterResult = createDataset(no_getgraphresult, args);
+    OwnedHqlExpr select = createNewSelectExpr(createRow(no_selectnth, LINK(counterResult), getSizetConstant(1)), LINK(counterField));
+    OwnedHqlExpr cast = ensureExprType(select, scalarType);
+    return createAlias(cast, internalAttrExpr);
+}
+
+IHqlExpression * createCounterAsResult(IHqlExpression * counter, IHqlExpression * represents, unsigned seq)
+{
+    return createScalarFromResult(counter->queryType(), unsignedType, represents, seq);
+}
+
 
 bool HqlCppTranslator::isCurrentActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag)
 {
@@ -7900,6 +7986,85 @@ bool HqlCppTranslator::isCurrentActiveGraph(BuildCtx & ctx, IHqlExpression * gra
 }
 
 
+IHqlExpression * HqlCppTranslator::createLoopSubquery(IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * filter, IHqlExpression * again, IHqlExpression * counter, bool multiInstance, unsigned & loopAgainResult)
+{
+    //Now need to generate the body of the loop.
+    //output dataset is result 0
+    //input dataset is fed in using result 1
+    //counter (if required) is fed in using <result-2>[0].counter;
+    ChildGraphExprBuilder graphBuilder(2);
+    IHqlExpression * graphid = graphBuilder.queryRepresents();
+
+    LinkedHqlExpr transformedBody = body;
+    LinkedHqlExpr transformedAgain = again;
+
+    //Result 1 is the input dataset.
+    HqlExprArray args;
+    args.append(*LINK(dataset->queryRecord()));
+    args.append(*LINK(graphid));
+    args.append(*getSizetConstant(1));
+    if (isGrouped(dataset))
+        args.append(*createAttribute(groupedAtom));
+    args.append(*createAttribute(_loop_Atom));
+    if (multiInstance)
+        args.append(*createAttribute(_streaming_Atom));
+    if (targetThor())    // MORE: && !isChildQuery(ctx)..
+        args.append(*createAttribute(_distributed_Atom));
+    OwnedHqlExpr inputResult= createDataset(no_getgraphresult, args);
+
+    //Result 2 is the counter - if present
+    OwnedHqlExpr counterResult;
+    if (counter)
+    {
+        OwnedHqlExpr select = createCounterAsResult(counter, graphid, 2);
+        transformedBody.setown(replaceExpression(transformedBody, counter, select));
+        if (transformedAgain)
+        {
+            //The COUNTER for the global termination condition is whether to execute iteration COUNTER, 1=1st iter
+            //Since we're evaluating the condition in the previous iteration it needs to be increased by 1.
+            OwnedHqlExpr nextCounter = adjustValue(select, 1);
+            transformedAgain.setown(replaceExpression(transformedAgain, counter, nextCounter));
+        }
+
+        graphBuilder.addInput();
+    }
+
+    //first create the result...
+    //Need to replace ROWS(LEFT) with the result1
+    OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
+    OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(left), LINK(rowsid));
+    transformedBody.setown(replaceExpression(transformedBody, rowsExpr, inputResult));
+
+    OwnedHqlExpr result = createValue(no_setgraphresult, makeVoidType(), LINK(transformedBody), LINK(graphid), getSizetConstant(0), createAttribute(_loop_Atom));
+    graphBuilder.addAction(result);
+
+    if (transformedAgain)
+    {
+        LinkedHqlExpr nextLoopDataset = transformedBody;
+        if (filter)
+        {
+            //If there is a loop filter then the global condition is applied to dataset filtered by that.
+            OwnedHqlExpr mappedFilter = replaceSelector(filter, left, nextLoopDataset);
+            nextLoopDataset.setown(createDataset(no_filter, nextLoopDataset.getClear(), LINK(mappedFilter)));
+        }
+        transformedAgain.setown(replaceExpression(transformedAgain, rowsExpr, nextLoopDataset));
+
+        loopAgainResult = graphBuilder.addInput();
+        //MORE: Don't let this get past review - purely to ensure the same code is generated.
+        if (loopAgainResult == 2)
+            loopAgainResult = graphBuilder.addInput();
+        //more: Add loopAgainResult as an attribute on the no_childquery rather than using a reference parameter
+
+        OwnedHqlExpr againResult = convertScalarToResult(transformedAgain, queryBoolType(), graphid, loopAgainResult);
+        graphBuilder.addAction(againResult);
+
+    }
+
+    return graphBuilder.getGraph();
+}
+
+
+
 ABoundActivity * HqlCppTranslator::doBuildActivityLoop(BuildCtx & ctx, IHqlExpression * expr)
 {
     IHqlExpression * dataset = expr->queryChild(0);
@@ -8009,14 +8174,13 @@ ABoundActivity * HqlCppTranslator::doBuildActivityLoop(BuildCtx & ctx, IHqlExpre
     subctx.addQuotedCompound("virtual void createParentExtract(rtlRowBuilder & builder)");
 
     //Now need to generate the body of the loop.
-    //output dataset is result 0
-    //input dataset is fed in using result 1
-    //counter (if required) is fed in using result 2[0].counter;
-    ChildGraphBuilder builder(*this);
-    unique_id_t loopId = builder.buildLoopBody(subctx, dataset, selSeq, rowsid, body->queryChild(0), filter, loopCond, counter, instance->activityId, (parallel != NULL));
+    unsigned loopAgainResult = 0;
+    OwnedHqlExpr childquery = createLoopSubquery(dataset, selSeq, rowsid, body->queryChild(0), filter, loopCond, counter, (parallel != NULL), loopAgainResult);
+
+    ChildGraphBuilder builder(*this, childquery);
+    unique_id_t loopId = builder.buildLoopBody(subctx, (parallel != NULL));
     instance->addAttributeInt("_loopid", loopId);
 
-    unsigned loopAgainResult = builder.queryLoopConditionResult();
     if (loopAgainResult)
         doBuildUnsignedFunction(instance->startctx, "loopAgainResult", loopAgainResult);
 
@@ -8066,7 +8230,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGraphLoop(BuildCtx & ctx, IHql
     //output dataset is result 0
     //input dataset is fed in using result 1
     //counter (if required) is fed in using result 2[0].counter;
-    unique_id_t loopId = buildGraphLoopSubgraph(subctx, dataset, selSeq, rowsid, body->queryChild(0), counter, instance->activityId, (parallel != NULL));
+    unique_id_t loopId = buildGraphLoopSubgraph(subctx, dataset, selSeq, rowsid, body->queryChild(0), counter, (parallel != NULL));
     instance->addAttributeInt("_loopid", loopId);
 
     buildInstanceSuffix(instance);
@@ -8113,7 +8277,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityRemote(BuildCtx & ctx, IHqlExp
     subctx.addQuotedCompound("virtual void createParentExtract(rtlRowBuilder & builder)");
 
     //output dataset is result 0
-    unique_id_t remoteId = buildRemoteSubgraph(subctx, dataset, instance->activityId);
+    unique_id_t remoteId = buildRemoteSubgraph(subctx, dataset);
     
     instance->addAttributeInt("_graphid", remoteId);
 
@@ -8306,9 +8470,11 @@ public:
     void flush(BuildCtx & ctx)
     {
         if (builder)
-            builder->generateGraph(ctx);
-
-        builder.clear();
+        {
+            OwnedHqlExpr childquery = builder->getGraph();
+            translator.buildStmt(ctx, childquery);
+            builder.clear();
+        }
     }
 
     bool requiresGraph(BuildCtx & ctx, IHqlExpression * expr)
@@ -8339,8 +8505,8 @@ public:
     void addGraphStmt(BuildCtx & ctx, IHqlExpression * expr)
     {
         if (!builder)
-            builder.setown(new ChildGraphBuilder(translator));
-        builder->buildStmt(ctx, expr);
+            builder.setown(new ChildGraphExprBuilder(0));
+        builder->addAction(expr);
     }
 
     void buildStmt(BuildCtx & ctx, IHqlExpression * expr)
@@ -8382,7 +8548,7 @@ public:
 
 protected:
     HqlCppTranslator & translator;
-    Owned<ChildGraphBuilder> builder;
+    Owned<ChildGraphExprBuilder> builder;
 };
 
 ABoundActivity * HqlCppTranslator::doBuildActivityApply(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
@@ -8509,20 +8675,25 @@ unsigned HqlCppTranslator::doBuildThorChildSubGraph(BuildCtx & ctx, IHqlExpressi
 {
 //NB: Need to create the graph in the correct order so the references to property trees we retain
 // remain live..
+    unsigned thisId = reservedId ? reservedId : nextActivityId();
+    unsigned graphId = thisId;
     SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
     IPropertyTree * node = createPTree("node");
     if (activeSubgraph)
+    {
         node = activeSubgraph->tree->addPropTree("node", node);
+        if (activeSubgraph->graphTag == graphTag)
+            graphId = activeSubgraph->graphId;
+    }
     else
         node = graph->addPropTree("node", node);
 
-    unsigned thisId = reservedId ? reservedId : nextActivityId();
     node->setPropInt("@id", thisId);
 
     IPropertyTree * graphAttr = node->addPropTree("att", createPTree("att"));
     IPropertyTree * subGraph = graphAttr->addPropTree("graph", createPTree("graph"));
 
-    Owned<SubGraphInfo> graphInfo = new SubGraphInfo(subGraph, thisId, graphTag, kind);
+    Owned<SubGraphInfo> graphInfo = new SubGraphInfo(subGraph, thisId, graphId, graphTag, kind);
     ctx.associate(*graphInfo);
 
     IHqlExpression * numResultsAttr = expr->queryProperty(numResultsAtom);
@@ -8580,7 +8751,7 @@ unsigned HqlCppTranslator::doBuildThorSubGraph(BuildCtx & ctx, IHqlExpression *
     if (!graphTag && outputLibraryId)
         graphTag = outputLibraryId;
     if (needToCreateGraph)
-        beginGraph(graphctx);
+        beginGraph();
 
     unsigned thisId = doBuildThorChildSubGraph(graphctx, expr, kind, reservedId, graphTag);
 
@@ -8591,14 +8762,10 @@ unsigned HqlCppTranslator::doBuildThorSubGraph(BuildCtx & ctx, IHqlExpression *
 }
 
 
-void HqlCppTranslator::beginGraph(BuildCtx & ctx, const char * _graphName)
+void HqlCppTranslator::beginGraph(const char * _graphName)
 {
     if (activeGraphName)
-    {
-        //buildStmt(ctx, expr->queryChild(0));
-        //return;
         throwError(HQLERR_NestedThorNodes);
-    }
 
     graphSeqNumber++;
     StringBuffer graphName;
@@ -8785,7 +8952,7 @@ void HqlCppTranslator::doBuildThorGraph(BuildCtx & ctx, IHqlExpression * expr)
         buildLibraryGraph(ctx, expr, NULL);
     else
     {
-        beginGraph(ctx);
+        beginGraph();
 
         unsigned id = 0;
         OwnedHqlExpr graphTag = NULL;//WIP:createAttribute(graphAtom, createConstant((__int64)id));
@@ -8799,7 +8966,7 @@ void HqlCppTranslator::doBuildThorGraph(BuildCtx & ctx, IHqlExpression * expr)
             Owned<SubGraphInfo> graphInfo;
             if (graphTag)
             {
-                graphInfo.setown(new SubGraphInfo(graph, 0, graphTag, SubGraphRoot));
+                graphInfo.setown(new SubGraphInfo(graph, 0, 0, graphTag, SubGraphRoot));
                 graphctx.associate(*graphInfo);
             }
 
@@ -11593,12 +11760,22 @@ ABoundActivity * HqlCppTranslator::doBuildActivityIterate(BuildCtx & ctx, IHqlEx
 
 IHqlExpression * HqlCppTranslator::queryExpandAliasScope(BuildCtx & ctx, IHqlExpression * expr)
 {
-    while (expr->getOperator() == no_alias_scope)
+    loop
     {
-        expandAliasScope(ctx, expr);
-        expr = expr->queryChild(0);
+        switch (expr->getOperator())
+        {
+        case no_alias_scope:
+            expandAliasScope(ctx, expr);
+            expr = expr->queryChild(0);
+            break;
+        case no_compound:
+            buildStmt(ctx, expr->queryChild(0));
+            expr = expr->queryChild(1);
+            break;
+        default:
+            return expr;
+        }
     }
-    return expr;
 }
 
 
@@ -13917,7 +14094,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityPrefetchProject(BuildCtx & ctx
         TransformBuilder builder(*this, postctx, record, selfCursor, assigns);
         filterExpandAssignments(postctx, &builder, assigns, transform);
         builder.buildTransformChildren(postctx, record, selfCursor->querySelector());
-        if (builder.hasGraphPending())
+        OwnedHqlExpr subgraph = builder.getPrefetchGraph();
+        if (subgraph)
         {
             //Generate the extract preparation function
             BuildCtx prectx(instance->startctx);
@@ -13931,23 +14109,20 @@ ABoundActivity * HqlCppTranslator::doBuildActivityPrefetchProject(BuildCtx & ctx
             if (counter)
                 associateCounter(prectx, counter, "counter");
         
-            OwnedHqlExpr graph, results;
-            builder.generatePrefetch(prectx, &graph, &results);
+            OwnedHqlExpr graphInstance;
+            ChildGraphBuilder graphBuilder(*this, subgraph);
+            graphBuilder.generatePrefetchGraph(prectx, &graphInstance);
             prectx.addReturn(queryBoolExpr(true));
 
             BuildCtx childctx(instance->startctx);
             childctx.addQuotedCompound("virtual IThorChildGraph *queryChild()");
-            childctx.addReturn(graph);
+            childctx.addReturn(graphInstance);
 
-            //hack!  I need to change the way results are bound so they aren't nesc. the same as the query name
-            StringBuffer s;
-            s.append("IEclGraphResults * ");
-            generateExprCpp(s, results);
-            s.append(" = results;");
-            postctx.addQuoted(s);
-            
             //Add an association for the results into the transform function.
-            postctx.associateExpr(results, results);
+            IHqlExpression * graph = subgraph->queryChild(0);
+            OwnedHqlExpr results = createAttribute(resultsAtom, LINK(graph));
+            OwnedHqlExpr resultsInstanceExpr = createQuoted("results", makeBoolType());
+            postctx.associateExpr(results, resultsInstanceExpr);
         }
         builder.flush(postctx);
     }

ecl/hqlcpp/hqlinline.cpp

@@ -342,6 +342,8 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
                 return 0;
             return RETassign|((childLFlags|childRFlags) & HEFspillinline);
         }
+    case no_compound:
+        return getInlineFlags(ctx, expr->queryChild(1));
     default:
         return 0;
     }

ecl/hqlcpp/hqllib.cpp

@@ -613,7 +613,7 @@ void HqlCppTranslator::buildLibraryGraph(BuildCtx & ctx, IHqlExpression * expr,
 {
     OwnedHqlExpr resourced = getResourcedGraph(expr->queryChild(0), NULL);
 
-    beginGraph(ctx, graphName);
+    beginGraph(graphName);
 
     traceExpression("beforeGenerate", resourced);
     BuildCtx initctx(ctx);

ecl/hqlcpp/hqlresource.cpp

@@ -2691,6 +2691,8 @@ bool EclResourcer::findSplitPoints(IHqlExpression * expr)
                 locator.analyseChild(expr->queryChild(0), true);
                 break;
             }
+        case no_childquery:
+            throwUnexpected();
         default:
             {
                 for (unsigned idx=first; idx < last; idx++)

ecl/hqlcpp/hqlstep.cpp

@@ -762,7 +762,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityRowsetRange(BuildCtx & ctx, IH
     if ((kind == TAKnwaygraphloopresultread) && isGrouped(rowset))
         doBuildBoolFunction(instance->classctx, "grouped", true);
     if (graphId && targetRoxie())
-        instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+        addGraphIdAttribute(instance, ctx, graphId);
 
     buildInstanceSuffix(instance);
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -1025,7 +1025,7 @@ void ThorScalarTransformer::setTransformedSelector(IHqlExpression * expr, IHqlEx
 
 
 static HqlTransformerInfo ThorScalarTransformerInfo("ThorScalarTransformer");
-ThorScalarTransformer::ThorScalarTransformer(const HqlCppOptions & _options) : HoistingHqlTransformer(ThorScalarTransformerInfo, HTFnoteconditionalactions), options(_options)
+ThorScalarTransformer::ThorScalarTransformer(const HqlCppOptions & _options) : HoistingHqlTransformer(ThorScalarTransformerInfo, CTFnoteifactions), options(_options)
 {
     isConditionalDepth = 0;
     seenCandidate = false;
@@ -6794,7 +6794,7 @@ inline bool isTypeToHoist(ITypeInfo * type)
 
 static HqlTransformerInfo scalarGlobalTransformerInfo("ScalarGlobalTransformer");
 ScalarGlobalTransformer::ScalarGlobalTransformer(HqlCppTranslator & _translator)
-: HoistingHqlTransformer(scalarGlobalTransformerInfo, HTFtraverseallnodes), translator(_translator)
+: HoistingHqlTransformer(scalarGlobalTransformerInfo, CTFtraverseallnodes), translator(_translator)
 {
     okToHoist = true;
     neverHoist = false;
@@ -7000,7 +7000,7 @@ IHqlExpression * ScalarGlobalTransformer::createTransformed(IHqlExpression * exp
 
 static HqlTransformerInfo explicitGlobalTransformerInfo("ExplicitGlobalTransformer");
 ExplicitGlobalTransformer::ExplicitGlobalTransformer(IWorkUnit * _wu, HqlCppTranslator & _translator)
-: HoistingHqlTransformer(explicitGlobalTransformerInfo, HTFnoteconditionalactions|HTFtraverseallnodes), translator(_translator)
+: HoistingHqlTransformer(explicitGlobalTransformerInfo, CTFnoteifactions|CTFtraverseallnodes), translator(_translator)
 {
     wu = _wu;
     isRoxie = (translator.getTargetClusterType() == RoxieCluster);
@@ -7140,7 +7140,7 @@ NewScopeMigrateTransformer::NewScopeMigrateTransformer(IWorkUnit * _wu, HqlCppTr
     wu = _wu;
     isRoxie = translator.targetRoxie();
     if (!isRoxie && !_translator.queryOptions().resourceConditionalActions)
-        setFlags(HTFnoteconditionalactions);
+        setFlags(CTFnoteifactions);
     minimizeWorkunitTemporaries = translator.queryOptions().minimizeWorkunitTemporaries;
 #ifdef REMOVE_GLOBAL_ANNOTATION
     activityDepth = 0;      // should be 0 to actually have any effect - but causes problems...
@@ -9547,7 +9547,7 @@ bool containsCompound(IHqlExpression * expr)
 
 static HqlTransformerInfo nestedCompoundTransformerInfo("NestedCompoundTransformer");
 NestedCompoundTransformer::NestedCompoundTransformer(HqlCppTranslator & _translator)
-: HoistingHqlTransformer(nestedCompoundTransformerInfo, HTFnoteconditionalactions), translator(_translator), translatorOptions(_translator.queryOptions())
+: HoistingHqlTransformer(nestedCompoundTransformerInfo, CTFnoteifactions), translator(_translator), translatorOptions(_translator.queryOptions())
 {
 }
 

ecl/regress/condchild1.ecl

@@ -0,0 +1,113 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//both uses are unconditional
+inRec t1(inRec l) := TRANSFORM
+    SELF.f1 := outofline(l, 1);
+    SELF.f2 := outofline(l, 1);
+    SELF := l;
+END;
+
+//One use is conditional, the other is unconditional
+inRec t2(inRec l) := TRANSFORM
+    SELF.f1 := outofline(l, 1);
+    SELF.f2 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF := l;
+END;
+
+//Conditional comes first
+inRec t3(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := outofline(l, 1);
+    SELF := l;
+END;
+
+
+//Both uses are conditional - on the same condition
+inRec t4(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := IF(l.f1 > 2, outofline(l, 1), 13);
+    SELF := l;
+END;
+
+//Both uses are conditional - different conditions
+inRec t5(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := IF(l.f1 > 3, outofline(l, 1), 13);
+    SELF := l;
+END;
+
+//Both uses are conditional - same condition, but not inside a statement
+inRec t6(inRec l) := TRANSFORM
+    SELF.f1 := 5 + IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := 10 + IF(l.f1 > 3, outofline(l, 1), 13);
+    SELF := l;
+END;
+
+//Both uses are conditional - inside same and different conditions
+inRec t7(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f2 > 8, IF(l.f1 > 2, outofline(l, 1), 12), 9);
+    SELF.f2 := IF(l.f2 > 8, IF(l.f1 > 3, outofline(l, 1), 13), 10);
+    SELF := l;
+END;
+
+
+//Both uses are conditional - on the same condition, different out of lines
+inRec t8(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := IF(l.f1 > 2, outofline(l, 2), 13);
+    SELF := l;
+END;
+
+#if (0)
+    output(project(nofold(input), t7(LEFT)));
+#else
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+    output(project(nofold(input), t2(LEFT)));
+    output(project(nofold(input), t3(LEFT)));
+    output(project(nofold(input), t4(LEFT)));
+    output(project(nofold(input), t5(LEFT)));
+    output(project(nofold(input), t6(LEFT)));
+    output(project(nofold(input), t7(LEFT)));
+    output(project(nofold(input), t8(LEFT)));
+);
+
+#end

ecl/regress/condchild2.ecl

@@ -0,0 +1,96 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//conditional - but unconditional
+inRec t1(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 10, 2 * outofline(l, 1), 3 * outofline(l, 1));
+    SELF := l;
+END;
+
+//conditional - but unconditional (from two different branches)
+inRec t2(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 10, 2 * outofline(l, 1), 10);
+    SELF.f2 := IF(l.f1 > 10, 20, 3 * outofline(l, 1));
+    SELF := l;
+END;
+
+//Guarded with a condition that is outofline
+inRec t3(inRec l) := TRANSFORM
+    test1 := IF(l.f1 > 2, outofline(l, 1), 12) != 3;
+    SELF.f1 := IF(test1, 2 * outofline(l, 2), 10);
+    SELF := l;
+END;
+
+
+//Guarded with a conditions that are ANDed outofline
+inRec t4(inRec l) := TRANSFORM
+    test1 := IF(l.f1 > 2, outofline(l, 1), 12) != 3;
+    test2 := IF(l.f2 > 2, outofline(l, 2), 12) != 5;
+    SELF.f1 := IF(test1 and test2, outofline(l, 3), 12);
+    SELF := l;
+END;
+
+//Guarded with a conditions that are ORed outofline
+inRec t5(inRec l) := TRANSFORM
+    test1 := IF(l.f1 > 2, outofline(l, 1), 12) != 3;
+    test2 := IF(l.f2 > 2, outofline(l, 2), 12) != 5;
+    SELF.f1 := IF(test1 or test2, outofline(l, 3), 12);
+    SELF := l;
+END;
+
+//Guarded with a conditions that are IFed outofline
+inRec t6(inRec l) := TRANSFORM
+    test1 := IF(l.f1 > 2, outofline(l, 1), outofline(l, 2)) != 3;
+    SELF.f1 := IF(test1, outofline(l, 3), 12);
+    SELF := l;
+END;
+
+#if (0)
+    output(project(nofold(input), t3(LEFT)));
+#else
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+    output(project(nofold(input), t2(LEFT)));
+    output(project(nofold(input), t3(LEFT)));
+    output(project(nofold(input), t4(LEFT)));
+    output(project(nofold(input), t5(LEFT)));
+    output(project(nofold(input), t6(LEFT)));
+);
+#end

ecl/regress/condchild3.ecl

@@ -0,0 +1,95 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//used in AND conditions => second use is guarded
+inRec t1(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    SELF.f1 := IF(t1 AND t2, 10, 12);
+    SELF := l;
+END;
+
+//used in OR conditions => second use is guarded
+inRec t2(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    SELF.f1 := IF(t1 OR t2, 10, 12);
+    SELF := l;
+END;
+
+//used in OR conditions => second use is guarded
+inRec t3(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    t3 := outofline(l, 3) < 8;
+    SELF.f1 := IF((t1 AND t2) OR t3, 10, 12);
+    SELF := l;
+END;
+
+//used in WHICH conditions => second use is guarded
+inRec t4(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    t3 := outofline(l, 3) < 8;
+    SELF.f1 := WHICH(t1, t2, t2);
+    SELF := l;
+END;
+
+//used in WHICH conditions => second use is guarded
+inRec t5(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    t3 := outofline(l, 3) < 8;
+    SELF.f1 := REJECTED(t1, t2, t2);
+    SELF := l;
+END;
+
+//MORE: CHOOSE
+//MORE: MAP/CASE
+//MORE: Some in REJECTED, some conditional
+//MORE: Some in REJECTED, some unconditional
+
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+    output(project(nofold(input), t2(LEFT)));
+    output(project(nofold(input), t3(LEFT)));
+    output(project(nofold(input), t4(LEFT)));
+    output(project(nofold(input), t5(LEFT)));
+);

ecl/regress/condchild4.ecl

@@ -0,0 +1,113 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//CHOOSE
+inRec t1(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    SELF.f1 := CHOOSE(l.f1, t1, t2, 0);
+    SELF := l;
+END;
+
+//CASE
+inRec t2(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1, 12=>t2, 27=>t3, t4);
+    SELF := l;
+END;
+
+//CASE, also in case conditions
+inRec t3(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1, t2=>t3, t4);
+    SELF := l;
+END;
+
+//CASE, value used in all results
+inRec t4(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1*5, t2=>t1*6, t3=>t1*8, t1*10);
+    SELF := l;
+END;
+
+//use of t1s actually unconditional (tricky to spot?)
+inRec t5(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1, t1*12=>t2, 27=>t3, t4);
+    SELF := l;
+END;
+
+//Some case uses are also conditional, some unconditional
+inRec t6(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1, 12=>t2, 27=>t3, t4);
+    SELF.f2 := t1 * IF(l.f2> 10, t2, t3);
+    SELF := l;
+END;
+
+//make cond5 same as this, but using map.
+//MORE: CHOOSE
+//MORE: MAP/CASE
+//MORE: Some in REJECTED, some conditional
+//MORE: Some in REJECTED, some unconditional
+
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+    output(project(nofold(input), t2(LEFT)));
+    output(project(nofold(input), t3(LEFT)));
+    output(project(nofold(input), t4(LEFT)));
+    output(project(nofold(input), t5(LEFT)));
+    output(project(nofold(input), t6(LEFT)));
+);

ecl/regress/condchild5.ecl

@@ -0,0 +1,59 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//guarded candidate is also used in the guard condition
+inRec t1(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 = 10 or outofline(l, 1) = 2, 10, outofline(l, 1));
+    SELF := l;
+END;
+
+//it is impossible to guard b and c since they are mutually dependent
+inRec t2(inRec l) := TRANSFORM
+    a := outofline(l, 1) = 1;
+    b := outofline(l, 2) = 1;
+    c := outofline(l, 3) = 1;
+    d := outofline(l, 4) = 1;
+    SELF.f1 := IF((a OR b OR c) AND (a OR c or b), outofline(l, 5), outofline(l, 6));
+    SELF := l;
+END;
+
+SEQUENTIAL(
+    output(project(nofold(input), t2(LEFT)));
+);

ecl/regress/condchild6.ecl

@@ -0,0 +1,52 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//complex condition on two branches
+inRec t1(inRec l) := TRANSFORM
+    cond1 := NOFOLD(CHOOSE(sum(l.ids,id), 'a', 'b', 'c', 'd', 'z', 'e')) != 'd';
+    cond2 := NOFOLD(CHOOSE(max(l.ids,id), 'a', 'b', 'c', 'd', 'z', 'e')) != 'd';
+    SELF.f1 := IF(cond1, outofline(l, 1), 1000);
+    SELF.f2 := IF(cond2, outofline(l, 1), 1001);
+    SELF := l;
+END;
+
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+);