Merge pull request #2710 from ghalliday/conditionals

Reimplement the way child queries are commoned up.

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -288,6 +288,7 @@ _ATOM rightAtom;
 _ATOM rightonlyAtom;
 _ATOM rightouterAtom;
 _ATOM rollbackAtom;
+_ATOM _root_Atom;
 _ATOM rowAtom;
 _ATOM _rowsid_Atom;
 _ATOM rowLimitAtom;
@@ -668,6 +669,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     rightonlyAtom = createLowerCaseAtom("right only");
     rightouterAtom = createLowerCaseAtom("right outer");
     MAKEATOM(rollback);
+    MAKESYSATOM(root);
     MAKEATOM(row);
     MAKESYSATOM(rowsid);
     MAKEATOM(rowLimit);

ecl/hql/hqlatoms.hpp

@@ -292,6 +292,7 @@ extern HQL_API _ATOM rightAtom;
 extern HQL_API _ATOM rightonlyAtom;
 extern HQL_API _ATOM rightouterAtom;
 extern HQL_API _ATOM rollbackAtom;
+extern HQL_API _ATOM _root_Atom;
 extern HQL_API _ATOM rowAtom;
 extern HQL_API _ATOM _rowsid_Atom;
 extern HQL_API _ATOM rowLimitAtom;

ecl/hql/hqlattr.cpp

@@ -435,6 +435,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_reference:
     case no_assign_addfiles:
     case no_nullptr:
+    case no_childquery:
 
 //Workflow
     case no_stored:
@@ -606,7 +607,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_persist_check:
     case no_dataset_from_transform:
 
-    case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:

ecl/hql/hqlexpr.cpp

@@ -1431,8 +1431,9 @@ const char *getOpString(node_operator op)
     case no_assign_addfiles: return "+=";
     case no_debug_option_value: return "__DEBUG__";
     case no_dataset_alias: return "TABLE";
+    case no_childquery: return "no_childquery";
 
-    case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
@@ -3769,27 +3770,6 @@ CHqlExpression::~CHqlExpression()
 {
 //  DBGLOG("%lx: Destroy", (unsigned)(IHqlExpression *)this);
     ::Release(type);
-//  RELEASE_TRANSFORM_EXTRA(transformDepth, transformExtra);
-#ifdef _DEBUG       // Not strictly correct, but good enough for me in debug mode...
-    if (!hashcode)
-    {
-
-        switch(op)
-        {
-        case no_mergedscope:
-        case no_remotescope:
-        case no_privatescope:
-        case no_service:
-        case no_none:
-            break;
-        default:
-
-            PrintLog("%s did not hash", getOpString(op));
-            // assertex(false);
-            break;
-        }
-    }
-#endif
 }
 
 IHqlScope * CHqlExpression::queryScope() 

ecl/hql/hqlexpr.hpp

@@ -353,7 +353,7 @@ enum _node_operator {
     no_unused23,
     no_unused24,
         no_dataset_from_transform,
-    no_unused2,
+        no_childquery,
         no_unknown,
     no_unused3,
     no_unused4,

ecl/hql/hqlir.cpp

@@ -797,8 +797,9 @@ static const char * getOperatorText(node_operator op)
     DUMP_CASE(no,assign_addfiles);
     DUMP_CASE(no,debug_option_value);
     DUMP_CASE(no,dataset_alias);
+    DUMP_CASE(no,childquery);
 
-    case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:

ecl/hql/hqltrans.cpp

@@ -1102,9 +1102,8 @@ NewHqlTransformer::NewHqlTransformer(HqlTransformerInfo & _info) : ANewHqlTransf
 }
 
 
-bool NewHqlTransformer::alreadyVisited(IHqlExpression * expr)
+bool NewHqlTransformer::alreadyVisited(ANewTransformInfo * extra)
 {
-    ANewTransformInfo * extra = queryTransformExtra(expr);
     if (extra->lastPass == pass)
         return true;
 
@@ -1117,6 +1116,12 @@ bool NewHqlTransformer::alreadyVisited(IHqlExpression * expr)
 }
 
 
+bool NewHqlTransformer::alreadyVisited(IHqlExpression * expr)
+{
+    return alreadyVisited(queryTransformExtra(expr));
+}
+
+
 void NewHqlTransformer::analyse(IHqlExpression * expr, unsigned _pass)
 {
     pass = _pass;
@@ -2093,80 +2098,42 @@ NestedHqlMapTransformer::NestedHqlMapTransformer() : NestedHqlTransformer(nested
 
 //---------------------------------------------------------------------------
 
-HoistingHqlTransformer::HoistingHqlTransformer(HqlTransformerInfo & _info, unsigned _flags) : NewHqlTransformer(_info)
+ConditionalHqlTransformer::ConditionalHqlTransformer(HqlTransformerInfo & _info, unsigned _flags) : NewHqlTransformer(_info)
 { 
     flags = _flags; 
-    target = NULL; 
     conditionDepth = 0; 
     containsUnknownIndependentContents = false;
 }
 
-void HoistingHqlTransformer::setParent(const HoistingHqlTransformer * parent)
-{
-    assertex(parent->independentCache);
-    independentCache.set(parent->independentCache);
-}
-
-
 
-void HoistingHqlTransformer::transformRoot(const HqlExprArray & in, HqlExprArray & out)
+ANewTransformInfo * ConditionalHqlTransformer::createTransformInfo(IHqlExpression * expr)
 {
-    HqlExprArray * savedTarget = target;
-    target = &out;
-
-    //If there is a single ensureresult, then transform it specially, so that any hoisted values are
-    //only inside the ensure result.  If multiple assume we want to create cses globally (e.g., shared stored)
-    if ((in.ordinality() == 1) && (in.item(0).getOperator() == no_ensureresult))
-        out.append(*transformEnsureResult(&in.item(0)));
-    else
-        NewHqlTransformer::transformRoot(in, out);
-    target = savedTarget;
-}
-
-
-void HoistingHqlTransformer::appendToTarget(IHqlExpression & cur)
-{
-    target->append(cur);
+    return CREATE_NEWTRANSFORMINFO(ConditionalTransformInfo, expr);
 }
 
-ANewTransformInfo * HoistingHqlTransformer::createTransformInfo(IHqlExpression * expr)
+bool ConditionalHqlTransformer::analyseThis(IHqlExpression * expr)
 {
-    return CREATE_NEWTRANSFORMINFO(HoistingTransformInfo, expr);
-}
-
-void HoistingHqlTransformer::transformArray(const HqlExprArray & in, HqlExprArray & out)
-{
-    HqlExprArray * savedTarget = target;
-    target = &out;
-    ForEachItemIn(idx, in)
-        out.append(*transform(&in.item(idx)));
-    target = savedTarget;
-}
-
-IHqlExpression * HoistingHqlTransformer::transformRoot(IHqlExpression * expr)
-{
-    HqlExprArray args, transformed;
-    unwindCommaCompound(args, expr);
-    transformRoot(args, transformed);
-    return createActionList(transformed);
-}
-
-
-bool HoistingHqlTransformer::analyseThis(IHqlExpression * expr)
-{
-    HoistingTransformInfo * extra = queryBodyExtra(expr);
-    if (alreadyVisited(expr->queryBody()))
+    if (pass == 0)
     {
-        if ((pass != 0) || extra->isUnconditional() || (conditionDepth > 0))
-            return false;
+        ConditionalTransformInfo * extra = queryBodyExtra(expr);
+        if (alreadyVisited(extra))
+        {
+            if (extra->isUnconditional() || (conditionDepth > 0))
+                return false;
+            extra->setUnconditional();
+        }
+        else
+        {
+            if (conditionDepth == 0)
+                extra->setFirstUnconditional();
+        }
+        return true;
     }
-    if (conditionDepth == 0)
-        extra->setUnconditional();
-    return true;
+    return !alreadyVisited(expr);
 }
 
 
-void HoistingHqlTransformer::analyseExpr(IHqlExpression * expr)
+void ConditionalHqlTransformer::analyseExpr(IHqlExpression * expr)
 {
     if (!analyseThis(expr))
         return;
@@ -2175,7 +2142,7 @@ void HoistingHqlTransformer::analyseExpr(IHqlExpression * expr)
 }
 
 
-void HoistingHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
+void ConditionalHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
 {
     node_operator op = expr->getOperator();
     switch (op)
@@ -2190,7 +2157,7 @@ void HoistingHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
         return;
     case no_allnodes:
     case no_thisnode:
-        if (!(flags & HTFtraverseallnodes))
+        if (!(flags & CTFtraverseallnodes))
             return;
         break;
     case no_globalscope:
@@ -2204,13 +2171,31 @@ void HoistingHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
         }
         break;
     case no_if:
-        if (checkConditional(expr))
+    case no_and:
+    case no_or:
+    case no_mapto:
+    case no_map:
+    case no_which:
+    case no_rejected:
+    case no_choose:
+        if (treatAsConditional(expr))
         {
             analyseExpr(expr->queryChild(0));
             conditionDepth++;
+            ForEachChildFrom(idx, expr, 1)
+                analyseExpr(expr->queryChild(idx));
+            conditionDepth--;
+            return;
+        }
+        break;
+    case no_case:
+        if (treatAsConditional(expr))
+        {
+            analyseExpr(expr->queryChild(0));
             analyseExpr(expr->queryChild(1));
-            if (expr->queryChild(2))
-                analyseExpr(expr->queryChild(2));
+            conditionDepth++;
+            ForEachChildFrom(idx, expr, 2)
+                analyseExpr(expr->queryChild(idx));
             conditionDepth--;
             return;
         }
@@ -2222,6 +2207,83 @@ void HoistingHqlTransformer::doAnalyseExpr(IHqlExpression * expr)
     NewHqlTransformer::analyseExpr(expr);
 }
 
+bool ConditionalHqlTransformer::treatAsConditional(IHqlExpression * expr)
+{
+    switch (expr->getOperator())
+    {
+    case no_if:
+        return ((flags & CTFnoteifall) ||
+            ((flags & CTFnoteifactions) && expr->isAction()) ||
+            ((flags & CTFnoteifdatasets) && expr->isDataset()) ||
+            ((flags & CTFnoteifdatarows) && expr->isDatarow()));
+    case no_or:
+        return (flags & CTFnoteor) != 0;
+    case no_and:
+        return (flags & CTFnoteor) != 0;
+    case no_case:
+    case no_map:
+    case no_mapto:
+        return (flags & CTFnotemap) != 0;
+    case no_which:
+    case no_rejected:
+    case no_choose:
+        return (flags & CTFnotewhich) != 0;
+    }
+    return false;
+}
+
+
+//---------------------------------------------------------------------------
+
+HoistingHqlTransformer::HoistingHqlTransformer(HqlTransformerInfo & _info, unsigned _flags) : ConditionalHqlTransformer(_info, _flags)
+{
+    target = NULL;
+}
+
+void HoistingHqlTransformer::setParent(const HoistingHqlTransformer * parent)
+{
+    assertex(parent->independentCache);
+    independentCache.set(parent->independentCache);
+}
+
+void HoistingHqlTransformer::transformRoot(const HqlExprArray & in, HqlExprArray & out)
+{
+    HqlExprArray * savedTarget = target;
+    target = &out;
+
+    //If there is a single ensureresult, then transform it specially, so that any hoisted values are
+    //only inside the ensure result.  If multiple assume we want to create cses globally (e.g., shared stored)
+    if ((in.ordinality() == 1) && (in.item(0).getOperator() == no_ensureresult))
+        out.append(*transformEnsureResult(&in.item(0)));
+    else
+        NewHqlTransformer::transformRoot(in, out);
+    target = savedTarget;
+}
+
+
+void HoistingHqlTransformer::appendToTarget(IHqlExpression & cur)
+{
+    target->append(cur);
+}
+
+void HoistingHqlTransformer::transformArray(const HqlExprArray & in, HqlExprArray & out)
+{
+    HqlExprArray * savedTarget = target;
+    target = &out;
+    ForEachItemIn(idx, in)
+        out.append(*transform(&in.item(idx)));
+    target = savedTarget;
+}
+
+IHqlExpression * HoistingHqlTransformer::transformRoot(IHqlExpression * expr)
+{
+    HqlExprArray args, transformed;
+    unwindCommaCompound(args, expr);
+    transformRoot(args, transformed);
+    return createActionList(transformed);
+}
+
+
 //A good idea, but I need to get my head around roxie/thor differences and see if we can execute graphs more dynamically.
 IHqlExpression * HoistingHqlTransformer::createTransformed(IHqlExpression * expr)
 {
@@ -2235,7 +2297,7 @@ IHqlExpression * HoistingHqlTransformer::createTransformed(IHqlExpression * expr
     case no_allnodes:               // MORE: This needs really needs to recurse, and substitute within no_thisnode - not quite sure how that would happen
     case no_thisnode:
         //I'm not sure this is a good solution - really contents of no_thisnode should be hoisted in common, would require dependence on insideAllNodes
-        if (!(flags & HTFtraverseallnodes))
+        if (!(flags & CTFtraverseallnodes))
             return LINK(expr);
         break;
     case no_cluster:
@@ -2276,7 +2338,7 @@ IHqlExpression * HoistingHqlTransformer::createTransformed(IHqlExpression * expr
             return completeTransform(expr, transformedArgs);
         }
     case no_if:
-        if (checkConditional(expr))
+        if (treatAsConditional(expr))
         {
             HqlExprArray args;
             args.append(*transform(expr->queryChild(0)));

ecl/hql/hqltrans.ipp

@@ -444,6 +444,7 @@ public:
     IHqlExpression * transformRoot(IHqlExpression * expr) { return doTransformRootExpr(expr); }
 
 protected:
+            bool alreadyVisited(ANewTransformInfo * extra);
             bool alreadyVisited(IHqlExpression * pass);
     virtual void analyseExpr(IHqlExpression * expr);
     virtual void analyseSelector(IHqlExpression * expr);
@@ -585,13 +586,19 @@ This provides a mechanism for handling the code correctly, but also allowing the
 Flags provided to the constructor indicate which options for transforming are used.
 */
 
-class HoistingTransformInfo : public NewTransformInfo
+//---------------------------------------------------------------------------------------------------------------------
+
+class ConditionalTransformInfo : public NewTransformInfo
 {
+    enum { CTFunconditional = 1, CTFfirstunconditional = 2 };
 public:
-    HoistingTransformInfo(IHqlExpression * _original) : NewTransformInfo(_original) { spareByte1 = false; }
+    ConditionalTransformInfo(IHqlExpression * _original) : NewTransformInfo(_original) { spareByte1 = 0; }
     
-    inline bool isUnconditional() { return spareByte1 != 0; }
-    inline void setUnconditional() { spareByte1 = true; }
+    inline bool isUnconditional() const { return (spareByte1 & CTFunconditional) != 0; }
+    inline bool isFirstUseUnconditional() const { return (spareByte1 & CTFfirstunconditional) != 0; }
+
+    inline void setUnconditional() { spareByte1 |= CTFunconditional; }
+    inline void setFirstUnconditional() { spareByte1 |= (CTFfirstunconditional|CTFunconditional); }
 
 private:
     using NewTransformInfo::spareByte1;             //prevent derived classes from also using this spare byte
@@ -600,7 +607,51 @@ private:
 //This allows expressions to be evaluated in a nested context, so that once that nested context is finished
 //all the mapped expressions relating to that nested context are removed, and not commoned up.
 //Useful for processing 
-class HQL_API HoistingHqlTransformer : public NewHqlTransformer
+class HQL_API ConditionalHqlTransformer : public NewHqlTransformer
+{
+public:
+    enum { CTFnoteifactions     = 0x0001,
+           CTFnoteifdatasets    = 0x0002,
+           CTFnoteifdatarows    = 0x0004,
+           CTFnoteifall         = 0x0008,
+           CTFnoteor            = 0x0010,
+           CTFnoteand           = 0x0020,
+           CTFnotemap           = 0x0040,
+           CTFnotewhich         = 0x0080,
+           CTFnoteall           = 0xFFFF,
+           CTFtraverseallnodes  = 0x10000,
+    };
+    ConditionalHqlTransformer(HqlTransformerInfo & _info, unsigned _flags);
+
+    void setFlags(unsigned _flags) { flags = _flags; }
+
+protected:
+    bool analyseThis(IHqlExpression * expr);
+
+    virtual void doAnalyseExpr(IHqlExpression * expr);
+
+    virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr);
+    inline ConditionalTransformInfo * queryBodyExtra(IHqlExpression * expr)     { return static_cast<ConditionalTransformInfo *>(queryTransformExtra(expr->queryBody())); }
+    inline bool isUsedUnconditionally(IHqlExpression * expr)                { return queryBodyExtra(expr)->isUnconditional(); }
+
+    virtual void analyseExpr(IHqlExpression * expr);
+
+    inline bool treatAsConditional(IHqlExpression * expr);
+
+protected:
+    unsigned conditionDepth;
+    unsigned flags;
+    bool containsUnknownIndependentContents;
+};
+
+
+//---------------------------------------------------------------------------------------------------------------------
+
+typedef ConditionalTransformInfo HoistingTransformInfo;
+
+//This allows expressions to be evaluated in a nested context, so that once that nested context is finished
+//all the mapped expressions relating to that nested context are removed, and not commoned up.
+class HQL_API HoistingHqlTransformer : public ConditionalHqlTransformer
 {
     class IndependentTransformMap : public CInterface
     {
@@ -611,30 +662,13 @@ class HQL_API HoistingHqlTransformer : public NewHqlTransformer
         HqlExprArray cache;
     };
 public:
-    enum { HTFnoteconditionalactions = 0x01, 
-           HTFnoteconditionaldatasets= 0x02,
-           HTFnoteconditionaldatarows= 0x04,
-           HTFtraverseallnodes = 0x08,
-           HTFnoteallconditionals = 0x10,
-    };
     HoistingHqlTransformer(HqlTransformerInfo & _info, unsigned _flags);
 
     void appendToTarget(IHqlExpression & curOwned);
-    void setFlags(unsigned _flags) { flags = _flags; }
-    void setParent(const HoistingHqlTransformer * parent);
     void transformRoot(const HqlExprArray & in, HqlExprArray & out);
     IHqlExpression * transformRoot(IHqlExpression * expr);
 
 protected:
-    bool analyseThis(IHqlExpression * expr);
-
-    virtual void doAnalyseExpr(IHqlExpression * expr);
-
-    virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr);
-    inline HoistingTransformInfo * queryBodyExtra(IHqlExpression * expr)        { return static_cast<HoistingTransformInfo *>(queryTransformExtra(expr->queryBody())); }
-    inline bool isUsedUnconditionally(IHqlExpression * expr)                { return queryBodyExtra(expr)->isUnconditional(); }
-
-    virtual void analyseExpr(IHqlExpression * expr);
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
     IHqlExpression * transformIndependent(IHqlExpression * expr);
     virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr) = 0;
@@ -642,22 +676,11 @@ protected:
     void transformArray(const HqlExprArray & in, HqlExprArray & out);
     IHqlExpression * transformEnsureResult(IHqlExpression * expr);
 
-    inline bool checkConditional(IHqlExpression * expr)
-    {
-        return ((flags & HTFnoteallconditionals) ||
-            ((flags & HTFnoteconditionalactions) && expr->isAction()) ||
-            ((flags & HTFnoteconditionaldatasets) && expr->isDataset()) ||
-            ((flags & HTFnoteconditionaldatarows) && expr->isDatarow()));
-    }
+    void setParent(const HoistingHqlTransformer * parent);
 
 private:
     HqlExprArray *      target;
-    unsigned flags;
     Owned<IndependentTransformMap> independentCache;
-
-protected:
-    unsigned conditionDepth;
-    bool containsUnknownIndependentContents;
 };
 
 

ecl/hql/hqlutil.cpp

@@ -515,6 +515,18 @@ IHqlExpression * queryVirtualFileposField(IHqlExpression * record)
 }
 
 
+IHqlExpression * queryLastNonAttribute(IHqlExpression * expr)
+{
+    unsigned max = expr->numChildren();
+    while (max--)
+    {
+        IHqlExpression * cur = expr->queryChild(max);
+        if (!cur->isAttribute())
+            return cur;
+    }
+    return NULL;
+}
+
 //---------------------------------------------------------------------------
 
 static IHqlExpression * queryOnlyTableChild(IHqlExpression * expr)
@@ -1158,6 +1170,7 @@ IHqlExpression * replaceChild(IHqlExpression * expr, unsigned childIndex, IHqlEx
 
 IHqlExpression * createIf(IHqlExpression * cond, IHqlExpression * left, IHqlExpression * right)
 {
+    assertex(right);
     if (left->isDataset() || right->isDataset())
         return createDataset(no_if, cond, createComma(left, right));
 
@@ -3234,6 +3247,35 @@ IHqlExpression * createGetResultFromSetResult(IHqlExpression * setResult, ITypeI
     return createValue(no_getresult, valueType.getLink(), LINK(seqAttr), LINK(aliasAttr));
 }
 
+
+//---------------------------------------------------------------------------------------------------------------------
+
+IHqlExpression * convertScalarToGraphResult(IHqlExpression * value, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq)
+{
+    OwnedHqlExpr row = convertScalarToRow(value, fieldType);
+    OwnedHqlExpr ds = createDatasetFromRow(LINK(row));
+    HqlExprArray args;
+    args.append(*LINK(ds));
+    args.append(*LINK(represents));
+    args.append(*getSizetConstant(seq));
+    args.append(*createAttribute(rowAtom));
+    return createValue(no_setgraphresult, makeVoidType(), args);
+}
+
+IHqlExpression * createScalarFromGraphResult(ITypeInfo * scalarType, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq)
+{
+    OwnedHqlExpr counterField = createField(unnamedAtom, LINK(fieldType), NULL, NULL);
+    OwnedHqlExpr counterRecord = createRecord(counterField);
+    HqlExprArray args;
+    args.append(*LINK(counterRecord));
+    args.append(*LINK(represents));
+    args.append(*getSizetConstant(seq));
+    args.append(*createAttribute(rowAtom));
+    OwnedHqlExpr counterResult = createDataset(no_getgraphresult, args);
+    OwnedHqlExpr select = createNewSelectExpr(createRow(no_selectnth, LINK(counterResult), getSizetConstant(1)), LINK(counterField));
+    return ensureExprType(select, scalarType);
+}
+
 _ATOM queryCsvEncoding(IHqlExpression * mode)
 {
     if (mode)
@@ -4165,6 +4207,17 @@ IHqlExpression * removeLocalAttribute(IHqlExpression * expr)
     return removeProperty(expr, localAtom);
 }
 
+bool hasOperand(IHqlExpression * expr, IHqlExpression * child)
+{
+    expr = expr->queryBody();
+    ForEachChild(i, expr)
+    {
+        if (expr->queryChild(i) == child)
+            return true;
+    }
+    return false;
+}
+
 //-------------------------------------------------------------------------------------------------------
 
 class HQL_API SplitDatasetAttributeTransformer : public NewHqlTransformer

ecl/hql/hqlutil.hpp

@@ -52,7 +52,8 @@ extern HQL_API IHqlExpression * swapDatasets(IHqlExpression * parent);
 extern HQL_API IHqlExpression * createCompare(node_operator op, IHqlExpression * l, IHqlExpression * r);    // handles cast insertion...
 extern HQL_API IHqlExpression * createRecord(IHqlExpression * field);
 extern HQL_API IHqlExpression * queryLastField(IHqlExpression * record);
-extern HQL_API IHqlExpression * queryNextRecordField(IHqlExpression * record, unsigned & idx);
+extern HQL_API IHqlExpression * queryLastNonAttribute(IHqlExpression * expr);
+extern HQL_API IHqlExpression * queryNextRecordField(IHqlExpression * recorhqlutid, unsigned & idx);
 extern HQL_API int compareSymbolsByName(IInterface * * pleft, IInterface * * pright);
 extern HQL_API int compareAtoms(IInterface * * pleft, IInterface * * pright);
 extern HQL_API IHqlExpression * getSizetConstant(unsigned size);
@@ -129,6 +130,10 @@ extern HQL_API void unwindFields(HqlExprCopyArray & fields, IHqlExpression * rec
 extern HQL_API unsigned numAttributes(const HqlExprArray & args);
 extern HQL_API unsigned numAttributes(const IHqlExpression * expr);
 extern HQL_API IHqlExpression * createGetResultFromSetResult(IHqlExpression * setExpr, ITypeInfo * type=NULL);
+
+extern HQL_API IHqlExpression * convertScalarToGraphResult(IHqlExpression * value, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq);
+extern HQL_API IHqlExpression * createScalarFromGraphResult(ITypeInfo * scalarType, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq);
+
 extern HQL_API IHqlExpression * createTrimExpr(IHqlExpression * value, IHqlExpression * flags);
 extern HQL_API bool isLengthPreservingCast(IHqlExpression * expr);
 
@@ -151,6 +156,8 @@ extern HQL_API IHqlExpression * appendOwnedOperand(IHqlExpression * expr, IHqlEx
 extern HQL_API IHqlExpression * replaceOwnedProperty(IHqlExpression * expr, IHqlExpression * ownedProeprty);
 extern HQL_API IHqlExpression * appendOwnedOperandsF(IHqlExpression * expr, ...);
 extern HQL_API IHqlExpression * inheritAttribute(IHqlExpression * expr, IHqlExpression * donor, _ATOM name);
+extern HQL_API bool hasOperand(IHqlExpression * expr, IHqlExpression * child);
+
 extern HQL_API unsigned numRealChildren(IHqlExpression * expr);
 
 extern HQL_API IHqlExpression * createEvaluateOutputModule(HqlLookupContext & ctx, IHqlExpression * scopeExpr, IHqlExpression * ifaceExpr, bool expandCallsWhenBound, node_operator outputOp);

ecl/hqlcpp/CMakeLists.txt

@@ -41,6 +41,7 @@ set (    SRCS
          hqlcset.cpp 
          hqlecl.cpp 
          hqlgraph.cpp 
+         hqlhoist.cpp
          hqlhtcpp.cpp 
          hqlinline.cpp 
          hqliproj.cpp 
@@ -66,6 +67,7 @@ set (    SRCS
          hqlcerrors.hpp
          hqlcpp.hpp
          hqlcppc.hpp
+         hqlcppds.hpp
          hqlcpputil.hpp
          hqlecl.hpp
          hqlfunc.hpp
@@ -82,6 +84,7 @@ set (    SRCS
          hqlcset.ipp
          hqlfunc.ipp
          hqlgraph.ipp
+         hqlhoist.hpp
          hqlhtcpp.ipp
          hqliproj.ipp
          hqliter.ipp

ecl/hqlcpp/hqlcatom.cpp

@@ -501,6 +501,7 @@ _ATOM reportFieldOverflowAtom;
 _ATOM reportRowOverflowAtom;
 _ATOM responseinfoAtom;
 _ATOM restoreClusterAtom;
+_ATOM resultsAtom;
 _ATOM returnAtom;
 _ATOM returnPersistVersionAtom;
 _ATOM reverseIntAtom[9][2];
@@ -1218,6 +1219,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(reportRowOverflow);
     MAKEATOM(responseinfo);
     MAKEATOM(restoreCluster);
+    MAKEATOM(results);
     MAKEATOM(return);
     MAKEATOM(returnPersistVersion);
 

ecl/hqlcpp/hqlcatom.hpp

@@ -501,6 +501,7 @@ extern _ATOM reportFieldOverflowAtom;
 extern _ATOM reportRowOverflowAtom;
 extern _ATOM responseinfoAtom;
 extern _ATOM restoreClusterAtom;
+extern _ATOM resultsAtom;
 extern _ATOM returnAtom;
 extern _ATOM returnPersistVersionAtom;
 extern _ATOM reverseIntAtom[9][2];

ecl/hqlcpp/hqlcpp.cpp

@@ -873,10 +873,11 @@ ExpressionFormat queryNaturalFormat(ITypeInfo * type)
 
 //===========================================================================
 
-SubGraphInfo::SubGraphInfo(IPropertyTree * _tree, unsigned _id, IHqlExpression * _graphTag, SubGraphType _type) 
-    : HqlExprAssociation(subGraphMarker), tree(_tree) 
+SubGraphInfo::SubGraphInfo(IPropertyTree * _tree, unsigned _id, unsigned _graphId, IHqlExpression * _graphTag, SubGraphType _type)
+    : HqlExprAssociation(subGraphMarker), tree(_tree)
 { 
-    id = _id; 
+    id = _id;
+    graphId = _graphId;
     type = _type; 
     graphTag.set(_graphTag);
 }
@@ -1486,7 +1487,6 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.checkRowOverflow,"checkRowOverflow", true),
         DebugOption(options.freezePersists,"freezePersists", false),
         DebugOption(options.maxRecordSize, "defaultMaxLengthRecord", MAX_RECORD_SIZE),
-        DebugOption(options.maxInlineDepth, "maxInlineDepth", 999),         // a debugging setting...
 
         DebugOption(options.checkRoxieRestrictions,"checkRoxieRestrictions", true),     // a debug aid for running regression suite
         DebugOption(options.checkThorRestrictions,"checkThorRestrictions", true),       // a debug aid for running regression suite
@@ -2721,6 +2721,13 @@ void HqlCppTranslator::buildFilter(BuildCtx & ctx, IHqlExpression * expr)
             buildFilter(ctx, expr->queryChild(0));
             return;
         }
+    case no_compound:
+        {
+            buildStmt(ctx, expr->queryChild(0));
+            buildFilter(ctx, expr->queryChild(1));
+            break;
+        }
+
     }
     buildFilterViaExpr(ctx, expr);
 }
@@ -3295,6 +3302,8 @@ bool HqlCppTranslator::specialCaseBoolReturn(BuildCtx & ctx, IHqlExpression * ex
         return false;
     if (expr->getOperator() == no_alias_scope)
         expr = expr->queryChild(0);
+    if (expr->getOperator() == no_compound)
+        expr = expr->queryChild(1);
     if ((expr->getOperator() == no_and) || (expr->getOperator() == no_or))
         return true;
     return false;
@@ -3575,11 +3584,14 @@ void HqlCppTranslator::buildStmt(BuildCtx & _ctx, IHqlExpression * expr)
     case no_persist_check:
         buildWorkflowPersistCheck(ctx, expr);
         return;
+    case no_childquery:
+        buildChildGraph(ctx, expr);
+        return;
     case no_evaluate_stmt:
         expr = expr->queryChild(0);
         if (expr->queryValue())
             return;
-        // fall through to default behaviour.
+        break; // evaluate default behaviour.
     }
     CHqlBoundExpr tgt;
     buildAnyExpr(ctx, expr, tgt);
@@ -4371,7 +4383,8 @@ void HqlCppTranslator::doBuildExprAlias(BuildCtx & ctx, IHqlExpression * expr, C
 {
     //MORE These will be declared in a different context later.
     IHqlExpression * value = expr->queryChild(0);
-    assertex(value->getOperator() != no_alias);
+    while (value->getOperator() == no_alias)
+        value = value->queryChild(0);
 
     //The second half of this test could cause aliases to be duplicated, but has the significant effect of reducing the amount of data that is serialised.
     //so far on my examples it does the latter, but doesn't seem to cause the former
@@ -7069,13 +7082,13 @@ void HqlCppTranslator::doBuildStmtIf(BuildCtx & ctx, IHqlExpression * expr)
     buildCachedExpr(subctx, expr->queryChild(0), cond);
 
     IHqlStmt * test = subctx.addFilter(cond.expr);
-    buildStmt(subctx, expr->queryChild(1));
+    optimizeBuildActionList(subctx, expr->queryChild(1));
 
     IHqlExpression * elseExpr = queryRealChild(expr, 2);
     if (elseExpr && elseExpr->getOperator() != no_null)
     {
         subctx.selectElse(test);
-        buildStmt(subctx, elseExpr);
+        optimizeBuildActionList(subctx, elseExpr);
     }
 }
 

ecl/hqlcpp/hqlcpp.ipp

@@ -485,13 +485,14 @@ enum SubGraphType { SubGraphRoot, SubGraphRemote, SubGraphChild };
 struct SubGraphInfo : public HqlExprAssociation
 {
 public:
-    SubGraphInfo(IPropertyTree * _tree, unsigned _id, IHqlExpression * graphTag, SubGraphType _type);
+    SubGraphInfo(IPropertyTree * _tree, unsigned _id, unsigned _graphId, IHqlExpression * graphTag, SubGraphType _type);
 
     virtual AssocKind getKind() { return AssocSubGraph; }
 
 public:
     Linked<IPropertyTree> tree;
     unsigned id;
+    unsigned graphId;
     LinkedHqlExpr graphTag;
     SubGraphType type;
 };
@@ -524,7 +525,6 @@ struct HqlCppOptions
     unsigned            optimizeDiskFlag;
     unsigned            activitiesPerCpp;
     unsigned            maxRecordSize;
-    unsigned            maxInlineDepth;
     unsigned            inlineStringThreshold;
     unsigned            maxRootMaybeThorActions;
     unsigned            maxStaticRowSize;
@@ -1216,8 +1216,9 @@ public:
     void doBuildReturnCompare(BuildCtx & ctx, IHqlExpression * expr, node_operator op, bool isBoolEquality);
     void buildReturnOrder(BuildCtx & ctx, IHqlExpression *sortList, const DatasetReference & dataset);
 
-    unique_id_t buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, unique_id_t containerId, bool multiInstance);
-    unique_id_t buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset, unique_id_t containerId);
+    IHqlExpression * createLoopSubquery(IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * filter, IHqlExpression * again, IHqlExpression * counter, bool multiInstance, unsigned & loopAgainResult);
+    unique_id_t buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, bool multiInstance);
+    unique_id_t buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset);
         
     void doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt, IHqlExpression * expr, CHqlBoundExpr * result);
     IHqlExpression * doBuildInternalFunction(IHqlExpression * funcdef);
@@ -1319,6 +1320,8 @@ public:
     void doBuildStmtUpdate(BuildCtx & ctx, IHqlExpression * expr);
     void doBuildStmtWait(BuildCtx & ctx, IHqlExpression * expr);
 
+    void optimizeBuildActionList(BuildCtx & ctx, IHqlExpression * exprs);
+
     bool buildNWayInputs(CIArrayOf<ABoundActivity> & inputs, BuildCtx & ctx, IHqlExpression * input);
 
 //Activities.   
@@ -1633,6 +1636,7 @@ public:
     bool insideChildQuery(BuildCtx & ctx);
     bool insideRemoteGraph(BuildCtx & ctx);
     bool isCurrentActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag);
+    void buildChildGraph(BuildCtx & ctx, IHqlExpression * expr);
 
     void buildXmlReadChildrenIterator(BuildCtx & subctx, const char * iterTag, IHqlExpression * rowName, SharedHqlExpr & subRowExpr);
     void buildXmlReadTransform(IHqlExpression * dataset, StringBuffer & className, bool & usesContents);
@@ -1782,7 +1786,7 @@ protected:
 
     void logGraphEdge(IPropertyTree * subGraph, unsigned __int64 source, unsigned __int64 target, unsigned sourceIndex, unsigned targetIndex, const char * label, bool nWay);
 
-    void beginGraph(BuildCtx & ctx, const char * graphName = NULL);
+    void beginGraph(const char * graphName = NULL);
     void clearGraph();
     void endGraph();
 
@@ -1907,40 +1911,7 @@ protected:
     unique_id_t         instance;
 };
 
-
-//===========================================================================
-
-class ChildGraphBuilder : public CInterface
-{
-public:
-    ChildGraphBuilder(HqlCppTranslator & _translator);
-
-    IHqlExpression * addDataset(IHqlExpression * expr);
-    void buildStmt(BuildCtx & ctx, IHqlExpression * expr);
-    unique_id_t buildGraphLoopBody(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, unique_id_t containerId, bool multiInstance);
-    unique_id_t buildLoopBody(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * filter, IHqlExpression * again, IHqlExpression * counter, unique_id_t containerId, bool multiInstance);
-    unique_id_t buildRemoteGraph(BuildCtx & ctx, IHqlExpression * ds, unique_id_t containerId);
-    void generateGraph(BuildCtx & ctx);
-    void generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * retGraphExpr, OwnedHqlExpr * retResultsExpr);
-    bool isDatasetPresent(IHqlExpression * expr);
-    unsigned queryLoopConditionResult() const { return loopAgainResult; }
-protected:
-    void createBuilderAlias(BuildCtx & ctx, ParentExtract * extractBuilder);
-
-protected:
-    HqlCppTranslator & translator;
-    unsigned id;
-    OwnedHqlExpr idExpr;
-    StringBuffer instanceName;
-    OwnedHqlExpr instanceExpr;
-    OwnedHqlExpr resultInstanceExpr;
-    OwnedHqlExpr represents;
-    unsigned loopAgainResult;
-    HqlExprArray results;
-    unsigned numResults;
-};
-
-//===========================================================================
+//---------------------------------------------------------------------------------------------------------------------
 
 class CompoundBuilder
 {
@@ -2019,6 +1990,8 @@ extern bool isGraphIndependent(IHqlExpression * expr, IHqlExpression * graph);
 extern IHqlExpression * adjustBoundIntegerValues(IHqlExpression * left, IHqlExpression * right, bool subtract);
 extern bool isNullAssign(const CHqlBoundTarget & target, IHqlExpression * expr);
 
+SubGraphInfo * matchActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag);
+bool isActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag);
 inline SubGraphInfo * queryActiveSubGraph(BuildCtx & ctx) 
 { 
     return static_cast<SubGraphInfo *>(ctx.queryFirstAssociation(AssocSubGraph));

ecl/hqlcpp/hqlcppds.cpp

@@ -49,10 +49,18 @@
 #include "hqliter.ipp"
 #include "hqlinline.hpp"
 #include "hqlusage.hpp"
+#include "hqlcppds.hpp"
 
 #define MAX_FIXED_SIZE_RAW 1024
 #define INLINE_TABLE_EXPAND_LIMIT 4
 
+void addGraphIdAttribute(ActivityInstance * instance, BuildCtx & ctx, IHqlExpression * graphId)
+{
+    SubGraphInfo * match = matchActiveGraph(ctx, graphId);
+    assertex(match);
+    instance->addAttributeInt("_graphId", match->graphId);
+}
+
 //===========================================================================
 
 
@@ -1145,54 +1153,102 @@ bool isGraphIndependent(IHqlExpression * expr, IHqlExpression * graph)
     return checker.isIndependent();
 }
 
-//---------------------------------------------------------------------------
-// Child dataset processing
+///--------------------------------------------------------------------------------------------------------------------
 
-static IHqlExpression * convertScalarToResult(IHqlExpression * value, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq)
+IHqlExpression * createCounterAsGraphResult(IHqlExpression * counter, IHqlExpression * represents, unsigned seq)
 {
-    OwnedHqlExpr row = convertScalarToRow(value, fieldType);
-    OwnedHqlExpr ds = createDatasetFromRow(LINK(row));
-    HqlExprArray args;
-    args.append(*LINK(ds));
-    args.append(*LINK(represents));
-    args.append(*getSizetConstant(seq));
-    args.append(*createAttribute(rowAtom));
-    return createValue(no_setgraphresult, makeVoidType(), args);
+    OwnedHqlExpr value = createScalarFromGraphResult(counter->queryType(), unsignedType, represents, seq);
+    OwnedHqlExpr internalAttr = createAttribute(internalAtom);
+    return createAlias(value, internalAttr);
+}
+
+ChildGraphExprBuilder::ChildGraphExprBuilder(unsigned _numInputs)
+: numInputs(_numInputs)
+{
+    numOutputs=0;
+    represents.setown(createAttribute(graphAtom, createUniqueId()));
+    resultsExpr.setown(createAttribute(resultsAtom, LINK(represents)));
 }
 
-static IHqlExpression * createScalarFromResult(ITypeInfo * scalarType, ITypeInfo * fieldType, IHqlExpression * represents, unsigned seq)
+IHqlExpression * ChildGraphExprBuilder::addDataset(IHqlExpression * expr)
 {
-    OwnedHqlExpr counterField = createField(unnamedAtom, LINK(fieldType), NULL, NULL);
-    OwnedHqlExpr counterRecord = createRecord(counterField);
+    OwnedHqlExpr resultNumExpr;
+    ForEachItemIn(i, results)
+    {
+        IHqlExpression & curSetResult = results.item(i);
+        if (expr->queryBody() == curSetResult.queryChild(0)->queryBody())
+        {
+            resultNumExpr.set(curSetResult.queryChild(2));
+            break;
+        }
+    }
+
+    if (!resultNumExpr)
+    {
+        resultNumExpr.setown(getSizetConstant(numResults()));
+        results.append(*createValue(no_setgraphresult, makeVoidType(), LINK(expr), LINK(represents), LINK(resultNumExpr)));
+        numOutputs++;
+    }
+
     HqlExprArray args;
-    args.append(*LINK(counterRecord));
+    args.append(*LINK(expr->queryRecord()));
     args.append(*LINK(represents));
-    args.append(*getSizetConstant(seq));
-    args.append(*createAttribute(rowAtom));
-    OwnedHqlExpr counterResult = createDataset(no_getgraphresult, args);
-    OwnedHqlExpr select = createNewSelectExpr(createRow(no_selectnth, LINK(counterResult), getSizetConstant(1)), LINK(counterField));
-    OwnedHqlExpr cast = ensureExprType(select, scalarType);
-    return createAlias(cast, internalAttrExpr);
+    args.append(*LINK(resultNumExpr));
+    if (isGrouped(expr))
+        args.append(*createAttribute(groupedAtom));
+    if (!expr->isDataset())
+        args.append(*createAttribute(rowAtom));
+    args.append(*createAttribute(externalAtom, LINK(resultsExpr)));
+    args.append(*createAttribute(_original_Atom, LINK(expr)));
+    IHqlExpression * recordCountAttr = queryRecordCountInfo(expr);
+    if (recordCountAttr)
+        args.append(*LINK(recordCountAttr));
+    OwnedHqlExpr ret = createDataset(no_getgraphresult, args);
+    if (expr->isDatarow())
+        ret.setown(createRow(no_selectnth, LINK(ret), createComma(getSizetConstant(1), createAttribute(noBoundCheckAtom))));
+    return ret.getClear();
+}
+
+void ChildGraphExprBuilder::addAction(IHqlExpression * expr)
+{
+    results.append(*LINK(expr));
 }
 
-static IHqlExpression * createCounterAsResult(IHqlExpression * counter, IHqlExpression * represents, unsigned seq)
+unsigned ChildGraphExprBuilder::addInput()
 {
-    return createScalarFromResult(counter->queryType(), unsignedType, represents, seq);
+    unsigned id = numResults();
+    numInputs++;
+    return id;
+}
+
+IHqlExpression * ChildGraphExprBuilder::getGraph()
+{
+    HqlExprArray args;
+    args.append(*LINK(represents));
+    args.append(*getSizetConstant(numResults()));
+    args.append(*createActionList(results));
+    return createValue(no_childquery, makeVoidType(), args);
 }
 
-ChildGraphBuilder::ChildGraphBuilder(HqlCppTranslator & _translator) 
+//---------------------------------------------------------------------------
+// Child dataset processing
+
+ChildGraphBuilder::ChildGraphBuilder(HqlCppTranslator & _translator, IHqlExpression * subgraph)
 : translator(_translator)
 {
+    represents.set(subgraph->queryChild(0));
     id = translator.nextActivityId();
-    idExpr.setown(createConstant((__int64)id));
+
     appendUniqueId(instanceName.append("child"), id);
     instanceExpr.setown(createQuoted(instanceName, makeBoolType()));
-    represents.setown(createAttribute(graphAtom, LINK(idExpr)));
-    numResults = 0;
-    loopAgainResult = 0;
+    resultsExpr.setown(createAttribute(resultsAtom, LINK(represents)));
 
     StringBuffer s;
     resultInstanceExpr.setown(createQuoted(appendUniqueId(s.append("res"), id), makeBoolType()));
+    numResults = (unsigned)getIntValue(subgraph->queryChild(1));
+
+    IHqlExpression * actions = subgraph->queryChild(2);
+    actions->unwindList(results, no_actionlist);
 }
 
 void ChildGraphBuilder::generateGraph(BuildCtx & ctx)
@@ -1241,10 +1297,10 @@ void ChildGraphBuilder::generateGraph(BuildCtx & ctx)
     graphctx.addQuoted(s);
 
     translator.endExtract(graphctx, extractBuilder);
-    ctx.associateExpr(resultInstanceExpr, resultInstanceExpr);
+    ctx.associateExpr(resultsExpr, resultInstanceExpr);
 }
 
-void ChildGraphBuilder::generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * retGraphExpr, OwnedHqlExpr * retResultsExpr)
+void ChildGraphBuilder::generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * retGraphExpr)
 {
     BuildCtx ctx(_ctx);
     ctx.addGroup();
@@ -1268,59 +1324,6 @@ void ChildGraphBuilder::generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * re
     assertex(retInstanceExpr == instanceExpr);
 
     retGraphExpr->setown(retInstanceExpr.getClear());
-    retResultsExpr->set(resultInstanceExpr);
-}
-
-void ChildGraphBuilder::buildStmt(BuildCtx & ctx, IHqlExpression * expr)
-{
-    results.append(*LINK(expr));
-}
-
-bool ChildGraphBuilder::isDatasetPresent(IHqlExpression * expr)
-{
-    ForEachItemIn(i, results)
-    {
-        IHqlExpression & curSetResult = results.item(i);
-        if (expr->queryBody() == curSetResult.queryChild(0)->queryBody())
-            return true;
-    }
-    return false;
-}
-
-
-IHqlExpression * ChildGraphBuilder::addDataset(IHqlExpression * expr)
-{
-    OwnedHqlExpr resultNumExpr;
-    ForEachItemIn(i, results)
-    {
-        IHqlExpression & curSetResult = results.item(i);
-        if (expr->queryBody() == curSetResult.queryChild(0)->queryBody())
-        {
-            resultNumExpr.set(curSetResult.queryChild(2));
-            break;
-        }
-    }
-
-    if (!resultNumExpr)
-    {
-        resultNumExpr.setown(getSizetConstant(numResults++));
-        results.append(*createValue(no_setgraphresult, makeVoidType(), LINK(expr), LINK(represents), LINK(resultNumExpr)));
-    }
-
-    HqlExprArray args;
-    args.append(*LINK(expr->queryRecord()));
-    args.append(*LINK(represents));
-    args.append(*LINK(resultNumExpr));
-    if (isGrouped(expr))
-        args.append(*createAttribute(groupedAtom));
-    if (!expr->isDataset())
-        args.append(*createAttribute(rowAtom));
-    args.append(*createAttribute(externalAtom, LINK(resultInstanceExpr)));
-    args.append(*createAttribute(_original_Atom, LINK(expr)));          
-    OwnedHqlExpr ret = createDataset(no_getgraphresult, args);
-    if (expr->isDatarow())
-        ret.setown(createRow(no_selectnth, LINK(ret), createComma(getSizetConstant(1), createAttribute(noBoundCheckAtom))));
-    return ret.getClear();
 }
 
 void ChildGraphBuilder::createBuilderAlias(BuildCtx & ctx, ParentExtract * extractBuilder)
@@ -1332,70 +1335,13 @@ void ChildGraphBuilder::createBuilderAlias(BuildCtx & ctx, ParentExtract * extra
     ctx.addQuoted(s);
 }
 
-unique_id_t ChildGraphBuilder::buildLoopBody(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * filter, IHqlExpression * again, IHqlExpression * counter, unique_id_t containerId, bool multiInstance)
+unique_id_t ChildGraphBuilder::buildLoopBody(BuildCtx & ctx, bool multiInstance)
 {
-    LinkedHqlExpr transformedBody = body;
-    LinkedHqlExpr transformedAgain = again;
-    numResults = 2;
-
-    //Result 1 is the input dataset.
-    HqlExprArray args;
-    args.append(*LINK(dataset->queryRecord()));
-    args.append(*LINK(represents));
-    args.append(*getSizetConstant(1));
-    if (isGrouped(dataset))
-        args.append(*createAttribute(groupedAtom));
-    args.append(*createAttribute(_loop_Atom));
-    if (multiInstance)
-        args.append(*createAttribute(_streaming_Atom));
-    if (translator.targetThor())    // MORE: && !isChildQuery(ctx)..
-        args.append(*createAttribute(_distributed_Atom));
-    OwnedHqlExpr inputResult= createDataset(no_getgraphresult, args);
-
-    //Result 2 is the counter - if present
-    OwnedHqlExpr counterResult;
-    if (counter)
-    {
-        OwnedHqlExpr select = createCounterAsResult(counter, represents, 2);
-        transformedBody.setown(replaceExpression(transformedBody, counter, select));
-        if (transformedAgain)
-        {
-            //The COUNTER for the global termination condition is whether to execute iteration COUNTER, 1=1st iter
-            //Since we're evaluating the condition in the previous iteration it needs to be increased by 1.
-            OwnedHqlExpr nextCounter = adjustValue(select, 1);
-            transformedAgain.setown(replaceExpression(transformedAgain, counter, nextCounter));
-        }
-        numResults = 3;
-    }
-
-    //first create the result...
-    //Need to replace ROWS(LEFT) with the result1
-    OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
-    OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(left), LINK(rowsid));
-    transformedBody.setown(replaceExpression(transformedBody, rowsExpr, inputResult));
-
-    OwnedHqlExpr result = createValue(no_setgraphresult, makeVoidType(), LINK(transformedBody), LINK(represents), getSizetConstant(0), createAttribute(_loop_Atom));
-
-    if (transformedAgain)
-    {
-        LinkedHqlExpr nextLoopDataset = transformedBody;
-        if (filter)
-        {
-            //If there is a loop filter then the global condition is applied to dataset filtered by that.
-            OwnedHqlExpr mappedFilter = replaceSelector(filter, left, nextLoopDataset);
-            nextLoopDataset.setown(createDataset(no_filter, nextLoopDataset.getClear(), LINK(mappedFilter)));
-        }
-        transformedAgain.setown(replaceExpression(transformedAgain, rowsExpr, nextLoopDataset));
-        OwnedHqlExpr againResult = convertScalarToResult(transformedAgain, queryBoolType(), represents, 3);
-        result.setown(createCompound(result.getClear(), againResult.getClear()));
-        loopAgainResult = 3;
-        numResults = 4;
-    }
-
     BuildCtx subctx(ctx);
     subctx.addGroup();
 
-    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, result, represents, numResults, no_loop);
+    OwnedHqlExpr query = createActionList(results);
+    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_loop);
     //Add a flag to indicate multi instance
     if (multiInstance)
         resourced.setown(appendOwnedOperand(resourced, createAttribute(multiInstanceAtom)));
@@ -1446,7 +1392,7 @@ public:
                 }
                 else
                 {
-                    counterResult.setown(createCounterAsResult(counter, represents, 0));
+                    counterResult.setown(createCounterAsGraphResult(counter, represents, 0));
                     return LINK(counterResult);
                 }
             }
@@ -1491,27 +1437,14 @@ protected:
 
 
 
-unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, unique_id_t containerId, bool isParallel)
+unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, bool isParallel)
 {
-    OwnedHqlExpr transformedBody;
-    OwnedHqlExpr counterResult;
-    {
-        GraphLoopReplacer replacer(rowsid, represents, counter, isParallel);
-        transformedBody.setown(replacer.transformRoot(body));
-        counterResult.set(replacer.queryCounterResult());
-    }
-
-    numResults = 0;
-    if (counterResult)
-        numResults++;
-
-    OwnedHqlExpr result = createValue(no_setgraphloopresult, makeVoidType(), LINK(transformedBody), LINK(represents));
-
     BuildCtx subctx(ctx);
     subctx.addGroup();
 
-    translator.traceExpression("Before Loop resource", result);
-    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, result, represents, numResults, no_loop);
+    OwnedHqlExpr query = createActionList(results);
+    translator.traceExpression("Before Loop resource", query);
+    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_loop);
     translator.traceExpression("After Loop resource", resourced);
 
     //Add a flag to indicate multi instance
@@ -1537,19 +1470,12 @@ unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, IHqlExpression
     return id;
 }
 
-unique_id_t ChildGraphBuilder::buildRemoteGraph(BuildCtx & ctx, IHqlExpression * ds, unique_id_t containerId)
+unique_id_t ChildGraphBuilder::buildRemoteGraph(BuildCtx & ctx)
 {
     BuildCtx subctx(ctx);
     subctx.addGroup();
 
-    OwnedHqlExpr query;
-    if (!ds->isAction())
-    {
-        OwnedHqlExpr result = addDataset(ds);
-        query.setown(createActionList(results));
-    }
-    else
-        query.set(ds);
+    OwnedHqlExpr query = createActionList(results);
     OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_allnodes);
 
     Owned<ParentExtract> extractBuilder = translator.createExtractBuilder(ctx, PETremote, represents, GraphRemote, false);
@@ -1564,6 +1490,18 @@ unique_id_t ChildGraphBuilder::buildRemoteGraph(BuildCtx & ctx, IHqlExpression *
 }
 
 
+void HqlCppTranslator::buildChildGraph(BuildCtx & ctx, IHqlExpression * expr)
+{
+    IHqlExpression * represents= expr->queryChild(0);
+    OwnedHqlExpr resultsExpr = createAttribute(resultsAtom, LINK(represents));
+    //Shouldn't really happen, but if this graph has already benn called just use the results
+    if (ctx.queryMatchExpr(resultsExpr))
+        return;
+
+    ChildGraphBuilder graphBuilder(*this, expr);
+    graphBuilder.generateGraph(ctx);
+}
+
 void HqlCppTranslator::beginExtract(BuildCtx & ctx, ParentExtract * extractBuilder)
 {
     ctx.associate(*extractBuilder);
@@ -1588,9 +1526,11 @@ void HqlCppTranslator::buildAssignChildDataset(BuildCtx & ctx, const CHqlBoundTa
 
     OwnedHqlExpr call;
     {
-        ChildGraphBuilder builder(*this);
+        ChildGraphExprBuilder builder(0);
         call.setown(builder.addDataset(expr));
-        builder.generateGraph(ctx);
+
+        OwnedHqlExpr subquery = builder.getGraph();
+        buildStmt(ctx, subquery);
     }
 
     buildExprAssign(ctx, target, call);
@@ -1679,6 +1619,7 @@ IHqlExpression * HqlCppTranslator::getResourcedChildGraph(BuildCtx & ctx, IHqlEx
     HqlExprArray children;
     resourced->unwindList(children, no_actionlist);
     children.append(*createAttribute(numResultsAtom, getSizetConstant(numResults)));
+    children.append(*LINK(graphIdExpr));
     resourced.setown(createValue(no_subgraph, makeVoidType(), children));
 
     if (options.paranoidCheckNormalized || options.paranoidCheckDependencies)
@@ -1709,16 +1650,46 @@ void HqlCppTranslator::buildChildDataset(BuildCtx & ctx, IHqlExpression * expr,
 }
 
 
-unique_id_t HqlCppTranslator::buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, unique_id_t containerId, bool multiInstance)
+unique_id_t HqlCppTranslator::buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, bool multiInstance)
 {
-    ChildGraphBuilder builder(*this);
-    return builder.buildGraphLoopBody(ctx, dataset, selSeq, rowsid, body, counter, containerId, multiInstance);
+    ChildGraphExprBuilder graphBuilder(0);
+
+    OwnedHqlExpr transformedBody;
+    OwnedHqlExpr counterResult;
+    IHqlExpression * graphid = graphBuilder.queryRepresents();
+    {
+        const bool isParallel = multiInstance;
+        GraphLoopReplacer replacer(rowsid, graphid, counter, isParallel);
+        transformedBody.setown(replacer.transformRoot(body));
+        counterResult.set(replacer.queryCounterResult());
+    }
+
+    if (counterResult)
+        graphBuilder.addInput();
+
+    OwnedHqlExpr result = createValue(no_setgraphloopresult, makeVoidType(), LINK(transformedBody), LINK(graphid));
+    graphBuilder.addAction(result);
+    OwnedHqlExpr subquery = graphBuilder.getGraph();
+
+    ChildGraphBuilder builder(*this, subquery);
+    return builder.buildGraphLoopBody(ctx, multiInstance);
 }
 
-unique_id_t HqlCppTranslator::buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset, unique_id_t containerId)
+unique_id_t HqlCppTranslator::buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset)
 {
-    ChildGraphBuilder builder(*this);
-    return builder.buildRemoteGraph(ctx, dataset, containerId);
+    ChildGraphExprBuilder graphBuilder(0);
+    if (dataset->isAction())
+    {
+        graphBuilder.addAction(dataset);
+    }
+    else
+    {
+        OwnedHqlExpr ignoredResult = graphBuilder.addDataset(dataset);
+    }
+
+    OwnedHqlExpr subquery = graphBuilder.getGraph();
+    ChildGraphBuilder builder(*this, subquery);
+    return builder.buildRemoteGraph(ctx);
 }
 
 //---------------------------------------------------------------------------
@@ -1756,7 +1727,7 @@ bool HqlCppTranslator::isInlineOk()
 {
     if (!activeGraphCtx)
         return true;
-    return (options.maxInlineDepth != 0);
+    return true;
 }
 
 IHqlExpression * HqlCppTranslator::buildSpillChildDataset(BuildCtx & ctx, IHqlExpression * expr)
@@ -1768,14 +1739,25 @@ IHqlExpression * HqlCppTranslator::buildSpillChildDataset(BuildCtx & ctx, IHqlEx
 
 IHqlExpression * HqlCppTranslator::forceInlineAssignDataset(BuildCtx & ctx, IHqlExpression * expr)
 {
-    CHqlBoundExpr bound;
-    if (expr->isPure() && ctx.getMatchExpr(expr, bound))
-        return bound.getTranslatedExpr();
+    loop
+    {
+        CHqlBoundExpr bound;
+        if (expr->isPure() && ctx.getMatchExpr(expr, bound))
+            return bound.getTranslatedExpr();
 
-    if (canProcessInline(&ctx, expr) || (expr->getOperator() == no_translated))
-        return LINK(expr);
+        if (canProcessInline(&ctx, expr) || (expr->getOperator() == no_translated))
+            return LINK(expr);
 
-    return buildSpillChildDataset(ctx, expr);
+        switch (expr->getOperator())
+        {
+        case no_compound:
+            buildStmt(ctx, expr->queryChild(0));
+            expr = expr->queryChild(1);
+            break;
+        default:
+            return buildSpillChildDataset(ctx, expr);
+        }
+    }
 }
 
 //---------------------------------------------------------------------------
@@ -2049,6 +2031,12 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
         if (doBuildDatasetInlineTable(ctx, expr, tgt, format))
             return;
         break;
+    case no_compound:
+        {
+            buildStmt(ctx, expr->queryChild(0));
+            buildDataset(ctx, expr->queryChild(1), tgt, format);
+            return;
+        }
     }
 
     bool singleRow = hasSingleRow(expr);
@@ -2219,6 +2207,12 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
             }
             break;
         }
+    case no_compound:
+        {
+            buildStmt(ctx, expr->queryChild(0));
+            buildDatasetAssign(ctx, target, expr->queryChild(1));
+            return;
+        }
     case no_compound_childread:
     case no_compound_childnormalize:
     case no_compound_childaggregate:
@@ -3736,6 +3730,7 @@ void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, IReferenceSelector * targe
                 return;
             }
         }
+        break;
         /*
     case no_externalcall:
         //MORE: Should assign directly to the target, but may not be very easy....
@@ -3748,6 +3743,11 @@ void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, IReferenceSelector * targe
         }
         break;
         */
+    case no_comma:
+    case no_compound:
+        buildStmt(ctx, expr->queryChild(0));
+        buildRowAssign(ctx, target, expr->queryChild(1));
+        return;
     }
 
     Owned<IReferenceSelector> src = buildNewRow(ctx, expr);
@@ -4235,7 +4235,8 @@ IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpre
     if (expr->hasProperty(externalAtom))
     {
         IHqlExpression * resultInstance = queryPropertyChild(expr, externalAtom, 0);
-        if (!ctx.queryMatchExpr(resultInstance))
+        HqlExprAssociation * matchedResults = ctx.queryMatchExpr(resultInstance);
+        if (!matchedResults)
         {
             //Very unusual - a result is required from a child query, but that child query is actually in
             //the parent/grandparent.  We need to evaluate in the parent instead.
@@ -4246,7 +4247,7 @@ IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpre
         }
 
         HqlExprArray args;
-        args.append(*LINK(resultInstance));
+        args.append(*LINK(matchedResults->queryExpr()));
         args.append(*LINK(resultNum));
         if (preferLinkedRows)
             return bindFunctionCall(getChildQueryLinkedResultAtom, args, exprType);
@@ -4255,7 +4256,10 @@ IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpre
 
     assertex(activeActivities.ordinality());
     queryAddResultDependancy(activeActivities.tos(), graphId, resultNum);
-    unique_id_t id = getIntValue(graphId->queryChild(0), -1);
+
+    SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
+    assertex(activeSubgraph && graphId == activeSubgraph->graphTag);
+    unique_id_t id = activeSubgraph->graphId;
 
     EvalContext * instance = queryEvalContext(ctx);
     OwnedHqlExpr retInstanceExpr;
@@ -4358,7 +4362,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGetGraphResult(BuildCtx & ctx,
     else
         instance->addConstructorParameter(resultNum);
 
-    instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+    addGraphIdAttribute(instance, ctx, graphId);
     buildInstanceSuffix(instance);
 
     queryAddResultDependancy(*instance->queryBoundActivity(), graphId, resultNum);
@@ -4423,7 +4427,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySetGraphResult(BuildCtx & ctx,
 
     instance->addAttributeBool("_isSpill", isSpill);
     if (targetRoxie())
-        instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+        addGraphIdAttribute(instance, ctx, graphId);
 
     buildInstanceSuffix(instance);
 
@@ -4498,7 +4502,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGetGraphLoopResult(BuildCtx &
 
     buildInstancePrefix(instance);
     doBuildUnsignedFunction(instance->startctx, "querySequence", resultNum);
-    instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+    addGraphIdAttribute(instance, ctx, graphId);
     buildInstanceSuffix(instance);
 
     return instance->getBoundActivity();
@@ -4525,7 +4529,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySetGraphLoopResult(BuildCtx &
     if (parentActivity && !insideRemoteGraph(ctx) && !isSpill)
         addDependency(ctx, instance->queryBoundActivity(), parentActivity, childAtom, "Body");
     if (targetRoxie())
-        instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+        addGraphIdAttribute(instance, ctx, graphId);
 
     buildInstanceSuffix(instance);
 
@@ -4608,4 +4612,3 @@ void HqlCppTranslator::doBuildStmtApply(BuildCtx & ctx, IHqlExpression * expr)
     if (end)
         buildStmt(ctx, end->queryChild(0));
 }
-

ecl/hqlcpp/hqlhoist.cpp

@@ -0,0 +1,864 @@
+/*##############################################################################
+
+    Copyright (C) 2012 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "platform.h"
+#include "jlib.hpp"
+
+#include "hqlexpr.hpp"
+#include "hqlfold.hpp"
+#include "hqlhoist.hpp"
+#include "hqlutil.hpp"
+#include "hqlcpputil.hpp"
+
+bool canSurroundWithAlias(IHqlExpression * expr)
+{
+    switch (expr->getOperator())
+    {
+    case no_range:
+    case no_rangefrom:
+    case no_rangeto:
+    case no_rangecommon:
+    case no_mapto:
+    case no_recordlist:
+    case no_transformlist:
+    case no_rowvalue:
+    case no_sortlist:
+    case no_attr_expr:
+    case no_transform:
+    case no_newtransform:
+        return false;
+    case no_alias_scope:
+        return canSurroundWithAlias(expr->queryChild(0));
+    default:
+        return true;
+    }
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+static HqlTransformerInfo externalToInternalResultMapperInfo("ExternalToInternalResultMapper");
+class ExternalToInternalResultMapper : public NewHqlTransformer
+{
+public:
+    ExternalToInternalResultMapper(IHqlExpression * _graph) : NewHqlTransformer(externalToInternalResultMapperInfo), graph(_graph)
+    {
+    }
+
+protected:
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr)
+    {
+        OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
+        if (transformed->getOperator() == no_getgraphresult)
+        {
+            if (hasOperand(transformed, graph))
+                return removeProperty(transformed, externalAtom);
+        }
+        return transformed.getClear();
+    }
+
+protected:
+    IHqlExpression * graph;
+};
+
+IHqlExpression * mapExternalToInternalResults(IHqlExpression * expr, IHqlExpression * graph)
+{
+    ExternalToInternalResultMapper mapper(graph);
+    return mapper.transformRoot(expr);
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+void CHqlExprMultiGuard::addGuarded(IHqlExpression * original)
+{
+    addGuarded(queryBoolExpr(true), original, false);
+}
+
+void CHqlExprMultiGuard::addGuarded(IHqlExpression * cond, IHqlExpression * original, bool guardContainsCandidate)
+{
+    guarded.append(*new CHqlExprGuard(cond, original, guardContainsCandidate));
+}
+
+void CHqlExprMultiGuard::combine(CHqlExprMultiGuard & other)
+{
+    //Potentially O(N^2).  If there are vast numbers of candidates this could become problematic.
+    ForEachItemIn(i, other.guarded)
+    {
+        CHqlExprGuard & cur = other.guarded.item(i);
+        combine(cur);
+    }
+}
+
+void CHqlExprMultiGuard::combine(CHqlExprGuard & other)
+{
+    ForEachItemIn(i, guarded)
+    {
+        CHqlExprGuard & cur = guarded.item(i);
+        if (cur.original == other.original)
+        {
+            //condition is now (a || b)
+            OwnedHqlExpr newCond;
+            if (matchesBoolean(cur.guard, true) || matchesBoolean(other.guard, true))
+                newCond.set(queryBoolExpr(true));
+            else
+                newCond.setown(createBoolExpr(no_or, LINK(cur.guard), LINK(other.guard)));
+
+            //MORE: Could sometimes reuse the existing guard if this was created by this node
+            Owned<CHqlExprGuard> newGuard = new CHqlExprGuard(newCond, cur.original, cur.guardContainsCandidate||other.guardContainsCandidate);
+            guarded.replace(*newGuard.getClear(), i);
+            return;
+        }
+    }
+    guarded.append(OLINK(other));
+}
+
+void CHqlExprMultiGuard::gatherCandidates(HqlExprCopyArray & candidates) const
+{
+    ForEachItemIn(i, guarded)
+    {
+        CHqlExprGuard & cur = guarded.item(i);
+        if (!candidates.contains(*cur.original))
+            candidates.append(*cur.original);
+    }
+}
+
+IHqlExpression * CHqlExprMultiGuard::queryGuardCondition(IHqlExpression * original) const
+{
+    ForEachItemIn(i, guarded)
+    {
+        CHqlExprGuard & cur = guarded.item(i);
+        if (cur.original == original)
+            return cur.guard;
+    }
+    return NULL;
+}
+
+bool CHqlExprMultiGuard::guardContainsCandidate(IHqlExpression * original) const
+{
+    ForEachItemIn(i, guarded)
+    {
+        CHqlExprGuard & cur = guarded.item(i);
+        if (cur.original == original)
+            return cur.guardContainsCandidate;
+    }
+    return false;
+}
+
+void gatherCandidates(HqlExprCopyArray & candidates, CHqlExprMultiGuard * guards)
+{
+    if (guards)
+        guards->gatherCandidates(candidates);
+}
+
+
+IHqlExpression * queryGuardCondition(CHqlExprMultiGuard * guards, IHqlExpression * original)
+{
+    if (guards)
+    {
+        IHqlExpression * condition = guards->queryGuardCondition(original);
+        if (condition)
+            return condition;
+    }
+    return queryBoolExpr(false);
+}
+
+
+
+//---------------------------------------------------------------------------------------------------------------------
+
+void ConditionalContextInfo::calcInheritedGuards()
+{
+    if (guards && definitions.ordinality() != 0)
+    {
+        Owned<CHqlExprMultiGuard> newGuards = new CHqlExprMultiGuard;
+        ForEachItemIn(i, guards->guarded)
+        {
+            CHqlExprGuard & cur = guards->guarded.item(i);
+            if (!definitions.contains(*cur.original))
+                newGuards->guarded.append(OLINK(cur));
+        }
+        if (newGuards->guarded.ordinality())
+            inheritedGuards.setown(newGuards.getClear());
+    }
+    else
+        inheritedGuards.set(guards);
+}
+
+
+bool ConditionalContextInfo::isCandidateThatMoves() const
+{
+    if (!isCandidateExpr)
+        return false;
+
+    if (!changesLocation())
+        return false;
+
+    return true;
+}
+
+bool ConditionalContextInfo::usedOnMultiplePaths() const
+{
+    if (extraParents.ordinality() != 0)
+        return true;
+    if (firstParent)
+        return firstParent->usedOnMultiplePaths();
+    return false;
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+
+ConditionalContextTransformer::ConditionalContextTransformer(HqlTransformerInfo & info, bool _alwaysEvaluateGuardedTogether)
+: ConditionalHqlTransformer(info, CTFnoteor|CTFnoteand|CTFnotemap|CTFnoteifall),
+    alwaysEvaluateGuardedTogether(_alwaysEvaluateGuardedTogether)
+{
+    seq = 0;
+    hasConditionalCandidate = false;
+    noteManyUnconditionalParents = false;//true;
+    createRootGraph = false;
+    rootExpr.setown(createValue(no_null, makeVoidType(), createAttribute(_root_Atom)));
+    activeParent = NULL;  // cannot call queryBodyExtra(rootExpr) since in constructor
+}
+
+
+ANewTransformInfo * ConditionalContextTransformer::createTransformInfo(IHqlExpression * expr)
+{
+    return CREATE_NEWTRANSFORMINFO(ConditionalContextInfo, expr);
+}
+
+
+void ConditionalContextTransformer::analyseExpr(IHqlExpression * expr)
+{
+    switch (pass)
+    {
+    case PassFindConditions:
+        ConditionalHqlTransformer::analyseExpr(expr);
+        return;
+    case PassFindParents:
+        analyseConditionalParents(expr);
+        return;
+    case PassGatherGuards:
+        analyseGatherGuards(expr);
+        break;
+    default:
+        ConditionalHqlTransformer::analyseExpr(expr);
+        break;
+    }
+}
+
+bool ConditionalContextTransformer::hasSingleConditionalCandidate() const
+{
+    if (candidates.ordinality() != 1)
+        return false;
+    ConditionalContextInfo & candidate = candidates.item(0);
+    if (candidate.isUnconditional())
+        return false;
+    return !candidate.usedOnMultiplePaths();
+}
+
+
+// ---- pass 1 --------
+
+void ConditionalContextTransformer::noteCandidate(ConditionalContextInfo * extra)
+{
+    extra->isCandidateExpr = true;
+    //MORE: If is used unconditionally enough???? then mark as such.  (don't mark as unconditional)
+    if (!extra->isUnconditional())
+        hasConditionalCandidate = true;
+    candidates.append(*LINK(extra));
+}
+
+
+void ConditionalContextTransformer::analyseConditionalParents(IHqlExpression * expr)
+{
+    ConditionalContextInfo * extra = queryBodyExtra(expr);
+    if (extra->seq)
+    {
+        extra->addExtraParent(activeParent, noteManyUnconditionalParents);
+        return;
+    }
+
+    if (!extra->firstAnnotatedExpr && (expr != expr->queryBody()))
+        extra->firstAnnotatedExpr = expr;
+
+    extra->setFirstParent(activeParent);
+    extra->seq = ++seq;
+
+    {
+        ConditionalContextInfo * savedParent = activeParent;
+        activeParent = extra;
+        ConditionalHqlTransformer::analyseExpr(expr);
+        activeParent = savedParent;
+    }
+}
+
+// ---- pass 1b --------
+
+void ConditionalContextTransformer::addDefinition(ConditionalContextInfo * location, ConditionalContextInfo * candidate)
+{
+    location->definitions.append(*LINK(candidate->original));
+    candidate->moveTo = location;
+
+    if (!insertLocations.contains(*location))
+        insertLocations.append(*location);
+}
+
+void ConditionalContextTransformer::removeDefinition(ConditionalContextInfo * location, ConditionalContextInfo * candidate)
+{
+    location->definitions.zap(*LINK(candidate->original));
+    candidate->moveTo = NULL;
+}
+
+bool ConditionalContextTransformer::findCommonLocations()
+{
+    bool changed = false;
+    ForEachItemIn(idx, candidates)
+    {
+        ConditionalContextInfo& cur = candidates.item(idx);
+        ConditionalContextInfo * candidateLocation = findCandidateLocation(&cur);
+        //A null alias location forces an expression to stay where it is
+        if (candidateLocation)
+        {
+            addDefinition(candidateLocation, &cur);
+            changed = true;
+        }
+    }
+    return changed;
+}
+
+//Find a single location to evaluate all the child items in
+bool ConditionalContextTransformer::findSingleCommonLocation()
+{
+    ConditionalContextInfo * bestLocation = NULL;
+    PointerArrayOf<ConditionalContextInfo> matched;
+    ForEachItemIn(idx, candidates)
+    {
+        ConditionalContextInfo& cur = candidates.item(idx);
+        ConditionalContextInfo * candidateLocation = findCandidateLocation(&cur);
+        if (candidateLocation)
+        {
+            matched.append(&cur);
+            if (!bestLocation)
+                bestLocation = candidateLocation;
+            else
+                bestLocation = findCommonPath(bestLocation, candidateLocation);
+        }
+    }
+
+    while (bestLocation)
+    {
+        if (canSurroundWithAlias(bestLocation->original))
+            break;
+        bestLocation = selectParent(bestLocation);
+    }
+
+    //If best location is NULL then all expressions are marked as staying where they are.
+    if (!bestLocation)
+        return false;
+
+    ForEachItemIn(i, matched)
+        addDefinition(bestLocation, matched.item(i));
+    return true;
+}
+
+
+//Find a single location to evaluate all the child items in
+bool ConditionalContextTransformer::associateCandidatesWithRoot()
+{
+    ConditionalContextInfo * rootLocation = queryBodyExtra(rootExpr);
+    PointerArrayOf<ConditionalContextInfo> matched;
+    ForEachItemIn(idx, candidates)
+    {
+        ConditionalContextInfo& cur = candidates.item(idx);
+        ConditionalContextInfo * candidateLocation = findCandidateLocation(&cur);
+        if (candidateLocation)
+            addDefinition(rootLocation, &cur);
+    }
+
+    return rootLocation->hasDefinitions();
+}
+
+
+//What location in the tree should extra be evaluated?
+//If it is unconditional, then if first use is unconditional, eval here, else in the root.
+//If this expression is conditional and has a single use,
+//   then evaluate here or globally - if always done globally.
+//if conditional, and multiple uses, then find the common evaluation condition of the parents.
+ConditionalContextInfo * ConditionalContextTransformer::calcCommonLocation(ConditionalContextInfo * extra)
+{
+    if (extra->calcedCommonLocation)
+        return extra->commonLocation;
+
+    if (!extra->firstParent)
+    {
+        dbgassertex(extra == queryBodyExtra(rootExpr));
+        return extra;
+    }
+
+    ConditionalContextInfo * commonLocation = extra;
+    if (extra->isUnconditional() && !alwaysEvaluateGuardedTogether)
+    {
+        if (!extra->isFirstUseUnconditional())
+            commonLocation = queryBodyExtra(rootExpr);
+    }
+    else
+    {
+        commonLocation = calcCommonLocation(extra->firstParent);
+        if (extra->hasSharedParent)
+        {
+            ForEachItemIn(i, extra->extraParents)
+            {
+                ConditionalContextInfo * curParent = extra->extraParents.item(i);
+                ConditionalContextInfo * nextExtra = calcCommonLocation(curParent);
+
+                //MORE: What should be done if some expressions can be guarded, and others are marked as unmovable?
+                commonLocation = findCommonPath(commonLocation, nextExtra);
+                if (!commonLocation)
+                    break;
+            }
+        }
+        else
+        {
+            if (commonLocation == extra->firstParent)
+                commonLocation = extra;
+        }
+
+        //MORE commonLocation == NULL if you never want to move this - e.g., if parent is a choose/which/rejected/....
+    }
+    extra->calcedCommonLocation = true;
+    extra->commonLocation = commonLocation;
+    return commonLocation;
+}
+
+ConditionalContextInfo * ConditionalContextTransformer::findCandidateLocation(ConditionalContextInfo * extra)
+{
+    ConditionalContextInfo * best = calcCommonLocation(extra);
+    loop
+    {
+        if (!best)
+            return NULL;
+        IHqlExpression * bestLocation = best->original;
+        if (canSurroundWithAlias(bestLocation))
+            return best;
+        best = selectParent(best);
+    }
+}
+
+
+ConditionalContextInfo * ConditionalContextTransformer::selectParent(ConditionalContextInfo * info)
+{
+    if (info->hasSharedParent)
+        return calcCommonLocation(info);
+    return info->firstParent;
+}
+
+ConditionalContextInfo * ConditionalContextTransformer::findCommonPath(ConditionalContextInfo * left, ConditionalContextInfo * right)
+{
+    loop
+    {
+        if (!left || !right)
+            return NULL;
+        if (left == right)
+            return left;
+
+        //Ensure if there is a child and a parent that the child's parent is selected
+        if (left->seq > right->seq)
+            left = selectParent(left);
+        else
+            right = selectParent(right);
+    }
+}
+
+
+// ---- pass 2 --------
+
+//Recursively walk the tree calculating guards, and gathering a list of guards for child expressions
+//Must be called after the common locations have been calculated
+void ConditionalContextTransformer::analyseGatherGuards(IHqlExpression * expr)
+{
+    ConditionalContextInfo * extra = queryBodyExtra(expr);
+    if (!alreadyVisited(extra))
+    {
+        unsigned prevGuards = childGuards.ordinality();
+
+        doAnalyseExpr(expr);
+
+        extra->guards.setown(calcGuard(extra, prevGuards));
+        extra->calcInheritedGuards();
+        childGuards.trunc(prevGuards);
+    }
+
+    if (extra->inheritedGuards)
+        childGuards.append(*extra->inheritedGuards);
+}
+
+
+//Calculate guards for all candidates - even those that will be defined here.
+CHqlExprMultiGuard * ConditionalContextTransformer::calcGuard(ConditionalContextInfo * cur, unsigned firstGuard)
+{
+    bool needToAddCandidate = cur->isCandidateThatMoves();
+    unsigned maxGuard = childGuards.ordinality();
+    if ((firstGuard == maxGuard) && !needToAddCandidate)
+        return NULL;
+
+    IHqlExpression * original = cur->original;
+    node_operator op = original->getOperator();
+    switch (op)
+    {
+    case no_if:
+        return createIfGuard(cur);
+    case no_case:
+    case no_map:
+        return createCaseMapGuard(cur, op);
+    case no_or:
+    case no_and:
+        return createAndOrGuard(cur);
+    case no_which:
+    case no_rejected:
+    case no_choose:
+        //MORE!!!
+        break;
+    }
+
+    //Get the union of all guards present for the child expressions
+    if ((firstGuard == maxGuard-1) && !needToAddCandidate)
+        return LINK(&childGuards.item(firstGuard));
+
+    Owned<CHqlExprMultiGuard> newGuard = new CHqlExprMultiGuard;
+    for (unsigned i=firstGuard; i < maxGuard; i++)
+        newGuard->combine(childGuards.item(i));
+    if (needToAddCandidate)
+        newGuard->addGuarded(original);
+    return newGuard.getClear();
+}
+
+CHqlExprMultiGuard * ConditionalContextTransformer::createIfGuard(IHqlExpression * ifCond, CHqlExprMultiGuard * condGuard, CHqlExprMultiGuard * trueGuard, CHqlExprMultiGuard * falseGuard)
+{
+    //If you want to common up the conditions between the child query and the parent code you might achieve
+    //it by forcing the condition into an alias.  E.g.,
+    //queryBodyExtra(ifCond)->createAlias = true;
+
+    Owned<CHqlExprMultiGuard> newGuards = new CHqlExprMultiGuard;
+    HqlExprCopyArray candidates;
+    gatherCandidates(candidates, trueGuard);
+    gatherCandidates(candidates, falseGuard);
+    ForEachItemIn(i, candidates)
+    {
+        IHqlExpression * candidate = &candidates.item(i);
+        IHqlExpression * trueCond = queryGuardCondition(trueGuard, candidate);
+        IHqlExpression * falseCond = queryGuardCondition(falseGuard, candidate);
+        OwnedHqlExpr newCond;
+        if (trueCond != falseCond)
+            newCond.setown(createValue(no_if, makeBoolType(), LINK(ifCond), LINK(trueCond), LINK(falseCond)));
+        else
+            newCond.set(trueCond);
+        newGuards->addGuarded(newCond, candidate, condGuard != NULL);
+    }
+
+    if (condGuard)
+        newGuards->combine(*condGuard);
+    return newGuards.getClear();
+}
+
+CHqlExprMultiGuard * ConditionalContextTransformer::createIfGuard(ConditionalContextInfo * cur)
+{
+    IHqlExpression * original = cur->original;
+    IHqlExpression * ifCond = original->queryChild(0);
+    CHqlExprMultiGuard * condGuard = queryGuards(ifCond);
+    CHqlExprMultiGuard * trueGuard = queryGuards(original->queryChild(1));
+    CHqlExprMultiGuard * falseGuard = queryGuards(original->queryChild(2));
+    if (!trueGuard && !falseGuard && !cur->isCandidateThatMoves())
+        return LINK(condGuard);
+
+    Owned<CHqlExprMultiGuard> newGuards = createIfGuard(ifCond, condGuard, trueGuard, falseGuard);
+    if (cur->isCandidateThatMoves())
+        newGuards->addGuarded(original);
+    return newGuards.getClear();
+}
+
+CHqlExprMultiGuard * ConditionalContextTransformer::createCaseMapGuard(ConditionalContextInfo * cur, node_operator op)
+{
+    IHqlExpression * original = cur->original;
+    IHqlExpression * testExpr = (op == no_case) ? original->queryChild(0) : NULL;
+
+    //MAP(k1=>v1,k2=>v2...,vn) is the same as IF(k1,v1,IF(k2,v2,...vn))
+    //So walk the MAP operator in reverse applying the guard conditions.
+    //Use queryLastNonAttribute() instead of max-1 to eventually cope with attributes
+    IHqlExpression * defaultExpr = queryLastNonAttribute(original);
+    Linked<CHqlExprMultiGuard> prevGuard = queryGuards(defaultExpr);
+    bool createdNewGuard = false;
+    unsigned first = (op == no_case) ? 1 : 0;
+    unsigned max = original->numChildren();
+    for (unsigned i= max-1; i-- != first; )
+    {
+        IHqlExpression * mapto = original->queryChild(i);
+        //In the unlikely event there are attributes this ensures only maps are processed
+        if (mapto->getOperator() != no_mapto)
+            continue;
+
+        IHqlExpression * testValue = mapto->queryChild(0);
+        CHqlExprMultiGuard * condGuard = queryGuards(testValue);
+        CHqlExprMultiGuard * trueGuard = queryGuards(mapto->queryChild(1));
+        if (trueGuard || prevGuard)
+        {
+            OwnedHqlExpr ifCond = testExpr ? createBoolExpr(no_eq, LINK(testExpr), LINK(testValue)) : LINK(testValue);
+            Owned<CHqlExprMultiGuard> newGuards = createIfGuard(ifCond, condGuard, trueGuard, prevGuard);
+            prevGuard.setown(newGuards.getClear());
+            createdNewGuard = true;
+        }
+        else
+        {
+            prevGuard.set(condGuard);
+            assertex(!createdNewGuard);
+        }
+    }
+
+    CHqlExprMultiGuard * testGuards = testExpr ? queryGuards(testExpr) : NULL;
+    if (testGuards || cur->isCandidateThatMoves())
+    {
+        if (!createdNewGuard)
+            prevGuard.setown(new CHqlExprMultiGuard(prevGuard));
+
+        if (testGuards)
+            prevGuard->combine(*testGuards);
+
+        if (cur->isCandidateThatMoves())
+            prevGuard->addGuarded(original);
+    }
+    return prevGuard.getClear();
+}
+
+CHqlExprMultiGuard * ConditionalContextTransformer::createAndOrGuard(ConditionalContextInfo * cur)
+{
+    IHqlExpression * original = cur->original;
+    IHqlExpression * left = original->queryChild(0);
+    CHqlExprMultiGuard * leftGuard = queryGuards(left);
+    CHqlExprMultiGuard * rightGuard = queryGuards(original->queryChild(1));
+    if (!rightGuard && !cur->isCandidateThatMoves())
+        return LINK(leftGuard);
+
+    node_operator op = original->getOperator();
+    Owned<CHqlExprMultiGuard> newGuards = new CHqlExprMultiGuard;
+    ForEachItemIn(i, rightGuard->guarded)
+    {
+        CHqlExprGuard & cur = rightGuard->guarded.item(i);
+        OwnedHqlExpr cond;
+        //(a || b):  b is evaluated if a is false.  => guard'(b,x) = !a && guard(b,x)
+        //(a && b):  b is evaluated if a is true.   => guard'(b,x) = a && guard(b,x)
+        if (op == no_or)
+            cond.setown(createBoolExpr(no_and, getInverse(left), LINK(cur.guard)));
+        else
+            cond.setown(createBoolExpr(no_and, LINK(left), LINK(cur.guard)));
+        newGuards->addGuarded(cond, cur.original, leftGuard != NULL);
+    }
+
+    //MORE: Is the reversal of the order going to matter?  E.g., instead of guard(a,x) || (a && guard(b,x))?
+    if (leftGuard)
+        newGuards->combine(*leftGuard);
+    if (cur->isCandidateThatMoves())
+        newGuards->addGuarded(cur->original);
+    return newGuards.getClear();
+}
+
+CHqlExprMultiGuard * ConditionalContextTransformer::queryGuards(IHqlExpression * expr)
+{
+    if (!expr)
+        return NULL;
+    return queryBodyExtra(expr)->guards;
+}
+
+// ---- pass 3 --------
+
+
+// ---- summary --------
+
+bool ConditionalContextTransformer::hasUnconditionalCandidate() const
+{
+    ForEachItemIn(i, candidates)
+    {
+        if (candidates.item(i).isUnconditional())
+            return true;
+    }
+    return false;
+}
+
+bool ConditionalContextTransformer::analyseNeedsTransform(const HqlExprArray & exprs)
+{
+    ConditionalContextInfo * rootInfo = queryBodyExtra(rootExpr);
+
+    activeParent = rootInfo;
+    rootInfo->setFirstUnconditional();
+
+    // Spot conditional and unconditional expressions
+    analyseArray(exprs, PassFindConditions);
+    analyseArray(exprs, PassFindCandidates);
+    if (candidates.ordinality() == 0)
+        return false;
+
+    // Tag expressions with their parent locations, and gather a list of candidates.
+    analyseArray(exprs, PassFindParents);
+
+    // Work out the locations conditional expressions need to be evaluated in
+    if (createRootGraph)
+        associateCandidatesWithRoot();
+    else if (!alwaysEvaluateGuardedTogether)
+        findCommonLocations();
+    else
+        findSingleCommonLocation();
+
+    if (hasConditionalCandidate)
+    {
+        // Calculate the guard conditions for each condition
+        analyseArray(exprs, PassGatherGuards);
+        rootInfo->guards.setown(calcGuard(rootInfo, 0));
+        childGuards.kill();
+
+        // Any guard conditions that are always true should become unconditional if context is unconditional
+        analyseArray(exprs, 3);
+    }
+
+    return hasConditionalCandidate || alwaysEvaluateGuardedTogether;
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+static IHqlExpression * createConditionalExpr(IHqlExpression * condition, IHqlExpression * expr, IHqlExpression * elseExpr)
+{
+    if (matchesBoolean(condition, true))
+        return LINK(expr);
+    if (matchesBoolean(condition, false))
+        return LINK(elseExpr);
+
+    //MORE: This code is currently disabled.
+    //The intent is to ensure that in the condition IF (a || b) that b is only evaluated if a is false.
+    //However there are complications and problems with nested guard conditions which need independently solving.
+    if (false)
+    {
+        switch (condition->getOperator())
+        {
+        case no_and:
+            if (expr->queryRecord())
+            {
+                //Use dataset if operators to ensure that the conditions are sortcircuited.
+                // if (a && b, c, -) => if (a, if (b, c, -), -)
+                OwnedHqlExpr second = createConditionalExpr(condition->queryChild(1), expr, elseExpr);
+                return createConditionalExpr(condition->queryChild(0), second, elseExpr);
+            }
+            break;
+        case no_or:
+            if (expr->queryRecord())
+            {
+                // if (a || b, c, -) => if (a, c, if (b, c, -))
+                OwnedHqlExpr second = createConditionalExpr(condition->queryChild(1), expr, elseExpr);
+                return createConditionalExpr(condition->queryChild(0), expr, second);
+            }
+            break;
+        case no_if:
+            {
+                OwnedHqlExpr trueExpr = createConditionalExpr(condition->queryChild(1), expr, elseExpr);
+                OwnedHqlExpr falseExpr = createConditionalExpr(condition->queryChild(2), expr, elseExpr);
+                return createIf(LINK(condition), trueExpr.getClear(), falseExpr.getClear());
+            }
+        }
+    }
+
+    return createIf(LINK(condition), LINK(expr), LINK(elseExpr));
+}
+
+IHqlExpression * ConditionalContextTransformer::getGuardCondition(ConditionalContextInfo * extra, IHqlExpression * expr)
+{
+    CHqlExprMultiGuard * guards = extra->guards;
+    if (guards)
+    {
+        if (expr != extra->original)
+            return foldHqlExpression(guards->queryGuardCondition(expr));
+    }
+    return LINK(queryBoolExpr(true));
+}
+
+IHqlExpression * ConditionalContextTransformer::createGuardedDefinition(ConditionalContextInfo * extra, IHqlExpression * expr)
+{
+    OwnedHqlExpr guard = getGuardCondition(extra, expr);
+    assertex(guard);
+    return createGuardedDefinition(extra, expr, guard);
+}
+
+IHqlExpression * ConditionalContextTransformer::createGuardedDefinition(ConditionalContextInfo * extra, IHqlExpression * expr, IHqlExpression * condition)
+{
+    if (matchesBoolean(condition, true))
+        return LINK(expr);
+
+    OwnedHqlExpr guard = transform(condition);
+    OwnedHqlExpr defaultExpr = createNullExpr(expr);
+    return createConditionalExpr(guard, expr, defaultExpr);
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+IHqlExpression * ConditionalContextTransformer::createTransformed(IHqlExpression * expr)
+{
+    OwnedHqlExpr transformed = ConditionalHqlTransformer::createTransformed(expr);
+    updateOrphanedSelectors(transformed, expr);
+    switch (transformed->getOperator())
+    {
+    case no_compound_indexread:
+    case no_compound_indexcount:
+    case no_compound_indexaggregate:
+    case no_compound_indexgroupaggregate:
+    case no_compound_indexnormalize:
+        if (!queryPhysicalRootTable(transformed))
+            return LINK(transformed->queryChild(0));
+        break;
+    }
+
+    if (queryBodyExtra(expr)->createAlias)
+    {
+        if (expr == expr->queryBody(true))
+            transformed.setown(createAlias(transformed, NULL));
+    }
+    return transformed.getClear();
+}
+
+void ConditionalContextTransformer::transformAllCandidates()
+{
+    //Process definitions in the order they were found - since this is a toplogical dependency ordering
+    ForEachItemIn(i, candidates)
+        transformCandidate(&candidates.item(i));
+}
+
+void ConditionalContextTransformer::transformAll(HqlExprArray & exprs, bool forceRootFirst)
+{
+    transformAllCandidates();
+    OwnedHqlExpr rootDefinitions = createDefinitions(queryBodyExtra(rootExpr));
+    HqlExprArray transformed;
+    if (rootDefinitions && forceRootFirst)
+        transformed.append(*rootDefinitions.getClear());
+
+    ForEachItemIn(i, exprs)
+    {
+        IHqlExpression * cur = &exprs.item(i);
+        OwnedHqlExpr mapped = transformRoot(cur);
+
+        //Add the child query etc. before the first expression that changes.
+        if (rootDefinitions && (mapped != cur))
+            transformed.append(*rootDefinitions.getClear());
+
+        transformed.append(*mapped.getClear());
+    }
+    assertex(!rootDefinitions);
+    exprs.swapWith(transformed);
+}

ecl/hqlcpp/hqlhoist.hpp

@@ -0,0 +1,200 @@
+/*##############################################################################
+
+    Copyright (C) 2012 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+#ifndef __HQLHOIST_HPP_
+#define __HQLHOIST_HPP_
+
+#include "hqltrans.ipp"
+
+bool canSurroundWithAlias(IHqlExpression * expr);
+
+//This class represents the guard/protection condition for a single candiate expression
+class CHqlExprGuard : public CInterface
+{
+public:
+    CHqlExprGuard(IHqlExpression * _guard, IHqlExpression * _original, bool _guardContainsCandidate)
+        : guard(_guard), original(_original), guardContainsCandidate(_guardContainsCandidate)
+    {
+    }
+
+    bool guardContainsCandidate;
+    LinkedHqlExpr guard;
+    LinkedHqlExpr original;
+};
+
+//This class represents a list of the guards for each of the guarded expressions used within the owning expression.
+class CHqlExprMultiGuard : public CInterface
+{
+public:
+    CHqlExprMultiGuard() {}
+    CHqlExprMultiGuard(const CHqlExprMultiGuard * other)
+    {
+        if (other)
+        {
+            CloneArray(guarded, other->guarded);
+        }
+    }
+
+    void addGuarded(IHqlExpression * original);
+    void addGuarded(IHqlExpression * cond, IHqlExpression * original, bool guardContainsCandidate);
+
+    void combine(CHqlExprMultiGuard & other);
+    void combine(CHqlExprGuard & other);
+    void gatherCandidates(HqlExprCopyArray & candidiates) const;
+    bool guardContainsCandidate(IHqlExpression * original) const;
+    IHqlExpression * queryGuardCondition(IHqlExpression * original) const;
+
+public:
+    CIArrayOf<CHqlExprGuard> guarded;
+};
+
+//---------------------------------------------------------------------------------------------------------------------
+
+class ConditionalContextInfo : public ConditionalTransformInfo
+{
+public:
+    ConditionalContextInfo(IHqlExpression * expr) : ConditionalTransformInfo(expr)
+    {
+        firstParent = NULL;
+        commonLocation = NULL;
+        moveTo = NULL;
+        firstAnnotatedExpr = NULL;
+        seq = 0;
+        hasSharedParent = false;
+        calcedCommonLocation = false;
+        isCandidateExpr = false;
+        createAlias = false;
+    }
+
+    void addExtraParent(ConditionalContextInfo * nextParent, bool noteManyUnconditionalParents)
+    {
+        hasSharedParent = true;
+        //We only care about parents of conditional expressions
+        if (!isUnconditional() || noteManyUnconditionalParents || canSurroundWithAlias(original))
+        {
+            extraParents.append(nextParent);
+        }
+    }
+
+    void calcInheritedGuards();
+    bool isCandidateThatMoves() const;
+    bool usedOnMultiplePaths() const;
+
+    inline bool hasDefinitions() const { return definitions.ordinality() != 0; }
+    inline bool changesLocation() const { return moveTo != this; }
+    inline void setFirstParent(ConditionalContextInfo * parent) { firstParent = parent; }
+
+public:
+    // use a pointer for the first match to avoid reallocation for unshared items
+    ConditionalContextInfo * firstParent;  // NULL for the root item
+    // if this expression is conditional, then which expression contains all uses of it?
+    ConditionalContextInfo * commonLocation;
+    // where will the definition be moved to.  NULL if it will not be moved.
+    ConditionalContextInfo * moveTo;
+    // First annotated expression seen (if any)
+    IHqlExpression * firstAnnotatedExpr;
+    //Which guarded expressions need to be created at this point?
+    HqlExprArray definitions;
+    //If conditional, a list of all the other places it is used.
+    PointerArrayOf<ConditionalContextInfo> extraParents;
+    // guards for all expressions that this expression contains
+    Owned<CHqlExprMultiGuard> guards;
+    //All guards that need to be passed on the container/parent expressions [excludes definitions]
+    Owned<CHqlExprMultiGuard> inheritedGuards;
+    //A sequence number use to ensure it is efficient to find a common parent
+    unsigned seq;
+    bool hasSharedParent;
+    bool calcedCommonLocation;
+    //Is this a candidate for evaluating in a single location with guards?
+    bool isCandidateExpr;
+    bool createAlias;
+};
+
+//---------------------------------------------------------------------------------------------------------------------
+
+class ConditionalContextTransformer : public ConditionalHqlTransformer
+{
+public:
+    enum { PassFindConditions,
+           PassFindCandidates,
+           PassFindParents,
+           PassGatherGuards,
+    };
+
+public:
+    virtual void analyseExpr(IHqlExpression * expr);
+            bool findSingleCommonLocation();
+            bool findCommonLocations();
+            bool associateCandidatesWithRoot();
+
+    bool analyseNeedsTransform(const HqlExprArray & exprs);
+    bool hasSingleConditionalCandidate() const;
+    void transformAll(HqlExprArray & exprs, bool forceRootFirst);
+
+protected:
+    virtual void transformCandidate(ConditionalContextInfo * candidate) = 0;
+    ConditionalContextTransformer(HqlTransformerInfo & info, bool _alwaysEvaluateGuardedTogether);
+    virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr);
+
+    bool hasUnconditionalCandidate() const;
+
+    void analyseConditionalParents(IHqlExpression * expr);
+    void analyseGatherGuards(IHqlExpression * expr);
+
+    ConditionalContextInfo * calcCommonLocation(ConditionalContextInfo * extra);
+    ConditionalContextInfo * findCandidateLocation(ConditionalContextInfo * extra);
+    ConditionalContextInfo * findCommonPath(ConditionalContextInfo * left, ConditionalContextInfo * right);
+    ConditionalContextInfo * selectParent(ConditionalContextInfo * info);
+
+    CHqlExprMultiGuard * calcGuard(ConditionalContextInfo * cur, unsigned firstGuard);
+    CHqlExprMultiGuard * createAndOrGuard(ConditionalContextInfo * cur);
+    CHqlExprMultiGuard * createCaseMapGuard(ConditionalContextInfo * cur, node_operator op);
+    CHqlExprMultiGuard * createIfGuard(ConditionalContextInfo * cur);
+    CHqlExprMultiGuard * createIfGuard(IHqlExpression * ifCond, CHqlExprMultiGuard * condGuard, CHqlExprMultiGuard * trueGuard, CHqlExprMultiGuard * falseGuard);
+    CHqlExprMultiGuard * queryGuards(IHqlExpression * expr);
+
+    void addDefinition(ConditionalContextInfo * location, ConditionalContextInfo * candidate);
+    void removeDefinition(ConditionalContextInfo * location, ConditionalContextInfo * candidate);
+    virtual IHqlExpression * createDefinitions(ConditionalContextInfo * extra) = 0;
+
+    inline ConditionalContextInfo * queryBodyExtra(IHqlExpression * expr) { return (ConditionalContextInfo*)NewHqlTransformer::queryTransformExtra(expr->queryBody()); }
+    void noteCandidate(ConditionalContextInfo * extra);
+    inline void noteCandidate(IHqlExpression * expr) { noteCandidate(queryBodyExtra(expr)); }
+
+    IHqlExpression * getGuardCondition(ConditionalContextInfo * extra, IHqlExpression * expr);
+    IHqlExpression * createGuardedDefinition(ConditionalContextInfo * extra, IHqlExpression * expr);
+    IHqlExpression * createGuardedDefinition(ConditionalContextInfo * extra, IHqlExpression * expr, IHqlExpression * condition);
+
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr);
+    void transformAllCandidates();
+
+protected:
+    CIArrayOf<ConditionalContextInfo> candidates;
+    ConditionalContextInfo * activeParent;
+    OwnedHqlExpr rootExpr;
+    CopyCIArrayOf<CHqlExprMultiGuard> childGuards;
+    CopyCIArrayOf<ConditionalContextInfo> insertLocations;
+    unsigned seq;
+    bool alwaysEvaluateGuardedTogether;
+    bool noteManyUnconditionalParents;
+    bool hasConditionalCandidate;
+    bool createRootGraph;
+};
+
+IHqlExpression * mapExternalToInternalResults(IHqlExpression * expr, IHqlExpression * graph);
+
+#endif

ecl/hqlcpp/hqlhtcpp.cpp

@@ -55,6 +55,8 @@
 #include "deffield.hpp"
 #include "hqlinline.hpp"
 #include "hqlusage.hpp"
+#include "hqlhoist.hpp"
+#include "hqlcppds.hpp"
 
 //The following are include to ensure they call compile...
 #include "eclhelper.hpp"
@@ -134,6 +136,25 @@ static void markSubGraphAsRoot(IPropertyTree * tree)
         addGraphAttributeBool(tree, "rootGraph", true);
 }
 
+SubGraphInfo * matchActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag)
+{
+    FilteredAssociationIterator iter(ctx, AssocSubGraph);
+    ForEach(iter)
+    {
+        SubGraphInfo & cur = static_cast<SubGraphInfo &>(iter.get());
+        if (graphTag == cur.graphTag)
+            return &cur;
+    }
+    return NULL;
+}
+
+
+bool isActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag)
+{
+    return matchActiveGraph(ctx, graphTag) != NULL;
+}
+
+
 //---------------------------------------------------------------------------
 
 class InternalResultTracker : public CInterface
@@ -338,81 +359,36 @@ MemberFunction::~MemberFunction()
 
 //---------------------------------------------------------------------------
 
-class ChildSpotterInfo : public HoistingTransformInfo
-{
-public:
-    ChildSpotterInfo(IHqlExpression * _original) : HoistingTransformInfo(_original) { spareByte2 = false; }
-    
-    inline bool getHoist() { return spareByte2 != 0; }
-    inline void setHoist() { spareByte2 = true; }
-
-private:
-    using HoistingTransformInfo::spareByte2;                //prevent derived classes from also using this spare byte
-};
-
-
 static HqlTransformerInfo childDatasetSpotterInfo("ChildDatasetSpotter");
-class ChildDatasetSpotter : public HoistingHqlTransformer
+class NewChildDatasetSpotter : public ConditionalContextTransformer
 {
 public:
-    ChildDatasetSpotter(HqlCppTranslator & _translator, BuildCtx & _ctx) 
-        : HoistingHqlTransformer(childDatasetSpotterInfo, HTFnoteallconditionals), translator(_translator), ctx(_ctx) 
-    { 
-        candidate = false; 
-    }
-
-    virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr)
+    NewChildDatasetSpotter(HqlCppTranslator & _translator, BuildCtx & _ctx, bool _forceRoot)
+        : ConditionalContextTransformer(childDatasetSpotterInfo, true), translator(_translator), ctx(_ctx), forceRoot(_forceRoot)
     {
-        return CREATE_NEWTRANSFORMINFO(ChildSpotterInfo, expr);
+        //The following line forces the conditionalContextTransformer code to generate a single root subgraph.
+        //An alternative would be to generate one (or more) graphs at the first unconditional place they are
+        //used.  e.g., adding a no_compound(no_childquery, f(no_getgraphresult)) into the tree.
+        //This was the initial approach, but it causes problems for subsequent optimizations -
+        //if an optimzation causes an expression containing the no_getgraphresult to be hoisted so it is
+        //evaluated before the no_childquery it creates an out-of-order dependency.
+        createRootGraph = true;
     }
-    inline ChildSpotterInfo * queryBodyExtra(IHqlExpression * expr)     { return static_cast<ChildSpotterInfo *>(queryTransformExtra(expr->queryBody())); }
 
     virtual void analyseExpr(IHqlExpression * expr)
     {
-        if (pass == 0)
-        {
-            if (!analyseThis(expr))
-                return;
-            switch (expr->getOperator())
-            {
-            case no_if:
-            case no_and:
-            case no_which:
-            case no_rejected:
-            case no_or:
-            case no_map:
-                {
-                    analyseExpr(expr->queryChild(0));
-                    conditionDepth++;
-                    ForEachChildFrom(i, expr, 1)
-                        analyseExpr(expr->queryChild(i));
-                    conditionDepth--;
-                    break;
-                }
-            case no_choose:
-            case no_case:
-                {
-                    analyseExpr(expr->queryChild(0));
-                    analyseExpr(expr->queryChild(1));
-                    conditionDepth++;
-                    ForEachChildFrom(i, expr, 2)
-                        analyseExpr(expr->queryChild(i));
-                    conditionDepth--;
-                    break;
-                }
-            default:
-                doAnalyseExpr(expr);
-                break;
-            }
-        }
-        else
+        switch (pass)
         {
+        case PassFindCandidates:
             if (!alreadyVisited(expr->queryBody()))
                 markHoistPoints(expr);
+            break;
+        default:
+            ConditionalContextTransformer::analyseExpr(expr);
+            break;
         }
     }
 
-
     //MORE: This is a bit of a hack, and should be improved (share code with resource child hoist?)
     inline bool walkFurtherDownTree(IHqlExpression * expr)
     {
@@ -437,27 +413,13 @@ public:
     void markHoistPoints(IHqlExpression * expr)
     {
         node_operator op = expr->getOperator();
-        switch (op)
-        {
-        case no_transform:
-        case no_assign:
-        case no_assignall:
-            break;
-        default:
-            if (!containsNonActiveDataset(expr))
-                return;
-            break;
-        }
-
         if (expr->isDataset() || (expr->isDatarow() && (op != no_select)))
         {
-            if (isUsedUnconditionallyEnough(expr) && !translator.canAssignInline(&ctx, expr))
+            if (!translator.canAssignInline(&ctx, expr))
             {
-                candidate = true;
-                queryBodyExtra(expr)->setHoist();
+                noteCandidate(expr);
                 return;
             }
-
             if (!walkFurtherDownTree(expr))
                 return;
         }
@@ -465,6 +427,102 @@ public:
         doAnalyseExpr(expr);
     }
 
+    IHqlExpression * createTransformed(IHqlExpression * expr)
+    {
+        IHqlExpression * body = expr->queryBody(true);
+        if (expr != body)
+            return createTransformedAnnotation(expr);
+
+        ConditionalContextInfo * extra = queryBodyExtra(expr);
+
+        //The following must preceed transforming the children
+        OwnedHqlExpr subgraph = createDefinitions(extra);
+
+        OwnedHqlExpr transformed = ConditionalContextTransformer::createTransformed(expr);
+        updateOrphanedSelectors(transformed, expr);
+
+        assertex(!extra->moveTo);
+
+        if (subgraph)
+            return createCompound(subgraph.getClear(), transformed.getClear());
+        return transformed.getClear();
+    }
+
+    virtual void transformCandidate(ConditionalContextInfo * candidate)
+    {
+        while (builders.ordinality() < insertLocations.ordinality())
+            builders.append(* new ChildGraphExprBuilder(0));
+
+        IHqlExpression * expr = candidate->original;
+        ConditionalContextInfo * moveTo = candidate->moveTo;
+        OwnedHqlExpr guard;
+        if (moveTo)
+        {
+            guard.setown(getGuardCondition(moveTo, expr));
+
+            //Expressions which are very simple functions of unconditional expressions are treated as if they
+            //are unconditional
+            if (moveTo->isUnconditional() && isUsedUnconditionallyEnough(expr))
+                guard.set(queryBoolExpr(true));
+
+            bool invalid = !guard->isPure();
+            //version 1: don't guard any child queries.
+            if (!matchesBoolean(guard, true))
+            {
+                assertex(moveTo->guards);
+                //MORE: For the moment disable any expressions that are only used conditionally.
+                //Often including conditions improves the code, but sometimes the duplicate evaluation of the
+                //guard conditions in the parent and the child causes excessive code generation.
+                //And forcing it into an alias doesn't help becuase that isn't currently executed in the parent.
+                //Uncomment: if (moveTo->guards->guardContainsCandidate(expr))
+                {
+                    invalid = true;
+                }
+            }
+
+            if (invalid)
+            {
+                removeDefinition(moveTo, candidate);
+                moveTo = NULL;
+            }
+        }
+
+        //A candidate inside a condition that prevents it being moved just creates a definition where it is.
+        if (!moveTo)
+            return;
+
+        ChildGraphExprBuilder & builder = queryBuilder(moveTo);
+        IHqlExpression * annotated = candidate->firstAnnotatedExpr ? candidate->firstAnnotatedExpr : expr;
+        OwnedHqlExpr guarded = createGuardedDefinition(moveTo, annotated, guard);
+        OwnedHqlExpr transformed = builder.addDataset(guarded);
+
+        if (moveTo == candidate)
+        {
+            OwnedHqlExpr subgraph = createDefinitions(moveTo);
+            transformed.setown(createCompound(subgraph.getClear(), transformed.getClear()));
+        }
+
+        setTransformed(expr, transformed);
+    }
+
+    virtual IHqlExpression * createDefinitions(ConditionalContextInfo * extra)
+    {
+        if (!extra->hasDefinitions())
+            return NULL;
+
+        ChildGraphExprBuilder & builder = queryBuilder(extra);
+        OwnedHqlExpr graph = builder.getGraph();
+        OwnedHqlExpr cleanedGraph = mapExternalToInternalResults(graph, builder.queryRepresents());
+        return cleanedGraph.getClear();
+    }
+
+    ChildGraphExprBuilder & queryBuilder(ConditionalContextInfo * extra)
+    {
+        unsigned match = insertLocations.find(*extra);
+        assertex(match != NotFound);
+        return builders.item(match);
+    }
+
     inline bool isUsedUnconditionallyEnough(IHqlExpression * expr)
     {
         IHqlExpression * search = expr;
@@ -489,47 +547,11 @@ public:
         }
     }
 
-
-    IHqlExpression * createTransformed(IHqlExpression * expr)
-    {
-        OwnedHqlExpr transformed = HoistingHqlTransformer::createTransformed(expr);
-        updateOrphanedSelectors(transformed, expr);
-
-        if ((expr->isDataset() || expr->isDatarow()) && (expr->getOperator() != no_select))
-        {
-            if (queryBodyExtra(expr)->getHoist() && !translator.canAssignInline(&ctx, transformed))
-            {
-                if (!builder)
-                    builder.setown(new ChildGraphBuilder(translator));
-                return builder->addDataset(expr);
-            }
-        }
-
-        //Very occasionally an index read being hoisted means that an index aggregate is now an aggregate
-        //on a hoisted result.
-        switch (transformed->getOperator())
-        {
-            case no_compound_indexcount:
-            case no_compound_indexaggregate:
-                if (!queryPhysicalRootTable(transformed))
-                    transformed.set(transformed->queryChild(0));
-                break;
-        }
-
-        return transformed.getClear();
-    }
-
-    virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr) { return LINK(expr); }
-
-    bool hasCandidate() const { return candidate; }
-    ChildGraphBuilder * getBuilder() { return builder.getClear(); };
-
-
 protected:
+    CIArrayOf<ChildGraphExprBuilder> builders;
     HqlCppTranslator & translator;
-    Owned<ChildGraphBuilder> builder;
     BuildCtx & ctx;
-    bool candidate;
+    bool forceRoot;
 };
 
 
@@ -667,27 +689,35 @@ public:
         pending.append(*LINK(stmt));
     }
 
+    void processStmts(IHqlExpression * expr)
+    {
+        expr->unwindList(pending, no_actionlist);
+    }
+
     void clear()
     {
-        builder.clear();
         pending.kill();
         processed = false;
     }
 
-    bool hasGraphPending()
+    IHqlExpression * getPrefetchGraph()
     {
-        spotChildDatasets();
-        return builder != NULL;
+        spotChildDatasets(true);
+        if (pending.ordinality() == 0)
+            return NULL;
+        IHqlExpression & subquery = pending.item(0);
+        if (subquery.getOperator() == no_childquery)
+        {
+            pending.remove(0, true);
+            return &subquery;
+        }
+
+        return NULL;
     }
 
     void flush(BuildCtx & ctx)
     {
-        spotChildDatasets();
-        if (builder)
-        {
-            builder->generateGraph(ctx);
-            builder.clear();
-        }
+        spotChildDatasets(false);
         combineConditions();
         optimizeAssigns();
         ForEachItemIn(i, pending)
@@ -695,10 +725,18 @@ public:
         pending.kill();
     }
 
-    void generatePrefetch(BuildCtx & ctx, OwnedHqlExpr * retGraphExpr, OwnedHqlExpr * retResultsExpr)
+    void optimize()
     {
-        builder->generatePrefetchGraph(ctx, retGraphExpr, retResultsExpr);
-        builder.clear();
+        spotChildDatasets(false);
+        combineConditions();
+        optimizeAssigns();
+    }
+
+    IHqlExpression * getActionList()
+    {
+        OwnedHqlExpr ret = createActionList(pending);
+        pending.kill();
+        return ret.getClear();
     }
 
 protected:
@@ -708,39 +746,49 @@ protected:
         pending.combineConditions();
     }
 
-    void spotChildDatasets()
+    void spotChildDatasets(bool forceRoot)
     {
         if (!processed && translator.queryCommonUpChildGraphs())
         {
-            ChildDatasetSpotter spotter(translator, buildctx);
-            for (unsigned pass=0; pass < 2; pass++)
+            HqlExprArray analyseExprs;
+            ForEachItemIn(i, pending)
             {
-                ForEachItemIn(i, pending)
+                IHqlExpression & cur = pending.item(i);
+                IHqlExpression * value = &cur;
+                switch (value->getOperator())
                 {
-                    IHqlExpression & cur = pending.item(i);
-                    IHqlExpression * value = &cur;
-                    switch (value->getOperator())
-                    {
-                    case no_assign:
-                        value = value->queryChild(1);
-                        break;
-                    case no_alias:
-                    case no_skip:
-                        value = value->queryChild(0);
-                        break;
-                    }
-                    if (value)
-                        spotter.analyse(value, pass);
+                case no_assign:
+                    value = value->queryChild(1);
+                    break;
+                case no_alias:
+                case no_skip:
+                    value = value->queryChild(0);
+                    break;
                 }
+                if (value)
+                    analyseExprs.append(*LINK(value));
             }
 
-            if (spotter.hasCandidate())
+            NewChildDatasetSpotter spotter(translator, buildctx, forceRoot);
+            if (spotter.analyseNeedsTransform(analyseExprs))
             {
-                HqlExprArray replacement;
-                ForEachItemIn(i, pending)
-                    replacement.append(*spotter.transformRoot(&pending.item(i)));
-                replaceArray(pending, replacement);
-                builder.setown(spotter.getBuilder());
+                //This could be conditional on whether or not there is an unconditional candidate, but that would stop
+                //the same expression being commoned up between two conditional branches.
+                //So, only avoid if 1 conditional candidate used in a single location.
+                bool worthHoisting = true;
+                if (!forceRoot)
+                {
+                    if (spotter.hasSingleConditionalCandidate())
+                        worthHoisting = false;
+                }
+
+                if (worthHoisting)
+                {
+                    //MORE: Remove || true to evaluate the subquery at the latest time.
+                    bool createSubQueryBeforeAll = forceRoot || true;
+                    spotter.transformAll(pending, createSubQueryBeforeAll);
+                    translator.traceExpressions("spotted child", pending);
+                }
             }
             processed = true;
         }
@@ -753,10 +801,24 @@ protected:
     HqlCppTranslator & translator;
     BuildCtx buildctx;
     StatementCollection pending;
-    Owned<ChildGraphBuilder> builder;
     bool processed;
 };
 
+void HqlCppTranslator::optimizeBuildActionList(BuildCtx & ctx, IHqlExpression * exprs)
+{
+    if ((exprs->getOperator() != no_actionlist) || !graph)
+    {
+        buildStmt(ctx, exprs);
+        return;
+    }
+
+    DelayedStatementExecutor delayed(*this, ctx);
+
+    delayed.processStmts(exprs);
+    delayed.flush(ctx);
+}
+
+
 //---------------------------------------------------------------------------
 
 static IHqlExpression * getExtractMatchingAssign(HqlExprArray & assigns, IHqlExpression * search, unsigned & expectedIndex, IHqlExpression * selfSelector)
@@ -2592,6 +2654,12 @@ AColumnInfo * DatasetSelector::resolveField(IHqlExpression * search) const
 
 void DatasetSelector::set(BuildCtx & ctx, IHqlExpression * expr)
 {
+    while (expr->getOperator() == no_compound)
+    {
+        translator.buildStmt(ctx, expr->queryChild(0));
+        expr = expr->queryChild(1);
+    }
+
     assertex(row->isModifyable());
     ITypeInfo * type = column->queryType();
     if (!hasReferenceModifier(type))
@@ -7890,8 +7958,6 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySpill(BuildCtx & ctx, IHqlExpr
 }
 
 
-//---------------------------------------------------------------------------
-
 bool HqlCppTranslator::isCurrentActiveGraph(BuildCtx & ctx, IHqlExpression * graphTag)
 {
     SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
@@ -7900,6 +7966,82 @@ bool HqlCppTranslator::isCurrentActiveGraph(BuildCtx & ctx, IHqlExpression * gra
 }
 
 
+IHqlExpression * HqlCppTranslator::createLoopSubquery(IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * filter, IHqlExpression * again, IHqlExpression * counter, bool multiInstance, unsigned & loopAgainResult)
+{
+    //Now need to generate the body of the loop.
+    //output dataset is result 0
+    //input dataset is fed in using result 1
+    //counter (if required) is fed in using <result-2>[0].counter;
+    ChildGraphExprBuilder graphBuilder(2);
+    IHqlExpression * graphid = graphBuilder.queryRepresents();
+
+    LinkedHqlExpr transformedBody = body;
+    LinkedHqlExpr transformedAgain = again;
+
+    //Result 1 is the input dataset.
+    HqlExprArray args;
+    args.append(*LINK(dataset->queryRecord()));
+    args.append(*LINK(graphid));
+    args.append(*getSizetConstant(1));
+    if (isGrouped(dataset))
+        args.append(*createAttribute(groupedAtom));
+    args.append(*createAttribute(_loop_Atom));
+    if (multiInstance)
+        args.append(*createAttribute(_streaming_Atom));
+    if (targetThor())    // MORE: && !isChildQuery(ctx)..
+        args.append(*createAttribute(_distributed_Atom));
+    OwnedHqlExpr inputResult= createDataset(no_getgraphresult, args);
+
+    //Result 2 is the counter - if present
+    OwnedHqlExpr counterResult;
+    if (counter)
+    {
+        OwnedHqlExpr select = createCounterAsGraphResult(counter, graphid, 2);
+        transformedBody.setown(replaceExpression(transformedBody, counter, select));
+        if (transformedAgain)
+        {
+            //The COUNTER for the global termination condition is whether to execute iteration COUNTER, 1=1st iter
+            //Since we're evaluating the condition in the previous iteration it needs to be increased by 1.
+            OwnedHqlExpr nextCounter = adjustValue(select, 1);
+            transformedAgain.setown(replaceExpression(transformedAgain, counter, nextCounter));
+        }
+
+        graphBuilder.addInput();
+    }
+
+    //first create the result...
+    //Need to replace ROWS(LEFT) with the result1
+    OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
+    OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(left), LINK(rowsid));
+    transformedBody.setown(replaceExpression(transformedBody, rowsExpr, inputResult));
+
+    OwnedHqlExpr result = createValue(no_setgraphresult, makeVoidType(), LINK(transformedBody), LINK(graphid), getSizetConstant(0), createAttribute(_loop_Atom));
+    graphBuilder.addAction(result);
+
+    if (transformedAgain)
+    {
+        LinkedHqlExpr nextLoopDataset = transformedBody;
+        if (filter)
+        {
+            //If there is a loop filter then the global condition is applied to dataset filtered by that.
+            OwnedHqlExpr mappedFilter = replaceSelector(filter, left, nextLoopDataset);
+            nextLoopDataset.setown(createDataset(no_filter, nextLoopDataset.getClear(), LINK(mappedFilter)));
+        }
+        transformedAgain.setown(replaceExpression(transformedAgain, rowsExpr, nextLoopDataset));
+
+        loopAgainResult = graphBuilder.addInput();
+
+        //MORE: Add loopAgainResult as an attribute on the no_childquery rather than using a reference parameter
+        OwnedHqlExpr againResult = convertScalarToGraphResult(transformedAgain, queryBoolType(), graphid, loopAgainResult);
+        graphBuilder.addAction(againResult);
+
+    }
+
+    return graphBuilder.getGraph();
+}
+
+
+
 ABoundActivity * HqlCppTranslator::doBuildActivityLoop(BuildCtx & ctx, IHqlExpression * expr)
 {
     IHqlExpression * dataset = expr->queryChild(0);
@@ -8009,14 +8151,13 @@ ABoundActivity * HqlCppTranslator::doBuildActivityLoop(BuildCtx & ctx, IHqlExpre
     subctx.addQuotedCompound("virtual void createParentExtract(rtlRowBuilder & builder)");
 
     //Now need to generate the body of the loop.
-    //output dataset is result 0
-    //input dataset is fed in using result 1
-    //counter (if required) is fed in using result 2[0].counter;
-    ChildGraphBuilder builder(*this);
-    unique_id_t loopId = builder.buildLoopBody(subctx, dataset, selSeq, rowsid, body->queryChild(0), filter, loopCond, counter, instance->activityId, (parallel != NULL));
+    unsigned loopAgainResult = 0;
+    OwnedHqlExpr childquery = createLoopSubquery(dataset, selSeq, rowsid, body->queryChild(0), filter, loopCond, counter, (parallel != NULL), loopAgainResult);
+
+    ChildGraphBuilder builder(*this, childquery);
+    unique_id_t loopId = builder.buildLoopBody(subctx, (parallel != NULL));
     instance->addAttributeInt("_loopid", loopId);
 
-    unsigned loopAgainResult = builder.queryLoopConditionResult();
     if (loopAgainResult)
         doBuildUnsignedFunction(instance->startctx, "loopAgainResult", loopAgainResult);
 
@@ -8066,7 +8207,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGraphLoop(BuildCtx & ctx, IHql
     //output dataset is result 0
     //input dataset is fed in using result 1
     //counter (if required) is fed in using result 2[0].counter;
-    unique_id_t loopId = buildGraphLoopSubgraph(subctx, dataset, selSeq, rowsid, body->queryChild(0), counter, instance->activityId, (parallel != NULL));
+    unique_id_t loopId = buildGraphLoopSubgraph(subctx, dataset, selSeq, rowsid, body->queryChild(0), counter, (parallel != NULL));
     instance->addAttributeInt("_loopid", loopId);
 
     buildInstanceSuffix(instance);
@@ -8113,7 +8254,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityRemote(BuildCtx & ctx, IHqlExp
     subctx.addQuotedCompound("virtual void createParentExtract(rtlRowBuilder & builder)");
 
     //output dataset is result 0
-    unique_id_t remoteId = buildRemoteSubgraph(subctx, dataset, instance->activityId);
+    unique_id_t remoteId = buildRemoteSubgraph(subctx, dataset);
     
     instance->addAttributeInt("_graphid", remoteId);
 
@@ -8306,9 +8447,11 @@ public:
     void flush(BuildCtx & ctx)
     {
         if (builder)
-            builder->generateGraph(ctx);
-
-        builder.clear();
+        {
+            OwnedHqlExpr childquery = builder->getGraph();
+            translator.buildStmt(ctx, childquery);
+            builder.clear();
+        }
     }
 
     bool requiresGraph(BuildCtx & ctx, IHqlExpression * expr)
@@ -8339,8 +8482,8 @@ public:
     void addGraphStmt(BuildCtx & ctx, IHqlExpression * expr)
     {
         if (!builder)
-            builder.setown(new ChildGraphBuilder(translator));
-        builder->buildStmt(ctx, expr);
+            builder.setown(new ChildGraphExprBuilder(0));
+        builder->addAction(expr);
     }
 
     void buildStmt(BuildCtx & ctx, IHqlExpression * expr)
@@ -8382,7 +8525,7 @@ public:
 
 protected:
     HqlCppTranslator & translator;
-    Owned<ChildGraphBuilder> builder;
+    Owned<ChildGraphExprBuilder> builder;
 };
 
 ABoundActivity * HqlCppTranslator::doBuildActivityApply(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
@@ -8509,20 +8652,25 @@ unsigned HqlCppTranslator::doBuildThorChildSubGraph(BuildCtx & ctx, IHqlExpressi
 {
 //NB: Need to create the graph in the correct order so the references to property trees we retain
 // remain live..
+    unsigned thisId = reservedId ? reservedId : nextActivityId();
+    unsigned graphId = thisId;
     SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
     IPropertyTree * node = createPTree("node");
     if (activeSubgraph)
+    {
         node = activeSubgraph->tree->addPropTree("node", node);
+        if (activeSubgraph->graphTag == graphTag)
+            graphId = activeSubgraph->graphId;
+    }
     else
         node = graph->addPropTree("node", node);
 
-    unsigned thisId = reservedId ? reservedId : nextActivityId();
     node->setPropInt("@id", thisId);
 
     IPropertyTree * graphAttr = node->addPropTree("att", createPTree("att"));
     IPropertyTree * subGraph = graphAttr->addPropTree("graph", createPTree("graph"));
 
-    Owned<SubGraphInfo> graphInfo = new SubGraphInfo(subGraph, thisId, graphTag, kind);
+    Owned<SubGraphInfo> graphInfo = new SubGraphInfo(subGraph, thisId, graphId, graphTag, kind);
     ctx.associate(*graphInfo);
 
     IHqlExpression * numResultsAttr = expr->queryProperty(numResultsAtom);
@@ -8580,7 +8728,7 @@ unsigned HqlCppTranslator::doBuildThorSubGraph(BuildCtx & ctx, IHqlExpression *
     if (!graphTag && outputLibraryId)
         graphTag = outputLibraryId;
     if (needToCreateGraph)
-        beginGraph(graphctx);
+        beginGraph();
 
     unsigned thisId = doBuildThorChildSubGraph(graphctx, expr, kind, reservedId, graphTag);
 
@@ -8591,14 +8739,10 @@ unsigned HqlCppTranslator::doBuildThorSubGraph(BuildCtx & ctx, IHqlExpression *
 }
 
 
-void HqlCppTranslator::beginGraph(BuildCtx & ctx, const char * _graphName)
+void HqlCppTranslator::beginGraph(const char * _graphName)
 {
     if (activeGraphName)
-    {
-        //buildStmt(ctx, expr->queryChild(0));
-        //return;
         throwError(HQLERR_NestedThorNodes);
-    }
 
     graphSeqNumber++;
     StringBuffer graphName;
@@ -8785,7 +8929,7 @@ void HqlCppTranslator::doBuildThorGraph(BuildCtx & ctx, IHqlExpression * expr)
         buildLibraryGraph(ctx, expr, NULL);
     else
     {
-        beginGraph(ctx);
+        beginGraph();
 
         unsigned id = 0;
         OwnedHqlExpr graphTag = NULL;//WIP:createAttribute(graphAtom, createConstant((__int64)id));
@@ -8799,7 +8943,7 @@ void HqlCppTranslator::doBuildThorGraph(BuildCtx & ctx, IHqlExpression * expr)
             Owned<SubGraphInfo> graphInfo;
             if (graphTag)
             {
-                graphInfo.setown(new SubGraphInfo(graph, 0, graphTag, SubGraphRoot));
+                graphInfo.setown(new SubGraphInfo(graph, 0, 0, graphTag, SubGraphRoot));
                 graphctx.associate(*graphInfo);
             }
 
@@ -11593,12 +11737,22 @@ ABoundActivity * HqlCppTranslator::doBuildActivityIterate(BuildCtx & ctx, IHqlEx
 
 IHqlExpression * HqlCppTranslator::queryExpandAliasScope(BuildCtx & ctx, IHqlExpression * expr)
 {
-    while (expr->getOperator() == no_alias_scope)
+    loop
     {
-        expandAliasScope(ctx, expr);
-        expr = expr->queryChild(0);
+        switch (expr->getOperator())
+        {
+        case no_alias_scope:
+            expandAliasScope(ctx, expr);
+            expr = expr->queryChild(0);
+            break;
+        case no_compound:
+            buildStmt(ctx, expr->queryChild(0));
+            expr = expr->queryChild(1);
+            break;
+        default:
+            return expr;
+        }
     }
-    return expr;
 }
 
 
@@ -13917,7 +14071,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityPrefetchProject(BuildCtx & ctx
         TransformBuilder builder(*this, postctx, record, selfCursor, assigns);
         filterExpandAssignments(postctx, &builder, assigns, transform);
         builder.buildTransformChildren(postctx, record, selfCursor->querySelector());
-        if (builder.hasGraphPending())
+        OwnedHqlExpr subgraph = builder.getPrefetchGraph();
+        if (subgraph)
         {
             //Generate the extract preparation function
             BuildCtx prectx(instance->startctx);
@@ -13931,23 +14086,20 @@ ABoundActivity * HqlCppTranslator::doBuildActivityPrefetchProject(BuildCtx & ctx
             if (counter)
                 associateCounter(prectx, counter, "counter");
         
-            OwnedHqlExpr graph, results;
-            builder.generatePrefetch(prectx, &graph, &results);
+            OwnedHqlExpr graphInstance;
+            ChildGraphBuilder graphBuilder(*this, subgraph);
+            graphBuilder.generatePrefetchGraph(prectx, &graphInstance);
             prectx.addReturn(queryBoolExpr(true));
 
             BuildCtx childctx(instance->startctx);
             childctx.addQuotedCompound("virtual IThorChildGraph *queryChild()");
-            childctx.addReturn(graph);
+            childctx.addReturn(graphInstance);
 
-            //hack!  I need to change the way results are bound so they aren't nesc. the same as the query name
-            StringBuffer s;
-            s.append("IEclGraphResults * ");
-            generateExprCpp(s, results);
-            s.append(" = results;");
-            postctx.addQuoted(s);
-            
             //Add an association for the results into the transform function.
-            postctx.associateExpr(results, results);
+            IHqlExpression * graph = subgraph->queryChild(0);
+            OwnedHqlExpr results = createAttribute(resultsAtom, LINK(graph));
+            OwnedHqlExpr resultsInstanceExpr = createQuoted("results", makeBoolType());
+            postctx.associateExpr(results, resultsInstanceExpr);
         }
         builder.flush(postctx);
     }

ecl/hqlcpp/hqlinline.cpp

@@ -342,6 +342,8 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
                 return 0;
             return RETassign|((childLFlags|childRFlags) & HEFspillinline);
         }
+    case no_compound:
+        return getInlineFlags(ctx, expr->queryChild(1));
     default:
         return 0;
     }
@@ -1095,6 +1097,9 @@ void ParentExtract::gatherActiveRows(BuildCtx & ctx)
             default:
                 if (cur.isResultAlias())
                     ok = false;
+
+                //MORE: This should only be done if the child query etc. actually references the datarow.
+                //ideally from a colocal activity.
 #if 0
             //Theoretically the following is true.  However it can mean that for the cost of serializing an extra 4 bytes here and
             //there you end up serializing several hundred in some other situations.

ecl/hqlcpp/hqllib.cpp

@@ -613,7 +613,7 @@ void HqlCppTranslator::buildLibraryGraph(BuildCtx & ctx, IHqlExpression * expr,
 {
     OwnedHqlExpr resourced = getResourcedGraph(expr->queryChild(0), NULL);
 
-    beginGraph(ctx, graphName);
+    beginGraph(graphName);
 
     traceExpression("beforeGenerate", resourced);
     BuildCtx initctx(ctx);

ecl/hqlcpp/hqlresource.cpp

@@ -2691,6 +2691,8 @@ bool EclResourcer::findSplitPoints(IHqlExpression * expr)
                 locator.analyseChild(expr->queryChild(0), true);
                 break;
             }
+        case no_childquery:
+            throwUnexpected();
         default:
             {
                 for (unsigned idx=first; idx < last; idx++)

ecl/hqlcpp/hqlstep.cpp

@@ -46,6 +46,7 @@
 #include "hqlgraph.ipp"
 #include "hqlscope.hpp"
 #include "hqlccommon.hpp"
+#include "hqlcppds.hpp"
 
 #include "eclhelper.hpp"
 
@@ -762,7 +763,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityRowsetRange(BuildCtx & ctx, IH
     if ((kind == TAKnwaygraphloopresultread) && isGrouped(rowset))
         doBuildBoolFunction(instance->classctx, "grouped", true);
     if (graphId && targetRoxie())
-        instance->addAttributeInt("_graphId", getIntValue(graphId->queryChild(0)));
+        addGraphIdAttribute(instance, ctx, graphId);
 
     buildInstanceSuffix(instance);
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -1025,7 +1025,7 @@ void ThorScalarTransformer::setTransformedSelector(IHqlExpression * expr, IHqlEx
 
 
 static HqlTransformerInfo ThorScalarTransformerInfo("ThorScalarTransformer");
-ThorScalarTransformer::ThorScalarTransformer(const HqlCppOptions & _options) : HoistingHqlTransformer(ThorScalarTransformerInfo, HTFnoteconditionalactions), options(_options)
+ThorScalarTransformer::ThorScalarTransformer(const HqlCppOptions & _options) : HoistingHqlTransformer(ThorScalarTransformerInfo, CTFnoteifactions), options(_options)
 {
     isConditionalDepth = 0;
     seenCandidate = false;
@@ -6794,7 +6794,7 @@ inline bool isTypeToHoist(ITypeInfo * type)
 
 static HqlTransformerInfo scalarGlobalTransformerInfo("ScalarGlobalTransformer");
 ScalarGlobalTransformer::ScalarGlobalTransformer(HqlCppTranslator & _translator)
-: HoistingHqlTransformer(scalarGlobalTransformerInfo, HTFtraverseallnodes), translator(_translator)
+: HoistingHqlTransformer(scalarGlobalTransformerInfo, CTFtraverseallnodes), translator(_translator)
 {
     okToHoist = true;
     neverHoist = false;
@@ -7000,7 +7000,7 @@ IHqlExpression * ScalarGlobalTransformer::createTransformed(IHqlExpression * exp
 
 static HqlTransformerInfo explicitGlobalTransformerInfo("ExplicitGlobalTransformer");
 ExplicitGlobalTransformer::ExplicitGlobalTransformer(IWorkUnit * _wu, HqlCppTranslator & _translator)
-: HoistingHqlTransformer(explicitGlobalTransformerInfo, HTFnoteconditionalactions|HTFtraverseallnodes), translator(_translator)
+: HoistingHqlTransformer(explicitGlobalTransformerInfo, CTFnoteifactions|CTFtraverseallnodes), translator(_translator)
 {
     wu = _wu;
     isRoxie = (translator.getTargetClusterType() == RoxieCluster);
@@ -7140,7 +7140,7 @@ NewScopeMigrateTransformer::NewScopeMigrateTransformer(IWorkUnit * _wu, HqlCppTr
     wu = _wu;
     isRoxie = translator.targetRoxie();
     if (!isRoxie && !_translator.queryOptions().resourceConditionalActions)
-        setFlags(HTFnoteconditionalactions);
+        setFlags(CTFnoteifactions);
     minimizeWorkunitTemporaries = translator.queryOptions().minimizeWorkunitTemporaries;
 #ifdef REMOVE_GLOBAL_ANNOTATION
     activityDepth = 0;      // should be 0 to actually have any effect - but causes problems...
@@ -9547,7 +9547,7 @@ bool containsCompound(IHqlExpression * expr)
 
 static HqlTransformerInfo nestedCompoundTransformerInfo("NestedCompoundTransformer");
 NestedCompoundTransformer::NestedCompoundTransformer(HqlCppTranslator & _translator)
-: HoistingHqlTransformer(nestedCompoundTransformerInfo, HTFnoteconditionalactions), translator(_translator), translatorOptions(_translator.queryOptions())
+: HoistingHqlTransformer(nestedCompoundTransformerInfo, CTFnoteifactions), translator(_translator), translatorOptions(_translator.queryOptions())
 {
 }
 

ecl/regress/condchild1.ecl

@@ -0,0 +1,113 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//both uses are unconditional
+inRec t1(inRec l) := TRANSFORM
+    SELF.f1 := outofline(l, 1);
+    SELF.f2 := outofline(l, 1);
+    SELF := l;
+END;
+
+//One use is conditional, the other is unconditional
+inRec t2(inRec l) := TRANSFORM
+    SELF.f1 := outofline(l, 1);
+    SELF.f2 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF := l;
+END;
+
+//Conditional comes first
+inRec t3(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := outofline(l, 1);
+    SELF := l;
+END;
+
+
+//Both uses are conditional - on the same condition
+inRec t4(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := IF(l.f1 > 2, outofline(l, 1), 13);
+    SELF := l;
+END;
+
+//Both uses are conditional - different conditions
+inRec t5(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := IF(l.f1 > 3, outofline(l, 1), 13);
+    SELF := l;
+END;
+
+//Both uses are conditional - same condition, but not inside a statement
+inRec t6(inRec l) := TRANSFORM
+    SELF.f1 := 5 + IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := 10 + IF(l.f1 > 3, outofline(l, 1), 13);
+    SELF := l;
+END;
+
+//Both uses are conditional - inside same and different conditions
+inRec t7(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f2 > 8, IF(l.f1 > 2, outofline(l, 1), 12), 9);
+    SELF.f2 := IF(l.f2 > 8, IF(l.f1 > 3, outofline(l, 1), 13), 10);
+    SELF := l;
+END;
+
+
+//Both uses are conditional - on the same condition, different out of lines
+inRec t8(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 2, outofline(l, 1), 12);
+    SELF.f2 := IF(l.f1 > 2, outofline(l, 2), 13);
+    SELF := l;
+END;
+
+#if (0)
+    output(project(nofold(input), t7(LEFT)));
+#else
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+    output(project(nofold(input), t2(LEFT)));
+    output(project(nofold(input), t3(LEFT)));
+    output(project(nofold(input), t4(LEFT)));
+    output(project(nofold(input), t5(LEFT)));
+    output(project(nofold(input), t6(LEFT)));
+    output(project(nofold(input), t7(LEFT)));
+    output(project(nofold(input), t8(LEFT)));
+);
+
+#end

ecl/regress/condchild2.ecl

@@ -0,0 +1,96 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//conditional - but unconditional
+inRec t1(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 10, 2 * outofline(l, 1), 3 * outofline(l, 1));
+    SELF := l;
+END;
+
+//conditional - but unconditional (from two different branches)
+inRec t2(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 > 10, 2 * outofline(l, 1), 10);
+    SELF.f2 := IF(l.f1 > 10, 20, 3 * outofline(l, 1));
+    SELF := l;
+END;
+
+//Guarded with a condition that is outofline
+inRec t3(inRec l) := TRANSFORM
+    test1 := IF(l.f1 > 2, outofline(l, 1), 12) != 3;
+    SELF.f1 := IF(test1, 2 * outofline(l, 2), 10);
+    SELF := l;
+END;
+
+
+//Guarded with a conditions that are ANDed outofline
+inRec t4(inRec l) := TRANSFORM
+    test1 := IF(l.f1 > 2, outofline(l, 1), 12) != 3;
+    test2 := IF(l.f2 > 2, outofline(l, 2), 12) != 5;
+    SELF.f1 := IF(test1 and test2, outofline(l, 3), 12);
+    SELF := l;
+END;
+
+//Guarded with a conditions that are ORed outofline
+inRec t5(inRec l) := TRANSFORM
+    test1 := IF(l.f1 > 2, outofline(l, 1), 12) != 3;
+    test2 := IF(l.f2 > 2, outofline(l, 2), 12) != 5;
+    SELF.f1 := IF(test1 or test2, outofline(l, 3), 12);
+    SELF := l;
+END;
+
+//Guarded with a conditions that are IFed outofline
+inRec t6(inRec l) := TRANSFORM
+    test1 := IF(l.f1 > 2, outofline(l, 1), outofline(l, 2)) != 3;
+    SELF.f1 := IF(test1, outofline(l, 3), 12);
+    SELF := l;
+END;
+
+#if (0)
+    output(project(nofold(input), t3(LEFT)));
+#else
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+    output(project(nofold(input), t2(LEFT)));
+    output(project(nofold(input), t3(LEFT)));
+    output(project(nofold(input), t4(LEFT)));
+    output(project(nofold(input), t5(LEFT)));
+    output(project(nofold(input), t6(LEFT)));
+);
+#end

ecl/regress/condchild3.ecl

@@ -0,0 +1,95 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//used in AND conditions => second use is guarded
+inRec t1(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    SELF.f1 := IF(t1 AND t2, 10, 12);
+    SELF := l;
+END;
+
+//used in OR conditions => second use is guarded
+inRec t2(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    SELF.f1 := IF(t1 OR t2, 10, 12);
+    SELF := l;
+END;
+
+//used in OR conditions => second use is guarded
+inRec t3(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    t3 := outofline(l, 3) < 8;
+    SELF.f1 := IF((t1 AND t2) OR t3, 10, 12);
+    SELF := l;
+END;
+
+//used in WHICH conditions => second use is guarded
+inRec t4(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    t3 := outofline(l, 3) < 8;
+    SELF.f1 := WHICH(t1, t2, t2);
+    SELF := l;
+END;
+
+//used in WHICH conditions => second use is guarded
+inRec t5(inRec l) := TRANSFORM
+    t1 := outofline(l, 1) > 10;
+    t2 := outofline(l, 2) < 12;
+    t3 := outofline(l, 3) < 8;
+    SELF.f1 := REJECTED(t1, t2, t2);
+    SELF := l;
+END;
+
+//MORE: CHOOSE
+//MORE: MAP/CASE
+//MORE: Some in REJECTED, some conditional
+//MORE: Some in REJECTED, some unconditional
+
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+    output(project(nofold(input), t2(LEFT)));
+    output(project(nofold(input), t3(LEFT)));
+    output(project(nofold(input), t4(LEFT)));
+    output(project(nofold(input), t5(LEFT)));
+);

ecl/regress/condchild4.ecl

@@ -0,0 +1,113 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//CHOOSE
+inRec t1(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    SELF.f1 := CHOOSE(l.f1, t1, t2, 0);
+    SELF := l;
+END;
+
+//CASE
+inRec t2(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1, 12=>t2, 27=>t3, t4);
+    SELF := l;
+END;
+
+//CASE, also in case conditions
+inRec t3(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1, t2=>t3, t4);
+    SELF := l;
+END;
+
+//CASE, value used in all results
+inRec t4(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1*5, t2=>t1*6, t3=>t1*8, t1*10);
+    SELF := l;
+END;
+
+//use of t1s actually unconditional (tricky to spot?)
+inRec t5(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1, t1*12=>t2, 27=>t3, t4);
+    SELF := l;
+END;
+
+//Some case uses are also conditional, some unconditional
+inRec t6(inRec l) := TRANSFORM
+    t1 := outofline(l, 1);
+    t2 := outofline(l, 2);
+    t3 := outofline(l, 3);
+    t4 := outofline(l, 4);
+    SELF.f1 := CASE(l.f1, 5=>t1, 12=>t2, 27=>t3, t4);
+    SELF.f2 := t1 * IF(l.f2> 10, t2, t3);
+    SELF := l;
+END;
+
+//make cond5 same as this, but using map.
+//MORE: CHOOSE
+//MORE: MAP/CASE
+//MORE: Some in REJECTED, some conditional
+//MORE: Some in REJECTED, some unconditional
+
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+    output(project(nofold(input), t2(LEFT)));
+    output(project(nofold(input), t3(LEFT)));
+    output(project(nofold(input), t4(LEFT)));
+    output(project(nofold(input), t5(LEFT)));
+    output(project(nofold(input), t6(LEFT)));
+);

ecl/regress/condchild5.ecl

@@ -0,0 +1,59 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//guarded candidate is also used in the guard condition
+inRec t1(inRec l) := TRANSFORM
+    SELF.f1 := IF(l.f1 = 10 or outofline(l, 1) = 2, 10, outofline(l, 1));
+    SELF := l;
+END;
+
+//it is impossible to guard b and c since they are mutually dependent
+inRec t2(inRec l) := TRANSFORM
+    a := outofline(l, 1) = 1;
+    b := outofline(l, 2) = 1;
+    c := outofline(l, 3) = 1;
+    d := outofline(l, 4) = 1;
+    SELF.f1 := IF((a OR b OR c) AND (a OR c or b), outofline(l, 5), outofline(l, 6));
+    SELF := l;
+END;
+
+SEQUENTIAL(
+    output(project(nofold(input), t2(LEFT)));
+);

ecl/regress/condchild6.ecl

@@ -0,0 +1,52 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+idRec := { UNSIGNED id; };
+
+inRec := RECORD
+    UNSIGNED seq;
+    DATASET(idRec) ids;
+    UNSIGNED f1;
+    UNSIGNED f2;
+END;
+
+input := DATASET([
+       {1, [1,2,2,3,3,3,4,4,4,4,5,5,5,5], 1, 2},
+       {2, [5,4,4,3,3,3,2,2,2,2,1,1,1,1], 2, 3}
+       ], inRec);
+
+outOfLine(inRec l, unsigned x) := FUNCTION
+    filtered := l.ids(id != x);
+    d := dedup(filtered, id);
+    RETURN COUNT(d);
+END;
+
+
+//complex condition on two branches
+inRec t1(inRec l) := TRANSFORM
+    cond1 := NOFOLD(CHOOSE(sum(l.ids,id), 'a', 'b', 'c', 'd', 'z', 'e')) != 'd';
+    cond2 := NOFOLD(CHOOSE(max(l.ids,id), 'a', 'b', 'c', 'd', 'z', 'e')) != 'd';
+    SELF.f1 := IF(cond1, outofline(l, 1), 1000);
+    SELF.f2 := IF(cond2, outofline(l, 1), 1001);
+    SELF := l;
+END;
+
+SEQUENTIAL(
+    output(project(nofold(input), t1(LEFT)));
+);