Initial work on implicit alias creation.

* USE TABLE(dataset) to introduce a dataset alias
* Fix problems with using dataset aliases in various contexts (e.g,
  inline aggregates)
* Fix the implicit alias creator to generate much better code.
  - the option is currently defaulted off
* IF(cond, ds(x), table(ds)(y)) now optimized to
  table(ds)(if(cond, x, y));
  (generating same code as if implicit aliases are disabled)

Some related changes

* Clean up the childdataset types
* Improve generation of splitters inside subgraphs

Previously a disk read of a spill file would often be duplicated
inside a subgraph that read it, and sibling subgraphs which read the
same spill files were not combined.  This change ensures a splitter
is now generated in those situations.

A splitter is only added if it would be balanced - otherwise the
potential spill in the splitter would outweigh any benefit.

* Splitters of splitter are now combined into a single splitter.  (Also
improves existing queries.)

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlatoms.cpp

@@ -218,6 +218,7 @@ _ATOM noCaseAtom;
 _ATOM _noHoist_Atom;
 _ATOM noLocalAtom;
 _ATOM noOverwriteAtom;
+_ATOM _normalized_Atom;
 _ATOM noRootAtom;
 _ATOM noScanAtom;
 _ATOM noSortAtom;
@@ -594,6 +595,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKESYSATOM(noHoist);
     MAKEATOM(noLocal);
     MAKEATOM(noOverwrite);
+    MAKESYSATOM(normalized);
     MAKEATOM(noRoot);
     MAKEATOM(noScan);
     MAKEATOM(noSort);

ecl/hql/hqlatoms.hpp

@@ -222,6 +222,7 @@ extern HQL_API _ATOM noCaseAtom;
 extern HQL_API _ATOM _noHoist_Atom;
 extern HQL_API _ATOM noLocalAtom;
 extern HQL_API _ATOM noOverwriteAtom;
+extern HQL_API _ATOM _normalized_Atom;
 extern HQL_API _ATOM noRootAtom;
 extern HQL_API _ATOM noScanAtom;
 extern HQL_API _ATOM noSortAtom;

ecl/hql/hqlattr.cpp

@@ -538,7 +538,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_notify:
     case no_setgraphresult:
     case no_extractresult:
-    case no_updatestate:
+    case no_unused81:
     case no_definesideeffect:
 
 // Scopes etc.
@@ -1918,6 +1918,7 @@ bool isTrivialDataset(IHqlExpression * expr)
         case no_distributed:
         case no_grouped:
         case no_preservemeta:
+        case no_dataset_alias:
         case no_filter:
             expr = expr->queryChild(0);
             break;

ecl/hql/hqlexpr.cpp

@@ -68,8 +68,8 @@
 //#define DEBUG_SCOPE
 //#define CHECK_RECORD_CONSITENCY
 //#define PARANOID
-//#define SEARCH_NAME1   "vCP4"
-//#define SEARCH_NAME2   "vIJ"
+//#define SEARCH_NAME1   "vQ8"
+//#define SEARCH_NAME2   "v19"
 //#define SEARCH_IEXPR 0x03289048
 //#define CHECK_SELSEQ_CONSISTENCY
 //#define GATHER_COMMON_STATS
@@ -1397,7 +1397,6 @@ const char *getOpString(node_operator op)
     case no_commonspill: return "no_commonspill";
     case no_forcegraph: return "GRAPH";
     case no_sectioninput: return "no_sectioninput";
-    case no_updatestate: return "UPDATE";
     case no_related: return "no_related";
     case no_definesideeffect: return "no_definessideeffect";
     case no_executewhen: return "WHEN";
@@ -1416,7 +1415,7 @@ const char *getOpString(node_operator op)
     case no_complex: return ",";
     case no_assign_addfiles: return "+=";
     case no_debug_option_value: return "__DEBUG__";
-    case no_dataset_alias: return "ALIAS";
+    case no_dataset_alias: return "TABLE";
 
     case no_unused1: case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused17: case no_unused18: case no_unused19:
@@ -1784,7 +1783,6 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_newusertable:
     case no_usertable:
     case no_alias_project:
-    case no_alias_scope:
     case no_cachealias:
     case no_choosen:
     case no_choosesets:
@@ -1809,15 +1807,8 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_compound_selectnew:
     case no_compound_inline:
     case no_metaactivity:
-    case no_split:
-    case no_spill:
-    case no_readspill:
-    case no_commonspill:
-    case no_writespill:
     case no_throughaggregate:
     case no_countcompare:
-    case no_limit:
-    case no_catchds:
     case no_fieldmap:
     case no_countfile:
     case NO_AGGREGATE:
@@ -1832,52 +1823,50 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_compound_fetch:
     case no_topn:
     case no_distributer:
-    case no_preload:
     case no_createset:
-    case no_activerow:
-    case no_newrow:
-    case no_keyedlimit:
     case no_keypatch:
-    case no_returnresult:
-    case no_setgraphresult:
-    case no_setgraphloopresult:
     case no_assert_ds:
-    case no_spillgraphresult:
     case no_assertsorted:
     case no_assertgrouped:
     case no_assertdistributed:
-    case no_deserialize:
-    case no_serialize:
+    case no_extractresult:
+        return childdataset_dataset;
+    case no_keyedlimit:
+    case no_preload:
+    case no_limit:
+    case no_catchds:
     case no_forcegraph:
     case no_owned_ds:
     case no_dataset_alias:
-        return childdataset_dataset;
-    case no_executewhen:
-        //second argument is independent of the other arguments
+    case no_split:
+    case no_spill:
+    case no_activerow:
+    case no_alias_scope:
+    case no_executewhen:  //second argument is independent of the other arguments
+    case no_selectnth:
+    case no_readspill:
+    case no_commonspill:
+    case no_writespill:
+    case no_newrow:
+    case no_returnresult:
+    case no_setgraphresult:
+    case no_setgraphloopresult:
+    case no_spillgraphresult:
         return childdataset_dataset_noscope;
-    case no_newxmlparse:
-    case no_newparse:
-    case no_soapcall_ds:
-    case no_soapaction_ds:
-    case no_newsoapcall_ds:
-    case no_newsoapaction_ds:
-        return childdataset_datasetleft;
-    case no_keyeddistribute:
-        return childdataset_leftright;
-    case no_extractresult:
-        return childdataset_dataset;
     case no_setresult:
     case no_sizeof:
     case no_offsetof:
     case no_nameof:
     case no_blob2id:
     case no_subgraph:
+    case no_deserialize:
+    case no_serialize:
         if (expr->queryChild(0)->isDataset())
-            return childdataset_dataset;
+            return childdataset_dataset_noscope;
         return childdataset_none;
-    case no_updatestate:
+    case no_pipe:
         if (expr->queryChild(0)->isDataset())
-            return childdataset_addfiles;
+            return childdataset_dataset;
         return childdataset_none;
     case no_cloned:
     case no_colon:
@@ -1886,7 +1875,6 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_nohoist:
     case no_section:
     case no_thor:
-    case no_pipe:
     case no_catch:
     case no_forcelocal:
     case no_nothor:
@@ -1897,16 +1885,26 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_sectioninput:
     case no_outofline:
         if (expr->isDataset())
-            return childdataset_dataset;
+            return childdataset_dataset_noscope;
         return childdataset_none;
+    case no_preservemeta:
+        //Only has a single dataset - but fields are referenced via active selector, so use the many option
+        return childdataset_many;
+    case no_newxmlparse:
+    case no_newparse:
+    case no_soapcall_ds:
+    case no_soapaction_ds:
+    case no_newsoapcall_ds:
+    case no_newsoapaction_ds:
+        return childdataset_datasetleft;
+    case no_keyeddistribute:
+        return childdataset_leftright;
     case no_select:
         if (expr->hasProperty(newAtom) && expr->isDataset())
             return childdataset_dataset;
         return childdataset_none;
-    case no_selectnth:
-        return childdataset_dataset_noscope;
     case no_sub:
-        return expr->isDataset() ? childdataset_addfiles : childdataset_none;
+        return expr->isDataset() ? childdataset_many_noscope : childdataset_none;
     case no_if:
         return expr->isDataset() ? childdataset_if : childdataset_none;
     case no_map:
@@ -1919,13 +1917,14 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
         return childdataset_left;
     case no_merge:
     case no_regroup:
-    case no_datasetlist:
-    case no_nonempty:
-    case no_preservemeta:
     case no_cogroup:
-        return childdataset_merge;              //NB: sorted() attribute on merge uses no_activetable as the selector, not the left dataset
+        return childdataset_many;              //NB: sorted() attribute on merge uses no_activetable as the selector, not the left dataset
+    case no_nonempty:
+    case no_datasetlist:
     case no_addfiles:
-        return childdataset_addfiles;
+    case no_keydiff:
+    case no_related:
+        return childdataset_many_noscope;   // two arbitrary inputs
     case no_hqlproject:
     case no_projectrow:
     case no_loop:
@@ -1933,9 +1932,6 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_filtergroup:
     case no_normalizegroup:
         return childdataset_left;
-    case no_keydiff:
-    case no_related:
-        return childdataset_addfiles;   // two arbitrary inputs
     case no_denormalize:
     case no_denormalizegroup:
     case no_fetch:
@@ -2183,10 +2179,6 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
         if (dataset->queryChild(0)->isDataset())
             return 1;
         return 0;
-    case no_updatestate:
-        if (dataset->queryChild(0)->isDataset())
-            return 2;
-        return 0;
     case no_select:
         if (dataset->hasProperty(newAtom) && dataset->isDataset())
             return 1;
@@ -2259,10 +2251,10 @@ unsigned getNumChildTables(IHqlExpression * dataset)
         if (dataset->getOperator() != no_aggregate)
             assertex(num==2); 
         break;
-    case childdataset_addfiles:
+    case childdataset_many_noscope:
         assertex(num==2); 
         break;
-    case childdataset_merge:
+    case childdataset_many:
         break;
     case childdataset_if:
     case childdataset_case:
@@ -3076,7 +3068,6 @@ void CHqlExpression::initFlagsBeforeOperands()
     case no_impure:
     case no_outputscalar:
     case no_ensureresult:
-    case no_updatestate:
     case no_definesideeffect:
     case no_callsideeffect:
         infoFlags2 &= ~(HEF2constant);
@@ -3113,6 +3104,10 @@ void CHqlExpression::initFlagsBeforeOperands()
     case no_alias:
         infoFlags |= HEFcontainsAlias|HEFcontainsAliasLocally;
         break;
+    case no_dataset_alias:
+        if (!queryProperty(_normalized_Atom))
+            infoFlags |= HEFcontainsDatasetAliasLocally;
+        break;
     case no_activetable:
     case no_activerow:
         infoFlags2 &= ~(HEF2constant);
@@ -3263,7 +3258,7 @@ void CHqlExpression::updateFlagsAfterOperands()
         if (queryChild(0) && (queryChild(0)->getOperator() == no_null))
             infoFlags2 |= HEF2constant;
         // don't percolate aliases beyond their subqueries at the moment.
-        infoFlags &= ~(HEFcontainsAliasLocally|HEFthrowscalar);
+        infoFlags &= ~(HEFcontainsAliasLocally|HEFthrowscalar|HEFcontainsDatasetAliasLocally);
         //a dataset fail, now becomes a scalar fail
         if (infoFlags & HEFthrowds)
             infoFlags = (infoFlags &~HEFthrowds)|HEFthrowscalar;
@@ -3300,7 +3295,7 @@ void CHqlExpression::updateFlagsAfterOperands()
         }
         else
         {
-            infoFlags &= ~(HEFcontextDependentException|HEFcontainsActiveDataset|HEFcontainsAliasLocally|HEFthrowscalar);       // don't percolate aliases beyond their subqueries at the moment.
+            infoFlags &= ~(HEFcontextDependentException|HEFcontainsActiveDataset|HEFcontainsAliasLocally|HEFcontainsDatasetAliasLocally|HEFthrowscalar);       // don't percolate aliases beyond their subqueries at the moment.
             infoFlags |= (queryChild(0)->getInfoFlags() & (HEFcontextDependentException|HEFcontainsActiveDataset));
             if (infoFlags & HEFthrowds)
                 infoFlags = (infoFlags &~HEFthrowds)|HEFthrowscalar;
@@ -4580,7 +4575,7 @@ void CHqlExpression::cacheTablesProcessChildScope()
     switch (getChildDatasetType(this))
     {
     case childdataset_none: 
-    case childdataset_addfiles:
+    case childdataset_many_noscope:
     case childdataset_if:
     case childdataset_case:
     case childdataset_map:
@@ -4588,7 +4583,7 @@ void CHqlExpression::cacheTablesProcessChildScope()
         cacheChildrenTablesUsed(0, max);
         //None of these have any scoped arguments, so no need to remove them
         break;
-    case childdataset_merge:
+    case childdataset_many:
         {
             //can now have sorted() attribute which is dependent on the no_activetable element.
             unsigned firstAttr = getNumChildTables(this);
@@ -5553,7 +5548,10 @@ IHqlExpression * queryRoot(IHqlExpression * expr)
 
 IHqlExpression * queryTable(IHqlExpression * dataset)
 {
-    return queryExpression(dataset->queryDataset()->queryTable());
+    IHqlDataset * ds = dataset->queryDataset();
+    if (!ds)
+        return NULL;
+    return queryExpression(ds->queryTable());
 }
 
 
@@ -8614,7 +8612,7 @@ CHqlSequence::CHqlSequence(node_operator _op, ITypeInfo * _type, _ATOM _name, un
 
 CHqlSequence *CHqlSequence::makeSequence(node_operator _op, ITypeInfo * _type, _ATOM _name, unsigned __int64 _seq) 
 {
-    if (!_type) _type = makeVoidType();
+    if (!_type) _type = makeNullType();
     return new CHqlSequence(_op, _type, _name, _seq);
 }
 

ecl/hql/hqlexpr.hpp

@@ -119,7 +119,7 @@ enum
 // generally applicable start from the top down
     HEFunbound                  = 0x00000010,
     HEFinternalVirtual          = 0x00000020,
-    HEF____unused1____          = 0x00000040,
+    HEFcontainsDatasetAliasLocally= 0x00000040,
     HEF____unused2____          = 0x00000080,
     HEF____unused3____          = 0x00000100,
     HEFfunctionOfGroupAggregate = 0x00000200,
@@ -165,7 +165,7 @@ enum
                                    HEFonFailDependent|HEFcontainsActiveDataset|HEFcontainsActiveNonSelector|HEFcontainsDataset|
                                    HEFtranslated|HEFgraphDependent|HEFcontainsNlpText|HEFcontainsXmlText|HEFtransformDependent|
                                    HEFcontainsSkip|HEFcontainsCounter|HEFassertkeyed|HEFcontextDependentException|HEFcontainsAlias|HEFcontainsAliasLocally|
-                                   HEFinternalVirtual|HEFcontainsThisNode),
+                                   HEFinternalVirtual|HEFcontainsThisNode|HEFcontainsDatasetAliasLocally),
 
     HEFcontextDependentNoThrow  = (HEFcontextDependent & ~(HEFthrowscalar|HEFthrowds|HEFoldthrows)),
     HEFcontextDependentDataset  = (HEFcontextDependent & ~(HEFthrowscalar)),
@@ -720,7 +720,7 @@ enum _node_operator {
         no_sectioninput,
         no_forcegraph,
         no_eventextra,
-        no_updatestate,
+    no_unused81,
         no_related,
         no_executewhen,
         no_definesideeffect,
@@ -1631,6 +1631,7 @@ inline bool containsNonActiveDataset(IHqlExpression * expr) { return (expr->getI
 inline bool containsAnyDataset(IHqlExpression * expr)   { return (expr->getInfoFlags() & (HEFcontainsDataset|HEFcontainsActiveDataset)) != 0; }
 inline bool containsAlias(IHqlExpression * expr)        { return (expr->getInfoFlags() & HEFcontainsAlias) != 0; }
 inline bool containsAliasLocally(IHqlExpression * expr) { return (expr->getInfoFlags() & HEFcontainsAliasLocally) != 0; }
+inline bool containsDatasetAliasLocally(IHqlExpression * expr) { return (expr->getInfoFlags() & HEFcontainsDatasetAliasLocally) != 0; }
 inline bool containsNonGlobalAlias(IHqlExpression * expr)   
                                                         { return (expr->getInfoFlags2() & HEF2containsNonGlobalAlias) != 0; }
 inline bool containsAssertKeyed(IHqlExpression * expr)  { return (expr->getInfoFlags() & HEFassertkeyed) != 0; }
@@ -1678,12 +1679,22 @@ extern HQL_API unsigned queryCurrentTransformDepth();                   // debug
 extern HQL_API bool isExternalFunction(IHqlExpression * funcdef);
 
 typedef enum { 
-    childdataset_none, childdataset_dataset, childdataset_left, childdataset_leftright, childdataset_same_left_right, 
-    childdataset_top_left_right,
+    childdataset_none,
+    childdataset_dataset_noscope, // single dataset but this operation doesn't use any fields from it.
+    childdataset_dataset,  // single dataset, fields are referenced by <dataset>.field
+    childdataset_datasetleft,  // single dataset, fields are referenced by <dataset>|LEFT.field
+    childdataset_left,  // single dataset, fields are referenced by LEFT.field
+    childdataset_leftright,   // two datasets, fields are referenced by LEFT|RIGHT.field
+    childdataset_same_left_right,  // single dataset, fields are referenced by LEFT|RIGHT.field
+    childdataset_top_left_right,  // single dataset, fields are referenced by <dataset>|LEFT|RIGHT.field
+    childdataset_many_noscope, // multiple input files, no reference to any fields.
+    childdataset_many,  // multiple input files, fields reference by <active>.field
+    childdataset_nway_left_right, // set of files for first parameter, fields accessed via LEFT and RIGHT
     //weird exceptions
-    childdataset_addfiles, childdataset_evaluate, childdataset_if, childdataset_case, childdataset_map, childdataset_merge,
-    childdataset_dataset_noscope, childdataset_datasetleft,
-    childdataset_nway_left_right,
+    childdataset_evaluate, // EVALUATE
+    childdataset_if, // IF - second and third are datasets
+    childdataset_case, // CASE
+    childdataset_map, // MAP
     childdataset_max
 } childDatasetType;
 extern HQL_API childDatasetType getChildDatasetType(IHqlExpression * expr);

ecl/hql/hqlfold.cpp

@@ -4909,14 +4909,14 @@ IHqlExpression * CExprFolderTransformer::percolateConstants(IHqlExpression * exp
             {
             case childdataset_none: 
             case childdataset_nway_left_right:
-            case childdataset_addfiles:
-            case childdataset_merge:
+            case childdataset_many_noscope:
+            case childdataset_many:
             case childdataset_if:
             case childdataset_case:
             case childdataset_map:
             case childdataset_evaluate:
-            case childdataset_dataset_noscope: 
                 break;
+            case childdataset_dataset_noscope:
             case childdataset_dataset:
                 updated.setown(percolateConstants(updated, child, no_none));
                 break;
@@ -5553,6 +5553,8 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
         break;
 
     default:
+        if (expr->isAction())
+            break;
         DBGLOG("Missing entry: %s", getOpString(expr->getOperator()));
         if (expr->isDatarow())
         {

ecl/hql/hqlgram.y

@@ -7776,6 +7776,10 @@ simpleDataSet
                             $$.setExpr(createDataset(no_newusertable, $3.getExpr(), createComma(LINK(tform->queryRecord()), ensureTransformType(tform, no_newtransform))));
                             $$.setPosition($1);
                         }
+    | TABLE '(' startTopFilter ')' endTopFilter
+                        {
+                            $$.setExpr(createDataset(no_dataset_alias, $3.getExpr(), ::createUniqueId()), $1);
+                        }
     | FETCH '(' startLeftSeqFilter ',' startRightFilterUpdateSeq ',' expression ',' transform optCommonAttrs ')' endRightFilter endLeftFilter endSelectorSequence
                         {
                             parser->normalizeExpression($7, type_int, false);
@@ -8014,20 +8018,20 @@ simpleDataSet
                             $$.setExpr(createDataset(no_workunit_dataset, $8.getExpr(), arg));
                             $$.setPosition($1);
                         }
-    | ENTH '(' startTopFilter ',' expression optCommonAttrs ')' endTopFilter
+    | ENTH '(' dataSet ',' expression optCommonAttrs ')'
                         {
                             parser->normalizeExpression($5, type_numeric, false);
                             $$.setExpr(createDataset(no_enth, $3.getExpr(), createComma($5.getExpr(), $6.getExpr())));
                             $$.setPosition($1);
                         }
-    | ENTH '(' startTopFilter ',' expression ',' expression optCommonAttrs ')' endTopFilter
+    | ENTH '(' dataSet ',' expression ',' expression optCommonAttrs ')'
                         {
                             parser->normalizeExpression($5, type_numeric, false);
                             parser->normalizeExpression($7, type_numeric, false);
                             $$.setExpr(createDataset(no_enth, $3.getExpr(), createComma($5.getExpr(), $7.getExpr(), $8.getExpr())));
                             $$.setPosition($1);
                         }
-    | ENTH '(' startTopFilter ',' expression ',' expression ',' expression optCommonAttrs ')' endTopFilter
+    | ENTH '(' dataSet ',' expression ',' expression ',' expression optCommonAttrs ')'
                         {
                             parser->normalizeExpression($5, type_numeric, false);
                             parser->normalizeExpression($7, type_numeric, false);
@@ -8064,7 +8068,7 @@ simpleDataSet
                             $$.setExpr(createDataset(no_pipe, $3.getExpr(), createComma($5.getExpr(), $7.getExpr(), LINK(attrs))));
                             $$.setPosition($1);
                         }
-    | PRELOAD '(' startTopFilter optConstExpression ')' endTopFilter
+    | PRELOAD '(' dataSet optConstExpression ')'
                         {
                             $$.setExpr(createDataset(no_preload, $3.getExpr(), $4.getExpr()));
                             $$.setPosition($1);

ecl/hql/hqlopt.cpp

@@ -74,6 +74,13 @@ IHqlExpression * createFilterCondition(const HqlExprArray & conds)
 }
 
 
+IHqlExpression * createFilterCondition(const HqlExprArray & conds, IHqlExpression * oldDataset, IHqlExpression * newDataset)
+{
+    OwnedHqlExpr mapped = createFilterCondition(conds);
+    return replaceSelector(mapped, oldDataset->queryNormalizedSelector(), newDataset->queryNormalizedSelector());
+}
+
+
 bool optimizeFilterConditions(HqlExprArray & conds)
 {
     ForEachItemInRev(i, conds)
@@ -682,20 +689,41 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateDataset(IHqlExpression * trans
     return transformed->clone(children);
 }
 
+static IHqlExpression * skipMetaAliases(IHqlExpression * expr)
+{
+    loop
+    {
+        switch (expr->getOperator())
+        {
+        case no_dataset_alias:
+            break;
+        default:
+            return expr;
+        }
+        expr = expr->queryChild(0);
+    }
+}
 
 IHqlExpression * CTreeOptimizer::optimizeDatasetIf(IHqlExpression * transformed)
 {
     //if(cond, ds(filt1), ds(filt2)) => ds(if(cond,filt1,filt2))
     HqlExprArray leftFilter, rightFilter;
-    IHqlExpression * left = extractFilterDs(leftFilter, transformed->queryChild(1));
-    IHqlExpression * right = extractFilterDs(rightFilter, transformed->queryChild(2));
+    IHqlExpression * unfilteredLeft = extractFilterDs(leftFilter, transformed->queryChild(1));
+    IHqlExpression * unfilteredRight = extractFilterDs(rightFilter, transformed->queryChild(2));
+    IHqlExpression * left = skipMetaAliases(unfilteredLeft);
+    IHqlExpression * right = skipMetaAliases(unfilteredRight);
     if (left->queryBody() == right->queryBody())
     {
+        //If one (or both) or the datasets are aliases then ensure that one of the of the
+        //aliases is used in the replacement.
+        IHqlExpression * baseDataset = unfilteredLeft;
+        if (right->queryNormalizedSelector() != unfilteredRight->queryNormalizedSelector())
+            baseDataset = unfilteredRight;
+
         HqlExprArray args;
-        args.append(*LINK(left));
-//                  intersectConditions(args, leftFilter, rightFilter);
-        OwnedHqlExpr leftCond = createFilterCondition(leftFilter);
-        OwnedHqlExpr rightCond = createFilterCondition(rightFilter);
+        args.append(*LINK(baseDataset));
+        OwnedHqlExpr leftCond = createFilterCondition(leftFilter, unfilteredLeft, baseDataset);
+        OwnedHqlExpr rightCond = createFilterCondition(rightFilter, unfilteredRight, baseDataset);
         if (leftCond == rightCond)
         {
             args.append(*leftCond.getClear());
@@ -712,7 +740,10 @@ IHqlExpression * CTreeOptimizer::optimizeDatasetIf(IHqlExpression * transformed)
 
         //NOTE: left and right never walk over any shared nodes, so don't need to decrement usage for 
         //child(1), child(2) or intermediate nodes to left/right, since not referenced any more.
-        noteUnused(right);      // dataset is now used one less time
+        if (baseDataset == left)
+            noteUnused(right);      // dataset is now used one less time
+        else
+            noteUnused(left);
         return transformed->cloneAllAnnotations(ret);
     }
     return LINK(transformed);
@@ -1721,8 +1752,8 @@ bool CTreeOptimizer::childrenAreShared(IHqlExpression * expr)
         case childdataset_map:
         case childdataset_nway_left_right:
             return true;    // stop any folding of these...
-        case childdataset_addfiles:
-        case childdataset_merge:
+        case childdataset_many_noscope:
+        case childdataset_many:
             {
                 ForEachChild(i, expr)
                 {
@@ -2033,8 +2064,8 @@ void CTreeOptimizer::recursiveDecChildUsage(IHqlExpression * expr)
     case childdataset_map:
     case childdataset_nway_left_right:
         break;  // who knows?
-    case childdataset_addfiles:
-    case childdataset_merge:
+    case childdataset_many_noscope:
+    case childdataset_many:
         {
             ForEachChild(i, expr)
                 recursiveDecUsage(expr->queryChild(i));

ecl/hql/hqltrans.cpp

@@ -270,8 +270,8 @@ unsigned activityHidesSelectorGetNumNonHidden(IHqlExpression * expr, IHqlExpress
     switch (getChildDatasetType(expr))
     {
     case childdataset_none:
-    case childdataset_addfiles:
-    case childdataset_merge:
+    case childdataset_many_noscope:
+    case childdataset_many:
     case childdataset_map:
     case childdataset_dataset_noscope:
     case childdataset_if:
@@ -1071,6 +1071,8 @@ ANewTransformInfo::ANewTransformInfo(IHqlExpression * _original)
     original = _original;
     lastPass = (byte) -1;
     flags = 0;
+    spareByte1 = 0;
+    spareByte2 = 0;
 }
 
 
@@ -1926,15 +1928,43 @@ IHqlExpression * HqlMapTransformer::queryMapping(IHqlExpression * oldValue)
     return queryTransformed(oldValue);
 }
 
+static HqlTransformerInfo hqlMapDatasetTransformerInfo("HqlMapDatasetTransformer");
+HqlMapDatasetTransformer::HqlMapDatasetTransformer() : NewHqlTransformer(hqlMapDatasetTransformerInfo)
+{
+}
+
+
 IHqlExpression * replaceExpression(IHqlExpression * expr, IHqlExpression * original, IHqlExpression * replacement)
 {
     if (expr == original)
         return LINK(replacement);
+
+    //if dataset that doesn't define columns is replaced with an expression that does then you need to remap any
+    //specially update selectors in nested expressions.
+    if (original->isDataset())
+    {
+        IHqlExpression * table = queryTable(original);
+        if (!definesColumnList(original) && definesColumnList(replacement))
+        {
+            HqlMapDatasetTransformer simpleTransformer;
+            simpleTransformer.setMapping(original, replacement);
+            return simpleTransformer.transformRoot(expr);
+        }
+    }
     HqlMapTransformer simpleTransformer;
     simpleTransformer.setMapping(original, replacement);
     return simpleTransformer.transformRoot(expr);
 }
 
+//---------------------------------------------------------------------------
+
+IHqlExpression * replaceDataset(IHqlExpression * expr, IHqlExpression * original, IHqlExpression * replacement)
+{
+    return replaceExpression(expr, original, replacement);
+}
+
+
+//---------------------------------------------------------------------------
 
 HqlMapSelectorTransformer::HqlMapSelectorTransformer(IHqlExpression * oldDataset, IHqlExpression * newDataset)
 {
@@ -1958,6 +1988,66 @@ HqlMapSelectorTransformer::HqlMapSelectorTransformer(IHqlExpression * oldDataset
     setSelectorMapping(oldDataset, newSelector);
 }
 
+IHqlExpression * HqlMapSelectorTransformer::createTransformed(IHqlExpression * expr)
+{
+    switch (getChildDatasetType(expr))
+    {
+    case childdataset_none:
+    case childdataset_many_noscope:
+    case childdataset_many:
+    case childdataset_map:
+    case childdataset_dataset_noscope:
+    case childdataset_if:
+    case childdataset_case:
+    case childdataset_evaluate:
+    case childdataset_left:
+    case childdataset_leftright:
+    case childdataset_same_left_right:
+    case childdataset_nway_left_right:
+        return HqlMapTransformer::createTransformed(expr);
+    case childdataset_dataset:
+    case childdataset_datasetleft:
+    case childdataset_top_left_right:
+        break;
+    default:
+        UNIMPLEMENTED;
+    }
+
+    //If this expression is a child dataset with a filter on the same dataset that is being replaced,
+    //then ensure the selectors aren't replaced in this operator's arguments.
+    IHqlExpression * dataset = expr->queryChild(0);
+    IHqlExpression * walker = dataset;
+    loop
+    {
+        IHqlExpression * table = queryTable(walker);
+        if (table)
+        {
+            if (table->queryNormalizedSelector() == oldSelector)
+                break;
+            if (table->getOperator() == no_select)
+            {
+                bool isNew;
+                walker = querySelectorDataset(table, isNew);
+                if (isNew)
+                    continue;
+            }
+        }
+
+        return HqlMapTransformer::createTransformed(expr);
+    }
+
+    OwnedHqlExpr transformedDataset = transform(dataset);
+    //optimization to avoid the clone
+    if (dataset == transformedDataset)
+        return LINK(expr);
+
+    HqlExprArray args;
+    args.append(*transformedDataset.getClear());
+    unwindChildren(args, expr, 1);
+    return expr->clone(args);
+}
+
+
 
 static HqlTransformerInfo hqlScopedMapSelectorTransformerInfo("HqlScopedMapSelectorTransformer");
 HqlScopedMapSelectorTransformer::HqlScopedMapSelectorTransformer(IHqlExpression * oldDataset, IHqlExpression * newDataset)
@@ -2679,7 +2769,7 @@ IHqlExpression * MergingHqlTransformer::createTransformed(IHqlExpression * expr)
             }
             return LINK(expr);
         }
-    case childdataset_merge:
+    case childdataset_many:
         {
             unsigned firstAttr = getNumChildTables(expr);
             IHqlExpression * arg0 = expr->queryChild(0);
@@ -3048,7 +3138,6 @@ void ScopedTransformer::analyseChildren(IHqlExpression * expr)
     case no_sizeof:
     case no_offsetof:
     case no_nameof:
-    case no_updatestate:
         if (!expr->queryChild(0)->isDataset())
         {
             NewHqlTransformer::analyseChildren(expr);
@@ -3308,7 +3397,7 @@ void ScopedTransformer::analyseChildren(IHqlExpression * expr)
             switch (getChildDatasetType(expr))
             {
             case childdataset_none:
-            case childdataset_addfiles:
+            case childdataset_many_noscope:
             case childdataset_map:
             case childdataset_dataset_noscope:
                 {
@@ -3316,7 +3405,7 @@ void ScopedTransformer::analyseChildren(IHqlExpression * expr)
                     NewHqlTransformer::analyseChildren(expr);
                     return;
                 }
-            case childdataset_merge:
+            case childdataset_many:
                 {
                     first = getNumChildTables(expr);
                     for (idx = 0; idx < first; idx++)
@@ -3433,10 +3522,6 @@ IHqlExpression * ScopedTransformer::createTransformed(IHqlExpression * expr)
 //      if (!expr->queryChild(0)->isDataset())
             return NewHqlTransformer::createTransformed(expr);
         throwUnexpected();
-    case no_updatestate:
-        if (!expr->queryChild(0)->isDataset())
-            return NewHqlTransformer::createTransformed(expr);
-        //fallthrough
     case NO_AGGREGATE:
     case no_joined:
     case no_countfile:
@@ -3713,13 +3798,13 @@ IHqlExpression * ScopedTransformer::createTransformed(IHqlExpression * expr)
             switch (getChildDatasetType(expr))
             {
             case childdataset_none:
-            case childdataset_addfiles:
+            case childdataset_many_noscope:
             case childdataset_map:
             case childdataset_dataset_noscope:
                 {
                     return NewHqlTransformer::createTransformed(expr);
                 }
-            case childdataset_merge:
+            case childdataset_many:
                 {
                     first = getNumChildTables(expr);
                     IHqlExpression * dataset = expr->queryChild(0);

ecl/hql/hqltrans.ipp

@@ -526,11 +526,35 @@ public:
     using NewHqlTransformer::setSelectorMapping;
 };
 
+class HQL_API HqlMapDatasetTransformer : public NewHqlTransformer
+{
+public:
+    HqlMapDatasetTransformer();
+
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr)
+    {
+        IHqlExpression * body = expr->queryBody(true);
+        if (expr == body)
+        {
+            OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
+            updateOrphanedSelectors(transformed, expr);
+            return transformed.getClear();
+        }
+        OwnedHqlExpr transformed = transform(body);
+        return expr->cloneAnnotation(transformed);
+    }
+
+    using NewHqlTransformer::setMapping;
+    using NewHqlTransformer::setSelectorMapping;
+};
+
 class HQL_API HqlMapSelectorTransformer : public HqlMapTransformer
 {
 public:
     HqlMapSelectorTransformer(IHqlExpression * oldValue, IHqlExpression * newValue);
 
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr);
+
 protected:
     OwnedHqlExpr oldSelector;
 };
@@ -672,6 +696,7 @@ public:
 
 
 extern HQL_API IHqlExpression * replaceExpression(IHqlExpression * expr, IHqlExpression * original, IHqlExpression * replacement);
+extern HQL_API IHqlExpression * replaceDataset(IHqlExpression * expr, IHqlExpression * original, IHqlExpression * replacement);
 
 
 //------------------------------------------------------------------------

ecl/hql/hqlutil.cpp

@@ -1711,7 +1711,6 @@ bool isSinkActivity(IHqlExpression * expr)
     case no_setresult:
     case no_setgraphresult:
     case no_setgraphloopresult:
-    case no_updatestate:
     case no_definesideeffect:
     //case no_callsideeffect:       //??
         return true;
@@ -4011,7 +4010,6 @@ bool hasActiveTopDataset(IHqlExpression * expr)
 {
     switch (getChildDatasetType(expr))
     {
-    case childdataset_merge:
     case childdataset_dataset:
     case childdataset_datasetleft:
     case childdataset_top_left_right:
@@ -7441,3 +7439,156 @@ IHqlExpression * replaceParameters(IHqlExpression * body, IHqlExpression * oldPa
 
     return simpleTransformer.transformRoot(body);
 }
+
+//---------------------------------------------------------------------------------------------------------------------
+
+/*
+Aliases are nasty...they can occur in two different situations
+i) The user specifies TABLE(x) to create an alias
+ii) The scope checking spots that an alias is being implicitly created.
+
+1) exists(join(ds, ds, left.id*3=right.id));
+ds_1 := table(ds);
+ds(exists(ds_1(ds_1.id=ds.id*3)));
+
+a) ds is a table
+b) ds is a filtered table.
+c) ds is an implicitly normalized dataset (ds.child);
+d) ds is a projected table
+e) ds is a filtered projected table.
+
+2) ds(exists(join(child, child, left.id*3=right.id)));
+child_1 = table(ds.child);
+ds(exists(child(exists(child1(child_1.id = child.id*3)));
+
+a) ds is a table
+b) ds is a filtered table.
+c) ds is an implicitly normalized dataset (ds.child);
+d) ds is a projected table
+e) ds is a filtered projected table.
+
+When either of these occurs a no_dataset_alias node is added to the tree with a unique id.  We don't want to modify
+any of the input datasets - because we want them to stay common as long as possible - otherwise code like
+ds(field in ds(filter))  would cause ds to become split in two - and it should mean the same thing.
+
+For implicit aliases they will be added around the dataset that is ambiguous.
+- It would be simpler to add them around the table that is ambiguous (Table is a dataset that defines a column list)
+  but that means that sort, filters etc. aren't commoned up.
+- When the code is actually generated the base table is modified - which ensures no ambiguous expressions are
+  actually present when generating.
+
+E.g,
+x := ds(a <> 0);
+x(b in set(x(c <> 0), b))
+becomes
+x := ds(a <> 0);
+x' = table(x);
+x'(b in set(x(c <> 0), b))
+
+To avoid that the aliases is not added around a dataset that has already been aliased in the dataset that uses it.
+
+When the expression comes to be generated/evaluated, the underlying table of the dataset expression is modified to
+include a unique id.  The root table doesn't need to be modified because no selectors for that can be in scope.
+
+*/
+
+IHqlExpression * queryTableOrSplitter(IHqlExpression * expr)
+{
+    loop
+    {
+        node_operator op = expr->getOperator();
+        if (op == no_compound)
+            expr = expr->queryChild(1);
+        else if (definesColumnList(expr))
+            return expr;
+        else if (op == no_split)
+            return expr;
+        else
+            expr = expr->queryChild(0);
+    }
+}
+
+//Convert no_dataset_alias(expr, uid) to expr'
+IHqlExpression * normalizeDatasetAlias(IHqlExpression * expr)
+{
+    IHqlExpression * uid = expr->queryProperty(_uid_Atom);
+    assertex(uid);
+    IHqlExpression * dataset = expr->queryChild(0);
+    IHqlExpression * table = queryTableOrSplitter(dataset);
+
+    //If the alias is based on a splitter then we need to ensure the splitter expression stays the same - otherwise
+    //if won't be commoned up.  So add a alias with a _normalized_Atom to ensure everything followed that will be
+    //unique.  Otherwise add a unique id onto the underlying table to ensure unique expressions.
+    OwnedHqlExpr newTable;
+    node_operator tableOp = table->getOperator();
+    if ((tableOp == no_split) || (tableOp == no_rows))
+        newTable.setown(createDataset(no_dataset_alias, LINK(table), createComma(createUniqueId(), createAttribute(_normalized_Atom))));
+    else
+        newTable.setown(appendOwnedOperand(table, LINK(uid)));
+    return replaceDataset(dataset, table, newTable);
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
+//This should only be called on source activities, and on inline datasets.
+IHqlExpression * normalizeAnyDatasetAliases(IHqlExpression * expr)
+{
+    //It is useful to also be able to call this on no_sum(aliased-dataset)
+    if (!containsDatasetAliasLocally(expr) && !expr->isAggregate())
+        return LINK(expr);
+
+    node_operator op = expr->getOperator();
+    IHqlExpression * selector = NULL;
+    switch (getChildDatasetType(expr))
+    {
+    case childdataset_none:
+        if ((op == no_select) && isNewSelector(expr))
+            break;
+        return LINK(expr);
+    case childdataset_dataset:
+    case childdataset_dataset_noscope:
+    case childdataset_datasetleft:
+    case childdataset_top_left_right:
+        selector = expr->queryChild(0)->queryNormalizedSelector();
+        break;
+    case childdataset_left:
+    case childdataset_leftright:
+    case childdataset_many:
+    case childdataset_many_noscope:
+        break;
+    default:
+        return LINK(expr);
+        throwUnexpected();
+    }
+
+    bool same = true;
+    HqlExprArray args;
+    unsigned max = getNumChildTables(expr);
+    for (unsigned i=0; i < max; i++)
+    {
+        IHqlExpression * dataset = expr->queryChild(i);
+        OwnedHqlExpr newDataset = normalizeAnyDatasetAliases(dataset);
+        if (dataset != newDataset)
+            same = false;
+        args.append(*newDataset.getClear());
+    }
+
+    OwnedHqlExpr transformed;
+    if (same)
+        transformed.set(expr);
+    else
+    {
+        if (selector)
+        {
+            assertex(max == 1);
+            replaceSelectors(args, expr, max, selector, args.item(0).queryNormalizedSelector());
+        }
+        else
+            unwindChildren(args, expr, max);
+        transformed.setown(expr->clone(args));
+    }
+
+    if ((op == no_dataset_alias) && !transformed->hasProperty(_normalized_Atom))
+        return normalizeDatasetAlias(transformed);
+    return transformed.getClear();
+}

ecl/hql/hqlutil.hpp

@@ -625,6 +625,9 @@ extern HQL_API IHqlExpression * ensureOwned(IHqlExpression * expr);
 extern HQL_API bool isSetWithUnknownElementSize(ITypeInfo * type);
 extern HQL_API IHqlExpression * replaceParameters(IHqlExpression * body, IHqlExpression * oldParams, IHqlExpression * newParams);
 
+extern HQL_API IHqlExpression * normalizeDatasetAlias(IHqlExpression * expr);
+extern HQL_API IHqlExpression * normalizeAnyDatasetAliases(IHqlExpression * expr);
+
 //In hqlgram2.cpp
 extern HQL_API IPropertyTree * queryEnsureArchiveModule(IPropertyTree * archive, const char * name, IHqlScope * rScope);
 extern HQL_API IPropertyTree * queryArchiveAttribute(IPropertyTree * module, const char * name);

ecl/hqlcpp/hqlcatom.cpp

@@ -416,7 +416,6 @@ _ATOM newWorkUnitReadArgAtom;
 _ATOM newWorkUnitWriteArgAtom;
 _ATOM _noAccess_Atom;
 _ATOM _noReplicate_Atom;
-_ATOM _normalized_Atom;
 _ATOM noSetAtom;
 _ATOM _noVirtual_Atom;
 _ATOM numResultsAtom;
@@ -1119,7 +1118,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(newWorkUnitWriteArg);
     MAKESYSATOM(noAccess);
     MAKESYSATOM(noReplicate);
-    MAKESYSATOM(normalized);
     MAKEATOM(noSet);
     MAKESYSATOM(noVirtual);
     MAKEATOM(numResults);

ecl/hqlcpp/hqlcatom.hpp

@@ -416,7 +416,6 @@ extern _ATOM newWorkUnitReadArgAtom;
 extern _ATOM newWorkUnitWriteArgAtom;
 extern _ATOM _noAccess_Atom;
 extern _ATOM _noReplicate_Atom;
-extern _ATOM _normalized_Atom;
 extern _ATOM noSetAtom;
 extern _ATOM _noVirtual_Atom;
 extern _ATOM numResultsAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -1671,6 +1671,9 @@ void HqlCppTranslator::cacheOptions()
         // The following works 99% of the time, but disabled due to potential problems with the ambiguity of LEFT
         //possibly causing filters on nested records to be incorrectly removed.
         DebugOption(options.optimizeNestedConditional,"optimizeNestedConditional", false),
+        DebugOption(options.createImplicitAliases,"createImplicitAliases", false),
+        DebugOption(options.combineSiblingGraphs,"combineSiblingGraphs", true),
+        DebugOption(options.optimizeSharedGraphInputs,"optimizeSharedGraphInputs", true),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -700,6 +700,9 @@ struct HqlCppOptions
     bool                standAloneExe;
     bool                enableCompoundCsvRead;
     bool                optimizeNestedConditional;
+    bool                createImplicitAliases;
+    bool                combineSiblingGraphs;
+    bool                optimizeSharedGraphInputs;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class

ecl/hqlcpp/hqlcppds.cpp

@@ -597,6 +597,13 @@ IReferenceSelector * HqlCppTranslator::buildActiveRow(BuildCtx & ctx, IHqlExpres
 
 void HqlCppTranslator::doBuildExprAggregate(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
 {
+    OwnedHqlExpr normalized = normalizeAnyDatasetAliases(expr);
+    if (expr != normalized)
+    {
+        buildExpr(ctx, normalized, tgt);
+        return;
+    }
+
     node_operator op = expr->getOperator();
     ITypeInfo * type = expr->queryType();
     ITypeInfo * tempType = op == no_count ? unsignedType : type;
@@ -765,8 +772,9 @@ static bool isNullValueMinimumValue(ITypeInfo * type)
     return false;
 }
 
-void HqlCppTranslator::doBuildAssignAggregate(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
+void HqlCppTranslator::doBuildAssignAggregate(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * _expr)
 {
+    OwnedHqlExpr expr = normalizeAnyDatasetAliases(_expr);
     if (assignAggregateDirect(target, expr))
     {
         IHqlExpression * dataset = expr->queryChild(0);
@@ -1835,15 +1843,28 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
     if (expr->isPure() && ctx.getMatchExpr(expr, tgt))
         return;
 
+/*
+    OwnedHqlExpr transformed = normalizeAnyDatasetAliases(expr);
+    if (transformed && (transformed != expr))
+    {
+        doBuildDataset(ctx, transformed, tgt, format);
+        ctx.associateExpr(expr, tgt);
+        return;
+    }
+*/
+
     node_operator op = expr->getOperator();
     switch (op)
     {
     case no_dataset_alias:
+        if (!expr->hasProperty(_normalized_Atom))
         {
             OwnedHqlExpr uniqueChild = normalizeDatasetAlias(expr);
             doBuildDataset(ctx, uniqueChild, tgt, format);
-            return;
         }
+        else
+            doBuildDataset(ctx, expr->queryChild(0), tgt, format);
+        return;
     case no_alias:
         doBuildExprAlias(ctx, expr, &tgt);
         return;
@@ -3119,11 +3140,20 @@ BoundRow * HqlCppTranslator::buildDatasetIterate(BuildCtx & ctx, IHqlExpression
     switch (expr->getOperator())
     {
     case no_dataset_alias:
+        if (!expr->hasProperty(_normalized_Atom))
         {
             OwnedHqlExpr uniqueChild = normalizeDatasetAlias(expr);
             BoundRow * childCursor = buildDatasetIterate(ctx, uniqueChild, needToBreak);
             return rebindTableCursor(ctx, expr, childCursor, no_none, NULL);
         }
+        else
+        {
+            throwUnexpected();
+            //The following would only be triggered for a splitter (not yet generated), and that would require
+            //disambiguation when that was built.
+            BoundRow * childCursor = buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
+            return rebindTableCursor(ctx, expr, childCursor, no_none, NULL);
+        }
     case no_null:
         buildFilter(ctx, queryBoolExpr(false));
         return NULL;
@@ -3891,6 +3921,7 @@ IHqlExpression * HqlCppTranslator::ensureIteratedRowIsLive(BuildCtx & initctx, B
         case no_compound_childnormalize:
         case no_compound_selectnew:
         case no_compound_childread:
+        case no_dataset_alias:
             ds = ds->queryChild(0);
             break;
         case no_select:
@@ -3958,7 +3989,7 @@ IHqlExpression * HqlCppTranslator::ensureIteratedRowIsLive(BuildCtx & initctx, B
 
 IReferenceSelector * HqlCppTranslator::buildDatasetIndexViaIterator(BuildCtx & ctx, IHqlExpression * expr)
 {
-    IHqlExpression * dataset = querySkipDatasetMeta(expr->queryChild(0));
+    OwnedHqlExpr dataset = normalizeAnyDatasetAliases(querySkipDatasetMeta(expr->queryChild(0)));
     IHqlExpression * index = expr->queryChild(1);
     IHqlExpression * childDataset = dataset;
     switch (dataset->getOperator())
@@ -4064,7 +4095,7 @@ IReferenceSelector * HqlCppTranslator::buildDatasetIndex(BuildCtx & ctx, IHqlExp
         return buildNewRow(ctx, optimized);
 #endif
 
-    IHqlExpression * dataset = expr->queryChild(0);
+    OwnedHqlExpr dataset = normalizeAnyDatasetAliases(expr->queryChild(0));
 
     //Special cases:
     //i) selecting row [1] from something that only has a single row
@@ -4093,7 +4124,7 @@ IReferenceSelector * HqlCppTranslator::buildDatasetIndex(BuildCtx & ctx, IHqlExp
     {
         //MORE? Following doesn't work for implicit normalize which iterates multiple levels
         bool specialCase = false;
-        dataset = querySkipDatasetMeta(dataset);
+        dataset.set(querySkipDatasetMeta(dataset));
         
         switch (dataset->getOperator())
         {

ecl/hqlcpp/hqlcpputil.cpp

@@ -215,14 +215,6 @@ bool storePointerInArray(ITypeInfo * type)
     return type->isReference() && isTypePassedByAddress(type); 
 }
 
-//Convert no_dataset_alias(expr, uid) to expr'
-IHqlExpression * normalizeDatasetAlias(IHqlExpression * expr)
-{
-    IHqlExpression * uid = expr->queryProperty(_uid_Atom);
-    assertex(uid);
-    return appendOwnedOperand(expr->queryChild(0), LINK(uid));
-}
-
 //---------------------------------------------------------------------------
 
 bool isSelectSortedTop(IHqlExpression * selectExpr)

ecl/hqlcpp/hqlcpputil.hpp

@@ -46,7 +46,6 @@ extern IHqlExpression * addMemberSelector(IHqlExpression * expr, IHqlExpression
 extern IHqlExpression * addExpressionModifier(IHqlExpression * expr, typemod_t modifier, IInterface * extra=NULL);
 extern void expandFieldNames(StringBuffer & out, IHqlExpression * record, const char * sep, IHqlExpression * formatFunc);
 extern IHqlExpression * ensurePositiveOrZeroInt64(IHqlExpression * expr);
-extern IHqlExpression * normalizeDatasetAlias(IHqlExpression * expr);
 
 extern void getOutputLibraryName(SCMStringBuffer & libraryName, IConstWorkUnit * wu);
 extern bool canCreateTemporary(IHqlExpression * expr);

ecl/hqlcpp/hqlcse.cpp

@@ -543,6 +543,7 @@ bool CseSpotter::checkPotentialCSE(IHqlExpression * expr, CseSpotterInfo * extra
     case no_xmlproject:
     case no_datasetfromrow:
     case no_preservemeta:
+    case no_dataset_alias:
     case no_workunit_dataset:
     case no_left:
     case no_right:

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5862,11 +5862,18 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
         cur = body;
     }
 
+    if (isCompoundSource(expr))
+    {
+        OwnedHqlExpr mapped = normalizeAnyDatasetAliases(expr);
+        if (mapped != expr)
+            return buildActivity(ctx, mapped, isRoot);
+    }
 
     ABoundActivity * result;
     try
     {
-        switch (expr->getOperator())
+        node_operator op = expr->getOperator();
+        switch (op)
         {
             case no_merge:
                 result = doBuildActivityMerge(ctx, expr);
@@ -6125,9 +6132,17 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_thisnode:
             case no_forcegraph:
             case no_keyed:
-            case no_dataset_alias:
                 result = buildCachedActivity(ctx, expr->queryChild(0));
                 break;
+            case no_dataset_alias:
+                if (!expr->hasProperty(_normalized_Atom))
+                {
+                    OwnedHqlExpr uniqueChild = normalizeDatasetAlias(expr);
+                    result = buildCachedActivity(ctx, uniqueChild);
+                }
+                else
+                    result = buildCachedActivity(ctx, expr->queryChild(0));
+                break;
             case no_alias_scope:
             case no_alias:
                 result = buildCachedActivity(ctx, expr->queryChild(0));
@@ -6311,7 +6326,7 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
                 }
                 else
                 {
-                    UNIMPLEMENTED_XY("Activity", getOpString(expr->getOperator()));
+                    UNIMPLEMENTED_XY("Activity", getOpString(op));
                 }
         }
     }
@@ -6335,8 +6350,6 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
 
 ABoundActivity * HqlCppTranslator::buildCachedActivity(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
 {
-//  if (isRoot && curActivityId > 400)
-//      return NULL;
     switch (expr->getOperator())
     {
     case no_split:
@@ -17585,6 +17598,7 @@ static bool needsRealThor(IHqlExpression *expr, unsigned flags)
     case no_compound_fetch:
     case no_addfiles:
     case no_nonempty:
+    case no_dataset_alias:
         //i.e. go through children...
         break;
     case no_compound:

ecl/hqlcpp/hqlinline.cpp

@@ -262,6 +262,7 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
     case no_alias_scope:
     case no_serialize:
     case no_deserialize:
+    case no_dataset_alias:
         return getInlineFlags(ctx, expr->queryChild(0));
     case no_forcegraph:
         return 0;

ecl/hqlcpp/hqliproj.cpp

@@ -326,6 +326,7 @@ static node_operator queryCompoundOp(IHqlExpression * expr)
         return no_compound_diskread;
     case no_newkeyindex:
         return no_compound_indexread;
+    case no_dataset_alias:
     case no_preservemeta:
         return queryCompoundOp(expr->queryChild(0));
     }
@@ -625,8 +626,8 @@ void ComplexImplicitProjectInfo::trace()
     switch (childDatasetType)
     {
     case childdataset_none: 
-    case childdataset_addfiles:
-    case childdataset_merge:
+    case childdataset_many_noscope:
+    case childdataset_many:
     case childdataset_if:
     case childdataset_case:
     case childdataset_map:
@@ -1225,7 +1226,7 @@ void ImplicitProjectTransformer::gatherFieldsUsed(IHqlExpression * expr, Implici
             switch (getChildDatasetType(expr))
             {
             case childdataset_none: 
-            case childdataset_addfiles:
+            case childdataset_many_noscope:
             case childdataset_if:
             case childdataset_case:
             case childdataset_map:
@@ -1233,7 +1234,7 @@ void ImplicitProjectTransformer::gatherFieldsUsed(IHqlExpression * expr, Implici
                 inheritActiveFields(expr, extra, 0, max);
                 //None of these have any scoped arguments, so no need to remove them
                 break;
-            case childdataset_merge:
+            case childdataset_many:
                 {
                     unsigned firstAttr = getNumChildTables(expr);
                     inheritActiveFields(expr, extra, firstAttr, max);
@@ -1583,11 +1584,10 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_writespill:
         return ActionSinkActivity;
     case no_preservemeta:
+    case no_dataset_alias:
         if (getProjectExprKind(expr->queryChild(0)) == CompoundableActivity)
             return CompoundableActivity;
         return PassThroughActivity;
-    case no_dataset_alias:
-        return PassThroughActivity;
     }
 
     ITypeInfo * type = expr->queryType();
@@ -2643,8 +2643,8 @@ IHqlExpression * ImplicitProjectTransformer::updateSelectors(IHqlExpression * ne
     switch (getChildDatasetType(newExpr))
     {
     case childdataset_none: 
-    case childdataset_addfiles:
-    case childdataset_merge:
+    case childdataset_many_noscope:
+    case childdataset_many:
     case childdataset_if:
     case childdataset_case:
     case childdataset_map:

ecl/hqlcpp/hqliter.cpp

@@ -100,6 +100,7 @@ bool canBuildSequenceInline(IHqlExpression * expr)
         case no_compound_selectnew:
         case no_compound_inline:
         case no_alias_scope:
+        case no_dataset_alias:
             expr = expr->queryChild(0);
             break;
         default:

ecl/hqlcpp/hqlresource.cpp

@@ -698,6 +698,41 @@ bool ResourceGraphInfo::addCondition(IHqlExpression * condition)
     return false;
 }
 
+bool ResourceGraphInfo::isSharedInput(IHqlExpression * expr)
+{
+    IHqlExpression * body = expr->queryBody();
+    if (unbalancedExternalSources.contains(*body))
+        return false;
+    if (queryResourceInfo(expr)->expandRatherThanSplit())
+        return false;
+    unsigned numUses = 0;
+    ForEachItemIn(i, balancedExternalSources)
+    {
+        if (body == &balancedExternalSources.item(i))
+            numUses++;
+    }
+    //NumUses could be zero if an input should be expanded, and that input is shared by another graph which also expands
+    //the input.  E.g. project(meta(diskread).
+    return numUses > 1;
+}
+
+void ResourceGraphInfo::addSharedInput(IHqlExpression * expr, IHqlExpression * mapped)
+{
+    sharedInputs.append(*LINK(expr));
+    sharedInputs.append(*LINK(mapped));
+}
+
+IHqlExpression * ResourceGraphInfo::queryMappedSharedInput(IHqlExpression * expr)
+{
+    unsigned max = sharedInputs.ordinality();
+    for (unsigned i=0; i < max; i+= 2)
+    {
+        if (expr == &sharedInputs.item(i))
+            return &sharedInputs.item(i+1);
+    }
+    return NULL;
+}
+
 bool ResourceGraphInfo::allocateResources(const CResources & value, const CResources & limit)
 {
     if (resources.addExceeds(value, limit))
@@ -897,6 +932,12 @@ bool ResourceGraphInfo::mergeInSource(ResourceGraphInfo & other, const CResource
     if (options->checkResources() && !allocateResources(other.resources, limit))
         return false;
 
+    mergeGraph(other, isConditionalLink, mergeConditions);
+    return true;
+}
+
+void ResourceGraphInfo::mergeGraph(ResourceGraphInfo & other, bool isConditionalLink, bool mergeConditions)
+{
 #ifdef TRACE_RESOURCING
     DBGLOG("Merging%s source into%s sink", other.isUnconditional ? "" : " conditional", isUnconditional ? "" : " conditional");
     other.display();
@@ -938,8 +979,22 @@ bool ResourceGraphInfo::mergeInSource(ResourceGraphInfo & other, const CResource
             conditions.append(OLINK(other.conditions.item(i)));
     }
     
-
     //sources and sinks are updated elsewhere...
+}
+
+
+bool ResourceGraphInfo::mergeInSibling(ResourceGraphInfo & other, const CResources & limit)
+{
+    if ((!isUnconditional || !other.isUnconditional) && !hasSameConditions(other))
+        return false;
+
+    if (isDependentOn(other, false) || other.isDependentOn(*this, false))
+        return false;
+
+    if (options->checkResources() && !allocateResources(other.resources, limit))
+        return false;
+
+    mergeGraph(other, false, false);
     return true;
 }
 
@@ -1323,9 +1378,9 @@ IHqlExpression * ResourcerInfo::createSpiller(IHqlExpression * transformed, bool
         else
         {
             if (transformed->isDataset())
-                split.setown(createDataset(no_split, LINK(transformed), createAttribute(balancedAtom)));
+                split.setown(createDataset(no_split, LINK(transformed), createComma(createAttribute(balancedAtom), createUniqueId())));
             else
-                split.setown(createRow(no_split, LINK(transformed), createAttribute(balancedAtom)));
+                split.setown(createRow(no_split, LINK(transformed), createComma(createAttribute(balancedAtom), createUniqueId())));
             split.setown(cloneInheritedAnnotations(original, split));
         }
 
@@ -1487,6 +1542,7 @@ bool ResourcerInfo::expandRatherThanSpill(bool noteOtherSpills)
         case no_nohoist:
         case no_section:
         case no_sectioninput:
+        case no_dataset_alias:
             expr = expr->queryChild(0);
             break;
         case no_newusertable:
@@ -1612,6 +1668,7 @@ bool ResourcerInfo::expandRatherThanSplit()
         case no_compound_inline:
         case no_section:
         case no_sectioninput:
+        case no_dataset_alias:
             break;
         case no_select:
             if (options->targetClusterType == RoxieCluster)
@@ -1786,6 +1843,8 @@ EclResourcer::EclResourcer(IErrorReceiver * _errors, IConstWorkUnit * _wu, Clust
     options.expandSingleConstRow = true;
     options.createSpillAsDataset = _translatorOptions.optimizeSpillProject && (targetClusterType != HThorCluster);
     options.useLinkedRawIterator = _translatorOptions.useLinkedRawIterator;
+    options.combineSiblings = _translatorOptions.combineSiblingGraphs && (targetClusterType != HThorCluster) && (targetClusterType != RoxieCluster);
+    options.optimizeSharedInputs = _translatorOptions.optimizeSharedGraphInputs && options.combineSiblings;
 }
 
 EclResourcer::~EclResourcer()               
@@ -2364,6 +2423,10 @@ protected:
 
     bool isEvaluateable(IHqlExpression * ds, bool ignoreInline = false)
     {
+        //Don't hoist an alias - it could create unnecessary duplicate spills - hoist its input
+        if (ds->getOperator() == no_dataset_alias)
+            return false;
+
         //Not allowed to hoist
         if (isContextDependent(ds, (conditionalDepth == 0), true))
             return false;
@@ -2583,6 +2646,25 @@ bool EclResourcer::findSplitPoints(IHqlExpression * expr)
             if (options.preventSteppedSplit)
                 insideNeverSplit = true;
             break;
+        case no_compound_diskread:
+        case no_compound_disknormalize:
+        case no_compound_diskaggregate:
+        case no_compound_diskcount:
+        case no_compound_diskgroupaggregate:
+        case no_compound_indexread:
+        case no_compound_indexnormalize:
+        case no_compound_indexaggregate:
+        case no_compound_indexcount:
+        case no_compound_indexgroupaggregate:
+        case no_compound_childread:
+        case no_compound_childnormalize:
+        case no_compound_childaggregate:
+        case no_compound_childcount:
+        case no_compound_childgroupaggregate:
+        case no_compound_selectnew:
+        case no_compound_inline:
+            insideNeverSplit = true;
+            break;
         }
 
         ITypeInfo * type = expr->queryType();
@@ -3453,18 +3535,26 @@ void EclResourcer::spotUnbalancedSplitters(IHqlExpression * expr, unsigned which
     if (!info)
         return;
 
-    if (info->currentSource == whichSource)
+    if (graph && info->graph && info->graph != graph)
     {
-        if (info->pathToSplitter != path)
-            info->balanced = false;
+        if ((info->currentSource == whichSource) && (info->pathToSplitter != path))
+            graph->unbalancedExternalSources.append(*LINK(expr->queryBody()));
+        info->currentSource = whichSource;
+        info->pathToSplitter.set(path);
         return;
     }
+    else
+    {
+        if (info->currentSource == whichSource)
+        {
+            if (info->pathToSplitter != path)
+                info->balanced = false;
+            return;
+        }
 
-    if (graph && info->graph && info->graph != graph)
-        return;
-
-    info->currentSource = whichSource;
-    info->pathToSplitter.set(path);
+        info->currentSource = whichSource;
+        info->pathToSplitter.set(path);
+    }
 
     if (info->containsActivity)
     {
@@ -3557,6 +3647,65 @@ void EclResourcer::spotUnbalancedSplitters(HqlExprArray & exprs)
     }
 }
 
+void EclResourcer::spotSharedInputs(IHqlExpression * expr, ResourceGraphInfo * graph)
+{
+    ResourcerInfo * info = queryResourceInfo(expr);
+    if (!info)
+        return;
+
+    if (info->graph && info->graph != graph)
+    {
+        IHqlExpression * body = expr->queryBody();
+        if (!graph->unbalancedExternalSources.contains(*body))
+            graph->balancedExternalSources.append(*LINK(body));
+
+        return;
+    }
+
+    if (info->isSplit())
+    {
+        //overload currentSource to track if we have visited this splitter before.  It cannot have value value NotFound up to now
+        if (info->currentSource == NotFound)
+            return;
+        info->currentSource = NotFound;
+    }
+
+    if (info->containsActivity)
+    {
+        unsigned first = getFirstActivityArgument(expr);
+        unsigned num = getNumActivityArguments(expr);
+        unsigned last = first + num;
+        for (unsigned idx = first; idx < last; idx++)
+        {
+            spotSharedInputs(expr->queryChild(idx), graph);
+        }
+    }
+}
+
+void EclResourcer::spotSharedInputs()
+{
+    //Thor only handles one graph at a time, so only walk expressions within a single graph.
+    ForEachItemIn(i1, graphs)
+    {
+        ResourceGraphInfo & curGraph = graphs.item(i1);
+        HqlExprCopyArray visited;
+        ForEachItemIn(i2, curGraph.sinks)
+        {
+            ResourceGraphLink & cur = curGraph.sinks.item(i2);
+            IHqlExpression * curExpr = cur.sourceNode;
+            if (!visited.contains(*curExpr))
+            {
+                ResourcerInfo * info = queryResourceInfo(curExpr);
+                if (!info->isExternalSpill() && !info->expandRatherThanSpill(true))
+                {
+                    spotSharedInputs(curExpr, &curGraph);
+                    visited.append(*curExpr);
+                }
+            }
+        }
+    }
+}
+
 //------------------------------------------------------------------------------------------
 // PASS6: Merge sub graphs that can share resources and don't have dependencies
 // MORE: Once sources are merged, should try merging between trees.
@@ -3638,7 +3787,7 @@ bool EclResourcer::queryMergeGraphLink(ResourceGraphLink & link)
 }
 
 
-void EclResourcer::mergeSubGraphs(unsigned pass)
+unsigned EclResourcer::getMaxDepth() const
 {
     unsigned maxDepth = 0;
     for (unsigned idx = 0; idx < graphs.ordinality(); idx++)
@@ -3647,7 +3796,13 @@ void EclResourcer::mergeSubGraphs(unsigned pass)
         if (depth > maxDepth)
             maxDepth = depth;
     }
+    return maxDepth;
+}
 
+
+void EclResourcer::mergeSubGraphs(unsigned pass)
+{
+    unsigned maxDepth = getMaxDepth();
     for (unsigned curDepth = maxDepth+1; curDepth-- != 0;)
     {
 mergeAgain:
@@ -3721,11 +3876,48 @@ mergeAgain:
             }
         }
     }
+}
 
-    ForEachItemInRev(idx2, graphs)
+void EclResourcer::mergeSiblings()
+{
+    unsigned maxDepth = getMaxDepth();
+    for (unsigned curDepth = maxDepth+1; curDepth-- != 0;)
     {
-        if (graphs.item(idx2).isDead)
-            graphs.remove(idx2);
+        for (unsigned idx = 0; idx < graphs.ordinality(); idx++)
+        {
+            ResourceGraphInfo & cur = graphs.item(idx);
+            if ((cur.getDepth() == curDepth) && !cur.isDead)
+            {
+                ForEachItemIn(idxSource, cur.sources)
+                {
+                    ResourceGraphLink & curLink = cur.sources.item(idxSource);
+                    ResourceGraphInfo * source = curLink.sourceGraph;
+                    IHqlExpression * sourceNode = curLink.sourceNode;
+                    ResourcerInfo * sourceInfo = queryResourceInfo(sourceNode);
+                    if (sourceInfo->neverSplit || sourceInfo->expandRatherThanSplit())
+                        continue;
+
+                    for (unsigned iSink = 0; iSink < source->sinks.ordinality(); )
+                    {
+                        ResourceGraphLink & secondLink = source->sinks.item(iSink);
+                        ResourceGraphInfo * sink = secondLink.sinkGraph;
+                        if (sink && (sink != &cur) && !sink->isDead && sourceNode->queryBody() == secondLink.sourceNode->queryBody())
+                        {
+                            if (cur.mergeInSibling(*sink, *resourceLimit))
+                            {
+                                //NB: Following cannot remove sources below the current index.
+                                replaceGraphReferences(sink, &cur);
+                                sink->isDead = true;
+                            }
+                            else
+                                iSink++;
+                        }
+                        else
+                           iSink++;
+                    }
+                }
+            }
+        }
     }
 }
 
@@ -3733,6 +3925,15 @@ void EclResourcer::mergeSubGraphs()
 {
     for (unsigned pass=0; pass < 2; pass++)
         mergeSubGraphs(pass);
+
+    if (options.combineSiblings)
+        mergeSiblings();
+
+    ForEachItemInRev(idx2, graphs)
+    {
+        if (graphs.item(idx2).isDead)
+            graphs.remove(idx2);
+    }
 }
 
 //------------------------------------------------------------------------------------------
@@ -4140,25 +4341,23 @@ IHqlExpression * EclResourcer::createResourced(IHqlExpression * expr, ResourceGr
     if (info->graph != ownerGraph)
     {
         assertex(!defineSideEffect);
+        bool isShared = options.optimizeSharedInputs && ownerGraph && ownerGraph->isSharedInput(expr);
+        if (isShared)
+        {
+            IHqlExpression * mapped = ownerGraph->queryMappedSharedInput(expr->queryBody());
+            if (mapped)
+                return LINK(mapped);
+        }
+
         IHqlExpression * source;
         if (info->expandRatherThanSpill(true))
         {
-            if (options.minimiseSpills)
-            {
-                OwnedHqlExpr resourced = doCreateResourced(expr, ownerGraph, expandInParent, false);
-                if (queryAddUniqueToActivity(resourced))
-                    source = appendUniqueAttr(resourced);
-                else
-                    source = LINK(resourced);
-            }
+            bool expandChildInParent = options.minimiseSpills ? expandInParent : true;
+            OwnedHqlExpr resourced = doCreateResourced(expr, ownerGraph, expandChildInParent, false);
+            if (queryAddUniqueToActivity(resourced))
+                source = appendUniqueAttr(resourced);
             else
-            {
-                OwnedHqlExpr child = doCreateResourced(expr, info->graph, true, false);
-                if (queryAddUniqueToActivity(child))
-                    source = appendUniqueAttr(child);
-                else
-                    source = LINK(child);
-            }
+                source = LINK(resourced);
         }
         else
         {
@@ -4186,6 +4385,12 @@ IHqlExpression * EclResourcer::createResourced(IHqlExpression * expr, ResourceGr
             }
         }
 
+        if (isShared)
+        {
+            source = createDatasetF(no_split, source, createAttribute(balancedAtom), createUniqueId(), NULL);
+            ownerGraph->addSharedInput(expr->queryBody(), source);
+        }
+
         return source;
     }
 
@@ -4420,6 +4625,8 @@ void EclResourcer::resourceGraph(HqlExprArray & exprs, HqlExprArray & transforme
     trace();
 #endif
     spotUnbalancedSplitters(exprs);
+    if (options.optimizeSharedInputs)
+        spotSharedInputs();
 
     if (spotThroughAggregate)
         optimizeAggregates();
@@ -4472,15 +4679,16 @@ void expandLists(HqlExprArray & args, IHqlExpression * expr)
 
 IHqlExpression * resourceThorGraph(HqlCppTranslator & translator, IHqlExpression * expr, ClusterType targetClusterType, unsigned clusterSize, IHqlExpression * graphIdExpr)
 {
-    EclResourcer resourcer(translator.queryErrors(), translator.wu(), targetClusterType, clusterSize, translator.queryOptions());
-    if (graphIdExpr)
-        resourcer.setNewChildQuery(graphIdExpr, 0);
-
-    HqlExprArray exprs;
-    expandLists(exprs, expr);
-
     HqlExprArray transformed;
-    resourcer.resourceGraph(exprs, transformed);
+    {
+        EclResourcer resourcer(translator.queryErrors(), translator.wu(), targetClusterType, clusterSize, translator.queryOptions());
+        if (graphIdExpr)
+            resourcer.setNewChildQuery(graphIdExpr, 0);
+
+        HqlExprArray exprs;
+        expandLists(exprs, expr);
+        resourcer.resourceGraph(exprs, transformed);
+    }
     hoistNestedCompound(translator, transformed);
     return createActionList(transformed);
 }
@@ -4490,21 +4698,23 @@ static IHqlExpression * doResourceGraph(HqlCppTranslator & translator, HqlExprCo
                                         ClusterType targetClusterType, unsigned clusterSize,
                                         IHqlExpression * graphIdExpr, unsigned * numResults, bool isChild, bool useGraphResults)
 {
-    EclResourcer resourcer(translator.queryErrors(), translator.wu(), targetClusterType, clusterSize, translator.queryOptions());
-    if (isChild)
-        resourcer.setChildQuery(true); 
-    resourcer.setNewChildQuery(graphIdExpr, *numResults);
-    resourcer.setUseGraphResults(useGraphResults);
+    HqlExprArray transformed;
+    {
+        EclResourcer resourcer(translator.queryErrors(), translator.wu(), targetClusterType, clusterSize, translator.queryOptions());
+        if (isChild)
+            resourcer.setChildQuery(true);
+        resourcer.setNewChildQuery(graphIdExpr, *numResults);
+        resourcer.setUseGraphResults(useGraphResults);
 
-    if (activeRows)
-        resourcer.tagActiveCursors(*activeRows);
+        if (activeRows)
+            resourcer.tagActiveCursors(*activeRows);
 
-    HqlExprArray exprs;
-    expandLists(exprs, expr);
+        HqlExprArray exprs;
+        expandLists(exprs, expr);
 
-    HqlExprArray transformed;
-    resourcer.resourceGraph(exprs, transformed);
-    *numResults = resourcer.numGraphResults();
+        resourcer.resourceGraph(exprs, transformed);
+        *numResults = resourcer.numGraphResults();
+    }
     hoistNestedCompound(translator, transformed);
     return createActionList(transformed);
 }
@@ -4528,13 +4738,15 @@ IHqlExpression * resourceLoopGraph(HqlCppTranslator & translator, HqlExprCopyArr
 
 IHqlExpression * resourceRemoteGraph(HqlCppTranslator & translator, IHqlExpression * expr, ClusterType targetClusterType, unsigned clusterSize)
 {
-    EclResourcer resourcer(translator.queryErrors(), translator.wu(), targetClusterType, clusterSize, translator.queryOptions());
+    HqlExprArray transformed;
+    {
+        EclResourcer resourcer(translator.queryErrors(), translator.wu(), targetClusterType, clusterSize, translator.queryOptions());
 
-    HqlExprArray exprs;
-    expandLists(exprs, expr);
+        HqlExprArray exprs;
+        expandLists(exprs, expr);
 
-    HqlExprArray transformed;
-    resourcer.resourceRemoteGraph(exprs, transformed);
+        resourcer.resourceRemoteGraph(exprs, transformed);
+    }
     hoistNestedCompound(translator, transformed);
     return createActionList(transformed);
 }
@@ -4555,12 +4767,30 @@ d) if (a, b(f1) +b(f2), c) needs to link b twice though!
 
 */
 
+/*
+This transformer converts spill activities to no_dataset/no_output, and also converts splitters of splitters into
+a single splitter.
+*/
+
 class SpillActivityTransformer : public NewHqlTransformer
 {
 public:
     SpillActivityTransformer();
 
+protected:
+    virtual void analyseExpr(IHqlExpression * expr);
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
+
+    bool isUnbalanced(IHqlExpression * body)
+    {
+        ANewTransformInfo * info = queryTransformExtra(body);
+        return info->spareByte1 != 0;
+    }
+    void setUnbalanced(IHqlExpression * body)
+    {
+        ANewTransformInfo * info = queryTransformExtra(body);
+        info->spareByte1 = true;
+    }
 };
 
 static HqlTransformerInfo spillActivityTransformerInfo("SpillActivityTransformer");
@@ -4569,6 +4799,30 @@ SpillActivityTransformer::SpillActivityTransformer()
 { 
 }
 
+void SpillActivityTransformer::analyseExpr(IHqlExpression * expr)
+{
+    IHqlExpression * body = expr->queryBody();
+    if (alreadyVisited(body))
+        return;
+    if (body->getOperator() == no_split)
+    {
+        IHqlExpression * input = body->queryChild(0);
+        if (input->getOperator() == no_split)
+        {
+            loop
+            {
+                IHqlExpression * cur = input->queryChild(0);
+                if (cur->getOperator() != no_split)
+                    break;
+                input = cur;
+            }
+            if (!body->hasProperty(balancedAtom))
+                setUnbalanced(input->queryBody());
+        }
+    }
+    NewHqlTransformer::analyseExpr(expr);
+}
+
 IHqlExpression * SpillActivityTransformer::createTransformed(IHqlExpression * expr)
 {
     IHqlExpression * annotation = queryTransformAnnotation(expr);
@@ -4577,6 +4831,16 @@ IHqlExpression * SpillActivityTransformer::createTransformed(IHqlExpression * ex
 
     switch (expr->getOperator())
     {
+    case no_split:
+        {
+            IHqlExpression * input = expr->queryChild(0);
+            if (input->getOperator() == no_split)
+                return transform(input);
+            OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
+            if (transformed->hasProperty(balancedAtom) && isUnbalanced(expr))
+                return removeProperty(transformed, balancedAtom);
+            return transformed.getClear();
+        }
     case no_writespill:
         {
             HqlExprArray args;
@@ -4608,5 +4872,6 @@ IHqlExpression * SpillActivityTransformer::createTransformed(IHqlExpression * ex
 IHqlExpression * convertSpillsToActivities(IHqlExpression * expr)
 {
     SpillActivityTransformer transformer;
+    transformer.analyse(expr, 0);
     return transformer.transformRoot(expr);
 }

ecl/hqlcpp/hqlresource.ipp

@@ -63,6 +63,8 @@ public:
     bool     expandSingleConstRow;
     bool     createSpillAsDataset;
     bool     useLinkedRawIterator;
+    bool     optimizeSharedInputs;
+    bool     combineSiblings;
 
     IHqlExpression * graphIdExpr;
     unsigned nextResult;
@@ -154,14 +156,21 @@ public:
     bool hasSameConditions(ResourceGraphInfo & other);
     bool isDependentOn(ResourceGraphInfo & other, bool allowDirect);
     bool isVeryCheap();
+    bool mergeInSibling(ResourceGraphInfo & other, const CResources & limit);
     bool mergeInSource(ResourceGraphInfo & other, const CResources & limit, bool isConditionalLink);
     void removeResources(const CResources & value);
     void replaceReferences(ResourceGraphInfo * oldGraph, ResourceGraphInfo * newGraph);
 
+    bool isSharedInput(IHqlExpression * expr);
+    void addSharedInput(IHqlExpression * expr, IHqlExpression * mapped);
+    IHqlExpression * queryMappedSharedInput(IHqlExpression * expr);
+
 protected:
     void display();
     void expandDependants(ResourceGraphArray & indirectSources);
     void gatherDependants(bool recalculate);
+    void mergeGraph(ResourceGraphInfo & other, bool isConditionalLink, bool mergeConditions);
+
 public:
     OwnedHqlExpr createdGraph;
     CResourceOptions * options;
@@ -170,6 +179,9 @@ public:
     GraphLinkArray sinks;
     ResourceGraphArray indirectSources;
     HqlExprArray conditions;
+    HqlExprArray sharedInputs;
+    HqlExprArray unbalancedExternalSources;
+    HqlExprArray balancedExternalSources;
     CResources resources;
     unsigned depth;
     bool beenResourced;
@@ -350,11 +362,16 @@ protected:
     bool queryMergeGraphLink(ResourceGraphLink & link);
     void mergeSubGraphs();
     void mergeSubGraphs(unsigned pass);
+    void mergeSiblings();
 
 //Pass 6b
     void spotUnbalancedSplitters(IHqlExpression * expr, unsigned whichSource, IHqlExpression * path, ResourceGraphInfo * graph);
     void spotUnbalancedSplitters(HqlExprArray & exprs);
 
+//Pass 6c
+    void spotSharedInputs(IHqlExpression * expr, ResourceGraphInfo * graph);
+    void spotSharedInputs();
+
 //Pass 7
     bool optimizeAggregate(IHqlExpression * expr);
     void optimizeAggregates();
@@ -378,6 +395,7 @@ protected:
     void doCheckRecursion(ResourceGraphInfo * graph, PointerArray & visited);
     void checkRecursion(ResourceGraphInfo * graph, PointerArray & visited);
     void checkRecursion(ResourceGraphInfo * graph);
+    unsigned getMaxDepth() const;
 
 protected:
     Owned<IConstWorkUnit> wu;

ecl/hqlcpp/hqlsource.cpp

@@ -196,6 +196,7 @@ bool isSimpleSource(IHqlExpression * expr)
         case no_sectioninput:
         case no_nofold:
         case no_nohoist:
+        case no_dataset_alias:
             break;
         default:
             return false;
@@ -950,6 +951,7 @@ void SourceBuilder::analyse(IHqlExpression * expr)
     case no_sectioninput:
     case no_nofold:
     case no_nohoist:
+    case no_dataset_alias:
         break;
     case no_preload:
         isPreloaded = true;
@@ -5902,6 +5904,7 @@ void MonitorExtractor::extractAllFilters(IHqlExpression * dataset)
         case no_stepped:
         case no_grouped:
         case no_alias_scope:
+        case no_dataset_alias:
             break;
         default:
             UNIMPLEMENTED;
@@ -7200,4 +7203,3 @@ ABoundActivity * HqlCppTranslator::doBuildActivityFetch(BuildCtx & ctx, IHqlExpr
     throwError1(HQLERR_FetchNotSupportMode, getOpString(kind));
     return NULL;
 }
-

ecl/hqlcpp/hqlttcpp.cpp

@@ -97,6 +97,7 @@ static bool isWorthHoisting(IHqlExpression * expr, bool asSubQuery)
         case no_nohoist:
         case no_section:
         case no_sectioninput:
+        case no_dataset_alias:
             expr = expr->queryChild(0);
             break;
         case no_fail:
@@ -3869,6 +3870,7 @@ void CompoundSourceTransformer::analyseGatherInfo(IHqlExpression * expr)
     case no_stepped:
     case no_section:
     case no_sectioninput:
+    case no_dataset_alias:
         {
             IHqlExpression * dataset = expr->queryChild(0);
             extra->inherit(*queryBodyExtra(dataset));
@@ -7586,6 +7588,7 @@ static bool isFilteredIndex(IHqlExpression * expr)
         case no_sorted:
         case no_stepped:
         case no_grouped:
+        case no_dataset_alias:
             break;
         case no_keyindex:
         case no_newkeyindex:
@@ -7984,8 +7987,8 @@ inline void getDatasetRange(IHqlExpression * expr, unsigned & first, unsigned &
     first = 0;
     switch (getChildDatasetType(expr))
     {
-    case childdataset_addfiles:
-    case childdataset_merge:
+    case childdataset_many_noscope:
+    case childdataset_many:
         max = expr->numChildren();
         break;
     case childdataset_if:
@@ -8221,8 +8224,8 @@ void LeftRightSelectorNormalizer::analyseExpr(IHqlExpression * expr)
         switch (getChildDatasetType(expr))
         {
         case childdataset_none:
-        case childdataset_addfiles:
-        case childdataset_merge:
+        case childdataset_many_noscope:
+        case childdataset_many:
         case childdataset_map:
         case childdataset_dataset_noscope:
         case childdataset_if:
@@ -8514,6 +8517,7 @@ IHqlExpression * HqlLinkedChildRowTransformer::createTransformedBody(IHqlExpress
 }
 
 //---------------------------------------------------------------------------
+
 HqlScopeTaggerInfo::HqlScopeTaggerInfo(IHqlExpression * _expr) : MergingTransformInfo(_expr)
 {
     if (!onlyTransformOnce() && isIndependentOfScope(_expr))
@@ -9086,8 +9090,225 @@ void HqlScopeTagger::reportError(const char * msg, bool warning)
 }
 
 
+//---------------------------------------------------------------------------------------------------------------------
+
+SharedTableInfo * ImplicitAliasTransformInfo::uses(IHqlExpression * tableBody) const
+{
+   ForEachItemIn(i, sharedTables)
+   {
+        SharedTableInfo & cur = sharedTables.item(i);
+        if (cur.dataset == tableBody)
+            return &cur;
+   }
+   return NULL;
+}
+
+void ImplicitAliasTransformInfo::add(SharedTableInfo * table)
+{
+    sharedTables.append(*LINK(table));
+}
+
+void ImplicitAliasTransformInfo::addAmbiguity(SharedTableInfo * table)
+{
+    containsAmbiguity = true;
+    merge(table);
+}
+
+void ImplicitAliasTransformInfo::merge(SharedTableInfo * table)
+{
+    ForEachItemIn(i, sharedTables)
+    {
+        SharedTableInfo & cur = sharedTables.item(i);
+        if (cur.dataset == table->dataset)
+        {
+            if (cur.depth < table->depth)
+                sharedTables.replace(*LINK(table), i);
+            return;
+        }
+    }
+    add(table);
+}
+
+void ImplicitAliasTransformInfo::inherit(const ImplicitAliasTransformInfo * other)
+{
+    ForEachItemIn(i, other->sharedTables)
+        merge(&other->sharedTables.item(i));
+}
+
+
+static HqlTransformerInfo implicitAliasTransformerInfo("ImplicitAliasTransformer");
+ImplicitAliasTransformer::ImplicitAliasTransformer() : NewHqlTransformer(implicitAliasTransformerInfo)
+{
+    seenShared = true;
+    seenAmbiguity = false;
+}
 
-//---------------------------------------------------------------------------
+
+void ImplicitAliasTransformer::analyseExpr(IHqlExpression * _expr)
+{
+    IHqlExpression * body = _expr->queryBody();
+    if (alreadyVisited(body))
+    {
+        if ((pass == 0) && body->isDataset())
+        {
+            switch (body->getOperator())
+            {
+            case no_rows:
+                break;
+            default:
+                seenShared = true;
+                queryExtra(body)->shared.setown(new SharedTableInfo(body, 0));
+                break;
+            }
+        }
+        return;
+    }
+
+    NewHqlTransformer::analyseExpr(body);
+    if (pass == 0)
+        return;
+
+    ImplicitAliasTransformInfo * extra = queryExtra(body);
+    if (extra->shared)
+        extra->add(extra->shared);
+
+    switch (body->getOperator())
+    {
+    case no_activerow:
+    case no_filepos:
+    case no_file_logicalname:
+    case no_offsetof:
+    case no_joined:
+    case no_colon:
+    case no_globalscope:
+    case no_attr:
+        return;
+    case no_select:
+        {
+            bool isNew;
+            IHqlExpression * ds = querySelectorDataset(body, isNew);
+            if (isNew)
+            {
+                ImplicitAliasTransformInfo * dsExtra = queryExtra(ds->queryBody());
+                extra->inherit(dsExtra);
+            }
+            return;
+        }
+    }
+
+    IHqlExpression * dataset = NULL;
+    switch (getChildDatasetType(body))
+    {
+    case childdataset_none:
+    case childdataset_many_noscope:
+    case childdataset_many:
+    case childdataset_map:
+    case childdataset_dataset_noscope:
+    case childdataset_if:
+    case childdataset_case:
+    case childdataset_evaluate:
+    case childdataset_left:
+    case childdataset_leftright:
+    case childdataset_same_left_right:
+    case childdataset_nway_left_right:
+        break;
+    case childdataset_dataset:
+    case childdataset_datasetleft:
+    case childdataset_top_left_right:
+        dataset = body->queryChild(0)->queryBody();
+        break;
+    default:
+        UNIMPLEMENTED;
+    }
+
+    ForEachChild(i, body)
+    {
+        IHqlExpression * cur = body->queryChild(i);
+        ImplicitAliasTransformInfo * childExtra = queryExtra(cur->queryBody());
+        //If this is one of the arguments to an operation which has an active top dataset,
+        //check to see if any of the contained expressions reference this item
+        if (dataset && (i != 0))
+        {
+            SharedTableInfo * match = childExtra->uses(dataset);
+            if (match)
+            {
+                seenAmbiguity = true;
+                SharedTableInfo * nested = createAmbiguityInfo(match->dataset, match->depth+1);
+                extra->addAmbiguity(nested);
+            }
+//            dbglogExpr(_expr);
+//            DBGLOG("Implicit nested table ambiguity spotted in expression");
+        }
+        extra->inherit(childExtra);
+    }
+}
+
+SharedTableInfo * ImplicitAliasTransformer::createAmbiguityInfo(IHqlExpression * dataset, unsigned depth)
+{
+    ForEachItemIn(i, ambiguousTables)
+    {
+        SharedTableInfo & cur = ambiguousTables.item(i);
+        if ((cur.dataset == dataset) && (depth == cur.depth))
+            return &cur;
+    }
+    ambiguousTables.append(*new SharedTableInfo(dataset, depth));
+    return &ambiguousTables.tos();
+}
+
+
+ANewTransformInfo * ImplicitAliasTransformer::createTransformInfo(IHqlExpression * expr)
+{
+    return CREATE_NEWTRANSFORMINFO(ImplicitAliasTransformInfo, expr);
+}
+
+IHqlExpression * ImplicitAliasTransformer::createTransformed(IHqlExpression * expr)
+{
+    IHqlExpression * body = expr->queryBody();
+    if (expr != body)
+    {
+        OwnedHqlExpr newBody = transform(body);
+        return expr->cloneAllAnnotations(newBody);
+    }
+
+    OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(expr);
+    updateOrphanedSelectors(transformed, expr);
+
+    ImplicitAliasTransformInfo * extra = queryExtra(body);
+    if (!extra->containsAmbiguity)
+        return transformed.getClear();
+
+    SharedTableInfo * match = extra->uses(body->queryChild(0)->queryBody());
+    assertex(match && match->depth != 0);
+    if (!match->uid)
+        match->uid.setown(createUniqueId());
+
+    OwnedHqlExpr aliased;
+    IHqlExpression * dataset = transformed->queryChild(0)->queryBody();
+    IHqlExpression * uid = match->uid;
+    aliased.setown(createDataset(no_dataset_alias, LINK(dataset), LINK(uid)));
+
+    //Replace dataset with an aliased variety, and remap all the selectors
+    HqlExprArray args;
+    args.append(*LINK(aliased));
+    replaceSelectors(args, transformed, 1, dataset->queryNormalizedSelector(), aliased->queryNormalizedSelector());
+    return transformed->clone(args);
+}
+
+void ImplicitAliasTransformer::process(HqlExprArray & exprs)
+{
+    analyseArray(exprs, 0);
+    if (!seenShared)
+        return;
+    analyseArray(exprs, 1);
+    if (hasAmbiguity())
+    {
+//        DBGLOG("Implicit nested dataset ambiguity spotted in expression");
+        HqlExprArray transformed;
+        transformRoot(exprs, transformed);
+        replaceArray(exprs, transformed);
+    }
+}
+//---------------------------------------------------------------------------------------------------------------------
 
 /*
   Common up expressions so that all references to the same expression have identical symbols, annotations.
@@ -11332,25 +11553,7 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
         }
 
     case no_call:
-        {
-            LinkedHqlExpr call = expr;
-#if 0
-            IHqlExpression * funcDef = expr->queryFunctionDefinition();
-            OwnedHqlExpr newFuncDef = normalizeRecord(translator, funcDef);
-            if (funcDef != newFuncDef)
-            {
-                HqlExprArray children;
-                transformChildren(expr, children);
-                call.setown(createReboundFunction(newFuncDef, children));
-            }
-#endif
-
-            if (options.ensureRecordsHaveSymbols)
-                if (call->queryRecord())
-                    return transformCall(call);
-            break;
-//          return call.getClear();
-        }
+        return transformCall(expr);
     case no_externalcall:
         //Yuk.... Because we ensure that all records have a name, we need to make sure that external functions that return records
         //also have there return value normalized - otherwise (jtolbert2.xhql) you can create an ambiguity
@@ -11850,6 +12053,18 @@ bool HqlCppTranslator::transformGraphForGeneration(IHqlExpression * query, Workf
     checkNormalized(workflow);
     DEBUG_TIMER("EclServer: tree transform: stored results", msTick()-time4);
 
+    if (queryOptions().createImplicitAliases)
+    {
+        unsigned time = msTick();
+        ForEachItemIn(i, workflow)
+        {
+            ImplicitAliasTransformer normalizer;
+            normalizer.process(workflow.item(i).queryExprs());
+        }
+        DEBUG_TIMERX(queryTimeReporter(), "EclServer: tree transform: implicit alias", msTick()-time);
+        //traceExpressions("after implicit alias", workflow);
+    }
+
     if (outputLibrary && workflow.ordinality() > 1)
     {
         unsigned cnt = 0;

ecl/hqlcpp/hqlttcpp.ipp

@@ -823,6 +823,59 @@ protected:
 
 //---------------------------------------------------------------------------
 
+class SharedTableInfo : public CInterface
+{
+public:
+    SharedTableInfo(IHqlExpression * _dataset, unsigned _depth) : dataset(_dataset), depth(_depth) {}
+
+    IHqlExpression * dataset;
+    unsigned depth;
+    OwnedHqlExpr uid;   // if (depth > 0)
+};
+
+class ImplicitAliasTransformInfo : public NewTransformInfo
+{
+public:
+    ImplicitAliasTransformInfo(IHqlExpression * _expr) : NewTransformInfo(_expr) { containsAmbiguity = false; }
+
+    void add(SharedTableInfo * table);
+    void addAmbiguity(SharedTableInfo * table);
+    void inherit(const ImplicitAliasTransformInfo * other);
+    void merge(SharedTableInfo * table);
+    SharedTableInfo * uses(IHqlExpression * tableBody) const;
+
+public:
+    CIArrayOf<SharedTableInfo> sharedTables;
+    Owned<SharedTableInfo> shared;
+    bool                containsAmbiguity;
+};
+
+
+class ImplicitAliasTransformer : public NewHqlTransformer
+{
+public:
+    ImplicitAliasTransformer();
+
+    void process(HqlExprArray & exprs);
+
+    inline bool hasAmbiguity() const { return seenAmbiguity; }
+
+protected:
+    virtual void analyseExpr(IHqlExpression * expr);
+    virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr);
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr);
+
+    inline ImplicitAliasTransformInfo * queryExtra(IHqlExpression * expr)  { return (ImplicitAliasTransformInfo *)queryTransformExtra(expr); }
+    SharedTableInfo * createAmbiguityInfo(IHqlExpression * dataset, unsigned depth);
+
+protected:
+    CIArrayOf<SharedTableInfo> ambiguousTables;
+    bool seenAmbiguity;
+    bool seenShared;
+};
+
+//---------------------------------------------------------------------------
+
 class ForceLocalTransformInfo : public NewTransformInfo
 {
     friend class ForceLocalTransformer;
@@ -879,6 +932,8 @@ protected:
     bool implicitLinkedChildRows;
 };
 
+//---------------------------------------------------------------------------
+
 class HqlScopeTaggerInfo : public MergingTransformInfo
 {
 public:

ecl/regress/sqfilt5.ecl

@@ -30,15 +30,10 @@ unsigned4 age(udecimal8 dob) := ((todaysDate - dob) / 10000D);
 //MORE: books[1] ave(books)
 
 // Different child operators, all inline.
-house := sqHousePersonBookDs.persons;
 persons := sqHousePersonBookDs.persons;
 books := persons.books;
-
-booksDs := sqBookDs(personid = persons.id);
-personsDs := sqPersonDs(houseid = sqHousePersonBookDs.id);
-booksDsDs := sqBookDs(personid = personsDs.id);
-personsDsDs := sqPersonDs(houseid = sqHouseDs.id);
-booksDsDsDs := sqBookDs(personid = personsDsDs.id);
+books_2 := table(books);
 
 //people with a book worth more than the rest of their books.
-output(sqHousePersonBookDs.persons, { exists(books(price > sum(__alias__(books)(id != books.id), price))); }, named('NumPeopleExceedBookLimit'));
+//Iterate books twice, and ensure that the two cursors are separately accessed.
+output(sqHousePersonBookDs.persons, { exists(books(price > sum(books_2(id != books.id), price))); });

ecl/regress/sqfilt6.ecl

@@ -0,0 +1,42 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+import sq;
+sq.DeclareCommon();
+
+#option ('childQueries', true);
+
+// Test filtering at different levels, making sure parent fields are available in the child query.
+// Also tests scoping of sub expressions using within.
+
+udecimal8 todaysDate := 20040602D;
+unsigned4 age(udecimal8 dob) := ((todaysDate - dob) / 10000D);
+
+//MORE: books[1] ave(books)
+
+// Different child operators, all inline.
+persons := sqHousePersonBookDs.persons;
+books := persons.books;
+books_2 := table(books);
+
+//people with a book worth more than the rest of their books.
+//Iterate books twice, and ensure that the two cursors are separately accessed.
+validBooks := nofold(books(max(id*7,99) != 0));
+validBooks2 := table(validBooks);
+
+output(sqHousePersonBookDs.persons, { exists(validBooks(price > sum(validBooks2(id != books.id), price))); });