Merge pull request #1056 from ghalliday/ds-alias

Initial work towards supporting aliases of datasets

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlattr.cpp

@@ -392,6 +392,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_xmlparse:
     case no_normalizegroup:
     case no_owned_ds:
+    case no_dataset_alias:
 
 //Multiple different kinds of values
     case no_select:
@@ -609,7 +610,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_unused1: case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused17: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
-    case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38: case no_unused39:
+    case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
     case no_is_null:
@@ -2364,6 +2365,7 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
     case no_deserialize:
     case no_executewhen:
     case no_owned_ds:
+    case no_dataset_alias:
         {
             return getRecordCountInfo(ds);
         }

ecl/hql/hqlexpr.cpp

@@ -1405,11 +1405,12 @@ const char *getOpString(node_operator op)
     case no_complex: return ",";
     case no_assign_addfiles: return "+=";
     case no_debug_option_value: return "__DEBUG__";
+    case no_dataset_alias: return "ALIAS";
 
     case no_unused1: case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused17: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
-    case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38: case no_unused39:
+    case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
         return "unused";
@@ -1838,6 +1839,7 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_serialize:
     case no_forcegraph:
     case no_owned_ds:
+    case no_dataset_alias:
         return childdataset_dataset;
     case no_executewhen:
         //second argument is independent of the other arguments
@@ -2109,6 +2111,7 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_executewhen:
     case no_normalizegroup:
     case no_owned_ds:
+    case no_dataset_alias:
         return 1;
     case no_childdataset:
     case no_left:
@@ -2468,6 +2471,7 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_fromxml:
     case no_normalizegroup:
     case no_cogroup:
+    case no_dataset_alias:
         return true;
     case no_select:
     case no_field:
@@ -4888,34 +4892,51 @@ void CHqlExpression::gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyAr
     }
 }
 
+IHqlExpression * CHqlExpression::calcNormalizedSelector() const
+{
+    IHqlExpression * left = &operands.item(0);
+    IHqlExpression * normalizedLeft = left->queryNormalizedSelector();
+    if ((normalizedLeft != left) || ((operands.ordinality() > 2) && hasProperty(newAtom)))
+    {
+        HqlExprArray args;
+        appendArray(args, operands);
+        args.replace(*LINK(normalizedLeft), 0);
+        removeProperty(args, newAtom);
+        return createSelectExpr(args);
+    }
+    return NULL;
+}
 
 //==============================================================================================================
 
-inline ITypeInfo * getSelectType(IHqlExpression * left, IHqlExpression * right)
+CHqlSelectExpression::CHqlSelectExpression(IHqlExpression * left, IHqlExpression * right, IHqlExpression * attr)
+: CHqlExpression(no_select, right->getType(), left, right, attr, NULL)
 {
-    return right->getType();
+#ifdef _DEBUG
+    assertex(!isDataset());
+    assertex(left->getOperator() != no_activerow);
+//  if ((left->getOperator() == no_select) && left->isDatarow())
+//      assertex(left->hasProperty(newAtom) == hasProperty(newAtom));
+#endif
+    normalized.setown(calcNormalizedSelector());
 }
 
-CHqlSelectExpression::CHqlSelectExpression(IHqlExpression * left, IHqlExpression * right, IHqlExpression * attr)
-: CHqlExpression(no_select, getSelectType(left, right), left, right, attr, NULL)
+CHqlSelectExpression::CHqlSelectExpression(HqlExprArray & _ownedOperands)
+: CHqlExpression(no_select, _ownedOperands.item(1).getType(), _ownedOperands)
 {
-    IHqlExpression * normalizedLeft = left->queryNormalizedSelector();
+    IHqlExpression * left = &operands.item(0);
 #ifdef _DEBUG
     assertex(!isDataset());
     assertex(left->getOperator() != no_activerow);
 //  if ((left->getOperator() == no_select) && left->isDatarow())
 //      assertex(left->hasProperty(newAtom) == hasProperty(newAtom));
 #endif
-    if (normalizedLeft != left)
-        normalized.setown(createSelectExpr(LINK(normalizedLeft), LINK(right)));
+    normalized.setown(calcNormalizedSelector());
 }
 
 IHqlExpression * CHqlSelectExpression::clone(HqlExprArray &newkids)
 {
-    if (newkids.ordinality() == 2)
-        return createSelectExpr(&OLINK(newkids.item(0)), &OLINK(newkids.item(1)), NULL);
-    assertex(newkids.ordinality() == 3);
-    return createSelectExpr(&OLINK(newkids.item(0)), &OLINK(newkids.item(1)), &OLINK(newkids.item(2)));
+    return createSelectExpr(newkids);
 }
 
 IHqlExpression * CHqlSelectExpression::queryNormalizedSelector(bool skipIndex)
@@ -5079,17 +5100,9 @@ CHqlRow::CHqlRow(node_operator op, ITypeInfo * type, HqlExprArray & _ownedOperan
     case no_select: 
         if (!hasProperty(newAtom))
         {
-            IHqlExpression * dataset = queryChild(0);
-            IHqlExpression * normalizedDataset = dataset->queryNormalizedSelector();
-            if (dataset != normalizedDataset)
-            {
-                HqlExprArray args;
-                args.append(*LINK(normalizedDataset));
-                args.append(*LINK(queryChild(1)));
-                normalized.setown(createRow(op, args));
-            }
-            break;
+            normalized.setown(calcNormalizedSelector());
         }
+        break;
     }
 
     switch (op)
@@ -5460,14 +5473,8 @@ void CHqlDataset::cacheParent()
     case no_select:
         {
             rootTable = this;
+            normalized.setown(calcNormalizedSelector());
             IHqlExpression * ds = queryChild(0);
-            IHqlExpression * normalizedLeft = ds->queryNormalizedSelector();
-
-            //Normalized form of select does not include any new attributes - since in scope
-            //I think this is best - since at point expression is evaluated, all parent tables should be in scope
-            if (normalizedLeft != ds || queryChild(2))
-                normalized.setown(createInScopeSelectExpr(LINK(normalizedLeft), LINK(queryChild(1))));
-
             container = LINK(queryDatasetCursor(ds)->queryNormalizedSelector(false));
 #ifdef _DEBUG
             assertex(!hasProperty(newAtom) || !isAlwaysActiveRow(ds));
@@ -10955,6 +10962,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
     case no_select:
     case no_fieldmap:
     case no_owned_ds:
+    case no_dataset_alias:
         type.set(datasetType);
         break;
     case no_serialize:
@@ -11470,23 +11478,20 @@ extern IHqlExpression *createDatasetFromRow(IHqlExpression * ownedRow)
 }
 
 
-extern IHqlExpression * createSelectExpr(IHqlExpression * _lhs, IHqlExpression * rhs, IHqlExpression * attr)
+inline IHqlExpression * normalizeSelectLhs(IHqlExpression * lhs, bool & isNew)
 {
-    OwnedHqlExpr lhs = _lhs;
-    bool done = false;
-    do
+    loop
     {
         switch (lhs->getOperator())
         {
         case no_newrow:
-            assertex(!attr);
-            attr = LINK(newSelectAttrExpr);
-            lhs.set(lhs->queryChild(0));
+            assertex(!isNew);
+            isNew = true;
+            lhs = lhs->queryChild(0);
             break;  // round the loop again
         case no_activerow:
-            ::Release(attr);
-            attr = NULL;
-            lhs.set(lhs->queryChild(0));
+            isNew = false;
+            lhs = lhs->queryChild(0);
             break;  // round the loop again
         case no_left:
         case no_right:
@@ -11494,50 +11499,76 @@ extern IHqlExpression * createSelectExpr(IHqlExpression * _lhs, IHqlExpression *
         case no_activetable:
         case no_self:
         case no_selfref:
-            if (attr)
-            {
-                attr->Release();
-                attr = NULL;
-            }
-            done = true;
-            break;
+            isNew = false;
+            return lhs;
         case no_select:
-            if (attr && isAlwaysActiveRow(lhs))
-            {
-                attr->Release();
-                attr = NULL;
-            }
-            done = true;
-            break;
+            if (isNew && isAlwaysActiveRow(lhs))
+                isNew = false;
+            return lhs;
         default:
-            done = true;
-            break;
+            return lhs;
         }
-    } while (!done);
+    }
+}
 
+inline void checkRhsSelect(IHqlExpression * rhs)
+{
 #ifdef _DEBUG
     node_operator rhsOp = rhs->getOperator();
-    assertex(rhsOp == no_field || rhsOp == no_ifblock || rhsOp == no_indirect || attr && attr->queryName() == internalAtom);
+    assertex(rhsOp == no_field || rhsOp == no_ifblock || rhsOp == no_indirect);
 #endif
+}
+
+extern IHqlExpression * createSelectExpr(IHqlExpression * _lhs, IHqlExpression * rhs, bool _isNew)
+{
+    OwnedHqlExpr lhs = _lhs;
+    bool isNew = _isNew;
+    IHqlExpression * normalLhs = normalizeSelectLhs(lhs, isNew);
+    IHqlExpression * newAttr = isNew ? newSelectAttrExpr : NULL;
+
+    checkRhsSelect(rhs);
+
     type_t t = rhs->queryType()->getTypeCode();
     if (t == type_table || t == type_groupedtable)
-        return createDataset(no_select, lhs.getClear(), createComma(rhs, attr));
+        return createDataset(no_select, LINK(normalLhs), createComma(rhs, LINK(newAttr)));
     if (t == type_row)
-        return createRow(no_select, lhs.getClear(), createComma(rhs, attr));
+        return createRow(no_select, LINK(normalLhs), createComma(rhs, LINK(newAttr)));
 
-    IHqlExpression * ret = new CHqlSelectExpression(lhs.getClear(), rhs, attr);
+    IHqlExpression * ret = new CHqlSelectExpression(LINK(normalLhs), rhs, LINK(newAttr));
     return ret->closeExpr();
 }
 
-extern IHqlExpression * createInScopeSelectExpr(IHqlExpression * lhs, IHqlExpression * rhs)
+extern IHqlExpression * createSelectExpr(HqlExprArray & args)
 {
-#ifdef _DEBUG
-    assertex(rhs->isDataset());
-#endif
-    HqlExprArray args;
-    args.append(*lhs);
-    args.append(*rhs);
-    return createDataset(no_select, args);
+    IHqlExpression * lhs = &args.item(0);
+    bool isNew = false;
+    for (unsigned i=2; i < args.ordinality(); i++)
+    {
+        if (args.item(i).queryName() == newAtom)
+        {
+            isNew = true;
+            args.remove(i);
+            break;
+        }
+    }
+
+    IHqlExpression * normalLhs = normalizeSelectLhs(lhs, isNew);
+    if (lhs != normalLhs)
+        args.replace(*LINK(normalLhs), 0);
+    if (isNew)
+        args.append(*LINK(newSelectAttrExpr));
+
+    IHqlExpression * rhs = &args.item(1);
+    checkRhsSelect(rhs);
+
+    type_t t = rhs->queryType()->getTypeCode();
+    if (t == type_table || t == type_groupedtable)
+        return createDataset(no_select, args);
+    if (t == type_row)
+        return createRow(no_select, args);
+
+    IHqlExpression * ret = new CHqlSelectExpression(args);
+    return ret->closeExpr();
 }
 
 IHqlExpression * ensureDataset(IHqlExpression * expr)
@@ -11573,20 +11604,6 @@ extern bool isAlwaysActiveRow(IHqlExpression * expr)
     return false;
 }
 
-extern IHqlExpression * createNewSelectExpr(IHqlExpression * lhs, IHqlExpression * rhs)
-{
-    switch (lhs->getOperator())
-    {
-    case no_left:
-    case no_right:
-    case no_top:
-    case no_activetable:
-        //Minor special cases
-        return createSelectExpr(lhs, rhs, NULL);
-    }
-    return createSelectExpr(lhs, rhs, LINK(newSelectAttrExpr));
-}
-
 IHqlExpression * ensureActiveRow(IHqlExpression * expr)
 {
     if (isAlwaysActiveRow(expr))
@@ -13236,13 +13253,13 @@ IHqlExpression * replaceSelectorDataset(IHqlExpression * expr, IHqlExpression *
 {
     assertex(expr->getOperator() == no_select);
     IHqlExpression * ds = expr->queryChild(0);
-    IHqlExpression * newSelector;
     if ((ds->getOperator() == no_select) && !ds->isDataset())
-        newSelector = replaceSelectorDataset(ds, newDataset);
+    {
+        OwnedHqlExpr newSelector = replaceSelectorDataset(ds, newDataset);
+        return replaceChild(expr, 0, newSelector);
+    }
     else
-        newSelector = LINK(newDataset);
-    //Call createSelectExpr instead of clone() so that no_newrow/no_activerow are handled correctly
-    return createSelectExpr(newSelector, LINK(expr->queryChild(1)), LINK(expr->queryChild(2)));
+        return replaceChild(expr, 0, newDataset);
 }
 
 IHqlExpression * querySkipDatasetMeta(IHqlExpression * dataset)

ecl/hql/hqlexpr.hpp

@@ -374,7 +374,7 @@ enum _node_operator {
     no_unused37,
     no_unused38,
         no_is_null,
-    no_unused39,
+        no_dataset_alias,
     no_unused40,
     no_unused41,
     no_unused52,
@@ -1145,7 +1145,7 @@ interface IHqlExpression : public IInterface
     inline bool isAnnotation() const { return getAnnotationKind() != annotate_none; }
     inline bool isNamedSymbol() const { return getAnnotationKind() == annotate_symbol; }
     inline bool isFunctionDefinition() const { return getOperator() == no_funcdef; }
-    inline bool hasProperty(_ATOM propName) { return queryProperty(propName) != NULL; }
+    inline bool hasProperty(_ATOM propName) const { return queryProperty(propName) != NULL; }
     inline bool hasText() const 
     { 
         IFileContents * contents = queryDefinitionText();
@@ -1328,9 +1328,10 @@ extern HQL_API IHqlExpression* createValue(node_operator op, HqlExprArray& opera
 extern HQL_API IHqlExpression *createValue(node_operator op, IHqlExpression *p1);
 extern HQL_API IHqlExpression* createConstant(int ival);
 extern HQL_API IHqlExpression* createBoolExpr(node_operator op, HqlExprArray& operands);
-extern HQL_API IHqlExpression* createInScopeSelectExpr(IHqlExpression * lhs, IHqlExpression * rhs);
-extern HQL_API IHqlExpression* createSelectExpr(IHqlExpression * lhs, IHqlExpression * rhs, IHqlExpression * attr = NULL);
-extern HQL_API IHqlExpression* createNewSelectExpr(IHqlExpression * lhs, IHqlExpression * rhs);
+extern HQL_API IHqlExpression* createSelectExpr(IHqlExpression * lhs, IHqlExpression * rhs, bool isNew);
+inline IHqlExpression* createSelectExpr(IHqlExpression * lhs, IHqlExpression * rhs) { return createSelectExpr(lhs, rhs, false); }
+inline IHqlExpression* createNewSelectExpr(IHqlExpression * lhs, IHqlExpression * rhs) { return createSelectExpr(lhs, rhs, true); }
+extern HQL_API IHqlExpression* createSelectExpr(HqlExprArray & args);
 
 inline IHqlExpression* createAction(node_operator op, HqlExprArray& operands) { return createValue(op, makeVoidType(), operands); }
 

ecl/hql/hqlexpr.ipp

@@ -97,6 +97,7 @@ protected:
     void initFlagsBeforeOperands();
     void updateFlagsAfterOperands();
 
+    IHqlExpression * calcNormalizedSelector() const;
     void mergeGathered(CopyArray &, CopyArray &);
     IHqlExpression *fixScope(IHqlDataset *table);
     CHqlExpression(node_operator op, ITypeInfo *type, ...);
@@ -247,6 +248,7 @@ class CHqlSelectExpression : public CHqlExpression
 {
 public:
     CHqlSelectExpression(IHqlExpression * left, IHqlExpression * right, IHqlExpression * attr);
+    CHqlSelectExpression(HqlExprArray & ownedOperands);
 
     virtual IHqlExpression *clone(HqlExprArray &newkids);
     virtual IHqlExpression *queryNormalizedSelector(bool skipIndex);

ecl/hql/hqlfold.cpp

@@ -5417,6 +5417,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_callsideeffect:
     case no_outofline:
     case no_owned_ds:
+    case no_dataset_alias:
         exprMapping.set(gatherConstants(expr->queryChild(0)));
         break;
     case no_normalizegroup:

ecl/hql/hqlgram.y

@@ -6991,9 +6991,7 @@ simpleDataSet
                         }
     | ALIAS '(' dataSet ')'
                         {
-                            //MORE: This needs to create a node which is then processed appropriately.
-                            $$.setExpr($3.getExpr());
-                            $$.setPosition($1);
+                            $$.setExpr(createDataset(no_dataset_alias, $3.getExpr(), ::createUniqueId()), $1);
                         }
     | EBCDIC '(' startTopFilter ')' endTopFilter
                         {

ecl/hql/hqlpmap.cpp

@@ -52,10 +52,10 @@ static IHqlExpression * optimizedReplaceSelector(IHqlExpression * expr, IHqlExpr
         {
             IHqlExpression * lhs = expr->queryChild(0);
             IHqlExpression * field = expr->queryChild(1);
-            IHqlExpression * newLhs;
+            OwnedHqlExpr newLhs;
             if (expr->hasProperty(newAtom))
             {
-                newLhs = optimizedReplaceSelector(lhs, oldDataset, newDataset);
+                newLhs.setown(optimizedReplaceSelector(lhs, oldDataset, newDataset));
             }
             else
             {
@@ -70,11 +70,11 @@ static IHqlExpression * optimizedReplaceSelector(IHqlExpression * expr, IHqlExpr
                     return createSelectExpr(LINK(newDataset->queryNormalizedSelector()), LINK(field));
                 }
                 else
-                    newLhs = optimizedReplaceSelector(lhs, oldDataset, newDataset);
+                    newLhs.setown(optimizedReplaceSelector(lhs, oldDataset, newDataset));
             }
 
             if (newLhs)
-                return createSelectExpr(newLhs, LINK(field), LINK(expr->queryChild(2)));
+                return replaceChild(expr, 0, newLhs);
             return NULL;
         }
     case no_implicitcast:

ecl/hql/hqlutil.cpp

@@ -1140,8 +1140,16 @@ IHqlExpression * replaceChild(IHqlExpression * expr, unsigned childIndex, IHqlEx
     if (oldChild == newChild)
         return LINK(expr);
     HqlExprArray args;
-    unwindChildren(args, expr);
-    args.replace(*LINK(newChild), childIndex);
+    if (childIndex == 0)
+    {
+        args.append(*LINK(newChild));
+        unwindChildren(args, expr, 1);
+    }
+    else
+    {
+        unwindChildren(args, expr);
+        args.replace(*LINK(newChild), childIndex);
+    }
     return expr->clone(args);
 }
 

ecl/hqlcpp/hqlcpp.ipp

@@ -760,7 +760,6 @@ class HQLCPP_API HqlCppTranslator : public CInterface, implements IHqlCppTransla
     friend class FetchBuilder;
     friend class MonitorExtractor;
     friend class NlpParseContext;
-    friend class FilterExtractor;
     friend class KeyedJoinInfo;
     friend class ChildGraphBuilder;
 public:
@@ -1570,7 +1569,6 @@ public:
     IHqlExpression * createOrderFromSortList(const DatasetReference & dataset, IHqlExpression * sortList, IHqlExpression * leftSelect, IHqlExpression * rightSelect);
 
     void buildSkewThresholdMembers(BuildCtx & ctx, IHqlExpression * expr);
-    IHqlExpression * doCompare(BuildCtx & ctx, IHqlExpression *sortList, const DatasetReference & dataset);
     void doCompareLeftRight(BuildCtx & ctx, const char * funcname, const DatasetReference & datasetLeft, const DatasetReference & datasetRight, HqlExprArray & left, HqlExprArray & right);
     void buildSlidingMatchFunction(BuildCtx & ctx, HqlExprArray & leftEq, HqlExprArray & rightEq, HqlExprArray & slidingMatches, const char * funcname, unsigned childIndex, const DatasetReference & datasetL, const DatasetReference & datasetR);
     void doBuildIndexOutputTransform(BuildCtx & ctx, IHqlExpression * record, SharedHqlExpr & rawRecord);

ecl/hqlcpp/hqlcppds.cpp

@@ -1838,6 +1838,12 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
     node_operator op = expr->getOperator();
     switch (op)
     {
+    case no_dataset_alias:
+        {
+            OwnedHqlExpr uniqueChild = normalizeDatasetAlias(expr);
+            doBuildDataset(ctx, uniqueChild, tgt, format);
+            return;
+        }
     case no_alias:
         doBuildExprAlias(ctx, expr, &tgt);
         return;
@@ -3112,6 +3118,12 @@ BoundRow * HqlCppTranslator::buildDatasetIterate(BuildCtx & ctx, IHqlExpression
 
     switch (expr->getOperator())
     {
+    case no_dataset_alias:
+        {
+            OwnedHqlExpr uniqueChild = normalizeDatasetAlias(expr);
+            BoundRow * childCursor = buildDatasetIterate(ctx, uniqueChild, needToBreak);
+            return rebindTableCursor(ctx, expr, childCursor, no_none, NULL);
+        }
     case no_null:
         buildFilter(ctx, queryBoolExpr(false));
         return NULL;

ecl/hqlcpp/hqlcpputil.cpp

@@ -215,6 +215,14 @@ bool storePointerInArray(ITypeInfo * type)
     return type->isReference() && isTypePassedByAddress(type); 
 }
 
+//Convert no_dataset_alias(expr, uid) to expr'
+IHqlExpression * normalizeDatasetAlias(IHqlExpression * expr)
+{
+    IHqlExpression * uid = expr->queryProperty(_uid_Atom);
+    assertex(uid);
+    return appendOwnedOperand(expr->queryChild(0), LINK(uid));
+}
+
 //---------------------------------------------------------------------------
 
 bool isSelectSortedTop(IHqlExpression * selectExpr)

ecl/hqlcpp/hqlcpputil.hpp

@@ -46,6 +46,7 @@ extern IHqlExpression * addMemberSelector(IHqlExpression * expr, IHqlExpression
 extern IHqlExpression * addExpressionModifier(IHqlExpression * expr, typemod_t modifier, IInterface * extra=NULL);
 extern void expandFieldNames(StringBuffer & out, IHqlExpression * record, const char * sep, IHqlExpression * formatFunc);
 extern IHqlExpression * ensurePositiveOrZeroInt64(IHqlExpression * expr);
+extern IHqlExpression * normalizeDatasetAlias(IHqlExpression * expr);
 
 extern void getOutputLibraryName(SCMStringBuffer & libraryName, IConstWorkUnit * wu);
 extern bool canCreateTemporary(IHqlExpression * expr);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5993,9 +5993,6 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_compound_selectnew:
                 result = doBuildActivityCompoundSelectNew(ctx, expr);
                 break;
-            case no_keyed:
-                result = buildCachedActivity(ctx, expr->queryChild(0));
-                break;
             case no_denormalize:
             case no_denormalizegroup:
                 result = doBuildActivityDenormalize(ctx, expr);
@@ -6127,6 +6124,8 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_globalscope:
             case no_thisnode:
             case no_forcegraph:
+            case no_keyed:
+            case no_dataset_alias:
                 result = buildCachedActivity(ctx, expr->queryChild(0));
                 break;
             case no_alias_scope:
@@ -7342,178 +7341,6 @@ public:
     CHqlBoundTarget     value;
 };
 
-static HqlTransformerInfo filterExtractorInfo("FilterExtractor");
-class FilterExtractor : public NewHqlTransformer
-{
-public:
-    FilterExtractor(HqlCppTranslator & _translator, BuildCtx & _classctx, BuildCtx & _ctx, IHqlExpression * _rootTable) 
-        : NewHqlTransformer(filterExtractorInfo), translator(_translator), classctx(_classctx), ctx(_ctx) { rootTable.set(_rootTable->queryNormalizedSelector()); }
-
-    void assignCursors(IHqlExpression * helper);
-    IHqlExpression * buildExtractLookupFields(IHqlExpression * expr);
-    IHqlExpression * createTransformed(IHqlExpression * expr);
-
-protected:
-    struct AssignPairs
-    {
-        HqlExprArray from;
-        CIArrayOf<CHqlBoundTargetItem> to;
-    };
-
-    void addTable(IHqlExpression * expr);
-    IHqlExpression * extractExpr(IHqlExpression * expr, AssignPairs & assigns);
-    IHqlExpression * querySelectorTable(IHqlExpression * expr);
-
-
-public:
-    HqlCppTranslator & translator;
-    BuildCtx & classctx;
-    BuildCtx & ctx;
-    CIArray tables;
-    AssignPairs  ctxAssign;
-    HqlExprArray ctxAssignDirectFrom;
-    HqlExprArray ctxAssignDirectTo;
-    AssignPairs  extractAssign;
-    HqlExprAttr rootTable;
-};
-
-void FilterExtractor::addTable(IHqlExpression * table)
-{
-    if (table == rootTable)
-        return;
-
-    BoundRow * cursor;
-    if (table)
-        cursor = static_cast<BoundRow *>(ctx.queryAssociation(table, AssocCursor, NULL));
-    else
-        cursor = (BoundRow *)ctx.queryFirstAssociation(AssocCursor);
-
-    assertex(cursor);
-    if (tables.find(*cursor) == NotFound)
-    {
-        StringBuffer newCursorName;
-        translator.getUniqueId(newCursorName.append("cur"));
-        BoundRow * newCursor = translator.bindTableCursor(classctx, cursor->queryDataset(), newCursorName, cursor->querySide(), cursor->querySelSeq());
-
-        IHqlExpression * oldVar = cursor->queryBound();
-        IHqlExpression * newVar = newCursor->queryBound();
-        ctxAssignDirectFrom.append(*LINK(oldVar));
-        ctxAssignDirectTo.append(*LINK(newVar));
-        tables.append(*LINK(cursor));
-
-        StringBuffer s;
-        s.append("const unsigned char * ").append(newCursorName).append(";");
-        classctx.addQuoted(s);
-    }
-}
-
-IHqlExpression * FilterExtractor::extractExpr(IHqlExpression * expr, AssignPairs & assigns)
-{
-    unsigned match = assigns.from.find(*expr);
-    if (match == NotFound)
-    {
-        CHqlBoundTargetItem & target = * new CHqlBoundTargetItem;
-        translator.createTempFor(classctx, expr, target.value);
-        assigns.from.append(*LINK(expr));
-        assigns.to.append(target);
-        translator.ensureSerialized(classctx, target.value);
-        match = assigns.to.ordinality()-1;
-    }
-
-    return assigns.to.item(match).value.getTranslatedExpr();
-}
-
-IHqlExpression * FilterExtractor::querySelectorTable(IHqlExpression * expr)
-{
-    loop
-    {
-        switch (expr->getOperator())
-        {
-        case no_select:
-            expr = expr->queryChild(0);
-            break;
-        case no_left:
-        case no_right:
-        case no_self:
-        case no_top:
-        case no_activetable:
-            return expr;
-        default:
-            if (expr->isDataset() || expr->isDatarow())
-                return expr->queryNormalizedSelector();
-            return NULL;
-        }
-    }
-}
-
-IHqlExpression * FilterExtractor::createTransformed(IHqlExpression * expr)
-{
-    switch (expr->getOperator())
-    {
-    case no_field:
-        return LINK(expr);
-    case no_select:
-        {
-            assertex(!expr->hasProperty(newAtom));
-
-            if (querySelectorTable(expr) == rootTable)
-                return LINK(expr);
-
-            addTable(querySelectorTable(expr));
-            return extractExpr(expr, extractAssign);
-        }
-    case no_counter:
-        return extractExpr(expr, ctxAssign);
-    }
-    return NewHqlTransformer::createTransformed(expr);
-}
-
-
-IHqlExpression * FilterExtractor::buildExtractLookupFields(IHqlExpression * expr)
-{
-    if (expr->getOperator() != no_filter)
-        return LINK(expr);
-
-    HqlExprArray args;
-    ForEachChild(idx, expr)
-    {
-        IHqlExpression * cur = expr->queryChild(idx);
-        if (idx == 0)
-            args.append(*LINK(cur));
-        else
-            args.append(*createTransformed(cur));
-    }
-
-    if ((ctxAssign.from.ordinality() == 0) && (ctxAssignDirectFrom.ordinality() == 0))
-        return LINK(expr);
-
-    //virtual void extractLookupFields() {}
-    BuildCtx funcctx(classctx);
-    funcctx.addQuotedCompound("virtual void extractLookupFields()");
-    ForEachItemIn(idx2, extractAssign.from)
-        translator.buildExprAssign(funcctx, extractAssign.to.item(idx2).value, &extractAssign.from.item(idx2));
-    return expr->clone(args);
-}
-
-
-void FilterExtractor::assignCursors(IHqlExpression * helper)
-{
-    ForEachItemIn(idx1, ctxAssignDirectFrom)
-    {
-        OwnedHqlExpr target = createSelectExpr(LINK(helper), LINK(&ctxAssignDirectTo.item(idx1)), createAttribute(internalAtom));
-        ctx.addAssign(target, &ctxAssignDirectFrom.item(idx1));
-    }
-    ForEachItemIn(idx2, ctxAssign.from)
-    {
-        CHqlBoundTarget target;
-        target.set(ctxAssign.to.item(idx2).value);
-        if (target.length)      target.length.setown(createSelectExpr(LINK(helper), LINK(target.length), createAttribute(internalAtom)));
-        if (target.expr)        target.expr.setown(createSelectExpr(LINK(helper), LINK(target.expr), createAttribute(internalAtom)));
-        translator.buildExprAssign(ctx, target, &ctxAssign.from.item(idx2));
-    }
-}
-
-
 void HqlCppTranslator::doBuildExprCountFile(BuildCtx & ctx, IHqlExpression * _expr, CHqlBoundExpr & tgt)
 {
     //NB: This is currently only applied to fixed width unfiltered files
@@ -11028,10 +10855,7 @@ void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx,
         funcctx.associateExpr(constantMemberMarkerExpr, constantMemberMarkerExpr);
 
         OwnedHqlExpr groupOrder = createValueSafe(no_sortlist, makeSortListType(NULL), sorts);
-        OwnedHqlExpr diff = doCompare(funcctx, groupOrder, dataset);
-
-        funcctx.setNextDestructor();
-        funcctx.addReturn(diff);
+        buildReturnOrder(funcctx, groupOrder, dataset);
 
         endNestedClass();
     }
@@ -15133,22 +14957,6 @@ IHqlExpression * HqlCppTranslator::createOrderFromSortList(const DatasetReferenc
 }
 
 
-IHqlExpression * HqlCppTranslator::doCompare(BuildCtx & ctx, IHqlExpression *sortList, const DatasetReference & dataset)
-{
-    OwnedHqlExpr selSeq = createDummySelectorSequence();
-    OwnedHqlExpr leftSelect = dataset.getSelector(no_left, selSeq);
-    OwnedHqlExpr rightSelect = dataset.getSelector(no_right, selSeq);
-    OwnedHqlExpr order = createOrderFromSortList(dataset, sortList, leftSelect, rightSelect);
-    
-    bindTableCursor(ctx, dataset.queryDataset(), "left", no_left, selSeq);
-    bindTableCursor(ctx, dataset.queryDataset(), "right", no_right, selSeq);
-
-    CHqlBoundExpr bound;
-    buildCachedExpr(ctx, order, bound);
-
-    return bound.expr.getClear();
-}
-
 void HqlCppTranslator::buildReturnOrder(BuildCtx & ctx, IHqlExpression *sortList, const DatasetReference & dataset)
 {
     OwnedHqlExpr selSeq = createDummySelectorSequence();

ecl/hqlcpp/hqlinline.cpp

@@ -101,6 +101,8 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
                 return RETevaluate;
             return RETevaluate|RETiterate;
         }
+    case no_dataset_alias:
+        return getInlineFlags(ctx, expr->queryChild(0));
     }
 
     if (isGrouped(expr))
@@ -579,6 +581,7 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
     case no_serialize:
     case no_actionlist:
     case no_definesideeffect:
+    case no_dataset_alias:
         return GraphNoAccess;               // Will never access any data values from anywhere
     case no_hqlproject:
     case no_newusertable:

ecl/hqlcpp/hqliproj.cpp

@@ -1586,6 +1586,8 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
         if (getProjectExprKind(expr->queryChild(0)) == CompoundableActivity)
             return CompoundableActivity;
         return PassThroughActivity;
+    case no_dataset_alias:
+        return PassThroughActivity;
     }
 
     ITypeInfo * type = expr->queryType();

ecl/hqlcpp/hqlttcpp.cpp

@@ -2738,19 +2738,7 @@ IHqlExpression * ThorHqlTransformer::normalizeScalarAggregate(IHqlExpression * e
     if (!project)
         throwUnexpected();
     IHqlExpression * field = project->queryRecord()->queryChild(0);
-
-    HqlExprArray args;
-    ForEachChild(i, expr)
-    {
-        IHqlExpression * cur = expr->queryChild(i);
-        if (cur->isAttribute())
-        {
-            _ATOM name = cur->queryName();
-            if ((name != keyedAtom) && (name != prefetchAtom))
-                args.append(*LINK(cur));
-        }
-    }
-    OwnedHqlExpr ret = createSelectExpr(project.getClear(), LINK(field), createExprAttribute(newAtom, args));
+    OwnedHqlExpr ret = createNewSelectExpr(project.getClear(), LINK(field));
     return expr->cloneAllAnnotations(ret);
 }
 
@@ -8743,10 +8731,7 @@ IHqlExpression * HqlScopeTagger::transformSelect(IHqlExpression * expr)
     }
     //MORE: What about child datasets - should really be tagged as
     //if (!isNewDataset && field->isDataset() && !containsSelf(ds) && !isDatasetActive(ds)) isNew = true;
-
-    HqlExprArray attrs;
-    endTableScope(attrs, ds, newDs);
-    return createSelectExpr(newDs.getClear(), LINK(field), createExprAttribute(newAtom, attrs));
+    return createNewSelectExpr(newDs.getClear(), LINK(field));
 }
 
 IHqlExpression * HqlScopeTagger::transformSelectorsAttr(IHqlExpression * expr)
@@ -8926,7 +8911,7 @@ IHqlExpression * HqlScopeTagger::transformWithin(IHqlExpression * dataset, IHqlE
         return createSelectExpr(newDs.getClear(), LINK(field));
     }
     OwnedHqlExpr newDs = transformWithin(ds, scope);
-    return createSelectExpr(newDs.getClear(), LINK(field), createExprAttribute(newAtom));
+    return createNewSelectExpr(newDs.getClear(), LINK(field));
 }
 
 IHqlExpression * HqlScopeTagger::transformRelated(IHqlExpression * expr)

ecl/regress/sqfilt.ecl

@@ -49,7 +49,7 @@ output(sqHouseDs, { addr, count(personsDsDs(exists(booksDsDsDs(price>personsDs.b
 //which means sizeof(row-of-books) is wrong, and the iterators etc. don't work correctly!
 
 // How many people have books worth more than twice their average book price
-output(sqHousePersonBookDs, { addr, count(persons(exists(books(price>ave(books, price)*2)))) });
+output(sqHousePersonBookDs, { addr, count(persons(exists(books(price>ave(__ALIAS__(books), price)*2)))) });
 output(sqHousePersonBookDs, { addr, count(persons(exists(booksDs(price>ave(books, price)*2)))) });
 output(sqHousePersonBookDs, { addr, count(persons(exists(booksDs(price>ave(booksDs, price)*2)))) });
 //output(sqHousePersonBookDs, { addr, count(personsDs(exists(booksDsDs(price>ave(booksDs, price)*2)))) });

ecl/regress/sqfilt5.ecl

@@ -0,0 +1,44 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+import sq;
+sq.DeclareCommon();
+
+#option ('childQueries', true);
+
+// Test filtering at different levels, making sure parent fields are available in the child query.
+// Also tests scoping of sub expressions using within.
+
+udecimal8 todaysDate := 20040602D;
+unsigned4 age(udecimal8 dob) := ((todaysDate - dob) / 10000D);
+
+//MORE: books[1] ave(books)
+
+// Different child operators, all inline.
+house := sqHousePersonBookDs.persons;
+persons := sqHousePersonBookDs.persons;
+books := persons.books;
+
+booksDs := sqBookDs(personid = persons.id);
+personsDs := sqPersonDs(houseid = sqHousePersonBookDs.id);
+booksDsDs := sqBookDs(personid = personsDs.id);
+personsDsDs := sqPersonDs(houseid = sqHouseDs.id);
+booksDsDsDs := sqBookDs(personid = personsDsDs.id);
+
+//people with a book worth more than the rest of their books.
+output(sqHousePersonBookDs.persons, { exists(books(price > sum(__alias__(books)(id != books.id), price))); }, named('NumPeopleExceedBookLimit'));