Merge pull request #263 from ghalliday/bug83964

Bug: #83964/83966 Add SKEW and THRESHOLD attributes

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlexpr.cpp

@@ -13371,6 +13371,22 @@ void unwindRealChildren(HqlExprArray & children, const IHqlExpression * expr, un
 }
 
 
+void unwindAttributes(HqlExprArray & children, const IHqlExpression * expr)
+{
+    unsigned max = expr->numChildren();
+    for (unsigned idx=0; idx < max; idx++)
+    {
+        IHqlExpression * child = expr->queryChild(idx);
+        if (child->isAttribute())
+        {
+            //Subsequent calls to ensure are quick no-ops
+            children.ensure(max - idx);
+            children.append(*LINK(child));
+        }
+    }
+}
+
+
 void unwindList(HqlExprArray &dst, IHqlExpression * expr, node_operator op)
 {
     while (expr->getOperator() == op)

ecl/hql/hqlexpr.hpp

@@ -1356,6 +1356,7 @@ extern HQL_API void unwindChildren(HqlExprArray & children, const IHqlExpression
 extern HQL_API void unwindChildren(HqlExprArray & children, const IHqlExpression * expr, unsigned from, unsigned to);
 extern HQL_API void unwindChildren(HqlExprCopyArray & children, const IHqlExpression * expr, unsigned first=0);
 extern HQL_API void unwindRealChildren(HqlExprArray & children, const IHqlExpression * expr, unsigned first);
+extern HQL_API void unwindAttributes(HqlExprArray & children, const IHqlExpression * expr);
 extern HQL_API void unwindList(HqlExprArray &dst, IHqlExpression * expr, node_operator op);
 extern HQL_API void unwindCopyList(HqlExprCopyArray &dst, IHqlExpression * expr, node_operator op);
 extern HQL_API void unwindCommaCompound(HqlExprArray & target, IHqlExpression * expr);

ecl/hql/hqlgram.y

@@ -2665,6 +2665,11 @@ buildFlag
                             $$.setPosition($1);
                         }
     | skewAttribute
+    | THRESHOLD '(' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_numeric, true);
+                            $$.setExpr(createAttribute(thresholdAtom, $3.getExpr()));
+                        }
     | FEW               {
                             $$.setExpr(createAttribute(fewAtom));
                             $$.setPosition($1);

ecl/hql/hqlgram2.cpp

@@ -5300,6 +5300,8 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
                     if (attr == manyAtom) ok = true;
                     if (attr == sortedAtom) ok = true;
                     if (attr == unsortedAtom) ok = true;
+                    if (attr == skewAtom) ok = true;
+                    if (attr == thresholdAtom) ok = true;
                     break;
                 case no_usertable:
                     if (attr == keyedAtom) ok = true;
@@ -5313,6 +5315,8 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
                     if (attr == manyAtom) ok = true;
                     if (attr == sortedAtom) ok = true;
                     if (attr == unsortedAtom) ok = true;
+                    if (attr == skewAtom) ok = true;
+                    if (attr == thresholdAtom) ok = true;
                     break;
                 case no_topn:
                     if (attr == bestAtom) ok = true;

ecl/hql/hqlutil.cpp

@@ -4017,9 +4017,15 @@ IHqlExpression * appendOwnedOperandsF(IHqlExpression * expr, ...)
 }
 
 
+IHqlExpression * inheritAttribute(IHqlExpression * expr, IHqlExpression * donor, _ATOM name)
+{
+    return appendOwnedOperand(expr, LINK(donor->queryProperty(name)));
+}
 
 IHqlExpression * appendOwnedOperand(IHqlExpression * expr, IHqlExpression * ownedOperand)
 {
+    if (!ownedOperand)
+        return LINK(expr);
     HqlExprArray args;
     unwindChildren(args, expr);
     args.append(*ownedOperand);

ecl/hql/hqlutil.hpp

@@ -143,6 +143,7 @@ extern HQL_API IHqlExpression * removeChildOp(IHqlExpression * expr, node_operat
 extern HQL_API IHqlExpression * appendOwnedOperand(IHqlExpression * expr, IHqlExpression * ownedOperand);
 extern HQL_API IHqlExpression * replaceOwnedProperty(IHqlExpression * expr, IHqlExpression * ownedProeprty);
 extern HQL_API IHqlExpression * appendOwnedOperandsF(IHqlExpression * expr, ...);
+extern HQL_API IHqlExpression * inheritAttribute(IHqlExpression * expr, IHqlExpression * donor, _ATOM name);
 extern HQL_API unsigned numRealChildren(IHqlExpression * expr);
 
 extern HQL_API IHqlExpression * createEvaluateOutputModule(HqlLookupContext & ctx, IHqlExpression * scopeExpr, IHqlExpression * ifaceExpr, bool expandCallsWhenBound, node_operator outputOp);

ecl/hqlcpp/hqlttcpp.cpp

@@ -1850,14 +1850,8 @@ static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortInde
         if (sorted == dataset)
             return NULL;
 
-        IHqlExpression * skew = expr->queryProperty(skewAtom);
-        if (skew)
-        {
-            HqlExprArray args;
-            unwindChildren(args, sorted);
-            args.append(*LINK(skew));
-            sorted.setown(sorted->clone(args));
-        }
+        sorted.setown(inheritAttribute(sorted, expr, skewAtom));
+        sorted.setown(inheritAttribute(sorted, expr, thresholdAtom));
 
         HqlExprArray args;
         args.append(*LINK(sorted));
@@ -2283,6 +2277,8 @@ IHqlExpression * ThorHqlTransformer::normalizeGroup(IHqlExpression * expr)
     if (sorted == dataset)
         return removeProperty(expr, allAtom);
     sorted.setown(cloneInheritedAnnotations(expr, sorted));
+    sorted.setown(inheritAttribute(sorted, expr, skewAtom));
+    sorted.setown(inheritAttribute(sorted, expr, thresholdAtom));
 
     if (!isLocal)
     {
@@ -3012,12 +3008,14 @@ IHqlExpression * ThorHqlTransformer::normalizePrefetchAggregate(IHqlExpression *
 static IHqlExpression * convertAggregateGroupingToGroupedAggregate(IHqlExpression * expr, IHqlExpression* groupBy)
 {
     IHqlExpression * dataset = expr->queryChild(0);
-    IHqlExpression * localAttr = expr->queryProperty(localAtom);
-    IHqlExpression * sortedAttr = expr->queryProperty(sortedAtom);
-    IHqlExpression * unsortedAttr = expr->queryProperty(unsortedAtom);
 
-    IHqlExpression * attr = createComma(createAttribute(allAtom), LINK(localAttr), LINK(sortedAttr), LINK(unsortedAttr));
-    OwnedHqlExpr result = createDatasetF(no_group, LINK(dataset), LINK(groupBy), attr, NULL);
+    HqlExprArray groupArgs;
+    groupArgs.append(*LINK(dataset));
+    groupArgs.append(*LINK(groupBy));
+    groupArgs.append(*createAttribute(allAtom));
+    unwindChildren(groupArgs, expr, 4);
+    OwnedHqlExpr result = createDataset(no_group, groupArgs);
+
     result.setown(cloneInheritedAnnotations(expr, result));
 
     HqlExprArray args;
@@ -3226,13 +3224,15 @@ IHqlExpression * ThorHqlTransformer::normalizeTableToAggregate(IHqlExpression *
     OwnedHqlExpr aggregateSelf = getSelf(aggregateRecord);
     replaceAssignSelector(aggregateAssigns, aggregateSelf);
     IHqlExpression * aggregateTransform = createValue(no_newtransform, makeTransformType(aggregateRecord->getType()), aggregateAssigns);
-    IHqlExpression * keyedAttr = expr->queryProperty(keyedAtom);
-    IHqlExpression * prefetchAttr = expr->queryProperty(prefetchAtom);
-    LinkedHqlExpr localAttr = expr->queryProperty(localAtom);
-    if (newGroupBy && !isGrouped(dataset) && isPartitionedForGroup(dataset, newGroupBy, true))
-        localAttr.setown(createLocalAttribute());
 
-    OwnedHqlExpr ret = createDataset(no_newaggregate, LINK(dataset), createComma(aggregateRecord, aggregateTransform, newGroupBy, createComma(LINK(keyedAttr), LINK(prefetchAttr), LINK(localAttr))));
+    HqlExprArray aggregateAttrs;
+    unwindAttributes(aggregateAttrs, expr);
+    removeProperty(aggregateAttrs, aggregateAtom);
+    removeProperty(aggregateAttrs, fewAtom);
+    if (!expr->hasProperty(localAtom) && newGroupBy && !isGrouped(dataset) && isPartitionedForGroup(dataset, newGroupBy, true))
+        aggregateAttrs.append(*createLocalAttribute());
+
+    OwnedHqlExpr ret = createDataset(no_newaggregate, LINK(dataset), createComma(aggregateRecord, aggregateTransform, newGroupBy, createComma(aggregateAttrs)));
     if (extraSelectNeeded)
         ret.setown(cloneInheritedAnnotations(expr, ret));
     else

ecl/regress/buildindex.ecl

@@ -34,4 +34,4 @@ nt2 := group(namesTable, age2);
 
 unsigned curVersion := 10;
 BUILDINDEX(nt2, { surname, forename, filepos }, 'name.idx', set('x','y'));
-BUILDINDEX(nt2, { age, age2, age8, filepos }, 'age.idx', dataset(namesTable2), skew(,2), backup,set('version',curVersion), set(U'user',U'ghalliday'),set('system','local'));
+BUILDINDEX(nt2, { age, age2, age8, filepos }, 'age.idx', dataset(namesTable2), skew(,2), threshold(3.1415), backup,set('version',curVersion), set(U'user',U'ghalliday'),set('system','local'));

ecl/regress/table7.ecl

@@ -0,0 +1,30 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('importAllModules', true);
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+namesTable := dataset('x',namesRecord,FLAT);
+
+output(table(namesTable, { count(group); }, surname, many, threshold(3.1415),skew(1.2345,9.876)));