HPCC-14586 Support HINT on COUNT() and DATASET([])

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlgram.hpp

@@ -456,6 +456,7 @@ public:
     void checkIntegerOrString(attribute & e1);
     void checkNumeric(attribute &e1);
     ITypeInfo *checkNumericGetType(attribute &e1);
+    void checkInlineDatasetOptions(const attribute & attr);
     void checkLibraryParametersMatch(const attribute & errpos, bool isParametered, const HqlExprArray & activeParameters, IHqlExpression * definition);
     void checkReal(attribute &e1);
     ITypeInfo *checkStringIndex(attribute & strAttr, attribute & idxAttr);

ecl/hql/hqlgram.y

@@ -3096,7 +3096,7 @@ datasetFlag
                             $$.setExpr(createExprAttribute(distributedAtom));
                             $$.setPosition($1);
                         }
-    | localAttribute
+    | commonAttribute
     ;
 
 optIndexFlags
@@ -5630,7 +5630,7 @@ primexpr1
                         {   $$.inherit($2); }
     | COUNT '(' startTopFilter aggregateFlags ')' endTopFilter
                         {
-                            $$.setExpr(createValue(no_count, LINK(parser->defaultIntegralType), $3.getExpr(), $4.getExpr()));
+                            $$.setExpr(createValueF(no_count, LINK(parser->defaultIntegralType), $3.getExpr(), $4.getExpr(), NULL));
                         }
     | COUNT '(' GROUP optExtraFilter ')'
                         {
@@ -5662,7 +5662,7 @@ primexpr1
                         }
     | EXISTS '(' dataSet aggregateFlags ')'
                         {
-                            $$.setExpr(createBoolExpr(no_exists, $3.getExpr(), $4.getExpr()));
+                            $$.setExpr(createValueF(no_exists, makeBoolType(), $3.getExpr(), $4.getExpr(), NULL));
                             $$.setPosition($1);
                         }
     | EXISTS '(' dictionary ')'
@@ -6023,7 +6023,7 @@ primexpr1
                         {
                             parser->normalizeExpression($5);
                             IHqlExpression *e5 = $5.getExpr();
-                            $$.setExpr(createValue(no_max, e5->getType(), $3.getExpr(), e5, $6.getExpr()));
+                            $$.setExpr(createValueF(no_max, e5->getType(), $3.getExpr(), e5, $6.getExpr(), NULL));
                         }
     | MAX '(' GROUP ',' expression ')'
                         {
@@ -6035,7 +6035,7 @@ primexpr1
                         {
                             parser->normalizeExpression($5);
                             IHqlExpression *e5 = $5.getExpr();
-                            $$.setExpr(createValue(no_min, e5->getType(), $3.getExpr(), e5, $6.getExpr()));
+                            $$.setExpr(createValueF(no_min, e5->getType(), $3.getExpr(), e5, $6.getExpr(), NULL));
                         }
     | MIN '(' GROUP ',' expression ')'
                         {
@@ -6056,7 +6056,7 @@ primexpr1
                             Owned<ITypeInfo> temp = parser->checkPromoteNumeric($5, true);
                             OwnedHqlExpr value = $5.getExpr();
                             Owned<ITypeInfo> type = getSumAggType(value);
-                            $$.setExpr(createValue(no_sum, LINK(type), $3.getExpr(), ensureExprType(value, type), $6.getExpr()));
+                            $$.setExpr(createValueF(no_sum, LINK(type), $3.getExpr(), ensureExprType(value, type), $6.getExpr(), NULL));
                         }
     | SUM '(' GROUP ',' expression optExtraFilter ')'
                         {
@@ -6064,12 +6064,12 @@ primexpr1
                             Owned<ITypeInfo> temp = parser->checkPromoteNumeric($5, true);
                             OwnedHqlExpr value = $5.getExpr();
                             Owned<ITypeInfo> type = getSumAggType(value);
-                            $$.setExpr(createValue(no_sumgroup, LINK(type), ensureExprType(value, type), $6.getExpr()));
+                            $$.setExpr(createValueF(no_sumgroup, LINK(type), ensureExprType(value, type), $6.getExpr(), NULL));
                         }
     | AVE '(' startTopFilter ',' expression aggregateFlags ')' endTopFilter
                         {
                             parser->normalizeExpression($5, type_numeric, false);
-                            $$.setExpr(createValue(no_ave, makeRealType(8), $3.getExpr(), $5.getExpr(), $6.getExpr()));
+                            $$.setExpr(createValueF(no_ave, makeRealType(8), $3.getExpr(), $5.getExpr(), $6.getExpr(), NULL));
                         }
     | AVE '(' GROUP ',' expression optExtraFilter')'
                         {
@@ -6079,7 +6079,7 @@ primexpr1
     | VARIANCE '(' startTopFilter ',' expression aggregateFlags ')' endTopFilter
                         {
                             parser->normalizeExpression($5, type_numeric, false);
-                            $$.setExpr(createValue(no_variance, makeRealType(8), $3.getExpr(), $5.getExpr(), $6.getExpr()));
+                            $$.setExpr(createValueF(no_variance, makeRealType(8), $3.getExpr(), $5.getExpr(), $6.getExpr(), NULL));
                         }
     | VARIANCE '(' GROUP ',' expression optExtraFilter')'
                         {
@@ -6090,7 +6090,7 @@ primexpr1
                         {
                             parser->normalizeExpression($5, type_numeric, false);
                             parser->normalizeExpression($7, type_numeric, false);
-                            $$.setExpr(createValue(no_covariance, makeRealType(8), $3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr()));
+                            $$.setExpr(createValueF(no_covariance, makeRealType(8), $3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr(), NULL));
                         }
     | COVARIANCE '(' GROUP ',' expression ',' expression optExtraFilter')'
                         {
@@ -6102,7 +6102,7 @@ primexpr1
                         {
                             parser->normalizeExpression($5, type_numeric, false);
                             parser->normalizeExpression($7, type_numeric, false);
-                            $$.setExpr(createValue(no_correlation, makeRealType(8), $3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr()));
+                            $$.setExpr(createValueF(no_correlation, makeRealType(8), $3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr(), NULL));
                         }
     | CORRELATION '(' GROUP ',' expression ',' expression optExtraFilter')'
                         {
@@ -6790,13 +6790,18 @@ xmlEncodeFlags
 
 aggregateFlags
     :                   { $$.setNullExpr(); }
-    | ',' KEYED         { $$.setExpr(createAttribute(keyedAtom)); $$.setPosition($2); }
-    | ',' prefetchAttribute
+    | aggregateFlags ',' aggregateFlag
                         {
-                            $$.setExpr($2.getExpr(), $2);
+                            $$.setExpr(createComma($1.getExpr(), $3.getExpr()), $1);
                         }
     ;
 
+aggregateFlag
+    : KEYED             { $$.setExpr(createAttribute(keyedAtom), $1); }
+    | prefetchAttribute
+    | hintAttribute
+    ;
+
 transfer
     : TYPE_LPAREN typeDef TYPE_RPAREN 
                         { $$ = $2; }
@@ -8761,14 +8766,8 @@ simpleDataSet
                             IHqlExpression * counter = $7.getExpr();
                             if (counter)
                                 counter = createAttribute(_countProject_Atom, counter);
-                            OwnedHqlExpr options = $8.getExpr();
-                            if (options)
-                            {
-                                if (options->numChildren() > 0)
-                                    parser->reportError(ERR_DSPARAM_INVALIDOPTCOMB, $8, "The DATASET options DISTRIBUTED, LOCAL, and NOLOCAL are not permutable.");
-                            }
-                            $$.setExpr(createDataset(no_dataset_from_transform, $3.getExpr(), createComma($6.getExpr(), counter, options.getClear())));
-                            $$.setPosition($1);
+                            parser->checkInlineDatasetOptions($8);
+                            $$.setExpr(createDataset(no_dataset_from_transform, $3.getExpr(), createComma($6.getExpr(), counter, $8.getExpr())), $1);
                         }
     | ENTH '(' dataSet ',' expression optCommonAttrs ')'
                         {

ecl/hql/hqlgram2.cpp

@@ -2152,6 +2152,20 @@ void HqlGram::checkConstant(attribute & attr)
 }
 
 
+void HqlGram::checkInlineDatasetOptions(const attribute & attr)
+{
+    IHqlExpression * options = attr.queryExpr();
+    unsigned optionCount = 0;
+    if (queryAttributeInList(distributedAtom, options))
+        optionCount++;
+    if (queryAttributeInList(localAtom, options))
+        optionCount++;
+    if (queryAttributeInList(noLocalAtom, options))
+        optionCount++;
+    if (optionCount > 1)
+        reportError(ERR_DSPARAM_INVALIDOPTCOMB, attr, "The DATASET options DISTRIBUTED, LOCAL, and NOLOCAL cannot be combined.");
+}
+
 IHqlExpression * HqlGram::checkConcreteModule(const attribute & errpos, IHqlExpression * expr)
 {
     return checkCreateConcreteModule(this, expr, errpos.pos);

ecl/hql/hqlutil.cpp

@@ -4594,11 +4594,16 @@ extern HQL_API IHqlExpression * convertScalarAggregateToDataset(IHqlExpression *
         field.setown(createField(valueId, expr->getType(), NULL));
 
     IHqlExpression * aggregateRecord = createRecord(field);
-    IHqlExpression * keyedAttr = expr->queryAttribute(keyedAtom);
-    IHqlExpression * prefetchAttr = expr->queryAttribute(prefetchAtom);
 
     HqlExprArray valueArgs;
-    unwindChildren(valueArgs, expr, 1);
+    ForEachChildFrom(i1, expr, 1)
+    {
+        IHqlExpression * cur = expr->queryChild(i1);
+        //keyed is currently required on the aggregate operator
+        if (!cur->isAttribute() || (cur->queryName() == keyedAtom))
+            valueArgs.append(*LINK(cur));
+    }
+
     IHqlExpression * newValue = createValue(newop, expr->getType(), valueArgs);
     IHqlExpression * assign = createAssign(createSelectExpr(getSelf(aggregateRecord), LINK(field)), newValue);
     IHqlExpression * transform = createValue(no_newtransform, makeTransformType(aggregateRecord->getType()), assign);
@@ -4608,7 +4613,17 @@ extern HQL_API IHqlExpression * convertScalarAggregateToDataset(IHqlExpression *
     if (dataset->queryType()->getTypeCode() == type_groupedtable)
         dataset = createDataset(no_group, dataset, NULL);
 
-    IHqlExpression * project = createDataset(no_newaggregate, dataset, createComma(aggregateRecord, transform, LINK(keyedAttr), LINK(prefetchAttr)));
+    HqlExprArray args;
+    args.append(*dataset);
+    args.append(*aggregateRecord);
+    args.append(*transform);
+    ForEachChild(i2, expr)
+    {
+        IHqlExpression * cur = expr->queryChild(i2);
+        if (cur->isAttribute())
+            args.append(*LINK(cur));
+    }
+    IHqlExpression * project = createDataset(no_newaggregate, args);
     return createRow(no_selectnth, project, createConstantOne());
 }
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -4680,7 +4680,9 @@ IHqlExpression * OptimizeActivityTransformer::optimizeCompare(IHqlExpression * l
         if (lhs->getOperator() == no_count)
         {
             IHqlExpression * ds = lhs->queryChild(0);
-            OwnedHqlExpr ret = createValue(no_exists, makeBoolType(), LINK(ds));
+            HqlExprArray args;
+            unwindChildren(args, lhs);
+            OwnedHqlExpr ret = createValue(no_exists, makeBoolType(), args);
             if (existOp == no_not)
                 return createValue(no_not, makeBoolType(), ret.getClear());
             return ret.getClear();

ecl/regress/count7.ecl

@@ -39,5 +39,5 @@ output(count(namesTable5) = 5);
 namesTable6 := dataset('x6',namesRecord,FLAT);
 output(count(namesTable6) != 5);
 namesTable7 := dataset('x7',namesRecord,FLAT);
-output(count(namesTable7) != 5);
-output(count(namesTable7) > 0);
+output(count(namesTable7, HINT(goQuitefast(true))) != 5);
+output(count(namesTable7, HINT(goReallyReallyFast(true))) > 0);

ecl/regress/dataset24.ecl

@@ -27,6 +27,6 @@ namesTable2 := dataset([
         {'Hawthorn','Gavin',31+zero},
         {'Hawthorn','Peter',30+zero},
         {'Smithe','Simon',10+zero},
-        {'X','Z',zero}], namesRecord);
+        {'X','Z',zero}], namesRecord, HINT(ThisIsAHint(true)));
 
 output(namesTable2,,'out.d00',overwrite);

ecl/regress/dataset_transform.ecl

@@ -31,7 +31,7 @@ end;
 
 // zero
 output(true);
-ds := DATASET(0, t1(COUNTER));
+ds := DATASET(0, t1(COUNTER), HINT(fasterThanAFastThing(true)));
 output(ds);
 
 // plain

ecl/regress/readahead2.ecl

@@ -506,7 +506,7 @@ outrec it(in L) := TRANSFORM
   self := L;
 END;
 
-OUTPUT(SUM(in, TS_WordIndex(KEYED(word = in.word))[1].doc, prefetch(5, parallel))) : independent;
+OUTPUT(SUM(in, TS_WordIndex(KEYED(word = in.word))[1].doc, prefetch(5, parallel), hint(thisIsAHint('Help!')))) : independent;
 OUTPUT(TABLE(in, { sum(group, TS_WordIndex(KEYED(word = in.word))[1].doc), count(group) }, prefetch(5, parallel))) : independent;
 OUTPUT(TABLE(in, { sum(group, TS_WordIndex(KEYED(word = in.word))[1].doc), count(group) }, word[1], prefetch(5, parallel))) : independent;