Merge pull request #11940 from ghalliday/issue20978

HPCC-20978 Ensure ,DISTRIBUTE creates a distributed dataset when possible

Reviewed-By: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlgram.hpp

@@ -842,7 +842,7 @@ protected:
     IHqlExpression * createClearTransform(IHqlExpression * record, const attribute & errpos);
     IHqlExpression * createDefaultAssignTransform(IHqlExpression * record, IHqlExpression * rowValue, const attribute & errpos);
     IHqlExpression * createDefaultProjectDataset(IHqlExpression * record, IHqlExpression * src, const attribute & errpos);
-    IHqlExpression * createDatasetFromList(attribute & listAttr, attribute & recordAttr);
+    IHqlExpression * createDatasetFromList(attribute & listAttr, attribute & recordAttr, IHqlExpression * attrs);
     IHqlExpression * getUnadornedRecord(IHqlExpression * record);
 
     void checkConditionalAggregates(IIdAtom * name, IHqlExpression * value, const attribute & errpos);

ecl/hql/hqlgram.y

@@ -9159,7 +9159,7 @@ simpleDataSet
                             $$.setExpr(createDataset(no_inlinetable, createValue(no_transformlist, makeNullType(), values), createComma(LINK(record), $7.getExpr())));
                             $$.setPosition($1);
                         }
-    | DATASET '(' thorFilenameOrList ',' beginCounterScope recordDef endCounterScope ')'
+    | DATASET '(' thorFilenameOrList ',' beginCounterScope dsRecordDef endCounterScope optDatasetFlags dsEnd
                         {
                             //NB: $3 is required to be a list, but uses thorfilename production to work around a s/r error
                             OwnedHqlExpr counter = $7.queryExpr();
@@ -9167,7 +9167,8 @@ simpleDataSet
                                 parser->reportError(ERR_ILL_HERE,$6,"Not expecting COUNTER for DATASET");
                             parser->normalizeExpression($3, type_set, false);
 
-                            $$.setExpr(parser->createDatasetFromList($3, $6), $1);
+                            OwnedHqlExpr attrs = $8.getExpr();
+                            $$.setExpr(parser->createDatasetFromList($3, $6, attrs), $1);
                         }
     | DATASET '(' WORKUNIT '(' expression ',' expression ')' ',' recordDef ')'
                         {

ecl/hql/hqlgram2.cpp

@@ -5788,7 +5788,7 @@ ITypeInfo *HqlGram::checkNumericGetType(attribute &a1)
 }
 
 
-IHqlExpression * HqlGram::createDatasetFromList(attribute & listAttr, attribute & recordAttr)
+IHqlExpression * HqlGram::createDatasetFromList(attribute & listAttr, attribute & recordAttr, IHqlExpression * attrs)
 {
     OwnedHqlExpr list = listAttr.getExpr();
     OwnedHqlExpr record = recordAttr.getExpr();
@@ -5801,7 +5801,7 @@ IHqlExpression * HqlGram::createDatasetFromList(attribute & listAttr, attribute
     if ((list->getOperator() == no_list) && (list->numChildren() == 0))
     {
         OwnedHqlExpr list = createValue(no_null);
-        OwnedHqlExpr table = createDataset(no_temptable, LINK(list), record.getClear());
+        OwnedHqlExpr table = createDataset(no_temptable, LINK(list), createComma(record.getClear(), LINK(attrs)));
         return convertTempTableToInlineTable(*errorHandler, listAttr.pos, table);
     }
 
@@ -5841,7 +5841,7 @@ IHqlExpression * HqlGram::createDatasetFromList(attribute & listAttr, attribute
     else if (childType && !field->queryType()->assignableFrom(childType))
         reportError(ERR_RECORD_NOT_MATCH_SET, recordAttr, "The field in the record does not match the type of the set elements");
     
-    OwnedHqlExpr table = createDataset(no_temptable, LINK(list), record.getClear());
+    OwnedHqlExpr table = createDataset(no_temptable, LINK(list), createComma(record.getClear(), LINK(attrs)));
     return convertTempTableToInlineTable(*errorHandler, listAttr.pos, table);
 }
 

ecl/hqlcpp/hqlinline.cpp

@@ -325,6 +325,8 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
     }
     case no_inlinetable:
         {
+            if (expr->hasAttribute(distributedAtom))
+                return 0;
             IHqlExpression * transforms = expr->queryChild(0);
             if (transformListContainsSkip(transforms))
                 return 0;

testing/regress/ecl/complexhoist2.ecl

@@ -0,0 +1,41 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//HOIST(dataset({unsigned i}) ds) := NOFOLD(SORT(NOFOLD(ds), i));
+HOIST( ds) := MACRO
+//NOFOLD(SORT(NOFOLD(ds), i))
+//NOFOLD(ds)
+ds
+ENDMACRO;
+
+mkRow(unsigned value) := TRANSFORM({ unsigned i }, SKIP(value = 1000000);   SELF.i := value);
+
+dsOuter  := HOIST(DATASET(3, mkRow(COUNTER)));
+dsInner1 := HOIST(DATASET(3, mkRow(COUNTER+10)));
+dsInner2 := HOIST(DATASET(3, mkRow(COUNTER+20)));
+
+innerSum1(unsigned x) := SUM(dsInner1, i * x);
+outerSum1(unsigned x) := SUM(dsOuter, x * innerSum1(i));
+innerSum2(unsigned x) := SUM(dsInner2, x * outerSum1(i));
+outerSum2 := SUM(dsOuter, innerSum2(i));
+
+sequential(
+output(innerSum1(1));   // 36
+output(outerSum1(1));   // 36 * 6 = 216
+output(innerSum2(1));   // 216 * 22 * 3 = 14256
+output(outerSum2);      // 14256 * 6 = 85536
+);

testing/regress/ecl/complexhoist3.ecl

@@ -0,0 +1,42 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//HOIST(dataset({unsigned i}) ds) := NOFOLD(SORT(NOFOLD(ds), i));
+HOIST( ds) := MACRO
+//NOFOLD(SORT(NOFOLD(ds), i))
+//NOFOLD(ds)
+ds
+ENDMACRO;
+
+mkRow(unsigned value) := TRANSFORM({ unsigned i }, SKIP(value = 1000000);   SELF.i := value);
+
+dsOuter  := HOIST(DATASET([1,2,3], { unsigned i}, DISTRIBUTED));
+dsInner1 := HOIST(DATASET([11,12,13], { unsigned i}, DISTRIBUTED));
+dsInner2 := HOIST(DATASET([21,22,23], { unsigned i}, DISTRIBUTED));
+
+innerSum1(unsigned x) := SUM(dsInner1(x != i), i);
+outerSum1(unsigned x) := SUM(dsOuter(x != i), innerSum1(i));
+innerSum2(unsigned x) := SUM(dsInner2(x != i), outerSum1(i));
+outerSum2 := SUM(dsOuter, innerSum2(i));
+
+sequential(
+output(innerSum1(1));   // 36
+output(outerSum1(1));   // 36 * 2 = 72
+output(outerSum1(21));   // 36 * 3 = 108
+output(innerSum2(1));   // 108 * 3 = 324
+output(outerSum2);      // 324 * 3 = 972
+);

testing/regress/ecl/key/complexhoist2.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>36</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>216</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>14256</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>85536</Result_4></Row>
+</Dataset>

testing/regress/ecl/key/complexhoist3.xml

@@ -0,0 +1,15 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>36</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>72</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>108</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>324</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>972</Result_5></Row>
+</Dataset>