Merge pull request #8226 from ghalliday/issue14872

HPCC-14872 Add new attributes for ordering and parallel execution

Reviewed-By: Jamie Noss <james.noss@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

docs/ECLLanguageReference/ECLR_mods/BltInFunc-LOOP.xml

@@ -13,16 +13,12 @@
   role="bold"> [, PARALLEL<indexterm>
       <primary>PARALLEL</primary>
     </indexterm>( </emphasis><emphasis>iterations </emphasis><emphasis
-  role="bold">| </emphasis><emphasis>iterationlist</emphasis><emphasis
-  role="bold"> [</emphasis><emphasis>, default </emphasis><emphasis
-  role="bold">] ) ] )</emphasis></para>
+  role="bold"> ) ] )</emphasis></para>
 
   <para><emphasis role="bold">LOOP(</emphasis><emphasis>
   dataset,</emphasis><emphasis role="bold"> </emphasis><emphasis>loopcount,
   loopfilter, loopbody </emphasis><emphasis role="bold"> [, PARALLEL(
-  </emphasis><emphasis>iterations </emphasis><emphasis role="bold">|
-  </emphasis><emphasis>iterationlist</emphasis><emphasis role="bold">
-  [</emphasis><emphasis>, default </emphasis><emphasis role="bold">] ) ]
+  </emphasis><emphasis>iterations </emphasis><emphasis role="bold">] ) ]
   )</emphasis></para>
 
   <para><emphasis role="bold">LOOP(</emphasis><emphasis>
@@ -86,24 +82,6 @@
         </row>
 
         <row>
-          <entry><emphasis>iterationlist</emphasis></entry>
-
-          <entry>A set of integers (contained in square brackets<indexterm>
-              <primary>square brackets</primary>
-            </indexterm>) specifying the number of parallel iterations for
-          each loop. The first set element specifies the parallel iterations
-          for the first loop, the second for the second, ...</entry>
-        </row>
-
-        <row>
-          <entry><emphasis>default</emphasis></entry>
-
-          <entry>Optional. The number of parallel iterations to execute once
-          all elements in the <emphasis>iterationlist</emphasis> have been
-          used.</entry>
-        </row>
-
-        <row>
           <entry><emphasis>loopfilter<indexterm>
               <primary>loopfilter</primary>
             </indexterm></emphasis></entry>
@@ -152,29 +130,8 @@
   <sect2 id="The_PARALLEL_Option">
     <title>The PARALLEL Option</title>
 
-    <para>The PARALLEL option is offered to solve the following type of
-    problem: When implementing a text search (A and B and C) or (D and E),
-    where each element in the search is evaluated on an iteration of a LOOP(),
-    you want to ensure that the execution is broken in the correct places. If
-    it were split every 2 iterations, the iterations would produce:</para>
-
-    <para>(A and B)</para>
-
-    <para>(A and B and C), (D)</para>
-
-    <para>(A and B and C) or (D and E)</para>
-
-    <para>The second iteration would potentially generate a very large number
-    of temporary records. To prevent this, the number of iterations at each
-    step can be controlled. For this specific case you would probably use
-    PARALLEL([3,3]). For more complicated search criteria the numbers would be
-    different.</para>
-
-    <para>If a very large number is provided as the
-    <emphasis>iterations</emphasis> or <emphasis>default </emphasis>value,
-    then the all the iterations will execute in parallel. Doing this will
-    likely significantly reduce the number of temporary rows stored in the
-    system, but may potentially use a large amount of resources.</para>
+    <para>The PARALLEL option allows multiple loop iterations to be executed in
+    parallel.</para>
 
     <para>There is a restriction: ROWS(LEFT) cannot be directly used in a
     sub-query of the <emphasis>loopbody</emphasis>.</para>

ecl/eclcc/reservedwords.cpp

@@ -312,6 +312,7 @@ static const char * eclReserved12[] = {//Attributes
     "__nostreaming__",
     "__owned__",
     "after",
+    "algorithm",
     "all",
     "any",
     "best",

ecl/hql/hqlatoms.cpp

@@ -302,8 +302,8 @@ IAtom * onceAtom;
 IAtom * onFailAtom;
 IAtom * onWarningAtom;
 IAtom * optAtom;
+IAtom * orderedAtom;
 IAtom * _ordered_Atom;
-IAtom * _orderedPull_Atom;
 IAtom * _origin_Atom;
 IAtom * _original_Atom;
 IAtom * outAtom;
@@ -426,7 +426,6 @@ IAtom * _uid_Atom;
 IAtom * unknownAtom;
 IAtom * unknownSizeFieldAtom;
 IAtom * unicodeAtom;
-IAtom * unorderedAtom;
 IAtom * unsortedAtom;
 IAtom * unstableAtom;
 IAtom * updateAtom;
@@ -743,8 +742,8 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(onFail);
     MAKEATOM(onWarning);
     MAKEATOM(opt);
+    MAKEATOM(ordered);
     MAKESYSATOM(ordered);
-    MAKESYSATOM(orderedPull);
     MAKESYSATOM(origin);
     MAKESYSATOM(original);
     MAKEATOM(out);
@@ -866,7 +865,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(unknown);
     MAKEATOM(unknownSizeField);
     MAKEATOM(unicode);
-    MAKEATOM(unordered);
     MAKEATOM(unsorted);
     MAKEATOM(unstable);
     MAKEATOM(update);

ecl/hql/hqlatoms.hpp

@@ -69,6 +69,8 @@ extern HQL_API IIdAtom * unknownId;
 extern HQL_API IIdAtom * unnamedId;
 extern HQL_API IIdAtom * valueId;
 
+//Any atom that is surrounded by underscores e.g., _aliased_Atom is an internal attribute.  They are used by the
+//code generator to annotate the graph (and are not included in regenerated ECL)
 extern HQL_API IAtom * abstractAtom;
 extern HQL_API IAtom * accessAtom;
 extern HQL_API IAtom * actionAtom;
@@ -304,8 +306,8 @@ extern HQL_API IAtom * onceAtom;
 extern HQL_API IAtom * onFailAtom;
 extern HQL_API IAtom * onWarningAtom;
 extern HQL_API IAtom * optAtom;
+extern HQL_API IAtom * orderedAtom;
 extern HQL_API IAtom * _ordered_Atom;
-extern HQL_API IAtom * _orderedPull_Atom;
 extern HQL_API IAtom * _origin_Atom;
 extern HQL_API IAtom * _original_Atom;
 extern HQL_API IAtom * outAtom;
@@ -429,7 +431,6 @@ extern HQL_API IAtom * _uid_Atom;
 extern HQL_API IAtom * unknownAtom;
 extern HQL_API IAtom * unknownSizeFieldAtom;
 extern HQL_API IAtom * unicodeAtom;
-extern HQL_API IAtom * unorderedAtom;
 extern HQL_API IAtom * unsortedAtom;
 extern HQL_API IAtom * unstableAtom;
 extern HQL_API IAtom * updateAtom;

ecl/hql/hqlattr.cpp

@@ -308,6 +308,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_normalize:
     case no_distributed:
     case no_preservemeta:
+    case no_unordered:
     case no_grouped:
     case no_denormalize:
     case no_newusertable:
@@ -627,7 +628,7 @@ unsigned getOperatorMetaFlags(node_operator op)
 
     case no_unused6:
     case no_unused13: case no_unused14: case no_unused15:
-    case no_unused28: case no_unused29:
+    case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
@@ -2015,6 +2016,7 @@ bool isTrivialDataset(IHqlExpression * expr)
         case no_preservemeta:
         case no_dataset_alias:
         case no_filter:
+        case no_unordered:
             expr = expr->queryChild(0);
             break;
         case no_inlinetable:
@@ -2531,6 +2533,7 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
     case no_owned_ds:
     case no_dataset_alias:
     case no_nocombine:
+    case no_unordered:
         {
             return getRecordCountInfo(ds);
         }

ecl/hql/hqlexpr.cpp

@@ -1519,10 +1519,11 @@ const char *getOpString(node_operator op)
     case no_unboundselect: return "no_unboundselect";
     case no_id: return "no_id";
     case no_orderedactionlist: return "ORDERED";
+    case no_unordered: return "UNORDERED";
 
     case no_unused6:
     case no_unused13: case no_unused14: case no_unused15:
-    case no_unused28: case no_unused29:
+    case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
@@ -1880,6 +1881,7 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_grouped:
     case no_distribute:
     case no_distributed:
+    case no_unordered:
     case no_cosort:
     case no_keyed:
     case no_sort:
@@ -2148,6 +2150,7 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_dedup:
     case no_distribute:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     case no_enth:
     case no_filter:
@@ -2444,6 +2447,7 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_dedup:
     case no_distribute:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     case no_enth:
     case no_filter:
@@ -6282,6 +6286,7 @@ void CHqlDataset::cacheParent()
     // distributing:
     case no_distribute:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     // fewer records
     case no_filter:
@@ -14194,6 +14199,7 @@ IHqlExpression * querySkipDatasetMeta(IHqlExpression * dataset)
         case no_sorted:
         case no_grouped:
         case no_distributed:
+        case no_unordered:
         case no_preservemeta:
         case no_stepped:
             break;
@@ -15591,7 +15597,7 @@ IHqlExpression * convertToSimpleAggregate(IHqlExpression * expr)
     loop
     {
         node_operator op = ds->getOperator();
-        if ((op != no_sorted) && (op != no_distributed) && (op != no_preservemeta) && (op != no_alias_scope))
+        if ((op != no_sorted) && (op != no_distributed) && (op != no_unordered) && (op != no_preservemeta) && (op != no_alias_scope))
             break;
         ds = ds->queryChild(0);
     }
@@ -15750,6 +15756,21 @@ bool getBoolAttribute(IHqlExpression * expr, IAtom * name, bool dft)
     return getBoolAttributeValue(attr);
 }
 
+IHqlExpression * queryBoolAttribute(IHqlExpression * expr, IAtom * name)
+{
+    if (!expr)
+        return NULL;
+    IHqlExpression * attr = expr->queryAttribute(name);
+    if (!attr)
+        return NULL;
+    return queryBoolExpr(getBoolAttributeValue(attr));
+}
+
+IHqlExpression * queryBoolExpr(bool value)
+{
+    return value ? constantTrue : constantFalse;
+}
+
 bool getBoolAttributeInList(IHqlExpression * expr, IAtom * search, bool dft)
 {
     IHqlExpression * match = queryAttributeInList(search, expr);

ecl/hql/hqlexpr.hpp

@@ -364,7 +364,7 @@ enum _node_operator {
         no_existsdict,
         no_quantile,
         no_nocombine,
-    no_unused28,  
+        no_unordered,
     no_unused29,
     no_unused30,
     no_unused31,
@@ -1853,6 +1853,8 @@ void addForwardDefinition(IHqlScope * scope, IIdAtom * symbolName, IIdAtom * mod
 extern HQL_API IPropertyTree * createAttributeArchive();
 extern HQL_API void ensureSymbolsDefined(IHqlExpression * scope, HqlLookupContext & ctx);
 extern HQL_API void ensureSymbolsDefined(IHqlScope * scope, HqlLookupContext & ctx);
+extern HQL_API IHqlExpression * queryBoolExpr(bool value);
+extern HQL_API IHqlExpression * queryBoolAttribute(IHqlExpression * expr, IAtom * name);    // true expr, false expr or NULL
 extern HQL_API bool getBoolAttribute(IHqlExpression * expr, IAtom * name, bool dft=false);
 extern HQL_API bool getBoolAttributeInList(IHqlExpression * expr, IAtom * name, bool dft);
 

ecl/hql/hqlfold.cpp

@@ -4147,6 +4147,8 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
     case no_filtergroup:
     case no_section:
     case no_related:
+    case no_unordered:
+    case no_preservemeta:
         if (isNull(child) || isFail(child))
             return removeParentNode(expr);
         break;
@@ -5985,6 +5987,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_assertgrouped:
     case no_distribute:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     case no_assertdistributed:
     case no_keyeddistribute:

ecl/hql/hqlgram.hpp

@@ -686,6 +686,7 @@ public:
 
     void createAppendDictionaries(attribute & targetAttr, attribute & leftAttr, attribute & rightAttr, IAtom * kind);
     void createAppendFiles(attribute & targetAttr, attribute & leftAttr, attribute & rightAttr, IAtom * kind);
+    IHqlExpression * createAppendFiles(attribute & filesAttr, IHqlExpression * _attrs);
     IHqlExpression * processIfProduction(attribute & condAttr, attribute & trueAttr, attribute * falseAttr);
 
     IHqlExpression * createSymbolFromValue(IHqlExpression * primaryExpr, IHqlExpression * value);

ecl/hql/hqlgram.y

@@ -111,6 +111,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   ACOS
   AFTER
   AGGREGATE
+  ALGORITHM
   ALIAS
   ALL
   ALLNODES
@@ -241,7 +242,6 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   HAVING
   HEADING
   HINT
-  HOLE
   HTTPCALL
   HTTPHEADER
   IF
@@ -2860,8 +2860,7 @@ buildFlags
     ;
 
 buildFlag
-    : PARALLEL          { $$.setExpr(createAttribute(parallelAtom)); $$.setPosition($1); }
-    | OVERWRITE         { $$.setExpr(createAttribute(overwriteAtom)); $$.setPosition($1); }
+    : OVERWRITE         { $$.setExpr(createAttribute(overwriteAtom)); $$.setPosition($1); }
     | NOOVERWRITE       { $$.setExpr(createAttribute(noOverwriteAtom)); $$.setPosition($1); }
     | BACKUP            { $$.setExpr(createAttribute(backupAtom)); $$.setPosition($1); }
     | NAMED '(' constExpression ')'
@@ -2999,6 +2998,7 @@ optCommonAttrs
 commonAttribute
     : localAttribute
     | hintAttribute
+    | orderAttribute
     ;
 
 hintAttribute
@@ -3079,6 +3079,79 @@ hintExpr
                         }
     ;
 
+orderAttribute
+    : UNORDERED
+                        {
+                            $$.setExpr(createExprAttribute(orderedAtom, createConstant(false)), $1);
+                        }
+    | ORDERED
+                        {
+                            $$.setExpr(createExprAttribute(orderedAtom), $1);
+                        }
+    | ORDERED '(' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_boolean, true);
+                            $$.setExpr(createExprAttribute(orderedAtom, $3.getExpr()), $1);
+                        }
+    | PARALLEL
+                        {
+                            $$.setExpr(createExprAttribute(parallelAtom), $1);
+                        }
+    | PARALLEL '(' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_int, true);
+                            $$.setExpr(createExprAttribute(parallelAtom, $3.getExpr()), $1);
+                        }
+    | STABLE
+                        {
+                            $$.setExpr(createExprAttribute(stableAtom));
+                            $$.setPosition($1);
+                        }
+    | STABLE '(' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_string, false);
+                            $$.setExpr(createExprAttribute(stableAtom, $3.getExpr()));
+                            $$.setPosition($1);
+                        }
+    | UNSTABLE
+                        {
+                            $$.setExpr(createExprAttribute(unstableAtom));
+                            $$.setPosition($1);
+                        }
+    | UNSTABLE '(' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_string, false);
+                            $$.setExpr(createExprAttribute(unstableAtom, $3.getExpr()));
+                            $$.setPosition($1);
+                        }
+    | ALGORITHM '(' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_string, true);
+                            $$.setExpr(createExprAttribute(algorithmAtom, $3.getExpr()), $1);
+                        }
+    ;
+
+optAppendAttrs
+    :                   { $$.setNullExpr(); }
+    | ',' commonAttribute optCommonAttrs
+                        {
+                            $$.setExpr(createComma($2.getExpr(), $3.getExpr()));
+                            $$.setPosition($3);
+                        }
+    | ',' pullAttr optCommonAttrs
+                        {
+                            $$.setExpr(createComma($2.getExpr(), $3.getExpr()));
+                            $$.setPosition($3);
+                        }
+    ;
+
+pullAttr
+    : PULL
+                        {
+                            $$.setExpr(createComma(createAttribute(pullAtom),createExprAttribute(orderedAtom)), $1);
+                        }
+    ;
+
 expireAttr
     : EXPIRE            {
                             $$.setExpr(createAttribute(expireAtom));
@@ -3171,7 +3244,6 @@ indexFlag
                             $$.setExpr(createAttribute(dynamicAtom));
                             $$.setPosition($1);
                         }
-    | UNORDERED         {   $$.setExpr(createAttribute(unorderedAtom)); $$.setPosition($1); }
     | FILEPOSITION optConstBoolArg
                         {
                             $$.setExpr(createExprAttribute(filepositionAtom, $2.getExpr()), $1);
@@ -3373,12 +3445,6 @@ soapFlag
                             $$.setExpr(createExprAttribute(mergeAtom, $3.getExpr()));
                             $$.setPosition($1);
                         }
-    | PARALLEL '(' expression ')'
-                        {
-                            parser->normalizeExpression($3, type_int, false);
-                            $$.setExpr(createExprAttribute(parallelAtom, $3.getExpr()));
-                            $$.setPosition($1);
-                        }
     | RETRY '(' expression ')'
                         {
                             parser->normalizeExpression($3, type_int, false);
@@ -5140,6 +5206,13 @@ endInlineFunctionToken
     | '}'               // see above
     ;
 
+optCondList
+    :                   {
+                            $$.setExpr(NULL);
+                        }
+    | condList
+    ;
+
 condList
     : booleanExpr       {
                             parser->normalizeExpression($1, type_boolean, false);
@@ -6145,12 +6218,13 @@ primexpr1
                             parser->normalizeExpression($7, type_numeric, false);
                             $$.setExpr(createValue(no_corrgroup, makeRealType(8), $5.getExpr(), $7.getExpr(), $8.getExpr()));
                         }
-    | WHICH conditions  {
-                            $$.setExpr(createList(no_which, LINK(parser->uint4Type), $2.getExpr()));
+    | WHICH '(' optCondList ')'
+                        {
+                            $$.setExpr(createList(no_which, LINK(parser->uint4Type), $3.getExpr()), $1);
                         }
-    | REJECTED conditions
+    | REJECTED '(' optCondList ')'
                         {
-                            $$.setExpr(createList(no_rejected, LINK(parser->uint4Type), $2.getExpr()));
+                            $$.setExpr(createList(no_rejected, LINK(parser->uint4Type), $3.getExpr()), $1);
                         }
     | SIZEOF '(' sizeof_type_target optMaxMin ')'
                         {
@@ -7102,6 +7176,22 @@ globalValueAttribute
                         }
     ;
 
+dataSetOrRowList
+    : dataSetOrRow
+    | dataSetOrRowList ',' dataSetOrRow
+                        {
+                            OwnedHqlExpr lhs = $1.getExpr();
+                            OwnedHqlExpr rhs = $3.getExpr();
+                            OwnedHqlExpr normRhs = parser->checkEnsureRecordsMatch(lhs, rhs, $2.pos, rhs->isDatarow());
+                            $$.setExpr(createComma(lhs.getClear(), normRhs.getClear()), $1);
+                        }
+    ;
+
+dataSetOrRow
+    : dataSet
+    | dataRow
+    ;
+
 dataRow
     : dataSet '[' expression ']'
                         {
@@ -7723,25 +7813,25 @@ dataSet
     | dataSet '&' dataSet
                         {   parser->createAppendFiles($$, $1, $3, _ordered_Atom);   }
     | dataSet ANDAND dataSet
-                        {   parser->createAppendFiles($$, $1, $3, _orderedPull_Atom);   }
+                        {   parser->createAppendFiles($$, $1, $3, pullAtom);   }
     | dataSet '+' dataRow
                         {   parser->createAppendFiles($$, $1, $3, NULL);    }
     | dataSet '&' dataRow
                         {   parser->createAppendFiles($$, $1, $3, _ordered_Atom);   }
     | dataSet ANDAND dataRow
-                        {   parser->createAppendFiles($$, $1, $3, _orderedPull_Atom);   }
+                        {   parser->createAppendFiles($$, $1, $3, pullAtom);   }
     | dataRow '+' dataSet
                         {   parser->createAppendFiles($$, $1, $3, NULL);    }
     | dataRow '&' dataSet
                         {   parser->createAppendFiles($$, $1, $3, _ordered_Atom);   }
     | dataRow ANDAND dataSet
-                        {   parser->createAppendFiles($$, $1, $3, _orderedPull_Atom);   }
+                        {   parser->createAppendFiles($$, $1, $3, pullAtom);   }
     | dataRow '+' dataRow
                         {   parser->createAppendFiles($$, $1, $3, NULL);    }
     | dataRow '&' dataRow
                         {   parser->createAppendFiles($$, $1, $3, _ordered_Atom);   }
     | dataRow ANDAND dataRow
-                        {   parser->createAppendFiles($$, $1, $3, _orderedPull_Atom);   }
+                        {   parser->createAppendFiles($$, $1, $3, pullAtom);   }
     | dataSet '[' expression DOTDOT expression ']'
                         {
                             parser->normalizeExpression($3, type_int, false);
@@ -7779,6 +7869,12 @@ simpleDataSet
                             $$.setExpr(createDataset(no_rowsetindex, $1.getExpr(), $3.getExpr()));
                             $$.setPosition($1);
                         }
+    // Prefix form of the dataset append operator.
+    // There are no equivalents to '&' and '&&' - the operator below defaults to ordered, and options can be used to change that
+    | '(' '+' ')' '(' dataSetOrRowList optAppendAttrs ')'
+                        {
+                            $$.setExpr(parser->createAppendFiles($5, $6.getExpr()), $1);
+                        }
     | ALIAS '(' dataSet ')'
                         {
                             $$.setExpr(createDataset(no_dataset_alias, $3.getExpr(), ::createUniqueId()), $1);
@@ -8503,11 +8599,11 @@ simpleDataSet
                             $$.setExpr(createDataset(no_regroup, args));
                             $$.setPosition($1);
                         }
-    | HAVING '(' startLeftRowsSeqFilter ',' condList ')' endRowsGroup endSelectorSequence
+    | HAVING '(' startLeftRowsSeqFilter ',' condList optCommonAttrs ')' endRowsGroup endSelectorSequence
                         {
                             parser->checkGrouped($3);
-                            IHqlExpression *attr = NULL;        // possibly local may make sense if thor supported it as a global operation, but it would be too painful.
-                            $$.setExpr(createDataset(no_filtergroup, $3.getExpr(), createComma($5.getExpr(), attr, $7.getExpr(), $8.getExpr())));
+                            IHqlExpression *attr = $6.getExpr();
+                            $$.setExpr(createDataset(no_filtergroup, $3.getExpr(), createComma($5.getExpr(), attr, $8.getExpr(), $9.getExpr())));
                             $$.setPosition($1);
                         }
     | KEYED '(' dataSet indexListOpt ')' endTopFilter
@@ -9320,11 +9416,11 @@ simpleDataSet
                             OwnedHqlExpr ds = $3.getExpr();
                             $$.setExpr(parser->resolveRows($1, ds), $1);
                         }
-    | XMLPROJECT '(' expression ',' transform ')'
+    | XMLPROJECT '(' expression ',' transform optCommonAttrs ')'
                         {
                             parser->normalizeExpression($3, type_string, false);
                             parser->validateXPath($3);
-                            $$.setExpr(createDatasetF(no_xmlproject, $3.getExpr(), $5.getExpr(), parser->createUniqueId(), NULL));
+                            $$.setExpr(createDatasetF(no_xmlproject, $3.getExpr(), $5.getExpr(), parser->createUniqueId(), $6.getExpr(), NULL));
                             $$.setPosition($1);
                         }
     | MAP '(' mapDatarowSpec ',' dataRow ')'
@@ -9403,25 +9499,10 @@ simpleDataSet
                             OwnedHqlExpr transform = parser->createDefaultAssignTransform(ds->queryRecord(), leftSelect, $1);
                             $$.setExpr(createDatasetF(no_quantile, ds.getClear(), $5.getExpr(), sortlist.getClear(), transform.getClear(), selSeq.getClear(), options.getClear(), NULL), $1);
                         }
-/*
-  //This may cause s/r problems with the attribute version if a dataset name clashes with a hint id
-    | HINT '(' dataSet ','  hintList ')'
+    | UNORDERED '(' startTopFilter ')' endTopFilter
                         {
-                            HqlExprArray hints;
-                            $5.unwindCommaList(hints);
-                            OwnedHqlExpr hint = createExprAttribute(hintAtom, hints);
-
-                            //Add a hint attribute to the dataset
-                            OwnedHqlExpr ds = $3.getExpr();
-                            $$.setExpr(appendOwnedOperand(ds, hint.getClear()), $1);
-
-                            //An alternative implementation is to add an annotation to the dataset, but these will then get commoned
-                            //up, and generally I suspect hints should be treated as not commoned up.  Maybe there should be a flag!
-                            //The code generator also doesn't yet preserve annotations from multiple branches.
-                            //$$.setExpr(createMetaAnnotation($3.getExpr(), args), $1);
+                            $$.setExpr(createDataset(no_unordered, $3.getExpr()), $1);
                         }
-
-*/
     ;
 
 
@@ -9629,20 +9710,7 @@ loopOptions
     ;
 
 loopOption
-    : PARALLEL '(' expression ')'
-                        {
-                            parser->normalizeExpression($3);    // could check more...
-                            $$.setExpr(createExprAttribute(parallelAtom, $3.getExpr()));
-                            $$.setPosition($1);
-                        }
-    | PARALLEL '(' expression ',' expression ')'
-                        {
-                            parser->normalizeExpression($3, type_set, false);
-                            parser->normalizeExpression($5, type_int, false);
-                            $$.setExpr(createExprAttribute(parallelAtom, $3.getExpr(), $5.getExpr()));
-                            $$.setPosition($1);
-                        }
-    | commonAttribute
+    : commonAttribute
     ;
 
 graphOptions
@@ -9656,12 +9724,7 @@ graphOptions
     ;
 
 graphOption
-    : PARALLEL
-                        {
-                            $$.setExpr(createExprAttribute(parallelAtom));
-                            $$.setPosition($1);
-                        }
-    | commonAttribute
+    : commonAttribute
     ;
 
 remoteOptions
@@ -10496,21 +10559,11 @@ JoinFlag
                             $$.setPosition($1);
                         }
     | onFailAction
-    | PARALLEL
-                        {
-                            $$.setExpr(createAttribute(parallelAtom));
-                            $$.setPosition($1);
-                        }
     | SEQUENTIAL
                         {
                             $$.setExpr(createAttribute(sequentialAtom));
                             $$.setPosition($1);
                         }
-    | UNORDERED
-                        {
-                            $$.setExpr(createAttribute(unorderedAtom));
-                            $$.setPosition($1);
-                        }
     | GROUP '(' startSortOrder heterogeneous_expr_list ')' endSortOrder
                         {
                             HqlExprArray args;
@@ -10520,16 +10573,6 @@ JoinFlag
                             OwnedHqlExpr impliedAttr = createComma(createAttribute(lookupAtom), createAttribute(manyAtom));
                             $$.setExpr(createComma(groupAttr.getClear(), impliedAttr.getClear()), $1);
                         }
-    | STABLE
-                        {
-                            $$.setExpr(createExprAttribute(stableAtom));
-                            $$.setPosition($1);
-                        }
-    | UNSTABLE
-                        {
-                            $$.setExpr(createExprAttribute(unstableAtom));
-                            $$.setPosition($1);
-                        }
     | STREAMED          {   $$.setExpr(createAttribute(streamedAtom)); $$.setPosition($1); }
     ;
 
@@ -10808,12 +10851,7 @@ parseFlag
                             parser->normalizeExpression($3, type_numeric, false);
                             $$.setExpr(createExprAttribute(maxLengthAtom, $3.getExpr()));
                         }
-    | PARALLEL
-                        {
-                            $$.setExpr(createAttribute(parallelAtom));
-                            $$.setPosition($1);
-                        }
-    | hintAttribute
+    | commonAttribute
     ;
 
 xmlParseFlags
@@ -11559,22 +11597,6 @@ sortItem
                             $$.setExpr(createExprAttribute(bestAtom, args));
                         }
     | mergeJoinFlag
-    | STABLE
-                        {
-                            $$.setExpr(createExprAttribute(stableAtom));
-                            $$.setPosition($1);
-                        }
-    | STABLE '(' expression ')'
-                        {
-                            parser->normalizeExpression($3, type_string, false);
-                            $$.setExpr(createExprAttribute(stableAtom, $3.getExpr()));
-                            $$.setPosition($1);
-                        }
-    | UNSTABLE
-                        {
-                            $$.setExpr(createExprAttribute(unstableAtom));
-                            $$.setPosition($1);
-                        }
     | KEYED
                         {
                             $$.setExpr(createAttribute(keyedAtom));
@@ -11584,16 +11606,6 @@ sortItem
                         {
                             $$.setExpr(createAttribute(groupedAtom), $1);
                         }
-    | UNSTABLE '(' expression ')'
-                        {
-                            parser->normalizeExpression($3, type_string, false);
-                            $$.setExpr(createExprAttribute(unstableAtom, $3.getExpr()));
-                            $$.setPosition($1);
-                        }
-    | PARALLEL
-                        {
-                            $$.setExpr(createAttribute(parallelAtom), $1);
-                        }
     | prefetchAttribute
     ;
 
@@ -11684,8 +11696,9 @@ rollupFlag
     ;
 
 conditions
-    : '(' condList ')'  {
-                            $$.setExpr($2.getExpr());
+    : '(' condList optCommonAttrs ')'
+                        {
+                            $$.setExpr(createComma($2.getExpr(), $3.getExpr()));
                         }
     | '(' ')'           {
                             $$.setNullExpr();

ecl/hql/hqlgram2.cpp

@@ -5688,7 +5688,9 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
             bool ok = false;
             if (attributes)
             {
-                if (attr == hintAtom) ok = true;
+                if ((attr == hintAtom) || (attr == parallelAtom) || (attr == orderedAtom) || (attr == algorithmAtom) ||
+                    (attr == stableAtom) || (attr == unstableAtom))
+                    ok = true;
 
                 switch (op)
                 {
@@ -5707,7 +5709,7 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
                     if (attr == prefetchAtom) ok = true;
                     if (attr == mergeAtom) ok = true;
                     if (attr == groupedAtom) ok = true;
-                    //fall through
+                    /* no break */
                 case no_group:
                     if (attr == allAtom) ok = true;
                     if (attr == localAtom) ok = true;
@@ -5720,7 +5722,7 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
                     break;
                 case no_topn:
                     if (attr == bestAtom) ok = true;
-                    //fall through
+                    /* no break */
                 case no_sort:
                     if (attr == localAtom) ok = true;
                     if (attr == skewAtom) ok = true;
@@ -5728,9 +5730,6 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
                     if (attr == manyAtom) ok = true;
                     if (attr == fewAtom) ok = true;
                     if (attr == assertAtom) ok = true;
-                    if (attr == stableAtom) ok = true;
-                    if (attr == unstableAtom) ok = true;
-                    if (attr == parallelAtom) ok = true;
                     break;
                 case no_nwaymerge:
                     if (attr == localAtom) ok = true;
@@ -5739,7 +5738,7 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
                 case no_mergejoin:
                     if (attr == dedupAtom) ok = true;
                     if (attr == assertAtom) ok = true;
-                    //fall through
+                    /* no break */
                 case no_nwayjoin:
                     if (attr == localAtom) ok = true;
                     if (attr == mofnAtom) ok = true;
@@ -8878,16 +8877,44 @@ void HqlGram::createAppendFiles(attribute & targetAttr, attribute & leftAttr, at
 {
     OwnedHqlExpr left = leftAttr.getExpr();
     OwnedHqlExpr right = rightAttr.getExpr();
-    if (left->isDatarow()) 
+    if (left->isDatarow())
         left.setown(createDatasetFromRow(LINK(left)));
     right.setown(checkEnsureRecordsMatch(left, right, rightAttr.pos, right->isDatarow()));
     if (right->isDatarow())
         right.setown(createDatasetFromRow(LINK(right)));
-    IHqlExpression * attr = kind ? createAttribute(kind) : NULL;
+    IHqlExpression * attr = NULL;
+    if (!kind)
+        attr = getOrderedAttribute(false);
+    else if (kind == pullAtom)
+        attr = createComma(createAttribute(_ordered_Atom), createAttribute(pullAtom));
+    else
+        attr = createAttribute(kind);
+
     targetAttr.setExpr(createDataset(no_addfiles, LINK(left), createComma(LINK(right), attr)));
     targetAttr.setPosition(leftAttr);
 }
 
+static IHqlExpression * createAppendFiles(IHqlExpression * expr, IHqlExpression * attrs)
+{
+    if (expr->getOperator() != no_comma)
+    {
+        if (expr->isDatarow())
+            return createDatasetFromRow(LINK(expr));
+        return LINK(expr);
+    }
+
+    IHqlExpression * lhs = createAppendFiles(expr->queryChild(0), attrs);
+    IHqlExpression * rhs = createAppendFiles(expr->queryChild(1), attrs);
+    return createDataset(no_addfiles, lhs, createComma(rhs, LINK(attrs)));
+}
+
+IHqlExpression * HqlGram::createAppendFiles(attribute & filesAttr, IHqlExpression * _attrs)
+{
+    OwnedHqlExpr files = filesAttr.getExpr();
+    OwnedHqlExpr attrs = _attrs;
+    return ::createAppendFiles(files, attrs);
+}
+
 void HqlGram::createAppendDictionaries(attribute & targetAttr, attribute & leftAttr, attribute & rightAttr, IAtom * kind)
 {
     OwnedHqlExpr left = leftAttr.getExpr();
@@ -10443,6 +10470,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case ACOS: msg.append("ACOS"); break;
     case AFTER: msg.append("AFTER"); break;
     case AGGREGATE: msg.append("AGGREGATE"); break;
+    case ALGORITHM: msg.append("ALGORITHM"); break;
     case ALIAS: msg.append("__ALIAS__"); break;
     case ALL: msg.append("ALL"); break;
     case ALLNODES: msg.append("ALLNODES"); break;
@@ -10578,7 +10606,6 @@ static void getTokenText(StringBuffer & msg, int token)
     case HAVING: msg.append("HAVING"); break;
     case HEADING: msg.append("HEADING"); break;
     case HINT: msg.append("HINT"); break;
-    case HOLE: msg.append("HOLE"); break;
     case IF: msg.append("IF"); break;
     case IFF: msg.append("IFF"); break;
     case IFBLOCK: msg.append("IFBLOCK"); break;
@@ -10958,7 +10985,7 @@ void HqlGram::simplifyExpected(int *expected)
                        GROUP, GROUPED, KEYED, UNGROUP, JOIN, PULL, ROLLUP, ITERATE, PROJECT, NORMALIZE, PIPE, DENORMALIZE, CASE, MAP, 
                        HTTPCALL, SOAPCALL, LIMIT, PARSE, FAIL, MERGE, PRELOAD, ROW, TOPN, ALIAS, LOCAL, NOFOLD, NOCOMBINE, NOHOIST, NOTHOR, IF, GLOBAL, __COMMON__, __COMPOUND__, TOK_ASSERT, _EMPTY_,
                        COMBINE, ROWS, REGROUP, XMLPROJECT, SKIP, LOOP, CLUSTER, NOLOCAL, REMOTE, PROCESS, ALLNODES, THISNODE, GRAPH, MERGEJOIN, STEPPED, NONEMPTY, HAVING,
-                       TOK_CATCH, '@', SECTION, WHEN, IFF, COGROUP, HINT, INDEX, PARTITION, AGGREGATE, SUBSORT, TOK_ERROR, CHOOSE, TRACE, QUANTILE, 0);
+                       TOK_CATCH, '@', SECTION, WHEN, IFF, COGROUP, HINT, INDEX, PARTITION, AGGREGATE, SUBSORT, TOK_ERROR, CHOOSE, TRACE, QUANTILE, UNORDERED, 0);
     simplify(expected, EXP, ABS, SIN, COS, TAN, SINH, COSH, TANH, ACOS, ASIN, ATAN, ATAN2, 
                        COUNT, CHOOSE, MAP, CASE, IF, HASH, HASH32, HASH64, HASHMD5, CRC, LN, TOK_LOG, POWER, RANDOM, ROUND, ROUNDUP, SQRT, 
                        TRUNCATE, LENGTH, TRIM, INTFORMAT, REALFORMAT, ASSTRING, TRANSFER, MAX, MIN, EVALUATE, SUM,

ecl/hql/hqlir.cpp

@@ -284,7 +284,7 @@ const char * getOperatorIRText(node_operator op)
     EXPAND_CASE(no,existsdict);
     EXPAND_CASE(no,quantile);
     EXPAND_CASE(no,nocombine);
-    EXPAND_CASE(no,unused28);
+    EXPAND_CASE(no,unordered);
     EXPAND_CASE(no,unused29);
     EXPAND_CASE(no,unused30);
     EXPAND_CASE(no,unused31);

ecl/hql/hqllex.l

@@ -620,6 +620,7 @@ ABS                 { RETURNSYM(ABS); }
 ACOS                { RETURNSYM(ACOS); }
 AFTER               { RETURNSYM(AFTER); }
 AGGREGATE           { RETURNSYM(AGGREGATE); }
+ALGORITHM           { RETURNSYM(ALGORITHM); }
 __ALIAS__           { RETURNSYM(ALIAS); }
 ALL                 { RETURNSYM(ALL); }
 ALLNODES            { RETURNSYM(ALLNODES); }
@@ -738,7 +739,6 @@ __GROUPED__         { RETURNSYM(__GROUPED__); }
 HAVING              { RETURNSYM(HAVING); }
 HEADING             { RETURNSYM(HEADING); }
 HINT                { RETURNSYM(HINT); }
-HOLE                { RETURNSYM(HOLE); }
 HTTPCALL            { RETURNSYM(HTTPCALL); }
 HTTPHEADER          { RETURNSYM(HTTPHEADER); }
 IF                  { RETURNSYM(IF); }

ecl/hql/hqlmeta.cpp

@@ -48,6 +48,8 @@ static IHqlExpression * cacheMatchGroupOrderSortlist;
 static IHqlExpression * cached_omitted_Attribute;
 static IHqlExpression * cacheAnyAttribute;
 static IHqlExpression * cacheAnyOrderSortlist;
+static IHqlExpression * cacheOrderedAttribute;
+static IHqlExpression * cacheUnorderedAttribute;
 static CHqlMetaProperty * nullMetaProperty;
 static CHqlMetaProperty * nullGroupedMetaProperty;
 
@@ -64,6 +66,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     cacheIndeterminateSortlist = createValue(no_sortlist, makeSortListType(NULL), LINK(cacheIndeterminateAttribute));
     cacheMatchGroupOrderSortlist = createValue(no_sortlist, makeSortListType(NULL), LINK(cacheGroupedElement));
     cacheAnyOrderSortlist = createValue(no_sortlist, makeSortListType(NULL), LINK(cacheAnyAttribute));
+    cacheOrderedAttribute = createExprAttribute(_ordered_Atom);
+    cacheUnorderedAttribute = createExprAttribute(_ordered_Atom, createConstant(false));
     nullMetaProperty = new CHqlMetaProperty;
     nullGroupedMetaProperty = new CHqlMetaProperty;
     nullGroupedMetaProperty->meta.setUnknownGrouping();
@@ -73,6 +77,8 @@ MODULE_EXIT()
 {
     nullGroupedMetaProperty->Release();
     nullMetaProperty->Release();
+    cacheUnorderedAttribute->Release();
+    cacheOrderedAttribute->Release();
     cached_omitted_Attribute->Release();
     cacheAnyOrderSortlist->Release();
     cacheMatchGroupOrderSortlist->Release();
@@ -284,6 +290,14 @@ bool CHqlMetaInfo::appearsToBeSorted(bool isLocal, bool ignoreGrouping)
     return globalSortOrder != NULL;
 }
 
+void CHqlMetaInfo::applyUnordered()
+{
+    if (!grouping)
+        groupSortOrder.clear();
+    else
+        removeAllSortOrders();
+}
+
 void CHqlMetaInfo::clearGrouping()
 {
     if (grouping)
@@ -1890,6 +1904,24 @@ IHqlExpression * preserveTableInfo(IHqlExpression * newTable, IHqlExpression * o
 
 //---------------------------------------------------------------------------------------------------------------------
 
+bool hasOrderedAttribute(IHqlExpression * expr)
+{
+    return expr->hasAttribute(orderedAtom) || expr->hasAttribute(_ordered_Atom);
+}
+
+bool isOrdered(IHqlExpression * expr)
+{
+    //If no ordered or _ordered_ attributes are present, assume the order of an activity's output is significant.
+    if (expr->hasAttribute(orderedAtom))
+        return getBoolAttribute(expr, orderedAtom, true);
+    return getBoolAttribute(expr, _ordered_Atom, true);
+}
+
+IHqlExpression * getOrderedAttribute(bool value)
+{
+    return value ? LINK(cacheOrderedAttribute) : LINK(cacheUnorderedAttribute);
+}
+
 static void getMetaIntersection(CHqlMetaInfo & meta, IHqlExpression * other)
 {
     CHqlMetaProperty * otherMetaProp = queryMetaProperty(other);
@@ -2221,9 +2253,9 @@ void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr)
                 meta.set(parentMeta);
                 if (!createDefaultLeft)
                 {
-                    bool preservesOrder = !expr->queryAttribute(unorderedAtom);
+                    bool preservesOrder = getBoolAttribute(expr, orderedAtom, true);
                     if (isKeyedJoin)
-                        preservesOrder = expr->queryAttribute(_ordered_Atom) != NULL;
+                        preservesOrder = getBoolAttribute(expr, _ordered_Atom, false);
                     if (!preservesOrder)
                         meta.removeAllSortOrders();
 
@@ -2276,7 +2308,7 @@ void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr)
                     mapper.setMapping(transform, leftSelect);
 
                     meta.set(ungroupedMeta);
-                    if (expr->hasAttribute(unorderedAtom))
+                    if (!getBoolAttribute(expr, orderedAtom, true))
                         meta.removeAllSortOrders();
                     meta.applyProject(mapper);
                 }
@@ -2385,6 +2417,12 @@ void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr)
             }
             break;
         }
+    case no_unordered:
+        {
+            extractMeta(meta, dataset);
+            meta.applyUnordered();
+            break;
+        }
     case no_preservemeta:
         {
             meta.distribution.set(queryRemoveOmitted(expr->queryChild(1)));
@@ -2870,6 +2908,9 @@ void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr)
         break;
     }
 
+    if (!isOrdered(expr))
+        meta.applyUnordered();
+
     assertex(isGrouped(expr) == (meta.grouping != NULL));
 #ifdef _DEBUG
     assertex(!meta.grouping || meta.grouping->getOperator() == no_sortlist);
@@ -3309,6 +3350,7 @@ ITypeInfo * calculateDatasetType(node_operator op, const HqlExprArray & parms)
         break;
     case no_distribute:
     case no_distributed:
+    case no_unordered:
     case no_assertdistributed:
         newRecordType.set(recordType);
         break;

ecl/hql/hqlmeta.hpp

@@ -17,6 +17,8 @@
 #ifndef __HQLMETA_HPP_
 #define __HQLMETA_HPP_
 
+#include "hqlexpr.hpp"
+
 class TableProjectMapper;
 
 class HQL_API CHqlMetaInfo
@@ -32,6 +34,7 @@ public:
     void applyGlobalSort(IHqlExpression * sortOrder);
     void applyProject(TableProjectMapper & mapper);
     void applySubSort(IHqlExpression * groupBy, IHqlExpression * sortOrder, bool isLocal);
+    void applyUnordered();
 
     void ensureAppearsSorted(bool isLocal, bool ignoreGrouping);
     void getIntersection(const CHqlMetaInfo & other);
@@ -100,6 +103,10 @@ extern HQL_API IHqlExpression * mapJoinDistribution(TableProjectMapper & mapper,
 extern HQL_API bool isPartitionedForGroup(IHqlExpression * table, IHqlExpression *grouping, bool isGroupAll);
 extern HQL_API bool isPartitionedForGroup(IHqlExpression * table, const HqlExprArray & grouping, bool isGroupAll);
 
+extern HQL_API bool hasOrderedAttribute(IHqlExpression * expr);
+extern HQL_API bool isOrdered(IHqlExpression * expr);  // Is the output of this activity considered to be ordered?
+extern HQL_API IHqlExpression * getOrderedAttribute(bool value);
+
 //The following only look at the sort order, and not the partitioning
 extern HQL_API bool isSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal);
 extern HQL_API IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal, bool alwaysLocal, bool allowSubSort);

ecl/hql/hqlopt.cpp

@@ -637,6 +637,7 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateDataset(IHqlExpression * trans
         case no_distributed:
         case no_keyeddistribute:
         case no_preservemeta:
+        case no_unordered:
             if (isScalarAggregate || !isGrouped(ds->queryChild(0)))
                 next = ds->queryChild(0);
             break;
@@ -2231,6 +2232,7 @@ IHqlExpression * CTreeOptimizer::queryMoveKeyedExpr(IHqlExpression * transformed
     case no_sorted:
     case no_stepped:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     case no_grouped:
     case no_nofold:
@@ -3010,6 +3012,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             case no_sorted:
             case no_stepped:
             case no_distributed:
+            case no_unordered:
             case no_distribute:
             case no_group:
             case no_grouped:
@@ -3200,6 +3203,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             switch(childOp)
             {
             case no_distributed:
+            case no_unordered:
             case no_sorted:
             case no_limit:
             case no_choosen:
@@ -3326,6 +3330,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                     break;
                 return moveProjectionOverSimple(transformed, true, false);
             case no_distributed:
+            case no_unordered:
             case no_sorted:
             case no_grouped:
                 if (transformedCountProject)
@@ -3519,6 +3524,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             case no_distributed:
             case no_sorted:
             case no_grouped:
+            case no_unordered:
                 return moveProjectionOverSimple(transformed, false, false);
             case no_stepped:
                 return moveProjectionOverSimple(transformed, false, true);
@@ -3977,7 +3983,17 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                     break;
                 }
             }
+            break;
+        }
+    case no_unordered:
+        //This is only added if not shared.  A future transform will optimize shared expressions where all uses are unordered.
+        if (!hasOrderedAttribute(child))
+        {
+            OwnedHqlExpr unorderedChild = appendOwnedOperand(child, getOrderedAttribute(false));
+            return replaceChild(transformed, unorderedChild);
         }
+        //MORE: Remove UNORDERED(SORT(ds,local|grouped)) and other optimizations see HPCC-10144
+        break;
     }
 
     return LINK(transformed);

ecl/hql/hqlthql.cpp

@@ -21,6 +21,7 @@
 
 #include "hqltrans.ipp"
 #include "hqlutil.hpp"
+#include "hqlmeta.hpp"
 #include "workunit.hpp"
 
 //The following constants can be uncommented to increase the level of detail which is added to the processed graphs
@@ -1280,16 +1281,24 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             const char * opText = getEclOpString(no);
             if ((no == no_div) && expr->queryType()->isInteger())
                 opText = "DIV";
-            if ((no == no_addfiles) && expr->hasAttribute(_ordered_Atom))
-                opText = "&";
-            if ((no == no_addfiles) && expr->hasAttribute(_orderedPull_Atom))
-                opText = "&&";
+            if (no == no_addfiles)
+            {
+                if (isOrdered(expr))
+                    opText = "&";
+                if (expr->hasAttribute(pullAtom))
+                    opText = "&&";
+            }
             unsigned num = expr->numChildren();
             for (unsigned i=1; i < num; i++)
             {
                 IHqlExpression * child = queryChild(expr, i);
                 if (child)
                 {
+                    if ((no == no_addfiles) && child->isAttribute())
+                    {
+                        if (child->queryName() == orderedAtom)
+                            continue;
+                    }
                     if (xgmmlGraphText && endsWithDotDotDot(s))
                         break;
                     s.append(' ').append(opText).append(' ');

ecl/hqlcpp/hqlckey.cpp

@@ -24,6 +24,7 @@
 #include "jdebug.hpp"
 
 #include "hql.hpp"
+#include "hqlmeta.hpp"
 #include "hqlthql.hpp"
 #include "hqlhtcpp.ipp"
 #include "hqlttcpp.ipp"
@@ -365,6 +366,7 @@ IHqlExpression * KeyedJoinInfo::querySimplifiedKey(IHqlExpression * expr)
         case no_nofold:
         case no_forcegraph:
         case no_nocombine:
+        case no_unordered:
             break;
         case no_newkeyindex:
             return LINK(expr);
@@ -1407,7 +1409,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedJoinOrDenormalize(BuildCt
         flags.append("|JFindexoptional");
     if (info.needToExtractJoinFields())
         flags.append("|JFextractjoinfields");
-    if (expr->hasAttribute(unorderedAtom))
+    if (!isOrdered(expr))
         flags.append("|JFreorderable");
     if (transformReturnsSide(expr, no_left, 0))
         flags.append("|JFtransformmatchesleft");

ecl/hqlcpp/hqlcppds.cpp

@@ -2172,6 +2172,7 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
     case no_compound_selectnew:
     case no_compound_inline:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     case no_sorted:
     case no_nofold:
@@ -2479,6 +2480,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
     case no_compound_selectnew:
     case no_compound_inline:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     case no_sorted:
     case no_nofold:
@@ -3485,6 +3487,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder
     case no_compound_selectnew:
     case no_compound_inline:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     case no_sorted:
     case no_nofold:
@@ -3947,6 +3950,7 @@ BoundRow * HqlCppTranslator::buildDatasetIterate(BuildCtx & ctx, IHqlExpression
     case no_compound_selectnew:
     case no_compound_inline:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     case no_sorted:
     case no_nofold:
@@ -4742,6 +4746,7 @@ IHqlExpression * HqlCppTranslator::ensureIteratedRowIsLive(BuildCtx & initctx, B
         case no_stepped:
         case no_sorted:
         case no_distributed:
+        case no_unordered:
         case no_preservemeta:
         case no_choosen:
         case no_selectnth:                      // can occur as the lhs of no_select

ecl/hqlcpp/hqlcpputil.cpp

@@ -37,7 +37,6 @@
 //===========================================================================
 
 static ITypeInfo * cachedVoidType;
-static IHqlExpression * cachedBoolValue[2];
 static IHqlExpression * cachedZero;
 static IHqlExpression * cachedNullChar;
 static IHqlExpression * defaultAttrExpr;
@@ -85,8 +84,6 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     doubleType = makeRealType(8);
 
     cachedVoidType = makeVoidType();
-    cachedBoolValue[false] = createConstant(false);
-    cachedBoolValue[true] = createConstant(true);
     cachedZero = createIntConstant(0);
     cachedNullChar = createConstant(createCharValue(0, makeCharType()));
     defaultAttrExpr = createAttribute(defaultAtom);
@@ -114,8 +111,6 @@ MODULE_EXIT()
     defaultAttrExpr->Release();
     boolType->Release();
     cachedVoidType->Release();
-    cachedBoolValue[false]->Release();
-    cachedBoolValue[true]->Release();
     cachedZero->Release();
     cachedNullChar->Release();
     unsignedType->Release();
@@ -140,7 +135,6 @@ IHqlExpression * getZero()                              { return LINK(cachedZero
 ITypeInfo *     queryBoolType()                     { return boolType; }
 ITypeInfo *     queryVoidType()                     { return cachedVoidType; }
 
-IHqlExpression * queryBoolExpr(bool value){ return cachedBoolValue[value]; }
 IHqlExpression * queryNullChar()                    { return cachedNullChar; }
 IHqlExpression * queryZero()                            { return cachedZero; }
 IHqlExpression * getDefaultAttr() { return LINK(defaultAttrExpr); }

ecl/hqlcpp/hqlcpputil.hpp

@@ -21,7 +21,6 @@ interface IConstWorkUnit;
 class CHqlBoundExpr;
 
 extern ITypeInfo * queryBoolType();
-extern IHqlExpression * queryBoolExpr(bool value);
 extern IHqlExpression * queryNullChar();
 extern IHqlExpression * queryZero();
 extern IHqlExpression * getZero();

ecl/hqlcpp/hqlgraph.cpp

@@ -60,9 +60,9 @@ void addGraphAttributeInt(IPropertyTree * node, const char * name, __int64 value
     addGraphAttribute(node, name)->setPropInt64("@value", value);
 }
 
-void addGraphAttributeBool(IPropertyTree * node, const char * name, bool value)
+void addGraphAttributeBool(IPropertyTree * node, const char * name, bool value, bool alwaysAdd)
 {
-    if (value)
+    if (value || alwaysAdd)
         addGraphAttribute(node, name)->setPropBool("@value", value);
 }
 

ecl/hqlcpp/hqlgraph.ipp

@@ -107,7 +107,7 @@ protected:
 IPropertyTree * addGraphAttribute(IPropertyTree * node, const char * name);
 void addGraphAttribute(IPropertyTree * node, const char * name, const char * value);
 void addGraphAttributeInt(IPropertyTree * node, const char * name, __int64 value);
-void addGraphAttributeBool(IPropertyTree * node, const char * name, bool value);
+void addGraphAttributeBool(IPropertyTree * node, const char * name, bool value, bool alwaysAdd=false);
 IPropertyTree * addSimpleGraphEdge(IPropertyTree * subGraph, unsigned __int64 source, unsigned __int64 target, unsigned outputIndex, unsigned inputIndex, IAtom * kind, const char * label, bool nWay);
 IPropertyTree * addComplexGraphEdge(IPropertyTree * graph, unsigned __int64 sourceGraph, unsigned __int64 targetGraph, unsigned __int64 sourceActivity, unsigned __int64 targetActivity, unsigned outputIndex, IAtom * kind, const char * label);
 void removeGraphAttribute(IPropertyTree * node, const char * name);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1822,9 +1822,9 @@ void ActivityInstance::addAttributeInt(const char * name, __int64 value)
     addGraphAttributeInt(graphNode, name, value);
 }
 
-void ActivityInstance::addAttributeBool(const char * name, bool value)
+void ActivityInstance::addAttributeBool(const char * name, bool value, bool alwaysAdd)
 {
-    addGraphAttributeBool(graphNode, name, value);
+    addGraphAttributeBool(graphNode, name, value, alwaysAdd);
 }
 
 void ActivityInstance::addAttribute(const char * name, IHqlExpression * expr)
@@ -2041,6 +2041,12 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
         addAttributeBool("coLocal", true);
     if (isNoAccess)
         addAttributeBool("noAccess", true);
+    if (dataset->hasAttribute(parallelAtom))
+        addAttributeInt("parallel", getIntValue(queryAttributeChild(dataset, parallelAtom, 0), -1));
+    if (hasOrderedAttribute(dataset))
+        addAttributeBool("ordered", isOrdered(dataset), true);
+    if (dataset->hasAttribute(algorithmAtom))
+        addAttribute("algorithm", queryAttributeChild(dataset, algorithmAtom, 0));
 
     if (!options.obfuscateOutput)
     {
@@ -6594,6 +6600,7 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             }
             case no_sorted:
             case no_preservemeta:
+            case no_unordered:
             case no_grouped:
             case no_nofold:
             case no_nohoist:
@@ -8181,12 +8188,12 @@ ABoundActivity * HqlCppTranslator::doBuildActivityCloned(BuildCtx & ctx, IHqlExp
 //---------------------------------------------------------------------------
 // no_addfiles
 
-static void unwindAddFiles(HqlExprArray & args, IHqlExpression * expr, bool isOrdered, bool isOrderedPull)
+static void unwindAddFiles(HqlExprArray & args, IHqlExpression * expr, bool reqIsOrdered, bool isOrderedPull)
 {
-    if ((expr->getOperator() == no_addfiles) && (expr->hasAttribute(_ordered_Atom) == isOrdered) && (expr->hasAttribute(_orderedPull_Atom) == isOrderedPull))
+    if ((expr->getOperator() == no_addfiles) && (isOrdered(expr) == reqIsOrdered) && (expr->hasAttribute(pullAtom) == isOrderedPull))
     {
-        unwindAddFiles(args, expr->queryChild(0), isOrdered, isOrderedPull);
-        unwindAddFiles(args, expr->queryChild(1), isOrdered, isOrderedPull);
+        unwindAddFiles(args, expr->queryChild(0), reqIsOrdered, isOrderedPull);
+        unwindAddFiles(args, expr->queryChild(1), reqIsOrdered, isOrderedPull);
     }
     else
         args.append(*LINK(expr));
@@ -8216,8 +8223,8 @@ static IHqlExpression * queryRootConcatActivity(IHqlExpression * expr)
 ABoundActivity * HqlCppTranslator::doBuildActivityConcat(BuildCtx & ctx, IHqlExpression * expr)
 {
     HqlExprArray inExprs;
-    bool ordered = expr->hasAttribute(_ordered_Atom);
-    bool orderedPull = expr->hasAttribute(_orderedPull_Atom);
+    bool ordered = isOrdered(expr);
+    bool orderedPull = expr->hasAttribute(pullAtom);
     unwindAddFiles(inExprs, expr, ordered, orderedPull);
 
     //If all coming from disk, probably better to pull them in order.
@@ -8596,34 +8603,9 @@ ABoundActivity * HqlCppTranslator::doBuildActivityLoop(BuildCtx & ctx, IHqlExpre
 
     if (parallel)
     {
-        IHqlExpression * arg0 = parallel->queryChild(0);
-        IHqlExpression * arg1 = parallel->queryChild(1);
-
-        LinkedHqlExpr parallelList;
-        LinkedHqlExpr numThreads;
-        if (arg0)
-        {
-            if (arg1)
-            {
-                parallelList.set(arg0);
-                numThreads.set(arg1);
-            }
-            else if (arg0->isList())
-                parallelList.set(arg0);
-            else
-                numThreads.set(arg0);
-        }
+        IHqlExpression * numThreads = parallel->queryChild(0);
         if (numThreads)
             doBuildUnsignedFunction(instance->startctx, "defaultParallelIterations", numThreads);
-
-        if (parallelList)
-        {
-            Owned<ITypeInfo> setType = makeSetType(LINK(unsignedType));
-            BuildCtx funcctx(instance->startctx);
-            funcctx.addQuotedCompoundLiteral("virtual void numParallelIterations(size32_t & __lenResult, void * & __result)");
-            funcctx.addQuotedLiteral("bool __isAllResult;");
-            doBuildFunctionReturn(funcctx, setType, parallelList);
-        }
     }
 
     StringBuffer flags;
@@ -12198,7 +12180,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
     if (isLookupJoin && isManyLookup) flags.append("|JFmanylookup");
     if (expr->hasAttribute(onFailAtom))
         flags.append("|JFonfail");
-    if (expr->hasAttribute(unorderedAtom))
+    if (!isOrdered(expr))
         flags.append("|JFreorderable");
     if (transformReturnsSide(expr, no_left, 0))
         flags.append("|JFtransformmatchesleft");
@@ -16585,6 +16567,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
             flags.append("|TAFunstable");
         }
     }
+    if (!method)
+        method = queryAttributeChild(expr, algorithmAtom, 0);
 
     if (spill)
         flags.append("|TAFspill");
@@ -18905,6 +18889,7 @@ static bool needsRealThor(IHqlExpression *expr, unsigned flags)
     case no_thor:
     case no_apply:
     case no_distributed:
+    case no_unordered:
     case no_preservemeta:
     case no_sorted:
     case no_limit:

ecl/hqlcpp/hqlhtcpp.ipp

@@ -149,7 +149,7 @@ public:
     void addAttribute(const char * name, const char * value);
     void addAttribute(const char * name, IHqlExpression * expr);
     void addAttributeInt(const char * name, __int64 value);
-    void addAttributeBool(const char * name, bool value);
+    void addAttributeBool(const char * name, bool value, bool alwaysAdd=false);
     void addLocationAttribute(IHqlExpression * location);
     void addNameAttribute(IHqlExpression * location);
     void removeAttribute(const char * name);

ecl/hqlcpp/hqlinline.cpp

@@ -270,6 +270,7 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
     case no_sorted:
     case no_distributed:
     case no_preservemeta:
+    case no_unordered:
     case no_nofold:
     case no_nohoist:
     case no_nocombine:
@@ -727,6 +728,7 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr, bool optimize
     case no_sorted:
     case no_distributed:
     case no_preservemeta:
+    case no_unordered:
     case no_grouped:
     case no_alias_scope:
     case no_sequential:

ecl/hqlcpp/hqliter.cpp

@@ -72,6 +72,7 @@ bool canBuildSequenceInline(IHqlExpression * expr)
         case no_sorted:
         case no_distributed:
         case no_preservemeta:
+        case no_unordered:
         case no_grouped:
         case no_preload:
         case no_limit:

ecl/hqlcpp/hqlresource.cpp

@@ -3149,6 +3149,7 @@ bool ResourcerInfo::expandRatherThanSpill(bool noteOtherSpills)
         case no_grouped:
         case no_distributed:
         case no_preservemeta:
+        case no_unordered:
         case no_nofold:
         case no_nohoist:
         case no_nocombine:
@@ -3262,6 +3263,7 @@ bool ResourcerInfo::expandRatherThanSplit()
         case no_grouped:
         case no_distributed:
         case no_preservemeta:
+        case no_unordered:
         case no_compound_diskread:
         case no_compound_disknormalize:
         case no_compound_diskaggregate:
@@ -3585,6 +3587,7 @@ static bool isPotentialCompoundSteppedIndexRead(IHqlExpression * expr)
         case no_sorted:
         case no_preservemeta:
         case no_distributed:
+        case no_unordered:
         case no_grouped:
         case no_stepped:
         case no_section:
@@ -4800,7 +4803,7 @@ void EclResourcer::oldSpotUnbalancedSplitters(IHqlExpression * expr, unsigned wh
             switch (expr->getOperator())
             {
             case no_addfiles:
-                if (expr->hasAttribute(_ordered_Atom) || expr->hasAttribute(_orderedPull_Atom) || isGrouped(expr))
+                if (isOrdered(expr) || isGrouped(expr))
                     modify = true;
                 break;
             default:
@@ -5061,7 +5064,7 @@ bool CSplitterInfo::allInputsPulledIndependently(IHqlExpression * expr)
     switch (expr->getOperator())
     {
     case no_addfiles:
-        if (expr->hasAttribute(_ordered_Atom) || expr->hasAttribute(_orderedPull_Atom) || isGrouped(expr))
+        if (isOrdered(expr) || isGrouped(expr))
             return false;
         return true;
     case no_parallel:
@@ -5109,6 +5112,9 @@ void CSplitterInfo::gatherPotentialSplitters(IHqlExpression * expr, IHqlExpressi
     {
     case no_null:
     case no_fail:
+        return;
+    case no_attr:
+    case no_attr_expr:
         //Any sources that never generate any rows are always fine as a splitter
         return;
     //MORE: A source that generates a single row, and subsequent rows are never read, is always fine.

ecl/hqlcpp/hqlsource.cpp

@@ -26,6 +26,7 @@
 
 #include "hql.hpp"
 #include "hqlattr.hpp"
+#include "hqlmeta.hpp"
 #include "hqlthql.hpp"
 #include "hqlhtcpp.ipp"
 #include "hqlttcpp.ipp"
@@ -179,6 +180,7 @@ bool isSimpleSource(IHqlExpression * expr)
         case no_stepped:
         case no_distributed:
         case no_preservemeta:
+        case no_unordered:
         case no_grouped:
         case no_compound_diskread:
         case no_compound_disknormalize:
@@ -965,6 +967,7 @@ void SourceBuilder::analyse(IHqlExpression * expr)
     case no_sorted:
     case no_distributed:
     case no_preservemeta:
+    case no_unordered:
     case no_grouped:
     case no_alias_scope:
     case no_section:
@@ -1350,6 +1353,7 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
     case no_stepped:
     case no_distributed:
     case no_preservemeta:
+    case no_unordered:
     case no_grouped:
     case no_preload:
     case no_limit:
@@ -5993,6 +5997,7 @@ void MonitorExtractor::extractAllFilters(IHqlExpression * dataset)
         case no_hqlproject:
         case no_distributed:
         case no_preservemeta:
+        case no_unordered:
         case no_sorted:
         case no_stepped:
         case no_grouped:
@@ -6249,7 +6254,7 @@ void IndexReadBuilderBase::buildFlagsMember(IHqlExpression * expr)
     StringBuffer flags;
     if (tableExpr->hasAttribute(sortedAtom))
         flags.append("|TIRsorted");
-    else if (tableExpr->hasAttribute(unorderedAtom))
+    else if (!isOrdered(tableExpr))
         flags.append("|TIRunordered");
     if (!monitors.isFiltered())
         flags.append("|TIRnofilter");

ecl/hqlcpp/hqlttcpp.cpp

@@ -31,6 +31,7 @@
 #include "hqlfold.hpp"
 #include "hqlgraph.ipp"
 #include "hqllib.ipp"
+#include "hqlmeta.hpp"
 #include "hqlpmap.hpp"
 #include "hqlopt.hpp"
 #include "hqlcerrors.hpp"
@@ -96,6 +97,7 @@ static bool isWorthHoisting(IHqlExpression * expr, bool asSubQuery)
         case no_stepped:
         case no_distributed:
         case no_preservemeta:
+        case no_unordered:
         case no_nofold:
         case no_nohoist:
         case no_nocombine:
@@ -2571,6 +2573,7 @@ IHqlExpression * ThorHqlTransformer::normalizeCoGroup(IHqlExpression * expr)
     }
     else
     {
+        inputs.append(*getOrderedAttribute(false));
         //otherwise append the datasets and then sort them all
         OwnedHqlExpr appended = createDataset(no_addfiles, inputs);
         appended.setown(cloneInheritedAnnotations(expr, appended));
@@ -2847,7 +2850,7 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
     //Tag a keyed join as ordered in the platforms that ensure it does remain ordered.  Extend if the others do.
     if (isKeyedJoin(expr))
     {
-        if ((translator.targetRoxie() || options.keyedJoinPreservesOrder) && !expr->hasAttribute(_ordered_Atom) && !expr->hasAttribute(unorderedAtom))
+        if ((translator.targetRoxie() || options.keyedJoinPreservesOrder) && !hasOrderedAttribute(expr))
             return appendOwnedOperand(expr, createAttribute(_ordered_Atom));
         return NULL;
     }
@@ -4378,6 +4381,7 @@ void CompoundSourceTransformer::analyseGatherInfo(IHqlExpression * expr)
     case no_sorted:
     case no_preservemeta:
     case no_distributed:
+    case no_unordered:
     case no_grouped:
     case no_stepped:
     case no_section:
@@ -12255,7 +12259,7 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
                 children.replace(*createAttribute(_selfJoinPlaceholder_Atom), 1);       // replace the 1st dataset with an attribute so parameters are still in the same place.
                 return createDataset(no_selfjoin, children);
             }
-            if (isKeyedJoin(transformed) && translator.targetRoxie() && !expr->hasAttribute(_ordered_Atom))
+            if (isKeyedJoin(transformed) && translator.targetRoxie() && !hasOrderedAttribute(expr))
                 return appendOwnedOperand(transformed, createAttribute(_ordered_Atom));
             return transformed.getClear();
         }

ecl/regress/loopparallel.ecl

@@ -67,8 +67,8 @@ end;
 
 
 initial := dataset([], rec);
-results := LOOP(initial, 4, processLoop(rows(left), counter), parallel([2,2]));
+results := LOOP(initial, 4, processLoop(rows(left), counter), parallel(2));
 output(results);
 
-results2 := LOOP(initial, 4, (left.id1 > counter), processLoop(rows(left), counter), parallel([2],2));
+results2 := LOOP(initial, 4, (left.id1 > counter), processLoop(rows(left), counter), parallel(2));
 output(results2);

ecl/regress/sort.ecl

@@ -36,7 +36,7 @@ sortedNames := sort(namesTable, -surname, forename);
 
 previousSort := sorted(thorNamesTable,(string25)thorNamesTable.forename,(string15)thorNamesTable.surname);
 
-myset := sort(person,person.per_last_name,person.per_first_name,stable('Insertion'));
+myset := sort(person,person.per_last_name,person.per_first_name,stable,algorithm('Insertion'));
 output(myset);
 string algo := '' : stored('algorithm');
 //myset2 := sort(namesTable,(string25)namesTable.surname,(string15)namesTable.forename,joined(myset));

roxie/ccd/ccdserver.cpp

@@ -14285,8 +14285,6 @@ class CRoxieServerParallelLoopActivity : public CRoxieServerLoopActivity
     QueueOf<const void, true> ready;
     CriticalSection helperCS;
     CriticalSection cs;
-    size32_t sizeNumParallel;
-    rtlDataAttr listNumParallel;
     unsigned defaultNumParallel;
     LoopExecutorThread executor;
     IProbeManager* probeManager;
@@ -14309,7 +14307,6 @@ public:
     {
         probeManager = _probeManager;
         defaultNumParallel = 0;
-        sizeNumParallel = 0;
     }
 
     virtual void onCreate(IHThorArg *_colocalParent)
@@ -14327,7 +14324,6 @@ public:
         defaultNumParallel = helper.defaultParallelIterations();
         if (!defaultNumParallel)
             defaultNumParallel = DEFAULT_PARALLEL_LOOP_THREADS;
-        helper.numParallelIterations(sizeNumParallel, listNumParallel.refdata());
 
         //MORE: If numIterations <= number of parallel iterations[1], 
         //then we don't need to create a separate thread to do the processing, and the results will also avoid
@@ -14398,9 +14394,7 @@ public:
 
     unsigned getNumParallel(unsigned iter)
     {
-        if (iter * sizeof(unsigned) >= sizeNumParallel)
-            return defaultNumParallel;
-        return ((unsigned *)listNumParallel.getdata())[iter];
+        return defaultNumParallel;
     }
 
     inline void enqueueResult(const void * row)

rtl/include/eclhelper.hpp

@@ -2545,7 +2545,6 @@ struct IHThorLoopArg : public IHThorArg
     virtual bool loopAgain(unsigned counter, unsigned numRows, const void * * _rows)    = 0;
     virtual void createParentExtract(rtlRowBuilder & builder) = 0;
     virtual unsigned defaultParallelIterations() = 0;
-    virtual void numParallelIterations(size32_t & retSize, void * & retData) = 0;
     //If new loop again is set the following should be used instead of loopAgain
     virtual bool loopFirstTime() = 0;
     virtual unsigned loopAgainResult() = 0;  // which result contains the indication of whether to loop again?

rtl/include/eclhelper_base.hpp

@@ -3225,7 +3225,6 @@ class CThorLoopArg : public CThorArg, implements IHThorLoopArg
     virtual unsigned numIterations() { return 0; }
     virtual bool loopAgain(unsigned counter, unsigned num, const void * * _rows)    { return num != 0; }
     virtual unsigned defaultParallelIterations() { return 0; }
-    virtual void numParallelIterations(size32_t & retSize, void * & retData) { retSize = 0; retData = NULL; }
     virtual bool loopFirstTime() { return false; }
     virtual unsigned loopAgainResult() { return 0; }
 };

testing/regress/ecl/childds1.ecl

@@ -53,6 +53,6 @@ cntBad := COUNT(ds(assertTrue(seq > 10, 'seq > 10'))) + NOFOLD(100000);
 
 //NOFOLD is required to stop code generator converting this to a filter (trueValue && ...) which means that cntBad is evaluated always
 //See childds1err.ecl
-cond := IF(trueValue, ds, NOFOLD(ds)(seq != cntBad));
+cond := IF(trueValue, ds, NOFOLD(ds)(seq != cntBad, hint(testHintIsSupported(1))));
 output(cond);
 

testing/regress/ecl/concat.ecl

@@ -62,7 +62,7 @@ output(SORT(TABLE(file1+g1+g1+g2, { value2, c := count(GROUP)} ), value2, c));
 
 output(TABLE(file1&g2, { value2, c := count(GROUP)} ));
 output(TABLE(file1&g2&g3, { value2, c := count(GROUP)} ));
-output(TABLE(file1&g1&g1&g2, { value2, c := count(GROUP)} ));
+output(TABLE((+)(file1,g1,g1,g2,ORDERED), { value2, c := count(GROUP)} ));
 
 // all inputs grouped - preserves grouping
 output(SORT(TABLE(g1+g2, { value2, c := count(GROUP)} ), value2, c));

testing/regress/ecl/loopparallel.ecl

@@ -73,7 +73,7 @@ end;
 //Simple parallel loop, 2 iterations each item
 initial := dataset([], rec);
 
-results := LOOP(initial, 4, processLoop(rows(left), counter), parallel([2],2));
+results := LOOP(initial, 4, processLoop(rows(left), counter), parallel(2));
 output(results);
 
 //Do the whole lot in a single go - check parallel is bounded by actual number of iterations
@@ -81,7 +81,7 @@ results2 := LOOP(initial, 4, processLoop(rows(left), counter), parallel(10000));
 output(results2);
 
 //Same as first, but also include a row filter
-results3 := LOOP(initial, 4, (left.id1 > counter), processLoop(rows(left), counter), parallel([2],2));
+results3 := LOOP(initial, 4, (left.id1 > counter), processLoop(rows(left), counter), parallel(2));
 output(sort(results3, id1, id2, score));
 
 //Partial reading of a loop's final output