Implement a SHUFFLE activity and keyword

- New SHUFFLE keyword (provisional)
  SHUFFLE(ds, { grouping }, { sort }, options);
  Equivalent to
  UNGROUP(SORT(GROUP(ds, grouping, options), sort));

- Add options to
  * Translate SHUFFLE operations to equivalent if engines don't support it
  * Create SHUFFLE for the inputs to thor local joins if possible.
  * Use SHUFFLE instead of a SORT if that is possible.
  * Use shuffle to reorder join conditions if one side is already sorted

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/commonext/commonext.cpp

@@ -200,6 +200,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     kindArray[TAKexternalsink] = "externalsink";
     kindArray[TAKexternalprocess] = "externalprocess";
     kindArray[TAKwhen_action] = "when_action";
+    kindArray[TAKshuffle] = "shuffle";
 
 //Non standard
     kindArray[TAKcountdisk] = "countdisk";

common/thorhelper/thorcommon.cpp

@@ -786,6 +786,7 @@ extern const char * getActivityText(ThorActivityKind kind)
     case TAKexternalsink:           return "User Output";
     case TAKexternalprocess:        return "User Proceess";
     case TAKwhen_action:            return "When";
+    case TAKshuffle:                return "Shuffle";
     }
     throwUnexpected();
 }

ecl/hql/hqlattr.cpp

@@ -291,6 +291,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_fetch:
     case no_join:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_dedup:
     case no_enth:
@@ -608,7 +609,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_dataset_from_transform:
 
     case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
-    case no_unused13: case no_unused14: case no_unused15: case no_unused17: case no_unused18: case no_unused19:
+    case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
@@ -1573,6 +1574,7 @@ bool isLocalActivity(IHqlExpression * expr)
     case no_cogroup:
     case no_cosort:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_topn:
     case no_iterate:
@@ -1760,6 +1762,7 @@ bool localChangesActivityAction(IHqlExpression * expr)
     case no_cogroup:
     case no_cosort:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_topn:
     case no_iterate:
@@ -2386,6 +2389,7 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
     case no_assertgrouped:
     case no_assertdistributed:
     case no_sort:
+    case no_shuffle:
     case no_nohoist:
     case no_section:
     case no_sectioninput:

ecl/hql/hqldsparam.cpp

@@ -264,7 +264,7 @@ IHqlExpression* HqlGram::bindFieldMap(IHqlExpression* expr, IHqlExpression* map)
                     newmaps.append(*createComma(createAttribute(mappedName), LINK(to)));
             }
             map->Release();
-            IHqlExpression * newmap = createValue(no_sortlist, makeSortListType(NULL), newmaps);
+            IHqlExpression * newmap = createSortList(newmaps);
 
             expr = clearFieldMap(expr);
             expr = createFieldMap(expr,newmap);

ecl/hql/hqlerrors.hpp

@@ -415,7 +415,8 @@
 #define HQLERR_CannotSubmitFunction 2385
 #define HQLERR_CannotSubmitModule   2386
 #define ERR_COUNTER_NOT_COUNT       2387
-#define HQLERR_CannotSubmitMacroX   2385
+#define HQLERR_CannotSubmitMacroX   2388
+#define HQLERR_CannotBeGrouped      2389
 
 #define ERR_ASSERTION_FAILS         100000
 

ecl/hql/hqlexpr.cpp

@@ -1000,6 +1000,7 @@ const char *getOpString(node_operator op)
     case no_enth: return "ENTH";
     case no_sample: return "SAMPLE";
     case no_sort: return "SORT";
+    case no_shuffle: return "SHUFFLE";
     case no_sorted: return "SORTED";
     case no_choosen: return "CHOOSEN";
     case no_choosesets: return "CHOOSESETS";
@@ -1426,7 +1427,7 @@ const char *getOpString(node_operator op)
     case no_dataset_alias: return "TABLE";
 
     case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
-    case no_unused13: case no_unused14: case no_unused15: case no_unused17: case no_unused18: case no_unused19:
+    case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
@@ -1782,6 +1783,7 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_cosort:
     case no_keyed:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_stepped:
     case no_transformebcdic:
@@ -2046,6 +2048,7 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_sample:
     case no_selectnth:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_stepped:
     case no_assertsorted:
@@ -2332,6 +2335,7 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_section:
     case no_sample:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_stepped:
     case no_assertsorted:
@@ -5377,6 +5381,7 @@ void CHqlDataset::cacheParent()
     case no_keyed:
     // change ordering
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_stepped:
     case no_cosort:
@@ -10587,6 +10592,16 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             type.setown(getTypeDistribute(datasetType, newDistribution, NULL));
             break;
         }
+    case no_shuffle:
+        {
+            bool isLocal = queryProperty(localAtom, parms) != NULL;
+            OwnedHqlExpr normalizedSortOrder = replaceSelector(&parms.item(1), dataset, cachedActiveTableExpr);
+            OwnedHqlExpr mappedGrouping = replaceSelector(&parms.item(2), dataset, cachedActiveTableExpr);
+            type.setown(getTypeGrouped(datasetType, mappedGrouping, isLocal));
+            type.setown(getTypeGroupSort(type, normalizedSortOrder));
+            type.setown(getTypeUngroup(type));
+            break;
+        }
     case no_cosort:
     case no_sort:
     case no_sorted:
@@ -10713,7 +10728,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
                 else
                     unwindRecordAsSelects(sortExprs, record, cachedActiveTableExpr);
 
-                OwnedHqlExpr sortOrder = createValue(no_sortlist, makeSortListType(NULL), sortExprs);
+                OwnedHqlExpr sortOrder = createSortList(sortExprs);
                 if (queryProperty(noRootAtom, parms))
                     type.setown(getTypeLocalSort(type, sortOrder));
                 else
@@ -10927,7 +10942,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             IHqlExpression * order= queryProperty(sortedAtom, parms);       // already uses no_activetable to refer to dataset
             assertex(order);
             unwindChildren(components, order);
-            OwnedHqlExpr sortlist = createValue(no_sortlist, makeSortListType(NULL), components);
+            OwnedHqlExpr sortlist = createSortList(components);
             if (queryProperty(localAtom, parms))
                 type.setown(getTypeLocalSort(datasetType, sortlist));
             else
@@ -10957,7 +10972,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             OwnedHqlExpr normalizedOrder = replaceSelector(order, selector, cachedActiveTableExpr);
             HqlExprArray components;
             unwindChildren(components, normalizedOrder);
-            OwnedHqlExpr sortlist = createValue(no_sortlist, makeSortListType(NULL), components);
+            OwnedHqlExpr sortlist = createSortList(components);
             //These are all currently implemented as local activities, need to change following if no longer true
             type.setown(getTypeLocalSort(datasetType, sortlist));
             break;
@@ -14978,6 +14993,11 @@ extern HQL_API IHqlExpression * createAbstractRecord(IHqlExpression * record)
     return createRecord(args);
 }
 
+extern HQL_API IHqlExpression * createSortList(HqlExprArray & elements)
+{
+     return createValue(no_sortlist, makeSortListType(NULL), elements);
+}
+
 //==============================================================================================================
 
 

ecl/hql/hqlexpr.hpp

@@ -343,7 +343,7 @@ enum _node_operator {
         no_crc,
         no_return_stmt,
         no_update,    
-    no_unused17,
+        no_shuffle,
     no_unused18,
         no_alias,
     no_unused19,
@@ -1580,6 +1580,7 @@ extern HQL_API IHqlScope * closeScope(IHqlScope * scope);
 extern HQL_API _ATOM queryPatternName(IHqlExpression * expr);
 extern HQL_API IHqlExpression * closeAndLink(IHqlExpression * expr);
 extern HQL_API IHqlExpression * createAbstractRecord(IHqlExpression * record);
+extern HQL_API IHqlExpression * createSortList(HqlExprArray & elements);
 
 // Same as expr->queryChild() except it doesn't return attributes.
 inline IHqlExpression * queryRealChild(IHqlExpression * expr, unsigned i)

ecl/hql/hqlfold.cpp

@@ -3307,6 +3307,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
             break;
         }
     case no_sort:
+    case no_shuffle:
     case no_sorted:
         {
             //If action does not change the type information, then it can't have done anything...
@@ -3314,7 +3315,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
                 return removeParentNode(expr);
             if (isNull(child) || hasNoMoreRowsThan(child, 1))
                 return removeParentNode(expr);
-            //If all arguments to sort are constnat then remove it, otherwise the activities will not like it.
+            //If all arguments to sort are constant then remove it, otherwise the activities will not like it.
             //NOTE: MERGE has its sort order preserved, so it won't cause issues there.
             bool allConst = true;
             ForEachChildFrom(i, expr, 1)
@@ -5397,6 +5398,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_keyeddistribute:
     case no_cosort:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_assertsorted:
     case no_topn:

ecl/hql/hqlgram.y

@@ -386,6 +386,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   SERVICE
   SET
   SHARED
+  SHUFFLE
   SIMPLE_TYPE
   SIN
   SINGLE
@@ -7110,7 +7111,7 @@ simpleDataSet
                             RecordSelectIterator iter(active->queryRecord(), active);
                             ForEach(iter)
                                 components.append(*iter.get());
-                            OwnedHqlExpr sortlist = createValue(no_sortlist, makeSortListType(NULL), components);
+                            OwnedHqlExpr sortlist = createSortList(components);
                             OwnedHqlExpr hash = createValue(no_hash32, makeIntType(4, false), LINK(sortlist), createAttribute(internalAtom));
                             $$.setExpr(createDataset(no_distribute, ds.getClear(), createComma(hash.getClear(), $5.getExpr())), $1);
                         }
@@ -7334,7 +7335,7 @@ simpleDataSet
                                     values.append(OLINK(cur));
                             }
                             if (values.ordinality())
-                                cond = createValue(no_sortlist, makeSortListType(NULL), values);
+                                cond = createSortList(values);
                             else
                                 cond = createConstant(true);
 
@@ -8164,6 +8165,14 @@ simpleDataSet
                             $$.setExpr(parser->createSortExpr(no_sort, $4, $7, sortItems));
                             $$.setPosition($1);
                         }
+    | SHUFFLE '(' startSortOrder startTopFilter ',' sortListExpr ',' sortListExpr optCommonAttrs ')'  endSortOrder endTopFilter
+                        {
+                            OwnedHqlExpr options = $9.getExpr();
+                            if (isGrouped($4.queryExpr()))
+                                parser->reportError(HQLERR_CannotBeGrouped, $1, "SHUFFLE not yet supported on grouped datasets");
+                            //NB: $6 and $8 are reversed in their internal representation to make consistent with no_sort
+                            $$.setExpr(createDataset(no_shuffle, $4.getExpr(), createComma($8.getExpr(), $6.getExpr(), options.getClear())), $1);
+                        }
     | SORTED '(' startSortOrder startTopFilter ',' beginList sortListOptCurleys ')' endSortOrder endTopFilter
                         {
                             HqlExprArray sortItems;
@@ -8181,7 +8190,7 @@ simpleDataSet
                             IHqlExpression * record = dataset->queryRecord();
                             unwindRecordAsSelects(sorted, record, dataset->queryNormalizedSelector());
                             args.append(*dataset.getClear());
-                            args.append(*createValue(no_sortlist, makeSortListType(NULL), sorted));
+                            args.append(*createSortList(sorted));
                             $$.setExpr(createDataset(no_sorted, args));
                             $$.setPosition($1);
                         }
@@ -8191,7 +8200,7 @@ simpleDataSet
 
                             HqlExprArray args;
                             $5.unwindCommaList(args);
-                            OwnedHqlExpr stepOrder = createValue(no_sortlist, makeSortListType(NULL), args);
+                            OwnedHqlExpr stepOrder = createSortList(args);
                             $$.setExpr(createDatasetF(no_stepped, dataset.getClear(), stepOrder.getClear(), $6.getExpr(), NULL));
                             $$.setPosition($1);
                         }
@@ -10445,7 +10454,7 @@ optFieldMaps
                         {
                             HqlExprArray args;
                             parser->endList(args);
-                            $$.setExpr(createValue(no_sortlist, makeSortListType(NULL), args), $1);
+                            $$.setExpr(createSortList(args), $1);
                         }
     |                   { $$.setNullExpr(); }
     | '{' beginList error '}'
@@ -10477,7 +10486,8 @@ fieldMap
 
 sortListOptCurleys
     : sortList
-    | '{' sortList '}'  
+    | '{' sortList '}'
+    | '{' sortList '}' ',' sortList         /* Allow trailing attributes */
     ;
 
 sortList
@@ -10719,6 +10729,15 @@ beginList
                         }
     ;
 
+sortListExpr
+    : beginList '{' sortList '}'
+                        {
+                            HqlExprArray elements;
+                            parser->endList(elements);
+                            $$.setExpr(createSortList(elements));
+                        }
+    ;
+
 ignoreDummyList
     :
                         {

ecl/hql/hqlgram2.cpp

@@ -5488,7 +5488,7 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
         }
     }
     if (items.ordinality())
-        return createValue(no_sortlist, makeSortListType(NULL), items);
+        return createSortList(items);
     if (op == no_list)
         return createValue(no_sortlist, makeSortListType(NULL), items);
     return NULL;
@@ -10084,6 +10084,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case HTTPHEADER: msg.append("HTTPHEADER"); break;
     case PROXYADDRESS: msg.append("PROXYADDRESS"); break;
     case HTTPCALL: msg.append("HTTPCALL"); break;
+    case SHUFFLE: msg.append("SHUFFLE"); break;
     case SOAPCALL: msg.append("SOAPCALL"); break;
     case SORT: msg.append("SORT"); break;
     case SORTED: msg.append("SORTED"); break;
@@ -10277,7 +10278,7 @@ void HqlGram::simplifyExpected(int *expected)
                        GROUP, GROUPED, KEYED, UNGROUP, JOIN, PULL, ROLLUP, ITERATE, PROJECT, NORMALIZE, PIPE, DENORMALIZE, CASE, MAP, 
                        HTTPCALL, SOAPCALL, LIMIT, PARSE, FAIL, MERGE, PRELOAD, ROW, TOPN, ALIAS, LOCAL, NOFOLD, NOHOIST, NOTHOR, IF, GLOBAL, __COMMON__, __COMPOUND__, TOK_ASSERT, _EMPTY_,
                        COMBINE, ROWS, REGROUP, XMLPROJECT, SKIP, LOOP, CLUSTER, NOLOCAL, REMOTE, PROCESS, ALLNODES, THISNODE, GRAPH, MERGEJOIN, STEPPED, NONEMPTY, HAVING,
-                       TOK_CATCH, '@', SECTION, WHEN, IFF, COGROUP, HINT, INDEX, PARTITION, AGGREGATE, 0);
+                       TOK_CATCH, '@', SECTION, WHEN, IFF, COGROUP, HINT, INDEX, PARTITION, AGGREGATE, SHUFFLE, 0);
     simplify(expected, EXP, ABS, SIN, COS, TAN, SINH, COSH, TANH, ACOS, ASIN, ATAN, ATAN2, 
                        COUNT, CHOOSE, MAP, CASE, IF, HASH, HASH32, HASH64, HASHMD5, CRC, LN, TOK_LOG, POWER, RANDOM, ROUND, ROUNDUP, SQRT, 
                        TRUNCATE, LENGTH, TRIM, INTFORMAT, REALFORMAT, ASSTRING, TRANSFER, MAX, MIN, EVALUATE, SUM,

ecl/hql/hqllex.l

@@ -878,6 +878,7 @@ SERVICE             { RETURNHARD(SERVICE); }
 SET                 { RETURNSYM(SET); }
 __SET_DEBUG_OPTION__ { RETURNSYM(HASH_OPTION); }
 SHARED              { RETURNHARD(SHARED); }
+SHUFFLE             { RETURNSYM(SHUFFLE); }
 SIN                 { RETURNSYM(SIN); }
 SINGLE              { RETURNSYM(SINGLE); }
 SINH                { RETURNSYM(SINH); }

ecl/hql/hqlmeta.cpp

@@ -39,7 +39,6 @@
 #include "hqlmeta.hpp"
 
 //#define OPTIMIZATION2
-//#define OPTIMIZE_SORT_WITH_GROUP
 
 static IHqlExpression * cacheGroupedElement;
 static IHqlExpression * cacheUnknownAttribute;
@@ -293,7 +292,7 @@ IHqlExpression * createSubSortlist(IHqlExpression * sortlist, unsigned from, uns
     unwindChildren(components, sortlist, from, to);
     if (subsetAttr)
         components.append(*LINK(subsetAttr));
-    return createValue(no_sortlist, makeSortListType(NULL), components);
+    return createSortList(components);
 }
 
 void removeDuplicates(HqlExprArray & components)
@@ -407,7 +406,7 @@ static IHqlExpression * normalizeSortlist(IHqlExpression * sortlist)
     unwindNormalizeSortlist(components, sortlist, false);
     removeDuplicates(components);
     //This never returns NULL if the input was non-null
-    return createValue(no_sortlist, makeSortListType(NULL), components);
+    return createSortList(components);
 }
 
 inline IHqlExpression * normalizeSortlist(IHqlExpression * sortlist, IHqlExpression * dataset)
@@ -505,7 +504,7 @@ IHqlExpression * getLocalSortOrder(ITypeInfo * type)
     if (components.ordinality())
     {
         removeDuplicates(components);
-        return createValue(no_sortlist, makeSortListType(NULL), components);
+        return createSortList(components);
     }
     return NULL;
 }
@@ -1033,7 +1032,7 @@ bool isSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isL
 }
 
 
-IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal, bool alwaysLocal)
+IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal, bool alwaysLocal, bool allowShuffle)
 {
     if (isSortedForGroup(table, sortList, isLocal||alwaysLocal))
         return LINK(table);
@@ -1221,31 +1220,11 @@ bool isAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLo
 {
     HqlExprArray components;
     normalizeComponents(components, newSort);
-    OwnedHqlExpr normalizedOrder = createValue(no_sortlist, makeSortListType(NULL), components);
+    OwnedHqlExpr normalizedOrder = createSortList(components);
     return normalizedIsAlreadySorted(dataset, normalizedOrder, isLocal, ignoreGrouping);
 }
 
 
-IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
-{
-    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
-        return LINK(dataset);
-
-    if (isLocal || alwaysLocal)
-    {
-        unsigned partialSorted = numElementsAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping);
-        if (partialSorted != 0)
-        {
-            OwnedHqlExpr shuffled = ensureShuffled(dataset, order, isLocal, ignoreGrouping, alwaysLocal, partialSorted);
-            if (shuffled)
-                return shuffled.getClear();
-        }
-    }
-
-    IHqlExpression * attr = isLocal ? createLocalAttribute() : (isGrouped(dataset) && ignoreGrouping) ? createAttribute(globalAtom) : NULL;
-    return createDatasetF(no_sort, LINK(dataset), LINK(order), attr, NULL);
-}
-
 //--------------------------------------------------------------------------------------------------------------------
 
 static unsigned numCompatibleSortElements(IHqlExpression * existingOrder, IHqlExpression * normalizedOrder)
@@ -1277,7 +1256,7 @@ static unsigned normalizedNumSortedElements(IHqlExpression * dataset, IHqlExpres
     return numCompatibleSortElements(existingOrder, normalizedOrder);
 }
 
-unsigned numElementsAlreadySorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping)
+static unsigned numElementsAlreadySorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping)
 {
 #ifdef OPTIMIZATION2
     if (hasNoMoreRowsThan(dataset, 1))
@@ -1288,29 +1267,55 @@ unsigned numElementsAlreadySorted(IHqlExpression * dataset, IHqlExpression * ord
     return normalizedNumSortedElements(dataset, normalizedOrder, isLocal, ignoreGrouping);
 }
 
-
 //Elements in the exprarray have already been mapped;
-unsigned numElementsAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
+static unsigned numElementsAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
 {
     HqlExprArray components;
     normalizeComponents(components, newSort);
-    OwnedHqlExpr normalizedOrder = createValue(no_sortlist, makeSortListType(NULL), components);
+    OwnedHqlExpr normalizedOrder = createSortList(components);
     return normalizedNumSortedElements(dataset, normalizedOrder, isLocal, ignoreGrouping);
 }
 
-IHqlExpression * ensureShuffled(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal, unsigned sortedElements)
+bool isWorthShuffling(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping)
 {
-#ifndef OPTIMIZE_SORT_WITH_GROUP
-    return NULL;
-#endif
-    if (sortedElements == (unsigned)-1)
-        sortedElements = numElementsAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping);
+    //MORE: Should this look at the cardinality of the already-sorted fields, and not transform if below a certain threshold?
+    return numElementsAlreadySorted(dataset, order, isLocal, ignoreGrouping) != 0;
+}
 
-#ifdef _DEBUG
-    assertex(numElementsAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping) >= sortedElements);
-#endif
+bool isWorthShuffling(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
+{
+    //MORE: Should this look at the cardinality of the already-sorted fields, and not transform if below a certain threshold?
+    return numElementsAlreadySorted(dataset, newSort, isLocal, ignoreGrouping) != 0;
+}
+
+//--------------------------------------------------------------------------------------------------------------------
+
+//Convert SHUFFLE(ds, <sort>, <grouping>, ?LOCAL, options) to
+//g := GROUP(ds, grouping, ?LOCAL); s := SORT(g, <sort>, options); GROUP(s);
+IHqlExpression * convertShuffleToGroupedSort(IHqlExpression * expr)
+{
+    IHqlExpression * dataset = expr->queryChild(0);
+    IHqlExpression * newOrder = expr->queryChild(1);
+    IHqlExpression * grouping = expr->queryChild(2);
 
-    if ((sortedElements == 0) || isGrouped(dataset))
+    assertex(!isGrouped(dataset) || expr->hasProperty(globalAtom));
+    OwnedHqlExpr attr = isLocalActivity(expr) ? createLocalAttribute() : NULL;
+    OwnedHqlExpr grouped = createDatasetF(no_group, LINK(dataset), LINK(grouping), LINK(attr), NULL);
+
+    HqlExprArray args;
+    args.append(*grouped.getClear());
+    args.append(*LINK(newOrder));
+    unwindChildren(args, expr, 3);
+    removeProperty(args, localAtom);
+    OwnedHqlExpr sorted = createDataset(no_sort, args);
+    return createDataset(no_group, LINK(sorted));
+}
+
+static IHqlExpression * createShuffled(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
+{
+    bool isGroupedShuffle = !ignoreGrouping && isGrouped(dataset);
+    unsigned sortedElements = numElementsAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping);
+    if ((sortedElements == 0) || isGroupedShuffle)
         return NULL;
 
     HqlExprArray components;
@@ -1323,13 +1328,15 @@ IHqlExpression * ensureShuffled(IHqlExpression * dataset, IHqlExpression * order
     OwnedHqlExpr newOrder = createValueSafe(no_sortlist, makeSortListType(NULL), components, sortedElements, components.ordinality());
 
     OwnedHqlExpr attr = isLocal ? createLocalAttribute() : (isGrouped(dataset) && ignoreGrouping) ? createAttribute(globalAtom) : NULL;
-    OwnedHqlExpr group = createDatasetF(no_group, LINK(dataset), LINK(alreadySorted), LINK(attr), NULL);
-    OwnedHqlExpr sorted = createDataset(no_sort, LINK(group), LINK(newOrder));
-    OwnedHqlExpr ungrouped = createDataset(no_group, LINK(sorted));
-    if (!isAlreadySorted(ungrouped, order, isLocal||alwaysLocal, ignoreGrouping))
-        dbglogExpr(ungrouped);
-    assertex(isAlreadySorted(ungrouped, order, isLocal||alwaysLocal, ignoreGrouping));
-    return ungrouped.getClear();
+    OwnedHqlExpr shuffle = createDatasetF(no_shuffle, LINK(dataset), LINK(newOrder), LINK(alreadySorted), LINK(attr), NULL);
+    //Grouped shuffles never generated, global shuffles (if generated) get converted to a global group
+    if (!isLocal && !alwaysLocal)
+        shuffle.setown(convertShuffleToGroupedSort(shuffle));
+
+    if (!isAlreadySorted(shuffle, order, isLocal||alwaysLocal, ignoreGrouping))
+        dbglogExpr(shuffle);
+    assertex(isAlreadySorted(shuffle, order, isLocal||alwaysLocal, ignoreGrouping));
+    return shuffle.getClear();
 }
 
 IHqlExpression * getShuffleSort(IHqlExpression * dataset, HqlExprArray & order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
@@ -1337,12 +1344,9 @@ IHqlExpression * getShuffleSort(IHqlExpression * dataset, HqlExprArray & order,
     if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
         return NULL;
 
-    unsigned numSortedElements = numElementsAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping);
-    if (numSortedElements == 0)
-        return NULL;
     OwnedHqlExpr sortlist = createValueSafe(no_sortlist, makeSortListType(NULL), order);
     OwnedHqlExpr mappedSortlist = replaceSelector(sortlist, queryActiveTableSelector(), dataset);
-    return ensureShuffled(dataset, mappedSortlist, isLocal, ignoreGrouping, alwaysLocal, numSortedElements);
+    return createShuffled(dataset, mappedSortlist, isLocal, ignoreGrouping, alwaysLocal);
 }
 
 IHqlExpression * getShuffleSort(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
@@ -1350,10 +1354,28 @@ IHqlExpression * getShuffleSort(IHqlExpression * dataset, IHqlExpression * order
     if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
         return NULL;
 
-    unsigned numSortedElements = numElementsAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping);
-    if (numSortedElements == 0)
-        return NULL;
-    return ensureShuffled(dataset, order, isLocal, ignoreGrouping, alwaysLocal, numSortedElements);
+    return createShuffled(dataset, order, isLocal, ignoreGrouping, alwaysLocal);
+}
+
+//--------------------------------------------------------------------------------------------------------------------
+
+IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal, bool allowShuffle)
+{
+    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
+        return LINK(dataset);
+
+    if (allowShuffle && (isLocal || alwaysLocal))
+    {
+        if (isWorthShuffling(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
+        {
+            OwnedHqlExpr shuffled = createShuffled(dataset, order, isLocal, ignoreGrouping, alwaysLocal);
+            if (shuffled)
+                return shuffled.getClear();
+        }
+    }
+
+    IHqlExpression * attr = isLocal ? createLocalAttribute() : (isGrouped(dataset) && ignoreGrouping) ? createAttribute(globalAtom) : NULL;
+    return createDatasetF(no_sort, LINK(dataset), LINK(order), attr, NULL);
 }
 
 //-------------------------------

ecl/hql/hqlmeta.hpp

@@ -63,20 +63,20 @@ extern HQL_API bool isPartitionedForGroup(IHqlExpression * table, const HqlExprA
 
 //The following only look at the sort order, and not the partitioning
 extern HQL_API bool isSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal);
-extern HQL_API IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal, bool alwaysLocal);
+extern HQL_API IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal, bool alwaysLocal, bool allowShuffle);
 
 extern HQL_API bool matchDedupDistribution(IHqlExpression * distn, const HqlExprArray & equalities);
 
 extern HQL_API bool appearsToBeSorted(ITypeInfo * type, bool isLocal, bool ignoreGrouping);
 extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
 extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * newSort, bool isLocal, bool ignoreGrouping);
-extern HQL_API IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal);
+extern HQL_API IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal, bool allowShuffle);
 
-extern HQL_API unsigned numElementsAlreadySorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping);
-extern HQL_API unsigned numElementsAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
-extern HQL_API IHqlExpression * ensureShuffled(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal, unsigned sortedElements);
+extern HQL_API bool isWorthShuffling(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping);
+extern HQL_API bool isWorthShuffling(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
 extern HQL_API IHqlExpression * getShuffleSort(IHqlExpression * dataset, HqlExprArray & order, bool isLocal, bool ignoreGrouping, bool alwaysLocal);
 extern HQL_API IHqlExpression * getShuffleSort(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal);
+extern HQL_API IHqlExpression * convertShuffleToGroupedSort(IHqlExpression * expr);
 
 extern HQL_API bool reorderMatchExistingLocalSort(HqlExprArray & sortedLeft, HqlExprArray & reorderedRight, IHqlExpression * dataset, const HqlExprArray & left, const HqlExprArray & right);
 

ecl/hql/hqlopt.cpp

@@ -484,6 +484,7 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateUnsharedDataset(IHqlExpression
     case no_newusertable:
     case no_newaggregate:
     case no_sort:
+    case no_shuffle:
     case no_distribute:
     case no_keyeddistribute:
     case no_fetch:
@@ -515,6 +516,7 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateUnsharedDataset(IHqlExpression
     switch (op)
     {
     case no_sort:
+    case no_shuffle:
     case no_distribute:
     case no_keyeddistribute:
         noteUnused(expr);
@@ -615,6 +617,7 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateDataset(IHqlExpression * trans
                 next = ds->queryChild(0);
             break;
         case no_sort:
+        case no_shuffle:
         case no_sorted:
             //MORE: Allowed if the transform is commutative for no_aggregate
             if (aggOp != no_aggregate)
@@ -2831,6 +2834,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             case no_grouped:
             case no_keyeddistribute:
             case no_sort:
+            case no_shuffle:
             case no_preload:
             case no_assertsorted:
             case no_assertgrouped:
@@ -3102,6 +3106,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                     return swapNodeWithChild(transformed);
                 break;
             case no_sort:
+            case no_shuffle:
                 if (transformedCountProject)
                     break;
                 if (increasesRowSize(transformed))
@@ -3294,6 +3299,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                 return swapNodeWithChild(transformed);
             case no_distribute:
             case no_sort:
+            case no_shuffle:
                 if (increasesRowSize(transformed))
                     break;
                 return moveProjectionOverSimple(transformed, true, false);
@@ -3409,6 +3415,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             switch(child->getOperator())
             {
             case no_sort:
+            case no_shuffle:
                 if (!isLocalActivity(transformed) || isLocalActivity(child))
                     return removeChildNode(transformed);
                 break;
@@ -3421,6 +3428,33 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             }
             break;
         }
+    case no_shuffle:
+        {
+            switch(child->getOperator())
+            {
+            case no_sort:
+                {
+                    if (isGrouped(transformed))
+                        break;
+                    //Convert shuffle(sort) back into a single sort.  Do not convert if it would change the distribution.
+                    if (!isAlwaysLocal() && (!isLocalActivity(transformed) || !isLocalActivity(child)))
+                        break;
+                    OwnedHqlExpr sortOrder = getExistingSortOrder(transformed, true, true);
+                    //A weird user defined SHUFFLE could create an unknown sort order
+                    if (!sortOrder)
+                        break;
+                    OwnedHqlExpr newOrder = replaceSelector(sortOrder, queryActiveTableSelector(), child->queryNormalizedSelector());
+                    decUsage(child);
+                    DBGLOG("Optimizer: Merge %s and %s", queryNode0Text(transformed), queryNode1Text(child));
+                    return ::replaceChild(child, 1, newOrder);
+                }
+
+            case no_shuffle:
+                //This should almost certainly be improved, but it might be a bit tricky!
+                break;
+            }
+            break;
+        }
     case no_keyeddistribute:
     case no_distribute:
         {
@@ -3434,6 +3468,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             case no_distribute:
             case no_keyeddistribute:
             case no_sort:
+            case no_shuffle:
                 if (!transformed->hasProperty(mergeAtom))
                     return removeChildNode(transformed);
                 break;

ecl/hql/hqlopt.hpp

@@ -31,6 +31,7 @@ enum
     HOOfiltersharedproject      = 0x0020,
     HOOhascompoundaggregate     = 0x0040,
     HOOfoldconstantdatasets     = 0x0080,
+    HOOalwayslocal              = 0x0100,
 };
 
 extern HQL_API IHqlExpression * optimizeHqlExpression(IHqlExpression * expr, unsigned options);

ecl/hql/hqlopt.ipp

@@ -130,6 +130,8 @@ protected:
 
     IHqlExpression * getNoHoistAttr();
 
+    inline bool isAlwaysLocal() const { return (options & HOOalwayslocal) != 0; }
+
 protected:
     typedef OPTTRANSFORMBASE PARENT;
     unsigned options;

ecl/hql/hqlthql.cpp

@@ -121,6 +121,7 @@ private:
     bool isServiceDefined(IHqlExpression * expr);
     bool isExportDefined(IHqlExpression * expr);
     void doFunctionDefinition(StringBuffer & newdef, IHqlExpression * funcdef, const char * name, bool inType);
+    void sortlistToEcl(IHqlExpression *expr, StringBuffer &s, bool addCurleys, bool inType);
 
     bool matchesActiveDataset(IHqlExpression * expr);
     void pushScope(IHqlExpression * expr);
@@ -1057,19 +1058,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
         }
         case no_sortlist:
         {
-            bool needComma = false;
-            ForEachChild(idx, expr)
-            {
-                IHqlExpression * child = queryChild(expr, idx);
-                if (child && (expandProcessed || !isInternalAttribute(expr)))
-                {
-                    if (needComma) queryNewline(s.append(", "));
-                    if (queryAddDotDotDot(s, startLength))
-                        break;
-                    needComma = true;
-                    toECL(child, s, false, inType);
-                }
-            }
+            sortlistToEcl(expr, s, false, inType);
             break;
         }
         case no_rowvalue:
@@ -1281,6 +1270,31 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             }
             break;
         }
+        case no_shuffle:
+            {
+                s.append(getEclOpString(expr->getOperator()));
+                s.append('(');
+                if (!xgmmlGraphText)
+                {
+                    toECL(child0, s, false, inType);
+                    queryNewline(s.append(", "));
+                }
+                pushScope(child0);
+                //MORE: Sortlists should always be generated using the {} syntax - then this could be simplified
+                //NOTE: child(1) and child(2) are output in a different order from their representation
+                s.append("{");
+                toECL(expr->queryChild(2), s, false, inType);
+                s.append("}");
+                queryNewline(s.append(", "));
+                s.append("{");
+                toECL(expr->queryChild(1), s, false, inType);
+                s.append("}");
+                childrenToECL(expr, s, inType, true, 3);
+                popScope();
+                s.append(')');
+                break;
+            }
+
         case no_select:
         {
             if (!expandProcessed && 
@@ -2632,6 +2646,28 @@ void HqltHql::defaultChildrenToECL(IHqlExpression *expr, StringBuffer &s, bool i
     }
 }
 
+void HqltHql::sortlistToEcl(IHqlExpression *expr, StringBuffer &s, bool addCurleys, bool inType)
+{
+    unsigned startLength = s.length();
+    if (addCurleys)
+        s.append("{ ");
+    bool needComma = false;
+    ForEachChild(idx, expr)
+    {
+        IHqlExpression * child = queryChild(expr, idx);
+        if (child && (expandProcessed || !isInternalAttribute(expr)))
+        {
+            if (needComma) queryNewline(s.append(", "));
+            if (queryAddDotDotDot(s, startLength))
+                break;
+            needComma = true;
+            toECL(child, s, false, inType);
+        }
+    }
+    if (addCurleys)
+        s.append(" }");
+}
+
 StringBuffer &HqltHql::getFieldTypeString(IHqlExpression * e, StringBuffer &s)
 {
     ITypeInfo * type = e->queryType();

ecl/hqlcpp/hqlcpp.cpp

@@ -1672,6 +1672,14 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.createImplicitAliases,"createImplicitAliases", false),
         DebugOption(options.combineSiblingGraphs,"combineSiblingGraphs", true),
         DebugOption(options.optimizeSharedGraphInputs,"optimizeSharedGraphInputs", true),
+        DebugOption(options.supportsShuffleActivity,"supportsShuffleActivity",false),
+        DebugOption(options.implicitShuffle,"implicitShuffle",false),
+        DebugOption(options.implicitBuildIndexShuffle,"implicitBuildIndexShuffle",false),
+        DebugOption(options.implicitJoinShuffle,"implicitJoinShuffle",false),
+        DebugOption(options.implicitGroupShuffle,"implicitGroupShuffle",false),
+        DebugOption(options.implicitGroupHashAggregate,"implicitGroupHashAggregate",false),
+        DebugOption(options.implicitGroupHashDedup,"implicitGroupHashDedup",false),
+        DebugOption(options.shuffleLocalJoinConditions,"shuffleLocalJoinConditions",false),
     };
 
     //get options values from workunit
@@ -1818,10 +1826,10 @@ unsigned HqlCppTranslator::getOptimizeFlags() const
     switch (targetClusterType)
     {
     case RoxieCluster:
-        optFlags |= HOOnoclonelimit;
+        optFlags |= HOOnoclonelimit|HOOalwayslocal;
         break;
     case HThorCluster:
-        optFlags |= HOOnocloneindexlimit;
+        optFlags |= HOOnocloneindexlimit|HOOalwayslocal;
         break;
     case ThorCluster:
     case ThorLCRCluster:

ecl/hqlcpp/hqlcpp.ipp

@@ -701,6 +701,14 @@ struct HqlCppOptions
     bool                createImplicitAliases;
     bool                combineSiblingGraphs;
     bool                optimizeSharedGraphInputs;
+    bool                supportsShuffleActivity;  // Does the target engine support SHUFFLE?
+    bool                implicitShuffle;  // convert sort when partially sortted to shuffle (group,sort,ungroup)
+    bool                implicitBuildIndexShuffle;  // use shuffle when building indexes?
+    bool                implicitJoinShuffle;  // use shuffle for paritially sorted join inputs when possible
+    bool                implicitGroupShuffle;  // use shuffle if some sort conditions match when grouping
+    bool                implicitGroupHashAggregate;  // convert aggreate(sort(x,a),{..},a,d) to aggregate(group(sort(x,a),a_,{},d))
+    bool                implicitGroupHashDedup;
+    bool                shuffleLocalJoinConditions;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class
@@ -1485,6 +1493,7 @@ public:
     void doBuildAggregateMergeFunc(BuildCtx & ctx, IHqlExpression * expr, bool & requiresOrderedMerge);
     void doBuildAggregateProcessTransform(BuildCtx & ctx, BoundRow * selfRow, IHqlExpression * expr, IHqlExpression * alreadyDoneExpr);
 
+    void doBuildFuncIsSameGroup(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * sortlist);
 
     void processUserAggregateTransform(IHqlExpression * expr, IHqlExpression * transform, SharedHqlExpr & firstTransform, SharedHqlExpr & nextTransform);
     void doBuildUserAggregateFuncs(BuildCtx & ctx, IHqlExpression * expr, bool & requiresOrderedMerge);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -3184,7 +3184,7 @@ void HqlCppTranslator::doBuildFunction(BuildCtx & ctx, ITypeInfo * type, const c
             HqlExprArray parameters;
             OwnedHqlExpr entrypoint = createAttribute(entrypointAtom, createConstant(name));
             OwnedHqlExpr body = createValue(no_null, LINK(type), LINK(entrypoint));
-            OwnedHqlExpr formals = createValue(no_sortlist, makeSortListType(NULL), parameters);
+            OwnedHqlExpr formals = createSortList(parameters);
             OwnedHqlExpr attrs = createAttribute(virtualAtom);
             OwnedHqlExpr function = createFunctionDefinition(NULL, LINK(body), LINK(formals), NULL, LINK(attrs));
             funcctx.addFunction(function);
@@ -5656,6 +5656,9 @@ double HqlCppTranslator::getComplexity(IHqlExpression * expr, ClusterType cluste
         if (isThorCluster(cluster))
             complexity = 5;
         break;
+    case no_shuffle:
+        complexity = 1;
+        break;
     case no_sort:
         if (isLocalActivity(expr) || !isThorCluster(cluster))
             complexity = 2;
@@ -6118,6 +6121,19 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
                 result = doBuildActivityProcess(ctx, expr);
                 break;
             case no_group:
+                //Special case group(shuffle) which will be mapped to group(group(sort(group))) to remove
+                //the redundant group
+                if (!options.supportsShuffleActivity && (expr->queryChild(0)->getOperator() == no_shuffle))
+                {
+                    IHqlExpression * shuffle = expr->queryChild(0);
+                    OwnedHqlExpr groupedSort = convertShuffleToGroupedSort(shuffle);
+                    assertex(groupedSort->getOperator() == no_group);
+                    OwnedHqlExpr newGroup = replaceChild(expr, 0, groupedSort->queryChild(0));
+                    result = doBuildActivityGroup(ctx, newGroup);
+                }
+                else
+                    result = doBuildActivityGroup(ctx, expr);
+                break;
             case no_cogroup:
             case no_assertgrouped:
                 result = doBuildActivityGroup(ctx, expr);
@@ -6171,6 +6187,15 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_sub:
                 result = doBuildActivitySub(ctx, expr);
                 break;
+            case no_shuffle:
+                if (!options.supportsShuffleActivity)
+                {
+                    OwnedHqlExpr groupedSort = convertShuffleToGroupedSort(expr);
+                    result = buildCachedActivity(ctx, groupedSort);
+                }
+                else
+                    result = doBuildActivitySort(ctx, expr);
+                break;
             case no_sort:
             case no_cosort:
             case no_topn:
@@ -12956,7 +12981,7 @@ void optimizeGroupOrder(HqlExprArray & optimized, IHqlExpression * dataset, HqlE
 
 IHqlExpression * createOrderFromCompareArray(HqlExprArray & exprs, IHqlExpression * dataset, IHqlExpression * left, IHqlExpression * right)
 {
-    OwnedHqlExpr equalExpr = createValue(no_sortlist, makeSortListType(NULL), exprs);
+    OwnedHqlExpr equalExpr = createSortList(exprs);
     OwnedHqlExpr lhs = replaceSelector(equalExpr, dataset, left);
     OwnedHqlExpr rhs = replaceSelector(equalExpr, dataset, right);
     return createValue(no_order, LINK(signedType), lhs.getClear(), rhs.getClear());
@@ -13122,7 +13147,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDedup(BuildCtx & ctx, IHqlExpr
         {
             if (info.equalities.ordinality() && !instance->isGrouped)
             {
-                OwnedHqlExpr order = createValue(no_sortlist, makeSortListType(NULL), info.equalities);
+                OwnedHqlExpr order = createValueSafe(no_sortlist, makeSortListType(NULL), info.equalities);
                 buildCompareMember(instance->nestedctx, "ComparePrimary", order, DatasetReference(dataset));
                 if (!targetThor())
                     equalities = &noEqualities;
@@ -14893,7 +14918,7 @@ IHqlExpression * HqlCppTranslator::createOrderFromSortList(const DatasetReferenc
         }
     }
 
-    return createValue(no_order, LINK(signedType), createValue(no_sortlist, makeSortListType(NULL), leftList), createValue(no_sortlist, makeSortListType(NULL), rightList));
+    return createValue(no_order, LINK(signedType), createSortList(leftList), createSortList(rightList));
 }
 
 
@@ -14910,6 +14935,97 @@ void HqlCppTranslator::buildReturnOrder(BuildCtx & ctx, IHqlExpression *sortList
     doBuildReturnCompare(ctx, order, no_order, false);
 }
 
+void HqlCppTranslator::doBuildFuncIsSameGroup(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * sortlist)
+{
+    BuildCtx funcctx(ctx);
+    funcctx.addQuotedCompound("virtual bool isSameGroup(const void * _left, const void * _right)");
+    if (sortlist->getOperator() == no_activetable)
+        buildReturn(funcctx, queryBoolExpr(false));
+    else
+    {
+        funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
+        funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
+
+        OwnedHqlExpr selSeq = createSelectorSequence();
+        OwnedHqlExpr leftSelect = createSelector(no_left, dataset, selSeq);
+        OwnedHqlExpr rightSelect = createSelector(no_right, dataset, selSeq);
+        HqlExprArray args;
+        HqlExprArray leftValues, rightValues;
+        HqlExprArray compares;
+        unwindChildren(compares, sortlist);
+
+        //Optimize the grouping conditions by ordering them by the fields in the record (so the
+        //doBuildReturnCompare() can combine as many as possible),  and remove duplicates
+        if (options.optimizeGrouping && (compares.ordinality() > 1))
+        {
+            HqlExprArray equalities;
+            optimizeGroupOrder(equalities, dataset, compares);
+            ForEachItemIn(i, equalities)
+            {
+                IHqlExpression * test = &equalities.item(i);
+                leftValues.append(*replaceSelector(test, dataset, leftSelect));
+                rightValues.append(*replaceSelector(test, dataset, rightSelect));
+            }
+        }
+
+        ForEachItemIn(idx, compares)
+        {
+            IHqlExpression * test = &compares.item(idx);
+            if (containsSelector(test, leftSelect) || containsSelector(test, rightSelect))
+                args.append(*LINK(test));
+            else
+            {
+                OwnedHqlExpr lhs = replaceSelector(test, dataset, leftSelect);
+                OwnedHqlExpr rhs = replaceSelector(test, dataset, rightSelect);
+                if (lhs != rhs)
+                {
+                    leftValues.append(*lhs.getClear());
+                    rightValues.append(*rhs.getClear());
+                }
+            }
+        }
+
+        OwnedHqlExpr result;
+        OwnedHqlExpr orderResult;
+        //Use the optimized equality code for more than one element - which often combines the comparisons.
+        if (leftValues.ordinality() != 0)
+        {
+            if (leftValues.ordinality() == 1)
+                args.append(*createValue(no_eq, makeBoolType(), LINK(&leftValues.item(0)), LINK(&rightValues.item(0))));
+            else
+                orderResult.setown(createValue(no_order, LINK(signedType), createSortList(leftValues), createSortList(rightValues)));
+        }
+
+        if (args.ordinality() == 1)
+            result.set(&args.item(0));
+        else if (args.ordinality() != 0)
+            result.setown(createValue(no_and, makeBoolType(), args));
+
+        bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
+        bindTableCursor(funcctx, dataset, "right", no_right, selSeq);
+        IHqlExpression * trueExpr = queryBoolExpr(true);
+        if (result)
+        {
+            if (orderResult)
+            {
+                buildFilteredReturn(funcctx, result, trueExpr);
+                doBuildReturnCompare(funcctx, orderResult, no_eq, true);
+            }
+            else
+            {
+                buildReturn(funcctx, result);
+            }
+        }
+        else
+        {
+            if (orderResult)
+                doBuildReturnCompare(funcctx, orderResult, no_eq, true);
+            else
+                buildReturn(funcctx, trueExpr);
+        }
+    }
+}
+
 ABoundActivity * HqlCppTranslator::doBuildActivityGroup(BuildCtx & ctx, IHqlExpression * expr)
 {
     IHqlExpression * dataset = expr->queryChild(0);
@@ -14945,93 +15061,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGroup(BuildCtx & ctx, IHqlExpr
         buildInstancePrefix(instance);
 
         //virtual bool isSameGroup(const void *left, const void *right);
-        BuildCtx funcctx(instance->startctx);
-        funcctx.addQuotedCompound("virtual bool isSameGroup(const void * _left, const void * _right)");
-        if (sortlist->getOperator() == no_activetable)
-            buildReturn(funcctx, queryBoolExpr(false));
-        else
-        {
-            funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
-            funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
-
-            OwnedHqlExpr selSeq = createSelectorSequence();
-            OwnedHqlExpr leftSelect = createSelector(no_left, dataset, selSeq);
-            OwnedHqlExpr rightSelect = createSelector(no_right, dataset, selSeq);
-            HqlExprArray args;
-            HqlExprArray leftValues, rightValues;
-            HqlExprArray compares;
-            unwindChildren(compares, sortlist);
-
-            //Optimize the grouping conditions by ordering them by the fields in the record (so the
-            //doBuildReturnCompare() can combine as many as possible),  and remove duplicates
-            if (options.optimizeGrouping && (compares.ordinality() > 1))
-            {
-                HqlExprArray equalities;
-                optimizeGroupOrder(equalities, dataset, compares);
-                ForEachItemIn(i, equalities)
-                {
-                    IHqlExpression * test = &equalities.item(i);
-                    leftValues.append(*replaceSelector(test, dataset, leftSelect));
-                    rightValues.append(*replaceSelector(test, dataset, rightSelect));
-                }
-            }
-
-            ForEachItemIn(idx, compares)
-            {
-                IHqlExpression * test = &compares.item(idx);
-                if (containsSelector(test, leftSelect) || containsSelector(test, rightSelect))
-                    args.append(*LINK(test));
-                else
-                {
-                    OwnedHqlExpr lhs = replaceSelector(test, dataset, leftSelect);
-                    OwnedHqlExpr rhs = replaceSelector(test, dataset, rightSelect);
-                    if (lhs != rhs)
-                    {
-                        leftValues.append(*lhs.getClear());
-                        rightValues.append(*rhs.getClear());
-                    }
-                }
-            }
-
-            OwnedHqlExpr result;
-            OwnedHqlExpr orderResult;
-            //Use the optimized equality code for more than one element - which often combines the comparisons.
-            if (leftValues.ordinality() != 0)
-            {
-                if (leftValues.ordinality() == 1)
-                    args.append(*createValue(no_eq, makeBoolType(), LINK(&leftValues.item(0)), LINK(&rightValues.item(0))));
-                else
-                    orderResult.setown(createValue(no_order, LINK(signedType), createValue(no_sortlist, makeSortListType(NULL), leftValues), createValue(no_sortlist, makeSortListType(NULL), rightValues)));
-            }
-
-            if (args.ordinality() == 1)
-                result.set(&args.item(0));
-            else if (args.ordinality() != 0)
-                result.setown(createValue(no_and, makeBoolType(), args));
-
-            bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
-            bindTableCursor(funcctx, dataset, "right", no_right, selSeq);
-            IHqlExpression * trueExpr = queryBoolExpr(true);
-            if (result)
-            {
-                if (orderResult)
-                {
-                    buildFilteredReturn(funcctx, result, trueExpr);
-                    doBuildReturnCompare(funcctx, orderResult, no_eq, true);
-                }
-                else
-                {
-                    buildReturn(funcctx, result);
-                }
-            }
-            else
-            {
-                if (orderResult)
-                    doBuildReturnCompare(funcctx, orderResult, no_eq, true);
-                else
-                    buildReturn(funcctx, trueExpr);
-            }
-        }
+        doBuildFuncIsSameGroup(instance->startctx, dataset, sortlist);
 
         buildInstanceSuffix(instance);
 
@@ -15292,6 +15322,10 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
             helper = "Sort";
             break;
         }
+    case no_shuffle:
+        actKind = TAKshuffle;
+        helper = "Shuffle";
+        break;
     default:
         {
             cosort = expr->queryChild(2);
@@ -15346,6 +15380,9 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
 
     buildSkewThresholdMembers(instance->classctx, expr);
 
+    if (expr->getOperator() == no_shuffle)
+        doBuildFuncIsSameGroup(instance->startctx, dataset, expr->queryChild(2));
+
     if (limit)
     {
         OwnedHqlExpr newLimit = ensurePositiveOrZeroInt64(limit);
@@ -15371,7 +15408,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
                 maxValues.replace(*LINK(cur.queryChild(0)), i);
             }
         }
-        OwnedHqlExpr order = createValue(no_order, LINK(signedType), createValue(no_sortlist, makeSortListType(NULL), sortValues), createValue(no_sortlist, makeSortListType(NULL), maxValues));
+        OwnedHqlExpr order = createValue(no_order, LINK(signedType), createSortList(sortValues), createSortList(maxValues));
 
         BuildCtx funcctx(instance->startctx);
         funcctx.addQuotedCompound("virtual int compareBest(const void * _self)");

ecl/hqlcpp/hqliproj.cpp

@@ -188,9 +188,14 @@ static unsigned getActivityCost(IHqlExpression * expr, ClusterType targetCluster
             switch (expr->getOperator())
             {
             case no_sort:
+                //MORE: What about checking for grouped!
                 if (!expr->hasProperty(localAtom))
                     return CostNetworkCopy;
                 return CostManyCopy;
+            case no_shuffle:
+                if (!expr->hasProperty(localAtom) && !isGrouped(expr))
+                    return CostNetworkCopy;
+                break;
             case no_group:
                 if (!expr->hasProperty(localAtom))
                     return CostNetworkGroup;

ecl/hqlcpp/hqllib.cpp

@@ -262,7 +262,7 @@ protected:
                     throwUnexpected();
                     HqlExprArray parms;
                     unwindChildren(parms, oldSymbol, 1);
-                    IHqlExpression * formals = createValue(no_sortlist, makeSortListType(NULL), parms);
+                    IHqlExpression * formals = createSortList(parms);
                     newValue.setown(createFunctionDefinition(name, newValue.getClear(), formals, NULL, NULL));
                 }
 

ecl/hqlcpp/hqlresource.cpp

@@ -141,6 +141,7 @@ void getResources(IHqlExpression * expr, CResources & resources, const CResource
         if (!isGrouped && !isLocal)
         {
             resources.set(RESslavememory, MEM_Const_Minimal+DEDUP_SMART_BUFFER_SIZE);
+            //MORE: Is this still correct?
             resources.setManyToMasterSockets(1);
         }
         break;
@@ -149,6 +150,12 @@ void getResources(IHqlExpression * expr, CResources & resources, const CResource
         resources.setLightweight();
         setHashResources(expr, resources, options);
         break;
+    case no_shuffle:
+        if (expr->hasProperty(manyAtom))
+            resources.setHeavyweight();
+        else
+            resources.setLightweight();
+        break;
     case no_sort:
         if (isGrouped)
         {

ecl/hqlcpp/hqlsource.cpp

@@ -2049,10 +2049,10 @@ void SourceBuilder::buildGroupAggregateCompareHelper(ParentExtract * extractBuil
         }
     }
 
-    OwnedHqlExpr leftList = createValue(no_sortlist, makeSortListType(NULL), optimizedLeft);
+    OwnedHqlExpr leftList = createSortList(optimizedLeft);
     DatasetReference datasetRight(aggregate, no_activetable, NULL);
     OwnedHqlExpr selSeq = createDummySelectorSequence();
-    OwnedHqlExpr rightList = createValue(no_sortlist, makeSortListType(NULL), optimizedRight);
+    OwnedHqlExpr rightList = createSortList(optimizedRight);
     OwnedHqlExpr rightSelect = datasetRight.getSelector(no_right, selSeq);
     OwnedHqlExpr rightResolved = datasetRight.mapCompound(rightList, rightSelect);
 

ecl/hqlcpp/hqlstep.cpp

@@ -931,7 +931,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityNWayMergeJoin(BuildCtx & ctx,
     {
         HqlExprArray skewArgs;
         unwindChildren(skewArgs, skew);
-        OwnedHqlExpr skewOrder = createValue(no_sortlist, makeSortListType(NULL), skewArgs);
+        OwnedHqlExpr skewOrder = createSortList(skewArgs);
         DatasetReference datasetRef(dataset);
         buildCompareEqMember(instance->classctx, "PartitionCompareEq", skewOrder, leftRef);
     }

ecl/hqlcpp/hqlttcpp.cpp

@@ -1718,12 +1718,12 @@ static IHqlExpression * simplifySortlistComplexity(IHqlExpression * sortlist)
             same = false;
     }
     if (!same)
-        return createValue(no_sortlist, makeSortListType(NULL), cpts);
+        return createSortList(cpts);
 
     return NULL;
 }
 
-static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortIndexPayload, bool alwaysLocal)
+static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortIndexPayload, bool alwaysLocal, bool allowImplicitShuffle)
 {
     LinkedHqlExpr dataset = expr->queryChild(0);
     IHqlExpression * normalizedDs = dataset->queryNormalizedSelector();
@@ -1736,7 +1736,7 @@ static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortInde
     HqlExprArray sorts;
     gatherIndexBuildSortOrder(sorts, expr, sortIndexPayload);
 
-    OwnedHqlExpr sortOrder = createValue(no_sortlist, makeSortListType(NULL), sorts);
+    OwnedHqlExpr sortOrder = createSortList(sorts);
     OwnedHqlExpr newsort = simplifySortlistComplexity(sortOrder);
     if (!newsort)
         newsort.set(sortOrder);
@@ -1844,7 +1844,7 @@ static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortInde
             }
         }
 
-        OwnedHqlExpr sorted = ensureSorted(dataset, newsort, expr->hasProperty(localAtom), true, alwaysLocal);
+        OwnedHqlExpr sorted = ensureSorted(dataset, newsort, expr->hasProperty(localAtom), true, alwaysLocal, allowImplicitShuffle);
         if (sorted == dataset)
             return NULL;
 
@@ -1914,6 +1914,9 @@ IHqlExpression * ThorHqlTransformer::createTransformed(IHqlExpression * expr)
     case no_assertsorted:
         normalized = normalizeSort(transformed);
         break;
+    case no_shuffle:
+        normalized = normalizeShuffle(transformed);
+        break;
     case no_cogroup:
         normalized = normalizeCoGroup(transformed);
         break;
@@ -2094,7 +2097,7 @@ IHqlExpression * ThorHqlTransformer::normalizeDedup(IHqlExpression * expr)
             if (hasLocal && translator.targetThor())
             {
                 HqlExprArray dedupArgs;
-                dedupArgs.append(*ensureSortedForGroup(dataset, groupOrder, true, false));
+                dedupArgs.append(*ensureSortedForGroup(dataset, groupOrder, true, false, options.implicitGroupShuffle));
                 unwindChildren(dedupArgs, expr, 1);
                 removeProperty(dedupArgs, allAtom);
                 return expr->clone(dedupArgs);
@@ -2271,7 +2274,7 @@ IHqlExpression * ThorHqlTransformer::normalizeGroup(IHqlExpression * expr)
 
     //First check to see if the dataset is already sorted by the group criteria, or more.
     //The the data could be globally sorted, but not distributed, and this is likely to be more efficient than redistributing...
-    OwnedHqlExpr sorted = ensureSortedForGroup(dataset, sortlist, hasLocal, !translator.targetThor());
+    OwnedHqlExpr sorted = ensureSortedForGroup(dataset, sortlist, hasLocal, !translator.targetThor(), options.implicitGroupShuffle);
     if (sorted == dataset)
         return removeProperty(expr, allAtom);
     sorted.setown(cloneInheritedAnnotations(expr, sorted));
@@ -2409,7 +2412,7 @@ IHqlExpression * ThorHqlTransformer::normalizeCoGroup(IHqlExpression * expr)
         {
             IHqlExpression & cur = inputs.item(i);
             OwnedHqlExpr mappedOrder = replaceSelector(bestSortOrder, queryActiveTableSelector(), &cur);
-            sortedInputs.append(*ensureSorted(&cur, mappedOrder, true, true, alwaysLocal));
+            sortedInputs.append(*ensureSorted(&cur, mappedOrder, true, true, alwaysLocal, options.implicitShuffle));
         }
         HqlExprArray sortedArgs;
         unwindChildren(sortedArgs, bestSortOrder);
@@ -2434,7 +2437,7 @@ IHqlExpression * ThorHqlTransformer::normalizeCoGroup(IHqlExpression * expr)
     return expr->cloneAllAnnotations(grouped);
 }
 
-static IHqlExpression * getNonThorSortedJoinInput(IHqlExpression * joinExpr, IHqlExpression * dataset, HqlExprArray & sorts)
+static IHqlExpression * getNonThorSortedJoinInput(IHqlExpression * joinExpr, IHqlExpression * dataset, HqlExprArray & sorts, bool implicitShuffle)
 {
     if (!sorts.length())
         return LINK(dataset);
@@ -2451,7 +2454,7 @@ static IHqlExpression * getNonThorSortedJoinInput(IHqlExpression * joinExpr, IHq
     groupOrder.setown(replaceSelector(groupOrder, queryActiveTableSelector(), expr->queryNormalizedSelector()));
 
     //not used for thor, so sort can be local
-    OwnedHqlExpr table = ensureSorted(expr, groupOrder, false, true, true);
+    OwnedHqlExpr table = ensureSorted(expr, groupOrder, false, true, true, implicitShuffle);
     if (table != expr)
         table.setown(cloneInheritedAnnotations(joinExpr, table));
 
@@ -2470,6 +2473,28 @@ static bool sameOrGrouped(IHqlExpression * newLeft, IHqlExpression * oldLeft)
     return (newLeft->queryBody() == oldLeft->queryBody());
 }
 
+bool canReorderMatchExistingLocalSort(HqlExprArray & newElements1, HqlExprArray & newElements2, IHqlExpression * ds1, Shared<IHqlExpression> & ds2, const HqlExprArray & elements1, HqlExprArray & elements2, bool canShuffle, bool isLocal, bool alwaysLocal)
+{
+    newElements1.kill();
+    newElements2.kill();
+    if (reorderMatchExistingLocalSort(newElements1, newElements2, ds1, elements1, elements2))
+    {
+        if (isAlreadySorted(ds2, newElements2, isLocal||alwaysLocal, true))
+            return true;
+
+        if (canShuffle && isWorthShuffling(ds2, newElements2, isLocal||alwaysLocal, true))
+        {
+            OwnedHqlExpr shuffled = getShuffleSort(ds2, newElements2, isLocal, true, alwaysLocal);
+            if (shuffled)
+            {
+                ds2.swap(shuffled);
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
 
 IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression * expr)
 {
@@ -2607,14 +2632,25 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
         //Check for a local join where we can reorder the condition so both sides match the existing sort orders.
         //could special case self-join to do less work, but probably not worth the effort.
         HqlExprArray sortedLeft, sortedRight;
-        if (!isLimitedSubstringJoin && reorderMatchExistingLocalSort(sortedLeft, sortedRight, leftDs, leftSorts, rightSorts))
+        if (!isLimitedSubstringJoin)
         {
-            if (isAlreadySorted(rightDs, sortedRight, true, true))
+            //Since the distribution and order of global joins is not defined this could probably be used for non-local as well.
+            LinkedHqlExpr newLeftDs = leftDs;
+            LinkedHqlExpr newRightDs = rightDs;
+            bool canShuffle = options.shuffleLocalJoinConditions;
+            bool reordered = canReorderMatchExistingLocalSort(sortedLeft, sortedRight, newLeftDs, newRightDs,
+                                                              leftSorts, rightSorts, canShuffle, isLocal, alwaysLocal);
+            //If allowed to shuffle then try the otherway around
+            if (!reordered && canShuffle)
+                reordered = canReorderMatchExistingLocalSort(sortedRight, sortedLeft, newRightDs, newLeftDs,
+                                                             rightSorts, leftSorts, canShuffle, isLocal, alwaysLocal);
+
+            if (reordered)
             {
                 //Recreate the join condition in the correct order to match the existing sorts...
                 HqlExprAttr newcond;
-                OwnedHqlExpr leftSelector = createSelector(no_left, leftDs, seq);
-                OwnedHqlExpr rightSelector = createSelector(no_right, rightDs, seq);
+                OwnedHqlExpr leftSelector = createSelector(no_left, newLeftDs, seq);
+                OwnedHqlExpr rightSelector = createSelector(no_right, newRightDs, seq);
                 ForEachItemIn(i, sortedLeft)
                 {
                     OwnedHqlExpr lc = replaceSelector(&sortedLeft.item(i), queryActiveTableSelector(), leftSelector);
@@ -2623,8 +2659,10 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
                 }
                 extendConditionOwn(newcond, no_and, fuzzyMatch.getClear());
                 HqlExprArray args;
-                unwindChildren(args, expr);
-                args.replace(*newcond.getClear(), 2);
+                args.append(*newLeftDs.getClear());
+                args.append(*newRightDs.getClear());
+                args.append(*newcond.getClear());
+                unwindChildren(args, expr, 3);
                 args.append(*createAttribute(_lightweight_Atom));
                 return expr->clone(args);
             }
@@ -2668,8 +2706,8 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
             return expr->clone(args);
         }
 
-        OwnedHqlExpr newLeft = getNonThorSortedJoinInput(expr, leftDs, leftSorts);
-        OwnedHqlExpr newRight = getNonThorSortedJoinInput(expr, rightDs, rightSorts);
+        OwnedHqlExpr newLeft = getNonThorSortedJoinInput(expr, leftDs, leftSorts, options.implicitShuffle);
+        OwnedHqlExpr newRight = getNonThorSortedJoinInput(expr, rightDs, rightSorts, options.implicitShuffle);
         try
         {
             if ((leftDs != newLeft) || (rightDs != newRight))
@@ -2726,7 +2764,7 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
         }
     }
 
-    if (isThorCluster(targetClusterType) && isLocal)
+    if (isThorCluster(targetClusterType) && isLocal && options.implicitJoinShuffle)
     {
         IHqlExpression * noSortAttr = expr->queryProperty(noSortAtom);
         OwnedHqlExpr newLeft;
@@ -2850,16 +2888,43 @@ IHqlExpression * ThorHqlTransformer::normalizeSort(IHqlExpression * expr)
         return LINK(dataset);
     if (op == no_sorted)
         return normalizeSortSteppedIndex(expr, sortedAtom);
-    if (op != no_assertsorted)
+
+    //NOTE: We can't convert a global sort to a shuffle because that will change the distribution
+    if (options.implicitShuffle && (isLocal || alwaysLocal) && (op != no_assertsorted))
     {
         OwnedHqlExpr shuffled = getShuffleSort(dataset, sortlist, isLocal, false, alwaysLocal);
         if (shuffled)
-            return shuffled.getClear();
+            return dataset->cloneAllAnnotations(shuffled);
     }
     return NULL;
 }
 
 
+IHqlExpression * ThorHqlTransformer::normalizeShuffle(IHqlExpression * expr)
+{
+    IHqlExpression * dataset = expr->queryChild(0);
+    IHqlExpression * sortlist = expr->queryChild(1);
+    IHqlExpression * grouping = expr->queryChild(2);
+    OwnedHqlExpr newsort = simplifySortlistComplexity(sortlist);
+    OwnedHqlExpr newgrouping = simplifySortlistComplexity(grouping);
+    if (newsort || newgrouping)
+    {
+        HqlExprArray args;
+        unwindChildren(args, expr);
+        if (newsort)
+            args.replace(*newsort.getClear(), 1);
+        if (newgrouping)
+            args.replace(*newgrouping.getClear(), 2);
+        return expr->clone(args);
+    }
+
+    if (translator.targetThor() && !expr->hasProperty(localAtom))
+        return convertShuffleToGroupedSort(expr);
+
+    return NULL;
+}
+
+
 IHqlExpression * ThorHqlTransformer::normalizeSortSteppedIndex(IHqlExpression * expr, _ATOM attrName)
 {
     node_operator op = expr->getOperator();
@@ -3236,7 +3301,7 @@ IHqlExpression * ThorHqlTransformer::normalizeTableToAggregate(IHqlExpression *
             }
             newGroupElement.append(*LINK(curGroup));
         }
-        newGroupBy = createValue(no_sortlist, makeSortListType(NULL), newGroupElement);
+        newGroupBy = createSortList(newGroupElement);
     }
 
     IHqlExpression * aggregateRecord = extraSelectNeeded ? translator.createRecordInheritMaxLength(aggregateFields, record) : LINK(record);
@@ -3309,7 +3374,7 @@ IHqlExpression * ThorHqlTransformer::normalizeTableGrouping(IHqlExpression * exp
                     ds.setown(createDataset(no_group, ds.getClear(), NULL));
                     ds.setown(cloneInheritedAnnotations(expr, ds));
                 }
-                OwnedHqlExpr sorted = ensureSortedForGroup(ds, newsort, expr->hasProperty(localAtom), !translator.targetThor());
+                OwnedHqlExpr sorted = ensureSortedForGroup(ds, newsort, expr->hasProperty(localAtom), !translator.targetThor(), options.implicitGroupShuffle);
 
                 //For thor a global grouped aggregate would transfer elements between nodes so it is still likely to
                 //be more efficient to do a hash aggregate.  Even better would be to check the distribution
@@ -4302,6 +4367,7 @@ static IHqlExpression * queryNormalizedAggregateParameter(IHqlExpression * expr)
                 return expr;
             break;
         case no_sort:
+        case no_shuffle:
         case no_distribute:
             break;
         default:
@@ -9738,6 +9804,7 @@ HqlTreeNormalizer::HqlTreeNormalizer(HqlCppTranslator & _translator) : NewHqlTra
     options.outputRowsAsDatasets = translator.targetRoxie();
     options.constantFoldNormalize = translatorOptions.constantFoldNormalize;
     options.allowActivityForKeyedJoin = translatorOptions.allowActivityForKeyedJoin;
+    options.implicitShuffle = translatorOptions.implicitBuildIndexShuffle;
     errors = translator.queryErrors();
     nextSequenceValue = 1;
 }
@@ -11393,7 +11460,7 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
             OwnedHqlExpr transformed = Parent::createTransformed(expr);
             loop
             {
-                IHqlExpression * ret = normalizeIndexBuild(transformed, options.sortIndexPayload, !translator.targetThor());
+                IHqlExpression * ret = normalizeIndexBuild(transformed, options.sortIndexPayload, !translator.targetThor(), options.implicitShuffle);
                 if (!ret)
                     return LINK(transformed);
                 transformed.setown(ret);

ecl/hqlcpp/hqlttcpp.ipp

@@ -202,6 +202,7 @@ protected:
     IHqlExpression * normalizeRollup(IHqlExpression * expr);
     IHqlExpression * normalizeScalarAggregate(IHqlExpression * expr);
     IHqlExpression * normalizeSelect(IHqlExpression * expr);
+    IHqlExpression * normalizeShuffle(IHqlExpression * expr);
     IHqlExpression * normalizeSort(IHqlExpression * expr);
     IHqlExpression * normalizeSortSteppedIndex(IHqlExpression * expr, _ATOM attrName);
     IHqlExpression * normalizeTempTable(IHqlExpression * expr);
@@ -1135,6 +1136,7 @@ protected:
         bool outputRowsAsDatasets;
         bool constantFoldNormalize;
         bool allowActivityForKeyedJoin;
+        bool implicitShuffle;
     } options;
     unsigned nextSequenceValue;
     bool seenForceLocal;

ecl/regress/shuffle.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+s2 := shuffle(s1, {d},{a,b},local); // sort and grouping are round the wrong way!
+s3 := sorted(s2, {a,b,d}, assert);
+output(s3);

ecl/regress/shuffle2.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+s2 := shuffle(nofold(s1), {a,b}, {d});
+s3 := sorted(nofold(s2), {a,b,d}, assert);
+output(s3);

ecl/regress/shuffle3.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+s2 := shuffle(s1, {a,b}, {d}, local);
+s3 := sorted(s2, {a,b,d}, assert);
+output(s3);

rtl/include/eclhelper.hpp

@@ -803,7 +803,7 @@ enum ThorActivityKind
     TAKcaseaction,
     TAKwhen_dataset,
     TAKwhen_action,
-        TAKunused2,
+    TAKshuffle,
     TAKindexgroupexists,
     TAKindexgroupcount,
     TAKhashdistributemerge,
@@ -957,6 +957,7 @@ enum ActivityInterfaceEnum
     TAIpipethrougharg_2,
     TAIpipewritearg_2,
     TAItemptablearg_2,
+    TAIshuffleextra_1,
 
 //Should remain as last of all meaningful tags, but before aliases
     TAImax,
@@ -1514,6 +1515,16 @@ struct IHThorTopNArg : public IHThorSortArg, public IHThorTopNExtra
     COMMON_NEWTHOR_FUNCTIONS
 };
 
+struct IHThorShuffleExtra : public IInterface
+{
+    virtual bool isSameGroup(const void * _left, const void * _right) = 0;
+};
+
+struct IHThorShuffleArg : public IHThorSortArg, public IHThorShuffleExtra
+{
+    COMMON_NEWTHOR_FUNCTIONS
+};
+
 // JoinFlags
 enum { 
     JFleftouter=1, JFrightouter=2, JFexclude=4,

rtl/include/eclhelper_base.hpp

@@ -1537,6 +1537,36 @@ class CThorTopNArg : public CThorArg, implements IHThorTopNArg
     virtual int compareBest(const void * _left) { return +1; }
 };
 
+class CThorShuffleArg : public CThorArg, implements IHThorShuffleArg
+{
+    virtual void Link() const { RtlCInterface::Link(); }
+    virtual bool Release() const { return RtlCInterface::Release(); }
+    virtual void onCreate(ICodeContext * _ctx, IHThorArg *, MemoryBuffer * in) { ctx = _ctx; }
+
+    virtual IInterface * selectInterface(ActivityInterfaceEnum which)
+    {
+        switch (which)
+        {
+        case TAIarg:
+        case TAIsortarg_1:
+            return static_cast<IHThorSortArg *>(this);
+        case TAIshuffleextra_1:
+            return static_cast<IHThorShuffleExtra *>(this);
+        }
+        return NULL;
+    }
+
+    virtual double getSkew()                            { return 0; }           // 0=default
+    virtual bool hasManyRecords() { return false; }
+    virtual double getTargetSkew()                      { return 0; }
+    virtual ISortKeySerializer * querySerialize() { return NULL; }
+    virtual unsigned __int64 getThreshold() { return 0; }
+    virtual IOutputMetaData * querySortedRecordSize() { return NULL; }
+    virtual const char * getSortedFilename() { return NULL; }
+    virtual ICompare * queryCompareLeftRight() { return NULL; }
+    virtual ICompare * queryCompareSerializedRow() { return NULL; }
+};
+
 class CThorKeyedJoinArg : public CThorArg, implements IHThorKeyedJoinArg
 {
     virtual void Link() const { RtlCInterface::Link(); }

system/jlib/jscm.hpp

@@ -73,6 +73,7 @@ public:
     }
     inline void set(const Shared<CLASS> &other) { this->set(other.get()); }
     inline void setown(CLASS * _ptr)            { CLASS * temp = ptr; ptr = _ptr; ::Release(temp); }
+    inline void swap(Shared<CLASS> & other)     { CLASS * temp = ptr; ptr = other.ptr; other.ptr = temp; }
     
 protected:
     inline Shared(CLASS * _ptr)                  { ptr = _ptr; } // deliberately protected