Merge pull request #1759 from ghalliday/sortwithin

Implement a SHUFFLE activity and keyword

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>
Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Renato Golin <rengolin@hpccsystems.com>

common/commonext/commonext.cpp

@@ -200,6 +200,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     kindArray[TAKexternalsink] = "externalsink";
     kindArray[TAKexternalprocess] = "externalprocess";
     kindArray[TAKwhen_action] = "when_action";
+    kindArray[TAKshuffle] = "shuffle";
 
 //Non standard
     kindArray[TAKcountdisk] = "countdisk";

common/thorhelper/thorcommon.cpp

@@ -786,6 +786,7 @@ extern const char * getActivityText(ThorActivityKind kind)
     case TAKexternalsink:           return "User Output";
     case TAKexternalprocess:        return "User Proceess";
     case TAKwhen_action:            return "When";
+    case TAKshuffle:                return "Shuffle";
     }
     throwUnexpected();
 }

ecl/hql/hqlattr.cpp

@@ -291,6 +291,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_fetch:
     case no_join:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_dedup:
     case no_enth:
@@ -608,7 +609,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_dataset_from_transform:
 
     case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
-    case no_unused13: case no_unused14: case no_unused15: case no_unused17: case no_unused18: case no_unused19:
+    case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
@@ -1573,6 +1574,7 @@ bool isLocalActivity(IHqlExpression * expr)
     case no_cogroup:
     case no_cosort:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_topn:
     case no_iterate:
@@ -1760,6 +1762,7 @@ bool localChangesActivityAction(IHqlExpression * expr)
     case no_cogroup:
     case no_cosort:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_topn:
     case no_iterate:
@@ -2386,6 +2389,7 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
     case no_assertgrouped:
     case no_assertdistributed:
     case no_sort:
+    case no_shuffle:
     case no_nohoist:
     case no_section:
     case no_sectioninput:

ecl/hql/hqldsparam.cpp

@@ -264,7 +264,7 @@ IHqlExpression* HqlGram::bindFieldMap(IHqlExpression* expr, IHqlExpression* map)
                     newmaps.append(*createComma(createAttribute(mappedName), LINK(to)));
             }
             map->Release();
-            IHqlExpression * newmap = createValue(no_sortlist, makeSortListType(NULL), newmaps);
+            IHqlExpression * newmap = createSortList(newmaps);
 
             expr = clearFieldMap(expr);
             expr = createFieldMap(expr,newmap);

ecl/hql/hqlerrors.hpp

@@ -415,7 +415,8 @@
 #define HQLERR_CannotSubmitFunction 2385
 #define HQLERR_CannotSubmitModule   2386
 #define ERR_COUNTER_NOT_COUNT       2387
-#define HQLERR_CannotSubmitMacroX   2385
+#define HQLERR_CannotSubmitMacroX   2388
+#define HQLERR_CannotBeGrouped      2389
 
 #define ERR_ASSERTION_FAILS         100000
 

ecl/hql/hqlexpr.cpp

@@ -1000,6 +1000,7 @@ const char *getOpString(node_operator op)
     case no_enth: return "ENTH";
     case no_sample: return "SAMPLE";
     case no_sort: return "SORT";
+    case no_shuffle: return "SHUFFLE";
     case no_sorted: return "SORTED";
     case no_choosen: return "CHOOSEN";
     case no_choosesets: return "CHOOSESETS";
@@ -1426,7 +1427,7 @@ const char *getOpString(node_operator op)
     case no_dataset_alias: return "TABLE";
 
     case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
-    case no_unused13: case no_unused14: case no_unused15: case no_unused17: case no_unused18: case no_unused19:
+    case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
@@ -1782,6 +1783,7 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_cosort:
     case no_keyed:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_stepped:
     case no_transformebcdic:
@@ -2046,6 +2048,7 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_sample:
     case no_selectnth:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_stepped:
     case no_assertsorted:
@@ -2332,6 +2335,7 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_section:
     case no_sample:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_stepped:
     case no_assertsorted:
@@ -5377,6 +5381,7 @@ void CHqlDataset::cacheParent()
     case no_keyed:
     // change ordering
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_stepped:
     case no_cosort:
@@ -10587,6 +10592,14 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             type.setown(getTypeDistribute(datasetType, newDistribution, NULL));
             break;
         }
+    case no_shuffle:
+        {
+            bool isLocal = queryProperty(localAtom, parms) != NULL;
+            OwnedHqlExpr normalizedSortOrder = replaceSelector(&parms.item(1), dataset, cachedActiveTableExpr);
+            OwnedHqlExpr mappedGrouping = replaceSelector(&parms.item(2), dataset, cachedActiveTableExpr);
+            type.setown(getTypeShuffle(datasetType, mappedGrouping, normalizedSortOrder, isLocal));
+            break;
+        }
     case no_cosort:
     case no_sort:
     case no_sorted:
@@ -10713,7 +10726,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
                 else
                     unwindRecordAsSelects(sortExprs, record, cachedActiveTableExpr);
 
-                OwnedHqlExpr sortOrder = createValue(no_sortlist, makeSortListType(NULL), sortExprs);
+                OwnedHqlExpr sortOrder = createSortList(sortExprs);
                 if (queryProperty(noRootAtom, parms))
                     type.setown(getTypeLocalSort(type, sortOrder));
                 else
@@ -10927,7 +10940,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             IHqlExpression * order= queryProperty(sortedAtom, parms);       // already uses no_activetable to refer to dataset
             assertex(order);
             unwindChildren(components, order);
-            OwnedHqlExpr sortlist = createValue(no_sortlist, makeSortListType(NULL), components);
+            OwnedHqlExpr sortlist = createSortList(components);
             if (queryProperty(localAtom, parms))
                 type.setown(getTypeLocalSort(datasetType, sortlist));
             else
@@ -10957,7 +10970,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             OwnedHqlExpr normalizedOrder = replaceSelector(order, selector, cachedActiveTableExpr);
             HqlExprArray components;
             unwindChildren(components, normalizedOrder);
-            OwnedHqlExpr sortlist = createValue(no_sortlist, makeSortListType(NULL), components);
+            OwnedHqlExpr sortlist = createSortList(components);
             //These are all currently implemented as local activities, need to change following if no longer true
             type.setown(getTypeLocalSort(datasetType, sortlist));
             break;
@@ -14978,6 +14991,11 @@ extern HQL_API IHqlExpression * createAbstractRecord(IHqlExpression * record)
     return createRecord(args);
 }
 
+extern HQL_API IHqlExpression * createSortList(HqlExprArray & elements)
+{
+     return createValue(no_sortlist, makeSortListType(NULL), elements);
+}
+
 //==============================================================================================================
 
 

ecl/hql/hqlexpr.hpp

@@ -343,7 +343,7 @@ enum _node_operator {
         no_crc,
         no_return_stmt,
         no_update,    
-    no_unused17,
+        no_shuffle,
     no_unused18,
         no_alias,
     no_unused19,
@@ -1580,6 +1580,7 @@ extern HQL_API IHqlScope * closeScope(IHqlScope * scope);
 extern HQL_API _ATOM queryPatternName(IHqlExpression * expr);
 extern HQL_API IHqlExpression * closeAndLink(IHqlExpression * expr);
 extern HQL_API IHqlExpression * createAbstractRecord(IHqlExpression * record);
+extern HQL_API IHqlExpression * createSortList(HqlExprArray & elements);
 
 // Same as expr->queryChild() except it doesn't return attributes.
 inline IHqlExpression * queryRealChild(IHqlExpression * expr, unsigned i)

ecl/hql/hqlfold.cpp

@@ -3307,6 +3307,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
             break;
         }
     case no_sort:
+    case no_shuffle:
     case no_sorted:
         {
             //If action does not change the type information, then it can't have done anything...
@@ -3314,7 +3315,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
                 return removeParentNode(expr);
             if (isNull(child) || hasNoMoreRowsThan(child, 1))
                 return removeParentNode(expr);
-            //If all arguments to sort are constnat then remove it, otherwise the activities will not like it.
+            //If all arguments to sort are constant then remove it, otherwise the activities will not like it.
             //NOTE: MERGE has its sort order preserved, so it won't cause issues there.
             bool allConst = true;
             ForEachChildFrom(i, expr, 1)
@@ -5397,6 +5398,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_keyeddistribute:
     case no_cosort:
     case no_sort:
+    case no_shuffle:
     case no_sorted:
     case no_assertsorted:
     case no_topn:

ecl/hql/hqlgram.y

@@ -386,6 +386,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   SERVICE
   SET
   SHARED
+  SHUFFLE
   SIMPLE_TYPE
   SIN
   SINGLE
@@ -7110,7 +7111,7 @@ simpleDataSet
                             RecordSelectIterator iter(active->queryRecord(), active);
                             ForEach(iter)
                                 components.append(*iter.get());
-                            OwnedHqlExpr sortlist = createValue(no_sortlist, makeSortListType(NULL), components);
+                            OwnedHqlExpr sortlist = createSortList(components);
                             OwnedHqlExpr hash = createValue(no_hash32, makeIntType(4, false), LINK(sortlist), createAttribute(internalAtom));
                             $$.setExpr(createDataset(no_distribute, ds.getClear(), createComma(hash.getClear(), $5.getExpr())), $1);
                         }
@@ -7334,7 +7335,7 @@ simpleDataSet
                                     values.append(OLINK(cur));
                             }
                             if (values.ordinality())
-                                cond = createValue(no_sortlist, makeSortListType(NULL), values);
+                                cond = createSortList(values);
                             else
                                 cond = createConstant(true);
 
@@ -8164,6 +8165,14 @@ simpleDataSet
                             $$.setExpr(parser->createSortExpr(no_sort, $4, $7, sortItems));
                             $$.setPosition($1);
                         }
+    | SHUFFLE '(' startSortOrder startTopFilter ',' sortListExpr ',' sortListExpr optCommonAttrs ')'  endSortOrder endTopFilter
+                        {
+                            OwnedHqlExpr options = $9.getExpr();
+                            if (isGrouped($4.queryExpr()))
+                                parser->reportError(HQLERR_CannotBeGrouped, $1, "SHUFFLE not yet supported on grouped datasets");
+                            //NB: $6 and $8 are reversed in their internal representation to make consistent with no_sort
+                            $$.setExpr(createDataset(no_shuffle, $4.getExpr(), createComma($8.getExpr(), $6.getExpr(), options.getClear())), $1);
+                        }
     | SORTED '(' startSortOrder startTopFilter ',' beginList sortListOptCurleys ')' endSortOrder endTopFilter
                         {
                             HqlExprArray sortItems;
@@ -8181,7 +8190,7 @@ simpleDataSet
                             IHqlExpression * record = dataset->queryRecord();
                             unwindRecordAsSelects(sorted, record, dataset->queryNormalizedSelector());
                             args.append(*dataset.getClear());
-                            args.append(*createValue(no_sortlist, makeSortListType(NULL), sorted));
+                            args.append(*createSortList(sorted));
                             $$.setExpr(createDataset(no_sorted, args));
                             $$.setPosition($1);
                         }
@@ -8191,7 +8200,7 @@ simpleDataSet
 
                             HqlExprArray args;
                             $5.unwindCommaList(args);
-                            OwnedHqlExpr stepOrder = createValue(no_sortlist, makeSortListType(NULL), args);
+                            OwnedHqlExpr stepOrder = createSortList(args);
                             $$.setExpr(createDatasetF(no_stepped, dataset.getClear(), stepOrder.getClear(), $6.getExpr(), NULL));
                             $$.setPosition($1);
                         }
@@ -10446,7 +10455,7 @@ optFieldMaps
                         {
                             HqlExprArray args;
                             parser->endList(args);
-                            $$.setExpr(createValue(no_sortlist, makeSortListType(NULL), args), $1);
+                            $$.setExpr(createSortList(args), $1);
                         }
     |                   { $$.setNullExpr(); }
     | '{' beginList error '}'
@@ -10478,7 +10487,8 @@ fieldMap
 
 sortListOptCurleys
     : sortList
-    | '{' sortList '}'  
+    | '{' sortList '}'
+    | '{' sortList '}' ',' sortList         /* Allow trailing attributes */
     ;
 
 sortList
@@ -10720,6 +10730,15 @@ beginList
                         }
     ;
 
+sortListExpr
+    : beginList '{' sortList '}'
+                        {
+                            HqlExprArray elements;
+                            parser->endList(elements);
+                            $$.setExpr(createSortList(elements));
+                        }
+    ;
+
 ignoreDummyList
     :
                         {

ecl/hql/hqlgram2.cpp

@@ -5493,7 +5493,7 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
         }
     }
     if (items.ordinality())
-        return createValue(no_sortlist, makeSortListType(NULL), items);
+        return createSortList(items);
     if (op == no_list)
         return createValue(no_sortlist, makeSortListType(NULL), items);
     return NULL;
@@ -10089,6 +10089,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case HTTPHEADER: msg.append("HTTPHEADER"); break;
     case PROXYADDRESS: msg.append("PROXYADDRESS"); break;
     case HTTPCALL: msg.append("HTTPCALL"); break;
+    case SHUFFLE: msg.append("SHUFFLE"); break;
     case SOAPCALL: msg.append("SOAPCALL"); break;
     case SORT: msg.append("SORT"); break;
     case SORTED: msg.append("SORTED"); break;
@@ -10282,7 +10283,7 @@ void HqlGram::simplifyExpected(int *expected)
                        GROUP, GROUPED, KEYED, UNGROUP, JOIN, PULL, ROLLUP, ITERATE, PROJECT, NORMALIZE, PIPE, DENORMALIZE, CASE, MAP, 
                        HTTPCALL, SOAPCALL, LIMIT, PARSE, FAIL, MERGE, PRELOAD, ROW, TOPN, ALIAS, LOCAL, NOFOLD, NOHOIST, NOTHOR, IF, GLOBAL, __COMMON__, __COMPOUND__, TOK_ASSERT, _EMPTY_,
                        COMBINE, ROWS, REGROUP, XMLPROJECT, SKIP, LOOP, CLUSTER, NOLOCAL, REMOTE, PROCESS, ALLNODES, THISNODE, GRAPH, MERGEJOIN, STEPPED, NONEMPTY, HAVING,
-                       TOK_CATCH, '@', SECTION, WHEN, IFF, COGROUP, HINT, INDEX, PARTITION, AGGREGATE, 0);
+                       TOK_CATCH, '@', SECTION, WHEN, IFF, COGROUP, HINT, INDEX, PARTITION, AGGREGATE, SHUFFLE, 0);
     simplify(expected, EXP, ABS, SIN, COS, TAN, SINH, COSH, TANH, ACOS, ASIN, ATAN, ATAN2, 
                        COUNT, CHOOSE, MAP, CASE, IF, HASH, HASH32, HASH64, HASHMD5, CRC, LN, TOK_LOG, POWER, RANDOM, ROUND, ROUNDUP, SQRT, 
                        TRUNCATE, LENGTH, TRIM, INTFORMAT, REALFORMAT, ASSTRING, TRANSFER, MAX, MIN, EVALUATE, SUM,

ecl/hql/hqllex.l

@@ -878,6 +878,7 @@ SERVICE             { RETURNHARD(SERVICE); }
 SET                 { RETURNSYM(SET); }
 __SET_DEBUG_OPTION__ { RETURNSYM(HASH_OPTION); }
 SHARED              { RETURNHARD(SHARED); }
+SHUFFLE             { RETURNSYM(SHUFFLE); }
 SIN                 { RETURNSYM(SIN); }
 SINGLE              { RETURNSYM(SINGLE); }
 SINH                { RETURNSYM(SINH); }

ecl/hql/hqlmeta.cpp

@@ -40,6 +40,7 @@
 
 //#define OPTIMIZATION2
 
+static IHqlExpression * cacheGroupedElement;
 static IHqlExpression * cacheUnknownAttribute;
 static IHqlExpression * cacheIndeterminateAttribute;
 static IHqlExpression * cacheUnknownSortlist;
@@ -49,11 +50,13 @@ static IHqlExpression * cached_omitted_Attribute;
 
 MODULE_INIT(INIT_PRIORITY_STANDARD)
 {
+    _ATOM groupedOrderAtom = createAtom("{group-order}");
+    cacheGroupedElement = createAttribute(groupedOrderAtom);
     cacheUnknownAttribute = createAttribute(unknownAtom);
     cacheIndeterminateAttribute = createAttribute(indeterminateAtom);
     cacheUnknownSortlist = createValue(no_sortlist, makeSortListType(NULL), LINK(cacheUnknownAttribute));
     cacheIndeterminateSortlist = createValue(no_sortlist, makeSortListType(NULL), LINK(cacheIndeterminateAttribute));
-    cacheMatchGroupOrderSortlist = createValue(no_sortlist, makeSortListType(NULL), createAttribute(groupedAtom));
+    cacheMatchGroupOrderSortlist = createValue(no_sortlist, makeSortListType(NULL), LINK(cacheGroupedElement));
     cached_omitted_Attribute = createAttribute(_omitted_Atom);
     return true;
 }
@@ -65,6 +68,7 @@ MODULE_EXIT()
     cacheUnknownSortlist->Release();
     cacheIndeterminateAttribute->Release();
     cacheUnknownAttribute->Release();
+    cacheGroupedElement->Release();
 }
 
 /*
@@ -244,6 +248,17 @@ IHqlExpression * getUnknownSortlist() { return LINK(cacheUnknownSortlist); }
 
 inline bool matchesGroupOrder(IHqlExpression * expr) { return expr == cacheMatchGroupOrderSortlist; }
 
+bool hasTrailingGroupOrder(IHqlExpression * expr)
+{
+    if (expr)
+    {
+        unsigned max = expr->numChildren();
+        if (max)
+            return expr->queryChild(max-1) == cacheGroupedElement;
+    }
+    return false;
+}
+
 //---------------------------------------------------------------------------------------------
 // Helper functions for processing the basic lists
 
@@ -266,15 +281,18 @@ bool intersectList(HqlExprArray & target, const HqlExprArray & left, const HqlEx
 }
 
 
-IHqlExpression * createSubSortlist(IHqlExpression * sortlist, unsigned from, unsigned to)
+IHqlExpression * createSubSortlist(IHqlExpression * sortlist, unsigned from, unsigned to, IHqlExpression * subsetAttr)
 {
     if (from == to)
         return NULL;
     if ((from == 0) && (to == sortlist->numChildren()))
         return LINK(sortlist);
+
     HqlExprArray components;
     unwindChildren(components, sortlist, from, to);
-    return createValue(no_sortlist, makeSortListType(NULL), components);
+    if (subsetAttr)
+        components.append(*LINK(subsetAttr));
+    return createSortList(components);
 }
 
 void removeDuplicates(HqlExprArray & components)
@@ -326,7 +344,7 @@ void normalizeComponents(HqlExprArray & args, const HqlExprArray & src)
     removeDuplicates(args);
 }
 
-IHqlExpression * getIntersectingSortlist(IHqlExpression * left, IHqlExpression * right)
+IHqlExpression * getIntersectingSortlist(IHqlExpression * left, IHqlExpression * right, IHqlExpression * subsetAttr)
 {
     if (!left || !right)
         return NULL;
@@ -335,16 +353,16 @@ IHqlExpression * getIntersectingSortlist(IHqlExpression * left, IHqlExpression *
     {
         //This test also covers the case where one list is longer than the other...
         if (left->queryChild(i) != right->queryChild(i))
-            return createSubSortlist(left, 0, i);
+            return createSubSortlist(left, 0, i, subsetAttr);
     }
     return LINK(left);
 }
 
 
 //Find the intersection between left and (localOrder+groupOrder)
-IHqlExpression * getIntersectingSortlist(IHqlExpression * left, IHqlExpression * localOrder, IHqlExpression * groupOrder)
+IHqlExpression * getModifiedGlobalOrder(IHqlExpression * globalOrder, IHqlExpression * localOrder, IHqlExpression * groupOrder)
 {
-    if (!left || !localOrder)
+    if (!globalOrder || !localOrder)
         return NULL;
 
     unsigned max1=0;
@@ -353,8 +371,13 @@ IHqlExpression * getIntersectingSortlist(IHqlExpression * left, IHqlExpression *
         ForEachChild(i1, localOrder)
         {
             //This test also covers the case where one list is longer than the other...
-            if (left->queryChild(i1) != localOrder->queryChild(i1))
-                return createSubSortlist(left, 0, i1);
+            IHqlExpression * curLocal = localOrder->queryChild(i1);
+            if (globalOrder->queryChild(i1) != curLocal)
+            {
+                if (curLocal == cacheGroupedElement)
+                    break;
+                return createSubSortlist(globalOrder, 0, i1, NULL);
+            }
         }
         max1 = localOrder->numChildren();
     }
@@ -365,12 +388,12 @@ IHqlExpression * getIntersectingSortlist(IHqlExpression * left, IHqlExpression *
         ForEachChild(i2, groupOrder)
         {
             //This test also covers the case where one list is longer than the other...
-            if (left->queryChild(i2+max1) != groupOrder->queryChild(i2))
-                return createSubSortlist(left, 0, i2+max1);
+            if (globalOrder->queryChild(i2+max1) != groupOrder->queryChild(i2))
+                return createSubSortlist(globalOrder, 0, i2+max1, NULL);
         }
         max2 = groupOrder->numChildren();
     }
-    return createSubSortlist(left, 0, max1+max2);
+    return createSubSortlist(globalOrder, 0, max1+max2, NULL);
 }
 
 
@@ -383,7 +406,7 @@ static IHqlExpression * normalizeSortlist(IHqlExpression * sortlist)
     unwindNormalizeSortlist(components, sortlist, false);
     removeDuplicates(components);
     //This never returns NULL if the input was non-null
-    return createValue(no_sortlist, makeSortListType(NULL), components);
+    return createSortList(components);
 }
 
 inline IHqlExpression * normalizeSortlist(IHqlExpression * sortlist, IHqlExpression * dataset)
@@ -399,6 +422,36 @@ IHqlExpression * normalizeDistribution(IHqlExpression * distribution)
     return LINK(distribution);
 }
 
+static bool sortComponentMatches(IHqlExpression * curNew, IHqlExpression * curExisting)
+{
+    IHqlExpression * newBody = curNew->queryBody();
+    IHqlExpression * existingBody = curExisting->queryBody();
+    if (newBody == existingBody)
+        return true;
+
+    ITypeInfo * newType = curNew->queryType();
+    ITypeInfo * existingType = curExisting->queryType();
+
+    //A local sort by (string)qstring is the same as by qstring....
+    if (isCast(curNew) && (curNew->queryChild(0)->queryBody() == existingBody))
+    {
+        if (preservesValue(newType, existingType) && preservesOrder(newType, existingType))
+            return true;
+    }
+    // a sort by qstring is the same as by (string)qstring.
+    if (isCast(curExisting) && (newBody == curExisting->queryChild(0)->queryBody()))
+    {
+        if (preservesValue(existingType, newType) && preservesOrder(existingType, newType))
+            return true;
+    }
+    // (cast:z)x should match (implicit-cast:z)x
+    if (isCast(curNew) && isCast(curExisting) && (newType==existingType))
+        if (curNew->queryChild(0)->queryBody() == curExisting->queryChild(0)->queryBody())
+            return true;
+
+    return false;
+}
+
 //---------------------------------------------------------------------------------------------
 
 bool isKnownDistribution(IHqlExpression * distribution)
@@ -440,6 +493,9 @@ IHqlExpression * getLocalSortOrder(ITypeInfo * type)
     unwindChildren(components, localOrder);
     if (!hasUnknownComponent(components))
     {
+        if (components.length() && (&components.tos() == cacheGroupedElement))
+            components.pop();
+
         if (groupOrder)
             unwindChildren(components, groupOrder);
     }
@@ -448,7 +504,7 @@ IHqlExpression * getLocalSortOrder(ITypeInfo * type)
     if (components.ordinality())
     {
         removeDuplicates(components);
-        return createValue(no_sortlist, makeSortListType(NULL), components);
+        return createSortList(components);
     }
     return NULL;
 }
@@ -566,16 +622,41 @@ ITypeInfo * getTypeUngroup(ITypeInfo * prev)
 }
 
 
+static bool matchesGroupBy(IHqlExpression * groupBy, IHqlExpression * cur)
+{
+    if (sortComponentMatches(groupBy, cur))
+        return true;
+    if (cur->getOperator() == no_negate)
+        return sortComponentMatches(groupBy, cur->queryChild(0));
+    return false;
+}
+
+static bool withinGroupBy(const HqlExprArray & groupBy, IHqlExpression * cur)
+{
+    ForEachItemIn(i, groupBy)
+    {
+        if (matchesGroupBy(&groupBy.item(i), cur))
+            return true;
+    }
+    return false;
+}
+
+static bool groupByWithinSortOrder(IHqlExpression * groupBy, IHqlExpression * order)
+{
+    ForEachChild(i, order)
+    {
+        if (matchesGroupBy(groupBy, order->queryChild(i)))
+            return true;
+    }
+    return false;
+}
+
 //NB: This does not handle ALL groups that is handled in createDataset()
 ITypeInfo * getTypeGrouped(ITypeInfo * prev, IHqlExpression * grouping, bool isLocal)
 {
     OwnedITypeInfo prevUngrouped = getTypeUngroup(prev);
     OwnedHqlExpr newGrouping = normalizeSortlist(grouping);
 
-    HqlExprArray groupBy;
-    if (newGrouping)
-        unwindChildren(groupBy, newGrouping);
-
     IHqlExpression * distribution = isLocal ? queryDistribution(prevUngrouped) : NULL;
     IHqlExpression * globalOrder = queryGlobalSortOrder(prevUngrouped);
     IHqlExpression * localOrder = queryLocalUngroupedSortOrder(prevUngrouped);
@@ -584,19 +665,58 @@ ITypeInfo * getTypeGrouped(ITypeInfo * prev, IHqlExpression * grouping, bool isL
 
     if (localOrder)
     {
-        //Find the last local order component that is included in the grouping condition
-        unsigned max = localOrder->numChildren();
-        unsigned firstGroup = 0;
-        for (unsigned i=max;i--!= 0;)
+        HqlExprArray groupBy;
+        if (newGrouping)
+            unwindChildren(groupBy, newGrouping);
+
+        //The local sort order is split into two.
+        //Where depends on whether all the grouping conditions match sort elements.
+        //MORE: Is there a good way to accomplish this withit iterating both ways round?
+        bool allGroupingMatch = true;
+        ForEachItemIn(i, groupBy)
         {
-            IHqlExpression * cur = localOrder->queryChild(i);
-            if (groupBy.contains(*cur))
+            IHqlExpression * groupElement = &groupBy.item(i);
+            if (!groupByWithinSortOrder(groupElement, localOrder))
             {
-                firstGroup = i+1;
+                allGroupingMatch = false;
                 break;
             }
         }
 
+        unsigned max = localOrder->numChildren();
+        unsigned firstGroup;
+        if (allGroupingMatch)
+        {
+            //All grouping conditions match known sorts.  Therefore the last local order component that is included in
+            //the grouping condition is important.  The order of all elements before that will be preserved if the
+            //group is sorted.
+            firstGroup = 0;
+            for (unsigned i=max;i--!= 0;)
+            {
+                IHqlExpression * cur = localOrder->queryChild(i);
+                if (withinGroupBy(groupBy, cur))
+                {
+                    firstGroup = i+1;
+                    break;
+                }
+            }
+        }
+        else
+        {
+            //If one of the grouping conditions is not included in the sort order, and if the group is subsequently
+            //sorted then the the state of the first element that doesn't match the grouping condition will be unknown.
+            firstGroup = max;
+            for (unsigned i=0;i<max;i++)
+            {
+                IHqlExpression * cur = localOrder->queryChild(i);
+                if (!withinGroupBy(groupBy, cur))
+                {
+                    firstGroup = i;
+                    break;
+                }
+            }
+        }
+
         if (firstGroup == 0)
         {
             //mark the local ungrouped sort order with a special value so we can restore if order doesn't change.
@@ -605,8 +725,10 @@ ITypeInfo * getTypeGrouped(ITypeInfo * prev, IHqlExpression * grouping, bool isL
         }
         else
         {
-            newLocalOrder.setown(createSubSortlist(localOrder, 0, firstGroup));
-            newGroupOrder.setown(createSubSortlist(localOrder, firstGroup, max));
+            //Add a marker to the end of the first order if it the rest will become invalidated by a group sort
+            IHqlExpression * subsetAttr = (!allGroupingMatch && (firstGroup != max)) ? cacheGroupedElement : NULL;
+            newLocalOrder.setown(createSubSortlist(localOrder, 0, firstGroup, subsetAttr));
+            newGroupOrder.setown(createSubSortlist(localOrder, firstGroup, max, NULL));
         }
     }
 
@@ -629,7 +751,7 @@ ITypeInfo * getTypeLocalSort(ITypeInfo * prev, IHqlExpression * sortOrder)
     IHqlExpression * globalSortOrder = queryGlobalSortOrder(prev);
     OwnedHqlExpr localSortOrder = normalizeSortlist(sortOrder);
     //The global sort order is maintained as the leading components that match.
-    OwnedHqlExpr newGlobalOrder = getIntersectingSortlist(globalSortOrder, localSortOrder);
+    OwnedHqlExpr newGlobalOrder = getIntersectingSortlist(globalSortOrder, localSortOrder, NULL);
     ITypeInfo * rowType = queryRowType(prev);
     return makeTableType(LINK(rowType), LINK(distribution), newGlobalOrder.getClear(), localSortOrder.getClear());
 }
@@ -644,9 +766,22 @@ ITypeInfo * getTypeGroupSort(ITypeInfo * prev, IHqlExpression * sortOrder)
     IHqlExpression * globalOrder = queryGlobalSortOrder(prev);
     IHqlExpression * localUngroupedOrder = queryLocalUngroupedSortOrder(prev);
     //Group sort => make sure we no longer track it as the localsort
-    IHqlExpression * newLocalUngroupedOrder = matchesGroupOrder(localUngroupedOrder) ? NULL : localUngroupedOrder;
+    OwnedHqlExpr newLocalUngroupedOrder;
+    if (localUngroupedOrder && !matchesGroupOrder(localUngroupedOrder))
+    {
+        if (hasTrailingGroupOrder(localUngroupedOrder))
+        {
+            HqlExprArray components;
+            unwindChildren(components, localUngroupedOrder);
+            components.pop();
+            components.append(*getUnknownAttribute());
+            newLocalUngroupedOrder.setown(localUngroupedOrder->clone(components));
+        }
+        else
+            newLocalUngroupedOrder.set(localUngroupedOrder);
+    }
 
-    OwnedHqlExpr newGlobalOrder = getIntersectingSortlist(globalOrder, newLocalUngroupedOrder, groupedOrder);
+    OwnedHqlExpr newGlobalOrder = getModifiedGlobalOrder(globalOrder, newLocalUngroupedOrder, groupedOrder);
     ITypeInfo * prevTable = prev->queryChildType();
     ITypeInfo * tableType;
     if ((globalOrder != newGlobalOrder) || (localUngroupedOrder != newLocalUngroupedOrder))
@@ -686,21 +821,23 @@ ITypeInfo * getTypeIntersection(ITypeInfo * leftType, ITypeInfo * rightType)
     else if (leftDist && rightDist)
         newDistributeInfo.set(queryUnknownAttribute());
 
-    OwnedHqlExpr globalOrder = getIntersectingSortlist(queryGlobalSortOrder(leftType), queryGlobalSortOrder(rightType));
+    OwnedHqlExpr globalOrder = getIntersectingSortlist(queryGlobalSortOrder(leftType), queryGlobalSortOrder(rightType), NULL);
     IHqlExpression * leftLocalOrder = queryLocalUngroupedSortOrder(leftType);
     IHqlExpression * rightLocalOrder = queryLocalUngroupedSortOrder(rightType);
     IHqlExpression * leftGrouping = queryGrouping(leftType);
     IHqlExpression * rightGrouping = queryGrouping(rightType);
     OwnedHqlExpr localOrder;
     OwnedHqlExpr grouping = (leftGrouping || rightGrouping) ? getUnknownSortlist() : NULL;
+    if (leftGrouping == rightGrouping)
+        grouping.set(leftGrouping);
+
     OwnedHqlExpr groupOrder;
     if (leftLocalOrder == rightLocalOrder)
     {
         localOrder.set(leftLocalOrder);
         if (leftGrouping == rightGrouping)
         {
-            grouping.set(leftGrouping);
-            groupOrder.setown(getIntersectingSortlist(queryGroupSortOrder(leftType), queryGroupSortOrder(rightType)));
+            groupOrder.setown(getIntersectingSortlist(queryGroupSortOrder(leftType), queryGroupSortOrder(rightType), NULL));
         }
         else
         {
@@ -711,7 +848,10 @@ ITypeInfo * getTypeIntersection(ITypeInfo * leftType, ITypeInfo * rightType)
     {
         //intersect local order - not worth doing anything else
         if (!matchesGroupOrder(leftLocalOrder) && !matchesGroupOrder(rightLocalOrder))
-            localOrder.setown(getIntersectingSortlist(leftLocalOrder, rightLocalOrder));
+        {
+            IHqlExpression * extraAttr = grouping ? queryUnknownAttribute() : NULL;
+            localOrder.setown(getIntersectingSortlist(leftLocalOrder, rightLocalOrder, extraAttr));
+        }
     }
 
     ITypeInfo * rowType = queryRowType(leftType);
@@ -835,6 +975,14 @@ ITypeInfo * getTypeFromMeta(IHqlExpression * record, IHqlExpression * meta, unsi
 }
 
 
+extern HQL_API ITypeInfo * getTypeShuffle(ITypeInfo * prevType, IHqlExpression * grouping, IHqlExpression * sortOrder, bool isLocal)
+{
+    Owned<ITypeInfo> groupedType = getTypeGrouped(prevType, grouping, isLocal);
+    Owned<ITypeInfo> sortedType = getTypeGroupSort(groupedType, sortOrder);
+    return getTypeUngroup(sortedType);
+}
+
+
 //---------------------------------------------------------------------------------------------
 
 bool appearsToBeSorted(ITypeInfo * type, bool isLocal, bool ignoreGrouping)
@@ -892,7 +1040,7 @@ bool isSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isL
 }
 
 
-IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal, bool alwaysLocal)
+IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal, bool alwaysLocal, bool allowShuffle)
 {
     if (isSortedForGroup(table, sortList, isLocal||alwaysLocal))
         return LINK(table);
@@ -1017,36 +1165,20 @@ IHqlExpression * getExistingSortOrder(IHqlExpression * dataset, bool isLocal, bo
 }
 
 
-static bool sortComponentMatches(IHqlExpression * curNew, IHqlExpression * curExisting)
+static bool isCorrectDistributionForSort(IHqlExpression * dataset, IHqlExpression * normalizedSortOrder, bool isLocal, bool ignoreGrouping)
 {
-    IHqlExpression * newBody = curNew->queryBody();
-    IHqlExpression * existingBody = curExisting->queryBody();
-    if (newBody == existingBody)
+    if (isLocal || (isGrouped(dataset) && !ignoreGrouping))
         return true;
-
-    ITypeInfo * newType = curNew->queryType();
-    ITypeInfo * existingType = curExisting->queryType();
-
-    //A local sort by (string)qstring is the same as by qstring....
-    if (isCast(curNew) && (curNew->queryChild(0)->queryBody() == existingBody))
-    {
-        if (preservesValue(newType, existingType) && preservesOrder(newType, existingType))
-            return true;
-    }
-    // a sort by qstring is the same as by (string)qstring.
-    if (isCast(curExisting) && (newBody == curExisting->queryChild(0)->queryBody()))
-    {
-        if (preservesValue(existingType, newType) && preservesOrder(existingType, newType))
-            return true;
-    }
-    // (cast:z)x should match (implicit-cast:z)x
-    if (isCast(curNew) && isCast(curExisting) && (newType==existingType))
-        if (curNew->queryChild(0)->queryBody() == curExisting->queryChild(0)->queryBody())
-            return true;
-
-    return false;
+    IHqlExpression * distribution = queryDistribution(dataset);
+    if (!isSortDistribution(distribution))
+        return false;
+    IHqlExpression * previousOrder = distribution->queryChild(0);           // Already normalized when it was created.
+    //MORE: We should possibly loosen this test to allow compatible casts etc.
+    //return isCompatibleSortOrder(existingOrder, normalizedSortOrder)
+    return (previousOrder == normalizedSortOrder);
 }
 
+//--------------------------------------------------------------------------------------------------------------------
 
 static bool isCompatibleSortOrder(IHqlExpression * existingOrder, IHqlExpression * normalizedOrder)
 {
@@ -1064,19 +1196,6 @@ static bool isCompatibleSortOrder(IHqlExpression * existingOrder, IHqlExpression
     return true;
 }
 
-static bool isCorrectDistributionForSort(IHqlExpression * dataset, IHqlExpression * normalizedSortOrder, bool isLocal, bool ignoreGrouping)
-{
-    if (isLocal || (isGrouped(dataset) && !ignoreGrouping))
-        return true;
-    IHqlExpression * distribution = queryDistribution(dataset);
-    if (!isSortDistribution(distribution))
-        return false;
-    IHqlExpression * previousOrder = distribution->queryChild(0);           // Already normalized when it was created.
-    //MORE: We should possibly loosen this test to allow compatible casts etc.
-    //return isCompatibleSortOrder(existingOrder, normalizedSortOrder)
-    return (previousOrder == normalizedSortOrder);
-}
-
 static bool normalizedIsAlreadySorted(IHqlExpression * dataset, IHqlExpression * normalizedOrder, bool isLocal, bool ignoreGrouping)
 {
 #ifdef OPTIMIZATION2
@@ -1109,15 +1228,158 @@ bool isAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLo
 {
     HqlExprArray components;
     normalizeComponents(components, newSort);
-    OwnedHqlExpr normalizedOrder = createValue(no_sortlist, makeSortListType(NULL), components);
+    OwnedHqlExpr normalizedOrder = createSortList(components);
     return normalizedIsAlreadySorted(dataset, normalizedOrder, isLocal, ignoreGrouping);
 }
 
-IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
+
+//--------------------------------------------------------------------------------------------------------------------
+
+static unsigned numCompatibleSortElements(IHqlExpression * existingOrder, IHqlExpression * normalizedOrder)
+{
+    if (!existingOrder)
+        return 0;
+    unsigned numExisting = existingOrder->numChildren();
+    unsigned numRequired = normalizedOrder->numChildren();
+    unsigned numToCompare = (numRequired > numExisting) ? numExisting : numRequired;
+    for (unsigned i=0; i < numToCompare; i++)
+    {
+        if (!sortComponentMatches(normalizedOrder->queryChild(i), existingOrder->queryChild(i)))
+            return i;
+    }
+    return numToCompare;
+}
+
+static unsigned normalizedNumSortedElements(IHqlExpression * dataset, IHqlExpression * normalizedOrder, bool isLocal, bool ignoreGrouping)
+{
+#ifdef OPTIMIZATION2
+    if (hasNoMoreRowsThan(dataset, 1))
+        return true;
+#endif
+    if (!isCorrectDistributionForSort(dataset, normalizedOrder, isLocal, ignoreGrouping))
+        return false;
+
+    //Constant items and duplicates should have been removed already.
+    OwnedHqlExpr existingOrder = getExistingSortOrder(dataset, isLocal, ignoreGrouping);
+    return numCompatibleSortElements(existingOrder, normalizedOrder);
+}
+
+static unsigned numElementsAlreadySorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping)
+{
+#ifdef OPTIMIZATION2
+    if (hasNoMoreRowsThan(dataset, 1))
+        return order->numChildren();
+#endif
+
+    OwnedHqlExpr normalizedOrder = normalizeSortlist(order, dataset);
+    return normalizedNumSortedElements(dataset, normalizedOrder, isLocal, ignoreGrouping);
+}
+
+//Elements in the exprarray have already been mapped;
+static unsigned numElementsAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
+{
+    HqlExprArray components;
+    normalizeComponents(components, newSort);
+    OwnedHqlExpr normalizedOrder = createSortList(components);
+    return normalizedNumSortedElements(dataset, normalizedOrder, isLocal, ignoreGrouping);
+}
+
+bool isWorthShuffling(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping)
+{
+    //MORE: Should this look at the cardinality of the already-sorted fields, and not transform if below a certain threshold?
+    return numElementsAlreadySorted(dataset, order, isLocal, ignoreGrouping) != 0;
+}
+
+bool isWorthShuffling(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
+{
+    //MORE: Should this look at the cardinality of the already-sorted fields, and not transform if below a certain threshold?
+    return numElementsAlreadySorted(dataset, newSort, isLocal, ignoreGrouping) != 0;
+}
+
+//--------------------------------------------------------------------------------------------------------------------
+
+//Convert SHUFFLE(ds, <sort>, <grouping>, ?LOCAL, options) to
+//g := GROUP(ds, grouping, ?LOCAL); s := SORT(g, <sort>, options); GROUP(s);
+IHqlExpression * convertShuffleToGroupedSort(IHqlExpression * expr)
+{
+    IHqlExpression * dataset = expr->queryChild(0);
+    IHqlExpression * newOrder = expr->queryChild(1);
+    IHqlExpression * grouping = expr->queryChild(2);
+
+    assertex(!isGrouped(dataset) || expr->hasProperty(globalAtom));
+    OwnedHqlExpr attr = isLocalActivity(expr) ? createLocalAttribute() : NULL;
+    OwnedHqlExpr grouped = createDatasetF(no_group, LINK(dataset), LINK(grouping), LINK(attr), NULL);
+
+    HqlExprArray args;
+    args.append(*grouped.getClear());
+    args.append(*LINK(newOrder));
+    unwindChildren(args, expr, 3);
+    removeProperty(args, localAtom);
+    OwnedHqlExpr sorted = createDataset(no_sort, args);
+    return createDataset(no_group, sorted.getClear());
+}
+
+static IHqlExpression * createShuffled(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
+{
+    bool isGroupedShuffle = !ignoreGrouping && isGrouped(dataset);
+    unsigned sortedElements = numElementsAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping);
+    if ((sortedElements == 0) || isGroupedShuffle)
+        return NULL;
+
+    HqlExprArray components;
+    unwindNormalizeSortlist(components, order, false);
+    removeDuplicates(components);
+    if (components.ordinality() == sortedElements)
+        return LINK(dataset);
+
+    OwnedHqlExpr alreadySorted = createValueSafe(no_sortlist, makeSortListType(NULL), components, 0, sortedElements);
+    OwnedHqlExpr newOrder = createValueSafe(no_sortlist, makeSortListType(NULL), components, sortedElements, components.ordinality());
+
+    OwnedHqlExpr attr = isLocal ? createLocalAttribute() : (isGrouped(dataset) && ignoreGrouping) ? createAttribute(globalAtom) : NULL;
+    OwnedHqlExpr shuffle = createDatasetF(no_shuffle, LINK(dataset), LINK(newOrder), LINK(alreadySorted), LINK(attr), NULL);
+    //Grouped shuffles never generated, global shuffles (if generated) get converted to a global group
+    if (!isLocal && !alwaysLocal)
+        shuffle.setown(convertShuffleToGroupedSort(shuffle));
+
+    assertex(isAlreadySorted(shuffle, order, isLocal||alwaysLocal, ignoreGrouping));
+    return shuffle.getClear();
+}
+
+IHqlExpression * getShuffleSort(IHqlExpression * dataset, HqlExprArray & order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
+{
+    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
+        return NULL;
+
+    OwnedHqlExpr sortlist = createValueSafe(no_sortlist, makeSortListType(NULL), order);
+    OwnedHqlExpr mappedSortlist = replaceSelector(sortlist, queryActiveTableSelector(), dataset);
+    return createShuffled(dataset, mappedSortlist, isLocal, ignoreGrouping, alwaysLocal);
+}
+
+IHqlExpression * getShuffleSort(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
+{
+    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
+        return NULL;
+
+    return createShuffled(dataset, order, isLocal, ignoreGrouping, alwaysLocal);
+}
+
+//--------------------------------------------------------------------------------------------------------------------
+
+IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal, bool allowShuffle)
 {
     if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
         return LINK(dataset);
 
+    if (allowShuffle && (isLocal || alwaysLocal))
+    {
+        if (isWorthShuffling(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
+        {
+            OwnedHqlExpr shuffled = createShuffled(dataset, order, isLocal, ignoreGrouping, alwaysLocal);
+            if (shuffled)
+                return shuffled.getClear();
+        }
+    }
+
     IHqlExpression * attr = isLocal ? createLocalAttribute() : (isGrouped(dataset) && ignoreGrouping) ? createAttribute(globalAtom) : NULL;
     return createDatasetF(no_sort, LINK(dataset), LINK(order), attr, NULL);
 }
@@ -1312,8 +1574,14 @@ static IHqlExpression * optimizePreserveMeta(IHqlExpression * expr)
     if (expr->getOperator() != no_preservemeta)
         return LINK(expr);
     IHqlExpression * ds = expr->queryChild(0);
-    if (ds->getOperator() != no_workunit_dataset)
+    switch (ds->getOperator())
+    {
+    case no_workunit_dataset:
+    case no_getgraphresult:
+        break;
+    default:
         return LINK(expr);
+    }
     HqlExprArray args;
     unwindChildren(args, expr, 1);
 

ecl/hql/hqlmeta.hpp

@@ -45,6 +45,7 @@ extern HQL_API ITypeInfo * getTypeDistribute(ITypeInfo * prev, IHqlExpression *
 extern HQL_API ITypeInfo * getTypeFromMeta(IHqlExpression * record, IHqlExpression * meta, unsigned firstChild);
 extern HQL_API ITypeInfo * getTypeIntersection(ITypeInfo * leftType, ITypeInfo * rightType);
 extern HQL_API ITypeInfo * getTypeProject(ITypeInfo * prev, IHqlExpression * newRecord, TableProjectMapper & mapper);
+extern HQL_API ITypeInfo * getTypeShuffle(ITypeInfo * prev, IHqlExpression * grouping, IHqlExpression * sortOrder, bool isLocal);
 extern HQL_API ITypeInfo * getTypeCannotProject(ITypeInfo * prev, IHqlExpression * newRecord); // preserve grouping, but that's it.
 extern HQL_API ITypeInfo * getTypeUnknownDistribution(ITypeInfo * prev);
 extern HQL_API ITypeInfo * getTypeRemoveDistribution(ITypeInfo * prev);
@@ -63,14 +64,20 @@ extern HQL_API bool isPartitionedForGroup(IHqlExpression * table, const HqlExprA
 
 //The following only look at the sort order, and not the partitioning
 extern HQL_API bool isSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal);
-extern HQL_API IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal, bool alwaysLocal);
+extern HQL_API IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHqlExpression *sortList, bool isLocal, bool alwaysLocal, bool allowShuffle);
 
 extern HQL_API bool matchDedupDistribution(IHqlExpression * distn, const HqlExprArray & equalities);
 
 extern HQL_API bool appearsToBeSorted(ITypeInfo * type, bool isLocal, bool ignoreGrouping);
 extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
 extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * newSort, bool isLocal, bool ignoreGrouping);
-extern HQL_API IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal);
+extern HQL_API IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal, bool allowShuffle);
+
+extern HQL_API bool isWorthShuffling(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping);
+extern HQL_API bool isWorthShuffling(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
+extern HQL_API IHqlExpression * getShuffleSort(IHqlExpression * dataset, HqlExprArray & order, bool isLocal, bool ignoreGrouping, bool alwaysLocal);
+extern HQL_API IHqlExpression * getShuffleSort(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal);
+extern HQL_API IHqlExpression * convertShuffleToGroupedSort(IHqlExpression * expr);
 
 extern HQL_API bool reorderMatchExistingLocalSort(HqlExprArray & sortedLeft, HqlExprArray & reorderedRight, IHqlExpression * dataset, const HqlExprArray & left, const HqlExprArray & right);
 

ecl/hql/hqlopt.cpp

@@ -484,6 +484,7 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateUnsharedDataset(IHqlExpression
     case no_newusertable:
     case no_newaggregate:
     case no_sort:
+    case no_shuffle:
     case no_distribute:
     case no_keyeddistribute:
     case no_fetch:
@@ -515,6 +516,7 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateUnsharedDataset(IHqlExpression
     switch (op)
     {
     case no_sort:
+    case no_shuffle:
     case no_distribute:
     case no_keyeddistribute:
         noteUnused(expr);
@@ -615,6 +617,7 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateDataset(IHqlExpression * trans
                 next = ds->queryChild(0);
             break;
         case no_sort:
+        case no_shuffle:
         case no_sorted:
             //MORE: Allowed if the transform is commutative for no_aggregate
             if (aggOp != no_aggregate)
@@ -2831,6 +2834,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             case no_grouped:
             case no_keyeddistribute:
             case no_sort:
+            case no_shuffle:
             case no_preload:
             case no_assertsorted:
             case no_assertgrouped:
@@ -3110,6 +3114,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                     return swapNodeWithChild(transformed);
                 break;
             case no_sort:
+            case no_shuffle:
                 if (transformedCountProject)
                     break;
                 if (increasesRowSize(transformed))
@@ -3302,6 +3307,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                 return swapNodeWithChild(transformed);
             case no_distribute:
             case no_sort:
+            case no_shuffle:
                 if (increasesRowSize(transformed))
                     break;
                 return moveProjectionOverSimple(transformed, true, false);
@@ -3417,6 +3423,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             switch(child->getOperator())
             {
             case no_sort:
+            case no_shuffle:
                 if (!isLocalActivity(transformed) || isLocalActivity(child))
                     return removeChildNode(transformed);
                 break;
@@ -3429,6 +3436,33 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             }
             break;
         }
+    case no_shuffle:
+        {
+            switch(child->getOperator())
+            {
+            case no_sort:
+                {
+                    if (isGrouped(transformed))
+                        break;
+                    //Convert shuffle(sort) back into a single sort.  Do not convert if it would change the distribution.
+                    if (!isAlwaysLocal() && (!isLocalActivity(transformed) || !isLocalActivity(child)))
+                        break;
+                    OwnedHqlExpr sortOrder = getExistingSortOrder(transformed, true, true);
+                    //A weird user defined SHUFFLE could create an unknown sort order
+                    if (!sortOrder)
+                        break;
+                    OwnedHqlExpr newOrder = replaceSelector(sortOrder, queryActiveTableSelector(), child->queryNormalizedSelector());
+                    decUsage(child);
+                    DBGLOG("Optimizer: Merge %s and %s", queryNode0Text(transformed), queryNode1Text(child));
+                    return ::replaceChild(child, 1, newOrder);
+                }
+
+            case no_shuffle:
+                //This should almost certainly be improved, but it might be a bit tricky!
+                break;
+            }
+            break;
+        }
     case no_keyeddistribute:
     case no_distribute:
         {
@@ -3442,6 +3476,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             case no_distribute:
             case no_keyeddistribute:
             case no_sort:
+            case no_shuffle:
                 if (!transformed->hasProperty(mergeAtom))
                     return removeChildNode(transformed);
                 break;

ecl/hql/hqlopt.hpp

@@ -31,6 +31,7 @@ enum
     HOOfiltersharedproject      = 0x0020,
     HOOhascompoundaggregate     = 0x0040,
     HOOfoldconstantdatasets     = 0x0080,
+    HOOalwayslocal              = 0x0100,
 };
 
 extern HQL_API IHqlExpression * optimizeHqlExpression(IHqlExpression * expr, unsigned options);

ecl/hql/hqlopt.ipp

@@ -130,6 +130,8 @@ protected:
 
     IHqlExpression * getNoHoistAttr();
 
+    inline bool isAlwaysLocal() const { return (options & HOOalwayslocal) != 0; }
+
 protected:
     typedef OPTTRANSFORMBASE PARENT;
     unsigned options;

ecl/hql/hqlthql.cpp

@@ -121,6 +121,7 @@ private:
     bool isServiceDefined(IHqlExpression * expr);
     bool isExportDefined(IHqlExpression * expr);
     void doFunctionDefinition(StringBuffer & newdef, IHqlExpression * funcdef, const char * name, bool inType);
+    void sortlistToEcl(IHqlExpression *expr, StringBuffer &s, bool addCurleys, bool inType);
 
     bool matchesActiveDataset(IHqlExpression * expr);
     void pushScope(IHqlExpression * expr);
@@ -1057,19 +1058,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
         }
         case no_sortlist:
         {
-            bool needComma = false;
-            ForEachChild(idx, expr)
-            {
-                IHqlExpression * child = queryChild(expr, idx);
-                if (child && (expandProcessed || !isInternalAttribute(expr)))
-                {
-                    if (needComma) queryNewline(s.append(", "));
-                    if (queryAddDotDotDot(s, startLength))
-                        break;
-                    needComma = true;
-                    toECL(child, s, false, inType);
-                }
-            }
+            sortlistToEcl(expr, s, false, inType);
             break;
         }
         case no_rowvalue:
@@ -1281,6 +1270,31 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             }
             break;
         }
+        case no_shuffle:
+            {
+                s.append(getEclOpString(expr->getOperator()));
+                s.append('(');
+                if (!xgmmlGraphText)
+                {
+                    toECL(child0, s, false, inType);
+                    queryNewline(s.append(", "));
+                }
+                pushScope(child0);
+                //MORE: Sortlists should always be generated using the {} syntax - then this could be simplified
+                //NOTE: child(1) and child(2) are output in a different order from their representation
+                s.append("{");
+                toECL(expr->queryChild(2), s, false, inType);
+                s.append("}");
+                queryNewline(s.append(", "));
+                s.append("{");
+                toECL(expr->queryChild(1), s, false, inType);
+                s.append("}");
+                childrenToECL(expr, s, inType, true, 3);
+                popScope();
+                s.append(')');
+                break;
+            }
+
         case no_select:
         {
             if (!expandProcessed && 
@@ -2632,6 +2646,28 @@ void HqltHql::defaultChildrenToECL(IHqlExpression *expr, StringBuffer &s, bool i
     }
 }
 
+void HqltHql::sortlistToEcl(IHqlExpression *expr, StringBuffer &s, bool addCurleys, bool inType)
+{
+    unsigned startLength = s.length();
+    if (addCurleys)
+        s.append("{ ");
+    bool needComma = false;
+    ForEachChild(idx, expr)
+    {
+        IHqlExpression * child = queryChild(expr, idx);
+        if (child && (expandProcessed || !isInternalAttribute(expr)))
+        {
+            if (needComma) queryNewline(s.append(", "));
+            if (queryAddDotDotDot(s, startLength))
+                break;
+            needComma = true;
+            toECL(child, s, false, inType);
+        }
+    }
+    if (addCurleys)
+        s.append(" }");
+}
+
 StringBuffer &HqltHql::getFieldTypeString(IHqlExpression * e, StringBuffer &s)
 {
     ITypeInfo * type = e->queryType();

ecl/hql/hqlutil.cpp

@@ -7668,3 +7668,20 @@ IHqlExpression * normalizeAnyDatasetAliases(IHqlExpression * expr)
         return normalizeDatasetAlias(transformed);
     return transformed.getClear();
 }
+
+bool userPreventsSort(IHqlExpression * noSortAttr, node_operator side)
+{
+    if (!noSortAttr)
+        return false;
+
+    IHqlExpression * child = noSortAttr->queryChild(0);
+    if (!child)
+        return true;
+
+    _ATOM name = child->queryName();
+    if (side == no_left)
+        return name == leftAtom;
+    if (side == no_right)
+        return name == rightAtom;
+    throwUnexpected();
+}

ecl/hql/hqlutil.hpp

@@ -637,5 +637,6 @@ extern HQL_API IPropertyTree * createArchiveAttribute(IPropertyTree * module, co
 extern HQL_API IECLError * annotateExceptionWithLocation(IException * e, IHqlExpression * location);
 extern HQL_API IHqlExpression * convertAttributeToQuery(IHqlExpression * expr, HqlLookupContext & ctx);
 extern HQL_API StringBuffer & appendLocation(StringBuffer & s, IHqlExpression * location, const char * suffix = NULL);
+extern HQL_API bool userPreventsSort(IHqlExpression * noSortAttr, node_operator side);
 
 #endif

ecl/hqlcpp/hqlcpp.cpp

@@ -1672,6 +1672,14 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.createImplicitAliases,"createImplicitAliases", false),
         DebugOption(options.combineSiblingGraphs,"combineSiblingGraphs", true),
         DebugOption(options.optimizeSharedGraphInputs,"optimizeSharedGraphInputs", true),
+        DebugOption(options.supportsShuffleActivity,"supportsShuffleActivity",false),
+        DebugOption(options.implicitShuffle,"implicitShuffle",false),
+        DebugOption(options.implicitBuildIndexShuffle,"implicitBuildIndexShuffle",false),
+        DebugOption(options.implicitJoinShuffle,"implicitJoinShuffle",false),
+        DebugOption(options.implicitGroupShuffle,"implicitGroupShuffle",false),
+        DebugOption(options.implicitGroupHashAggregate,"implicitGroupHashAggregate",false),
+        DebugOption(options.implicitGroupHashDedup,"implicitGroupHashDedup",false),
+        DebugOption(options.shuffleLocalJoinConditions,"shuffleLocalJoinConditions",false),
     };
 
     //get options values from workunit
@@ -1818,10 +1826,10 @@ unsigned HqlCppTranslator::getOptimizeFlags() const
     switch (targetClusterType)
     {
     case RoxieCluster:
-        optFlags |= HOOnoclonelimit;
+        optFlags |= HOOnoclonelimit|HOOalwayslocal;
         break;
     case HThorCluster:
-        optFlags |= HOOnocloneindexlimit;
+        optFlags |= HOOnocloneindexlimit|HOOalwayslocal;
         break;
     case ThorCluster:
     case ThorLCRCluster:

ecl/hqlcpp/hqlcpp.ipp

@@ -701,6 +701,14 @@ struct HqlCppOptions
     bool                createImplicitAliases;
     bool                combineSiblingGraphs;
     bool                optimizeSharedGraphInputs;
+    bool                supportsShuffleActivity;  // Does the target engine support SHUFFLE?
+    bool                implicitShuffle;  // convert sort when partially sortted to shuffle (group,sort,ungroup)
+    bool                implicitBuildIndexShuffle;  // use shuffle when building indexes?
+    bool                implicitJoinShuffle;  // use shuffle for paritially sorted join inputs when possible
+    bool                implicitGroupShuffle;  // use shuffle if some sort conditions match when grouping
+    bool                implicitGroupHashAggregate;  // convert aggreate(sort(x,a),{..},a,d) to aggregate(group(sort(x,a),a_,{},d))
+    bool                implicitGroupHashDedup;
+    bool                shuffleLocalJoinConditions;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class
@@ -1485,6 +1493,7 @@ public:
     void doBuildAggregateMergeFunc(BuildCtx & ctx, IHqlExpression * expr, bool & requiresOrderedMerge);
     void doBuildAggregateProcessTransform(BuildCtx & ctx, BoundRow * selfRow, IHqlExpression * expr, IHqlExpression * alreadyDoneExpr);
 
+    void doBuildFuncIsSameGroup(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * sortlist);
 
     void processUserAggregateTransform(IHqlExpression * expr, IHqlExpression * transform, SharedHqlExpr & firstTransform, SharedHqlExpr & nextTransform);
     void doBuildUserAggregateFuncs(BuildCtx & ctx, IHqlExpression * expr, bool & requiresOrderedMerge);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -3184,7 +3184,7 @@ void HqlCppTranslator::doBuildFunction(BuildCtx & ctx, ITypeInfo * type, const c
             HqlExprArray parameters;
             OwnedHqlExpr entrypoint = createAttribute(entrypointAtom, createConstant(name));
             OwnedHqlExpr body = createValue(no_null, LINK(type), LINK(entrypoint));
-            OwnedHqlExpr formals = createValue(no_sortlist, makeSortListType(NULL), parameters);
+            OwnedHqlExpr formals = createSortList(parameters);
             OwnedHqlExpr attrs = createAttribute(virtualAtom);
             OwnedHqlExpr function = createFunctionDefinition(NULL, LINK(body), LINK(formals), NULL, LINK(attrs));
             funcctx.addFunction(function);
@@ -5656,6 +5656,9 @@ double HqlCppTranslator::getComplexity(IHqlExpression * expr, ClusterType cluste
         if (isThorCluster(cluster))
             complexity = 5;
         break;
+    case no_shuffle:
+        complexity = 1;
+        break;
     case no_sort:
         if (isLocalActivity(expr) || !isThorCluster(cluster))
             complexity = 2;
@@ -6118,6 +6121,19 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
                 result = doBuildActivityProcess(ctx, expr);
                 break;
             case no_group:
+                //Special case group(shuffle) which will be mapped to group(group(sort(group))) to remove
+                //the redundant group
+                if (!options.supportsShuffleActivity && (expr->queryChild(0)->getOperator() == no_shuffle))
+                {
+                    IHqlExpression * shuffle = expr->queryChild(0);
+                    OwnedHqlExpr groupedSort = convertShuffleToGroupedSort(shuffle);
+                    assertex(groupedSort->getOperator() == no_group);
+                    OwnedHqlExpr newGroup = replaceChild(expr, 0, groupedSort->queryChild(0));
+                    result = doBuildActivityGroup(ctx, newGroup);
+                }
+                else
+                    result = doBuildActivityGroup(ctx, expr);
+                break;
             case no_cogroup:
             case no_assertgrouped:
                 result = doBuildActivityGroup(ctx, expr);
@@ -6171,6 +6187,15 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_sub:
                 result = doBuildActivitySub(ctx, expr);
                 break;
+            case no_shuffle:
+                if (!options.supportsShuffleActivity)
+                {
+                    OwnedHqlExpr groupedSort = convertShuffleToGroupedSort(expr);
+                    result = buildCachedActivity(ctx, groupedSort);
+                }
+                else
+                    result = doBuildActivitySort(ctx, expr);
+                break;
             case no_sort:
             case no_cosort:
             case no_topn:
@@ -10742,19 +10767,8 @@ void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx,
 
     assertex(dataset.querySide() == no_activetable);
     bool noNeedToSort = canRemoveSort && isAlreadySorted(dataset.queryDataset(), sorts, canRemoveSort, true);
-    if (noSortAttr)
-    {
-        IHqlExpression * child = noSortAttr->queryChild(0);
-        if (!child)
-            noNeedToSort = true;
-        else
-        {
-            if (side == no_left)
-                noNeedToSort = child->queryName() == leftAtom;
-            else if (side == no_left)
-                noNeedToSort = child->queryName() == rightAtom;
-        }
-    }
+    if (userPreventsSort(noSortAttr, side))
+        noNeedToSort = true;
     
     if (noNeedToSort || isLightweight)
     {
@@ -12967,7 +12981,7 @@ void optimizeGroupOrder(HqlExprArray & optimized, IHqlExpression * dataset, HqlE
 
 IHqlExpression * createOrderFromCompareArray(HqlExprArray & exprs, IHqlExpression * dataset, IHqlExpression * left, IHqlExpression * right)
 {
-    OwnedHqlExpr equalExpr = createValue(no_sortlist, makeSortListType(NULL), exprs);
+    OwnedHqlExpr equalExpr = createSortList(exprs);
     OwnedHqlExpr lhs = replaceSelector(equalExpr, dataset, left);
     OwnedHqlExpr rhs = replaceSelector(equalExpr, dataset, right);
     return createValue(no_order, LINK(signedType), lhs.getClear(), rhs.getClear());
@@ -13133,7 +13147,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDedup(BuildCtx & ctx, IHqlExpr
         {
             if (info.equalities.ordinality() && !instance->isGrouped)
             {
-                OwnedHqlExpr order = createValue(no_sortlist, makeSortListType(NULL), info.equalities);
+                OwnedHqlExpr order = createValueSafe(no_sortlist, makeSortListType(NULL), info.equalities);
                 buildCompareMember(instance->nestedctx, "ComparePrimary", order, DatasetReference(dataset));
                 if (!targetThor())
                     equalities = &noEqualities;
@@ -14904,7 +14918,7 @@ IHqlExpression * HqlCppTranslator::createOrderFromSortList(const DatasetReferenc
         }
     }
 
-    return createValue(no_order, LINK(signedType), createValue(no_sortlist, makeSortListType(NULL), leftList), createValue(no_sortlist, makeSortListType(NULL), rightList));
+    return createValue(no_order, LINK(signedType), createSortList(leftList), createSortList(rightList));
 }
 
 
@@ -14921,6 +14935,97 @@ void HqlCppTranslator::buildReturnOrder(BuildCtx & ctx, IHqlExpression *sortList
     doBuildReturnCompare(ctx, order, no_order, false);
 }
 
+void HqlCppTranslator::doBuildFuncIsSameGroup(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * sortlist)
+{
+    BuildCtx funcctx(ctx);
+    funcctx.addQuotedCompound("virtual bool isSameGroup(const void * _left, const void * _right)");
+    if (sortlist->getOperator() == no_activetable)
+        buildReturn(funcctx, queryBoolExpr(false));
+    else
+    {
+        funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
+        funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
+
+        OwnedHqlExpr selSeq = createSelectorSequence();
+        OwnedHqlExpr leftSelect = createSelector(no_left, dataset, selSeq);
+        OwnedHqlExpr rightSelect = createSelector(no_right, dataset, selSeq);
+        HqlExprArray args;
+        HqlExprArray leftValues, rightValues;
+        HqlExprArray compares;
+        unwindChildren(compares, sortlist);
+
+        //Optimize the grouping conditions by ordering them by the fields in the record (so the
+        //doBuildReturnCompare() can combine as many as possible),  and remove duplicates
+        if (options.optimizeGrouping && (compares.ordinality() > 1))
+        {
+            HqlExprArray equalities;
+            optimizeGroupOrder(equalities, dataset, compares);
+            ForEachItemIn(i, equalities)
+            {
+                IHqlExpression * test = &equalities.item(i);
+                leftValues.append(*replaceSelector(test, dataset, leftSelect));
+                rightValues.append(*replaceSelector(test, dataset, rightSelect));
+            }
+        }
+
+        ForEachItemIn(idx, compares)
+        {
+            IHqlExpression * test = &compares.item(idx);
+            if (containsSelector(test, leftSelect) || containsSelector(test, rightSelect))
+                args.append(*LINK(test));
+            else
+            {
+                OwnedHqlExpr lhs = replaceSelector(test, dataset, leftSelect);
+                OwnedHqlExpr rhs = replaceSelector(test, dataset, rightSelect);
+                if (lhs != rhs)
+                {
+                    leftValues.append(*lhs.getClear());
+                    rightValues.append(*rhs.getClear());
+                }
+            }
+        }
+
+        OwnedHqlExpr result;
+        OwnedHqlExpr orderResult;
+        //Use the optimized equality code for more than one element - which often combines the comparisons.
+        if (leftValues.ordinality() != 0)
+        {
+            if (leftValues.ordinality() == 1)
+                args.append(*createValue(no_eq, makeBoolType(), LINK(&leftValues.item(0)), LINK(&rightValues.item(0))));
+            else
+                orderResult.setown(createValue(no_order, LINK(signedType), createSortList(leftValues), createSortList(rightValues)));
+        }
+
+        if (args.ordinality() == 1)
+            result.set(&args.item(0));
+        else if (args.ordinality() != 0)
+            result.setown(createValue(no_and, makeBoolType(), args));
+
+        bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
+        bindTableCursor(funcctx, dataset, "right", no_right, selSeq);
+        IHqlExpression * trueExpr = queryBoolExpr(true);
+        if (result)
+        {
+            if (orderResult)
+            {
+                buildFilteredReturn(funcctx, result, trueExpr);
+                doBuildReturnCompare(funcctx, orderResult, no_eq, true);
+            }
+            else
+            {
+                buildReturn(funcctx, result);
+            }
+        }
+        else
+        {
+            if (orderResult)
+                doBuildReturnCompare(funcctx, orderResult, no_eq, true);
+            else
+                buildReturn(funcctx, trueExpr);
+        }
+    }
+}
+
 ABoundActivity * HqlCppTranslator::doBuildActivityGroup(BuildCtx & ctx, IHqlExpression * expr)
 {
     IHqlExpression * dataset = expr->queryChild(0);
@@ -14956,93 +15061,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGroup(BuildCtx & ctx, IHqlExpr
         buildInstancePrefix(instance);
 
         //virtual bool isSameGroup(const void *left, const void *right);
-        BuildCtx funcctx(instance->startctx);
-        funcctx.addQuotedCompound("virtual bool isSameGroup(const void * _left, const void * _right)");
-        if (sortlist->getOperator() == no_activetable)
-            buildReturn(funcctx, queryBoolExpr(false));
-        else
-        {
-            funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
-            funcctx.addQuoted("const unsigned char * right = (const unsigned char *) _right;");
-
-            OwnedHqlExpr selSeq = createSelectorSequence();
-            OwnedHqlExpr leftSelect = createSelector(no_left, dataset, selSeq);
-            OwnedHqlExpr rightSelect = createSelector(no_right, dataset, selSeq);
-            HqlExprArray args;
-            HqlExprArray leftValues, rightValues;
-            HqlExprArray compares;
-            unwindChildren(compares, sortlist);
-
-            //Optimize the grouping conditions by ordering them by the fields in the record (so the
-            //doBuildReturnCompare() can combine as many as possible),  and remove duplicates
-            if (options.optimizeGrouping && (compares.ordinality() > 1))
-            {
-                HqlExprArray equalities;
-                optimizeGroupOrder(equalities, dataset, compares);
-                ForEachItemIn(i, equalities)
-                {
-                    IHqlExpression * test = &equalities.item(i);
-                    leftValues.append(*replaceSelector(test, dataset, leftSelect));
-                    rightValues.append(*replaceSelector(test, dataset, rightSelect));
-                }
-            }
-
-            ForEachItemIn(idx, compares)
-            {
-                IHqlExpression * test = &compares.item(idx);
-                if (containsSelector(test, leftSelect) || containsSelector(test, rightSelect))
-                    args.append(*LINK(test));
-                else
-                {
-                    OwnedHqlExpr lhs = replaceSelector(test, dataset, leftSelect);
-                    OwnedHqlExpr rhs = replaceSelector(test, dataset, rightSelect);
-                    if (lhs != rhs)
-                    {
-                        leftValues.append(*lhs.getClear());
-                        rightValues.append(*rhs.getClear());
-                    }
-                }
-            }
-
-            OwnedHqlExpr result;
-            OwnedHqlExpr orderResult;
-            //Use the optimized equality code for more than one element - which often combines the comparisons.
-            if (leftValues.ordinality() != 0)
-            {
-                if (leftValues.ordinality() == 1)
-                    args.append(*createValue(no_eq, makeBoolType(), LINK(&leftValues.item(0)), LINK(&rightValues.item(0))));
-                else
-                    orderResult.setown(createValue(no_order, LINK(signedType), createValue(no_sortlist, makeSortListType(NULL), leftValues), createValue(no_sortlist, makeSortListType(NULL), rightValues)));
-            }
-
-            if (args.ordinality() == 1)
-                result.set(&args.item(0));
-            else if (args.ordinality() != 0)
-                result.setown(createValue(no_and, makeBoolType(), args));
-
-            bindTableCursor(funcctx, dataset, "left", no_left, selSeq);
-            bindTableCursor(funcctx, dataset, "right", no_right, selSeq);
-            IHqlExpression * trueExpr = queryBoolExpr(true);
-            if (result)
-            {
-                if (orderResult)
-                {
-                    buildFilteredReturn(funcctx, result, trueExpr);
-                    doBuildReturnCompare(funcctx, orderResult, no_eq, true);
-                }
-                else
-                {
-                    buildReturn(funcctx, result);
-                }
-            }
-            else
-            {
-                if (orderResult)
-                    doBuildReturnCompare(funcctx, orderResult, no_eq, true);
-                else
-                    buildReturn(funcctx, trueExpr);
-            }
-        }
+        doBuildFuncIsSameGroup(instance->startctx, dataset, sortlist);
 
         buildInstanceSuffix(instance);
 
@@ -15303,6 +15322,10 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
             helper = "Sort";
             break;
         }
+    case no_shuffle:
+        actKind = TAKshuffle;
+        helper = "Shuffle";
+        break;
     default:
         {
             cosort = expr->queryChild(2);
@@ -15357,6 +15380,9 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
 
     buildSkewThresholdMembers(instance->classctx, expr);
 
+    if (expr->getOperator() == no_shuffle)
+        doBuildFuncIsSameGroup(instance->startctx, dataset, expr->queryChild(2));
+
     if (limit)
     {
         OwnedHqlExpr newLimit = ensurePositiveOrZeroInt64(limit);
@@ -15382,7 +15408,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
                 maxValues.replace(*LINK(cur.queryChild(0)), i);
             }
         }
-        OwnedHqlExpr order = createValue(no_order, LINK(signedType), createValue(no_sortlist, makeSortListType(NULL), sortValues), createValue(no_sortlist, makeSortListType(NULL), maxValues));
+        OwnedHqlExpr order = createValue(no_order, LINK(signedType), createSortList(sortValues), createSortList(maxValues));
 
         BuildCtx funcctx(instance->startctx);
         funcctx.addQuotedCompound("virtual int compareBest(const void * _self)");

ecl/hqlcpp/hqliproj.cpp

@@ -188,9 +188,14 @@ static unsigned getActivityCost(IHqlExpression * expr, ClusterType targetCluster
             switch (expr->getOperator())
             {
             case no_sort:
+                //MORE: What about checking for grouped!
                 if (!expr->hasProperty(localAtom))
                     return CostNetworkCopy;
                 return CostManyCopy;
+            case no_shuffle:
+                if (!expr->hasProperty(localAtom) && !isGrouped(expr))
+                    return CostNetworkCopy;
+                break;
             case no_group:
                 if (!expr->hasProperty(localAtom))
                     return CostNetworkGroup;

ecl/hqlcpp/hqllib.cpp

@@ -262,7 +262,7 @@ protected:
                     throwUnexpected();
                     HqlExprArray parms;
                     unwindChildren(parms, oldSymbol, 1);
-                    IHqlExpression * formals = createValue(no_sortlist, makeSortListType(NULL), parms);
+                    IHqlExpression * formals = createSortList(parms);
                     newValue.setown(createFunctionDefinition(name, newValue.getClear(), formals, NULL, NULL));
                 }
 

ecl/hqlcpp/hqlresource.cpp

@@ -142,6 +142,7 @@ void getResources(IHqlExpression * expr, CResources & resources, const CResource
         if (!isGrouped && !isLocal)
         {
             resources.set(RESslavememory, MEM_Const_Minimal+DEDUP_SMART_BUFFER_SIZE);
+            //MORE: Is this still correct?
             resources.setManyToMasterSockets(1);
         }
         break;
@@ -150,6 +151,12 @@ void getResources(IHqlExpression * expr, CResources & resources, const CResource
         resources.setLightweight();
         setHashResources(expr, resources, options);
         break;
+    case no_shuffle:
+        if (expr->hasProperty(manyAtom))
+            resources.setHeavyweight();
+        else
+            resources.setLightweight();
+        break;
     case no_sort:
         if (isGrouped)
         {

ecl/hqlcpp/hqlsource.cpp

@@ -2051,10 +2051,10 @@ void SourceBuilder::buildGroupAggregateCompareHelper(ParentExtract * extractBuil
         }
     }
 
-    OwnedHqlExpr leftList = createValue(no_sortlist, makeSortListType(NULL), optimizedLeft);
+    OwnedHqlExpr leftList = createSortList(optimizedLeft);
     DatasetReference datasetRight(aggregate, no_activetable, NULL);
     OwnedHqlExpr selSeq = createDummySelectorSequence();
-    OwnedHqlExpr rightList = createValue(no_sortlist, makeSortListType(NULL), optimizedRight);
+    OwnedHqlExpr rightList = createSortList(optimizedRight);
     OwnedHqlExpr rightSelect = datasetRight.getSelector(no_right, selSeq);
     OwnedHqlExpr rightResolved = datasetRight.mapCompound(rightList, rightSelect);
 

ecl/hqlcpp/hqlstep.cpp

@@ -931,7 +931,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityNWayMergeJoin(BuildCtx & ctx,
     {
         HqlExprArray skewArgs;
         unwindChildren(skewArgs, skew);
-        OwnedHqlExpr skewOrder = createValue(no_sortlist, makeSortListType(NULL), skewArgs);
+        OwnedHqlExpr skewOrder = createSortList(skewArgs);
         DatasetReference datasetRef(dataset);
         buildCompareEqMember(instance->classctx, "PartitionCompareEq", skewOrder, leftRef);
     }

ecl/hqlcpp/hqlttcpp.cpp

@@ -1718,12 +1718,12 @@ static IHqlExpression * simplifySortlistComplexity(IHqlExpression * sortlist)
             same = false;
     }
     if (!same)
-        return createValue(no_sortlist, makeSortListType(NULL), cpts);
+        return createSortList(cpts);
 
     return NULL;
 }
 
-static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortIndexPayload, bool alwaysLocal)
+static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortIndexPayload, bool alwaysLocal, bool allowImplicitShuffle)
 {
     LinkedHqlExpr dataset = expr->queryChild(0);
     IHqlExpression * normalizedDs = dataset->queryNormalizedSelector();
@@ -1736,7 +1736,7 @@ static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortInde
     HqlExprArray sorts;
     gatherIndexBuildSortOrder(sorts, expr, sortIndexPayload);
 
-    OwnedHqlExpr sortOrder = createValue(no_sortlist, makeSortListType(NULL), sorts);
+    OwnedHqlExpr sortOrder = createSortList(sorts);
     OwnedHqlExpr newsort = simplifySortlistComplexity(sortOrder);
     if (!newsort)
         newsort.set(sortOrder);
@@ -1844,7 +1844,7 @@ static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortInde
             }
         }
 
-        OwnedHqlExpr sorted = ensureSorted(dataset, newsort, expr->hasProperty(localAtom), true, alwaysLocal);
+        OwnedHqlExpr sorted = ensureSorted(dataset, newsort, expr->hasProperty(localAtom), true, alwaysLocal, allowImplicitShuffle);
         if (sorted == dataset)
             return NULL;
 
@@ -1914,6 +1914,9 @@ IHqlExpression * ThorHqlTransformer::createTransformed(IHqlExpression * expr)
     case no_assertsorted:
         normalized = normalizeSort(transformed);
         break;
+    case no_shuffle:
+        normalized = normalizeShuffle(transformed);
+        break;
     case no_cogroup:
         normalized = normalizeCoGroup(transformed);
         break;
@@ -2094,7 +2097,7 @@ IHqlExpression * ThorHqlTransformer::normalizeDedup(IHqlExpression * expr)
             if (hasLocal && translator.targetThor())
             {
                 HqlExprArray dedupArgs;
-                dedupArgs.append(*ensureSortedForGroup(dataset, groupOrder, true, false));
+                dedupArgs.append(*ensureSortedForGroup(dataset, groupOrder, true, false, options.implicitGroupShuffle));
                 unwindChildren(dedupArgs, expr, 1);
                 removeProperty(dedupArgs, allAtom);
                 return expr->clone(dedupArgs);
@@ -2271,7 +2274,7 @@ IHqlExpression * ThorHqlTransformer::normalizeGroup(IHqlExpression * expr)
 
     //First check to see if the dataset is already sorted by the group criteria, or more.
     //The the data could be globally sorted, but not distributed, and this is likely to be more efficient than redistributing...
-    OwnedHqlExpr sorted = ensureSortedForGroup(dataset, sortlist, hasLocal, !translator.targetThor());
+    OwnedHqlExpr sorted = ensureSortedForGroup(dataset, sortlist, hasLocal, !translator.targetThor(), options.implicitGroupShuffle);
     if (sorted == dataset)
         return removeProperty(expr, allAtom);
     sorted.setown(cloneInheritedAnnotations(expr, sorted));
@@ -2409,7 +2412,7 @@ IHqlExpression * ThorHqlTransformer::normalizeCoGroup(IHqlExpression * expr)
         {
             IHqlExpression & cur = inputs.item(i);
             OwnedHqlExpr mappedOrder = replaceSelector(bestSortOrder, queryActiveTableSelector(), &cur);
-            sortedInputs.append(*ensureSorted(&cur, mappedOrder, true, true, alwaysLocal));
+            sortedInputs.append(*ensureSorted(&cur, mappedOrder, true, true, alwaysLocal, options.implicitShuffle));
         }
         HqlExprArray sortedArgs;
         unwindChildren(sortedArgs, bestSortOrder);
@@ -2434,7 +2437,7 @@ IHqlExpression * ThorHqlTransformer::normalizeCoGroup(IHqlExpression * expr)
     return expr->cloneAllAnnotations(grouped);
 }
 
-static IHqlExpression * getNonThorSortedJoinInput(IHqlExpression * joinExpr, IHqlExpression * dataset, HqlExprArray & sorts)
+static IHqlExpression * getNonThorSortedJoinInput(IHqlExpression * joinExpr, IHqlExpression * dataset, HqlExprArray & sorts, bool implicitShuffle)
 {
     if (!sorts.length())
         return LINK(dataset);
@@ -2451,7 +2454,7 @@ static IHqlExpression * getNonThorSortedJoinInput(IHqlExpression * joinExpr, IHq
     groupOrder.setown(replaceSelector(groupOrder, queryActiveTableSelector(), expr->queryNormalizedSelector()));
 
     //not used for thor, so sort can be local
-    OwnedHqlExpr table = ensureSorted(expr, groupOrder, false, true, true);
+    OwnedHqlExpr table = ensureSorted(expr, groupOrder, false, true, true, implicitShuffle);
     if (table != expr)
         table.setown(cloneInheritedAnnotations(joinExpr, table));
 
@@ -2470,6 +2473,28 @@ static bool sameOrGrouped(IHqlExpression * newLeft, IHqlExpression * oldLeft)
     return (newLeft->queryBody() == oldLeft->queryBody());
 }
 
+bool canReorderMatchExistingLocalSort(HqlExprArray & newElements1, HqlExprArray & newElements2, IHqlExpression * ds1, Shared<IHqlExpression> & ds2, const HqlExprArray & elements1, HqlExprArray & elements2, bool canShuffle, bool isLocal, bool alwaysLocal)
+{
+    newElements1.kill();
+    newElements2.kill();
+    if (reorderMatchExistingLocalSort(newElements1, newElements2, ds1, elements1, elements2))
+    {
+        if (isAlreadySorted(ds2, newElements2, isLocal||alwaysLocal, true))
+            return true;
+
+        if (canShuffle && isWorthShuffling(ds2, newElements2, isLocal||alwaysLocal, true))
+        {
+            OwnedHqlExpr shuffled = getShuffleSort(ds2, newElements2, isLocal, true, alwaysLocal);
+            if (shuffled)
+            {
+                ds2.swap(shuffled);
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
 
 IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression * expr)
 {
@@ -2491,7 +2516,8 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
     }
 
     bool hasLocal = isLocalActivity(expr);
-    bool isLocal = hasLocal || !translator.targetThor();
+    bool alwaysLocal = !translator.targetThor();
+    bool isLocal = hasLocal || alwaysLocal;
     //hash,local doesn't make sense (hash is only used for distribution) => remove hash
     //but also prevent it being converted to a lookup join??
     if (isLocal && expr->hasProperty(hashAtom))
@@ -2606,14 +2632,25 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
         //Check for a local join where we can reorder the condition so both sides match the existing sort orders.
         //could special case self-join to do less work, but probably not worth the effort.
         HqlExprArray sortedLeft, sortedRight;
-        if (!isLimitedSubstringJoin && reorderMatchExistingLocalSort(sortedLeft, sortedRight, leftDs, leftSorts, rightSorts))
+        if (!isLimitedSubstringJoin)
         {
-            if (isAlreadySorted(rightDs, sortedRight, true, true))
+            //Since the distribution and order of global joins is not defined this could probably be used for non-local as well.
+            LinkedHqlExpr newLeftDs = leftDs;
+            LinkedHqlExpr newRightDs = rightDs;
+            bool canShuffle = options.shuffleLocalJoinConditions;
+            bool reordered = canReorderMatchExistingLocalSort(sortedLeft, sortedRight, newLeftDs, newRightDs,
+                                                              leftSorts, rightSorts, canShuffle, isLocal, alwaysLocal);
+            //If allowed to shuffle then try the otherway around
+            if (!reordered && canShuffle)
+                reordered = canReorderMatchExistingLocalSort(sortedRight, sortedLeft, newRightDs, newLeftDs,
+                                                             rightSorts, leftSorts, canShuffle, isLocal, alwaysLocal);
+
+            if (reordered)
             {
                 //Recreate the join condition in the correct order to match the existing sorts...
                 HqlExprAttr newcond;
-                OwnedHqlExpr leftSelector = createSelector(no_left, leftDs, seq);
-                OwnedHqlExpr rightSelector = createSelector(no_right, rightDs, seq);
+                OwnedHqlExpr leftSelector = createSelector(no_left, newLeftDs, seq);
+                OwnedHqlExpr rightSelector = createSelector(no_right, newRightDs, seq);
                 ForEachItemIn(i, sortedLeft)
                 {
                     OwnedHqlExpr lc = replaceSelector(&sortedLeft.item(i), queryActiveTableSelector(), leftSelector);
@@ -2622,8 +2659,10 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
                 }
                 extendConditionOwn(newcond, no_and, fuzzyMatch.getClear());
                 HqlExprArray args;
-                unwindChildren(args, expr);
-                args.replace(*newcond.getClear(), 2);
+                args.append(*newLeftDs.getClear());
+                args.append(*newRightDs.getClear());
+                args.append(*newcond.getClear());
+                unwindChildren(args, expr, 3);
                 args.append(*createAttribute(_lightweight_Atom));
                 return expr->clone(args);
             }
@@ -2641,9 +2680,6 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
                     createLookup = !expr->hasProperty(_lightweight_Atom);
         }
 
-        OwnedHqlExpr newLeft = getNonThorSortedJoinInput(expr, leftDs, leftSorts);
-        OwnedHqlExpr newRight = getNonThorSortedJoinInput(expr, rightDs, rightSorts);
-
         if (isLimitedSubstringJoin)
             createLookup = false;           //doesn't support it yet
         else if (createLookup && leftSorts.ordinality() && rightSorts.ordinality())
@@ -2670,6 +2706,8 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
             return expr->clone(args);
         }
 
+        OwnedHqlExpr newLeft = getNonThorSortedJoinInput(expr, leftDs, leftSorts, options.implicitShuffle);
+        OwnedHqlExpr newRight = getNonThorSortedJoinInput(expr, rightDs, rightSorts, options.implicitShuffle);
         try
         {
             if ((leftDs != newLeft) || (rightDs != newRight))
@@ -2726,6 +2764,32 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
         }
     }
 
+    if (isThorCluster(targetClusterType) && isLocal && options.implicitJoinShuffle)
+    {
+        IHqlExpression * noSortAttr = expr->queryProperty(noSortAtom);
+        OwnedHqlExpr newLeft;
+        OwnedHqlExpr newRight;
+        if (!userPreventsSort(noSortAttr, no_left))
+            newLeft.setown(getShuffleSort(leftDs, leftSorts, isLocal, true, alwaysLocal));
+        if (!userPreventsSort(noSortAttr, no_right))
+            newRight.setown(getShuffleSort(rightDs, rightSorts, isLocal, true, alwaysLocal));
+        if (newLeft || newRight)
+        {
+            HqlExprArray args;
+            if (newLeft)
+                args.append(*newLeft.getClear());
+            else
+                args.append(*LINK(leftDs));
+            if (newRight)
+                args.append(*newRight.getClear());
+            else
+                args.append(*LINK(rightDs));
+            unwindChildren(args, expr, 2);
+            return expr->clone(args);
+        }
+    }
+
+
     return NULL;
 }
 
@@ -2818,11 +2882,45 @@ IHqlExpression * ThorHqlTransformer::normalizeSort(IHqlExpression * expr)
             return normalized;
     }
 
-    bool isLocal = !translator.targetThor() || expr->hasProperty(localAtom);
-    if ((op != no_assertsorted) && isAlreadySorted(dataset, sortlist, isLocal, false))
+    bool isLocal = expr->hasProperty(localAtom);
+    bool alwaysLocal = !translator.targetThor();
+    if ((op != no_assertsorted) && isAlreadySorted(dataset, sortlist, isLocal||alwaysLocal, false))
         return LINK(dataset);
     if (op == no_sorted)
         return normalizeSortSteppedIndex(expr, sortedAtom);
+
+    //NOTE: We can't convert a global sort to a shuffle because that will change the distribution
+    if (options.implicitShuffle && (isLocal || alwaysLocal) && (op != no_assertsorted))
+    {
+        OwnedHqlExpr shuffled = getShuffleSort(dataset, sortlist, isLocal, false, alwaysLocal);
+        if (shuffled)
+            return dataset->cloneAllAnnotations(shuffled);
+    }
+    return NULL;
+}
+
+
+IHqlExpression * ThorHqlTransformer::normalizeShuffle(IHqlExpression * expr)
+{
+    IHqlExpression * dataset = expr->queryChild(0);
+    IHqlExpression * sortlist = expr->queryChild(1);
+    IHqlExpression * grouping = expr->queryChild(2);
+    OwnedHqlExpr newsort = simplifySortlistComplexity(sortlist);
+    OwnedHqlExpr newgrouping = simplifySortlistComplexity(grouping);
+    if (newsort || newgrouping)
+    {
+        HqlExprArray args;
+        unwindChildren(args, expr);
+        if (newsort)
+            args.replace(*newsort.getClear(), 1);
+        if (newgrouping)
+            args.replace(*newgrouping.getClear(), 2);
+        return expr->clone(args);
+    }
+
+    if (translator.targetThor() && !expr->hasProperty(localAtom))
+        return convertShuffleToGroupedSort(expr);
+
     return NULL;
 }
 
@@ -3203,7 +3301,7 @@ IHqlExpression * ThorHqlTransformer::normalizeTableToAggregate(IHqlExpression *
             }
             newGroupElement.append(*LINK(curGroup));
         }
-        newGroupBy = createValue(no_sortlist, makeSortListType(NULL), newGroupElement);
+        newGroupBy = createSortList(newGroupElement);
     }
 
     IHqlExpression * aggregateRecord = extraSelectNeeded ? translator.createRecordInheritMaxLength(aggregateFields, record) : LINK(record);
@@ -3276,7 +3374,7 @@ IHqlExpression * ThorHqlTransformer::normalizeTableGrouping(IHqlExpression * exp
                     ds.setown(createDataset(no_group, ds.getClear(), NULL));
                     ds.setown(cloneInheritedAnnotations(expr, ds));
                 }
-                OwnedHqlExpr sorted = ensureSortedForGroup(ds, newsort, expr->hasProperty(localAtom), !translator.targetThor());
+                OwnedHqlExpr sorted = ensureSortedForGroup(ds, newsort, expr->hasProperty(localAtom), !translator.targetThor(), options.implicitGroupShuffle);
 
                 //For thor a global grouped aggregate would transfer elements between nodes so it is still likely to
                 //be more efficient to do a hash aggregate.  Even better would be to check the distribution
@@ -4269,6 +4367,7 @@ static IHqlExpression * queryNormalizedAggregateParameter(IHqlExpression * expr)
                 return expr;
             break;
         case no_sort:
+        case no_shuffle:
         case no_distribute:
             break;
         default:
@@ -9705,6 +9804,7 @@ HqlTreeNormalizer::HqlTreeNormalizer(HqlCppTranslator & _translator) : NewHqlTra
     options.outputRowsAsDatasets = translator.targetRoxie();
     options.constantFoldNormalize = translatorOptions.constantFoldNormalize;
     options.allowActivityForKeyedJoin = translatorOptions.allowActivityForKeyedJoin;
+    options.implicitShuffle = translatorOptions.implicitBuildIndexShuffle;
     errors = translator.queryErrors();
     nextSequenceValue = 1;
 }
@@ -11360,7 +11460,7 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
             OwnedHqlExpr transformed = Parent::createTransformed(expr);
             loop
             {
-                IHqlExpression * ret = normalizeIndexBuild(transformed, options.sortIndexPayload, !translator.targetThor());
+                IHqlExpression * ret = normalizeIndexBuild(transformed, options.sortIndexPayload, !translator.targetThor(), options.implicitShuffle);
                 if (!ret)
                     return LINK(transformed);
                 transformed.setown(ret);

ecl/hqlcpp/hqlttcpp.ipp

@@ -202,6 +202,7 @@ protected:
     IHqlExpression * normalizeRollup(IHqlExpression * expr);
     IHqlExpression * normalizeScalarAggregate(IHqlExpression * expr);
     IHqlExpression * normalizeSelect(IHqlExpression * expr);
+    IHqlExpression * normalizeShuffle(IHqlExpression * expr);
     IHqlExpression * normalizeSort(IHqlExpression * expr);
     IHqlExpression * normalizeSortSteppedIndex(IHqlExpression * expr, _ATOM attrName);
     IHqlExpression * normalizeTempTable(IHqlExpression * expr);
@@ -1135,6 +1136,7 @@ protected:
         bool outputRowsAsDatasets;
         bool constantFoldNormalize;
         bool allowActivityForKeyedJoin;
+        bool implicitShuffle;
     } options;
     unsigned nextSequenceValue;
     bool seenForceLocal;

ecl/regress/selfjoin6.ecl

@@ -0,0 +1,39 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        surname := '?????????????';
+string10        forename := '?????????????';
+integer2        age := 25;
+            END;
+
+namesTable := dataset([
+        {'Smithe','Pru',10},
+        {'Hawthorn','Gavin',31},
+        {'Hawthorn','Mia',30},
+        {'Smith','Jo'},
+        {'Smith','Matthew'},
+        {'X','Z'}], namesRecord);
+
+sorted1 := NOFOLD(SORT(NOFOLD(namesTable), surname, age));
+
+Joined := join (sorted1, sorted1, LEFT.surname = RIGHT.surname AND LEFT.forename = RIGHT.forename, TRANSFORM(LEFT));
+output(Joined);

ecl/regress/shuffle.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+s2 := shuffle(s1, {d},{a,b},local); // sort and grouping are round the wrong way!
+s3 := sorted(s2, {a,b,d}, assert);
+output(s3);

ecl/regress/shuffle2.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+s2 := shuffle(nofold(s1), {a,b}, {d});
+s3 := sorted(nofold(s2), {a,b,d}, assert);
+output(s3);

ecl/regress/shuffle3.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+s2 := shuffle(s1, {a,b}, {d}, local);
+s3 := sorted(s2, {a,b,d}, assert);
+output(s3);

ecl/regress/sortgroup.ecl

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+g1 := group(s1, a, d);
+s2 := sort(g1, c, d);
+g2 := group(s2);
+// sort order of g2 should be a, <unknown>
+s3 := sort(g2, a, c, d);        // invalid to be optimized away
+output(s3);

ecl/regress/sortgroup2.ecl

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+g1 := group(s1, a, d);
+s2 := having(g1, count(rows(left)) < 100);
+g2 := group(s2);
+// sort order of g2 should be a, <unknown>
+s3 := sort(g2, a, b, c);        // should be optimized away
+output(s3);

ecl/regress/sortgroup3.ecl

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+g1 := group(s1, a, c);
+s2 := sort(g1, c, d);
+g2 := group(s2);
+// sort order of g2 should be a, <unknown>
+s3 := sort(g2, a, b, c, d);        // should be optimized away
+output(s3);

ecl/regress/sortgroup4.ecl

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+namesRecord :=
+            RECORD
+string20        a;
+string10        b;
+integer2        c;
+integer2        d;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+s1 := sort(namesTable, a, b, c);
+g1 := group(s1, a, c);
+s2 := having(g1, count(rows(left)) < 100);
+g2 := group(s2);
+// sort order of g2 should be a, <unknown>
+s3 := sort(g2, a, b, c);        // should be optimized away
+output(s3);

rtl/include/eclhelper.hpp

@@ -803,7 +803,7 @@ enum ThorActivityKind
     TAKcaseaction,
     TAKwhen_dataset,
     TAKwhen_action,
-        TAKunused2,
+    TAKshuffle,
     TAKindexgroupexists,
     TAKindexgroupcount,
     TAKhashdistributemerge,
@@ -957,6 +957,7 @@ enum ActivityInterfaceEnum
     TAIpipethrougharg_2,
     TAIpipewritearg_2,
     TAItemptablearg_2,
+    TAIshuffleextra_1,
 
 //Should remain as last of all meaningful tags, but before aliases
     TAImax,
@@ -1514,6 +1515,16 @@ struct IHThorTopNArg : public IHThorSortArg, public IHThorTopNExtra
     COMMON_NEWTHOR_FUNCTIONS
 };
 
+struct IHThorShuffleExtra : public IInterface
+{
+    virtual bool isSameGroup(const void * _left, const void * _right) = 0;
+};
+
+struct IHThorShuffleArg : public IHThorSortArg, public IHThorShuffleExtra
+{
+    COMMON_NEWTHOR_FUNCTIONS
+};
+
 // JoinFlags
 enum { 
     JFleftouter=1, JFrightouter=2, JFexclude=4,

rtl/include/eclhelper_base.hpp

@@ -1537,6 +1537,36 @@ class CThorTopNArg : public CThorArg, implements IHThorTopNArg
     virtual int compareBest(const void * _left) { return +1; }
 };
 
+class CThorShuffleArg : public CThorArg, implements IHThorShuffleArg
+{
+    virtual void Link() const { RtlCInterface::Link(); }
+    virtual bool Release() const { return RtlCInterface::Release(); }
+    virtual void onCreate(ICodeContext * _ctx, IHThorArg *, MemoryBuffer * in) { ctx = _ctx; }
+
+    virtual IInterface * selectInterface(ActivityInterfaceEnum which)
+    {
+        switch (which)
+        {
+        case TAIarg:
+        case TAIsortarg_1:
+            return static_cast<IHThorSortArg *>(this);
+        case TAIshuffleextra_1:
+            return static_cast<IHThorShuffleExtra *>(this);
+        }
+        return NULL;
+    }
+
+    virtual double getSkew()                            { return 0; }           // 0=default
+    virtual bool hasManyRecords() { return false; }
+    virtual double getTargetSkew()                      { return 0; }
+    virtual ISortKeySerializer * querySerialize() { return NULL; }
+    virtual unsigned __int64 getThreshold() { return 0; }
+    virtual IOutputMetaData * querySortedRecordSize() { return NULL; }
+    virtual const char * getSortedFilename() { return NULL; }
+    virtual ICompare * queryCompareLeftRight() { return NULL; }
+    virtual ICompare * queryCompareSerializedRow() { return NULL; }
+};
+
 class CThorKeyedJoinArg : public CThorArg, implements IHThorKeyedJoinArg
 {
     virtual void Link() const { RtlCInterface::Link(); }

system/jlib/jscm.hpp

@@ -73,6 +73,7 @@ public:
     }
     inline void set(const Shared<CLASS> &other) { this->set(other.get()); }
     inline void setown(CLASS * _ptr)            { CLASS * temp = ptr; ptr = _ptr; ::Release(temp); }
+    inline void swap(Shared<CLASS> & other)     { CLASS * temp = ptr; ptr = other.ptr; other.ptr = temp; }
     
 protected:
     inline Shared(CLASS * _ptr)                  { ptr = _ptr; } // deliberately protected