Merge pull request #7820 from ghalliday/issue12267

HPCC-12267 Initial implementation of QUANTILE activity in roxie

Reviewed-By: Jamie Noss <james.noss@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/roxiehelper.cpp

@@ -618,7 +618,22 @@ extern IGroupedInput *createSortedGroupedInputReader(IInputBase *_input, const I
 
 //========================================================================================= 
 
-class CInplaceSortAlgorithm : implements CInterfaceOf<ISortAlgorithm>
+class CSortAlgorithm : implements CInterfaceOf<ISortAlgorithm>
+{
+public:
+    virtual void getSortedGroup(ConstPointerArray & result)
+    {
+        loop
+        {
+            const void * row = next();
+            if (!row)
+                return;
+            result.append(row);
+        }
+    }
+};
+
+class CInplaceSortAlgorithm : public CSortAlgorithm
 {
 protected:
     unsigned curIndex;
@@ -645,6 +660,11 @@ public:
         curIndex = 0;
         sorted.kill();
     }
+    virtual void getSortedGroup(ConstPointerArray & result)
+    {
+        sorted.swapWith(result);
+        curIndex = 0;
+    }
 };
 
 class CQuickSortAlgorithm : public CInplaceSortAlgorithm
@@ -802,7 +822,7 @@ public:
     }
 };
 
-class CInsertionSortAlgorithm : implements CInterfaceOf<ISortAlgorithm>
+class CInsertionSortAlgorithm : public CSortAlgorithm
 {
     SortedBlock *curBlock;
     unsigned blockNo;
@@ -937,7 +957,7 @@ public:
     }
 };
 
-class CHeapSortAlgorithm : implements CInterfaceOf<ISortAlgorithm>
+class CHeapSortAlgorithm : public CSortAlgorithm
 {
     unsigned curIndex;
     ConstPointerArray sorted;
@@ -1137,7 +1157,7 @@ public:
     }
 };
 
-class CSpillingSortAlgorithm : implements CInterfaceOf<ISortAlgorithm>, implements roxiemem::IBufferedRowCallback
+class CSpillingSortAlgorithm : public CSortAlgorithm, implements roxiemem::IBufferedRowCallback
 {
     enum {
         InitialSortElements = 0,
@@ -1362,6 +1382,16 @@ extern ISortAlgorithm *createHeapSortAlgorithm(ICompare *_compare)
     return new CHeapSortAlgorithm(_compare);
 }
 
+extern ISortAlgorithm *createMergeSortAlgorithm(ICompare *_compare)
+{
+    return new CMergeSortAlgorithm(_compare);
+}
+
+extern ISortAlgorithm *createParallelMergeSortAlgorithm(ICompare *_compare)
+{
+    return new CParallelMergeSortAlgorithm(_compare);
+}
+
 extern ISortAlgorithm *createSpillingQuickSortAlgorithm(ICompare *_compare, roxiemem::IRowManager &_rowManager, IOutputMetaData * _rowMeta, ICodeContext *_ctx, const char *_tempDirectory, unsigned _activityId, bool _stable)
 {
     return new CSpillingQuickSortAlgorithm(_compare, _rowManager, _rowMeta, _ctx, _tempDirectory, _activityId, _stable);

common/thorhelper/roxiehelper.hpp

@@ -110,6 +110,7 @@ interface ISortAlgorithm : extends IInterface
     virtual void prepare(IInputBase *input) = 0;
     virtual const void *next() = 0;
     virtual void reset() = 0;
+    virtual void getSortedGroup(ConstPointerArray & result) = 0;
 };
 
 extern THORHELPER_API ISortAlgorithm *createQuickSortAlgorithm(ICompare *_compare);
@@ -117,6 +118,8 @@ extern THORHELPER_API ISortAlgorithm *createStableQuickSortAlgorithm(ICompare *_
 extern THORHELPER_API ISortAlgorithm *createInsertionSortAlgorithm(ICompare *_compare, roxiemem::IRowManager *_rowManager, unsigned _activityId);
 extern THORHELPER_API ISortAlgorithm *createHeapSortAlgorithm(ICompare *_compare);
 extern THORHELPER_API ISortAlgorithm *createSpillingQuickSortAlgorithm(ICompare *_compare, roxiemem::IRowManager &_rowManager, IOutputMetaData * _rowMeta, ICodeContext *_ctx, const char *_tempDirectory, unsigned _activityId, bool _stable);
+extern THORHELPER_API ISortAlgorithm *createMergeSortAlgorithm(ICompare *_compare);
+extern THORHELPER_API ISortAlgorithm *createParallelMergeSortAlgorithm(ICompare *_compare);
 
 extern THORHELPER_API ISortAlgorithm *createSortAlgorithm(RoxieSortAlgorithm algorithm, ICompare *_compare, roxiemem::IRowManager &_rowManager, IOutputMetaData * _rowMeta, ICodeContext *_ctx, const char *_tempDirectory, unsigned _activityId);
 

common/thorhelper/thorcommon.cpp

@@ -779,6 +779,7 @@ extern const char * getActivityText(ThorActivityKind kind)
     case TAKselfdenormalize:        return "Self Denormalize";
     case TAKselfdenormalizegroup:   return "Self Denormalize Group";
     case TAKtrace:                  return "Trace";
+    case TAKquantile:               return "Quantile";
     }
     throwUnexpected();
 }

ecl/eclcc/reservedwords.cpp

@@ -277,6 +277,7 @@ static const char * eclReserved11[] = {//Activities
     "parse",
     "process",
     "project",
+    "quantile",
     "range",
     "rank",
     "ranked",

ecl/hql/hqlatoms.cpp

@@ -365,6 +365,7 @@ IAtom * scanAtom;
 IAtom * scanAllAtom;
 IAtom * scopeAtom;
 IAtom * scopeCheckingAtom;
+IAtom * scoreAtom;
 IAtom * sectionAtom;
 IAtom * _selectors_Atom;
 IAtom * _selectorSequence_Atom;
@@ -808,6 +809,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(scanAll);
     MAKEATOM(scope);
     MAKEATOM(scopeChecking);
+    MAKEATOM(score);
     MAKEATOM(section);
     MAKESYSATOM(selectors);
     MAKESYSATOM(selectorSequence);

ecl/hql/hqlatoms.hpp

@@ -368,6 +368,7 @@ extern HQL_API IAtom * scanAtom;
 extern HQL_API IAtom * scanAllAtom;
 extern HQL_API IAtom * scopeAtom;
 extern HQL_API IAtom * scopeCheckingAtom;
+extern HQL_API IAtom * scoreAtom;
 extern HQL_API IAtom * sectionAtom;
 extern HQL_API IAtom * _selectorSequence_Atom;
 extern HQL_API IAtom * selfAtom;

ecl/hql/hqlattr.cpp

@@ -347,6 +347,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_soapcall_ds:
     case no_newsoapcall:
     case no_newsoapcall_ds:
+    case no_quantile:
     case no_nonempty:
     case no_filtergroup:
     case no_limit:
@@ -629,7 +630,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
     case no_unused80: case no_unused83:
-    case no_unused101: case no_unused102:
+    case no_unused102:
     case no_is_null:
     case no_position:
     case no_current_time:
@@ -2813,6 +2814,27 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
                 info.limitMin(choosenLimit);
         }
         break;
+    case no_quantile:
+        {
+            __int64 parts = getIntValue(expr->queryChild(1), 0);
+            if ((parts > 0) && !isGrouped(expr) && !isLocalActivity(expr))
+            {
+                if (expr->hasAttribute(firstAtom))
+                    parts++;
+                if (expr->hasAttribute(lastAtom))
+                    parts++;
+
+                IHqlExpression * transform = queryNewColumnProvider(expr);
+                if (transformContainsSkip(transform) || expr->hasAttribute(dedupAtom))
+                    info.setRange(0,parts-1);
+                else
+                    info.setN(parts-1);
+            }
+            else
+                info.setUnknown(RCMfew);
+        }
+        break;
+
     case no_topn:
         {
             retrieveRowInformation(info, ds);

ecl/hql/hqlexpr.cpp

@@ -1349,6 +1349,7 @@ const char *getOpString(node_operator op)
     case no_newsoapcall: return "SOAPCALL";
     case no_newsoapcall_ds: return "SOAPCALL";
     case no_soapaction_ds: return "SOAPCALL";
+    case no_quantile: return "QUANTILE";
     case no_newsoapaction_ds: return "SOAPCALL";
     case no_temprow: return "ROW"; 
     case no_projectrow: return "ROW"; 
@@ -1528,7 +1529,6 @@ const char *getOpString(node_operator op)
     case no_unused80:
     case no_unused81:
     case no_unused83:
-    case no_unused101:
     case no_unused102:
         return "unused";
     /* if fail, use "hqltest -internal" to find out why. */
@@ -1602,6 +1602,7 @@ bool checkConstant(node_operator op)
     case no_newsoapcall:
     case no_newsoapcall_ds:
     case no_newsoapaction_ds:
+    case no_quantile:
     case no_filepos:
     case no_file_logicalname:
     case no_failcode:
@@ -2009,6 +2010,7 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_soapaction_ds:
     case no_newsoapcall_ds:
     case no_newsoapaction_ds:
+    case no_quantile:
         return childdataset_datasetleft;
     case no_keyeddistribute:
         return childdataset_leftright;
@@ -2216,6 +2218,7 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_soapaction_ds:
     case no_newsoapcall_ds:
     case no_newsoapaction_ds:
+    case no_quantile:
     case no_activerow:
     case no_newrow:
     case no_keyedlimit:
@@ -2567,6 +2570,7 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_soapcall_ds:
     case no_newsoapcall:
     case no_newsoapcall_ds:
+    case no_quantile:
     case no_alias:
     case no_id2blob:
     case no_embedbody:
@@ -2672,6 +2676,7 @@ unsigned queryTransformIndex(IHqlExpression * expr)
     case no_parse:
     case no_soapcall:
     case no_newxmlparse:
+    case no_quantile:
         pos = 3;
         break;
     case no_newparse:
@@ -2737,6 +2742,7 @@ IHqlExpression * queryNewColumnProvider(IHqlExpression * expr)
     case no_soapcall:
     case no_httpcall:
     case no_newxmlparse:
+    case no_quantile:
         return expr->queryChild(3);
     case no_newparse:
     case no_newsoapcall:            // 4 because input(2) gets transformed.
@@ -11826,6 +11832,7 @@ extern IHqlExpression *createRow(node_operator op, HqlExprArray & args)
     switch (op)
     {
     case no_soapcall:
+    case no_quantile:
         {
             IHqlExpression & record = args.item(3);
             type = makeRowType(record.getType());

ecl/hql/hqlexpr.hpp

@@ -362,7 +362,7 @@ enum _node_operator {
         no_countdict,
         no_any,
         no_existsdict,
-    no_unused101,
+        no_quantile,
     no_unused25,
     no_unused28,  
     no_unused29,

ecl/hql/hqlfold.cpp

@@ -5923,6 +5923,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_projectrow:
     case no_createrow:
     case no_dataset_from_transform:
+    case no_quantile:
         {
             IHqlExpression * transform = queryNewColumnProvider(expr);
             exprMapping.setown(HqlConstantPercolator::extractConstantMapping(transform));

ecl/hql/hqlgram.y

@@ -362,6 +362,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   PROXYADDRESS
   PULL
   PULLED
+  QUANTILE
   QUOTE
   RANDOM
   RANGE
@@ -393,6 +394,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   RULE
   SAMPLE
   SCAN
+  SCORE
   SECTION
   SELF
   SEPARATOR
@@ -9327,6 +9329,31 @@ simpleDataSet
                         {
                             $$.setExpr(parser->processUserAggregate($1, $3, $5, $7, &$10, &$12, $14, $15), $1);
                         }
+    | QUANTILE '(' startTopLeftSeqFilter ',' expression ',' sortListExpr beginCounterScope ',' transform optQuantileOptions endCounterScope ')' endTopLeftFilter endSelectorSequence
+                        {
+                            parser->normalizeExpression($5, type_int, false);
+                            OwnedHqlExpr ds = $3.getExpr();
+                            OwnedHqlExpr sortlist = $7.getExpr();
+                            OwnedHqlExpr transform = $10.getExpr();
+                            OwnedHqlExpr options = $11.getExpr();
+                            OwnedHqlExpr counter = $12.getExpr();
+                            if (counter)
+                                options.setown(createComma(options.getClear(), createAttribute(_countProject_Atom, counter.getClear())));
+                            OwnedHqlExpr selSeq = $15.getExpr();
+                            $$.setExpr(createDatasetF(no_quantile, ds.getClear(), $5.getExpr(), sortlist.getClear(), transform.getClear(), selSeq.getClear(), options.getClear(), NULL), $1);
+                        }
+    | QUANTILE '(' startTopLeftSeqFilter ',' expression ',' sortListExpr beginCounterScope optQuantileOptions endCounterScope ')' endTopLeftFilter endSelectorSequence
+                        {
+                            parser->normalizeExpression($5, type_int, false);
+                            OwnedHqlExpr ds = $3.getExpr();
+                            OwnedHqlExpr sortlist = $7.getExpr();
+                            OwnedHqlExpr options = $9.getExpr();
+                            OwnedHqlExpr counter = $10.getExpr();
+                            OwnedHqlExpr selSeq = $13.getExpr();
+                            OwnedHqlExpr leftSelect = createSelector(no_left, ds, selSeq);
+                            OwnedHqlExpr transform = parser->createDefaultAssignTransform(ds->queryRecord(), leftSelect, $1);
+                            $$.setExpr(createDatasetF(no_quantile, ds.getClear(), $5.getExpr(), sortlist.getClear(), transform.getClear(), selSeq.getClear(), options.getClear(), NULL), $1);
+                        }
 /*
   //This may cause s/r problems with the attribute version if a dataset name clashes with a hint id
     | HINT '(' dataSet ','  hintList ')'
@@ -9348,7 +9375,7 @@ simpleDataSet
 */
     ;
 
-    
+
 dataSetList
     : dataSet
     | dataSet ',' dataSetList
@@ -9427,6 +9454,52 @@ sideEffectOptions
                         }
     ;
 
+optQuantileOptions
+    :
+                        {
+                            $$.setNullExpr();
+                        }
+    | quantileOptions
+    ;
+
+quantileOptions
+    : ',' quantileOption
+                        {
+                            $$.inherit($2);
+                        }
+    | quantileOptions ',' quantileOption
+                        {
+                            $$.setExpr(createComma($1.getExpr(), $3.getExpr()), $1);
+                        }
+    ;
+
+quantileOption
+    : FIRST
+                        {
+                            $$.setExpr(createExprAttribute(firstAtom), $1);
+                        }
+    | LAST
+                        {
+                            $$.setExpr(createExprAttribute(lastAtom), $1);
+                        }
+    | SCORE '(' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_int, false);
+                            $$.setExpr(createExprAttribute(scoreAtom, $3.getExpr()), $1);
+                        }
+    | DEDUP
+                        {
+                            $$.setExpr(createExprAttribute(dedupAtom), $1);
+                        }
+    | RANGE '(' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_set, false);
+                            $$.setExpr(createExprAttribute(rangeAtom, $3.getExpr()), $1);
+                        }
+    | skewAttribute
+    | commonAttribute
+    ;
+
 limitOptions
     :                   {   $$.setNullExpr(); }
     | ',' limitOption limitOptions
@@ -11376,7 +11449,7 @@ nonDatasetExpr
     | dataRow
     | '-' dataRow       {
                             //This is an example of a semantic check that should really take place one everything is
-                            //parsed.  The beingSortOrder productions are a pain, and not strictly correct.
+                            //parsed.  The beginSortOrder productions are a pain, and not strictly correct.
                             if (parser->sortDepth == 0)
                                 parser->normalizeExpression($2, type_numeric, false);
 
@@ -11387,9 +11460,14 @@ nonDatasetExpr
     | dictionary
     ;
 
-sortItem
+simpleSortItem
     : nonDatasetExpr
     | dataSet
+    | expandedSortListByReference
+    ;
+
+sortItem
+    : simpleSortItem
     | FEW               {   $$.setExpr(createAttribute(fewAtom));   }
     | MANY              {   $$.setExpr(createAttribute(manyAtom));  }
     | MERGE             {   $$.setExpr(createAttribute(mergeAtom)); }
@@ -11468,7 +11546,6 @@ sortItem
                             $$.setExpr(createAttribute(parallelAtom), $1);
                         }
     | prefetchAttribute
-    | expandedSortListByReference
     ;
 
 expandedSortListByReference

ecl/hql/hqlgram2.cpp

@@ -5786,6 +5786,7 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
     return NULL;
 }
 
+
 IHqlExpression * HqlGram::createDistributeCond(IHqlExpression * leftDs, IHqlExpression * rightDs, const attribute & err, const attribute & seqAttr)
 {
     IHqlSimpleScope * leftScope = leftDs->queryRecord()->querySimpleScope();
@@ -10679,6 +10680,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case PROJECT: msg.append("PROJECT"); break;
     case PULL: msg.append("PULL"); break;
     case PULLED: msg.append("PULLED"); break;
+    case QUANTILE: msg.append("QUANTILE"); break;
     case QUOTE: msg.append("QUOTE"); break;
     case RANDOM: msg.append("RANDOM"); break;
     case RANGE: msg.append("RANGE"); break;
@@ -10710,6 +10712,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case RULE: msg.append("RULE"); break;
     case SAMPLE: msg.append("SAMPLE"); break;
     case SCAN: msg.append("SCAN"); break;
+    case SCORE: msg.append("SCORE"); break;
     case SECTION: msg.append("SECTION"); break;
     case SELF: msg.append("SELF"); break;
     case SEPARATOR: msg.append("SEPARATOR"); break;
@@ -10934,7 +10937,7 @@ void HqlGram::simplifyExpected(int *expected)
                        GROUP, GROUPED, KEYED, UNGROUP, JOIN, PULL, ROLLUP, ITERATE, PROJECT, NORMALIZE, PIPE, DENORMALIZE, CASE, MAP, 
                        HTTPCALL, SOAPCALL, LIMIT, PARSE, FAIL, MERGE, PRELOAD, ROW, TOPN, ALIAS, LOCAL, NOFOLD, NOHOIST, NOTHOR, IF, GLOBAL, __COMMON__, __COMPOUND__, TOK_ASSERT, _EMPTY_,
                        COMBINE, ROWS, REGROUP, XMLPROJECT, SKIP, LOOP, CLUSTER, NOLOCAL, REMOTE, PROCESS, ALLNODES, THISNODE, GRAPH, MERGEJOIN, STEPPED, NONEMPTY, HAVING,
-                       TOK_CATCH, '@', SECTION, WHEN, IFF, COGROUP, HINT, INDEX, PARTITION, AGGREGATE, SUBSORT, TOK_ERROR, CHOOSE, TRACE, 0);
+                       TOK_CATCH, '@', SECTION, WHEN, IFF, COGROUP, HINT, INDEX, PARTITION, AGGREGATE, SUBSORT, TOK_ERROR, CHOOSE, TRACE, QUANTILE, 0);
     simplify(expected, EXP, ABS, SIN, COS, TAN, SINH, COSH, TANH, ACOS, ASIN, ATAN, ATAN2, 
                        COUNT, CHOOSE, MAP, CASE, IF, HASH, HASH32, HASH64, HASHMD5, CRC, LN, TOK_LOG, POWER, RANDOM, ROUND, ROUNDUP, SQRT, 
                        TRUNCATE, LENGTH, TRIM, INTFORMAT, REALFORMAT, ASSTRING, TRANSFER, MAX, MIN, EVALUATE, SUM,

ecl/hql/hqlir.cpp

@@ -282,7 +282,7 @@ const char * getOperatorIRText(node_operator op)
     EXPAND_CASE(no,countdict);
     EXPAND_CASE(no,any);
     EXPAND_CASE(no,existsdict);
-    EXPAND_CASE(no,unused101);
+    EXPAND_CASE(no,quantile);
     EXPAND_CASE(no,unused25);
     EXPAND_CASE(no,unused28);
     EXPAND_CASE(no,unused29);

ecl/hql/hqllex.l

@@ -860,6 +860,7 @@ PROJECT             { RETURNSYM(PROJECT); }
 PROXYADDRESS        { RETURNSYM(PROXYADDRESS); }
 PULL                { RETURNSYM(PULL); }
 PULLED              { RETURNSYM(PULLED); }
+QUANTILE            { RETURNSYM(QUANTILE); }
 QUOTE               { RETURNSYM(QUOTE); }
 RANGE               { RETURNSYM(RANGE); }
 RANK                { RETURNSYM(RANK); }
@@ -893,6 +894,7 @@ ROWDIFF             { RETURNSYM(ROWDIFF); }
 RULE                { RETURNSYM(RULE); }
 SAMPLE              { RETURNSYM(SAMPLE); }
 SCAN                { RETURNSYM(SCAN); }
+SCORE               { RETURNSYM(SCORE); }
 SECTION             { RETURNSYM(SECTION); }
 SELF                { RETURNSYM(SELF); }
 SEPARATOR           { RETURNSYM(SEPARATOR); }

ecl/hql/hqlmeta.cpp

@@ -1393,13 +1393,13 @@ static bool isCompatibleSortOrder(IHqlExpression * existingOrder, IHqlExpression
     return true;
 }
 
-static bool normalizedIsAlreadySorted(IHqlExpression * dataset, IHqlExpression * normalizedOrder, bool isLocal, bool ignoreGrouping)
+static bool normalizedIsAlreadySorted(IHqlExpression * dataset, IHqlExpression * normalizedOrder, bool isLocal, bool ignoreGrouping, bool requireDistribution)
 {
 #ifdef OPTIMIZATION2
     if (hasNoMoreRowsThan(dataset, 1))
         return true;
 #endif
-    if (!isCorrectDistributionForSort(dataset, normalizedOrder, isLocal, ignoreGrouping))
+    if (requireDistribution && !isCorrectDistributionForSort(dataset, normalizedOrder, isLocal, ignoreGrouping))
         return false;
 
     //Constant items and duplicates should have been removed already.
@@ -1408,7 +1408,7 @@ static bool normalizedIsAlreadySorted(IHqlExpression * dataset, IHqlExpression *
 }
 
 
-bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping)
+bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool requireDistribution)
 {
 #ifdef OPTIMIZATION2
     if (hasNoMoreRowsThan(dataset, 1))
@@ -1416,17 +1416,17 @@ bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * order, bool isLo
 #endif
 
     OwnedHqlExpr normalizedOrder = normalizeSortlist(order, dataset);
-    return normalizedIsAlreadySorted(dataset, normalizedOrder, isLocal, ignoreGrouping);
+    return normalizedIsAlreadySorted(dataset, normalizedOrder, isLocal, ignoreGrouping, requireDistribution);
 }
 
 
 //Elements in the exprarray have already been mapped;
-bool isAlreadySorted(IHqlExpression * dataset, const HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
+bool isAlreadySorted(IHqlExpression * dataset, const HqlExprArray & newSort, bool isLocal, bool ignoreGrouping, bool requireDistribution)
 {
     HqlExprArray components;
     normalizeComponents(components, newSort);
     OwnedHqlExpr normalizedOrder = createSortList(components);
-    return normalizedIsAlreadySorted(dataset, normalizedOrder, isLocal, ignoreGrouping);
+    return normalizedIsAlreadySorted(dataset, normalizedOrder, isLocal, ignoreGrouping, requireDistribution);
 }
 
 
@@ -1540,13 +1540,13 @@ static IHqlExpression * createSubSorted(IHqlExpression * dataset, IHqlExpression
     if (!isLocal && !alwaysLocal)
         subsort.setown(convertSubSortToGroupedSort(subsort));
 
-    assertex(isAlreadySorted(subsort, order, isLocal||alwaysLocal, ignoreGrouping));
+    assertex(isAlreadySorted(subsort, order, isLocal||alwaysLocal, ignoreGrouping, false));
     return subsort.getClear();
 }
 
 IHqlExpression * getSubSort(IHqlExpression * dataset, const HqlExprArray & order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
 {
-    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
+    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping, true))  // could possible have requireDistribution = false
         return NULL;
 
     OwnedHqlExpr sortlist = createValueSafe(no_sortlist, makeSortListType(NULL), order);
@@ -1556,7 +1556,7 @@ IHqlExpression * getSubSort(IHqlExpression * dataset, const HqlExprArray & order
 
 IHqlExpression * getSubSort(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
 {
-    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
+    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping, true))  // could possible have requireDistribution = false
         return NULL;
 
     return createSubSorted(dataset, order, isLocal, ignoreGrouping, alwaysLocal);
@@ -1566,7 +1566,7 @@ IHqlExpression * getSubSort(IHqlExpression * dataset, IHqlExpression * order, bo
 
 IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, IHqlExpression * parentExpr, bool isLocal, bool ignoreGrouping, bool alwaysLocal, bool allowSubSort, bool requestSpilling)
 {
-    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
+    if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping, true))
         return LINK(dataset);
 
     if (allowSubSort && (isLocal || alwaysLocal))
@@ -2443,6 +2443,22 @@ void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr)
             }
             break;
         }
+    case no_quantile:
+        //It is not be safe to assume that all implementations of the quantile activity will generate their results in order.
+        if (expr->hasAttribute(localAtom) || isGrouped(dataset))
+        {
+            IHqlExpression * transform = expr->queryChild(3);
+            OwnedHqlExpr leftSelect = createSelector(no_left, dataset, expr->queryAttribute(_selectorSequence_Atom));
+            TableProjectMapper mapper;
+            mapper.setMapping(transform, leftSelect);
+
+            extractMeta(meta, dataset);
+            meta.removeAllSortOrders();
+            meta.applyProject(mapper);  // distribution preserved if local..
+        }
+        else
+            meta.preserveGrouping(dataset);
+        break;
     case no_iterate:
     case no_transformebcdic:
     case no_transformascii:
@@ -3073,6 +3089,10 @@ ITypeInfo * calculateDatasetType(node_operator op, const HqlExprArray & parms)
     case no_newsoapcall:
         recordArg = 4;
         break;
+    case no_quantile:
+        recordArg = 3;
+        nowGrouped = isGrouped(datasetType);
+        break;
     case no_soapcall_ds:
         recordArg = 4;
         nowGrouped = isGrouped(datasetType);

ecl/hql/hqlmeta.hpp

@@ -108,8 +108,8 @@ extern HQL_API bool matchDedupDistribution(IHqlExpression * distn, const HqlExpr
 extern HQL_API bool matchesAnyDistribution(IHqlExpression * distn);
 
 extern HQL_API bool appearsToBeSorted(IHqlExpression * dataset, bool isLocal, bool ignoreGrouping);
-extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, const HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
-extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * newSort, bool isLocal, bool ignoreGrouping);
+extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, const HqlExprArray & newSort, bool isLocal, bool ignoreGrouping, bool requireDistribution);
+extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * newSort, bool isLocal, bool ignoreGrouping, bool requireDistribution);
 extern HQL_API IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, IHqlExpression * parentExpr, bool isLocal, bool ignoreGrouping, bool alwaysLocal, bool allowSubSort, bool requestSpilling);
 
 extern HQL_API bool isWorthShuffling(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping);

ecl/hqlcpp/hqlcpp.ipp

@@ -1478,6 +1478,7 @@ public:
     ABoundActivity * doBuildActivityProject(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityProcess(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityPullActivity(BuildCtx & ctx, IHqlExpression * expr);
+    ABoundActivity * doBuildActivityQuantile(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityRegroup(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityRemote(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);
     ABoundActivity * doBuildActivityReturnResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -6539,6 +6539,9 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_map:
                 result = doBuildActivityCase(ctx, expr, isRoot);
                 break;
+            case no_quantile:
+                result = doBuildActivityQuantile(ctx, expr);
+                break;
             case no_chooseds:
             case no_choose:
                 result = doBuildActivityChoose(ctx, expr, isRoot);
@@ -11484,7 +11487,7 @@ void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx,
     compareName.append("compare").append(sideText);
 
     assertex(dataset.querySide() == no_activetable);
-    bool noNeedToSort = isAlreadySorted(dataset.queryDataset(), sorts, isLocal, true);
+    bool noNeedToSort = isAlreadySorted(dataset.queryDataset(), sorts, isLocal, true, true);
     if (userPreventsSort(noSortAttr, side))
         noNeedToSort = true;
 
@@ -12179,9 +12182,9 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
     if (joinInfo.hasOptionalEqualities())
         flags.append("|JFlimitedprefixjoin");
 
-    if (isAlreadySorted(dataset1, joinInfo.queryLeftSort(), true, true) || userPreventsSort(noSortAttr, no_left))
+    if (isAlreadySorted(dataset1, joinInfo.queryLeftSort(), true, true, false) || userPreventsSort(noSortAttr, no_left))
         flags.append("|JFleftSortedLocally");
-    if (isAlreadySorted(dataset2, joinInfo.queryRightSort(), true, true) || userPreventsSort(noSortAttr, no_right))
+    if (isAlreadySorted(dataset2, joinInfo.queryRightSort(), true, true, false) || userPreventsSort(noSortAttr, no_right))
         flags.append("|JFrightSortedLocally");
     if (isSmartJoin) flags.append("|JFsmart|JFmanylookup");
     if (isSmartJoin || expr->hasAttribute(unstableAtom))
@@ -16584,6 +16587,99 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
     return instance->getBoundActivity();
 }
 
+ABoundActivity * HqlCppTranslator::doBuildActivityQuantile(BuildCtx & ctx, IHqlExpression * expr)
+{
+    IHqlExpression * dataset = expr->queryChild(0);
+    Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
+
+    Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKquantile, expr, "Quantile");
+    buildActivityFramework(instance);
+    buildInstancePrefix(instance);
+
+    IHqlExpression * number = expr->queryChild(1);
+    IHqlExpression * sortlist = expr->queryChild(2);
+    IHqlExpression * transform = expr->queryChild(3);
+    IHqlExpression * score = queryAttributeChild(expr, scoreAtom, 0);
+    IHqlExpression * skew = queryAttributeChild(expr, skewAtom, 0);
+    IHqlExpression * dedupAttr = expr->queryAttribute(dedupAtom);
+    IHqlExpression * range = queryAttributeChild(expr, rangeAtom, 0);
+
+    instance->classctx.addQuotedLiteral("virtual ICompare * queryCompare() { return &compare; }");
+
+    buildCompareClass(instance->nestedctx, "compare", sortlist, DatasetReference(dataset));
+
+    doBuildUnsigned64Function(instance->startctx, "getNumDivisions", number);
+
+    if (skew)
+        doBuildFunction(instance->startctx, doubleType, "getSkew", skew);
+
+    if (range)
+    {
+        Owned<ITypeInfo> setType = makeSetType(makeIntType(8, false));
+        doBuildFunction(instance->startctx, setType, "getRange", range);
+    }
+
+    if (score)
+    {
+        BuildCtx funcctx(instance->startctx);
+        funcctx.addQuotedCompound("virtual unsigned __int64 getScore(const void * _self)");
+        funcctx.addQuotedLiteral("unsigned char * self = (unsigned char *) _self;");
+        bindTableCursor(funcctx, dataset, "self");
+        buildReturn(funcctx, score);
+    }
+
+    IHqlExpression * counter = queryAttributeChild(expr, _countProject_Atom, 0);
+    IHqlExpression * selSeq = querySelSeq(expr);
+    {
+        BuildCtx funcctx(instance->startctx);
+        funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, unsigned __int64 counter)");
+        if (counter)
+            associateCounter(funcctx, counter, "counter");
+        buildTransformBody(funcctx, transform, dataset, NULL, instance->dataset, selSeq);
+    }
+
+    buildClearRecordMember(instance->createctx, "", dataset);
+
+    //If a dataset is sorted by all fields then it is impossible to determine if the original order
+    //was preserved - so mark the sort as potentially unstable (to reduce memory usage at runtime)
+    bool unstable = expr->hasAttribute(unstableAtom);
+    if (options.optimizeSortAllFields &&
+        allFieldsAreSorted(expr->queryRecord(), sortlist, dataset->queryNormalizedSelector(), options.optimizeSortAllFieldsStrict))
+        unstable = true;
+
+    StringBuffer flags;
+    if (expr->hasAttribute(firstAtom))
+        flags.append("|TQFfirst");
+    if (expr->hasAttribute(lastAtom))
+        flags.append("|TQFlast");
+    if (isAlreadySorted(dataset, sortlist, false, false, false))
+        flags.append("|TQFsorted|TQFlocalsorted");
+    else if (isAlreadySorted(dataset, sortlist, true, false, false))
+        flags.append("|TQFlocalsorted");
+    if (score)
+        flags.append("|TQFhasscore");
+    if (range)
+        flags.append("|TQFhasrange");
+    if (skew)
+        flags.append("|TQFhasskew");
+    if (dedupAttr)
+        flags.append("|TQFdedup");
+    if (unstable)
+        flags.append("|TQFunstable");
+    if (!number->queryValue())
+        flags.append("|TQFvariabledivisions");
+    if (!transformReturnsSide(expr, no_left, 0))
+        flags.append("|TQFneedtransform");
+
+    if (flags.length())
+        instance->classctx.addQuotedF("virtual unsigned getFlags() { return %s; }", flags.str()+1);
+
+    buildInstanceSuffix(instance);
+
+    buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
+    return instance->getBoundActivity();
+}
+
 //---------------------------------------------------------------------------
 
 void HqlCppTranslator::doBuildXmlReadMember(ActivityInstance & instance, IHqlExpression * expr, const char * functionName, bool & usesContents)

ecl/hqlcpp/hqliproj.cpp

@@ -2006,6 +2006,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_createrow:
     case no_rollupgroup:
     case no_projectrow:
+    case no_quantile:
         return CreateRecordActivity;
     case no_inlinetable:
     case no_dataset_from_transform:

ecl/hqlcpp/hqlttcpp.cpp

@@ -2526,7 +2526,7 @@ IHqlExpression * ThorHqlTransformer::normalizeCoGroup(IHqlExpression * expr)
             {
                 OwnedHqlExpr mappedDistribution = replaceSelector(distribution, queryActiveTableSelector(), &cur);
                 OwnedHqlExpr mergeAttr;
-                if (bestSortOrder && isAlreadySorted(&cur, bestSortOrder, true, true))
+                if (bestSortOrder && isAlreadySorted(&cur, bestSortOrder, true, true, false))
                     mergeAttr.setown(createExprAttribute(mergeAtom, replaceSelector(bestSortOrder, queryActiveTableSelector(), &cur)));
                 OwnedHqlExpr distributedInput = createDatasetF(no_distribute, LINK(&cur), LINK(mappedDistribution), mergeAttr.getClear(), NULL);
                 distributedInput.setown(cloneInheritedAnnotations(expr, distributedInput));
@@ -2575,7 +2575,7 @@ static bool canReorderMatchExistingLocalSort(HqlExprArray & newElements1, HqlExp
     newElements2.kill();
     if (reorderMatchExistingLocalSort(newElements1, newElements2, ds1, elements1, elements2))
     {
-        if (isAlreadySorted(ds2, newElements2, isLocal||alwaysLocal, true))
+        if (isAlreadySorted(ds2, newElements2, isLocal||alwaysLocal, true, true))
             return true;
 
         if (canSubSort && isWorthShuffling(ds2, newElements2, isLocal||alwaysLocal, true))
@@ -2926,8 +2926,8 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
     //Worthwhile even for lookup joins
     if (isLightweightJoinCandidate(expr, isLocal, joinInfo.hasOptionalEqualities()))
     {
-        if (isAlreadySorted(leftDs, joinInfo.queryLeftSort(), true, true) &&
-            isAlreadySorted(rightDs, joinInfo.queryRightSort(), true, true))
+        if (isAlreadySorted(leftDs, joinInfo.queryLeftSort(), true, true, false) &&
+            isAlreadySorted(rightDs, joinInfo.queryRightSort(), true, true, false))
         {
             //If this is a lookup join without a many then we need to make sure only the first match is retained.
             return appendOwnedOperand(expr, createAttribute(_lightweight_Atom));
@@ -3171,7 +3171,7 @@ IHqlExpression * ThorHqlTransformer::normalizeSort(IHqlExpression * expr)
 
     bool isLocal = expr->hasAttribute(localAtom);
     bool alwaysLocal = !translator.targetThor();
-    if ((op != no_assertsorted) && isAlreadySorted(dataset, sortlist, isLocal||alwaysLocal, false))
+    if ((op != no_assertsorted) && isAlreadySorted(dataset, sortlist, isLocal||alwaysLocal, false, true))
         return LINK(dataset);
     if (op == no_sorted)
         return normalizeSortSteppedIndex(expr, sortedAtom);

ecl/regress/quantile_e1.ecl

@@ -0,0 +1,44 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+rawRec := { unsigned id; };
+
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt));
+END;
+
+ascending := createDataset(100, 1, 0);
+
+integer zero := 0;
+integer minusOne := -1;
+integer oneHundred := 100;
+
+//Out of range number of items
+OUTPUT(QUANTILE(ascending, zero, {id}));    // 10, 20, 30, 40, 50, 60, 70, 80, 90  
+//Out of range number of items
+OUTPUT(QUANTILE(ascending, minusOne, {id}));    // 10, 20, 30, 40, 50, 60, 70, 80, 90  
+
+//Number of items is sensible, but range items are invalid
+OUTPUT(QUANTILE(ascending, 5, {id}, range([minusOne,oneHundred]));    // 10, 20, 30, 40, 50, 60, 70, 80, 90  

roxie/ccd/ccdquery.cpp

@@ -751,6 +751,8 @@ protected:
             return createRoxieServerWorkUnitReadActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKxmlparse:
             return createRoxieServerXmlParseActivityFactory(id, subgraphId, *this, helperFactory, kind);
+        case TAKquantile:
+            return createRoxieServerQuantileActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKregroup:
             return createRoxieServerRegroupActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKcombine:

roxie/ccd/ccdserver.cpp

@@ -7466,6 +7466,268 @@ IRoxieServerActivityFactory *createRoxieServerSortActivityFactory(unsigned _id,
 
 //=====================================================================================================
 
+static int compareUint64(const void * _left, const void * _right)
+{
+    const unsigned __int64 left = *static_cast<const unsigned __int64 *>(_left);
+    const unsigned __int64 right = *static_cast<const unsigned __int64 *>(_right);
+    if (left < right)
+        return -1;
+    if (left > right)
+        return -1;
+    return 0;
+}
+
+class CRoxieServerQuantileActivity : public CRoxieServerActivity
+{
+protected:
+    Owned<ISortAlgorithm> sorter;
+    IHThorQuantileArg &helper;
+    ConstPointerArray sorted;
+    ICompare *compare;
+    unsigned flags;
+    double skew;
+    unsigned __int64 numDivisions;
+    bool rangeIsAll;
+    size32_t rangeSize;
+    rtlDataAttr rangeValues;
+    bool calculated;
+    bool processedAny;
+    bool anyThisGroup;
+    bool eof;
+    unsigned curQuantile;
+    unsigned curIndex;
+    unsigned curIndexExtra;
+    unsigned skipSize;
+    unsigned skipExtra;
+    unsigned prevIndex;
+
+public:
+    CRoxieServerQuantileActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _flags)
+        : CRoxieServerActivity(_factory, _probeManager), helper((IHThorQuantileArg &)basehelper), flags(_flags)
+    {
+        compare = helper.queryCompare();
+        skew = 0.0;
+        numDivisions = 0;
+        calculated = false;
+        processedAny = false;
+        anyThisGroup = false;
+        eof = false;
+        rangeIsAll = true;
+        rangeSize = 0;
+        curQuantile = 0;
+        prevIndex = 0;
+        curIndex = 0;
+        curIndexExtra = 0;
+        skipSize = 0;
+        skipExtra = 0;
+        if (flags & TQFunstable)
+            sorter.setown(createQuickSortAlgorithm(compare));
+        else
+            sorter.setown(createMergeSortAlgorithm(compare));
+    }
+
+    virtual void reset()
+    {
+        sorter->reset();
+        calculated = false;
+        processedAny = false;
+        anyThisGroup = false;
+        CRoxieServerActivity::reset();
+    }
+
+    virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
+    {
+        CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
+        skew = helper.getSkew();
+        numDivisions = helper.getNumDivisions();
+        //Check for -ve integer values and treat as out of range
+        if ((__int64)numDivisions < 1)
+            numDivisions = 1;
+        if (flags & TQFhasrange)
+            helper.getRange(rangeIsAll, rangeSize, rangeValues.refdata());
+        else
+        {
+            rangeIsAll = true;
+            rangeSize = 0;
+        }
+        if (rangeSize)
+        {
+            //Sort the range items into order to allow quick comparison
+            qsort(rangeValues.getdata(), rangeSize / sizeof(unsigned __int64), sizeof(unsigned __int64), compareUint64);
+        }
+        calculated = false;
+        processedAny = false;
+        anyThisGroup = false;
+        eof = false;
+        curQuantile = 0;
+        prevIndex = 0;
+        curIndex = 0;
+        curIndexExtra = 0;
+    }
+
+    virtual bool needsAllocator() const { return true; }
+
+    virtual const void * nextInGroup()
+    {
+        ActivityTimer t(totalCycles, timeActivities);
+        if (eof)
+            return NULL;
+
+        const void * ret = NULL;
+        for(;;)
+        {
+            if (!calculated)
+            {
+                if (flags & TQFlocalsorted)
+                {
+                    for (;;)
+                    {
+                        const void * next = input->nextInGroup();
+                        if (!next)
+                            break;
+                        sorted.append(next);
+                    }
+                }
+                else
+                {
+                    sorter->prepare(input);
+                    sorter->getSortedGroup(sorted);
+                }
+
+                if (sorted.ordinality() == 0)
+                {
+                    if (processedAny)
+                    {
+                        eof = true;
+                        return NULL;
+                    }
+
+                    //Unusual case 0 rows - add a default row instead
+                    RtlDynamicRowBuilder rowBuilder(rowAllocator);
+                    size32_t thisSize = helper.createDefault(rowBuilder);
+                    sorted.append(rowBuilder.finalizeRowClear(thisSize));
+                }
+
+                calculated = true;
+                processedAny = true;
+                anyThisGroup = false;
+                curQuantile = 0;
+                curIndex = 0;
+                curIndexExtra = (numDivisions-1) / 2;   // to ensure correctly rounded up
+                prevIndex = curIndex-1; // Ensure it doesn't match
+                skipSize = (sorted.ordinality() / numDivisions);
+                skipExtra = (sorted.ordinality() % numDivisions);
+            }
+
+            if (isQuantileIncluded(curQuantile))
+            {
+                if (!(flags & TQFdedup) || (prevIndex != curIndex))
+                {
+                    const void * lhs = sorted.item(curIndex);
+                    if (flags & TQFneedtransform)
+                    {
+                        unsigned outSize;
+                        RtlDynamicRowBuilder rowBuilder(rowAllocator);
+                        try
+                        {
+                            outSize = helper.transform(rowBuilder, lhs, curQuantile);
+                        }
+                        catch (IException *E)
+                        {
+                            throw makeWrappedException(E);
+                        }
+
+                        if (outSize)
+                            ret = rowBuilder.finalizeRowClear(outSize);
+                    }
+                    else
+                    {
+                        LinkRoxieRow(lhs);
+                        ret = lhs;
+                    }
+                    prevIndex = curIndex; // even if output was skipped?
+                }
+            }
+
+            curIndex += skipSize;
+            curIndexExtra += skipExtra;
+            if (curIndexExtra >= numDivisions)
+            {
+                curIndex++;
+                curIndexExtra -= numDivisions;
+            }
+            //Ensure the current index always stays valid.
+            if (curIndex >= sorted.ordinality())
+                curIndex = sorted.ordinality()-1;
+            curQuantile++;
+            if (curQuantile > numDivisions)
+            {
+                sorted.kill();
+                sorter->reset();
+                calculated = false; // ready for next group
+            }
+
+            if (ret)
+            {
+                anyThisGroup = true;
+                processed++;
+                return ret;
+            }
+
+            if (curQuantile > numDivisions)
+            {
+                if (anyThisGroup)
+                    return NULL;
+            }
+        }
+    }
+
+    bool isQuantileIncluded(unsigned quantile) const
+    {
+        if (quantile == 0)
+            return (flags & TQFfirst) != 0;
+        if (quantile == numDivisions)
+            return (flags & TQFlast) != 0;
+        if (rangeIsAll)
+            return true;
+        //Compare against the list of ranges provided
+        //MORE: Since the list is sorted should only need to compare the next (and allow for dups)
+        unsigned rangeNum = rangeSize / sizeof(unsigned __int64);
+        const unsigned __int64 * ranges = static_cast<const unsigned __int64 *>(rangeValues.getdata());
+        for (unsigned i = 0; i < rangeNum; i++)
+        {
+            if (ranges[i] == quantile)
+                return true;
+        }
+        return false;
+    }
+};
+
+class CRoxieServerQuantileActivityFactory : public CRoxieServerActivityFactory
+{
+    unsigned flags;
+
+public:
+    CRoxieServerQuantileActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind)
+        : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind)
+    {
+        Owned<IHThorQuantileArg> quantileHelper = (IHThorQuantileArg *) helperFactory();
+        flags = quantileHelper->getFlags();
+    }
+
+    virtual IRoxieServerActivity *createActivity(IProbeManager *_probeManager) const
+    {
+        return new CRoxieServerQuantileActivity(this, _probeManager, flags);
+    }
+};
+
+IRoxieServerActivityFactory *createRoxieServerQuantileActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind)
+{
+    return new CRoxieServerQuantileActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind);
+}
+
+//=====================================================================================================
+
 class CRoxieServerSortedActivity : public CRoxieServerActivity
 {
     IHThorSortedArg &helper;

roxie/ccd/ccdserver.hpp

@@ -409,6 +409,7 @@ extern IRoxieServerActivityFactory *createRoxieServerSoapRowActionActivityFactor
 extern IRoxieServerActivityFactory *createRoxieServerSoapDatasetCallActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
 extern IRoxieServerActivityFactory *createRoxieServerSoapDatasetActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, bool _isRoot);
 extern IRoxieServerActivityFactory *createRoxieServerLinkedRawIteratorActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
+extern IRoxieServerActivityFactory *createRoxieServerQuantileActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
 
 extern IRoxieServerActivityFactory *createRoxieServerNWayGraphLoopResultReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, unsigned graphId);
 extern IRoxieServerActivityFactory *createRoxieServerNWayInputActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);

rtl/include/eclhelper.hpp

@@ -923,6 +923,7 @@ enum ThorActivityKind
     TAKjsonwrite,
     TAKjsonread,
     TAKtrace,
+    TAKquantile,
 
     TAKlast
 };
@@ -1062,7 +1063,9 @@ enum ActivityInterfaceEnum
     TAIdictionaryworkunitwritearg_1,
     TAIdictionaryresultwritearg_1,
     TAItracearg_1,
-//Should remain as last of all meaningful tags, but before aliases
+    TAIquantilearg_1,
+
+    //Should remain as last of all meaningful tags, but before aliases
     TAImax,
 
 //Only aliases follow - for interfaces implemented via typedefs
@@ -1322,6 +1325,32 @@ struct IHThorSelectNArg : public IHThorArg
     virtual size32_t createDefault(ARowBuilder & rowBuilder) = 0;
 };
 
+enum
+{
+    TQFfirst            = 0x0001,       // default flags is zero
+    TQFlast             = 0x0002,
+    TQFsorted           = 0x0004,
+    TQFlocalsorted      = 0x0008,
+    TQFhasscore         = 0x0010,
+    TQFhasrange         = 0x0020,
+    TQFhasskew          = 0x0040,
+    TQFdedup            = 0x0080,
+    TQFunstable         = 0x0100,
+    TQFvariabledivisions= 0x0200,       // num divisions is not a constant
+    TQFneedtransform    = 0x0400,       // if not set the records are returned as-is
+};
+
+struct IHThorQuantileArg : public IHThorArg
+{
+    virtual unsigned getFlags() = 0;
+    virtual unsigned __int64 getNumDivisions() = 0;
+    virtual double getSkew() = 0;
+    virtual ICompare * queryCompare() = 0;
+    virtual size32_t createDefault(ARowBuilder & rowBuilder) = 0;
+    virtual size32_t transform(ARowBuilder & rowBuilder, const void * _left, unsigned __int64 _counter) = 0;
+    virtual unsigned __int64 getScore(const void * _left) = 0;
+    virtual void getRange(bool & isAll, size32_t & tlen, void * & tgt) = 0;
+};
 
 struct IHThorCombineArg : public IHThorArg
 {

rtl/include/eclhelper_base.hpp

@@ -1,4 +1,3 @@
-
 /*##############################################################################
 #    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
 #
@@ -708,6 +707,30 @@ class CThorProjectArg : public CThorArg, implements IHThorProjectArg
     virtual bool canFilter()                                { return false; }
 };
 
+class CThorQuantileArg : public CThorArg, implements IHThorQuantileArg
+{
+    virtual void Link() const { RtlCInterface::Link(); }
+    virtual bool Release() const { return RtlCInterface::Release(); }
+    virtual void onCreate(ICodeContext * _ctx, IHThorArg *, MemoryBuffer * in) { ctx = _ctx; }
+
+    virtual IInterface * selectInterface(ActivityInterfaceEnum which)
+    {
+        switch (which)
+        {
+        case TAIarg:
+        case TAIquantilearg_1:
+            return static_cast<IHThorQuantileArg *>(this);
+        }
+        return NULL;
+    }
+
+    virtual unsigned getFlags() { return 0; }
+    virtual unsigned __int64 getNumDivisions() { return 2; }
+    virtual double getSkew() { return 0; }
+    virtual unsigned __int64 getScore(const void * _left) { return 1; }
+    virtual void getRange(bool & isAll, size32_t & tlen, void * & tgt) { isAll = true; tlen = 0; tgt = NULL; }
+};
+
 class CThorPrefetchProjectArg : public CThorArg, implements IHThorPrefetchProjectArg
 {
     virtual void Link() const { RtlCInterface::Link(); }

testing/regress/ecl/key/quantile1.xml

@@ -0,0 +1,75 @@
+<Dataset name='Result 1'>
+ <Row><id>10</id></Row>
+ <Row><id>20</id></Row>
+ <Row><id>30</id></Row>
+ <Row><id>40</id></Row>
+ <Row><id>50</id></Row>
+ <Row><id>60</id></Row>
+ <Row><id>70</id></Row>
+ <Row><id>80</id></Row>
+ <Row><id>90</id></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>10</id><quant>1</quant></Row>
+ <Row><id>20</id><quant>2</quant></Row>
+ <Row><id>30</id><quant>3</quant></Row>
+ <Row><id>40</id><quant>4</quant></Row>
+ <Row><id>50</id><quant>5</quant></Row>
+ <Row><id>60</id><quant>6</quant></Row>
+ <Row><id>70</id><quant>7</quant></Row>
+ <Row><id>80</id><quant>8</quant></Row>
+ <Row><id>90</id><quant>9</quant></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>10</id><quant>1</quant></Row>
+ <Row><id>20</id><quant>2</quant></Row>
+ <Row><id>30</id><quant>3</quant></Row>
+ <Row><id>40</id><quant>4</quant></Row>
+ <Row><id>50</id><quant>5</quant></Row>
+ <Row><id>60</id><quant>6</quant></Row>
+ <Row><id>70</id><quant>7</quant></Row>
+ <Row><id>80</id><quant>8</quant></Row>
+ <Row><id>90</id><quant>9</quant></Row>
+ <Row><id>99</id><quant>10</quant></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>10</id><quant>1</quant></Row>
+ <Row><id>20</id><quant>2</quant></Row>
+ <Row><id>30</id><quant>3</quant></Row>
+ <Row><id>40</id><quant>4</quant></Row>
+ <Row><id>50</id><quant>5</quant></Row>
+ <Row><id>60</id><quant>6</quant></Row>
+ <Row><id>70</id><quant>7</quant></Row>
+ <Row><id>80</id><quant>8</quant></Row>
+ <Row><id>90</id><quant>9</quant></Row>
+ <Row><id>99</id><quant>10</quant></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>10</id><quant>1</quant></Row>
+ <Row><id>20</id><quant>2</quant></Row>
+ <Row><id>30</id><quant>3</quant></Row>
+ <Row><id>40</id><quant>4</quant></Row>
+ <Row><id>50</id><quant>5</quant></Row>
+ <Row><id>60</id><quant>6</quant></Row>
+ <Row><id>70</id><quant>7</quant></Row>
+ <Row><id>80</id><quant>8</quant></Row>
+ <Row><id>90</id><quant>9</quant></Row>
+ <Row><id>99</id><quant>10</quant></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><id>50</id><quant>1</quant></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><id>50</id><quant>1</quant></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><id>50</id><quant>1</quant></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>50</id><quant>1</quant></Row>
+ <Row><id>99</id><quant>2</quant></Row>
+</Dataset>

testing/regress/ecl/key/quantile10.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><rid>1</rid><whichpercentile>2</whichpercentile><ids><Row><id>1.0</id></Row></ids></Row>
+ <Row><rid>2</rid><whichpercentile>13</whichpercentile><ids><Row><id>2.8</id></Row></ids></Row>
+ <Row><rid>3</rid><whichpercentile>15</whichpercentile><ids><Row><id>16.0</id></Row></ids></Row>
+ <Row><rid>4</rid><whichpercentile>3</whichpercentile><ids><Row><id>984.0</id></Row></ids></Row>
+ <Row><rid>5</rid><whichpercentile>27</whichpercentile><ids><Row><id>1.81</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/quantile11.xml

@@ -0,0 +1,15 @@
+<Dataset name='Result 1'>
+ <Row><id>25</id></Row>
+ <Row><id>50</id></Row>
+ <Row><id>75</id></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>25</id></Row>
+ <Row><id>49</id></Row>
+ <Row><id>74</id></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>25</id></Row>
+ <Row><id>50</id></Row>
+ <Row><id>76</id></Row>
+</Dataset>

testing/regress/ecl/key/quantile12.xml

@@ -0,0 +1,36 @@
+<Dataset name='Result 1'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>20</id><quant>1</quant></Row>
+ <Row><id>40</id><quant>2</quant></Row>
+ <Row><id>60</id><quant>3</quant></Row>
+ <Row><id>80</id><quant>4</quant></Row>
+ <Row><id>99</id><quant>5</quant></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>4</id><quant>1</quant></Row>
+ <Row><id>7</id><quant>2</quant></Row>
+ <Row><id>11</id><quant>3</quant></Row>
+ <Row><id>15</id><quant>4</quant></Row>
+ <Row><id>19</id><quant>5</quant></Row>
+ <Row><id>22</id><quant>6</quant></Row>
+ <Row><id>26</id><quant>7</quant></Row>
+ <Row><id>30</id><quant>8</quant></Row>
+ <Row><id>33</id><quant>9</quant></Row>
+ <Row><id>37</id><quant>10</quant></Row>
+ <Row><id>41</id><quant>11</quant></Row>
+ <Row><id>44</id><quant>12</quant></Row>
+ <Row><id>48</id><quant>13</quant></Row>
+ <Row><id>52</id><quant>14</quant></Row>
+ <Row><id>56</id><quant>15</quant></Row>
+ <Row><id>59</id><quant>16</quant></Row>
+ <Row><id>63</id><quant>17</quant></Row>
+ <Row><id>67</id><quant>18</quant></Row>
+ <Row><id>70</id><quant>19</quant></Row>
+ <Row><id>74</id><quant>20</quant></Row>
+ <Row><id>78</id><quant>21</quant></Row>
+ <Row><id>81</id><quant>22</quant></Row>
+ <Row><id>85</id><quant>23</quant></Row>
+ <Row><id>89</id><quant>24</quant></Row>
+ <Row><id>93</id><quant>25</quant></Row>
+ <Row><id>96</id><quant>26</quant></Row>
+</Dataset>

testing/regress/ecl/key/quantile13.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>4</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>6</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>1</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>3</Result_4></Row>
+</Dataset>

testing/regress/ecl/key/quantile1a.xml

@@ -0,0 +1,42 @@
+<Dataset name='Result 1'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>2</id><quant>1</quant></Row>
+ <Row><id>5</id><quant>2</quant></Row>
+ <Row><id>7</id><quant>3</quant></Row>
+ <Row><id>9</id><quant>4</quant></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>2</id><quant>1</quant></Row>
+ <Row><id>5</id><quant>2</quant></Row>
+ <Row><id>7</id><quant>3</quant></Row>
+ <Row><id>9</id><quant>4</quant></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>2</id><quant>1</quant></Row>
+ <Row><id>5</id><quant>2</quant></Row>
+ <Row><id>7</id><quant>3</quant></Row>
+ <Row><id>9</id><quant>4</quant></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>2</id><quant>1</quant></Row>
+ <Row><id>4</id><quant>2</quant></Row>
+ <Row><id>7</id><quant>3</quant></Row>
+ <Row><id>8</id><quant>4</quant></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>2</id><quant>1</quant></Row>
+ <Row><id>4</id><quant>2</quant></Row>
+ <Row><id>7</id><quant>3</quant></Row>
+ <Row><id>8</id><quant>4</quant></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>2</id><quant>1</quant></Row>
+ <Row><id>4</id><quant>2</quant></Row>
+ <Row><id>7</id><quant>3</quant></Row>
+ <Row><id>8</id><quant>4</quant></Row>
+</Dataset>

testing/regress/ecl/key/quantile1b.xml

@@ -0,0 +1,54 @@
+<Dataset name='Result 1'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>3</id><quant>1</quant></Row>
+ <Row><id>6</id><quant>2</quant></Row>
+ <Row><id>8</id><quant>3</quant></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>3</id><quant>1</quant></Row>
+ <Row><id>6</id><quant>2</quant></Row>
+ <Row><id>8</id><quant>3</quant></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>3</id><quant>1</quant></Row>
+ <Row><id>6</id><quant>2</quant></Row>
+ <Row><id>8</id><quant>3</quant></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>3</id><quant>1</quant></Row>
+ <Row><id>5</id><quant>2</quant></Row>
+ <Row><id>7</id><quant>3</quant></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>3</id><quant>1</quant></Row>
+ <Row><id>5</id><quant>2</quant></Row>
+ <Row><id>7</id><quant>3</quant></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>3</id><quant>1</quant></Row>
+ <Row><id>5</id><quant>2</quant></Row>
+ <Row><id>7</id><quant>3</quant></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>3</id><quant>1</quant></Row>
+ <Row><id>7</id><quant>2</quant></Row>
+ <Row><id>9</id><quant>3</quant></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>3</id><quant>1</quant></Row>
+ <Row><id>7</id><quant>2</quant></Row>
+ <Row><id>9</id><quant>3</quant></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>3</id><quant>1</quant></Row>
+ <Row><id>7</id><quant>2</quant></Row>
+ <Row><id>9</id><quant>3</quant></Row>
+</Dataset>

testing/regress/ecl/key/quantile2.xml

@@ -0,0 +1,86 @@
+<Dataset name='Result 1'>
+ <Row><id>3</id></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>1</id></Row>
+ <Row><id>1</id></Row>
+ <Row><id>2</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>4</id></Row>
+ <Row><id>5</id></Row>
+ <Row><id>6</id></Row>
+ <Row><id>6</id></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>1</id><quant>1</quant></Row>
+ <Row><id>1</id><quant>2</quant></Row>
+ <Row><id>2</id><quant>3</quant></Row>
+ <Row><id>3</id><quant>4</quant></Row>
+ <Row><id>3</id><quant>5</quant></Row>
+ <Row><id>4</id><quant>6</quant></Row>
+ <Row><id>5</id><quant>7</quant></Row>
+ <Row><id>6</id><quant>8</quant></Row>
+ <Row><id>6</id><quant>9</quant></Row>
+ <Row><id>6</id><quant>10</quant></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><id>0</id><quant>0</quant></Row>
+ <Row><id>1</id><quant>1</quant></Row>
+ <Row><id>2</id><quant>3</quant></Row>
+ <Row><id>3</id><quant>4</quant></Row>
+ <Row><id>4</id><quant>6</quant></Row>
+ <Row><id>5</id><quant>7</quant></Row>
+ <Row><id>6</id><quant>8</quant></Row>
+</Dataset>
+<Dataset name='Result 5'>
+</Dataset>
+<Dataset name='Result 6'>
+</Dataset>
+<Dataset name='Result 7'>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><id>0</id></Row>
+ <Row><id>6</id></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><id>0</id></Row>
+ <Row><id>1</id></Row>
+ <Row><id>2</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>4</id></Row>
+ <Row><id>5</id></Row>
+ <Row><id>6</id></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><id>0</id></Row>
+ <Row><id>0</id></Row>
+ <Row><id>0</id></Row>
+ <Row><id>0</id></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><id>0</id></Row>
+ <Row><id>0</id></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><id>0</id></Row>
+ <Row><id>0</id></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><id>0</id></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><id>0</id></Row>
+</Dataset>
+<Dataset name='Result 15'>
+</Dataset>
+<Dataset name='Result 16'>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><id>0</id></Row>
+ <Row><id>0</id></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><id>0</id></Row>
+</Dataset>

testing/regress/ecl/key/quantile3.xml

@@ -0,0 +1,11 @@
+<Dataset name='Result 1'>
+ <Row><id>0</id></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>0</id></Row>
+ <Row><id>0</id></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>0</id></Row>
+ <Row><id>0</id></Row>
+</Dataset>

testing/regress/ecl/key/quantile4.xml

@@ -0,0 +1,72 @@
+<Dataset name='Result 1'>
+ <Row><id>54011519</id><quant>0</quant></Row>
+ <Row><id>932021114</id><quant>1</quant></Row>
+ <Row><id>1737233514</id><quant>2</quant></Row>
+ <Row><id>2615243109</id><quant>3</quant></Row>
+ <Row><id>3470969220</id><quant>4</quant></Row>
+ <Row><id>4276181620</id><quant>5</quant></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>171375682</id><quant>1</quant></Row>
+ <Row><id>339253556</id><quant>2</quant></Row>
+ <Row><id>478901203</id><quant>3</quant></Row>
+ <Row><id>669062561</id><quant>4</quant></Row>
+ <Row><id>859223919</id><quant>5</quant></Row>
+ <Row><id>1004818309</id><quant>6</quant></Row>
+ <Row><id>1144465956</id><quant>7</quant></Row>
+ <Row><id>1261830119</id><quant>8</quant></Row>
+ <Row><id>1429707993</id><quant>9</quant></Row>
+ <Row><id>1619869351</id><quant>10</quant></Row>
+ <Row><id>1810030709</id><quant>11</quant></Row>
+ <Row><id>1927394872</id><quant>12</quant></Row>
+ <Row><id>2095272746</id><quant>13</quant></Row>
+ <Row><id>2234920393</id><quant>14</quant></Row>
+ <Row><id>2425081751</id><quant>15</quant></Row>
+ <Row><id>2592959625</id><quant>16</quant></Row>
+ <Row><id>2710323788</id><quant>17</quant></Row>
+ <Row><id>2900485146</id><quant>18</quant></Row>
+ <Row><id>2995565825</id><quant>19</quant></Row>
+ <Row><id>3185727183</id><quant>20</quant></Row>
+ <Row><id>3375888541</id><quant>21</quant></Row>
+ <Row><id>3493252704</id><quant>22</quant></Row>
+ <Row><id>3661130578</id><quant>23</quant></Row>
+ <Row><id>3800778225</id><quant>24</quant></Row>
+ <Row><id>3990939583</id><quant>25</quant></Row>
+ <Row><id>4158817457</id><quant>26</quant></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>54011519</id><quant>0</quant></Row>
+ <Row><id>932021114</id><quant>1</quant></Row>
+ <Row><id>1737233514</id><quant>2</quant></Row>
+ <Row><id>2615243109</id><quant>3</quant></Row>
+ <Row><id>3470969220</id><quant>4</quant></Row>
+ <Row><id>4276181620</id><quant>5</quant></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><id>171375682</id><quant>1</quant></Row>
+ <Row><id>339253556</id><quant>2</quant></Row>
+ <Row><id>478901203</id><quant>3</quant></Row>
+ <Row><id>669062561</id><quant>4</quant></Row>
+ <Row><id>859223919</id><quant>5</quant></Row>
+ <Row><id>1004818309</id><quant>6</quant></Row>
+ <Row><id>1144465956</id><quant>7</quant></Row>
+ <Row><id>1261830119</id><quant>8</quant></Row>
+ <Row><id>1429707993</id><quant>9</quant></Row>
+ <Row><id>1619869351</id><quant>10</quant></Row>
+ <Row><id>1810030709</id><quant>11</quant></Row>
+ <Row><id>1927394872</id><quant>12</quant></Row>
+ <Row><id>2095272746</id><quant>13</quant></Row>
+ <Row><id>2234920393</id><quant>14</quant></Row>
+ <Row><id>2425081751</id><quant>15</quant></Row>
+ <Row><id>2592959625</id><quant>16</quant></Row>
+ <Row><id>2710323788</id><quant>17</quant></Row>
+ <Row><id>2900485146</id><quant>18</quant></Row>
+ <Row><id>2995565825</id><quant>19</quant></Row>
+ <Row><id>3185727183</id><quant>20</quant></Row>
+ <Row><id>3375888541</id><quant>21</quant></Row>
+ <Row><id>3493252704</id><quant>22</quant></Row>
+ <Row><id>3661130578</id><quant>23</quant></Row>
+ <Row><id>3800778225</id><quant>24</quant></Row>
+ <Row><id>3990939583</id><quant>25</quant></Row>
+ <Row><id>4158817457</id><quant>26</quant></Row>
+</Dataset>

testing/regress/ecl/key/quantile5.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><rid>1</rid><ids><Row><id>6</id></Row></ids></Row>
+ <Row><rid>2</rid><ids><Row><id>2</id></Row></ids></Row>
+ <Row><rid>3</rid><ids><Row><id>76</id></Row></ids></Row>
+ <Row><rid>4</rid><ids><Row><id>16384</id></Row></ids></Row>
+ <Row><rid>5</rid><ids><Row><id>2</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/quantile6.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><rid>1</rid><id>6</id></Row>
+ <Row><rid>2</rid><id>2</id></Row>
+ <Row><rid>3</rid><id>76</id></Row>
+ <Row><rid>4</rid><id>16384</id></Row>
+ <Row><rid>5</rid><id>2</id></Row>
+</Dataset>

testing/regress/ecl/key/quantile6b.xml

@@ -0,0 +1,5 @@
+<Dataset name='Result 1'>
+ <Row><rid>1</rid><id>6</id></Row>
+ <Row><rid>4</rid><id>16384</id></Row>
+ <Row><rid>5</rid><id>2</id></Row>
+</Dataset>

testing/regress/ecl/key/quantile7.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><id>1000000000000000000</id></Row>
+</Dataset>

testing/regress/ecl/key/quantile8.xml

@@ -0,0 +1,4 @@
+<Dataset name='Result 1'>
+ <Row><id>-0.9375</id></Row>
+ <Row><id>0.9375</id></Row>
+</Dataset>

testing/regress/ecl/key/quantile9.xml

@@ -0,0 +1,7 @@
+<Dataset name='Result 1'>
+ <Row><rid>1</rid><numparts>2</numparts><ids><Row><id>6</id></Row></ids></Row>
+ <Row><rid>2</rid><numparts>3</numparts><ids><Row><id>1</id></Row><Row><id>1</id></Row></ids></Row>
+ <Row><rid>3</rid><numparts>5</numparts><ids><Row><id>31</id></Row><Row><id>61</id></Row><Row><id>91</id></Row><Row><id>121</id></Row></ids></Row>
+ <Row><rid>4</rid><numparts>3</numparts><ids><Row><id>10923</id></Row><Row><id>21846</id></Row></ids></Row>
+ <Row><rid>5</rid><numparts>7</numparts><ids><Row><id>1</id></Row><Row><id>1</id></Row><Row><id>1</id></Row><Row><id>1</id></Row><Row><id>1</id></Row><Row><id>1</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/quantile_e1.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>

testing/regress/ecl/quantile1.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt));
+END;
+
+show(virtual dataset({ unsigned id }) ds) := OUTPUT(SORT(ds, {id}));
+
+ascending := createDataset(100, 1, 0);
+ascendingBy3 := createDataset(100, 3, 0);
+descending := createDataset(100, -1, 99);
+
+show(QUANTILE(ascending, 10, {id}));    // 10, 20, 30, 40, 50, 60, 70, 80, 90  
+show(QUANTILE(ascending, 10, {id}, createQuantile(LEFT, COUNTER)));    // 10, 20, 30, 40, 50, 60, 70, 80, 90  
+show(QUANTILE(ascending, 10, {id}, createQuantile(LEFT, COUNTER), FIRST, LAST));        // 0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 99
+show(QUANTILE(descending, 10, {id}, createQuantile(LEFT, COUNTER), FIRST, LAST));        // 0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 99
+show(QUANTILE(ascendingBy3, 10, {id}, createQuantile(LEFT, COUNTER), FIRST, LAST));        // 0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 99
+
+//Calculate median
+show(QUANTILE(ascending, 2, {id}, createQuantile(LEFT, COUNTER)));        // 50
+show(QUANTILE(descending, 2, {id}, createQuantile(LEFT, COUNTER)));        // 50
+show(QUANTILE(ascendingBy3, 2, {id}, createQuantile(LEFT, COUNTER)));        // 50
+
+//Check that first/last work with median - may need to fall back to simplest implementation
+show(QUANTILE(ascendingBy3, 2, {id}, createQuantile(LEFT, COUNTER), FIRST, LAST));        // 50

testing/regress/ecl/quantile10.ecl

@@ -0,0 +1,69 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRecord := { real id; };
+
+quantRec := RECORD(rawRecord)
+    UNSIGNED quant;
+END;
+
+rawRecord createRaw(REAL id) := TRANSFORM
+    SELF.id := id;
+END;
+
+inRecord := RECORD
+    UNSIGNED rid;
+    UNSIGNED whichPercentile;
+    DATASET(rawRecord) ids;
+END;
+
+quantRec createQuantile(rawRecord l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, real scale, unsigned delta = 0) := FUNCTION
+    RETURN NOFOLD(SORT(DATASET(cnt, createRaw((COUNTER-1) * scale + delta), DISTRIBUTED), HASH(id)));
+END;
+
+inRecord createIn(unsigned rid, unsigned whichPercentile, unsigned cnt, real scale, unsigned delta) := TRANSFORM
+    SELF.rid := rid;
+    SELF.whichPercentile := whichPercentile; 
+    SELF.ids := createDataset(cnt, scale, delta);
+END;
+
+
+inDs := DATASET([
+            createIn(1, 2, 10, 1, 1),
+            createIn(2, 13, 50, 0.3, 1),
+            createIn(3, 15, 10, 15, 1),
+            createIn(4, 3, 32767, 1, 1),
+            createIn(5, 27, 99, 0.03, 1)
+            ]);
+
+//Check quantile inside a child query
+inRecord t(inRecord l) := TRANSFORM
+    SELF.ids := QUANTILE(l.ids, 100, { id }, RANGE([l.whichPercentile]));
+    SELF := l;
+END;
+
+
+output(PROJECT(inDs, t(LEFT)));

testing/regress/ecl/quantile11.ecl

@@ -0,0 +1,47 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt));
+END;
+
+show(virtual dataset({ unsigned id }) ds) := OUTPUT(SORT(ds, {id}));
+
+ascending100 := createDataset(100, 1, 0);
+show(QUANTILE(ascending100, 4, {id}));    // 25, 50, 75  
+ascending99 := createDataset(99, 1, 0);
+show(QUANTILE(ascending99, 4, {id}));    // 25, 49, 74  
+ascending101 := createDataset(101, 1, 0);
+show(QUANTILE(ascending101, 4, {id}));    // 25, 50, 76

testing/regress/ecl/quantile12.ecl

@@ -0,0 +1,59 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt), DISTRIBUTED);
+END;
+
+createHashDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(HASH32((COUNTER-1) * scale + delta)), DISTRIBUTED);
+END;
+
+calcQuantile(unsigned c, unsigned num, unsigned total) := IF(c = total, num, ((c-1) * num + (num DIV 2)) DIV total);
+
+//Does not work if num < count(in)
+simpleQuantile(dataset(rawRec) in, unsigned num, boolean first = false, boolean last = false) := FUNCTION
+    s := SORT(in, id);
+    q := PROJECT(s, createQuantile(LEFT, calcQuantile(COUNTER, num, COUNT(in))));
+    RETURN DEDUP(q, quant)(first OR quant!=0, last OR quant!=num);
+END;
+
+
+//Check that quantile on a sorted dataset produces the same results.
+ds100 := createDataset(100, 1, 0);
+ds100sort := SORTED(ds100, { id });
+output(QUANTILE(ds100sort, 5, { id }, createQuantile(LEFT, COUNTER), FIRST, LAST));
+output(QUANTILE(ds100sort, 27, { id }, createQuantile(LEFT, COUNTER)));

testing/regress/ecl/quantile13.ecl

@@ -0,0 +1,50 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt), DISTRIBUTED);
+END;
+
+rawRec createSkipMatch(rawRec l, UNSIGNED c) := TRANSFORM, SKIP(c IN [2,3,4])
+    SELF := l;
+END;
+
+//Check that skip on a quantile prevents the fixed numbers of outputs being generated.
+ds100 := createDataset(100, 1, 0);
+output(COUNT(QUANTILE(ds100, 5, { id }))); // 4
+output(COUNT(QUANTILE(ds100, 5, { id }, FIRST, LAST))); // 6
+output(COUNT(QUANTILE(ds100, 5, { id }, createSkipMatch(LEFT, COUNTER)))); // 1
+output(COUNT(QUANTILE(ds100, 5, { id }, createSkipMatch(LEFT, COUNTER), FIRST, LAST))); // 3

testing/regress/ecl/quantile1a.ecl

@@ -0,0 +1,74 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt), DISTRIBUTED);
+END;
+
+createHashDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(HASH32((COUNTER-1) * scale + delta)), DISTRIBUTED);
+END;
+
+calcQuantile(unsigned c, unsigned num, unsigned total) := IF(c = total, num, ((c-1) * num + (num DIV 2)) DIV total);
+
+//Does not work if num < count(in)
+//Version 1: Calculate which quantile each element should belong to
+simpleQuantile1(dataset(rawRec) in, unsigned num, boolean first = false, boolean last = false) := FUNCTION
+    s := SORT(in, id);
+    q := PROJECT(s, createQuantile(LEFT, calcQuantile(COUNTER, num, COUNT(in))));
+    RETURN DEDUP(q, quant)(first OR quant!=0, last OR quant!=num);
+END;
+
+//Version 1: Calculate which rank corresponds to which quantile, and then extract those
+simpleQuantile2(dataset(rawRec) in, unsigned num, boolean first = false, boolean last = false) := FUNCTION
+    numRows := COUNT(in);
+    s := SORT(in, id);
+    quantiles := DATASET(num+1, TRANSFORM({unsigned quant, unsigned idx}, SELF.quant := COUNTER-1; SELF.idx := IF(COUNTER = num+1, numRows, ((numRows * (COUNTER-1) + (num-1) DIV 2) DIV num)+1)));
+    q := PROJECT(s, createQuantile(LEFT, COUNTER));
+    j := JOIN(q, quantiles, LEFT.quant = RIGHT.idx, createQuantile(LEFT, RIGHT.quant), LOOKUP);
+    RETURN j(first OR quant!=0, last OR quant!=num);
+END;
+
+//Compare results of a trivial implementation with results from the activity
+ds10 := createDataset(10, 1, 0);
+output(simpleQuantile1(ds10, 4, first := true, last := true));      // (0,2,5,7,9)
+output(simpleQuantile2(ds10, 4, first := true, last := true));
+output(QUANTILE(ds10, 4, { id }, createQuantile(LEFT, COUNTER), FIRST, LAST));
+
+ds9 := createDataset(9, 1, 0);
+output(simpleQuantile1(ds9, 4, first := true, last := true));       // (0,2,4,7,8)
+output(simpleQuantile2(ds9, 4, first := true, last := true));
+output(QUANTILE(ds9, 4, { id }, createQuantile(LEFT, COUNTER), FIRST, LAST));

testing/regress/ecl/quantile1b.ecl

@@ -0,0 +1,80 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt), DISTRIBUTED);
+END;
+
+createHashDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(HASH32((COUNTER-1) * scale + delta)), DISTRIBUTED);
+END;
+
+calcQuantile(unsigned c, unsigned num, unsigned total) := IF(c = total, num, ((c-1) * num + (num DIV 2)) DIV total);
+
+//Does not work if num < count(in)
+//Version 1: Calculate which quantile each element should belong to
+simpleQuantile1(dataset(rawRec) in, unsigned num, boolean first = false, boolean last = false) := FUNCTION
+    s := SORT(in, id);
+    q := PROJECT(s, createQuantile(LEFT, calcQuantile(COUNTER, num, COUNT(in))));
+    RETURN DEDUP(q, quant)(first OR quant!=0, last OR quant!=num);
+END;
+
+//Version 1: Calculate which rank corresponds to which quantile, and then extract those
+simpleQuantile2(dataset(rawRec) in, unsigned num, boolean first = false, boolean last = false) := FUNCTION
+    numRows := COUNT(in);
+    s := SORT(in, id);
+    quantiles := DATASET(num+1, TRANSFORM({unsigned quant, unsigned idx}, SELF.quant := COUNTER-1; SELF.idx := IF(COUNTER = num+1, numRows, ((numRows * (COUNTER-1) + (num-1) DIV 2) DIV num)+1)));
+    q := PROJECT(s, createQuantile(LEFT, COUNTER));
+    j := JOIN(q, quantiles, LEFT.quant = RIGHT.idx, createQuantile(LEFT, RIGHT.quant), LOOKUP);
+    RETURN j(first OR quant!=0, last OR quant!=num);
+END;
+
+//Compare results of a trivial implementation with results from the activity
+ds9 := createDataset(9, 1, 0);
+output(simpleQuantile1(ds9, 3, first := true, last := true));      // (0,3,6,8)
+output(simpleQuantile2(ds9, 3, first := true, last := true));
+output(QUANTILE(ds9, 3, { id }, createQuantile(LEFT, COUNTER), FIRST, LAST));
+
+ds8 := createDataset(8, 1, 0);
+output(simpleQuantile1(ds8, 3, first := true, last := true));      // (0,3,5,7)
+output(simpleQuantile2(ds8, 3, first := true, last := true));
+output(QUANTILE(ds8, 3, { id }, createQuantile(LEFT, COUNTER), FIRST, LAST));
+
+ds10 := createDataset(10, 1, 0);
+output(simpleQuantile1(ds10, 3, first := true, last := true));      // (0,3,7,9)
+output(simpleQuantile2(ds10, 3, first := true, last := true));
+output(QUANTILE(ds10, 3, { id }, createQuantile(LEFT, COUNTER), FIRST, LAST));
+

testing/regress/ecl/quantile2.ecl

@@ -0,0 +1,71 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt));
+END;
+
+show(virtual dataset({ unsigned id }) ds) := OUTPUT(SORT(ds, {id}));
+
+nullDataset := DATASET([], rawRec);
+emptyDataset := NOFOLD(DATASET([], rawRec));
+singleDataset := NOFOLD(createDataset(1, 1, 0)); 
+smallDataset := createDataset(7, 3, 0); 
+
+show(QUANTILE(smallDataset, 2, {id}));    // 3
+
+//Two few entries in the input  
+show(QUANTILE(smallDataset, 10, {id}));    // 0,1,2,2,3,4,4,5,6 
+show(QUANTILE(smallDataset, 10, {id}, createQuantile(LEFT, COUNTER), FIRST, LAST));   // 0,0,1,2,2,3,4,4,5,6,6 
+show(QUANTILE(smallDataset, 10, {id}, createQuantile(LEFT, COUNTER), FIRST, LAST, DEDUP)); // 0,1,2,3,4,5,6    
+
+//Unusual values for the number of blocks
+show(QUANTILE(smallDataset, 0, {id}));  // nothing - illegal really  
+show(QUANTILE(smallDataset, 1, {id}));  // nothing  
+show(QUANTILE(smallDataset, NOFOLD(1), {id}));  // nothing  
+show(QUANTILE(smallDataset, NOFOLD(1), {id}, FIRST, LAST));  // 0,6  
+show(QUANTILE(smallDataset, NOFOLD(999999999), {id}, DEDUP));  // 0,1,2,3,4,5,6  
+show(QUANTILE(singleDataset, 5, {id}));  // 0,0,0,0  
+
+//Check null/empty datasets are not optimized away incorrectly
+show(QUANTILE(nullDataset, NOFOLD(3), {id}));  // 0, 0  
+show(QUANTILE(emptyDataset, NOFOLD(3), {id}));  // 0, 0  
+show(QUANTILE(nullDataset, NOFOLD(3), {id}, DEDUP));  // 0  
+show(QUANTILE(emptyDataset, NOFOLD(3), {id}, DEDUP));  // 0 
+
+show(QUANTILE(emptyDataset, NOFOLD(0), {id}));  // nothing  
+show(QUANTILE(emptyDataset, NOFOLD(1), {id}));  // nothing  
+show(QUANTILE(emptyDataset, NOFOLD(1), {id}, FIRST, LAST));  // 0,0  
+show(QUANTILE(emptyDataset, NOFOLD(1), {id}, FIRST, LAST, DEDUP));  // 0   

testing/regress/ecl/quantile3.ecl

@@ -0,0 +1,47 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt));
+END;
+
+show(virtual dataset({ unsigned id }) ds) := OUTPUT(SORT(ds, {id}));
+
+nullDataset := DATASET([], rawRec);
+dupsDataset := createDataset(47, 0, 0);
+
+show(QUANTILE(dupsDataset, 2, {id}));    // 0
+show(QUANTILE(dupsDataset, 3, {id}));    // 0,0
+show(QUANTILE(dupsDataset, 3, {id}, DEDUP));    // 0,0  - DEDUP does not remove duplicate entries, it prevents QUANTILE returning duplicated rows. 

testing/regress/ecl/quantile4.ecl

@@ -0,0 +1,60 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt), DISTRIBUTED);
+END;
+
+createHashDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(HASH32((COUNTER-1) * scale + delta)), DISTRIBUTED);
+END;
+
+calcQuantile(unsigned c, unsigned num, unsigned total) := IF(c = total, num, ((c-1) * num + (num DIV 2)) DIV total);
+
+//Does not work if num < count(in)
+simpleQuantile(dataset(rawRec) in, unsigned num, boolean first = false, boolean last = false) := FUNCTION
+    s := SORT(in, id);
+    q := PROJECT(s, createQuantile(LEFT, calcQuantile(COUNTER, num, COUNT(in))));
+    RETURN DEDUP(q, quant)(first OR quant!=0, last OR quant!=num);
+END;
+
+
+//Compare results of a trivial implementation with results from the activity
+ds100 := createHashDataset(100, 1, 0);
+output(simpleQuantile(ds100, 5, first := true, last := true));
+output(simpleQuantile(ds100, 27));
+output(QUANTILE(ds100, 5, { id }, createQuantile(LEFT, COUNTER), FIRST, LAST));
+output(QUANTILE(ds100, 27, { id }, createQuantile(LEFT, COUNTER)));

testing/regress/ecl/quantile5.ecl

@@ -0,0 +1,68 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRecord := { unsigned id; };
+
+quantRec := RECORD(rawRecord)
+    UNSIGNED4 quant;
+END;
+
+rawRecord createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+inRecord := RECORD
+    UNSIGNED rid;
+    DATASET(rawRecord) ids;
+END;
+
+quantRec createQuantile(rawRecord l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, real scale, unsigned delta = 0) := FUNCTION
+    RETURN NOFOLD(SORT(DATASET(cnt, createRaw((COUNTER-1) * scale + delta), DISTRIBUTED), HASH(id)));
+END;
+
+inRecord createIn(unsigned rid, unsigned cnt, real scale, unsigned delta) := TRANSFORM
+    SELF.rid := rid;
+    SELF.ids := createDataset(cnt, scale, delta);
+END;
+
+
+inDs := DATASET([
+            createIn(1, 10, 1, 1),
+            createIn(2, 10, 0.3, 1),
+            createIn(3, 10, 15, 1),
+            createIn(4, 32767, 1, 1),
+            createIn(5, 99, 0.03, 1)
+            ]);
+
+//Check quantile inside a child query
+selectMedian(dataset(rawRecord) inDs) := QUANTILE(inDs, 2, { id });
+
+inRecord t(inRecord l) := TRANSFORM
+    SELF.ids := selectMedian(l.ids);
+    SELF := l;
+END;
+
+output(PROJECT(inDs, t(LEFT)));

testing/regress/ecl/quantile6.ecl

@@ -0,0 +1,65 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRecord := { unsigned id; };
+
+quantRec := RECORD(rawRecord)
+    UNSIGNED4 quant;
+END;
+
+rawRecord createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+inRecord := RECORD
+    UNSIGNED rid;
+    DATASET(rawRecord) ids;
+END;
+
+quantRec createQuantile(rawRecord l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, real scale, unsigned delta = 0) := FUNCTION
+    RETURN NOFOLD(SORT(DATASET(cnt, createRaw((COUNTER-1) * scale + delta), DISTRIBUTED), HASH(id)));
+END;
+
+inRecord createIn(unsigned rid, unsigned cnt, real scale, unsigned delta) := TRANSFORM
+    SELF.rid := rid;
+    SELF.ids := createDataset(cnt, scale, delta);
+END;
+
+
+inDs := DATASET([
+            createIn(1, 10, 1, 1),
+            createIn(2, 10, 0.3, 1),
+            createIn(3, 10, 15, 1),
+            createIn(4, 32767, 1, 1),
+            createIn(5, 99, 0.03, 1)
+            ]);
+
+normRec := { UNSIGNED rid; UNSIGNED id, };
+norm := NORMALIZE(inDs, LEFT.ids, TRANSFORM(normRec, SELF := LEFT; SELF := RIGHT));
+
+//Grouped quantile..
+gr := GROUP(norm, rid);
+output(QUANTILE(gr, 2, { id }));

testing/regress/ecl/quantile6b.ecl

@@ -0,0 +1,70 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRecord := { unsigned id; };
+
+quantRec := RECORD(rawRecord)
+    UNSIGNED4 quant;
+END;
+
+rawRecord createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+inRecord := RECORD
+    UNSIGNED rid;
+    DATASET(rawRecord) ids;
+END;
+
+quantRec createQuantile(rawRecord l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, real scale, unsigned delta = 0) := FUNCTION
+    RETURN NOFOLD(SORT(DATASET(cnt, createRaw((COUNTER-1) * scale + delta), DISTRIBUTED), HASH(id)));
+END;
+
+inRecord createIn(unsigned rid, unsigned cnt, real scale, unsigned delta) := TRANSFORM
+    SELF.rid := rid;
+    SELF.ids := createDataset(cnt, scale, delta);
+END;
+
+
+inDs := DATASET([
+            createIn(1, 10, 1, 1),
+            createIn(2, 10, 0.3, 1),
+            createIn(3, 10, 15, 1),
+            createIn(4, 32767, 1, 1),
+            createIn(5, 99, 0.03, 1)
+            ]);
+
+normRecord := { UNSIGNED rid; UNSIGNED id, };
+norm := NORMALIZE(inDs, LEFT.ids, TRANSFORM(normRecord, SELF := LEFT; SELF := RIGHT));
+
+normRecord createSkipQuantile(normRecord l) := TRANSFORM,SKIP(l.rid IN [2,3])
+    SELF := l;
+END;
+
+//Grouped quantile..
+gr := GROUP(norm, rid);
+q := QUANTILE(gr, 2, { id }, createSkipQuantile(LEFT));
+output(q);

testing/regress/ecl/quantile7.ecl

@@ -0,0 +1,55 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+//version scale=10
+//version scale=100
+//version scale=1000
+//version scale=10000
+//version scale=100000
+//version scale=1000000
+//version scale=10000000
+
+import ^ as root;
+scale := #IFDEFINED(root.scale, 1000000);
+
+rawRec := { unsigned id; };
+
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, integer delta) := FUNCTION
+    prime := 181;
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * prime) % cnt + delta), DISTRIBUTED);
+END;
+
+midPoint := 1000000000000000000;
+inDs := createDataset(scale*2+1, midPoint - scale);
+output(QUANTILE(inDs, 2, { id }));

testing/regress/ecl/quantile8.ecl

@@ -0,0 +1,62 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+//version scale=32
+//version scale=0x400
+//version scale=0x10000
+//version scale=0x100000
+//version scale=0x1000000
+//xversion scale=0x10000000
+
+import ^ as root;
+scale := #IFDEFINED(root.scale, 1024);
+
+rawRec := { REAL id; };
+
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(real id) := TRANSFORM
+    SELF.id := id;
+END;
+
+quantRec createQuantile(rawRec l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+calcId(integer cnt, integer delta, unsigned c) := FUNCTION
+    prime := 181;
+    x1 := (c-1)*prime;
+    x2 := x1 % cnt;
+    x3 := (integer)x2;
+    x4 := (integer)(x3 - delta);
+    x5 := x4 / delta;
+    RETURN x5;
+END;
+    
+createDataset(integer cnt, integer delta) := FUNCTION
+    RETURN DATASET(cnt, createRaw(calcId(cnt, delta, COUNTER)), DISTRIBUTED);
+END;
+
+inDs := createDataset(scale*2, scale);
+output(QUANTILE(inDs, 64, { id }, RANGE([2,62])));

testing/regress/ecl/quantile9.ecl

@@ -0,0 +1,68 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRecord := { unsigned id; };
+
+quantRec := RECORD(rawRecord)
+    UNSIGNED4 quant;
+END;
+
+rawRecord createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+inRecord := RECORD
+    UNSIGNED rid;
+    UNSIGNED numParts;
+    DATASET(rawRecord) ids;
+END;
+
+quantRec createQuantile(rawRecord l, UNSIGNED quant) := TRANSFORM
+    SELF := l;
+    SELF.quant := quant;
+END;
+
+createDataset(unsigned cnt, real scale, unsigned delta = 0) := FUNCTION
+    RETURN NOFOLD(SORT(DATASET(cnt, createRaw((COUNTER-1) * scale + delta), DISTRIBUTED), HASH(id)));
+END;
+
+inRecord createIn(unsigned rid, unsigned numParts, unsigned cnt, integer scale, unsigned delta) := TRANSFORM
+    SELF.rid := rid;
+    SELF.numParts := numParts;
+    SELF.ids := createDataset(cnt, scale, delta);
+END;
+
+
+inDs := DATASET([
+            createIn(1, 2, 10, 1, 1),
+            createIn(2, 3, 10, 0.3, 1),
+            createIn(3, 5, 10, 15, 1),
+            createIn(4, 3, 32767, 1, 1),
+            createIn(5, 7, 99, 0.03, 1)
+            ]);
+
+//Check quantile inside a child query
+inRecord t(inRecord l) := TRANSFORM
+    SELF.ids := QUANTILE(l.ids, l.numParts, { id });
+    SELF := l;
+END;
+
+output(PROJECT(inDs, t(LEFT)));

testing/regress/ecl/quantile_e1.ecl

@@ -0,0 +1,48 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Temporarily disable tests on hthor/thor, until activity is implemented
+//nohthor
+//nothor
+
+rawRec := { unsigned id; };
+
+quantRec := RECORD(rawRec)
+    UNSIGNED4 quant;
+END;
+
+rawRec createRaw(UNSIGNED id) := TRANSFORM
+    SELF.id := id;
+END;
+
+createDataset(unsigned cnt, integer scale, unsigned delta = 0) := FUNCTION
+    RETURN DATASET(cnt, createRaw(((COUNTER-1) * scale + delta) % cnt));
+END;
+
+ascending := createDataset(100, 1, 0);
+
+integer zero := 0 : stored('zero');
+integer minusOne := -1 : stored('minusOne');
+integer oneHundred := 100 : stored('oneHundred');
+
+//Out of range number of items
+OUTPUT(QUANTILE(ascending, zero, {id}));  
+//Out of range number of items
+OUTPUT(QUANTILE(ascending, minusOne, {id}));  
+
+//Number of items is sensible, but range items are invalid
+OUTPUT(QUANTILE(ascending, 5, {id}, range([minusOne,oneHundred])));