HPCC-12267 Add QUANTILE support to the code generator

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/thorhelper/thorcommon.cpp

@@ -779,6 +779,7 @@ extern const char * getActivityText(ThorActivityKind kind)
     case TAKselfdenormalize:        return "Self Denormalize";
     case TAKselfdenormalizegroup:   return "Self Denormalize Group";
     case TAKtrace:                  return "Trace";
+    case TAKquantile:               return "Quantile";
     }
     throwUnexpected();
 }

ecl/hqlcpp/hqlcpp.ipp

@@ -1480,6 +1480,7 @@ public:
     ABoundActivity * doBuildActivityProject(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityProcess(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityPullActivity(BuildCtx & ctx, IHqlExpression * expr);
+    ABoundActivity * doBuildActivityQuantile(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityRegroup(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityRemote(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);
     ABoundActivity * doBuildActivityReturnResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -6539,6 +6539,9 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_map:
                 result = doBuildActivityCase(ctx, expr, isRoot);
                 break;
+            case no_quantile:
+                result = doBuildActivityQuantile(ctx, expr);
+                break;
             case no_chooseds:
             case no_choose:
                 result = doBuildActivityChoose(ctx, expr, isRoot);
@@ -16584,6 +16587,99 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySort(BuildCtx & ctx, IHqlExpre
     return instance->getBoundActivity();
 }
 
+ABoundActivity * HqlCppTranslator::doBuildActivityQuantile(BuildCtx & ctx, IHqlExpression * expr)
+{
+    IHqlExpression * dataset = expr->queryChild(0);
+    Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
+
+    Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKquantile, expr, "Quantile");
+    buildActivityFramework(instance);
+    buildInstancePrefix(instance);
+
+    IHqlExpression * number = expr->queryChild(1);
+    IHqlExpression * sortlist = expr->queryChild(2);
+    IHqlExpression * transform = expr->queryChild(3);
+    IHqlExpression * score = queryAttributeChild(expr, scoreAtom, 0);
+    IHqlExpression * skew = queryAttributeChild(expr, skewAtom, 0);
+    IHqlExpression * dedupAttr = expr->queryAttribute(dedupAtom);
+    IHqlExpression * range = queryAttributeChild(expr, rangeAtom, 0);
+
+    instance->classctx.addQuotedLiteral("virtual ICompare * queryCompare() { return &compare; }");
+
+    buildCompareClass(instance->nestedctx, "compare", sortlist, DatasetReference(dataset));
+
+    doBuildUnsigned64Function(instance->startctx, "getNumDivisions", number);
+
+    if (skew)
+        doBuildFunction(instance->startctx, doubleType, "getSkew", skew);
+
+    if (range)
+    {
+        Owned<ITypeInfo> setType = makeSetType(makeIntType(8, false));
+        doBuildFunction(instance->startctx, setType, "getRange", range);
+    }
+
+    if (score)
+    {
+        BuildCtx funcctx(instance->startctx);
+        funcctx.addQuotedCompound("virtual unsigned __int64 getScore(const void * _self)");
+        funcctx.addQuotedLiteral("unsigned char * self = (unsigned char *) _self;");
+        bindTableCursor(funcctx, dataset, "self");
+        buildReturn(funcctx, score);
+    }
+
+    IHqlExpression * counter = queryAttributeChild(expr, _countProject_Atom, 0);
+    IHqlExpression * selSeq = querySelSeq(expr);
+    {
+        BuildCtx funcctx(instance->startctx);
+        funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, unsigned __int64 counter)");
+        if (counter)
+            associateCounter(funcctx, counter, "counter");
+        buildTransformBody(funcctx, transform, dataset, NULL, instance->dataset, selSeq);
+    }
+
+    buildClearRecordMember(instance->createctx, "", dataset);
+
+    //If a dataset is sorted by all fields then it is impossible to determine if the original order
+    //was preserved - so mark the sort as potentially unstable (to reduce memory usage at runtime)
+    bool unstable = expr->hasAttribute(unstableAtom);
+    if (options.optimizeSortAllFields &&
+        allFieldsAreSorted(expr->queryRecord(), sortlist, dataset->queryNormalizedSelector(), options.optimizeSortAllFieldsStrict))
+        unstable = true;
+
+    StringBuffer flags;
+    if (expr->hasAttribute(firstAtom))
+        flags.append("|TQFfirst");
+    if (expr->hasAttribute(lastAtom))
+        flags.append("|TQFlast");
+    if (isAlreadySorted(dataset, sortlist, false, false))
+        flags.append("|TQFsorted|TQFlocalsorted");
+    else if (isAlreadySorted(dataset, sortlist, true, false))
+        flags.append("|TQFlocalsorted");
+    if (score)
+        flags.append("|TQFhasscore");
+    if (range)
+        flags.append("|TQFhasrange");
+    if (skew)
+        flags.append("|TQFhasskew");
+    if (dedupAttr)
+        flags.append("|TQFdedup");
+    if (unstable)
+        flags.append("|TQFunstable");
+    if (!number->queryValue())
+        flags.append("|TQFvariabledivisions");
+    if (!transformReturnsSide(expr, no_left, 0))
+        flags.append("|TQFneedtransform");
+
+    if (flags.length())
+        instance->classctx.addQuotedF("virtual unsigned getFlags() { return %s; }", flags.str()+1);
+
+    buildInstanceSuffix(instance);
+
+    buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
+    return instance->getBoundActivity();
+}
+
 //---------------------------------------------------------------------------
 
 void HqlCppTranslator::doBuildXmlReadMember(ActivityInstance & instance, IHqlExpression * expr, const char * functionName, bool & usesContents)

ecl/hqlcpp/hqliproj.cpp

@@ -2006,6 +2006,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_createrow:
     case no_rollupgroup:
     case no_projectrow:
+    case no_quantile:
         return CreateRecordActivity;
     case no_inlinetable:
     case no_dataset_from_transform: