HPCC-13602 Add NOCOMBINE a less intrusive version of NOFOLD

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlattr.cpp

@@ -593,6 +593,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_rangecommon:
     case no_nofold:
     case no_nohoist:
+    case no_nocombine:
     case no_section:
     case no_sectioninput:
     case no_alias:
@@ -625,7 +626,7 @@ unsigned getOperatorMetaFlags(node_operator op)
 
     case no_unused6:
     case no_unused13: case no_unused14: case no_unused15:
-    case no_unused25: case no_unused28: case no_unused29:
+    case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
@@ -2528,6 +2529,7 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
     case no_executewhen:
     case no_owned_ds:
     case no_dataset_alias:
+    case no_nocombine:
         {
             return getRecordCountInfo(ds);
         }

ecl/hql/hqlexpr.cpp

@@ -1247,6 +1247,7 @@ const char *getOpString(node_operator op)
     case no_template_context: return "no_template_context";
     case no_nofold: return "NOFOLD";
     case no_nohoist: return "NOHOIST";
+    case no_nocombine: return "NOCOMBINE";
     case no_fail: return "FAIL";
     case no_filepos: return "no_filepos";
     case no_file_logicalname: return "no_file_logicalname";
@@ -1522,7 +1523,7 @@ const char *getOpString(node_operator op)
 
     case no_unused6:
     case no_unused13: case no_unused14: case no_unused15:
-    case no_unused25: case no_unused28: case no_unused29:
+    case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
@@ -1948,7 +1949,6 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_preload:
     case no_limit:
     case no_catchds:
-    case no_forcegraph:
     case no_owned_ds:
     case no_dataset_alias:
     case no_split:
@@ -1985,6 +1985,8 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_globalscope:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:
     case no_thor:
     case no_catch:
@@ -2234,7 +2236,6 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_graphloop:
     case no_extractresult:
     case no_filtergroup:
-    case no_forcegraph:
     case no_normalizegroup:
     case no_owned_ds:
     case no_dataset_alias:
@@ -2319,6 +2320,8 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_globalscope:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:
     case no_thor:
     case no_pipe:
@@ -2451,6 +2454,8 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_metaactivity:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:
     case no_sample:
     case no_sort:
@@ -2496,7 +2501,6 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_assert_ds:
     case no_spillgraphresult:
     case no_sectioninput:
-    case no_forcegraph:
     case no_related:
     case no_outofline:
     case no_fieldmap:
@@ -3257,6 +3261,8 @@ void CHqlExpression::initFlagsBeforeOperands()
         infoFlags2 &= ~HEF2constant;
         break;
     case no_nofold:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:            // not so sure about this...
     case no_sectioninput:
     case no_wuid:
@@ -10810,6 +10816,8 @@ IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
         }
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_thor:
     case no_nothor:
     case no_alias:

ecl/hql/hqlexpr.hpp

@@ -363,7 +363,7 @@ enum _node_operator {
         no_any,
         no_existsdict,
         no_quantile,
-    no_unused25,
+        no_nocombine,
     no_unused28,  
     no_unused29,
     no_unused30,

ecl/hql/hqlfold.cpp

@@ -6021,6 +6021,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_owned_ds:
     case no_dataset_alias:
     case no_createdictionary:
+    case no_nocombine:
         exprMapping.set(gatherConstants(expr->queryChild(0)));
         break;
     case no_normalizegroup:

ecl/hql/hqlgram.y

@@ -312,6 +312,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   NAMESPACE
   NOBOUNDCHECK
   NOCASE
+  NOCOMBINE
   NOFOLD
   NOHOIST
   NOLOCAL
@@ -2568,6 +2569,10 @@ actionStmt
                         {
                             $$.setExpr(createValue(no_nofold, makeVoidType(), $3.getExpr()), $1);
                         }
+    | NOCOMBINE '(' action ')'
+                        {
+                            $$.setExpr(createValue(no_nocombine, makeVoidType(), $3.getExpr()), $1);
+                        }
     | NOTHOR '(' action ')'
                         {
                             $$.setExpr(createValue(no_nothor, makeVoidType(), $3.getExpr()), $1);
@@ -5887,6 +5892,12 @@ primexpr1
                             IHqlExpression * expr = $3.getExpr();
                             $$.setExpr(createValue(no_nofold, expr->getType(), expr));
                         }
+    | NOCOMBINE '(' expression ')'
+                        {
+                            parser->normalizeExpression($3);
+                            IHqlExpression * expr = $3.getExpr();
+                            $$.setExpr(createValue(no_nocombine, expr->getType(), expr));
+                        }
     | NOHOIST '(' expression ')'
                         {
                             parser->normalizeExpression($3);
@@ -7286,6 +7297,11 @@ simpleDataRow
                             $$.setExpr(createRow(no_nofold, $3.getExpr(), NULL));
                             $$.setPosition($1);
                         }
+    | NOCOMBINE '(' dataRow ')'
+                        {
+                            $$.setExpr(createRow(no_nocombine, $3.getExpr(), NULL));
+                            $$.setPosition($1);
+                        }
     | NOHOIST '(' dataRow ')'
                         {
                             $$.setExpr(createRow(no_nohoist, $3.getExpr(), NULL));
@@ -7382,6 +7398,11 @@ simpleDictionary
                             $$.setExpr(createDictionary(no_nofold, $3.getExpr(), NULL));
                             $$.setPosition($1);
                         }
+    | NOCOMBINE '(' dictionary ')'
+                        {
+                            $$.setExpr(createDictionary(no_nocombine, $3.getExpr(), NULL));
+                            $$.setPosition($1);
+                        }
     | NOHOIST '(' dictionary ')'
                         {
                             $$.setExpr(createDictionary(no_nohoist, $3.getExpr(), NULL));
@@ -8363,6 +8384,11 @@ simpleDataSet
                             $$.setExpr(createDataset(no_nofold, $3.getExpr(), NULL));
                             $$.setPosition($1);
                         }
+    | NOCOMBINE '(' dataSet ')'
+                        {
+                            $$.setExpr(createDataset(no_nocombine, $3.getExpr(), NULL));
+                            $$.setPosition($1);
+                        }
     | NOHOIST '(' dataSet ')'
                         {
                             $$.setExpr(createDataset(no_nohoist, $3.getExpr(), NULL));

ecl/hql/hqlgram2.cpp

@@ -10630,6 +10630,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case NAMESPACE: msg.append("NAMESPACE"); break;
     case NOBOUNDCHECK: msg.append("NOBOUNDCHECK"); break;
     case NOCASE: msg.append("NOCASE"); break;
+    case NOCOMBINE: msg.append("NOCOMBINE"); break;
     case NOFOLD: msg.append("NOFOLD"); break;
     case NOHOIST: msg.append("NOHOIST"); break;
     case NOLOCAL: msg.append("NOLOCAL"); break;
@@ -10935,7 +10936,7 @@ void HqlGram::simplifyExpected(int *expected)
     //simplify checks if the first item in the list is expected next, and if so it removes all of the others as expected tokens.
     simplify(expected, DISTRIBUTE, DISTRIBUTE, ASCII, CHOOSEN, CHOOSESETS, DEDUP, DISTRIBUTED, EBCDIC, ENTH, SAMPLE, SORT, SORTED, TABLE, DATASET, FETCH,
                        GROUP, GROUPED, KEYED, UNGROUP, JOIN, PULL, ROLLUP, ITERATE, PROJECT, NORMALIZE, PIPE, DENORMALIZE, CASE, MAP, 
-                       HTTPCALL, SOAPCALL, LIMIT, PARSE, FAIL, MERGE, PRELOAD, ROW, TOPN, ALIAS, LOCAL, NOFOLD, NOHOIST, NOTHOR, IF, GLOBAL, __COMMON__, __COMPOUND__, TOK_ASSERT, _EMPTY_,
+                       HTTPCALL, SOAPCALL, LIMIT, PARSE, FAIL, MERGE, PRELOAD, ROW, TOPN, ALIAS, LOCAL, NOFOLD, NOCOMBINE, NOHOIST, NOTHOR, IF, GLOBAL, __COMMON__, __COMPOUND__, TOK_ASSERT, _EMPTY_,
                        COMBINE, ROWS, REGROUP, XMLPROJECT, SKIP, LOOP, CLUSTER, NOLOCAL, REMOTE, PROCESS, ALLNODES, THISNODE, GRAPH, MERGEJOIN, STEPPED, NONEMPTY, HAVING,
                        TOK_CATCH, '@', SECTION, WHEN, IFF, COGROUP, HINT, INDEX, PARTITION, AGGREGATE, SUBSORT, TOK_ERROR, CHOOSE, TRACE, QUANTILE, 0);
     simplify(expected, EXP, ABS, SIN, COS, TAN, SINH, COSH, TANH, ACOS, ASIN, ATAN, ATAN2, 

ecl/hql/hqlir.cpp

@@ -283,7 +283,7 @@ const char * getOperatorIRText(node_operator op)
     EXPAND_CASE(no,any);
     EXPAND_CASE(no,existsdict);
     EXPAND_CASE(no,quantile);
-    EXPAND_CASE(no,unused25);
+    EXPAND_CASE(no,nocombine);
     EXPAND_CASE(no,unused28);
     EXPAND_CASE(no,unused29);
     EXPAND_CASE(no,unused30);

ecl/hql/hqllex.l

@@ -808,6 +808,7 @@ __NAMEOF__          { RETURNSYM(NAMEOF); }
 NAMESPACE           { RETURNSYM(NAMESPACE); }
 NOBOUNDCHECK        { RETURNSYM(NOBOUNDCHECK); }
 NOCASE              { RETURNSYM(NOCASE); }
+NOCOMBINE           { RETURNSYM(NOCOMBINE); }
 NOFOLD              { RETURNSYM(NOFOLD); }
 NOHOIST             { RETURNSYM(NOHOIST); }
 NOLOCAL             { RETURNSYM(NOLOCAL); }

ecl/hql/hqlmeta.cpp

@@ -1945,6 +1945,8 @@ CHqlMetaProperty * querySimpleDatasetMeta(IHqlExpression * expr)
     case no_keyed:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:
     case no_sectioninput:
     case no_sub:
@@ -1990,7 +1992,6 @@ CHqlMetaProperty * querySimpleDatasetMeta(IHqlExpression * expr)
     case no_thisnode:
     case no_forcelocal:
     case no_filtergroup:
-    case no_forcegraph:
     case no_related:
     case no_executewhen:
     case no_outofline:
@@ -2058,6 +2059,8 @@ void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr)
     case no_keyed:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:
     case no_sectioninput:
     case no_sub:
@@ -2103,7 +2106,6 @@ void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr)
     case no_thisnode:
     case no_forcelocal:
     case no_filtergroup:
-    case no_forcegraph:
     case no_related:
     case no_executewhen:
     case no_outofline:
@@ -3243,6 +3245,8 @@ ITypeInfo * calculateDatasetType(node_operator op, const HqlExprArray & parms)
     case no_keyed:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:
     case no_sectioninput:
     case no_sub:
@@ -3289,7 +3293,6 @@ ITypeInfo * calculateDatasetType(node_operator op, const HqlExprArray & parms)
     case no_thisnode:
     case no_forcelocal:
     case no_filtergroup:
-    case no_forcegraph:
     case no_related:
     case no_executewhen:
     case no_outofline:

ecl/hql/hqlopt.cpp

@@ -2235,6 +2235,8 @@ IHqlExpression * CTreeOptimizer::queryMoveKeyedExpr(IHqlExpression * transformed
     case no_grouped:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_filter:
         {
             OwnedHqlExpr newChild = queryMoveKeyedExpr(child);

ecl/hql/hqlthql.cpp

@@ -2041,6 +2041,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             }
         case no_nofold:
         case no_nohoist:
+        case no_nocombine:
         case no_selectfields:
             if (expandProcessed)
                 defaultToECL(expr, s, inType);

ecl/hqlcpp/hqlckey.cpp

@@ -363,6 +363,8 @@ IHqlExpression * KeyedJoinInfo::querySimplifiedKey(IHqlExpression * expr)
         case no_assertgrouped:
         case no_assertdistributed:
         case no_nofold:
+        case no_forcegraph:
+        case no_nocombine:
             break;
         case no_newkeyindex:
             return LINK(expr);

ecl/hqlcpp/hqlcpp.cpp

@@ -2385,6 +2385,8 @@ void HqlCppTranslator::buildExprAssign(BuildCtx & ctx, const CHqlBoundTarget & t
     case no_nohoist:
     case no_section:
     case no_sectioninput:
+    case no_forcegraph:
+    case no_nocombine:
         buildExprAssign(ctx, target, expr->queryChild(0));
         break;
     case no_realformat:
@@ -3251,6 +3253,8 @@ void HqlCppTranslator::buildExpr(BuildCtx & ctx, IHqlExpression * expr, CHqlBoun
     case no_section:
     case no_sectioninput:
     case no_pure:
+    case no_forcegraph:
+    case no_nocombine:
         buildExpr(ctx, expr->queryChild(0), tgt);
         return;
     case no_band:
@@ -3633,6 +3637,8 @@ void HqlCppTranslator::buildStmt(BuildCtx & _ctx, IHqlExpression * expr)
     case no_nothor:
     case no_section:
     case no_sectioninput:
+    case no_forcegraph:
+    case no_nocombine:
         buildStmt(ctx, expr->queryChild(0));
         return;
     case no_null:

ecl/hqlcpp/hqlcppds.cpp

@@ -560,6 +560,8 @@ IReferenceSelector * HqlCppTranslator::buildNewRow(BuildCtx & ctx, IHqlExpressio
     case no_datasetfromrow:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:
     case no_sectioninput:
         return buildNewRow(ctx, expr->queryChild(0));
@@ -2163,6 +2165,7 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
     case no_sorted:
     case no_nofold:
     case no_nohoist:
+    case no_nocombine:
     case no_section:
     case no_sectioninput:
         buildDataset(ctx, expr->queryChild(0), tgt, format);
@@ -2469,6 +2472,8 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
     case no_sorted:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:
     case no_sectioninput:
         buildDatasetAssign(ctx, target, expr->queryChild(0));
@@ -3475,6 +3480,8 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder
     case no_sorted:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
     case no_section:
     case no_sectioninput:
         buildDatasetAssign(subctx, target, expr->queryChild(0));
@@ -3936,6 +3943,8 @@ BoundRow * HqlCppTranslator::buildDatasetIterate(BuildCtx & ctx, IHqlExpression
     case no_sorted:
     case no_nofold:
     case no_nohoist:
+    case no_forcegraph:
+    case no_nocombine:
         return buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
     case no_sectioninput:
     case no_section:
@@ -4439,6 +4448,8 @@ void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, IReferenceSelector * targe
         doBuildRowAssignNullRow(ctx, target, expr);
         return;
     case no_nofold:
+    case no_forcegraph:
+    case no_nocombine:
         buildRowAssign(ctx, target, expr->queryChild(0));
         return;
     case no_serialize:

ecl/hqlcpp/hqlhtcpp.cpp

@@ -6593,6 +6593,7 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_grouped:
             case no_nofold:
             case no_nohoist:
+            case no_nocombine:
             case no_globalscope:
             case no_thisnode:
             case no_forcegraph:
@@ -8187,6 +8188,27 @@ static void unwindAddFiles(HqlExprArray & args, IHqlExpression * expr, bool isOr
         args.append(*LINK(expr));
 }
 
+static IHqlExpression * queryRootConcatActivity(IHqlExpression * expr)
+{
+    loop
+    {
+        node_operator curOp = expr->getOperator();
+        switch (curOp)
+        {
+        case no_nofold:
+        case no_section:
+        case no_sectioninput:
+        case no_preservemeta:
+        case no_nocombine:
+        case no_forcegraph:
+            break;
+        default:
+            return expr;
+        }
+        expr = expr->queryChild(0);
+    }
+}
+
 ABoundActivity * HqlCppTranslator::doBuildActivityConcat(BuildCtx & ctx, IHqlExpression * expr)
 {
     HqlExprArray inExprs;
@@ -8202,13 +8224,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityConcat(BuildCtx & ctx, IHqlExp
         IHqlExpression * cur = &inExprs.item(idx);
         bound.append(*buildCachedActivity(ctx, cur));
 
-        loop
-        {
-            node_operator curOp = cur->getOperator();
-            if ((curOp != no_nofold) && (curOp != no_section) && (curOp != no_sectioninput) && (curOp != no_preservemeta))
-                break;
-            cur = cur->queryChild(0);
-        }
+        cur = queryRootConcatActivity(cur);
 
         switch (cur->getOperator())
         {
@@ -18882,6 +18898,7 @@ static bool needsRealThor(IHqlExpression *expr, unsigned flags)
     case no_forcegraph:
     case no_nofold:
     case no_nohoist:
+    case no_nocombine:
     case no_actionlist:
     case no_orderedactionlist:
     case no_compound_fetch:

ecl/hqlcpp/hqlinline.cpp

@@ -272,6 +272,7 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
     case no_preservemeta:
     case no_nofold:
     case no_nohoist:
+    case no_nocombine:
     case no_alias_scope:
     case no_serialize:
     case no_deserialize:
@@ -721,6 +722,7 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr, bool optimize
     case no_subgraph:
     case no_nofold:
     case no_nohoist:
+    case no_nocombine:
     case no_regroup:
     case no_sorted:
     case no_distributed:

ecl/hqlcpp/hqliproj.cpp

@@ -2148,6 +2148,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_nonempty:
     case no_cogroup:
     case no_chooseds:
+    case no_nocombine:
         return PassThroughActivity;
     case no_keydiff:
     case no_keypatch:

ecl/hqlcpp/hqlresource.cpp

@@ -3146,6 +3146,7 @@ bool ResourcerInfo::expandRatherThanSpill(bool noteOtherSpills)
         case no_preservemeta:
         case no_nofold:
         case no_nohoist:
+        case no_nocombine:
         case no_section:
         case no_sectioninput:
         case no_dataset_alias:

ecl/hqlcpp/hqlsource.cpp

@@ -201,6 +201,7 @@ bool isSimpleSource(IHqlExpression * expr)
         case no_sectioninput:
         case no_nofold:
         case no_nohoist:
+        case no_nocombine:
         case no_dataset_alias:
             break;
         default:
@@ -970,6 +971,7 @@ void SourceBuilder::analyse(IHqlExpression * expr)
     case no_sectioninput:
     case no_nofold:
     case no_nohoist:
+    case no_nocombine:
     case no_dataset_alias:
         break;
     case no_preload:
@@ -1359,6 +1361,7 @@ void SourceBuilder::buildTransformElements(BuildCtx & ctx, IHqlExpression * expr
     case no_sectioninput:
     case no_nofold:
     case no_nohoist:
+    case no_nocombine:
         break;
     case no_filter:
         {

ecl/hqlcpp/hqlttcpp.cpp

@@ -98,6 +98,7 @@ static bool isWorthHoisting(IHqlExpression * expr, bool asSubQuery)
         case no_preservemeta:
         case no_nofold:
         case no_nohoist:
+        case no_nocombine:
         case no_section:
         case no_sectioninput:
         case no_dataset_alias:
@@ -7198,6 +7199,7 @@ bool ScalarGlobalTransformer::isCandidate(IHqlExpression * expr)
         switch (op)
         {
         case no_nofold:
+        case no_nocombine:
             //try to hoist the thing that is nofolded instead
             return false;
         default:

ecl/regress/nocombine.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+ds := DATASET(100, TRANSFORM({ unsigned id, unsigned value }, SELF.id := COUNTER; SELF.value := 10;));
+
+dsNoFold := NOFOLD(ds)(value between 10 and 20);
+dsNoCombine := NOCOMBINE(ds)(value between 10 and 20);
+
+sequential(
+    output(dsNoFold, { value });
+    output(dsNoCombine, { value }); // check fields are projected and value are percolated
+    output(count(dsNoFold));
+    output(count(dsNoCombine));
+);