HPCC-17179 Add FEW option to LOOP

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlgram.y

@@ -9824,6 +9824,7 @@ loopOptions
 
 loopOption
     : commonAttribute
+    | FEW               {   $$.setExpr(createAttribute(fewAtom), $1); }
     ;
 
 graphOptions

ecl/hqlcpp/hqlcpp.ipp

@@ -1317,7 +1317,7 @@ public:
     void buildReturnOrder(BuildCtx & ctx, IHqlExpression *sortList, const DatasetReference & dataset);
 
     IHqlExpression * createLoopSubquery(IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * filter, IHqlExpression * again, IHqlExpression * counter, bool multiInstance, unsigned & loopAgainResult);
-    unique_id_t buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, bool multiInstance);
+    unique_id_t buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, bool multiInstance, bool unlimitedResources);
     unique_id_t buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset);
         
     void doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt, IHqlExpression * expr, CHqlBoundExpr * result);
@@ -1926,7 +1926,7 @@ protected:
     double getComplexity(IHqlExpression * expr, ClusterType cluster);
     bool prepareToGenerate(HqlQueryContext & query, WorkflowArray & exprs, bool isEmbeddedLibrary);
     IHqlExpression * getResourcedGraph(IHqlExpression * expr, IHqlExpression * graphIdExpr);
-    IHqlExpression * getResourcedChildGraph(BuildCtx & ctx, IHqlExpression * childQuery, unsigned numResults, node_operator graphKind);
+    IHqlExpression * getResourcedChildGraph(BuildCtx & ctx, IHqlExpression * childQuery, unsigned numResults, node_operator graphKind, bool unlimitedResources);
     IHqlExpression * optimizeCompoundSource(IHqlExpression * expr, unsigned flags);
     IHqlExpression * optimizeGraphPostResource(IHqlExpression * expr, unsigned csfFlags, bool projectBeforeSpill);
     bool isInlineOk();

ecl/hqlcpp/hqlcppds.cpp

@@ -1512,7 +1512,7 @@ void ChildGraphBuilder::generateGraph(BuildCtx & ctx)
     //Remove this line once all engines use the new child queries exclusively
     if (numResults == 0) numResults++;
 
-    OwnedHqlExpr resourced = translator.getResourcedChildGraph(graphctx, childQuery, numResults, no_none);
+    OwnedHqlExpr resourced = translator.getResourcedChildGraph(graphctx, childQuery, numResults, no_none, false);
 
     Owned<ParentExtract> extractBuilder = translator.createExtractBuilder(graphctx, PETchild, represents, resourced, true);
     if (!translator.queryOptions().serializeRowsetInExtract)
@@ -1560,7 +1560,7 @@ void ChildGraphBuilder::generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * re
     BuildCtx aliasctx(ctx);
     aliasctx.addGroup();
 
-    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, childQuery, numResults, no_none);
+    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, childQuery, numResults, no_none, false);
 
     Owned<ParentExtract> extractBuilder = translator.createExtractBuilder(ctx, PETchild, represents, resourced, false);
     createBuilderAlias(aliasctx, extractBuilder);
@@ -1586,12 +1586,12 @@ void ChildGraphBuilder::createBuilderAlias(BuildCtx & ctx, ParentExtract * extra
     ctx.addQuoted(s);
 }
 
-unique_id_t ChildGraphBuilder::buildLoopBody(BuildCtx & ctx, bool multiInstance)
+unique_id_t ChildGraphBuilder::buildLoopBody(BuildCtx & ctx, bool multiInstance, bool unlimitedResources)
 {
     BuildCtx subctx(ctx);
     subctx.addGroup();
 
-    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, childQuery, numResults, no_loop);
+    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, childQuery, numResults, no_loop, unlimitedResources);
     //Add a flag to indicate multi instance
     if (multiInstance)
         resourced.setown(appendOwnedOperand(resourced, createAttribute(multiInstanceAtom)));
@@ -1687,14 +1687,14 @@ protected:
 
 
 
-unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, bool isParallel)
+unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, bool isParallel, bool unlimitedResources)
 {
     BuildCtx subctx(ctx);
     subctx.addGroup();
 
     IHqlExpression * query = childQuery->queryChild(2);
     translator.traceExpression("Before Loop resource", query);
-    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, childQuery, numResults, no_loop);
+    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, childQuery, numResults, no_loop, unlimitedResources);
     translator.traceExpression("After Loop resource", resourced);
 
     //Add a flag to indicate multi instance
@@ -1725,7 +1725,7 @@ unique_id_t ChildGraphBuilder::buildRemoteGraph(BuildCtx & ctx)
     BuildCtx subctx(ctx);
     subctx.addGroup();
 
-    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, childQuery, numResults, no_allnodes);
+    OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, childQuery, numResults, no_allnodes, false);
 
     Owned<ParentExtract> extractBuilder = translator.createExtractBuilder(ctx, PETremote, represents, GraphRemote, false);
 
@@ -1786,7 +1786,7 @@ void HqlCppTranslator::buildAssignChildDataset(BuildCtx & ctx, const CHqlBoundTa
 }
 
 
-IHqlExpression * HqlCppTranslator::getResourcedChildGraph(BuildCtx & ctx, IHqlExpression * childQuery, unsigned numResults, node_operator graphKind)
+IHqlExpression * HqlCppTranslator::getResourcedChildGraph(BuildCtx & ctx, IHqlExpression * childQuery, unsigned numResults, node_operator graphKind, bool unlimitedResources)
 {
     if (options.paranoidCheckNormalized || options.paranoidCheckDependencies)
         DBGLOG("Before resourcing a child graph");
@@ -1835,7 +1835,7 @@ IHqlExpression * HqlCppTranslator::getResourcedChildGraph(BuildCtx & ctx, IHqlEx
     if (graphKind == no_loop)
     {
         bool insideChild = insideChildQuery(ctx);
-        resourced.setown(resourceLoopGraph(*this, activeRows, resourced, targetClusterType, graphIdExpr, numResults, insideChild));
+        resourced.setown(resourceLoopGraph(*this, activeRows, resourced, targetClusterType, graphIdExpr, numResults, insideChild, unlimitedResources));
     }
     else
         resourced.setown(resourceNewChildGraph(*this, activeRows, resourced, targetClusterType, graphIdExpr, numResults));
@@ -1879,7 +1879,7 @@ void HqlCppTranslator::buildChildDataset(BuildCtx & ctx, IHqlExpression * expr,
 }
 
 
-unique_id_t HqlCppTranslator::buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, bool multiInstance)
+unique_id_t HqlCppTranslator::buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, bool multiInstance, bool unlimitedResources)
 {
     ChildGraphExprBuilder graphBuilder(0);
 
@@ -1901,7 +1901,7 @@ unique_id_t HqlCppTranslator::buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpress
     OwnedHqlExpr subquery = graphBuilder.getGraph();
 
     ChildGraphBuilder builder(*this, subquery);
-    return builder.buildGraphLoopBody(ctx, multiInstance);
+    return builder.buildGraphLoopBody(ctx, multiInstance, unlimitedResources);
 }
 
 unique_id_t HqlCppTranslator::buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset)

ecl/hqlcpp/hqlcppds.hpp

@@ -48,8 +48,8 @@ class ChildGraphBuilder : public CInterface
 public:
     ChildGraphBuilder(HqlCppTranslator & _translator, IHqlExpression * subgraph);
 
-    unique_id_t buildGraphLoopBody(BuildCtx & ctx, bool multiInstance);
-    unique_id_t buildLoopBody(BuildCtx & ctx, bool multiInstance);
+    unique_id_t buildGraphLoopBody(BuildCtx & ctx, bool multiInstance, bool unlimitedResources);
+    unique_id_t buildLoopBody(BuildCtx & ctx, bool multiInstance, bool unlimitedResources);
     unique_id_t buildRemoteGraph(BuildCtx & ctx);
     void generateGraph(BuildCtx & ctx);
     void generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * retGraphExpr);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -8705,7 +8705,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityLoop(BuildCtx & ctx, IHqlExpre
     OwnedHqlExpr childquery = createLoopSubquery(dataset, selSeq, rowsid, body->queryChild(0), filter, loopCond, counter, (parallel != NULL), loopAgainResult);
 
     ChildGraphBuilder builder(*this, childquery);
-    unique_id_t loopId = builder.buildLoopBody(subctx, (parallel != NULL));
+    unique_id_t loopId = builder.buildLoopBody(subctx, (parallel != NULL), getBoolAttribute(expr, fewAtom));
     instance->addAttributeInt("_loopid", loopId);
 
     if (loopAgainResult)
@@ -8757,7 +8757,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGraphLoop(BuildCtx & ctx, IHql
     //output dataset is result 0
     //input dataset is fed in using result 1
     //counter (if required) is fed in using result 2[0].counter;
-    unique_id_t loopId = buildGraphLoopSubgraph(subctx, dataset, selSeq, rowsid, body->queryChild(0), counter, (parallel != NULL));
+    unique_id_t loopId = buildGraphLoopSubgraph(subctx, dataset, selSeq, rowsid, body->queryChild(0), counter, (parallel != NULL), getBoolAttribute(expr, fewAtom));
     instance->addAttributeInt("_loopid", loopId);
 
     buildInstanceSuffix(instance);

ecl/hqlcpp/hqlresource.cpp

@@ -3475,7 +3475,7 @@ EclResourcer::EclResourcer(IErrorReceiver & _errors, IConstWorkUnit * _wu, const
         break;
     }
 
-    if (_translatorOptions.unlimitedResources)
+    if (_translatorOptions.unlimitedResources || options.unlimitedResources)
     {
         resourceLimit->set(RESheavy, 0xffffffff).set(REShashdist, 0xffffffff);
         resourceLimit->set(RESmastersocket, 0xffffffff).set(RESslavememory,0xffffffff);
@@ -6548,7 +6548,7 @@ IHqlExpression * resourceThorGraph(HqlCppTranslator & translator, IHqlExpression
 
 static IHqlExpression * doResourceGraph(HqlCppTranslator & translator, HqlExprCopyArray * activeRows, IHqlExpression * _expr,
                                         ClusterType targetClusterType, unsigned clusterSize,
-                                        IHqlExpression * graphIdExpr, unsigned numResults, bool isChild, bool useGraphResults)
+                                        IHqlExpression * graphIdExpr, unsigned numResults, bool isChild, bool useGraphResults, bool unlimitedResources)
 {
     LinkedHqlExpr expr = _expr;
     HqlExprArray transformed;
@@ -6556,6 +6556,8 @@ static IHqlExpression * doResourceGraph(HqlCppTranslator & translator, HqlExprCo
     CResourceOptions options(targetClusterType, clusterSize, translator.queryOptions(), translator.querySpillSequence());
     if (isChild)
         options.setChildQuery(true);
+    if (unlimitedResources)
+        options.unlimitedResources = true;
     options.setNewChildQuery(graphIdExpr, numResults);
     options.setUseGraphResults(useGraphResults);
 
@@ -6588,18 +6590,18 @@ static IHqlExpression * doResourceGraph(HqlCppTranslator & translator, HqlExprCo
 
 IHqlExpression * resourceLibraryGraph(HqlCppTranslator & translator, IHqlExpression * expr, ClusterType targetClusterType, unsigned clusterSize, IHqlExpression * graphIdExpr, unsigned numResults)
 {
-    return doResourceGraph(translator, NULL, expr, targetClusterType, clusterSize, graphIdExpr, numResults, false, true);       //?? what value for isChild (e.g., thor library call).  Need to gen twice?
+    return doResourceGraph(translator, NULL, expr, targetClusterType, clusterSize, graphIdExpr, numResults, false, true, false);       //?? what value for isChild (e.g., thor library call).  Need to gen twice?
 }
 
 
 IHqlExpression * resourceNewChildGraph(HqlCppTranslator & translator, HqlExprCopyArray & activeRows, IHqlExpression * expr, ClusterType targetClusterType, IHqlExpression * graphIdExpr, unsigned numResults)
 {
-    return doResourceGraph(translator, &activeRows, expr, targetClusterType, 0, graphIdExpr, numResults, true, true);
+    return doResourceGraph(translator, &activeRows, expr, targetClusterType, 0, graphIdExpr, numResults, true, true, false);
 }
 
-IHqlExpression * resourceLoopGraph(HqlCppTranslator & translator, HqlExprCopyArray & activeRows, IHqlExpression * expr, ClusterType targetClusterType, IHqlExpression * graphIdExpr, unsigned numResults, bool insideChildQuery)
+IHqlExpression * resourceLoopGraph(HqlCppTranslator & translator, HqlExprCopyArray & activeRows, IHqlExpression * expr, ClusterType targetClusterType, IHqlExpression * graphIdExpr, unsigned numResults, bool insideChildQuery, bool unlimitedResources)
 {
-    return doResourceGraph(translator, &activeRows, expr, targetClusterType, 0, graphIdExpr, numResults, insideChildQuery, true);
+    return doResourceGraph(translator, &activeRows, expr, targetClusterType, 0, graphIdExpr, numResults, insideChildQuery, true, unlimitedResources);
 }
 
 IHqlExpression * resourceRemoteGraph(HqlCppTranslator & translator, IHqlExpression * _expr, ClusterType targetClusterType, unsigned clusterSize)

ecl/hqlcpp/hqlresource.hpp

@@ -23,7 +23,7 @@
 
 IHqlExpression * resourceThorGraph(HqlCppTranslator & translator, IHqlExpression * expr, ClusterType targetClusterType, unsigned clusterSize, IHqlExpression * graphIdExpr);
 IHqlExpression * resourceLibraryGraph(HqlCppTranslator & translator, IHqlExpression * expr, ClusterType targetClusterType, unsigned clusterSize, IHqlExpression * graphIdExpr, unsigned numResults);
-IHqlExpression * resourceLoopGraph(HqlCppTranslator & translator, HqlExprCopyArray & activeRows, IHqlExpression * expr, ClusterType targetClusterType, IHqlExpression * graphIdExpr, unsigned numResults, bool insideChildQuery);
+IHqlExpression * resourceLoopGraph(HqlCppTranslator & translator, HqlExprCopyArray & activeRows, IHqlExpression * expr, ClusterType targetClusterType, IHqlExpression * graphIdExpr, unsigned numResults, bool insideChildQuery, bool unlimitedResources);
 IHqlExpression * resourceNewChildGraph(HqlCppTranslator & translator, HqlExprCopyArray & activeRows, IHqlExpression * expr, ClusterType targetClusterType, IHqlExpression * graphIdExpr, unsigned numResults);
 IHqlExpression * resourceRemoteGraph(HqlCppTranslator & translator, IHqlExpression * expr, ClusterType targetClusterType, unsigned clusterSize);
 

ecl/hqlcpp/hqlresource.ipp

@@ -85,6 +85,7 @@ public:
     bool     spillMultiCondition;
     bool     spotThroughAggregate;
     bool     alwaysReuseGlobalSpills;
+    bool     unlimitedResources = false;
 
     IHqlExpression * graphIdExpr;
     unsigned nextResult;

testing/regress/ecl/looprpn2.ecl

@@ -15,7 +15,6 @@
     limitations under the License.
 ############################################################################## */
 
-//nothor
 stepRecord := 
             record
 string          next{maxlength(20)};
@@ -60,7 +59,7 @@ processExpression(dataset(stepRecord) actions) := function
     END;
                 
     initial := dataset([], stateRecord);
-    result := LOOP(initial, count(actions), processNext(rows(left), counter, actions[NOBOUNDCHECK counter]));
+    result := LOOP(initial, count(actions), processNext(rows(left), counter, actions[NOBOUNDCHECK counter]), FEW);
     return result;
 end;