Merge pull request #3962 from ghalliday/issue8748

HPCC-8748 Start treating constant rows as link counted

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlcpp.cpp

@@ -1712,6 +1712,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.normalizeSelectorSequence,"normalizeSelectorSequence",false),  // For tracking down why projects are not commoned up
         DebugOption(options.transformCaseToChoose,"transformCaseToChoose",true),
         DebugOption(options.removeXpathFromOutput,"removeXpathFromOutput",false),
+        DebugOption(options.canLinkConstantRows,"canLinkConstantRows",true),
     };
 
     //get options values from workunit
@@ -5864,17 +5865,12 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
             }
         case type_row:
             {
+                Owned<IReferenceSelector> selector = buildNewRow(ctx, castParam);
+
                 if (hasLinkCountedModifier(argType))
-                {
-                    doBuildAliasValue(ctx, castParam, bound);
-//                    buildTempExpr(ctx, castParam, bound, FormatLinkedDataset);
-                }
-                else
-                {
-                    Owned<IReferenceSelector> selector = buildNewRow(ctx, castParam);
-                    selector->buildAddress(ctx, bound);
-                }
-    //          buildExpr(ctx, castParam, bound);       // more this needs more work I think
+                    selector.setown(ensureLinkCountedRow(ctx, selector));
+
+                selector->buildAddress(ctx, bound);
                 break;
             }
         case type_set:

ecl/hqlcpp/hqlcpp.ipp

@@ -723,6 +723,7 @@ struct HqlCppOptions
     bool                normalizeSelectorSequence;
     bool                transformCaseToChoose;
     bool                removeXpathFromOutput;
+    bool                canLinkConstantRows;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class
@@ -831,6 +832,8 @@ public:
     IReferenceSelector * buildNewOrActiveRow(BuildCtx & ctx, IHqlExpression * expr, bool isNew);
     void buildRowAssign(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr);
     void buildRowAssign(BuildCtx & ctx, IReferenceSelector * target, IReferenceSelector * source);
+    BoundRow * ensureLinkCountedRow(BuildCtx & ctx, BoundRow * row);
+    IReferenceSelector * ensureLinkCountedRow(BuildCtx & ctx, IReferenceSelector * source);
 
 //Dataset processing.
     void buildAnyExpr(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);

ecl/hqlcpp/hqlcppds.cpp

@@ -148,6 +148,8 @@ void HqlCppTranslator::buildConstRow(IHqlExpression * record, IHqlExpression * r
 
     //MORE: This probably needs to go in the header as well...
     Owned<ITypeInfo> rowType = makeConstantModifier(makeRowType(record->getType()));
+    if (options.canLinkConstantRows)
+        rowType.setown(setLinkCountedAttr(rowType, true));
 
     StringBuffer rowName;
     getUniqueId(rowName.append("r"));
@@ -633,6 +635,52 @@ IReferenceSelector * HqlCppTranslator::buildActiveRow(BuildCtx & ctx, IHqlExpres
     return NULL; //remove warning about control paths
 }
 
+BoundRow * HqlCppTranslator::ensureLinkCountedRow(BuildCtx & ctx, BoundRow * row)
+{
+    if (row->isLinkCounted())
+        return row;
+
+    OwnedHqlExpr srcRow = createTranslated(row->queryBound());
+    Owned<BoundRow> tempRow = declareLinkedRow(ctx, row->queryDataset(), false);
+
+    OwnedHqlExpr source = getPointer(row->queryBound());
+    BuildCtx subctx(ctx);
+    if (row->isConditional())
+        subctx.addFilter(source);
+
+    IHqlExpression * sourceExpr = row->querySelector();
+    OwnedHqlExpr rowExpr = sourceExpr->isDataset() ? ensureActiveRow(sourceExpr) : LINK(sourceExpr);
+    OwnedHqlExpr size = createSizeof(rowExpr);
+    CHqlBoundExpr boundSize;
+    buildExpr(subctx, size, boundSize);
+
+    StringBuffer allocatorName;
+    ensureRowAllocator(allocatorName, ctx, row->queryRecord(), getCurrentActivityId(subctx));
+
+    StringBuffer s;
+    s.append("rtlCloneRow(").append(allocatorName).append(",");
+    generateExprCpp(s, boundSize.expr).append(",");
+    generateExprCpp(s, source);
+    s.append(")");
+    OwnedHqlExpr call = createQuoted(s, tempRow->queryBound()->queryType());
+
+    subctx.addAssign(tempRow->queryBound(), call);
+
+    ctx.associate(*tempRow);
+    return tempRow;
+}
+
+IReferenceSelector * HqlCppTranslator::ensureLinkCountedRow(BuildCtx & ctx, IReferenceSelector * source)
+{
+    if (!source->isRoot() || !source->queryRootRow()->isLinkCounted())
+    {
+        BoundRow * row = source->getRow(ctx);
+        BoundRow * lcrRow = ensureLinkCountedRow(ctx, row);
+        assertex(row != lcrRow);
+        return createReferenceSelector(lcrRow, source->queryExpr());
+    }
+    return LINK(source);
+}
 
 //---------------------------------------------------------------------------
 
@@ -1142,7 +1190,8 @@ void HqlCppTranslator::doBuildAggregateList(BuildCtx & ctx, const CHqlBoundTarge
     }
 
     ITypeInfo * elemType = list->queryType()->queryChildType();
-    if (!elemType) elemType = defaultIntegralType;
+    if (!elemType)
+        elemType = defaultIntegralType;
 
     //Default implementation in terms of a dataset
     OwnedHqlExpr field = createField(valueAtom, LINK(elemType), NULL);
@@ -2184,6 +2233,12 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             ctx.associateExpr(expr, tgt);
             return;
         }
+    case no_if:
+        if (::canEvaluateInline(&ctx, expr->queryChild(1)) && ::canEvaluateInline(&ctx, expr->queryChild(2)))
+        {
+            buildTempExpr(ctx, expr, tgt, format);
+            return;
+        }
     }
 
     if (expr->isDictionary())
@@ -2331,7 +2386,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
         {
             CHqlBoundExpr bound;
             buildDataset(ctx, expr, bound, isArrayRowset(target.queryType()) ? FormatLinkedDataset : FormatBlockedDataset);
-            if (hasWrapperModifier(target.queryType()))
+            if (hasWrapperModifier(target.queryType()) && hasLinkCountedModifier(target.queryType()))
             {
                 OwnedHqlExpr complex = bound.getComplexExpr();
                 ctx.addAssign(target.expr, complex);
@@ -2347,7 +2402,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
     case no_inlinetable:
         {
             //This will typically generate a loop.  If few items then it is more efficient to expand the assigns/clones out.
-            if (expr->queryChild(0)->numChildren() > INLINE_TABLE_EXPAND_LIMIT)
+            if (options.canLinkConstantRows || (expr->queryChild(0)->numChildren() > INLINE_TABLE_EXPAND_LIMIT))
             {
                 CHqlBoundExpr bound;
                 if (doBuildDatasetInlineTable(ctx, expr, bound, FormatNatural))
@@ -2739,6 +2794,8 @@ bool HqlCppTranslator::doBuildDatasetInlineTable(BuildCtx & ctx, IHqlExpression
 
     Owned<ITypeInfo> tableType = makeConstantModifier(makeArrayType(LINK(rowType), maxRows));
     OwnedITypeInfo rowsType = makeOutOfLineModifier(makeTableType(LINK(rowType), NULL, NULL, NULL));
+    if (options.canLinkConstantRows)
+        rowsType.setown(setLinkCountedAttr(rowsType, true));
 
     OwnedHqlExpr table = declareCtx.getTempDeclare(tableType, values);
     if (options.spanMultipleCpp)
@@ -3859,7 +3916,7 @@ void HqlCppTranslator::doBuildRowAssignAggregateNext(BuildCtx & ctx, IReferenceS
             break;
         }
         if (targetSelect->queryType()->getSize() == UNKNOWN_LENGTH)
-           isVariableOffset = true;
+            isVariableOffset = true;
     }
     if (alreadyDoneExpr)
         buildAssignToTemp(ctx, alreadyDoneExpr, queryBoolExpr(true));

ecl/hqlcpp/hqlcset.cpp

@@ -1696,7 +1696,7 @@ bool LinkedDatasetBuilderBase::buildLinkRow(BuildCtx & ctx, BoundRow * sourceRow
         OwnedHqlExpr rowExpr = sourceExpr->isDataset() ? ensureActiveRow(sourceExpr) : LINK(sourceExpr);
         OwnedHqlExpr size = createSizeof(rowExpr);
         CHqlBoundExpr boundSize;
-        translator.buildExpr(ctx, size, boundSize);
+        translator.buildExpr(subctx, size, boundSize);
 
         StringBuffer s;
         s.append(instanceName).append(".cloneRow(");

ecl/hqlcpp/hqlhtcpp.cpp

@@ -3370,9 +3370,15 @@ void HqlCppTranslator::ensureRowAllocator(StringBuffer & allocatorName, BuildCtx
     StringBuffer uid;
     getUniqueId(uid.append("alloc"));
 
-    BuildCtx * declarectx = &ctx;
-    BuildCtx * callctx = &ctx;
-    getInvariantMemberContext(ctx, &declarectx, &callctx, true, false);
+    BuildCtx subctx(ctx);
+    BuildCtx * declarectx = &subctx;
+    BuildCtx * callctx = &subctx;
+    if (!getInvariantMemberContext(ctx, &declarectx, &callctx, true, false))
+    {
+        //The following will not currently work because not all compound statements are correctly marked as
+        //complete/incomplete
+        //subctx.selectOutermostScope();
+    }
 
     StringBuffer s;
     s.append("Owned<IEngineRowAllocator> ").append(uid).append(";");

ecl/hqlcpp/hqlinline.cpp

@@ -303,8 +303,11 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
     }
     case no_inlinetable:
         {
-            if (transformListContainsSkip(expr->queryChild(0)))
+            IHqlExpression * transforms = expr->queryChild(0);
+            if (transformListContainsSkip(transforms))
                 return 0;
+            if (isConstantDataset(expr))
+                return RETevaluate;
             return RETassign;
         }
     case no_createrow:

ecl/regress/inlinedatasetif.ecl

@@ -0,0 +1,52 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+namesTable1 := dataset([
+        {'Smith','Gavin',31},
+        {'Smythe','Mia',30},
+        {'Smiph','Pru',10},
+        {'Y','Z'}], namesRecord);
+
+namesTable2 := dataset([
+        {'Hawthorn','Gavin',31},
+        {'Hawthorn','Mia',30},
+        {'Smithe','Pru',10},
+        {'X','Z'}], namesRecord);
+
+boolean bool1 := true : stored('bool1');
+
+
+ds3 := DATASET([1,2,3,4,10,99],{integer searchAge});
+
+r := { integer age; string surname; };
+
+r t(ds3 l) := TRANSFORM
+    ds := IF(l.searchAge % 2 = 1, namesTable1, namesTable2);
+    SELF.age := l.searchAge;
+    SELF.surname := ds(age = l.searchAge)[1].surname;
+END;
+
+p := PROJECT(ds3, t(LEFT));
+
+output(p);

rtl/eclrtl/rtlds.cpp

@@ -499,6 +499,24 @@ void appendRowsToRowset(size32_t & targetCount, byte * * & targetRowset, IEngine
     }
 }
 
+const void * rtlCloneRow(IEngineRowAllocator * rowAllocator, size32_t len, const void * row)
+{
+    RtlDynamicRowBuilder builder(rowAllocator);
+
+    byte * self = builder.ensureCapacity(len, NULL);
+    memcpy(self, row, len);
+
+    IOutputMetaData * meta = rowAllocator->queryOutputMeta();
+    if (meta->getMetaFlags() & MDFneeddestruct)
+    {
+        RtlChildRowLinkerWalker walker;
+        meta->walkIndirectMembers(self, walker);
+    }
+
+    return builder.finalizeRowClear(len);
+}
+
+
 
 //---------------------------------------------------------------------------
 

rtl/eclrtl/rtlds_imp.hpp

@@ -324,6 +324,7 @@ protected:
     size32_t suffix;
 };
 
+extern ECLRTL_API const void * rtlCloneRow(IEngineRowAllocator * rowAllocator, size32_t len, const void * row);
 
 //---------------------------------------------------------------------------