HPCC-21642 Fix various minor codegen issues, generally had no no symptoms

* Improved offset calculation for a few archived queries.

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlgram.y

@@ -8270,7 +8270,7 @@ simpleDataSet
                             OwnedHqlExpr active = ensureActiveRow(ds);
                             //Expand row components (see parser->processSortList() for similar code for hash)
                             HqlExprArray components;
-                            RecordSelectIterator iter(active->queryRecord(), active);
+                            RecordSelectIterator iter(active->queryRecord(), active, false);
                             ForEach(iter)
                                 components.append(*iter.get());
                             OwnedHqlExpr sortlist = createSortList(components);

ecl/hql/hqlgram2.cpp

@@ -6085,7 +6085,7 @@ IHqlExpression * HqlGram::processSortList(const attribute & errpos, node_operato
         {
             //Expanding the row selectors at this point generates better code if the hash is done inside a compound read activity
             LinkedHqlExpr row = &e;
-            RecordSelectIterator iter(e.queryRecord(), row);
+            RecordSelectIterator iter(e.queryRecord(), row, false);
             items.remove(idx);
             unsigned i=0;
             ForEach(iter)

ecl/hql/hqlmeta.cpp

@@ -3578,7 +3578,7 @@ extern HQL_API bool allFieldsAreSorted(IHqlExpression * record, IHqlExpression *
     markValidSelectors(sortOrder, dsSelector);
 
     //Now expand all the selectors from the record, and check that they have been tagged
-    RecordSelectIterator iter(record, dsSelector);
+    RecordSelectIterator iter(record, dsSelector, false);
     ForEach(iter)
     {
         IHqlExpression * select = iter.query();

ecl/hql/hqltrans.cpp

@@ -1509,7 +1509,7 @@ IHqlExpression * NewHqlTransformer::createTransformed(IHqlExpression * expr)
             same = transformChildren(expr, children);
             IHqlExpression & ds = children.item(0);
             node_operator dsOp = ds.getOperator();
-            if (dsOp == no_activetable)
+            if (dsOp == no_activerow)
             {
                 children.replace(*LINK(ds.queryChild(0)), 0);
                 removeAttribute(children, newAtom);

ecl/hql/hqlutil.cpp

@@ -1462,7 +1462,7 @@ IHqlExpression * JoinSortInfo::getContiguousJoinCondition(unsigned numRhsFields)
     //Ensure that numRhsFields from RIGHT are joined, and if so return the join condition
     IHqlExpression * rightRecord = rhs->queryRecord();
     HqlExprCopyArray leftMatches, rightMatches;
-    RecordSelectIterator iter(rightRecord, queryActiveTableSelector());
+    RecordSelectIterator iter(rightRecord, queryActiveTableSelector(), false);
     unsigned numMatched = 0;
     ForEach(iter)
     {
@@ -2137,7 +2137,7 @@ void expandRowSelectors(HqlExprArray & target, HqlExprArray const & source)
         IHqlExpression & cur = source.item(i);
         if (cur.isDatarow())
         {
-            RecordSelectIterator iter(cur.queryRecord(), &cur);
+            RecordSelectIterator iter(cur.queryRecord(), &cur, false);
             ForEach(iter)
                 target.append(*iter.get());
         }
@@ -3320,7 +3320,7 @@ extern HQL_API bool introducesNewDependencies(IHqlExpression * oldExpr, IHqlExpr
 //---------------------------------------------------------------------------
 
 
-RecordSelectIterator::RecordSelectIterator(IHqlExpression * record, IHqlExpression * selector)
+RecordSelectIterator::RecordSelectIterator(IHqlExpression * record, IHqlExpression * selector, bool newSelect) : rootNewSelect(newSelect)
 {
     rootRecord.set(record);
     rootSelector.set(selector);
@@ -3351,18 +3351,21 @@ bool RecordSelectIterator::doNext()
                 beginRecord(cur);
                 break;
             case no_field:
+            {
+                bool isNew = (rootNewSelect && (nestingDepth == 0));
                 switch (cur->queryType()->getTypeCode())
                 {
                 case type_row:
                     beginRecord(cur->queryRecord());
-                    selector.setown(createSelectExpr(LINK(selector), LINK(cur)));
+                    selector.setown(createSelectExpr(LINK(selector), LINK(cur), isNew));
                     nestingDepth++;
                     break;
                 default:
-                    curSelector.setown(createSelectExpr(LINK(selector), LINK(cur)));
+                    curSelector.setown(createSelectExpr(LINK(selector), LINK(cur), isNew));
                     return true;
                 }
                 break;
+            }
             case no_ifblock:
                 beginRecord(cur->queryChild(1));
                 ifblockDepth++;

ecl/hql/hqlutil.hpp

@@ -440,7 +440,7 @@ public:
 class HQL_API RecordSelectIterator
 {
 public:
-    RecordSelectIterator(IHqlExpression * record, IHqlExpression * selector);
+    RecordSelectIterator(IHqlExpression * record, IHqlExpression * selector, bool newSelect);
 
     bool first();
     bool next();
@@ -464,6 +464,7 @@ protected:
     UnsignedArray indices;
     unsigned ifblockDepth;
     unsigned nestingDepth;
+    bool rootNewSelect;
 };
 
 //===========================================================================

ecl/hqlcpp/hqlckey.cpp

@@ -346,9 +346,9 @@ void KeyedJoinInfo::buildClearRightFunction(BuildCtx & classctx)
         
         BoundRow * selfCursor = translator.bindSelf(func.ctx, rawKey, "crSelf");
         IHqlExpression * rawSelf = selfCursor->querySelector();
-        RecordSelectIterator rawIter(rawKey->queryRecord(), rawSelf);
+        RecordSelectIterator rawIter(rawKey->queryRecord(), rawSelf, false);
 
-        RecordSelectIterator keyIter(key->queryRecord(), key);
+        RecordSelectIterator keyIter(key->queryRecord(), key, false);
         buildClearRecord(func.ctx, rawIter, keyIter);
         translator.buildReturnRecordSize(func.ctx, selfCursor);
     }

ecl/hqlcpp/hqlcpp.cpp

@@ -8404,12 +8404,12 @@ void HqlCppTranslator::expandOrder(IHqlExpression * expr, HqlExprArray & leftVal
 }
 
 
-IHqlExpression * HqlCppTranslator::querySimpleOrderSelector(IHqlExpression * expr)
+IHqlExpression * HqlCppTranslator::querySimpleOrderSelector(IHqlExpression * expr, bool & isNew)
 {
     if (expr->getOperator() != no_select)
         return NULL;
 
-    return queryDatasetCursor(expr);
+    return querySelectorDataset(expr, isNew);
 }
 
 static unsigned getMemcmpSize(IHqlExpression * left, IHqlExpression * right, bool isEqualityCompare)
@@ -8452,8 +8452,10 @@ void HqlCppTranslator::optimizeOrderValues(HqlExprArray & leftValues, HqlExprArr
     {
         IHqlExpression * curFirstLeft = &leftValues.item(i);
         IHqlExpression * curFirstRight = &rightValues.item(i);
-        IHqlExpression * leftSel = querySimpleOrderSelector(curFirstLeft);
-        IHqlExpression * rightSel = querySimpleOrderSelector(curFirstRight);
+        bool leftIsNew;
+        bool rightIsNew;
+        IHqlExpression * leftSel = querySimpleOrderSelector(curFirstLeft, leftIsNew);
+        IHqlExpression * rightSel = querySimpleOrderSelector(curFirstRight, rightIsNew);
         if (!leftSel || !rightSel)
             continue;
 
@@ -8461,21 +8463,23 @@ void HqlCppTranslator::optimizeOrderValues(HqlExprArray & leftValues, HqlExprArr
         if (!compareSize)
             continue;
 
+        bool nextLeftIsNew;
+        bool nextRightIsNew;
         IHqlExpression * nextLeft = &leftValues.item(i+1);
         IHqlExpression * nextRight = &rightValues.item(i+1);
-        if (querySimpleOrderSelector(nextLeft) != leftSel || querySimpleOrderSelector(nextRight) != rightSel ||
+        if (querySimpleOrderSelector(nextLeft, nextLeftIsNew) != leftSel || querySimpleOrderSelector(nextRight, nextRightIsNew) != rightSel ||
             (getMemcmpSize(nextLeft, nextRight, isEqualityCompare) == 0))
             continue;
 
         //Worth iterating the selectors...
-        RecordSelectIterator leftIter(leftSel->queryRecord(), leftSel);
+        RecordSelectIterator leftIter(leftSel->queryRecord(), leftSel, leftIsNew);
         ForEach(leftIter)
             if (leftIter.query() == curFirstLeft)
                 break;
         if (!leftIter.isValid() || leftIter.isInsideIfBlock())
             continue;
 
-        RecordSelectIterator rightIter(rightSel->queryRecord(), rightSel);
+        RecordSelectIterator rightIter(rightSel->queryRecord(), rightSel, rightIsNew);
         ForEach(rightIter)
             if (rightIter.query() == curFirstRight)
                 break;

ecl/hqlcpp/hqlcpp.ipp

@@ -1894,7 +1894,7 @@ protected:
     void expandSimpleOrder(IHqlExpression * left, IHqlExpression * right, HqlExprArray & leftValues, HqlExprArray & rightValues);
     void expandOrder(IHqlExpression * expr, HqlExprArray & leftValues, HqlExprArray & rightValues, SharedHqlExpr & defaultValue);
     void optimizeOrderValues(HqlExprArray & leftValues, HqlExprArray & rightValues, bool isEqualityCompare);
-    IHqlExpression * querySimpleOrderSelector(IHqlExpression * expr);
+    IHqlExpression * querySimpleOrderSelector(IHqlExpression * expr, bool & isNew);
 
     unsigned doBuildThorChildSubGraph(BuildCtx & ctx, IHqlExpression * expr, SubGraphType kind, unsigned thisId=0, IHqlExpression * represents=NULL);
     unsigned doBuildThorSubGraph(BuildCtx & ctx, IHqlExpression * expr, SubGraphType kind, unsigned thisId=0, IHqlExpression * represents=NULL);

ecl/hqlcpp/hqlecl.cpp

@@ -386,7 +386,7 @@ bool HqlDllGenerator::generateFullFieldUsageStatistics(HqlCppTranslator & transl
     //Now project the output dataset down to a single field, and generate the file/field dependencies for each of those
     OwnedHqlExpr selSeq = createUniqueSelectorSequence();
     OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
-    RecordSelectIterator iter(dataset->queryRecord(), left);
+    RecordSelectIterator iter(dataset->queryRecord(), left, false);
     ForEach(iter)
     {
         IHqlExpression * cur = iter.query();

ecl/hqlcpp/hqlhtcpp.cpp

@@ -2825,7 +2825,10 @@ IReferenceSelector * DatasetSelector::select(BuildCtx & ctx, IHqlExpression * se
     else if (selectExpr->queryChild(0) == path)
         selected.set(selectExpr);
     else
-        selected.setown(createSelectExpr(LINK(path), LINK(selectExpr->queryChild(1))));
+    {
+        bool isNew = selectExpr->hasAttribute(newAtom);
+        selected.setown(createSelectExpr(LINK(path), LINK(selectExpr->queryChild(1)), isNew));
+    }
 
     IHqlExpression * selectedField = selected->queryChild(1);
     AColumnInfo * newColumn = resolveField(selectedField);
@@ -14248,7 +14251,7 @@ static void gatherDedupCompareExpr(HqlExprArray & equalities, HqlExprArray & com
 
 void optimizeGroupOrder(HqlExprArray & optimized, IHqlExpression * dataset, HqlExprArray & exprs)
 {
-    RecordSelectIterator iter(dataset->queryRecord(), dataset->queryNormalizedSelector());
+    RecordSelectIterator iter(dataset->queryRecord(), dataset->queryNormalizedSelector(), false);
     ForEach(iter)
     {
         IHqlExpression * select = iter.query();
@@ -15310,7 +15313,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySelectNew(BuildCtx & ctx, IHql
         bindTableCursor(func.ctx, anon->queryNormalizedSelector(), "left");
 
         OwnedHqlExpr activeAnon = ensureActiveRow(anon);
-        OwnedHqlExpr value = replaceSelectorDataset(expr, activeAnon);
+        OwnedHqlExpr value = replaceExpression(expr, dataset, activeAnon);
         buildAssign(func.ctx, selfCursor->querySelector(), value);
 
         buildReturnRecordSize(func.ctx, selfCursor);

ecl/hqlcpp/hqlresource.cpp

@@ -1344,7 +1344,7 @@ IHqlExpression * ActivityInvariantHoister::projectChildDependent(IHqlExpression
         while (fields.contains(*projectedField))
             projectedField.setown(cloneFieldMangleName(field));  // Generates a new mangled name each time
 
-        OwnedHqlExpr activeDs = createRow(no_activetable, LINK(ds->queryNormalizedSelector()));
+        OwnedHqlExpr activeDs = createRow(no_activerow, LINK(ds->queryNormalizedSelector()));
         fields.append(*LINK(projectedField));
         values.append(*replaceSelector(value, row, activeDs));
     }

ecl/hqlcpp/hqlttcpp.cpp

@@ -8320,6 +8320,7 @@ IHqlExpression * NewScopeMigrateTransformer::createTransformed(IHqlExpression *
         return ret;
 
     OwnedHqlExpr transformed = HoistingHqlTransformer::createTransformed(expr);
+    updateOrphanedSelectors(transformed, expr);
 
     ScopeMigrateInfo * extra = queryBodyExtra(expr);
     node_operator op = expr->getOperator();