Merge pull request #7942 from ghalliday/issue14052

HPCC-14052 Fix error on implicit project of MERGE(,SORTED(childrow.field))

Reviewed-By: Jamie Noss <james.noss@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlpmap.hpp

@@ -173,6 +173,7 @@ extern HQL_API IHqlExpression * updateMappedFields(IHqlExpression * expr, IHqlEx
 extern HQL_API void replaceSelectors(HqlExprArray & out, unsigned first, IHqlExpression * oldDataset, IHqlExpression * newDataset);
 extern HQL_API IHqlExpression * scopedReplaceSelector(IHqlExpression * expr, IHqlExpression * oldDataset, IHqlExpression * newDataset);
 extern HQL_API IHqlExpression * replaceSelfRefSelector(IHqlExpression * expr, IHqlExpression * newDataset);
+extern HQL_API IHqlExpression * updateActiveSelectorFields(IHqlExpression * expr, IHqlExpression * oldRecord, IHqlExpression * newRecord, unsigned firstChild);
 
 extern HQL_API bool isNullProject(IHqlExpression * expr, bool canIgnorePayload, bool canLoseFieldsFromEnd);
 extern HQL_API bool isSimpleProject(IHqlExpression * expr);                             // Restriction or rearrangement only

ecl/hql/hqltrans.cpp

@@ -3212,6 +3212,11 @@ void NewSelectorReplacingTransformer::setNestedMapping(IHqlExpression * oldSel,
     }
 }
 
+void NewSelectorReplacingTransformer::setActiveSelectorMapping(IHqlExpression * oldRecord, IHqlExpression * newRecord)
+{
+    setNestedMapping(queryActiveTableSelector(), queryActiveTableSelector(), oldRecord->querySimpleScope(), newRecord, false);
+}
+
 void NewSelectorReplacingTransformer::setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlExpression * oldRecord, bool isSelector)
 {
     if (isSelector)
@@ -3355,6 +3360,34 @@ IHqlExpression * updateMappedFields(IHqlExpression * expr, IHqlExpression * oldS
     return expr->clone(args);
 }
 
+IHqlExpression * updateActiveSelectorFields(IHqlExpression * expr, IHqlExpression * oldRecord, IHqlExpression * newRecord, unsigned firstChild)
+{
+    if (oldRecord == newRecord)
+        return LINK(expr);
+
+    unsigned max = expr->numChildren();
+    unsigned i;
+    HqlExprArray args;
+    args.ensure(max);
+    for (i = 0; i < firstChild; i++)
+        args.append(*LINK(expr->queryChild(i)));
+
+    NewSelectorReplacingTransformer transformer;
+    transformer.setActiveSelectorMapping(oldRecord, newRecord);
+    bool same = true;
+    for (; i < max; i++)
+    {
+        IHqlExpression * cur = expr->queryChild(i);
+        IHqlExpression * transformed = transformer.transformRoot(cur);
+        args.append(*transformed);
+        if (cur != transformed)
+            same = false;
+    }
+    if (same)
+        return LINK(expr);
+    return expr->clone(args);
+}
+
 //---------------------------------------------------------------------------
 
 /*

ecl/hql/hqltrans.ipp

@@ -991,6 +991,7 @@ public:
     inline bool foundAmbiguity() const { return introducesAmbiguity; }
 
     void setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlExpression * record, bool isSelector);
+    void setActiveSelectorMapping(IHqlExpression * oldRecord, IHqlExpression * newRecord);
 
 protected:
     void setNestedMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlSimpleScope * oldScope, IHqlExpression * newRecord, bool isSelector);

ecl/hqlcpp/hqliproj.cpp

@@ -2925,6 +2925,7 @@ IHqlExpression * ImplicitProjectTransformer::createTransformed(IHqlExpression *
                 args.append(*next.getClear());
             }
             transformed.setown(expr->clone(args));
+            transformed.setown(updateSelectors(transformed, expr));
             logChange("Passthrough modified", expr, complexExtra->outputFields);
         }
         else
@@ -3196,7 +3197,6 @@ IHqlExpression * ImplicitProjectTransformer::updateSelectors(IHqlExpression * ne
     {
     case childdataset_none: 
     case childdataset_many_noscope:
-    case childdataset_many:
     case childdataset_if:
     case childdataset_case:
     case childdataset_map:
@@ -3204,6 +3204,9 @@ IHqlExpression * ImplicitProjectTransformer::updateSelectors(IHqlExpression * ne
         return LINK(newExpr);
         //None of these have any scoped arguments, so no need to remove them
         break;
+    case childdataset_many:
+        //The selectors listed in the sorted list may need updating if nested records have changed
+        return updateActiveSelectorFields(newExpr, oldExpr->queryRecord(), newExpr->queryRecord(), getNumChildTables(newExpr));
     case childdataset_dataset:
         {
             return updateMappedFields(newExpr, oldDs->queryNormalizedSelector(), newDs->queryNormalizedSelector(), 1);

ecl/regress/issue14052.ecl

@@ -0,0 +1,54 @@
+layout1 := {
+  unsigned4 doc_id;
+  unicode uni_val{maxlength(30)}
+};
+
+derived1 := {
+  layout1 ref;
+  layout1 def;
+  unsigned4 doc_id;
+};
+
+inline := dataset([
+   {1,u'one',3,u'three',3}
+  ,{2,u'two',4,u'four',4}
+  ,{3,u'three',5,u'five',5}
+  ,{4,u'four',6,u'six',6}
+],derived1);
+
+getStuff(dataset(derived1) ds1) := function
+
+  ds1s := sort(ds1,ref.doc_id,def.doc_id,local);
+
+  ds1sb := ds1s(doc_id % 2 = 1);
+
+  ds1m := merge(ds1s(doc_id % 2 = 0),
+                ds1sb,
+                sorted(ref.doc_id,def.doc_id),
+                // dedup,   // fails to compile with or without...leaving out for simplicity
+                local);
+  // This will work
+  // ds1m := dedup(sort(ds1s(doc_id % 2 = 0)+ds1sb,ref.doc_id,def.doc_id,local),ref.doc_id,def.doc_id,local);
+
+  ds2 := project(ds1m,transform(layout1,
+                                self.doc_id:= left.def.doc_id;
+                                self.uni_val := left.def.uni_val));
+
+  debug() := parallel(
+     evaluate('')
+    ,output(ds1m,,named('ds1m'))
+  );
+
+  ret := module
+    export debugger := debug();
+    export finalOut := ds2;
+    // export finalOut := ds1m;  // works
+  end;
+
+  return ret;
+end;
+
+example := getStuff(inline);
+
+// example.debugger;  // works if not commented
+output(example.finalout,all);