Merge pull request #13228 from ghalliday/issue23226

HPCC-23226 Substitute default values for RIGHT in JOIN ONFAIL clause

Reviewed-By: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlpmap.cpp

@@ -204,6 +204,13 @@ void replaceSelectors(HqlExprArray & exprs, unsigned first, IHqlExpression * old
     }
 }
 
+IHqlExpression * replaceSelectorWithNull(IHqlExpression * expr, IHqlExpression * selector)
+{
+    NewSelectorReplacingTransformer transformer;
+    transformer.initNullMapping(selector);
+    return transformer.transformRoot(expr);
+}
+
 
 //NB: This can not be derived from NewHqlTransformer since it is called before the tree is normalised, and it creates
 //inconsistent expression trees.

ecl/hql/hqlpmap.hpp

@@ -174,6 +174,8 @@ extern HQL_API IHqlExpression * scopedReplaceSelector(IHqlExpression * expr, IHq
 extern HQL_API IHqlExpression * replaceSelfRefSelector(IHqlExpression * expr, IHqlExpression * newDataset);
 extern HQL_API IHqlExpression * updateActiveSelectorFields(IHqlExpression * expr, IHqlExpression * oldRecord, IHqlExpression * newRecord, unsigned firstChild);
 
+extern HQL_API IHqlExpression * replaceSelectorWithNull(IHqlExpression * expr, IHqlExpression * selector);
+
 extern HQL_API bool isNullProject(IHqlExpression * expr, bool canIgnorePayload, bool canLoseFieldsFromEnd);
 extern HQL_API bool isSimpleProject(IHqlExpression * expr);                             // Restriction or rearrangement only
 extern HQL_API bool leftRecordIsSubsetOfRight(IHqlExpression * left, IHqlExpression * right);

ecl/hql/hqltrans.cpp

@@ -3234,6 +3234,26 @@ NewSelectorReplacingTransformer::NewSelectorReplacingTransformer() : NewHqlTrans
     isHidden=false; 
 }
 
+void NewSelectorReplacingTransformer::initNullMapping(IHqlExpression * selector)
+{
+    node_operator op = selector->getOperator();
+    assertex(op == no_left || op == no_right);
+
+    setNullMapping(selector, selector->queryRecord(), false);
+    setNullMapping(selector, selector->queryRecord(), true);
+
+    //Also map the selector to the a null row
+    OwnedHqlExpr nullValue = createNullExpr(selector);
+    setRootMapping(selector, nullValue, false);
+    setRootMapping(selector, nullValue, true);
+
+    //And map MATCHED(RIGHT) -> false since that can be used inside a JOIN ONFAIL() clause.
+    OwnedHqlExpr matchedExpr = createValue(no_matched_injoin, makeBoolType(), LINK(selector));
+    setSelectorMapping(matchedExpr, queryBoolExpr(false));
+    setMappingOnly(matchedExpr, queryBoolExpr(false));
+
+    oldSelector.set(selector);
+}
 
 void NewSelectorReplacingTransformer::initSelectorMapping(IHqlExpression * oldDataset, IHqlExpression * newDataset)
 {
@@ -3289,6 +3309,32 @@ void NewSelectorReplacingTransformer::setNestedMapping(IHqlExpression * oldSel,
                     OwnedHqlExpr newSelected = createSelectExpr(LINK(newSel), LINK(cur));
                     setRootMapping(oldSelected, newSelected, oldField->queryRecord(), isSelector);
                 }
+                break;
+            }
+        }
+    }
+}
+
+void NewSelectorReplacingTransformer::setNullMapping(IHqlExpression * selector, IHqlExpression * record, bool isSelector)
+{
+    ForEachChild(i, record)
+    {
+        IHqlExpression * cur = record->queryChild(i);
+        switch (cur->getOperator())
+        {
+        case no_record:
+            setNullMapping(selector, cur, isSelector);
+            break;
+        case no_ifblock:
+            setNullMapping(selector, cur->queryChild(1), isSelector);
+            break;
+        case no_field:
+            {
+                IHqlExpression * field = cur;
+                OwnedHqlExpr selected = createSelectExpr(LINK(selector), LINK(field));
+                OwnedHqlExpr newValue = createNullExpr(selected);
+                setRootMapping(selected, newValue, field->queryRecord(), isSelector);
+                break;
             }
         }
     }

ecl/hql/hqltrans.ipp

@@ -989,6 +989,7 @@ public:
     NewSelectorReplacingTransformer();
 
     void initSelectorMapping(IHqlExpression * oldValue, IHqlExpression * newValue);
+    void initNullMapping(IHqlExpression * selector);
 
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
 
@@ -1018,6 +1019,7 @@ public:
     void setActiveSelectorMapping(IHqlExpression * oldRecord, IHqlExpression * newRecord);
 
 protected:
+    void setNullMapping(IHqlExpression * selector, IHqlExpression * record, bool isSelector);
     void setNestedMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlSimpleScope * oldScope, IHqlExpression * newRecord, bool isSelector);
     void setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, bool isSelector);
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -3057,6 +3057,18 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
     IHqlExpression * rightDs = queryJoinRhs(expr);
     IHqlExpression * seq = querySelSeq(expr);
     node_operator op = expr->getOperator();
+    IHqlExpression * onFail = expr->queryAttribute(onFailAtom);
+    if (onFail)
+    {
+        OwnedHqlExpr right = createSelector(no_right, rightDs, seq);
+        if (onFail->usesSelector(right))
+        {
+            //Replace all references to right with the default values
+            OwnedHqlExpr newFail = replaceSelectorWithNull(onFail, right);
+            assertex(!newFail->usesSelector(right));
+            return replaceOwnedAttribute(expr, newFail.getClear());
+        }
+    }
 
     if (op == no_join)
     {