Merge pull request #4876 from ghalliday/issue9378

HPCC-9378 Remove DISTRIBUTE from the RHS of a global lookup join

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlopt.cpp

@@ -2176,6 +2176,17 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             //If left outer join, and transform doesn't reference RIGHT, and only one rhs record  could match each lhs record (e.g., it was rolled
             //up, or a non-many lookup join, then the join could be converted into a project
             //Can occur once fields get implicitly removed from transforms etc. - e.g., bc10.xhql, although that code has since been fixed.
+
+            //There is no point in distributing the rhs of a global lookup join => remove it.
+            if (transformed->hasAttribute(lookupAtom) && !transformed->hasAttribute(localAtom))
+            {
+                IHqlExpression * rhs = transformed->queryChild(1);
+                if (rhs->getOperator() == no_distribute)
+                {
+                    DBGLOG("Optimizer: Remove %s from RHS of global LOOKUP JOIN", queryNode0Text(rhs));
+                    return ::replaceChild(transformed, 1, rhs->queryChild(0));
+                }
+            }
             break;
         }
     case no_dedup:

ecl/regress/issue9378.ecl

@@ -0,0 +1,37 @@
+namesRecord :=
+            RECORD
+string20        surname := '?????????????';
+string10        forename := '?????????????';
+integer2        age := 25;
+            END;
+
+addressRecord :=
+            RECORD
+string30        addr := 'Unknown';
+string20        surname;
+            END;
+
+namesTable := dataset([
+        {'Smithe','Pru',10},
+        {'Hawthorn','Gavin',31},
+        {'Hawthorn','Mia',30},
+        {'Smith','Jo'},
+        {'Smith','Matthew'},
+        {'X','Z'}], namesRecord);
+
+addressTable := dataset([
+        {'Hawthorn','10 Slapdash Lane'},
+        {'Smith','Leicester'},
+        {'Smith','China'},
+        {'X','12 The burrows'},
+        {'X','14 The crescent'},
+        {'Z','The end of the world'}
+        ], addressRecord);
+
+dNamesTable := namesTable;
+dAddressTable := distribute(addressTable, hash(surname));
+
+JoinedF := join (dNamesTable, dAddressTable,
+                LEFT.surname = RIGHT.surname, MANY LOOKUP);
+
+output(JoinedF,,'out.d00');

testing/ecl/lookupjoin.ecl

@@ -46,7 +46,7 @@ j4 := join(lhs,rhs,left.did=right.did,testLookup(left, right, 'LOOKUP, MANY'),LO
 j5 := join(lhs,rhs,left.did=right.did,testLookup(left, right, 'LOOKUP, MANY, ATMOST'),LOOKUP, MANY, ATMOST(2));
 j6 := join(lhs,rhs,left.did=right.did,testLookup(left, right, 'LOOKUP, MANY, LIMIT(SKIP)'),LOOKUP, MANY, LIMIT(2, SKIP));
 j7 := join(lhs,rhs,left.did=right.did,testLookup(left, right, 'LOOKUP, MANY, LIMIT, ONFAIL'),LOOKUP, MANY, LIMIT(2), ONFAIL(testLookup(left, right, 'FAILED: LOOKUP, MANY, LIMIT, ONFAIL')));
-j8 := join(GROUP(dlhs,let),drhs,left.let=right.let,testLookup(left, right, 'LOOKUP LHSGROUPED'),LOOKUP, MANY);
+j8 := join(GROUP(dlhs,let),NOFOLD(drhs),left.let=right.let,testLookup(left, right, 'LOOKUP LHSGROUPED'),LOOKUP, MANY);
 tj8 := SORT(TABLE(j8, { letL, unsigned4 c := COUNT(GROUP), unsigned4 tot := SUM(group,did), label }), letL);
 j9 := SORT(join(dlhs,drhs,left.let=right.let,testLookup(left, right, 'LOOKUP MANY LOCAL'),LOOKUP, LOCAL), did);
 output(j1);