HPPC-10068 Various changes following review and new test case

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/thorhelper/commonext.cpp

@@ -200,6 +200,9 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     kindArray[TAKsubsort] = "subsort";
     kindArray[TAKdictionaryworkunitwrite] = "dictionaryworkunitwrite";
     kindArray[TAKdictionaryresultwrite] = "dictionaryresultwrite";
+    kindArray[TAKsmartjoin] = "smartjoin";
+    kindArray[TAKsmartdenormalize] = "smartdenormalize";
+    kindArray[TAKsmartdenormalizegroup] = "smartdenormalizegroup";
 
 //Non standard
     kindArray[TAKcountdisk] = "countdisk";

common/thorhelper/thorcommon.cpp

@@ -772,6 +772,9 @@ extern const char * getActivityText(ThorActivityKind kind)
     case TAKsubsort:                return "Sub Sort";
     case TAKdictionaryworkunitwrite:return "Dictionary Write";
     case TAKdictionaryresultwrite:  return "Dictionary Result";
+    case TAKsmartjoin:              return "Smart Join";
+    case TAKsmartdenormalize:       return "Smart Denormalize";
+    case TAKsmartdenormalizegroup:  return "Smart Denormalize Group";
     }
     throwUnexpected();
 }

ecl/eclagent/eclgraph.cpp

@@ -91,6 +91,9 @@ static IHThorActivity * createActivity(IAgentContext & agent, unsigned activityI
     case TAKlookupjoin:
     case TAKlookupdenormalize:
     case TAKlookupdenormalizegroup:
+    case TAKsmartjoin:
+    case TAKsmartdenormalize:
+    case TAKsmartdenormalizegroup:
         return createLookupJoinActivity(agent, activityId, subgraphId, (IHThorHashJoinArg &)arg, kind);
     case TAKalljoin:
     case TAKalldenormalize:

ecl/hql/hqlgram2.cpp

@@ -7654,14 +7654,33 @@ void HqlGram::checkJoinFlags(const attribute &err, IHqlExpression * join)
     bool ro = join->hasAttribute(rightouterAtom) || ronly;
     bool fo = join->hasAttribute(fullouterAtom) || fonly;
     bool keep = join->hasAttribute(keepAtom);
+    bool isLookup = join->hasAttribute(lookupAtom);
+    bool isSmart = join->hasAttribute(smartAtom);
+    bool isAll = join->hasAttribute(allAtom);
     IHqlExpression * rowLimit = join->queryAttribute(rowLimitAtom);
-
     IHqlExpression * keyed = join->queryAttribute(keyedAtom);
+
     if (keyed)
     {
-        if (join->hasAttribute(allAtom) || join->hasAttribute(lookupAtom) || join->hasAttribute(smartAtom))
-            reportError(ERR_KEYEDINDEXINVALID, err, "LOOKUP/ALL/SMART not compatible with KEYED");
+        if (isAll || isLookup || isSmart)
+            reportError(ERR_KEYEDINDEXINVALID, err, "LOOKUP/ALL/SMART is not compatible with KEYED");
+    }
+    else if (isLookup)
+    {
+        //The following should be, and will become, an error.  However too many legacy queries have it, so make a warning for now.
+        if (isAll)
+            reportWarning(ERR_KEYEDINDEXINVALID, err.pos, "ALL is not compatible with LOOKUP");
+        if (isSmart)
+            reportError(ERR_KEYEDINDEXINVALID, err.pos, "SMART is not compatible with LOOKUP");
+    }
+    else if (isSmart)
+    {
+        if (isAll)
+            reportError(ERR_KEYEDINDEXINVALID, err, "ALL is not compatible with KEYED");
+    }
 
+    if (keyed)
+    {
         IHqlExpression * index = keyed->queryChild(0);
         if (index)
         {
@@ -7705,21 +7724,22 @@ void HqlGram::checkJoinFlags(const attribute &err, IHqlExpression * join)
             }
         }
     }
-    if (join->hasAttribute(lookupAtom) || join->hasAttribute(smartAtom))
+    if (isLookup || isSmart)
     {
-        bool isMany = join->hasAttribute(manyAtom) || join->hasAttribute(smartAtom);
+        bool isMany = join->hasAttribute(manyAtom) || isSmart;
+        const char * joinText = isSmart ? "Smart" : "Lookup";
         if (ro || fo)
-            reportError(ERR_BADKIND_LOOKUPJOIN, err, "JOIN(LOOKUP) only supports INNER, LEFT OUTER, and LEFT ONLY joins");
+            reportError(ERR_BADKIND_LOOKUPJOIN, err, "%s joins only support INNER, LEFT OUTER, and LEFT ONLY joins", joinText);
         if (join->hasAttribute(partitionRightAtom))
-            reportError(ERR_BADKIND_LOOKUPJOIN, err, "Lookup joins do not support PARTITION RIGHT");
+            reportError(ERR_BADKIND_LOOKUPJOIN, err, "%s joins do not support PARTITION RIGHT", joinText);
         if (keep && !isMany)
-            reportError(ERR_BADKIND_LOOKUPJOIN, err, "Lookup joins do not support KEEP");
+            reportError(ERR_BADKIND_LOOKUPJOIN, err, "%s joins do not support KEEP", joinText);
         if (join->hasAttribute(atmostAtom) && !isMany)
-            reportError(ERR_BADKIND_LOOKUPJOIN, err, "Lookup joins do not support ATMOST");
+            reportError(ERR_BADKIND_LOOKUPJOIN, err, "%s joins do not support ATMOST", joinText);
         if (rowLimit && !isMany)
-            reportError(ERR_BADKIND_LOOKUPJOIN, err, "Lookup joins do not support LIMIT (they can only match 1 entry)");
+            reportError(ERR_BADKIND_LOOKUPJOIN, err, "%s joins do not support LIMIT (they can only match 1 entry)", joinText);
         if (isKey(join->queryChild(1)))
-            reportWarning(ERR_BADKIND_LOOKUPJOIN, err.pos, "Lookup specified on an unfiltered keyed join - was this intended?");
+            reportWarning(ERR_BADKIND_LOOKUPJOIN, err.pos, "%s specified on an unfiltered keyed join - was this intended?", joinText);
     }
     else if (isKeyedJoin(join))
     {
@@ -7769,6 +7789,7 @@ void HqlGram::checkJoinFlags(const attribute &err, IHqlExpression * join)
         if (isKey(cur))
             reportWarning(ERR_BAD_JOINFLAG, err.pos, "Filtered RIGHT prevents a keyed join being used.  Consider including the filter in the join condition.");
     }
+
 }
 
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -11629,11 +11629,16 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
             kind = TAKalljoin;
             argName = "AllJoin";
         }
-        else if (isLookupJoin || isSmartJoin)
+        else if (isLookupJoin)
         {
             kind = TAKlookupjoin;
             argName = "HashJoin";
         }
+        else if (isSmartJoin)
+        {
+            kind = TAKsmartjoin;
+            argName = "HashJoin";
+        }
         else if (isHashJoin)
         {
             kind = TAKhashjoin;
@@ -11652,11 +11657,16 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
             kind = TAKalldenormalize;
             argName = "AllDenormalize";
         }
-        else if (isLookupJoin || isSmartJoin)
+        else if (isLookupJoin)
         {
             kind = TAKlookupdenormalize;
             argName = "HashDenormalize";
         }
+        else if (isSmartJoin)
+        {
+            kind = TAKsmartdenormalize;
+            argName = "HashDenormalize";
+        }
         else if (isHashJoin)
         {
             kind = TAKhashdenormalize;
@@ -11675,11 +11685,16 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
             kind = TAKalldenormalizegroup;
             argName = "AllDenormalizeGroup";
         }
-        else if (isLookupJoin || isSmartJoin)
+        else if (isLookupJoin)
         {
             kind = TAKlookupdenormalizegroup;
             argName = "HashDenormalizeGroup";
         }
+        else if (isSmartJoin)
+        {
+            kind = TAKsmartdenormalizegroup;
+            argName = "HashDenormalizeGroup";
+        }
         else if (isHashJoin)
         {
             kind = TAKhashdenormalizegroup;

ecl/hqlcpp/hqlresource.cpp

@@ -91,7 +91,6 @@ void getResources(IHqlExpression * expr, CResources & resources, const CResource
         }
         else if (expr->hasAttribute(smartAtom))
         {
-            //GH->JCS what should we assume here?
             resources.setHeavyweight();
             setHashResources(expr, resources, options);
         }

ecl/hthor/hthor.cpp

@@ -5243,6 +5243,7 @@ void CHThorLookupJoinActivity::ready()
         atmostLimit = static_cast<unsigned>(-1);
     if(limitLimit==0)
         limitLimit = static_cast<unsigned>(-1);
+    isSmartJoin = (helper.getJoinFlags() & JFsmart) != 0;
     getLimitType(helper.getJoinFlags(), limitFail, limitOnFail);
 
     if((leftOuterJoin || limitOnFail) && !defaultRight)
@@ -5364,9 +5365,12 @@ const void * CHThorLookupJoinActivity::nextInGroup()
     switch (kind)
     {
     case TAKlookupjoin:
+    case TAKsmartjoin:
         return nextInGroupJoin();
     case TAKlookupdenormalize:
     case TAKlookupdenormalizegroup:
+    case TAKsmartdenormalize:
+    case TAKsmartdenormalizegroup:
         return nextInGroupDenormalize();
     }
     throwUnexpected();
@@ -5383,14 +5387,20 @@ const void * CHThorLookupJoinActivity::nextInGroupJoin()
             keepCount = keepLimit;
             if(!left)
             {
-                if(matchedGroup || eog)
+                if (isSmartJoin)
+                    left.setown(input->nextInGroup());
+
+                if(!left)
                 {
-                    matchedGroup = false;
+                    if(matchedGroup || eog)
+                    {
+                        matchedGroup = false;
+                        eog = true;
+                        return NULL;
+                    }
                     eog = true;
-                    return NULL;
+                    continue;
                 }
-                eog = true;
-                continue;
             }
             eog = false;
             gotMatch = false;
@@ -5447,7 +5457,7 @@ const void * CHThorLookupJoinActivity::nextInGroupDenormalize()
         left.setown(input->nextInGroup());
         if(!left)
         {
-            if (!matchedGroup)
+            if (!matchedGroup || isSmartJoin)
                 left.setown(input->nextInGroup());
 
             if (!left)
@@ -5462,7 +5472,7 @@ const void * CHThorLookupJoinActivity::nextInGroupDenormalize()
         const void * ret = NULL;
         if (failingLimit)
             ret = joinException(left, failingLimit);
-        else if (kind == TAKlookupdenormalize)
+        else if (kind == TAKlookupdenormalize || kind == TAKsmartdenormalize)
         {
             OwnedConstRoxieRow newLeft(left.getLink());
             unsigned rowSize = 0;

ecl/hthor/hthor.ipp

@@ -1371,6 +1371,7 @@ private:
     bool limitFail;
     bool limitOnFail;
     bool hasGroupLimit;
+    bool isSmartJoin;
     unsigned keepCount;
     OwnedConstRoxieRow defaultRight;
     RtlDynamicRowBuilder outBuilder;

ecl/regress/smartjoin_err.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+addressRecord :=
+            RECORD
+string20        surname;
+string30        addr;
+            END;
+
+namesTable := dataset('n', namesRecord, thor);
+
+addressTable := dataset('a', addressRecord, thor);
+
+JoinedF := join (namesTable, addressTable, LEFT.surname = RIGHT.surname, SMART, LOOKUP);
+output(JoinedF,,'out.d00',overwrite);
+
+JoinedF2 := join (namesTable, addressTable, LEFT.surname = RIGHT.surname, SMART, RIGHT ONLY);
+output(JoinedF2,,'out.d00',overwrite);

roxie/ccd/ccdquery.cpp

@@ -479,6 +479,9 @@ protected:
         case TAKlookupjoin:
         case TAKlookupdenormalize:
         case TAKlookupdenormalizegroup:
+        case TAKsmartjoin:
+        case TAKsmartdenormalize:
+        case TAKsmartdenormalizegroup:
             return createRoxieServerLookupJoinActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKmerge:
             return createRoxieServerMergeActivityFactory(id, subgraphId, *this, helperFactory, kind);

roxie/ccd/ccdserver.cpp

@@ -17095,6 +17095,7 @@ private:
     bool limitFail;
     bool limitOnFail;
     bool hasGroupLimit;
+    bool isSmartJoin;
     unsigned keepCount;
     bool gotMatch;
     bool cloneLeft;
@@ -17140,7 +17141,8 @@ public:
         atmostsTriggered = 0;
         limitLimit = 0;
         hasGroupLimit = false;
-        getLimitType(helper.getJoinFlags(), limitFail, limitOnFail);
+        isSmartJoin = (joinFlags & JFsmart) != 0;
+        getLimitType(joinFlags, limitFail, limitOnFail);
     }
 
     void loadRight()
@@ -17201,7 +17203,17 @@ public:
         if(atmostLimit==0) atmostLimit = static_cast<unsigned>(-1);
         if(limitLimit==0) limitLimit = static_cast<unsigned>(-1);
         getLimitType(helper.getJoinFlags(), limitFail, limitOnFail);
-        if (((activityKind==TAKlookupjoin || activityKind==TAKlookupdenormalizegroup) && leftOuterJoin) || limitOnFail)
+        switch (activityKind)
+        {
+        case TAKlookupjoin:
+        case TAKlookupdenormalizegroup:
+        case TAKsmartjoin:
+        case TAKsmartdenormalizegroup:
+            if (leftOuterJoin)
+                createDefaultRight();
+            break;
+        }
+        if (limitOnFail)
             createDefaultRight();
     }
 
@@ -17229,9 +17241,12 @@ public:
         switch (activityKind)
         {
             case TAKlookupjoin:
+            case TAKsmartjoin:
                 return nextInGroupJoin();
             case TAKlookupdenormalize:
             case TAKlookupdenormalizegroup:
+            case TAKsmartdenormalize:
+            case TAKsmartdenormalizegroup:
                 return nextInGroupDenormalize();
         }
         throwUnexpected();
@@ -17251,14 +17266,20 @@ private:
                 keepCount = keepLimit;
                 if(!left)
                 {
-                    if(matchedGroup || eog)
+                    if (isSmartJoin)
+                        left = input->nextInGroup();
+
+                    if (!left)
                     {
-                        matchedGroup = false;
+                        if(matchedGroup || eog)
+                        {
+                            matchedGroup = false;
+                            eog = true;
+                            return NULL;
+                        }
                         eog = true;
-                        return NULL;
+                        continue;
                     }
-                    eog = true;
-                    continue;
                 }
                 eog = false;
                 gotMatch = false;
@@ -17315,7 +17336,7 @@ private:
             left = input->nextInGroup();
             if(!left)
             {
-                if (!matchedGroup)
+                if (!matchedGroup || isSmartJoin)
                     left = input->nextInGroup();
 
                 if (!left)
@@ -17330,7 +17351,7 @@ private:
             const void * ret = NULL;
             if (failingLimit)
                 ret = joinException(left, failingLimit);
-            else if (activityKind == TAKlookupdenormalize)
+            else if (activityKind == TAKlookupdenormalize || activityKind == TAKsmartdenormalize)
             {
                 OwnedConstRoxieRow newLeft;
                 newLeft.set(left);

rtl/include/eclhelper.hpp

@@ -850,6 +850,9 @@ enum ThorActivityKind
     TAKexternalprocess,
     TAKdictionaryworkunitwrite,
     TAKdictionaryresultwrite,
+    TAKsmartjoin,
+    TAKsmartdenormalize,
+    TAKsmartdenormalizegroup,
 
     TAKlast
 };

testing/ecl/key/smartjoin.xml

@@ -0,0 +1,67 @@
+<Dataset name='Result 1'>
+ <Row><i>1</i><idl>A</idl><idr>a</idr></Row>
+ <Row><i>1</i><idl>A</idl><idr>b</idr></Row>
+ <Row><i>1</i><idl>A</idl><idr>c</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>a</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>b</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>c</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>a</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>b</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>c</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>x</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>y</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>x</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>y</idr></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><cnt>13</cnt></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><i>1</i><idl>A</idl><idr>A</idr></Row>
+ <Row><i>1</i><idl>A</idl><idr>B</idr></Row>
+ <Row><i>1</i><idl>A</idl><idr>C</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>A</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>B</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>C</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>A</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>B</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>C</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>X</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>Y</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>X</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>Y</idr></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><i>1</i><idl>A</idl><idr>abc</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>abc</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>abc</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>xy</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>xy</idr></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><cnt>5</cnt></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><i>1</i><idl>A</idl><idr>ABC</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>ABC</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>ABC</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>XY</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>XY</idr></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><i>1</i><idl>A</idl><idr>abc</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>abc</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>abc</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>xy</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>xy</idr></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><cnt>5</cnt></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><i>1</i><idl>A</idl><idr>ABC</idr></Row>
+ <Row><i>1</i><idl>B</idl><idr>ABC</idr></Row>
+ <Row><i>1</i><idl>C</idl><idr>ABC</idr></Row>
+ <Row><i>2</i><idl>X</idl><idr>XY</idr></Row>
+ <Row><i>2</i><idl>Y</idl><idr>XY</idr></Row>
+</Dataset>

testing/ecl/smartjoin.ecl

@@ -0,0 +1,84 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+rec := RECORD
+        integer i;
+        string1 id;
+END;
+
+recout := RECORD
+        integer i;
+        string idL;
+        string idR;
+END;
+
+recout createOut(integer i, string idL, string idR) := TRANSFORM
+            SELF.i := i;
+            SELF.idl := idL;
+            SELF.idr := idR;
+END;
+
+ds1 := DATASET([{1,'A'}, {1,'B'}, {1,'C'}, {2,'X'}, {2,'Y'}], rec);
+ds2 := DATASET([{1,'a'}, {1,'b'}, {1,'c'}, {2,'x'}, {2,'y'}], rec);
+
+recout trans(rec L, rec R) := createOut(L.i, L.id, R.id);
+
+gr1 := GROUP(ds1, i, id);
+j1 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), SMART);
+j2 := JOIN(gr1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), SMART);
+j3 := JOIN(gr1, gr1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), SMART);
+
+p1 := PROJECT(ds1, createOut(LEFT.i, LEFT.id, ''));
+gp1 := GROUP(p1, i, idL);
+
+recout tr2(recout L, rec R) := TRANSFORM
+            SELF.idR := L.idR + R.id;
+            SELF := L;
+END;
+
+recout tr3(recout L, recout R) := TRANSFORM
+            SELF.idR := L.idR + R.idL;
+            SELF := L;
+END;
+
+d1 := DENORMALIZE(p1, ds2, LEFT.i = RIGHT.i, tr2(LEFT, RIGHT), SMART);
+d2 := DENORMALIZE(gp1, ds2, LEFT.i = RIGHT.i, tr2(LEFT, RIGHT), SMART);
+d3 := DENORMALIZE(gp1, gp1, LEFT.i = RIGHT.i, tr3(LEFT, RIGHT), SMART);
+
+strRec := { string text };
+
+recout trg(rec L, dataset(rec) Rs) := TRANSFORM
+            SELF.i := L.i;
+            SELF.idL := L.id;
+            SELF.idR := AGGREGATE(Rs, strRec, TRANSFORM(strRec, SELF.text := RIGHT.text + LEFT.id))[1].text;
+END;
+
+dg1 := DENORMALIZE(ds1, ds2, LEFT.i = RIGHT.i, GROUP, trg(LEFT, ROWS(RIGHT)), SMART);
+dg2 := DENORMALIZE(gr1, ds2, LEFT.i = RIGHT.i, GROUP, trg(LEFT, ROWS(RIGHT)), SMART);
+dg3 := DENORMALIZE(gr1, gr1, LEFT.i = RIGHT.i, GROUP, trg(LEFT, ROWS(RIGHT)), SMART);
+
+sequential(
+    output(j1),
+    output(TABLE(j2, { cnt := COUNT(GROUP) })),  // check output is not grouped
+    output(j3),
+    output(d1),
+    output(TABLE(d2, { cnt := COUNT(GROUP) })),  // check output is not grouped
+    output(d3),
+    output(dg1),
+    output(TABLE(dg2, { cnt := COUNT(GROUP) })),  // check output is not grouped
+    output(dg3),
+);

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -497,7 +497,7 @@ class CLookupJoinActivity : public CSlaveActivity, public CThorDataLink, impleme
     inline bool isLookup() { return (joinKind==join_lookup)||(joinKind==denormalize_lookup); }
     inline bool isAll() { return (joinKind==join_all)||(joinKind==denormalize_all); }
     inline bool isDenormalize() { return (joinKind==denormalize_all)||(joinKind==denormalize_lookup); }
-    inline bool isGroupOp() { return (TAKlookupdenormalizegroup == container.getKind() || TAKalldenormalizegroup == container.getKind()); }
+    inline bool isGroupOp() { return (TAKlookupdenormalizegroup == container.getKind() || TAKsmartdenormalizegroup == container.getKind() || TAKalldenormalizegroup == container.getKind()); }
     StringBuffer &getJoinTypeStr(StringBuffer &str)
     {
         switch(joinType)

thorlcr/graph/thgraph.cpp

@@ -877,16 +877,19 @@ bool isGlobalActivity(CGraphElementBase &container)
         case TAKjoin:
         case TAKselfjoin:
         case TAKhashjoin:
+        case TAKsmartjoin:
         case TAKkeyeddenormalize:
         case TAKhashdenormalize:
         case TAKdenormalize:
-        case TAKlookupdenormalize:
+        case TAKlookupdenormalize: //GH->JCS why are these here, and join not?
         case TAKalldenormalize:
+        case TAKsmartdenormalize:
         case TAKdenormalizegroup:
         case TAKhashdenormalizegroup:
         case TAKlookupdenormalizegroup:
         case TAKkeyeddenormalizegroup:
         case TAKalldenormalizegroup:
+        case TAKsmartdenormalizegroup:
         case TAKaggregate:
         case TAKexistsaggregate:
         case TAKcountaggregate:

thorlcr/master/thactivitymaster.cpp

@@ -262,6 +262,9 @@ public:
             case TAKalljoin:
             case TAKlookupdenormalize:
             case TAKlookupdenormalizegroup:
+            case TAKsmartjoin:
+            case TAKsmartdenormalize:
+            case TAKsmartdenormalizegroup:
             case TAKalldenormalize:
             case TAKalldenormalizegroup:
                 ret = createLookupJoinActivityMaster(this);

thorlcr/slave/slave.cpp

@@ -438,6 +438,7 @@ public:
                 ret = createHashJoinSlave(this);
                 break;
             case TAKlookupjoin:
+            case TAKsmartjoin:
                 ret = createLookupJoinSlave(this);
                 break;
             case TAKalljoin:
@@ -540,6 +541,8 @@ public:
                 break;
             case TAKlookupdenormalize:
             case TAKlookupdenormalizegroup:
+            case TAKsmartdenormalize:
+            case TAKsmartdenormalizegroup:
                 ret = createLookupDenormalizeSlave(this);
                 break;
             case TAKchilddataset: