HPCC-9286 Allow multiple optional atmost conditions.

Also fixes:

HPCC-9711 Ensure only the atmost condition is used for calculating the hard matches.
HPCC-10005 Global ATMOST prefix join in thor may lose results

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlattr.cpp

@@ -3195,7 +3195,7 @@ unsigned getMaxRecordSize(IHqlExpression * record, unsigned defaultMaxRecordSize
         OwnedHqlExpr folded = foldHqlExpression(value);
         assertex(folded);
         maxSize = (unsigned)getIntValue(folded);
-        unsigned minSize = getIntValue(minSizeExpr);
+        unsigned minSize = (unsigned)getIntValue(minSizeExpr);
         if (maxSize < minSize)
             maxSize = minSize;
         usedDefault = true;

ecl/hql/hqlerrors.hpp

@@ -465,6 +465,11 @@
 #define HQLERR_PayloadMismatch                  3127
 #define HQLERR_MemberXContainsVirtualRef        3128
 #define HQLERR_FieldHasNoDefaultValue           3129
+#define HQLERR_AtmostFailMatchCondition         3130
+#define HQLERR_AtmostCannotImplementOpt         3131
+#define HQLERR_PrefixJoinRequiresEquality       3132
+#define HQLERR_AtmostFollowUnknownSubstr        3133
+#define HQLERR_AtmostLegacyMismatch             3134
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contain an illegal cycle/recursive definition %s"
@@ -496,6 +501,11 @@
 #define HQLERR_PayloadMismatch_Text             "Mismatched => in inline dictionary definition"
 #define HQLERR_MemberXContainsVirtualRef_Text   "Member %s contains virtual references but not supported as virtual"
 #define HQLERR_FieldHasNoDefaultValue_Text      "Field '%s' doesn't have a defined value"
+#define HQLERR_AtmostFailMatchCondition_Text    "ATMOST(%s) failed to match part of the join condition"
+#define HQLERR_AtmostCannotImplementOpt_Text    "ATMOST() optional condition is too complex"
+#define HQLERR_PrefixJoinRequiresEquality_Text  "Global JOIN with no required equalities requires ALL"
+#define HQLERR_AtmostFollowUnknownSubstr_Text   "ATMOST [1..*] on an unknown length string must be last in the optional list"
+#define HQLERR_AtmostLegacyMismatch_Text        "Legacy JOIN condition on field[1..*] should be included in the optional fields"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */

ecl/hql/hqlexpr.cpp

@@ -14400,12 +14400,11 @@ bool isSelfJoin(IHqlExpression * expr)
     if (!joinSortOrdersMatch(expr->queryChild(2)))
         return false;
 
-    //Check this isn't going to generate a between join - if it is that takes precedence.  A bit arbitary
+    //Check this isn't going to generate a between join - if it is that takes precedence.  A bit arbitrary
     //when one is more efficient.
-    HqlExprArray leftSorts, rightSorts, slidingMatches;
-    bool isLimitedSubstringJoin;
-    OwnedHqlExpr fuzzy = findJoinSortOrders(expr, leftSorts, rightSorts, isLimitedSubstringJoin, &slidingMatches);
-    if ((slidingMatches.ordinality() != 0) && (leftSorts.ordinality() == slidingMatches.ordinality()))
+    JoinSortInfo joinInfo;
+    joinInfo.findJoinSortOrders(expr, true);
+    if ((joinInfo.slidingMatches.ordinality() != 0) && (joinInfo.queryLeftReq().ordinality() == joinInfo.slidingMatches.ordinality()))
         return false;
     return true;
 }

ecl/hql/hqlexpr.hpp

@@ -1644,11 +1644,11 @@ inline bool isCast(IHqlExpression * expr)
 }
 extern HQL_API bool isKey(IHqlExpression * expr);
 
-IHqlExpression * queryGrouping(IHqlExpression * expr);
-IHqlExpression * queryDistribution(IHqlExpression * expr);
-IHqlExpression * queryGlobalSortOrder(IHqlExpression * expr);
-IHqlExpression * queryLocalUngroupedSortOrder(IHqlExpression * expr);
-IHqlExpression * queryGroupSortOrder(IHqlExpression * expr);
+extern HQL_API IHqlExpression * queryGrouping(IHqlExpression * expr);
+extern HQL_API IHqlExpression * queryDistribution(IHqlExpression * expr);
+extern HQL_API IHqlExpression * queryGlobalSortOrder(IHqlExpression * expr);
+extern HQL_API IHqlExpression * queryLocalUngroupedSortOrder(IHqlExpression * expr);
+extern HQL_API IHqlExpression * queryGroupSortOrder(IHqlExpression * expr);
 
 inline bool isGrouped(ITypeInfo * type)                     { return type && (type->getTypeCode() == type_groupedtable); }
 inline bool isGrouped(IHqlExpression * expr)                { return isGrouped(expr->queryType()); }

ecl/hql/hqlfold.cpp

@@ -5113,12 +5113,11 @@ IHqlExpression * CExprFolderTransformer::percolateConstants(IHqlExpression * exp
                     IHqlExpression * selSeq = querySelSeq(updated);
                     IHqlExpression * updatedLhs = updated->queryChild(0);
                     IHqlExpression * updatedRhs = (op == no_selfjoin) ? updatedLhs : updated->queryChild(1);
-                    HqlExprArray leftSorts, rightSorts;
-                    bool isLimitedSubstringJoin;
-                    OwnedHqlExpr extra = findJoinSortOrders(updatedCond, updatedLhs, updatedRhs, selSeq, leftSorts, rightSorts, isLimitedSubstringJoin, NULL);
+                    JoinSortInfo joinInfo;
+                    joinInfo.findJoinSortOrders(updatedCond, updatedLhs, updatedRhs, selSeq, false);
 
                     //if will convert to an all join, then restore the old condition,
-                    if (leftSorts.ordinality() == 0)
+                    if (!joinInfo.hasRequiredEqualities())
                         updated.setown(replaceChild(updated, 2, oldCond));
                     else
                         updated.setown(appendOwnedOperand(updated, createAttribute(_conditionFolded_Atom)));

ecl/hql/hqlgram.y

@@ -7645,13 +7645,12 @@ simpleDataSet
 
                             IHqlExpression * ds = createDataset(no_keyeddistribute, left, createComma(right, cond, LINK(attr), $12.getExpr()));
 
-                            HqlExprArray leftSorts, rightSorts;
-                            bool isLimitedSubstringJoin;
-                            OwnedHqlExpr match = findJoinSortOrders(cond, NULL, NULL, NULL, leftSorts, rightSorts, isLimitedSubstringJoin, NULL);
+                            JoinSortInfo joinInfo;
+                            joinInfo.findJoinSortOrders(cond, NULL, NULL, NULL, false);
                             unsigned numUnsortedFields = numPayloadFields(right);
-                            if (match || (!ds->hasAttribute(firstAtom) && (leftSorts.ordinality() != getFieldCount(right->queryRecord())-numUnsortedFields)))
+                            if (joinInfo.extraMatch || (!ds->hasAttribute(firstAtom) && (joinInfo.queryLeftReq().ordinality() != getFieldCount(right->queryRecord())-numUnsortedFields)))
                                 parser->reportError(ERR_MATCH_KEY_EXACTLY,$9,"Condition on DISTRIBUTE must match the key exactly");
-                            if (isLimitedSubstringJoin)
+                            if (joinInfo.hasOptionalEqualities())
                                 parser->reportError(ERR_MATCH_KEY_EXACTLY,$9,"field[1..*] is not supported for a keyed distribute");
 
                             //Should check that all index fields are accounted for...
@@ -10030,7 +10029,7 @@ JoinFlag
                         {
                             parser->normalizeExpression($3, type_numeric, false);
                             if ($3.isZero())
-                                parser->reportError(ERR_BAD_JOINFLAG, $4, "ATMOST(0) doesn't make any sense");
+                                parser->reportError(ERR_BAD_JOINFLAG, $3, "ATMOST(0) doesn't make any sense");
                             $$.setExpr(createExprAttribute(atmostAtom, $3.getExpr()));
                             $$.setPosition($1);
                         }
@@ -10039,7 +10038,24 @@ JoinFlag
                             parser->normalizeExpression($3, type_boolean, false);
                             parser->normalizeExpression($5, type_numeric, false);
                             if ($5.isZero())
-                                parser->reportError(ERR_BAD_JOINFLAG, $6, "ATMOST(0) doesn't make any sense");
+                                parser->reportError(ERR_BAD_JOINFLAG, $5, "ATMOST(0) doesn't make any sense");
+                            $$.setExpr(createExprAttribute(atmostAtom, $3.getExpr(), $5.getExpr()));
+                            $$.setPosition($1);
+                        }
+    | ATMOST '(' expression ',' sortListExpr ',' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_boolean, false);
+                            parser->normalizeExpression($7, type_numeric, false);
+                            if ($7.isZero())
+                                parser->reportError(ERR_BAD_JOINFLAG, $7, "ATMOST(0) doesn't make any sense");
+                            $$.setExpr(createExprAttribute(atmostAtom, $3.getExpr(), $5.getExpr(), $7.getExpr()));
+                            $$.setPosition($1);
+                        }
+    | ATMOST '(' sortListExpr ',' expression ')'
+                        {
+                            parser->normalizeExpression($5, type_numeric, false);
+                            if ($5.isZero())
+                                parser->reportError(ERR_BAD_JOINFLAG, $5, "ATMOST(0) doesn't make any sense");
                             $$.setExpr(createExprAttribute(atmostAtom, $3.getExpr(), $5.getExpr()));
                             $$.setPosition($1);
                         }

ecl/hql/hqlmeta.cpp

@@ -1379,7 +1379,7 @@ bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * order, bool isLo
 
 
 //Elements in the exprarray have already been mapped;
-bool isAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
+bool isAlreadySorted(IHqlExpression * dataset, const HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
 {
     HqlExprArray components;
     normalizeComponents(components, newSort);
@@ -1431,7 +1431,7 @@ static unsigned numElementsAlreadySorted(IHqlExpression * dataset, IHqlExpressio
 }
 
 //Elements in the exprarray have already been mapped;
-static unsigned numElementsAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
+static unsigned numElementsAlreadySorted(IHqlExpression * dataset, const HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
 {
     HqlExprArray components;
     normalizeComponents(components, newSort);
@@ -1445,7 +1445,7 @@ bool isWorthShuffling(IHqlExpression * dataset, IHqlExpression * order, bool isL
     return numElementsAlreadySorted(dataset, order, isLocal, ignoreGrouping) != 0;
 }
 
-bool isWorthShuffling(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
+bool isWorthShuffling(IHqlExpression * dataset, const HqlExprArray & newSort, bool isLocal, bool ignoreGrouping)
 {
     //MORE: Should this look at the cardinality of the already-sorted fields, and not transform if below a certain threshold?
     return numElementsAlreadySorted(dataset, newSort, isLocal, ignoreGrouping) != 0;
@@ -1502,7 +1502,7 @@ static IHqlExpression * createSubSorted(IHqlExpression * dataset, IHqlExpression
     return subsort.getClear();
 }
 
-IHqlExpression * getSubSort(IHqlExpression * dataset, HqlExprArray & order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
+IHqlExpression * getSubSort(IHqlExpression * dataset, const HqlExprArray & order, bool isLocal, bool ignoreGrouping, bool alwaysLocal)
 {
     if (isAlreadySorted(dataset, order, isLocal||alwaysLocal, ignoreGrouping))
         return NULL;

ecl/hql/hqlmeta.hpp

@@ -107,13 +107,13 @@ extern HQL_API bool matchDedupDistribution(IHqlExpression * distn, const HqlExpr
 extern HQL_API bool matchesAnyDistribution(IHqlExpression * distn);
 
 extern HQL_API bool appearsToBeSorted(IHqlExpression * dataset, bool isLocal, bool ignoreGrouping);
-extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
+extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, const HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
 extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * newSort, bool isLocal, bool ignoreGrouping);
 extern HQL_API IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal, bool allowSubSort);
 
 extern HQL_API bool isWorthShuffling(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping);
-extern HQL_API bool isWorthShuffling(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
-extern HQL_API IHqlExpression * getSubSort(IHqlExpression * dataset, HqlExprArray & order, bool isLocal, bool ignoreGrouping, bool alwaysLocal);
+extern HQL_API bool isWorthShuffling(IHqlExpression * dataset, const HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
+extern HQL_API IHqlExpression * getSubSort(IHqlExpression * dataset, const HqlExprArray & order, bool isLocal, bool ignoreGrouping, bool alwaysLocal);
 extern HQL_API IHqlExpression * getSubSort(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal);
 extern HQL_API IHqlExpression * convertSubSortToGroupedSort(IHqlExpression * expr);
 

ecl/hql/hqlutil.cpp

@@ -560,7 +560,7 @@ void expandRecord(HqlExprArray & selects, IHqlExpression * selector, IHqlExpress
     }
 }
 
-//---------------------------------------------------------------------------
+//---------------------------------------------------------------------------------------------------------------------
 
 static IHqlExpression * queryOnlyTableChild(IHqlExpression * expr)
 {
@@ -607,13 +607,134 @@ static IHqlExpression * findCommonExpression(IHqlExpression * lower, IHqlExpress
 }
 
 
+//---------------------------------------------------------------------------------------------------------------------
+
+bool isCommonSubstringRange(IHqlExpression * expr)
+{
+    if (expr->getOperator() != no_substring)
+        return false;
+    IHqlExpression * range = expr->queryChild(1);
+    return (range->getOperator() == no_rangecommon);
+}
+
+IHqlExpression * removeCommonSubstringRange(IHqlExpression * expr)
+{
+    if (expr->getOperator() != no_substring)
+        return LINK(expr);
+    IHqlExpression * range = expr->queryChild(1);
+    if (range->getOperator() != no_rangecommon)
+        return LINK(expr);
+    IHqlExpression * value = expr->queryChild(0);
+    IHqlExpression * from = range->queryChild(0);
+    if (matchesConstantValue(from, 1))
+        return LINK(value);
+    OwnedHqlExpr newRange = createValue(no_rangefrom, makeNullType(), LINK(from));
+    return replaceChild(expr, 1, newRange.getClear());
+}
+
+void AtmostLimit::extractAtmostArgs(IHqlExpression * atmost)
+{
+    limit.setown(getSizetConstant(0));
+    if (atmost)
+    {
+        unsigned cur = 0;
+        IHqlExpression * arg = atmost->queryChild(0);
+        if (arg && arg->isBoolean())
+        {
+            required.set(arg);
+            arg = atmost->queryChild(++cur);
+        }
+        if (arg && arg->queryType()->getTypeCode() == type_sortlist)
+        {
+            unwindChildren(optional, arg);
+            arg = atmost->queryChild(++cur);
+        }
+        if (arg)
+            limit.set(arg);
+    }
+}
+
+
+static bool matchesAtmostCondition(IHqlExpression * cond, HqlExprArray & atConds, unsigned & numMatched)
+{
+    if (atConds.find(*cond) != NotFound)
+    {
+        numMatched++;
+        return true;
+    }
+    if (cond->getOperator() != no_assertkeyed)
+        return false;
+    unsigned savedMatched = numMatched;
+    HqlExprArray conds;
+    cond->queryChild(0)->unwindList(conds, no_and);
+    ForEachItemIn(i, conds)
+    {
+        if (!matchesAtmostCondition(&conds.item(i), atConds, numMatched))
+        {
+            numMatched = savedMatched;
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool doSplitFuzzyCondition(IHqlExpression * condition, IHqlExpression * atmostCond, SharedHqlExpr & fuzzy, SharedHqlExpr & hard)
+{
+    if (atmostCond)
+    {
+        //If join condition has evaluated to a constant then allow any atmost condition.
+        if (!condition->isConstant())
+        {
+            HqlExprArray conds, atConds;
+            condition->unwindList(conds, no_and);
+            atmostCond->unwindList(atConds, no_and);
+            unsigned numAtmostMatched = 0;
+            ForEachItemIn(i, conds)
+            {
+                IHqlExpression & cur = conds.item(i);
+                if (matchesAtmostCondition(&cur, atConds, numAtmostMatched))
+                    extendConditionOwn(hard, no_and, LINK(&cur));
+                else
+                    extendConditionOwn(fuzzy, no_and, LINK(&cur));
+            }
+            if (atConds.ordinality() != numAtmostMatched)
+            {
+                hard.clear();
+                fuzzy.clear();
+                return false;
+            }
+        }
+    }
+    else
+        hard.set(condition);
+    return true;
+}
+
+void splitFuzzyCondition(IHqlExpression * condition, IHqlExpression * atmostCond, SharedHqlExpr & fuzzy, SharedHqlExpr & hard)
+{
+    if (!doSplitFuzzyCondition(condition, atmostCond, fuzzy, hard))
+    {
+        //Ugly, but sometimes the condition only matches after it has been constant folded.
+        //And this function can be called from the normalizer before the expression tree is folded.
+        OwnedHqlExpr foldedCond = foldHqlExpression(condition);
+        OwnedHqlExpr foldedAtmost = foldHqlExpression(atmostCond);
+        if (!doSplitFuzzyCondition(foldedCond, foldedAtmost, fuzzy, hard))
+        {
+            StringBuffer s;
+            getExprECL(atmostCond, s);
+            throwError1(HQLERR_AtmostFailMatchCondition, s.str());
+        }
+    }
+}
+
+//---------------------------------------------------------------------------------------------------------------------
 
 
 
 class JoinOrderSpotter
 {
 public:
-    JoinOrderSpotter(IHqlExpression * _leftDs, IHqlExpression * _rightDs, IHqlExpression * seq, HqlExprArray & _leftSorts, HqlExprArray & _rightSorts) : leftSorts(_leftSorts), rightSorts(_rightSorts)
+    JoinOrderSpotter(IHqlExpression * _leftDs, IHqlExpression * _rightDs, IHqlExpression * seq, JoinSortInfo & _joinOrder) : joinOrder(_joinOrder)
     {
         if (_leftDs)
             left.setown(createSelector(no_left, _leftDs, seq));
@@ -621,7 +742,8 @@ public:
             right.setown(createSelector(no_right, _rightDs, seq));
     }
 
-    IHqlExpression * doFindJoinSortOrders(IHqlExpression * condition, HqlExprArray * slidingMatches, HqlExprCopyArray & matched);
+    IHqlExpression * doFindJoinSortOrders(IHqlExpression * condition, bool allowSlidingMatch, HqlExprCopyArray & matched);
+    bool doProcessOptional(IHqlExpression * condition);
     void findImplicitBetween(IHqlExpression * condition, HqlExprArray & slidingMatches, HqlExprCopyArray & matched, HqlExprCopyArray & pending);
 
 protected:
@@ -633,8 +755,7 @@ protected:
 protected:
     OwnedHqlExpr left;
     OwnedHqlExpr right;
-    HqlExprArray & leftSorts;
-    HqlExprArray & rightSorts;
+    JoinSortInfo & joinOrder;
 };
 
 
@@ -737,7 +858,7 @@ void JoinOrderSpotter::unwindSelectorRecord(HqlExprArray & target, IHqlExpressio
     }
 }
 
-IHqlExpression * JoinOrderSpotter::doFindJoinSortOrders(IHqlExpression * condition, HqlExprArray * slidingMatches, HqlExprCopyArray & matched)
+IHqlExpression * JoinOrderSpotter::doFindJoinSortOrders(IHqlExpression * condition, bool allowSlidingMatch, HqlExprCopyArray & matched)
 {
     IHqlExpression *l = condition->queryChild(0);
     IHqlExpression *r = condition->queryChild(1);
@@ -746,8 +867,8 @@ IHqlExpression * JoinOrderSpotter::doFindJoinSortOrders(IHqlExpression * conditi
     {
     case no_and:
         {
-            IHqlExpression *lmatch = doFindJoinSortOrders(l, slidingMatches, matched);
-            IHqlExpression *rmatch = doFindJoinSortOrders(r, slidingMatches, matched);
+            IHqlExpression *lmatch = doFindJoinSortOrders(l, allowSlidingMatch, matched);
+            IHqlExpression *rmatch = doFindJoinSortOrders(r, allowSlidingMatch, matched);
             if (lmatch)
             {
                 if (rmatch)
@@ -772,26 +893,26 @@ IHqlExpression * JoinOrderSpotter::doFindJoinSortOrders(IHqlExpression * conditi
 
             if ((leftSelectKind == no_left) && (rightSelectKind == no_right))
             {
-                leftSorts.append(*leftStrip.getClear());
-                rightSorts.append(*rightStrip.getClear());
+                joinOrder.leftReq.append(*leftStrip.getClear());
+                joinOrder.rightReq.append(*rightStrip.getClear());
                 return NULL;
             }
             if ((leftSelectKind == no_right) && (rightSelectKind == no_left))
             {
-                leftSorts.append(*rightStrip.getClear());
-                rightSorts.append(*leftStrip.getClear());
+                joinOrder.leftReq.append(*rightStrip.getClear());
+                joinOrder.rightReq.append(*leftStrip.getClear());
                 return NULL;
             }
             if (((l == left) && (r == right)) || ((l == right) && (r == left)))
             {
-                unwindSelectorRecord(leftSorts, queryActiveTableSelector(), left->queryRecord());
-                unwindSelectorRecord(rightSorts, queryActiveTableSelector(), right->queryRecord());
+                unwindSelectorRecord(joinOrder.leftReq, queryActiveTableSelector(), left->queryRecord());
+                unwindSelectorRecord(joinOrder.rightReq, queryActiveTableSelector(), right->queryRecord());
                 return NULL;
             }
         }
         return LINK(condition);
     case no_between:
-        if (slidingMatches)
+        if (allowSlidingMatch)
         {
             node_operator leftSelectKind = no_none;
             node_operator rightSelectKind = no_none;
@@ -805,7 +926,7 @@ IHqlExpression * JoinOrderSpotter::doFindJoinSortOrders(IHqlExpression * conditi
                 IHqlExpression * common = findCommonExpression(lowerStrip,upperStrip);
                 if (common)
                 {
-                    slidingMatches->append(*createValue(no_between, makeBoolType(), LINK(leftStrip), LINK(lowerStrip), LINK(upperStrip), createExprAttribute(commonAtom, LINK(common))));
+                    joinOrder.slidingMatches.append(*createValue(no_between, makeBoolType(), LINK(leftStrip), LINK(lowerStrip), LINK(upperStrip), createExprAttribute(commonAtom, LINK(common))));
                     return NULL;
                 }
             }
@@ -823,6 +944,50 @@ IHqlExpression * JoinOrderSpotter::doFindJoinSortOrders(IHqlExpression * conditi
     }
 }
 
+bool JoinOrderSpotter::doProcessOptional(IHqlExpression * condition)
+{
+    switch(condition->getOperator())
+    {
+    //MORE We could support no_and by adding a list to both sides, but I can't see it being worth the effort.
+    case no_constant:
+        //remove silly "and true" conditions
+        if (condition->queryValue()->getBoolValue())
+            return true;
+        return false;
+    case no_eq:
+        {
+            IHqlExpression *l = condition->queryChild(0);
+            IHqlExpression *r = condition->queryChild(1);
+            node_operator leftSelectKind = no_none;
+            node_operator rightSelectKind = no_none;
+            OwnedHqlExpr leftStrip = traverseStripSelect(l, leftSelectKind);
+            OwnedHqlExpr rightStrip = traverseStripSelect(r, rightSelectKind);
+
+            if ((leftSelectKind == no_left) && (rightSelectKind == no_right))
+            {
+                joinOrder.leftOpt.append(*leftStrip.getClear());
+                joinOrder.rightOpt.append(*rightStrip.getClear());
+                return true;
+            }
+            if ((leftSelectKind == no_right) && (rightSelectKind == no_left))
+            {
+                joinOrder.leftOpt.append(*rightStrip.getClear());
+                joinOrder.rightOpt.append(*leftStrip.getClear());
+                return true;
+            }
+            if (((l == left) && (r == right)) || ((l == right) && (r == left)))
+            {
+                unwindSelectorRecord(joinOrder.leftOpt, queryActiveTableSelector(), left->queryRecord());
+                unwindSelectorRecord(joinOrder.rightOpt, queryActiveTableSelector(), right->queryRecord());
+                return true;
+            }
+        }
+        return false;
+    default:
+        return false;
+    }
+}
+
 void JoinOrderSpotter::findImplicitBetween(IHqlExpression * condition, HqlExprArray & slidingMatches, HqlExprCopyArray & matched, HqlExprCopyArray & pending)
 {
     IHqlExpression *l = condition->queryChild(0);
@@ -875,57 +1040,42 @@ void JoinOrderSpotter::findImplicitBetween(IHqlExpression * condition, HqlExprAr
     }
 }
 
-static bool isCommonSubstringRange(IHqlExpression * expr)
+JoinSortInfo::JoinSortInfo()
 {
-    if (expr->getOperator() != no_substring)
-        return false;
-    IHqlExpression * range = expr->queryChild(1);
-    return (range->getOperator() == no_rangecommon);
+    conditionAllEqualities = false;
 }
 
-static IHqlExpression * getSimplifiedCommonSubstringRange(IHqlExpression * expr)
+void JoinSortInfo::findJoinSortOrders(IHqlExpression * condition, IHqlExpression * leftDs, IHqlExpression * rightDs, IHqlExpression * seq, bool allowSlidingMatch)
 {
-    IHqlExpression * rawSelect = expr->queryChild(0);
-    IHqlExpression * range = expr->queryChild(1);
-    IHqlExpression * rangeLow = range->queryChild(0);
-    if (matchesConstantValue(rangeLow, 1))
-        return LINK(rawSelect);
-    HqlExprArray args;
-    args.append(*LINK(rawSelect));
-    args.append(*createValue(no_rangefrom, makeNullType(), LINK(rangeLow)));
-    return expr->clone(args);
-}
-
-
-IHqlExpression * findJoinSortOrders(IHqlExpression * condition, IHqlExpression * leftDs, IHqlExpression * rightDs, IHqlExpression * seq, HqlExprArray &leftSorts, HqlExprArray &rightSorts, bool & isLimitedSubstringJoin, HqlExprArray * slidingMatches)
-{
-    JoinOrderSpotter spotter(leftDs, rightDs, seq, leftSorts, rightSorts);
+    JoinOrderSpotter spotter(leftDs, rightDs, seq, *this);
     HqlExprCopyArray matched;
-    if (slidingMatches)
+    if (allowSlidingMatch)
     {
         //First spot any implicit betweens using x >= a and x <= b.  Do it first so that the second pass doesn't
         //reorder the join condition (this still reorders it slightly by moving the implicit betweens before explicit)
         HqlExprCopyArray pending;
-        spotter.findImplicitBetween(condition, *slidingMatches, matched, pending);
+        spotter.findImplicitBetween(condition, slidingMatches, matched, pending);
     }
-    OwnedHqlExpr ret = spotter.doFindJoinSortOrders(condition, slidingMatches, matched);
     
-    //Check for x[n..*] - a no_rangecommon, and ensure they are tagged as the last sorts.
-    unsigned numCommonRange = 0;
-    ForEachItemInRev(i, leftSorts)
+    extraMatch.setown(spotter.doFindJoinSortOrders(condition, allowSlidingMatch, matched));
+    conditionAllEqualities = (extraMatch == NULL);
+
+    //Support for legacy syntax where x[n..*] was present in join and atmost condition
+    //Ensure they are tagged as optional join fields.
+    ForEachItemInRev(i, leftReq)
     {
-        IHqlExpression & left = leftSorts.item(i);
-        IHqlExpression & right = rightSorts.item(i);
+        IHqlExpression & left = leftReq.item(i);
+        IHqlExpression & right = rightReq.item(i);
         if (isCommonSubstringRange(&left))
         {
             if (isCommonSubstringRange(&right))
             {
-                //MORE: May be best to remove the substring syntax as this point - or modify it if start != 1.
-                leftSorts.append(*getSimplifiedCommonSubstringRange(&left));
-                leftSorts.remove(i);
-                rightSorts.append(*getSimplifiedCommonSubstringRange(&right));
-                rightSorts.remove(i);
-                numCommonRange++;
+                if (leftOpt.ordinality())
+                    throwError(HQLERR_AtmostLegacyMismatch);
+                leftOpt.append(OLINK(left));
+                leftReq.remove(i);
+                rightOpt.append(OLINK(right));
+                rightReq.remove(i);
             }
             else
                 throwError(HQLERR_AtmostSubstringNotMatch);
@@ -936,30 +1086,65 @@ IHqlExpression * findJoinSortOrders(IHqlExpression * condition, IHqlExpression *
                 throwError(HQLERR_AtmostSubstringNotMatch);
         }
     }
-    isLimitedSubstringJoin = numCommonRange != 0;
-    if (numCommonRange > 1)
-        throwError(HQLERR_AtmostSubstringSingleInstance);
 
-    if ((numCommonRange == 0) && slidingMatches)
+    if (!hasOptionalEqualities() && allowSlidingMatch)
     {
-        ForEachItemIn(i, *slidingMatches)
+        ForEachItemIn(i, slidingMatches)
         {
-            IHqlExpression & cur = slidingMatches->item(i);
-            leftSorts.append(*LINK(cur.queryChild(0)));
-            rightSorts.append(*LINK(cur.queryChild(3)->queryChild(0)));
+            IHqlExpression & cur = slidingMatches.item(i);
+            leftReq.append(*LINK(cur.queryChild(0)));
+            rightReq.append(*LINK(cur.queryChild(3)->queryChild(0)));
         }
     }
 
-
-    return ret.getClear();
 }
 
-extern HQL_API IHqlExpression * findJoinSortOrders(IHqlExpression * expr, HqlExprArray &leftSorts, HqlExprArray &rightSorts, bool & isLimitedSubstringJoin, HqlExprArray * slidingMatches)
+void JoinSortInfo::findJoinSortOrders(IHqlExpression * expr, bool allowSlidingMatch)
 {
     IHqlExpression * lhs = expr->queryChild(0);
-    return findJoinSortOrders(expr->queryChild(2), lhs, queryJoinRhs(expr), querySelSeq(expr), leftSorts, rightSorts, isLimitedSubstringJoin, slidingMatches);
+    IHqlExpression * rhs = queryJoinRhs(expr);
+    IHqlExpression * selSeq = querySelSeq(expr);
+
+    atmost.extractAtmostArgs(expr->queryAttribute(atmostAtom));
+    if (atmost.optional.ordinality())
+    {
+        JoinOrderSpotter spotter(lhs, rhs, selSeq, *this);
+        ForEachItemIn(i, atmost.optional)
+        {
+            if (!spotter.doProcessOptional(&atmost.optional.item(i)))
+                throwError(HQLERR_AtmostCannotImplementOpt);
+        }
+    }
+
+    OwnedHqlExpr fuzzy, hard;
+    splitFuzzyCondition(expr->queryChild(2), atmost.required, fuzzy, hard);
+    findJoinSortOrders(hard, lhs, rhs, selSeq, allowSlidingMatch);
+
+    extraMatch.setown(extendConditionOwn(no_and, extraMatch.getClear(), fuzzy.getClear()));
+}
+
+static void appendOptElements(HqlExprArray & target, const HqlExprArray & src)
+{
+    ForEachItemIn(i, src)
+    {
+        IHqlExpression & cur = src.item(i);
+        //Strip the substring syntax when adding the optional compares to the sort list
+        target.append(*removeCommonSubstringRange(&cur));
+    }
+}
+
+void JoinSortInfo::initSorts()
+{
+    if (!leftSorts.ordinality())
+    {
+        appendArray(leftSorts, leftReq);
+        appendOptElements(leftSorts, leftOpt);
+        appendArray(rightSorts, rightReq);
+        appendOptElements(rightSorts, rightOpt);
+    }
 }
 
+
 IHqlExpression * createImpureOwn(IHqlExpression * expr)
 {
     return createValue(no_impure, expr->getType(), expr);
@@ -5509,7 +5694,7 @@ void TempTableTransformer::reportWarning(IHqlExpression * location, int code,con
 IHqlExpression *getDictionaryKeyRecord(IHqlExpression *record)
 {
     IHqlExpression * payload = record->queryAttribute(_payload_Atom);
-    unsigned payloadSize = payload ? getIntValue(payload->queryChild(0)) : 0;
+    unsigned payloadSize = payload ? (unsigned)getIntValue(payload->queryChild(0)) : 0;
     unsigned max = record->numChildren() - payloadSize;
     IHqlExpression *newrec = createRecord();
     for (unsigned idx = 0; idx < max; idx++)

ecl/hql/hqlutil.hpp

@@ -26,8 +26,9 @@ extern HQL_API node_operator queryTransformSingleAggregate(IHqlExpression * tran
 extern HQL_API bool containsOnlyLeft(IHqlExpression * expr,bool ignoreSelfOrFilepos = false);
 extern HQL_API IHqlExpression * queryPhysicalRootTable(IHqlExpression * expr);
 extern HQL_API IHqlExpression * queryTableFilename(IHqlExpression * expr);
-extern HQL_API IHqlExpression * findJoinSortOrders(IHqlExpression * condition, IHqlExpression * leftDs, IHqlExpression * rightDs, IHqlExpression * seq, HqlExprArray &leftSorts, HqlExprArray &rightSorts, bool & isLimitedSubstringJoin, HqlExprArray * slidingMatches);
-extern HQL_API IHqlExpression * findJoinSortOrders(IHqlExpression * expr, HqlExprArray &leftSorts, HqlExprArray &rightSorts, bool & isLimitedSubstringJoin, HqlExprArray * slidingMatches);
+
+extern HQL_API void splitFuzzyCondition(IHqlExpression * condition, IHqlExpression * atmostCond, SharedHqlExpr & fuzzy, SharedHqlExpr & hard);
+
 extern HQL_API IHqlExpression * createRawIndex(IHqlExpression * index);
 extern HQL_API IHqlExpression * createImpureOwn(IHqlExpression * expr);
 extern HQL_API IHqlExpression * getNormalizedFilename(IHqlExpression * filename);
@@ -672,4 +673,56 @@ extern HQL_API bool userPreventsSort(IHqlExpression * noSortAttr, node_operator
 extern HQL_API IHqlExpression * queryTransformAssign(IHqlExpression * transform, IHqlExpression * searchField);
 extern HQL_API IHqlExpression * queryTransformAssignValue(IHqlExpression * transform, IHqlExpression * searchField);
 
+extern HQL_API bool isCommonSubstringRange(IHqlExpression * expr);
+
+class HQL_API AtmostLimit
+{
+public:
+    AtmostLimit(IHqlExpression * expr = NULL)
+    {
+        extractAtmostArgs(expr);
+    }
+    void extractAtmostArgs(IHqlExpression * atmost);
+
+public:
+    OwnedHqlExpr required;
+    HqlExprArray optional;
+    OwnedHqlExpr limit;
+};
+
+class HQL_API JoinSortInfo
+{
+    friend class JoinOrderSpotter;
+public:
+    JoinSortInfo();
+
+    void findJoinSortOrders(IHqlExpression * condition, IHqlExpression * leftDs, IHqlExpression * rightDs, IHqlExpression * seq, bool allowSlidingMatch);
+    void findJoinSortOrders(IHqlExpression * expr, bool allowSlidingMatch);
+
+    inline bool hasRequiredEqualities() const { return leftReq.ordinality() != 0; }
+    inline bool hasOptionalEqualities() const { return leftOpt.ordinality() != 0; }
+    inline const HqlExprArray & queryLeftReq() { return leftReq; }
+    inline const HqlExprArray & queryRightReq() { return rightReq; }
+    inline const HqlExprArray & queryLeftOpt() { return leftOpt; }
+    inline const HqlExprArray & queryRightOpt() { return rightOpt; }
+    inline const HqlExprArray & queryLeftSort() { initSorts(); return leftSorts; }
+    inline const HqlExprArray & queryRightSort() { initSorts(); return rightSorts; }
+
+protected:
+    void initSorts();
+
+public:
+    AtmostLimit atmost;
+    OwnedHqlExpr extraMatch;
+    HqlExprArray slidingMatches;
+    bool conditionAllEqualities;
+protected:
+    HqlExprArray leftReq;
+    HqlExprArray leftOpt;
+    HqlExprArray leftSorts;
+    HqlExprArray rightReq;
+    HqlExprArray rightOpt;
+    HqlExprArray rightSorts;
+};
+
 #endif

ecl/hqlcpp/hqlcerrors.hpp

@@ -92,7 +92,6 @@
 #define HQLERR_BadJoinConditionAtMost           4067
 #define HQLERR_BadKeyedJoinConditionAtMost      4068
 #define HQLERR_FullJoinNeedDataset              4069
-#define HQLERR_AtmostFailMatchCondition         4070
 #define HQLERR_KeyedLimitNotKeyed               4071
 #define HQLERR_ExtendMismatch                   4072
 #define HQLERR_DuplicateNameOutput              4073
@@ -376,7 +375,6 @@
 #define HQLERR_BadJoinConditionAtMost_Text      "ATMOST JOIN cannot be evaluated with this join condition%s"
 #define HQLERR_BadKeyedJoinConditionAtMost_Text "ATMOST JOIN cannot be evaluated with this join condition%s"
 #define HQLERR_FullJoinNeedDataset_Text         "RIGHT for a full keyed join must be a disk based DATASET"
-#define HQLERR_AtmostFailMatchCondition_Text    "ATMOST(%s) failed to match part of the join condition"
 #define HQLERR_KeyedLimitNotKeyed_Text          "LIMIT(%s, KEYED) could not be merged into an index read"
 #define HQLERR_ExtendMismatch_Text              "EXTEND is required on all outputs to NAMED(%s)"
 #define HQLERR_DuplicateNameOutput_Text         "Duplicate output to NAMED(%s).  EXTEND/OVERWRITE required"

ecl/hqlcpp/hqlckey.cpp

@@ -1104,13 +1104,12 @@ void KeyedJoinInfo::optimizeExtractJoinFields()
 
 bool KeyedJoinInfo::processFilter()
 {
-    OwnedHqlExpr atmostCond, atmostLimit;
-    IHqlExpression * atmost = expr->queryAttribute(atmostAtom);
-    extractAtmostArgs(atmost, atmostCond, atmostLimit);
+    IHqlExpression * atmostAttr = expr->queryAttribute(atmostAtom);
+    AtmostLimit atmost(atmostAttr);
 
     IHqlExpression * cond = expr->queryChild(2);
     OwnedHqlExpr fuzzy, hard;
-    translator.splitFuzzyCondition(cond, atmostCond, fuzzy, hard);
+    splitFuzzyCondition(cond, atmost.required, fuzzy, hard);
 
     OwnedHqlExpr keyedKeyFilter, fuzzyKeyFilter;
     splitFilter(hard, keyedKeyFilter);
@@ -1128,7 +1127,7 @@ bool KeyedJoinInfo::processFilter()
         else
             leftOnlyMatch.set(cond);
     }
-    if (atmost && fileFilter)
+    if (atmostAttr && fileFilter)
     {
         StringBuffer s;
         translator.throwError1(HQLERR_BadKeyedJoinConditionAtMost,getExprECL(fileFilter, s.append(" (")).append(")").str());
@@ -1154,7 +1153,7 @@ bool KeyedJoinInfo::processFilter()
     if (newFilter)
         monitors->extractFilters(newFilter, extra);
 
-    if (atmost && extra && (atmostCond || !monitors->isCleanlyKeyedExplicitly()))
+    if (atmostAttr && extra && (atmost.required || !monitors->isCleanlyKeyedExplicitly()))
     {
         StringBuffer s;
         //map the key references back so the error message refers to RIGHT instead of a weird key expression.
@@ -1380,9 +1379,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedJoinOrDenormalize(BuildCt
     StringBuffer s;
     buildInstancePrefix(instance);
 
-    OwnedHqlExpr atmostCond, atmostLimit;
-    IHqlExpression * atmost = expr->queryAttribute(atmostAtom);
-    extractAtmostArgs(atmost, atmostCond, atmostLimit);
+    IHqlExpression * atmostAttr = expr->queryAttribute(atmostAtom);
+    AtmostLimit atmost(atmostAttr);
 
     //virtual unsigned getJoinFlags()
     StringBuffer flags;
@@ -1435,15 +1433,15 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedJoinOrDenormalize(BuildCt
         doBuildUnsignedFunction(instance->classctx, "getFetchFlags", flags.str()+1);
 
     //virtual unsigned getJoinLimit()
-    if (!isZero(atmostLimit))
-        doBuildUnsignedFunction(instance->startctx, "getJoinLimit", atmostLimit);
+    if (!isZero(atmost.limit))
+        doBuildUnsignedFunction(instance->startctx, "getJoinLimit", atmost.limit);
 
     //virtual unsigned getKeepLimit()
     LinkedHqlExpr keepLimit = queryAttributeChild(expr, keepAtom, 0);
     if (keepLimit)
         doBuildUnsignedFunction(instance->startctx, "getKeepLimit", keepLimit);
 
-    bool implicitLimit = !rowlimit && !atmost &&
+    bool implicitLimit = !rowlimit && !atmostAttr &&
                         (!keepLimit || info.hasPostFilter()) &&
                         !expr->hasAttribute(leftonlyAtom);
 
@@ -1498,20 +1496,19 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedDistribute(BuildCtx & ctx
     if (!targetThor() || insideChildQuery(ctx))
         return buildCachedActivity(ctx, expr->queryChild(0));
 
-    HqlExprArray leftSorts, rightSorts;
     IHqlExpression * left = expr->queryChild(0);
     IHqlExpression * right = expr->queryChild(1);
     IHqlExpression * indexRecord = right->queryRecord();
     IHqlExpression * seq = querySelSeq(expr);
-    bool isLimitedSubstringJoin;
-    OwnedHqlExpr match = findJoinSortOrders(expr, leftSorts, rightSorts, isLimitedSubstringJoin, NULL);
-    assertex(leftSorts.ordinality() == rightSorts.ordinality());
-    if (isLimitedSubstringJoin)
+
+    JoinSortInfo joinInfo;
+    joinInfo.findJoinSortOrders(expr, false);
+    if (joinInfo.hasOptionalEqualities())
         throwError(HQLERR_KeyedDistributeNoSubstringJoin);
 
     unsigned numUnsortedFields = numPayloadFields(right);
     unsigned numKeyedFields = getFlatFieldCount(indexRecord)-numUnsortedFields;
-    if (match || (!expr->hasAttribute(firstAtom) && (leftSorts.ordinality() != numKeyedFields)))
+    if (joinInfo.extraMatch || (!expr->hasAttribute(firstAtom) && (joinInfo.queryLeftReq().ordinality() != numKeyedFields)))
         throwError(HQLERR_MustMatchExactly);    //Should already be caught in parser
 
     KeyedJoinInfo info(*this, expr, false);
@@ -1558,15 +1555,15 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedDistribute(BuildCtx & ctx
     TableProjectMapper mapper(expandedIndex);
 
     HqlExprArray normalizedRight;
-    ForEachItemIn(i, rightSorts)
+    ForEachItemIn(i, joinInfo.queryRightReq())
     {
-        IHqlExpression & curRight = rightSorts.item(i);
+        IHqlExpression & curRight = joinInfo.queryRightReq().item(i);
         normalizedRight.append(*mapper.expandFields(&curRight, oldSelector, newSelector));
     }
     DatasetReference    leftDs(left, no_activetable, seq);
     DatasetReference    rightDs(rawIndex, no_activetable, seq);
 
-    doCompareLeftRight(instance->nestedctx, "CompareRowKey", leftDs, rightDs, leftSorts, normalizedRight);
+    doCompareLeftRight(instance->nestedctx, "CompareRowKey", leftDs, rightDs, joinInfo.queryLeftReq(), normalizedRight);
 
     buildFormatCrcFunction(instance->classctx, "getFormatCrc", info.queryRawKey(), info.queryRawKey(), 1);
     buildSerializedLayoutMember(instance->classctx, indexRecord, "getIndexLayout", numKeyedFields);

ecl/hqlcpp/hqlcpp.ipp

@@ -1625,8 +1625,8 @@ public:
     IHqlExpression * createOrderFromSortList(const DatasetReference & dataset, IHqlExpression * sortList, IHqlExpression * leftSelect, IHqlExpression * rightSelect);
 
     void buildSkewThresholdMembers(BuildCtx & ctx, IHqlExpression * expr);
-    void doCompareLeftRight(BuildCtx & ctx, const char * funcname, const DatasetReference & datasetLeft, const DatasetReference & datasetRight, HqlExprArray & left, HqlExprArray & right);
-    void buildSlidingMatchFunction(BuildCtx & ctx, HqlExprArray & leftEq, HqlExprArray & rightEq, HqlExprArray & slidingMatches, const char * funcname, unsigned childIndex, const DatasetReference & datasetL, const DatasetReference & datasetR);
+    void doCompareLeftRight(BuildCtx & ctx, const char * funcname, const DatasetReference & datasetLeft, const DatasetReference & datasetRight, const HqlExprArray & left, const HqlExprArray & right);
+    void buildSlidingMatchFunction(BuildCtx & ctx, const HqlExprArray & leftEq, const HqlExprArray & rightEq, const HqlExprArray & slidingMatches, const char * funcname, unsigned childIndex, const DatasetReference & datasetL, const DatasetReference & datasetR);
     void doBuildIndexOutputTransform(BuildCtx & ctx, IHqlExpression * record, SharedHqlExpr & rawRecord);
 
     void buildKeyedJoinExtra(ActivityInstance & instance, IHqlExpression * expr, KeyedJoinInfo * joinKey);
@@ -1758,7 +1758,6 @@ protected:
     IHqlExpression * normalizeGlobalIfCondition(BuildCtx & ctx, IHqlExpression * expr);
     void substituteClusterSize(HqlExprArray & exprs);
     void throwCannotCast(ITypeInfo * from, ITypeInfo * to);
-    void splitFuzzyCondition(IHqlExpression * condition, IHqlExpression * atmostCond, SharedHqlExpr & fuzzy, SharedHqlExpr & hard);
 
     void ensureSerialized(BuildCtx & ctx, const CHqlBoundTarget & variable);
 
@@ -1769,8 +1768,8 @@ protected:
     void doFilterAssignments(BuildCtx & ctx, TransformBuilder * builder, HqlExprArray & assigns, IHqlExpression * expr);
     void generateSerializeAssigns(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * selector, IHqlExpression * selfSelect, IHqlExpression * leftSelect, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, bool needToClear, node_operator serializeOp, IAtom * serialForm);
     void generateSerializeFunction(BuildCtx & ctx, const char * funcName, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, node_operator serializeOp, IAtom * serialForm);
-    void generateSerializeKey(BuildCtx & ctx, node_operator side, const DatasetReference & dataset, HqlExprArray & sorts, bool isGlobal, bool generateCompares, bool canReuseLeft);             //NB: sorts are ats.xyz
-    void generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx, node_operator index, const DatasetReference & dataset, HqlExprArray & sorts, bool canRemoveSort, IHqlExpression * noSortAttr, bool canReuseLeft, bool isLightweight, bool isLocal);
+    void generateSerializeKey(BuildCtx & ctx, node_operator side, const DatasetReference & dataset, const HqlExprArray & sorts, bool isGlobal, bool generateCompares, bool canReuseLeft);             //NB: sorts are ats.xyz
+    void generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx, node_operator index, const DatasetReference & dataset, const HqlExprArray & sorts, bool canRemoveSort, IHqlExpression * noSortAttr, bool canReuseLeft, bool isLightweight, bool isLocal);
     void addSchemaField(IHqlExpression *field, MemoryBuffer &schema, IHqlExpression *selector);
     void addSchemaFields(IHqlExpression * record, MemoryBuffer &schema, IHqlExpression *selector);
     void addSchemaResource(int seq, const char * name, IHqlExpression * record);

ecl/hqlcpp/hqlcppsys.ecl

@@ -227,8 +227,8 @@ const char * cppSystemText[]  = {
     "   integer4 compareVUnicodeVUnicode(const varunicode l, const varunicode r, const varstring loc) : eclrtl,pure,library='eclrtl',entrypoint='rtlCompareVUnicodeVUnicode';",
     "   integer4 compareVUnicodeVUnicodeStrength(const varunicode l, const varunicode r, const varstring loc, unsigned4 str) : eclrtl,pure,library='eclrtl',entrypoint='rtlCompareVUnicodeVUnicodeStrength';",
 
-    "   integer4 prefixDiffStr(const string l, const string r) : eclrtl,pure,library='eclrtl',entrypoint='rtlPrefixDiffStr';",
-    "   integer4 prefixDiffUnicode(const unicode l, const unicode r, const varstring loc) : eclrtl,pure,library='eclrtl',entrypoint='rtlPrefixDiffUnicode';",
+    "   integer4 prefixDiffStr(const string l, const string r, unsigned4 origin) : eclrtl,pure,library='eclrtl',entrypoint='rtlPrefixDiffStrEx';",
+    "   integer4 prefixDiffUnicode(const unicode l, const unicode r, const varstring loc, unsigned4 origin) : eclrtl,pure,library='eclrtl',entrypoint='rtlPrefixDiffUnicodeEx';",
 
     "   createOrder(data1 tgt, const data1 src, unsigned4 num, unsigned4 width, const data1 compare) : eclrtl,library='eclrtl',entrypoint='rtlCreateOrder';",
     "   unsigned4 rankFromOrder(unsigned4 idx, unsigned4 num, const data1 order) : eclrtl,pure,library='eclrtl',entrypoint='rtlRankFromOrder';",

ecl/hqlcpp/hqlhtcpp.cpp

@@ -249,78 +249,6 @@ static IHqlExpression * createResultName(IHqlExpression * name)
 
 //---------------------------------------------------------------------------
 
-void extractAtmostArgs(IHqlExpression * atmost, SharedHqlExpr & atmostCond, SharedHqlExpr & atmostLimit)
-{
-    atmostLimit.set(queryZero());
-    if (atmost)
-    {
-        IHqlExpression * arg0 = atmost->queryChild(0);
-        if (arg0->isBoolean())
-        {
-            atmostCond.set(arg0);
-            atmostLimit.set(atmost->queryChild(1));
-        }
-        else
-            atmostLimit.set(arg0);
-    }
-}
-
-static bool matchesAtmostCondition(IHqlExpression * cond, HqlExprArray & atConds, unsigned & numMatched)
-{
-    if (atConds.find(*cond) != NotFound)
-    {
-        numMatched++;
-        return true;
-    }
-    if (cond->getOperator() != no_assertkeyed)
-        return false;
-    unsigned savedMatched = numMatched;
-    HqlExprArray conds;
-    cond->queryChild(0)->unwindList(conds, no_and);
-    ForEachItemIn(i, conds)
-    {
-        if (!matchesAtmostCondition(&conds.item(i), atConds, numMatched))
-        {
-            numMatched = savedMatched;
-            return false;
-        }
-    }
-    return true;
-}
-
-void HqlCppTranslator::splitFuzzyCondition(IHqlExpression * condition, IHqlExpression * atmostCond, SharedHqlExpr & fuzzy, SharedHqlExpr & hard)
-{
-    if (atmostCond)
-    {
-        //If join condition has evaluated to a constant then allow any atmost condition.
-        if (!condition->isConstant())
-        {
-            HqlExprArray conds, atConds;
-            condition->unwindList(conds, no_and);
-            atmostCond->unwindList(atConds, no_and);
-            unsigned numAtmostMatched = 0;
-            ForEachItemIn(i, conds)
-            {
-                IHqlExpression & cur = conds.item(i);
-                if (matchesAtmostCondition(&cur, atConds, numAtmostMatched))
-                    extendConditionOwn(hard, no_and, LINK(&cur));
-                else
-                    extendConditionOwn(fuzzy, no_and, LINK(&cur));
-            }
-            if (atConds.ordinality() != numAtmostMatched)
-            {
-                StringBuffer s;
-                getExprECL(atmostCond, s);
-                throwError1(HQLERR_AtmostFailMatchCondition, s.str());
-            }
-        }
-    }
-    else
-        hard.set(condition);
-}
-
-//---------------------------------------------------------------------------
-
 ColumnToOffsetMap * RecordOffsetMap::queryMapping(IHqlExpression * record, unsigned maxRecordSize)
 {
     ColumnToOffsetMap * match = find(record);
@@ -11178,7 +11106,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityPipeThrough(BuildCtx & ctx, IH
 //-- no_join [JOIN] --
 
 /* in parms: NOT linked */
-void HqlCppTranslator::doCompareLeftRight(BuildCtx & ctx, const char * funcname, const DatasetReference & datasetLeft, const DatasetReference & datasetRight, HqlExprArray & left, HqlExprArray & right)
+void HqlCppTranslator::doCompareLeftRight(BuildCtx & ctx, const char * funcname, const DatasetReference & datasetLeft, const DatasetReference & datasetRight, const HqlExprArray & left, const HqlExprArray & right)
 {
     OwnedHqlExpr selSeq = createDummySelectorSequence();
     OwnedHqlExpr leftList = createValueSafe(no_sortlist, makeSortListType(NULL), left);
@@ -11192,7 +11120,7 @@ void HqlCppTranslator::doCompareLeftRight(BuildCtx & ctx, const char * funcname,
     buildCompareMemberLR(ctx, funcname, order, datasetLeft.queryDataset(), datasetRight.queryDataset(), selSeq);
 }
 
-void HqlCppTranslator::buildSlidingMatchFunction(BuildCtx & ctx, HqlExprArray & leftEq, HqlExprArray & rightEq, HqlExprArray & slidingMatches, const char * funcname, unsigned childIndex, const DatasetReference & datasetL, const DatasetReference & datasetR)
+void HqlCppTranslator::buildSlidingMatchFunction(BuildCtx & ctx, const HqlExprArray & leftEq, const HqlExprArray & rightEq, const HqlExprArray & slidingMatches, const char * funcname, unsigned childIndex, const DatasetReference & datasetL, const DatasetReference & datasetR)
 {
     HqlExprArray left, right;
     unsigned numSimple = leftEq.ordinality() - slidingMatches.ordinality();
@@ -11211,7 +11139,7 @@ void HqlCppTranslator::buildSlidingMatchFunction(BuildCtx & ctx, HqlExprArray &
     doCompareLeftRight(ctx, funcname, datasetL, datasetR, left, right);
 }
 
-void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx, node_operator side, const DatasetReference & dataset, HqlExprArray & sorts, bool canRemoveSort, IHqlExpression * noSortAttr, bool canReuseLeft, bool isLightweight, bool isLocal)
+void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx, node_operator side, const DatasetReference & dataset, const HqlExprArray & sorts, bool canRemoveSort, IHqlExpression * noSortAttr, bool canReuseLeft, bool isLightweight, bool isLocal)
 {
     StringBuffer s, compareName;
 
@@ -11342,7 +11270,7 @@ void HqlCppTranslator::generateSerializeFunction(BuildCtx & ctx, const char * fu
     buildReturnRecordSize(r2kctx, serialize ? tgtCursor : srcCursor);
 }
 
-void HqlCppTranslator::generateSerializeKey(BuildCtx & nestedctx, node_operator side, const DatasetReference & dataset, HqlExprArray & sorts, bool isGlobal, bool generateCompares, bool canReuseLeft)
+void HqlCppTranslator::generateSerializeKey(BuildCtx & nestedctx, node_operator side, const DatasetReference & dataset, const HqlExprArray & sorts, bool isGlobal, bool generateCompares, bool canReuseLeft)
 {
     //check if there are any ifblocks, and if so don't allow it.  Even more accurate would be no join fields used in ifblocks
     IHqlExpression * record = dataset.queryDataset()->queryRecord();
@@ -11498,6 +11426,37 @@ void HqlCppTranslator::doBuildJoinRowLimitHelper(ActivityInstance & instance, IH
 }
 
 
+static size32_t getMaxSubstringLength(IHqlExpression * expr)
+{
+    IHqlExpression * rawSelect = expr->queryChild(0);
+    IHqlExpression * range = expr->queryChild(1);
+    IHqlExpression * rangeLow = range->queryChild(0);
+    unsigned rawLength = rawSelect->queryType()->getStringLen();
+    if (matchesConstantValue(rangeLow, 1))
+        return rawLength;
+
+    __int64 lowValue = getIntValue(rangeLow, UNKNOWN_LENGTH);
+    size32_t resultLength = UNKNOWN_LENGTH;
+    if ((rawLength != UNKNOWN_LENGTH) && (lowValue >= 1) && (lowValue <= rawLength))
+        resultLength = rawLength - (size32_t)(lowValue - 1);
+    return resultLength;
+}
+
+static IHqlExpression * getSimplifiedCommonSubstringRange(IHqlExpression * expr)
+{
+    IHqlExpression * rawSelect = expr->queryChild(0);
+    IHqlExpression * range = expr->queryChild(1);
+    IHqlExpression * rangeLow = range->queryChild(0);
+    if (matchesConstantValue(rangeLow, 1))
+        return LINK(rawSelect);
+
+    HqlExprArray args;
+    args.append(*LINK(rawSelect));
+    args.append(*createValue(no_rangefrom, makeNullType(), LINK(rangeLow)));
+    return expr->clone(args);
+}
+
+
 ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & ctx, IHqlExpression * expr)
 {
     node_operator op = expr->getOperator();
@@ -11550,7 +11509,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
     bool isLocalJoin = !isHashJoin && expr->hasAttribute(localAtom);
     bool joinToSelf = (op == no_selfjoin);
     bool allowAllToLookupConvert = !options.noAllToLookupConversion;
-    IHqlExpression * atmost = expr->queryAttribute(atmostAtom);
+    IHqlExpression * atmostAttr = expr->queryAttribute(atmostAtom);
     //Delay removing ungroups until this point because they can be useful for reducing the size of spill files.
     if (isUngroup(dataset1) && !isLookupJoin)
         dataset1.set(dataset1->queryChild(0));
@@ -11575,34 +11534,25 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
     }
 
 
-    OwnedHqlExpr atmostCond, atmostLimit;
-    extractAtmostArgs(atmost, atmostCond, atmostLimit);
-
-    HqlExprArray leftSorts, rightSorts, slidingMatches;
     bool slidingAllowed = options.slidingJoins && canBeSlidingJoin(expr);
-    OwnedHqlExpr match;
-
-    OwnedHqlExpr fuzzy, hard;
-    bool isLimitedSubstringJoin;
-    splitFuzzyCondition(condition, atmostCond, fuzzy, hard);
-    match.setown(findJoinSortOrders(hard, dataset1, dataset2, selSeq, leftSorts, rightSorts, isLimitedSubstringJoin, slidingAllowed ? &slidingMatches : NULL));
+    JoinSortInfo joinInfo;
+    joinInfo.findJoinSortOrders(expr, slidingAllowed);
 
-    if (atmost && match)
+    if (atmostAttr && !joinInfo.conditionAllEqualities)
     {
         if (isAllJoin)
             allowAllToLookupConvert = false;
         else
         {
             StringBuffer s;
-            throwError1(HQLERR_BadJoinConditionAtMost,getExprECL(match, s.append(" (")).append(")").str());
+            throwError1(HQLERR_BadJoinConditionAtMost,getExprECL(joinInfo.extraMatch, s.append(" (")).append(")").str());
         }
     }
-    extendConditionOwn(match, no_and, fuzzy.getClear());
 
     LinkedHqlExpr rhs = dataset2;
     if (isAllJoin)
     {
-        if (leftSorts.ordinality() && allowAllToLookupConvert)
+        if (joinInfo.hasRequiredEqualities() && allowAllToLookupConvert)
         {
             //Convert an all join to a many lookup if it can be done that way - more efficient, and same resourcing/semantics ...
             isManyLookup = true;
@@ -11610,7 +11560,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
             isLookupJoin = true;
         }
     }
-    else if (leftSorts.ordinality() == 0)
+    else if (!joinInfo.hasRequiredEqualities() && !joinInfo.hasOptionalEqualities())
     {
         if (expr->hasAttribute(_conditionFolded_Atom))
         {
@@ -11766,19 +11716,19 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
         //more could use the compareLeftRight function instead of generating the same code 
         //several time....
     }
-    bool canReuseLeft = recordTypesMatch(dataset1, dataset2) && arraysMatch(leftSorts, rightSorts);
+    bool canReuseLeftCompare = recordTypesMatch(dataset1, dataset2) && arraysMatch(joinInfo.queryLeftSort(), joinInfo.queryRightSort());
     if (!isAllJoin)
     {
         bool isLocalSort = isLocalJoin || !targetThor();
         //Lookup join doesn't need the left sort (unless it is reused elsewhere), or the right sort unless it is deduping.
-        if (canReuseLeft || !isLookupJoin)
-            generateSortCompare(instance->nestedctx, instance->classctx, no_left, lhsDsRef, leftSorts, true, noSortAttr, false, isLightweight, isLocalSort);
+        if (canReuseLeftCompare || !isLookupJoin)
+            generateSortCompare(instance->nestedctx, instance->classctx, no_left, lhsDsRef, joinInfo.queryLeftSort(), true, noSortAttr, false, isLightweight, isLocalSort);
         if (!(isLookupJoin && isManyLookup && !couldBeKeepOne && !targetThor()))            // many lookup doesn't need to dedup the rhs
-            generateSortCompare(instance->nestedctx, instance->classctx, no_right, rhsDsRef, rightSorts, isLocalSort, noSortAttr, canReuseLeft, isLightweight, isLocalSort);
+            generateSortCompare(instance->nestedctx, instance->classctx, no_right, rhsDsRef, joinInfo.queryRightSort(), isLocalSort, noSortAttr, canReuseLeftCompare, isLightweight, isLocalSort);
 
         bool isGlobal = !isLocalJoin && !instance->isChildActivity();
-        generateSerializeKey(instance->nestedctx, no_left, lhsDsRef, leftSorts, isGlobal, false, false);
-        generateSerializeKey(instance->nestedctx, no_right, rhsDsRef, rightSorts, isGlobal, false, canReuseLeft);
+        generateSerializeKey(instance->nestedctx, no_left, lhsDsRef, joinInfo.queryLeftSort(), isGlobal, false, false);
+        generateSerializeKey(instance->nestedctx, no_right, rhsDsRef, joinInfo.queryRightSort(), isGlobal, false, canReuseLeftCompare);
     }
 
     StringBuffer flags;
@@ -11797,8 +11747,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
         flags.append("|JFmatchAbortLimitSkips");
     if (rowlimit && rowlimit->hasAttribute(countAtom))
         flags.append("|JFcountmatchabortlimit");
-    if (slidingMatches.ordinality()) flags.append("|JFslidingmatch");
-    if (match) flags.append("|JFmatchrequired");
+    if (joinInfo.slidingMatches.ordinality()) flags.append("|JFslidingmatch");
+    if (joinInfo.extraMatch) flags.append("|JFmatchrequired");
     if (isLookupJoin && isManyLookup) flags.append("|JFmanylookup");
     if (expr->hasAttribute(onFailAtom))
         flags.append("|JFonfail");
@@ -11806,12 +11756,12 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
         flags.append("|JFreorderable");
     if (transformReturnsSide(expr, no_left, 0))
         flags.append("|JFtransformmatchesleft");
-    if (isLimitedSubstringJoin)
+    if (joinInfo.hasOptionalEqualities())
         flags.append("|JFlimitedprefixjoin");
 
-    if (isAlreadySorted(dataset1, leftSorts, true, true) || userPreventsSort(noSortAttr, no_left))
+    if (isAlreadySorted(dataset1, joinInfo.queryLeftSort(), true, true) || userPreventsSort(noSortAttr, no_left))
         flags.append("|JFleftSortedLocally");
-    if (isAlreadySorted(dataset2, rightSorts, true, true) || userPreventsSort(noSortAttr, no_right))
+    if (isAlreadySorted(dataset2, joinInfo.queryRightSort(), true, true) || userPreventsSort(noSortAttr, no_right))
         flags.append("|JFrightSortedLocally");
 
     if (flags.length())
@@ -11821,8 +11771,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
     {
         buildSkewThresholdMembers(instance->classctx, expr);
 
-        if (!isZero(atmostLimit))
-            doBuildUnsignedFunction(instance->startctx, "getJoinLimit", atmostLimit);
+        if (!isZero(joinInfo.atmost.limit))
+            doBuildUnsignedFunction(instance->startctx, "getJoinLimit", joinInfo.atmost.limit);
     }
 
     if (keepLimit)
@@ -11878,21 +11828,10 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
     if (!isAllJoin)
     {
         //if left and right match, then leftright compare function is also the same
-        if (isLimitedSubstringJoin)
-        {
-            HqlExprArray compareLeftSorts, compareRightSorts;
-            unsigned max = leftSorts.ordinality()-1;
-            for (unsigned i=0; i < max; i++)
-            {
-                compareLeftSorts.append(OLINK(leftSorts.item(i)));
-                compareRightSorts.append(OLINK(rightSorts.item(i)));
-            }
-            doCompareLeftRight(instance->nestedctx, "CompareLeftRight", lhsDsRef, rhsDsRef, compareLeftSorts, compareRightSorts);
-        }
-        else if (canReuseLeft)
+        if (canReuseLeftCompare && !joinInfo.hasOptionalEqualities())
             instance->nestedctx.addQuoted("virtual ICompare * queryCompareLeftRight() { return &compareLeft; }");
         else
-            doCompareLeftRight(instance->nestedctx, "CompareLeftRight", lhsDsRef, rhsDsRef, leftSorts, rightSorts);
+            doCompareLeftRight(instance->nestedctx, "CompareLeftRight", lhsDsRef, rhsDsRef, joinInfo.queryLeftReq(), joinInfo.queryRightReq());
     }
 
     doBuildJoinRowLimitHelper(*instance, rowlimit, NULL, false);
@@ -11903,44 +11842,111 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
         buildClearRecordMember(instance->createctx, "Left", dataset1);
     if (createDefaultRight)
         buildClearRecordMember(instance->createctx, "Right", dataset2);
-    buildJoinMatchFunction(instance->startctx, "match", dataset1, dataset2, match, selSeq);
+    buildJoinMatchFunction(instance->startctx, "match", dataset1, dataset2, joinInfo.extraMatch, selSeq);
 
-    if (slidingMatches.ordinality())
+    if (joinInfo.slidingMatches.ordinality())
     {
-        buildSlidingMatchFunction(instance->nestedctx, leftSorts, rightSorts, slidingMatches, "CompareLeftRightLower", 1, lhsDsRef, rhsDsRef);
-        buildSlidingMatchFunction(instance->nestedctx, leftSorts, rightSorts, slidingMatches, "CompareLeftRightUpper", 2, lhsDsRef, rhsDsRef);
+        buildSlidingMatchFunction(instance->nestedctx, joinInfo.queryLeftSort(), joinInfo.queryRightSort(), joinInfo.slidingMatches, "CompareLeftRightLower", 1, lhsDsRef, rhsDsRef);
+        buildSlidingMatchFunction(instance->nestedctx, joinInfo.queryLeftSort(), joinInfo.queryRightSort(), joinInfo.slidingMatches, "CompareLeftRightUpper", 2, lhsDsRef, rhsDsRef);
     }
 
     if (isHashJoin||isLookupJoin)
     {
-        OwnedHqlExpr leftList = createValueSafe(no_sortlist, makeSortListType(NULL), leftSorts);
+        OwnedHqlExpr leftList = createValueSafe(no_sortlist, makeSortListType(NULL), joinInfo.queryLeftReq());
         buildHashOfExprsClass(instance->nestedctx, "HashLeft", leftList, lhsDsRef, false);
 
-        if (!canReuseLeft)
+        bool canReuseLeftHash = recordTypesMatch(dataset1, dataset2) && arraysMatch(joinInfo.queryLeftReq(), joinInfo.queryRightReq());
+        if (!canReuseLeftHash)
         {
-            OwnedHqlExpr rightList = createValueSafe(no_sortlist, makeSortListType(NULL), rightSorts);
+            OwnedHqlExpr rightList = createValueSafe(no_sortlist, makeSortListType(NULL), joinInfo.queryRightReq());
             buildHashOfExprsClass(instance->nestedctx, "HashRight", rightList, rhsDsRef, false);
         }
         else
             instance->nestedctx.addQuoted("virtual IHash * queryHashRight() { return &HashLeft; }");
     }
 
-    if (isLimitedSubstringJoin)
+    if (joinInfo.hasOptionalEqualities())
     {
         OwnedHqlExpr leftSelect = createSelector(no_left, dataset1, selSeq);
         OwnedHqlExpr rightSelect = createSelector(no_right, dataset2, selSeq);
-        HqlExprArray args;
-        args.append(*lhsDsRef.mapCompound(&leftSorts.tos(), leftSelect));
-        args.append(*rhsDsRef.mapCompound(&rightSorts.tos(), rightSelect));
 
-        IIdAtom * func = prefixDiffStrId;
-        ITypeInfo * lhsType = args.item(0).queryType();
-        if (isUnicodeType(lhsType))
+        UnsignedArray origins;
+        unsigned origin = 0;
+        ForEachItemIn(i, joinInfo.queryLeftOpt())
         {
-            func = prefixDiffUnicodeId;
-            args.append(*createConstant(lhsType->queryLocale()->str()));
+            IHqlExpression & left = joinInfo.queryLeftOpt().item(i);
+            IHqlExpression & right = joinInfo.queryRightOpt().item(i);
+            unsigned delta;
+            if (origin == UNKNOWN_LENGTH)
+                throwError(HQLERR_AtmostFollowUnknownSubstr);
+
+            if (isCommonSubstringRange(&left))
+            {
+                size32_t leftLen = getMaxSubstringLength(&left);
+                size32_t rightLen = getMaxSubstringLength(&right);
+                if (leftLen == rightLen)
+                    delta = leftLen;
+                else
+                    delta = UNKNOWN_LENGTH;
+            }
+            else
+                delta = 1;
+            origins.append(origin);
+            if (delta != UNKNOWN_LENGTH)
+                origin += delta;
+            else
+                origin = UNKNOWN_LENGTH;
+        }
+
+        OwnedHqlExpr compare;
+        OwnedITypeInfo retType = makeIntType(4, true);
+        OwnedHqlExpr zero = createConstant(retType->castFrom(true, 0));
+        ForEachItemInRev(i1, joinInfo.queryLeftOpt())
+        {
+            IHqlExpression & left = joinInfo.queryLeftOpt().item(i1);
+            IHqlExpression & right = joinInfo.queryRightOpt().item(i1);
+
+            unsigned origin = origins.item(i1);
+            if (isCommonSubstringRange(&left))
+            {
+                OwnedHqlExpr simpleLeft = getSimplifiedCommonSubstringRange(&left);
+                OwnedHqlExpr simpleRight = getSimplifiedCommonSubstringRange(&right);
+                HqlExprArray args;
+                args.append(*lhsDsRef.mapCompound(simpleLeft, leftSelect));
+                args.append(*rhsDsRef.mapCompound(simpleRight, rightSelect));
+
+                IIdAtom * func = prefixDiffStrId;
+                ITypeInfo * lhsType = args.item(0).queryType();
+                if (isUnicodeType(lhsType))
+                {
+                    func = prefixDiffUnicodeId;
+                    args.append(*createConstant(lhsType->queryLocale()->str()));
+                }
+                args.append(*getSizetConstant(origin));
+                OwnedHqlExpr diff = bindFunctionCall(func, args);
+                if (compare)
+                {
+                    OwnedHqlExpr alias = createAlias(diff, NULL);
+                    OwnedHqlExpr compareNe = createValue(no_ne, makeBoolType(), LINK(alias), LINK(zero));
+                    compare.setown(createValue(no_if, LINK(retType), compareNe.getClear(), LINK(alias), compare.getClear()));
+                }
+                else
+                    compare.set(diff);
+            }
+            else
+            {
+                OwnedHqlExpr leftExpr = lhsDsRef.mapCompound(&left, leftSelect);
+                OwnedHqlExpr rightExpr = lhsDsRef.mapCompound(&right, rightSelect);
+                OwnedHqlExpr compareGt = createValue(no_gt, makeBoolType(), LINK(leftExpr), LINK(rightExpr));
+                OwnedHqlExpr gtValue = createConstant(retType->castFrom(true, origin+1));
+                OwnedHqlExpr ltValue = createConstant(retType->castFrom(true, -(int)(origin+1)));
+                OwnedHqlExpr mismatch = createValue(no_if, LINK(retType), compareGt.getClear(), gtValue.getClear(), ltValue.getClear());
+                OwnedHqlExpr compareNe = createValue(no_ne, makeBoolType(), LINK(leftExpr), LINK(rightExpr));
+                OwnedHqlExpr eqValue = compare ? LINK(compare) : LINK(zero);
+                compare.setown(createValue(no_if, LINK(retType), compareNe.getClear(), mismatch.getClear(), eqValue.getClear()));
+                origin += 1;
+            }
         }
-        OwnedHqlExpr compare = bindFunctionCall(func, args);
 
         buildCompareMemberLR(instance->nestedctx, "PrefixCompare", compare, dataset1, dataset2, selSeq);
     }

ecl/hqlcpp/hqlhtcpp.ipp

@@ -295,8 +295,6 @@ protected:
     bool            matchedDataset;
 };
 
-void extractAtmostArgs(IHqlExpression * atmost, SharedHqlExpr & atmostCond, SharedHqlExpr & atmostLimit);
-
 IHqlExpression * extractFilterConditions(HqlExprAttr & invariant, IHqlExpression * expr, IHqlExpression * dataset, bool spotCSE);
 bool isLibraryScope(IHqlExpression * expr);
 extern IHqlExpression * constantMemberMarkerExpr;

ecl/hqlcpp/hqlttcpp.cpp

@@ -2485,7 +2485,7 @@ IHqlExpression * ThorHqlTransformer::normalizeCoGroup(IHqlExpression * expr)
     return expr->cloneAllAnnotations(grouped);
 }
 
-static IHqlExpression * getNonThorSortedJoinInput(IHqlExpression * joinExpr, IHqlExpression * dataset, HqlExprArray & sorts, bool implicitSubSort)
+static IHqlExpression * getNonThorSortedJoinInput(IHqlExpression * joinExpr, IHqlExpression * dataset, const HqlExprArray & sorts, bool implicitSubSort)
 {
     if (!sorts.length())
         return LINK(dataset);
@@ -2521,7 +2521,7 @@ static bool sameOrGrouped(IHqlExpression * newLeft, IHqlExpression * oldLeft)
     return (newLeft->queryBody() == oldLeft->queryBody());
 }
 
-bool canReorderMatchExistingLocalSort(HqlExprArray & newElements1, HqlExprArray & newElements2, IHqlExpression * ds1, Shared<IHqlExpression> & ds2, const HqlExprArray & elements1, HqlExprArray & elements2, bool canSubSort, bool isLocal, bool alwaysLocal)
+static bool canReorderMatchExistingLocalSort(HqlExprArray & newElements1, HqlExprArray & newElements2, IHqlExpression * ds1, Shared<IHqlExpression> & ds2, const HqlExprArray & elements1, const HqlExprArray & elements2, bool canSubSort, bool isLocal, bool alwaysLocal)
 {
     newElements1.kill();
     newElements2.kill();
@@ -2674,15 +2674,14 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
     }
 
 
-    HqlExprArray leftSorts, rightSorts, slidingMatches;
-    bool isLimitedSubstringJoin;
-    OwnedHqlExpr fuzzyMatch = findJoinSortOrders(expr, leftSorts, rightSorts, isLimitedSubstringJoin, canBeSlidingJoin(expr) ? &slidingMatches : NULL);
+    JoinSortInfo joinInfo;
+    joinInfo.findJoinSortOrders(expr, canBeSlidingJoin(expr));
 
     //If the data is already distributed so the data is on the correct machines then perform the join locally.
     //Should be equally applicable to lookup, hash, all and normal joins.
-    if (!isLocal && !isLimitedSubstringJoin && leftSorts.ordinality())
+    if (!isLocal && !joinInfo.hasOptionalEqualities() && joinInfo.queryLeftReq().ordinality())
     {
-        if (isDistributedCoLocally(leftDs, rightDs, leftSorts, rightSorts))
+        if (isDistributedCoLocally(leftDs, rightDs, joinInfo.queryLeftReq(), joinInfo.queryRightReq()))
             return appendOwnedOperand(expr, createLocalAttribute());
 
         if (options.matchExistingDistributionForJoin)
@@ -2698,10 +2697,10 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
             if (matchesAnyDistribution(leftDistribution))
                 return appendOwnedOperand(expr, createLocalAttribute());
 
-            if (!isPersistDistribution(leftDistribution) && !isSortedDistribution(leftDistribution) && isPartitionedForGroup(leftDs, leftSorts, true))
+            if (!isPersistDistribution(leftDistribution) && !isSortedDistribution(leftDistribution) && isPartitionedForGroup(leftDs, joinInfo.queryLeftReq(), true))
             {
                 //MORE: May need a flag to stop this - to prevent issues with skew.
-                OwnedHqlExpr newHash = createMatchingDistribution(leftDistribution, leftSorts, rightSorts);
+                OwnedHqlExpr newHash = createMatchingDistribution(leftDistribution, joinInfo.queryLeftReq(), joinInfo.queryRightReq());
                 if (newHash)
                 {
                     OwnedHqlExpr dist = replaceSelector(newHash, queryActiveTableSelector(), rightDs);
@@ -2714,11 +2713,18 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
         else
         {
             IHqlExpression * leftDistribution = queryDistribution(leftDs);
-            if (!isPersistDistribution(leftDistribution) && !isSortedDistribution(leftDistribution) && isPartitionedForGroup(leftDs, leftSorts, true))
+            if (!isPersistDistribution(leftDistribution) && !isSortedDistribution(leftDistribution) && isPartitionedForGroup(leftDs, joinInfo.queryLeftReq(), true))
                 DBGLOG("MORE: Potential for distributed join optimization");
         }
     }
 
+    if (joinInfo.hasOptionalEqualities() && !isLocal && !expr->hasAttribute(hashAtom) && !expr->hasAttribute(allAtom))
+    {
+        if (joinInfo.hasRequiredEqualities())
+            return appendAttribute(expr, hashAtom);
+        throwError(HQLERR_PrefixJoinRequiresEquality);
+    }
+
     if (expr->hasAttribute(allAtom))
         return NULL;
 
@@ -2728,10 +2734,10 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
     //Try and convert local joins to a lightweight join that doesn't require any sorting of the inputs.
     //Improves resourcing for thor, and prevents lookup conversion for hthor/roxie
     //Worthwhile even for lookup joins
-    if (isLightweightJoinCandidate(expr, isLocal, isLimitedSubstringJoin))
+    if (isLightweightJoinCandidate(expr, isLocal, joinInfo.hasOptionalEqualities()))
     {
-        if (isAlreadySorted(leftDs, leftSorts, true, true) &&
-            isAlreadySorted(rightDs, rightSorts, true, true))
+        if (isAlreadySorted(leftDs, joinInfo.queryLeftSort(), true, true) &&
+            isAlreadySorted(rightDs, joinInfo.queryRightSort(), true, true))
         {
             //If this is a lookup join without a many then we need to make sure only the first match is retained.
             return appendOwnedOperand(expr, createAttribute(_lightweight_Atom));
@@ -2740,18 +2746,18 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
         //Check for a local join where we can reorder the condition so both sides match the existing sort orders.
         //could special case self-join to do less work, but probably not worth the effort.
         HqlExprArray sortedLeft, sortedRight;
-        if (!isLimitedSubstringJoin)
+        if (!joinInfo.hasOptionalEqualities())
         {
             //Since the distribution and order of global joins is not defined this could probably be used for non-local as well.
             LinkedHqlExpr newLeftDs = leftDs;
             LinkedHqlExpr newRightDs = rightDs;
             bool canSubSort = options.subsortLocalJoinConditions;
             bool reordered = canReorderMatchExistingLocalSort(sortedLeft, sortedRight, newLeftDs, newRightDs,
-                                                              leftSorts, rightSorts, canSubSort, isLocal, alwaysLocal);
+                                                              joinInfo.queryLeftSort(), joinInfo.queryRightSort(), canSubSort, isLocal, alwaysLocal);
             //If allowed to subsort then try the otherway around
             if (!reordered && canSubSort)
                 reordered = canReorderMatchExistingLocalSort(sortedRight, sortedLeft, newRightDs, newLeftDs,
-                                                             rightSorts, leftSorts, canSubSort, isLocal, alwaysLocal);
+                                                             joinInfo.queryRightSort(), joinInfo.queryLeftSort(), canSubSort, isLocal, alwaysLocal);
 
             if (reordered)
             {
@@ -2765,7 +2771,7 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
                     OwnedHqlExpr rc = replaceSelector(&sortedRight.item(i), queryActiveTableSelector(), rightSelector);
                     extendConditionOwn(newcond, no_and, createValue(no_eq, makeBoolType(), lc.getClear(), rc.getClear()));
                 }
-                extendConditionOwn(newcond, no_and, fuzzyMatch.getClear());
+                extendConditionOwn(newcond, no_and, LINK(joinInfo.extraMatch));
                 HqlExprArray args;
                 args.append(*newLeftDs.getClear());
                 args.append(*newRightDs.getClear());
@@ -2788,12 +2794,12 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
                     createLookup = !expr->hasAttribute(_lightweight_Atom);
         }
 
-        if (isLimitedSubstringJoin)
+        if (joinInfo.hasOptionalEqualities())
             createLookup = false;           //doesn't support it yet
-        else if (createLookup && leftSorts.ordinality() && rightSorts.ordinality())
+        else if (createLookup && joinInfo.queryLeftSort().ordinality())
         {
             //Check this isn't going to generate a between join - if it is that takes precedence.
-            if ((slidingMatches.ordinality() != 0) && (leftSorts.ordinality() == slidingMatches.ordinality()))
+            if ((joinInfo.slidingMatches.ordinality() != 0) && (joinInfo.queryLeftSort().ordinality() == joinInfo.slidingMatches.ordinality()))
                 createLookup = false;
         }
 
@@ -2814,8 +2820,8 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
             return expr->clone(args);
         }
 
-        OwnedHqlExpr newLeft = getNonThorSortedJoinInput(expr, leftDs, leftSorts, options.implicitSubSort);
-        OwnedHqlExpr newRight = getNonThorSortedJoinInput(expr, rightDs, rightSorts, options.implicitSubSort);
+        OwnedHqlExpr newLeft = getNonThorSortedJoinInput(expr, leftDs, joinInfo.queryLeftSort(), options.implicitSubSort);
+        OwnedHqlExpr newRight = getNonThorSortedJoinInput(expr, rightDs, joinInfo.queryRightSort(), options.implicitSubSort);
         try
         {
             if ((leftDs != newLeft) || (rightDs != newRight))
@@ -2840,15 +2846,9 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
     if ((op == no_selfjoin) && expr->hasAttribute(hashAtom))
     {
         assertex(!isLocal);
-        if (isLimitedSubstringJoin)
-        {
-            leftSorts.pop();
-            rightSorts.pop();
-        }
-
-        if (leftSorts.ordinality())
+        if (joinInfo.hasRequiredEqualities())
         {
-            OwnedHqlExpr sortlist = createValueSafe(no_sortlist, makeSortListType(NULL), leftSorts);
+            OwnedHqlExpr sortlist = createValueSafe(no_sortlist, makeSortListType(NULL), joinInfo.queryLeftReq());
             OwnedHqlExpr distribute;
             //Only likely to catch this partition test if isLimitedSubstringJoin true, otherwise caught above
             if (!isPartitionedForGroup(leftDs, sortlist, true))
@@ -2868,11 +2868,11 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
         }
     }
 
-    if (options.expandHashJoin && isThorCluster(targetClusterType) && expr->hasAttribute(hashAtom) && !isLimitedSubstringJoin)
+    if (options.expandHashJoin && isThorCluster(targetClusterType) && expr->hasAttribute(hashAtom) && !joinInfo.hasOptionalEqualities())
     {
         HqlExprArray args;
-        args.append(*createDistributedInput(leftDs, leftSorts, false));
-        args.append(*createDistributedInput(rightDs, rightSorts, false));
+        args.append(*createDistributedInput(leftDs, joinInfo.queryLeftReq(), false));
+        args.append(*createDistributedInput(rightDs, joinInfo.queryRightReq(), false));
         unwindChildren(args, expr, 2);
         removeProperty(args, hashAtom);
         args.append(*createLocalAttribute());
@@ -2885,9 +2885,9 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
         OwnedHqlExpr newLeft;
         OwnedHqlExpr newRight;
         if (!userPreventsSort(noSortAttr, no_left))
-            newLeft.setown(getSubSort(leftDs, leftSorts, isLocal, true, alwaysLocal));
+            newLeft.setown(getSubSort(leftDs, joinInfo.queryLeftSort(), isLocal, true, alwaysLocal));
         if (!userPreventsSort(noSortAttr, no_right))
-            newRight.setown(getSubSort(rightDs, rightSorts, isLocal, true, alwaysLocal));
+            newRight.setown(getSubSort(rightDs, joinInfo.queryRightSort(), isLocal, true, alwaysLocal));
         if (newLeft || newRight)
         {
             HqlExprArray args;

ecl/regress/limmatch.ecl

@@ -56,7 +56,7 @@ recO T(recL l,recR r) := TRANSFORM
 END;
 
 
-J1 := JOIN(dsL,dsR,left.name[1..*]=right.name[3..*] and left.val<right.val,T(LEFT,RIGHT), ATMOST(left.name[1..*]=right.name[3..*],3));
+J1 := JOIN(dsL,dsR,left.name[1..*]=right.name[3..*] and left.val<right.val,T(LEFT,RIGHT), ATMOST(left.name[1..*]=right.name[3..*],3), LOCAL);
 
 output(J1);
 

rtl/eclrtl/eclrtl.cpp

@@ -2544,7 +2544,7 @@ void rtlKeyUnicodeStrengthX(unsigned & tlen, void * & tgt, unsigned slen, const
     ucol_getSortKey(coll, src, slen, (unsigned char *)tgt, tlen);
 }
 
-ECLRTL_API int rtlPrefixDiffStr(unsigned l1, const char * p1, unsigned l2, const char * p2)
+ECLRTL_API int rtlPrefixDiffStrEx(unsigned l1, const char * p1, unsigned l2, const char * p2, unsigned origin)
 {
     unsigned len = l1 < l2 ? l1 : l2;
     const byte * str1 = (const byte *)p1;
@@ -2556,18 +2556,23 @@ ECLRTL_API int rtlPrefixDiffStr(unsigned l1, const char * p1, unsigned l2, const
         if (c1 != c2)
         {
             if (c1 < c2)
-                return -(int)(i+1);
+                return -(int)(i+origin+1);
             else
-                return (int)(i+1);
+                return (int)(i+origin+1);
         }
     }
     if (l1 != l2)
-        return (l1 < l2) ? -(int)(len+1) : (int)(len + 1);
+        return (l1 < l2) ? -(int)(len+origin+1) : (int)(len+origin+1);
     return 0;
 }
 
+ECLRTL_API int rtlPrefixDiffStr(unsigned l1, const char * p1, unsigned l2, const char * p2)
+{
+    return rtlPrefixDiffStrEx(l1, p1, l2, p2, 0);
+}
+
 //MORE: I'm not sure this can really be implemented....
-ECLRTL_API int rtlPrefixDiffUnicode(unsigned l1, const UChar * p1, unsigned l2, const UChar * p2, char const * locale)
+ECLRTL_API int rtlPrefixDiffUnicodeEx(unsigned l1, const UChar * p1, unsigned l2, const UChar * p2, char const * locale, unsigned origin)
 {
     while(l1 && u_isUWhiteSpace(p1[l1-1])) l1--;
     while(l2 && u_isUWhiteSpace(p2[l2-1])) l2--;
@@ -2578,18 +2583,20 @@ ECLRTL_API int rtlPrefixDiffUnicode(unsigned l1, const UChar * p1, unsigned l2,
         {
             int c = ucol_strcoll(queryRTLLocale(locale)->queryCollator(), p1+i, l1-i, p2+i, l2-i);
             if (c < 0)
-                return -(int)(i+1);
+                return -(int)(i+origin+1);
             else if (c > 0)
-                return (int)(i+1);
-            else
-                return 0;       //weird!
+                return (int)(i+origin+1);
         }
     }
     if (l1 != l2)
-        return (l1 < l2) ? -(int)(len+1) : (int)(len + 1);
+        return (l1 < l2) ? -(int)(len+origin+1) : (int)(len+origin+1);
     return 0;
 }
 
+ECLRTL_API int rtlPrefixDiffUnicode(unsigned l1, const UChar * p1, unsigned l2, const UChar * p2, char const * locale)
+{
+    return rtlPrefixDiffUnicodeEx(l1, p1, l2, p2, locale, 0);
+}
 
 //-----------------------------------------------------------------------------
 

rtl/eclrtl/eclrtl.hpp

@@ -250,7 +250,9 @@ ECLRTL_API void rtlKeyUnicodeStrengthX(unsigned & tlen, void * & tgt, unsigned s
 ECLRTL_API bool rtlGetNormalizedUnicodeLocaleName(unsigned len, char const * in, char * out);
 
 ECLRTL_API int rtlPrefixDiffStr(unsigned l1, const char * p1, unsigned l2, const char * p2);
+ECLRTL_API int rtlPrefixDiffStrEx(unsigned l1, const char * p1, unsigned l2, const char * p2, unsigned origin);
 ECLRTL_API int rtlPrefixDiffUnicode(unsigned l1, const UChar * p1, unsigned l2, const UChar * p2, const char * locale);
+ECLRTL_API int rtlPrefixDiffUnicodeEx(unsigned l1, const UChar * p1, unsigned l2, const UChar * p2, const char * locale, unsigned origin);
 
 ECLRTL_API void rtlTrimRight(unsigned &tlen, char * &tgt, unsigned slen, const char * src); // YMA
 ECLRTL_API void rtlTrimUnicodeRight(unsigned &tlen, UChar * &tgt, unsigned slen, UChar const * src);

testing/ecl/key/prefixjoin2.xml

@@ -0,0 +1,4 @@
+<Dataset name='Result 1'>
+ <Row><l><id>1</id><name>ACAAB     </name><id2>3</id2><val>0</val></l><r><id>1</id><name>ACAAB</name><id2>1</id2><val>6</val></r></Row>
+ <Row><l><id>1</id><name>FAAAA     </name><id2>1</id2><val>0</val></l><r><id>1</id><name>FAAAA</name><id2>1</id2><val>9</val></r></Row>
+</Dataset>

testing/ecl/key/prefixjoin3.xml

@@ -0,0 +1,110 @@
+<Dataset name='Result 1'>
+ <Row><sect>1</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>1</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>1</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>1</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>20</rightid></Row>
+ <Row><sect>2</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>2</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>2</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>2</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>2</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>2</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>2</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>2</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>2</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>2</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>2</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>2</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>2</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>2</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>2</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>2</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>2</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>2</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>2</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>2</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>2</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>2</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>2</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>2</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>2</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>2</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>2</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>2</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>2</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>2</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>2</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>2</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>2</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>2</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>2</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>2</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>2</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>2</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>2</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>2</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>2</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>2</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>2</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>2</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>2</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>2</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>2</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>2</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>2</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>2</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>2</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>2</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>2</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>2</sect><leftid>20</leftid><rightid>20</rightid></Row>
+</Dataset>

testing/ecl/key/prefixjoin4.xml

@@ -0,0 +1,380 @@
+<Dataset name='Result 1'>
+ <Row><sect>1</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>1</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>1</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>1</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>20</rightid></Row>
+ <Row><sect>2</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>2</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>2</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>2</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>2</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>2</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>2</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>2</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>2</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>2</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>2</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>2</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>2</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>2</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>2</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>2</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>2</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>2</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>2</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>2</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>2</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>2</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>2</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>2</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>2</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>2</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>2</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>2</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>2</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>2</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>2</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>2</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>2</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>2</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>2</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>2</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>2</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>2</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>2</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>2</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>2</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>2</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>2</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>2</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>2</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>2</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>2</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>2</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>2</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>2</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>2</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>2</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>2</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>2</sect><leftid>20</leftid><rightid>20</rightid></Row>
+ <Row><sect>3</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>3</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>3</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>3</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>3</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>3</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>3</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>3</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>3</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>3</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>3</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>3</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>3</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>3</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>3</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>3</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>3</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>3</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>3</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>3</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>3</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>3</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>3</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>3</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>3</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>3</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>3</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>3</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>3</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>3</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>3</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>3</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>3</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>3</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>3</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>3</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>3</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>3</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>3</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>3</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>3</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>3</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>3</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>3</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>3</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>3</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>3</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>3</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>3</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>3</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>3</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>3</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>3</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>3</sect><leftid>20</leftid><rightid>20</rightid></Row>
+ <Row><sect>4</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>4</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>4</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>4</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>4</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>4</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>4</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>4</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>4</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>4</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>4</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>4</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>4</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>4</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>4</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>4</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>4</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>4</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>4</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>4</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>4</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>4</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>4</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>4</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>4</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>4</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>4</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>4</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>4</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>4</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>4</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>4</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>4</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>4</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>4</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>4</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>4</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>4</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>4</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>4</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>4</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>4</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>4</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>4</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>4</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>4</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>4</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>4</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>4</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>4</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>4</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>4</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>4</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>4</sect><leftid>20</leftid><rightid>20</rightid></Row>
+ <Row><sect>5</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>5</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>5</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>5</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>5</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>5</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>5</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>5</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>5</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>5</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>5</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>5</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>5</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>5</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>5</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>5</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>5</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>5</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>5</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>5</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>5</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>5</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>5</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>5</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>5</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>5</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>5</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>5</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>5</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>5</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>5</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>5</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>5</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>5</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>5</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>5</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>5</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>5</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>5</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>5</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>5</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>5</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>5</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>5</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>5</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>5</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>5</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>5</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>5</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>5</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>5</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>5</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>5</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>5</sect><leftid>20</leftid><rightid>20</rightid></Row>
+ <Row><sect>6</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>6</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>6</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>6</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>6</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>6</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>6</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>6</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>6</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>6</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>6</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>6</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>6</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>6</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>6</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>6</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>6</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>6</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>6</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>6</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>6</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>6</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>6</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>6</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>6</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>6</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>6</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>6</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>6</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>6</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>6</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>6</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>6</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>6</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>6</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>6</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>6</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>6</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>6</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>6</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>6</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>6</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>6</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>6</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>6</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>6</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>6</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>6</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>6</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>6</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>6</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>6</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>6</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>6</sect><leftid>20</leftid><rightid>20</rightid></Row>
+ <Row><sect>7</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>7</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>7</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>7</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>7</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>7</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>7</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>7</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>7</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>7</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>7</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>7</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>7</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>7</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>7</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>7</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>7</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>7</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>7</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>7</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>7</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>7</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>7</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>7</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>7</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>7</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>7</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>7</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>7</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>7</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>7</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>7</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>7</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>7</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>7</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>7</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>7</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>7</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>7</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>7</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>7</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>7</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>7</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>7</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>7</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>7</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>7</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>7</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>7</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>7</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>7</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>7</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>7</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>7</sect><leftid>20</leftid><rightid>20</rightid></Row>
+</Dataset>

testing/ecl/key/prefixjoin5.xml

@@ -0,0 +1,406 @@
+<Dataset name='Result 1'>
+ <Row><sect>1</sect><leftid>1</leftid><rightid>1</rightid></Row>
+ <Row><sect>1</sect><leftid>1</leftid><rightid>2</rightid></Row>
+ <Row><sect>1</sect><leftid>2</leftid><rightid>1</rightid></Row>
+ <Row><sect>1</sect><leftid>2</leftid><rightid>2</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>3</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>4</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>5</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>6</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>7</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>8</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>9</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>10</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>11</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>12</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>13</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>14</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>15</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>16</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>17</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>18</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>19</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>20</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>21</leftid><rightid>22</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>3</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>4</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>5</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>6</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>7</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>8</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>9</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>10</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>11</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>12</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>13</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>14</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>15</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>16</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>17</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>18</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>19</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>20</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>21</rightid></Row>
+ <Row><sect>1</sect><leftid>22</leftid><rightid>22</rightid></Row>
+</Dataset>

testing/ecl/prefixjoin2.ecl

@@ -0,0 +1,81 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+// limited prefix match join tests
+
+recL := record
+    unsigned id;
+    STRING10 name;
+  unsigned      id2;
+  unsigned  val;
+END;
+
+recR := record
+  unsigned      id;
+  STRING        name {MAXLENGTH(12)};
+  unsigned      id2;
+  unsigned2     val;
+END;
+
+outR := RECORD
+    recL l;
+    recR r;
+END;
+
+dsL := DATASET([
+    { 1, 'ABAAA', 3, 0 },
+    { 1, 'ABAAA', 3, 3 },
+    { 1, 'ACAAB', 3, 0 },
+    { 1, 'ABAAA', 4, 0 },
+    { 1, 'FAAAA', 1, 0 },
+    { 2, 'ABAAB', 1, 0 }
+    ], recL);
+
+dsR := DATASET([
+    { 1, 'ABAAA', 1, 1 },   // atmost ok if including id2
+    { 1, 'ABAAA', 2, 2 },
+    { 1, 'ABAAA', 3, 3 },
+    { 1, 'ABAAA', 4, 4 },
+    { 1, 'ACAAA', 1, 5 },   //atmost ok if including str[name[1..4]
+    { 1, 'ACAAB', 1, 6 },
+    { 1, 'ACAAC', 1, 7 },
+    { 1, 'ACAAD', 1, 8 },
+    { 1, 'DAAAA', 1, 9 },
+    { 1, 'EAAAA', 1, 9 },
+    { 1, 'FAAAA', 1, 9 },
+    { 2, 'ABAAA', 1, 1 },   // atmost ok if including id2
+    { 2, 'ABAAA', 2, 2 },
+    { 2, 'ABAAA', 3, 3 },
+    { 2, 'ABAAA', 4, 4 },
+    { 2, 'ACAAA', 1, 5 },   //atmost ok if including str[name[1..4]
+    { 2, 'ACAAB', 1, 6 },
+    { 2, 'ACAAC', 1, 7 },
+    { 2, 'ACAAD', 1, 8 },
+    { 2, 'DAAAA', 1, 9 },
+    { 2, 'EAAAA', 1, 9 },
+    { 2, 'FAAAA', 1, 9 }
+    ], recR);
+
+
+sdsL := NOFOLD(dsL);
+sdsR := NOFOLD(dsR);
+
+JT := JOIN(sdsL, sdsR, left.id = right.id AND left.name[1..*]=right.name[1..*] and left.val<=right.val,
+            TRANSFORM(outR, SELF.l := LEFT; SELF.r := RIGHT), ATMOST(left.name[1..*]=right.name[1..*],3), LOCAL);
+
+OUTPUT(JT);

testing/ecl/prefixjoin3.ecl

@@ -0,0 +1,75 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('smallSortThreshold', 0);  // Stop all the records ending up on node 1
+
+// limited prefix match join tests
+
+rec := record
+  unsigned      id;
+  unsigned      sect;
+  STRING2       s1;
+  boolean       b1;
+  STRING2       s2;
+  unsigned      i1;
+END;
+
+outR := RECORD
+    rec l;
+    rec r;
+END;
+
+ds := DATASET([
+    //Match less than 3 on s1[1]
+    { 1, 1, 'AA', false, 'AA', 10 },
+    { 2, 1, 'AZ', true, 'QA', 12 },
+    //Match less than 3 on s1[2]
+    { 3, 1, 'BA', false, 'AA', 10 },
+    { 4, 1, 'BA', true, 'QA', 12 },
+    { 5, 1, 'BA', true, 'QA', 12 },
+    //Match less than 3 on s1, b1
+    { 6, 1, 'BB', true, 'QA', 12 },
+    { 7, 1, 'BB', true, 'QA', 12 },
+    { 8, 1, 'BB', true, 'QA', 12 },
+    //Match less than 3 on s1, b1, s2[1]
+    { 9, 1, 'BB', false, 'AA', 12 },
+    { 10, 1, 'BB', false, 'AB', 12 },
+    { 11, 1, 'BB', false, 'AC', 12 },
+    //Match less than 3 on s1, b1, s2
+    { 12, 1, 'BB', false, 'BA', 1 },
+    { 13, 1, 'BB', false, 'BA', 2 },
+    { 14, 1, 'BB', false, 'BA', 3 },
+    //Match less than 3 on s1, b1, s2, i1
+    { 15, 1, 'BB', false, 'BB', 1 },
+    { 16, 1, 'BB', false, 'BB', 1 },
+    { 17, 1, 'BB', false, 'BB', 1 },
+    //
+    { 18, 1, 'BB', false, 'BB', 2 },
+    { 19, 1, 'BB', false, 'BB', 2 },
+    //
+    { 20, 1, 'BB', false, 'BB', 3 }
+    ], rec);
+
+
+//Create duplicate sections
+dds := NOFOLD(DISTRIBUTE(ds, RANDOM()));
+sds := NOFOLD(NORMALIZE(dds, 2, TRANSFORM(rec, SELF.sect := COUNTER, SELF := LEFT)));
+
+JT := JOIN(sds, sds, left.sect = right.sect,
+            TRANSFORM(outR, SELF.l := LEFT; SELF.r := RIGHT), ATMOST({ left.s1[1..*]=right.s1[1..*], left.b1=right.b1, left.s2[1..*]=right.s2[1..*], left.i1=right.i1 },3));
+
+OUTPUT(SORT(JT, l.sect, l.id, r.id), { l.sect, leftid := l.id, rightid := r.id });

testing/ecl/prefixjoin4.ecl

@@ -0,0 +1,72 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('smallSortThreshold', 0);  // Stop all the records ending up on node 1
+
+// limited prefix match join tests
+
+rec := record
+  unsigned      id;
+  unsigned      sect;
+  STRING6       s;
+END;
+
+outR := RECORD
+    rec l;
+    rec r;
+END;
+
+ds := DATASET([
+    //Match less than 3 on s1[1]
+    { 1, 1, 'AAFAA0' },
+    { 2, 1, 'AZTQA2' },
+    //Match less than 3 on s1[2]
+    { 3, 1, 'BAFAA0' },
+    { 4, 1, 'BATQA2' },
+    { 5, 1, 'BATQA2' },
+    //Match less than 3 on s1, b1
+    { 6, 1, 'BBTQA2' },
+    { 7, 1, 'BBTQA2' },
+    { 8, 1, 'BBTQA2' },
+    //Match less than 3 on s1, b1, s2[1]
+    { 9, 1, 'BBFAA2' },
+    { 10, 1, 'BBFAB2' },
+    { 11, 1, 'BBFAC2' },
+    //Match less than 3 on s1, b1, s2
+    { 12, 1, 'BBFBA1' },
+    { 13, 1, 'BBFBA2' },
+    { 14, 1, 'BBFBA3' },
+    //Match less than 3 on s1, b1, s2, i1
+    { 15, 1, 'BBFBB1' },
+    { 16, 1, 'BBFBB1' },
+    { 17, 1, 'BBFBB1' },
+    //
+    { 18, 1, 'BBFBB2' },
+    { 19, 1, 'BBFBB2' },
+    //
+    { 20, 1, 'BBFBB3' }
+    ], rec);
+
+
+//Create duplicate sections
+dds := NOFOLD(DISTRIBUTE(ds, id));
+sds := NOFOLD(NORMALIZE(dds, 7, TRANSFORM(rec, SELF.sect := COUNTER, SELF := LEFT)));
+
+JT := JOIN(sds, sds, left.sect = right.sect AND left.s[1..*] = right.s[1..*],
+            TRANSFORM(outR, SELF.l := LEFT; SELF.r := RIGHT), ATMOST(left.sect = right.sect AND left.s[1..*] = right.s[1..*],3));
+
+OUTPUT(SORT(JT, l.sect, l.id, r.id), { l.sect, leftid := l.id, rightid := r.id });

testing/ecl/prefixjoin5.ecl

@@ -0,0 +1,66 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('smallSortThreshold', 0);  // Stop all the records ending up on node 1
+
+// limited prefix match join tests
+
+rec := record
+  unsigned      id;
+  unsigned      sect;
+  STRING6       s;
+END;
+
+outR := RECORD
+    rec l;
+    rec r;
+END;
+
+ds := DATASET([
+    //Match less than 3 on s1[1]
+    { 1, 1, 'AAFAA0' },
+    { 2, 1, 'AZTQA2' },
+    { 3, 1, 'BAAAAA' },
+    { 4, 1, 'BAAAAB' },
+    { 5, 1, 'BAAAAC' },
+    { 6, 1, 'BAAAAD' },
+    { 7, 1, 'BAAAAE' },
+    { 8, 1, 'BAAAAF' },
+    { 9, 1, 'BAAAAG' },
+    { 10, 1, 'BAAAAH' },
+    { 11, 1, 'BAAAAI' },
+    { 12, 1, 'BAAAAJ' },
+    { 13, 1, 'BAAAAK' },
+    { 14, 1, 'BAAAAL' },
+    { 15, 1, 'BAAAAM' },
+    { 16, 1, 'BAAAAN' },
+    { 17, 1, 'BAAAAO' },
+    { 18, 1, 'BAAAAP' },
+    { 19, 1, 'BAAAAQ' },
+    { 20, 1, 'BAAAAR' },
+    { 21, 1, 'BAAAAS' },
+    { 22, 1, 'BAAAAT' }
+    ], rec);
+
+
+//Create duplicate sections
+sds := NOFOLD(DISTRIBUTE(ds, id));
+
+JT := JOIN(sds, sds, left.sect = right.sect AND left.s[1..*] = right.s[1..*],
+            TRANSFORM(outR, SELF.l := LEFT; SELF.r := RIGHT), ATMOST(left.sect = right.sect AND left.s[1..*] = right.s[1..*],20));
+
+OUTPUT(SORT(JT, l.sect, l.id, r.id), { l.sect, leftid := l.id, rightid := r.id });