Merge pull request #6643 from ghalliday/issue8833

HPCC-8833 Assume keyed join is sorted unless UNORDERED

Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlcpp.cpp

@@ -1729,6 +1729,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.alwaysUseGraphResults,"alwaysUseGraphResults",false),
         DebugOption(options.reportAssertFilenameTail,"reportAssertFilenameTail",false),        
         DebugOption(options.newBalancedSpotter,"newBalancedSpotter",true),
+        DebugOption(options.keyedJoinPreservesOrder,"keyedJoinPreservesOrder",true),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -735,6 +735,7 @@ struct HqlCppOptions
     bool                alwaysUseGraphResults;
     bool                reportAssertFilenameTail;
     bool                newBalancedSpotter;
+    bool                keyedJoinPreservesOrder;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class

ecl/hqlcpp/hqlttcpp.cpp

@@ -2813,7 +2813,7 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
     //Tag a keyed join as ordered in the platforms that ensure it does remain ordered.  Extend if the others do.
     if (isKeyedJoin(expr))
     {
-        if (translator.targetRoxie() && !expr->hasAttribute(_ordered_Atom))
+        if ((translator.targetRoxie() || options.keyedJoinPreservesOrder) && !expr->hasAttribute(_ordered_Atom) && !expr->hasAttribute(unorderedAtom))
             return appendOwnedOperand(expr, createAttribute(_ordered_Atom));
         return NULL;
     }

ecl/hthor/hthorkey.cpp

@@ -28,7 +28,6 @@
 
 #define MAX_FETCH_LOOKAHEAD 1000
 #define IGNORE_FORMAT_CRC_MISMATCH_WHEN_NO_METADATA
-#define DEFAULT_KJ_PRESERVES_ORDER 1
 
 using roxiemem::IRowManager;
 using roxiemem::OwnedRoxieRow;
@@ -3398,7 +3397,7 @@ public:
     { 
         CHThorThreadedActivityBase::ready(); 
 
-        preserveOrder = ((helper.getJoinFlags() & JFkeepsorted) != 0) || agent.queryWorkUnit()->getDebugValueBool("__hthor_kj_always_preserve_order", DEFAULT_KJ_PRESERVES_ORDER);
+        preserveOrder = ((helper.getJoinFlags() & JFreorderable) == 0);
         preserveGroups = helper.queryOutputMeta()->isGrouped();
         needsDiskRead = helper.diskAccessRequired();
         extractJoinFields = ((helper.getJoinFlags() & JFextractjoinfields) != 0);

ecl/regress/issue8833.ecl

@@ -0,0 +1,15 @@
+#option('showMetaInGraph', true);
+layout := {unsigned u1};
+ds := dataset([{3},{2},{1}], layout);
+key := index(ds, {u1},{}, '~dskaggs::delete::key');
+
+dsSort := sort(ds, u1);
+dsJoin := join(dsSort, key,
+               keyed(left.u1 = right.u1),
+               transform(left), keyed);
+dsSort2 := sort(dsJoin, u1);
+
+sequential(
+    build(key, overwrite),
+    output(dsSort2,, '~dskaggs::delete::dsSort2', overwrite),
+);

ecl/regress/issue8833b.ecl

@@ -0,0 +1,15 @@
+#option('showMetaInGraph', true);
+layout := {unsigned u1};
+ds := dataset([{3},{2},{1}], layout);
+key := index(ds, {u1},{}, '~dskaggs::delete::key');
+
+dsSort := sort(ds, u1);
+dsJoin := join(dsSort, key,
+               keyed(left.u1 = right.u1),
+               transform(left), keyed, unordered);
+dsSort2 := sort(dsJoin, u1);
+
+sequential(
+    build(key, overwrite),
+    output(dsSort2,, '~dskaggs::delete::dsSort2', overwrite),
+);

rtl/include/eclhelper.hpp

@@ -1642,7 +1642,6 @@ enum {
     JFmanylookup                 = 0x00008000,
     JFparallel                   = 0x00010000,
     JFsequential                 = 0x00020000,
-    JFkeepsorted                 = 0x00040000,
     JFcountmatchabortlimit       = 0x00080000,
     JFreorderable                = 0x00100000,
     JFtransformmatchesleft       = 0x00200000,

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -69,7 +69,6 @@
 #define FETCHKEY_HEADER_SIZE (sizeof(offset_t)+sizeof(void *))
 #define DEFAULTMAXRESULTPULLPOOL 1
 #define DEFAULTFREEQSIZE 10
-#define DEFAULT_KJ_PRESERVES_ORDER 1
 #define LOWTHROTTLE_GRANULARITY 10
 
 class CJoinGroup;
@@ -2039,7 +2038,7 @@ public:
         startInput(input);
         inputHelper = LINK(input->queryFromActivity()->queryContainer().queryHelper());
         inputStopped = false;
-        preserveOrder = ((joinFlags & JFkeepsorted) != 0) || (container.queryJob().getWorkUnitValueInt("kj_always_preserve_order", DEFAULT_KJ_PRESERVES_ORDER) != 0);
+        preserveOrder = ((joinFlags & JFreorderable) == 0);
         preserveGroups = input->isGrouped();
         ActPrintLog("KJ: parallelLookups=%d, freeQSize=%d, preserveGroups=%s, preserveOrder=%s", parallelLookups, freeQSize, preserveGroups?"true":"false", preserveOrder?"true":"false");