Merge pull request #12265 from richardkchapman/hpcc-21436

HPCC-21436 pickBestEngine should not go to thor just because of a filter

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/eclcc/eclcc.hpp

@@ -137,7 +137,8 @@ const char * const helpText[] = {
     "?!  -fmaxCompileThreads     Number of compiler instances to compile the c++",
     "?!  -fmaxErrors             Maximum number of errors to report",
     "?!  -fnoteRecordSizeInGraph Add estimates of record sizes to the graph",
-    "?!  -fpickBestEngine        Allow simple thor queries to be passed to thor",
+    "?!  -fpickBestEngine        Allow simple thor queries to be run without using thor",
+    "?!  -fdiskReadsAreSimple    Count disk reads as simple when using pickBestEngine",
     "?!  -fobfuscateOutput       Remove details of the original ECL from output",
     "?!  -freportCppWarnings     Report warnings from c++ compilation",
     "?!  -fsaveCpp -fsaveCppTempFiles  Retain the generated c++ files",

ecl/hqlcpp/hqlcpp.cpp

@@ -1855,6 +1855,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.newIndexReadMapping, "newIndexReadMapping", false), // Not yet enabled due to problems with merging mapped fields and roxie/thor integration
         DebugOption(options.checkDuplicateThreshold, "checkDuplicateThreshold", 0), // If non zero, create a warning if duplicates > this percentage increase
         DebugOption(options.checkDuplicateMinActivities, "checkDuplicateMinActivities", 100),
+        DebugOption(options.diskReadsAreSimple, "diskReadsAreSimple", false), // Not yet enabled - needs filters to default to generating keyed info first
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.hpp

@@ -45,6 +45,7 @@ The following debug options are currently supported by the code generator:
 "orderDiskFunnel"        true   - if all inputs to a funnel are disk reads, pull in order.
 "parseDfaComplexity"     2000   - maximum complexity of expression to convert to a DFA.
 "pickBestEngine"         true   - use hthor if it is more efficient than thor
+"diskReadsAreSimple"     false  - treat disk reads as simple when evaluating pickBestEngine
 "sortIndexPayload"       true   - do we sort by the non-keyed elements in an index.
 "targetClusterType"      ----   - hthor|thor|roxie - who are we generating code for?
 "topnLimit"              10000  - maximum number of records to do topN on.

ecl/hqlcpp/hqlcpp.ipp

@@ -821,6 +821,7 @@ struct HqlCppOptions
     bool                transformNestedSequential;
     bool                forceAllProjectedDiskSerialized;
     bool                newIndexReadMapping;
+    bool                diskReadsAreSimple;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class
@@ -1953,6 +1954,8 @@ protected:
     void convertLogicalToActivities(WorkflowItem & curWorkflow);
     void flattenDatasets(WorkflowArray & array);
 
+    bool needsRealThor(IHqlExpression *expr, unsigned flags);
+
     void spotGlobalCSE(WorkflowItem & curWorkflow);
     IHqlExpression * spotGlobalCSE(IHqlExpression * _expr);
     void spotGlobalCSE(HqlExprArray & exprs);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -19448,16 +19448,19 @@ void HqlCppTranslator::flattenDatasets(WorkflowArray & array)
 // 2. Filtered index count.
 // 3. Restricted set of records from an index/table.
 
-enum { NRTfiltered = 0x0001, NRTcount = 0x0002, NRTlimited = 0x0004 };
-static bool needsRealThor(IHqlExpression *expr, unsigned flags)
+bool HqlCppTranslator::needsRealThor(IHqlExpression *expr, unsigned flags)
 {
+    enum { NRTfiltered = 0x0001, NRTcount = 0x0002, NRTlimited = 0x0004 };
     unsigned numChildrenToCheck = (unsigned)-1;
     switch (expr->getOperator())
     {
     case no_table:
-        //only allow non filtered limited outputs, and non filtered counts
-        return !((flags == NRTlimited) || (flags == NRTcount));
-
+        if (!options.diskReadsAreSimple)
+        {
+            //only allow non filtered limited outputs, and non filtered counts
+            return !((flags == NRTlimited) || (flags == NRTcount));
+        }
+        //fallthrough...
     case no_newkeyindex:
     case no_keyindex:
     case no_compound_indexread:
@@ -19623,7 +19626,7 @@ static bool needsRealThor(IHqlExpression *expr, unsigned flags)
             }
             if (!containsAnyDataset(child0))
                 return false;
-//          return needsRealThor(child0, isFiltered);
+//          return needsRealThor(child0, isFiltered, diskReadsOnHThor);
             //fallthrough...
         }
 
@@ -19648,12 +19651,6 @@ static bool needsRealThor(IHqlExpression *expr, unsigned flags)
     return false;
 }
 
-bool needsRealThor(IHqlExpression *expr)
-{
-    return needsRealThor(expr, 0);
-}
-
-
 IHqlExpression * HqlCppTranslator::getDefaultOutputAttr(IHqlExpression * expr)
 {
     return createAttribute(workunitAtom);
@@ -19699,11 +19696,11 @@ void HqlCppTranslator::pickBestEngine(HqlExprArray & exprs)
     {
         ForEachItemIn(idx, exprs)
         {
-            if (needsRealThor(&exprs.item(idx)))
+            if (needsRealThor(&exprs.item(idx), 0))
                 return;
         }
         // if we got this far, thor not required
-        setTargetClusterType(HThorCluster);
+        setTargetClusterType(HThorCluster);  // MORE - what about Roxie?
         DBGLOG("Thor query redirected to hthor instead");
         Owned<IWUException> we = wu()->createException();
         we->setSeverity(SeverityInformation);
@@ -19721,7 +19718,7 @@ void HqlCppTranslator::pickBestEngine(WorkflowArray & workflow)
             HqlExprArray & exprs = workflow.item(idx2).queryExprs();
             ForEachItemIn(idx, exprs)
             {
-                if (needsRealThor(&exprs.item(idx)))
+                if (needsRealThor(&exprs.item(idx), 0))
                     return;
             }
             // if we got this far, thor not required