HPCC-25133 Decrease hard workunit result limit to 100MB

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

common/thorhelper/thorcommon.hpp

@@ -29,7 +29,6 @@
 #include "thorhelper.hpp"
 #include "thorxmlwrite.hpp"
 
-static unsigned const defaultDaliResultOutputMax = 2000; // MB
 static unsigned const defaultDaliResultLimit = 10; // MB
 static unsigned const defaultMaxCsvRowSize = 10; // MB
 

ecl/hqlcpp/hqlcerrors.hpp

@@ -225,6 +225,7 @@
 #define HQLERR_CannotInterpretRecord            4213
 #define HQLERR_NoVirtualAndAlien                4214
 #define HQLERR_CannotFilterLimitInsideActivity  4215
+#define HQLERR_OutputLimitMaxExceeded           4216
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -533,6 +534,7 @@
 #define HQLERR_CannotInterpretRecord_Text       "This dataset contains deprecated record formats and virtual fields.  Remove the alien data types, or temporarily add __OPTION__(LEGACY) to the table definition"
 #define HQLERR_NoVirtualAndAlien_Text           "GenericDiskReads: VIRTUAL fields are not currently supported with ALIEN types"
 #define HQLERR_CannotFilterLimitInsideActivity_Text "Cannot filter a non keyed limit within a compound activity"
+#define HQLERR_OutputLimitMaxExceeded_Text      "Dali result outputs are restricted to an absolute maximum of %u MB (%u MB specified by option). A huge dali result usually indicates the ECL needs altering."
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlcpp.cpp

@@ -1539,6 +1539,9 @@ void HqlCppTranslator::checkAbort()
 // Without this restriction it becomes much easier.
 void HqlCppTranslator::cacheOptions()
 {
+    size32_t outputLimit = wu()->getDebugValueInt("outputLimitMb", 0);
+    if (outputLimit > daliResultOutputMax)
+        throwError2(HQLERR_OutputLimitMaxExceeded, daliResultOutputMax, outputLimit);
     SCMStringBuffer targetText;
     wu()->getDebugValue("targetClusterType", targetText);
     ClusterType clusterType = getClusterType(targetText.s.str());

ecl/hthor/hthor.cpp

@@ -6308,8 +6308,8 @@ void CHThorWorkUnitWriteActivity::execute()
     size32_t outputLimit = agent.queryWorkUnit()->getDebugValueInt(OPT_OUTPUTLIMIT, agent.queryWorkUnit()->getDebugValueInt(OPT_OUTPUTLIMIT_LEGACY, defaultDaliResultLimit));
     if (flags & POFmaxsize)
         outputLimit = helper.getMaxSize();
-    if (outputLimit>defaultDaliResultOutputMax)
-        throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", defaultDaliResultOutputMax, defaultDaliResultLimit);
+    if (outputLimit>daliResultOutputMax)
+        throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", daliResultOutputMax, defaultDaliResultLimit);
     assertex(outputLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
     outputLimit *= 0x100000;
     MemoryBuffer rowdata;

roxie/ccd/ccdserver.cpp

@@ -21575,8 +21575,8 @@ public:
                 // In absense of OPT_OUTPUTLIMIT check pre 5.2 legacy name OPT_OUTPUTLIMIT_LEGACY
                 outputLimit = workunit->getDebugValueInt(OPT_OUTPUTLIMIT, workunit->getDebugValueInt(OPT_OUTPUTLIMIT_LEGACY, defaultDaliResultLimit));
             }
-            if (outputLimit>defaultDaliResultOutputMax)
-                throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", defaultDaliResultOutputMax, defaultDaliResultLimit);
+            if (outputLimit>daliResultOutputMax)
+                throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", daliResultOutputMax, defaultDaliResultLimit);
             assertex(outputLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
             outputLimitBytes = outputLimit * 0x100000;
         }

rtl/include/eclhelper.hpp

@@ -2966,6 +2966,8 @@ protected:
     Owned<IColumnProvider> cur;
 };
 
+constexpr unsigned daliResultOutputMax = 100; // MB
+
 #ifdef STARTQUERY_EXPORTS
 #define STARTQUERY_API DECL_EXPORT
 #else

thorlcr/activities/wuidwrite/thwuidwrite.cpp

@@ -80,8 +80,8 @@ public:
         CMasterActivity::init();
         // In absense of OPT_OUTPUTLIMIT check pre 5.2 legacy name OPT_OUTPUTLIMIT_LEGACY
         workunitWriteLimit = activityMaxSize ? activityMaxSize : getOptInt(OPT_OUTPUTLIMIT, getOptInt(OPT_OUTPUTLIMIT_LEGACY, defaultDaliResultLimit));
-        if (workunitWriteLimit>defaultDaliResultOutputMax)
-            throw MakeActivityException(this, 0, "Configured max result size, %d MB, exceeds absolute max limit of %d MB. A huge Dali result usually indicates the ECL needs altering.", workunitWriteLimit, defaultDaliResultOutputMax);
+        if (workunitWriteLimit>daliResultOutputMax)
+            throw MakeActivityException(this, 0, "Configured max result size, %d MB, exceeds absolute max limit of %d MB. A huge Dali result usually indicates the ECL needs altering.", workunitWriteLimit, daliResultOutputMax);
         assertex(workunitWriteLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
         workunitWriteLimit *= 0x100000;
     }