Merge pull request #4102 from jakesmith/hpcc-8969

HPCC-8969 - Enforce an absolute hard limit on dali results

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorcommon.hpp

@@ -25,6 +25,7 @@
 #include "thorhelper.hpp"
 #include "thorxmlwrite.hpp"
 
+#define DALI_RESULT_OUTPUTMAX 2000 // MB
 class THORHELPER_API CSizingSerializer : implements IRowSerializerTarget
 {
     size32_t totalsize;

ecl/hthor/hthor.cpp

@@ -5772,7 +5772,11 @@ CHThorWorkUnitWriteActivity::CHThorWorkUnitWriteActivity(IAgentContext &_agent,
 void CHThorWorkUnitWriteActivity::execute()
 {
     grouped = (POFgrouped & helper.getFlags()) != 0;
-    size32_t outputLimit = agent.queryWorkUnit()->getDebugValueInt("outputLimit", defaultWorkUnitWriteLimit) * 0x100000;
+    size32_t outputLimit = agent.queryWorkUnit()->getDebugValueInt("outputLimit", defaultWorkUnitWriteLimit);
+    if (outputLimit>DALI_RESULT_OUTPUTMAX)
+        throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the default limit is %d MB. A huge dali result usually indicates the ECL needs altering.", DALI_RESULT_OUTPUTMAX, defaultWorkUnitWriteLimit);
+    assertex(outputLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
+    outputLimit *= 0x100000;
     MemoryBuffer rowdata;
     __int64 rows = 0;
     IRecordSize * inputMeta = input->queryOutputMeta();

thorlcr/activities/wuidwrite/thwuidwrite.cpp

@@ -50,7 +50,12 @@ public:
     }
     void init()
     {
-        workunitWriteLimit = (unsigned)container.queryJob().getWorkUnitValueInt("outputLimit", DEFAULT_WUIDWRITE_LIMIT) * 0x100000;
+        workunitWriteLimit = getOptInt(THOROPT_OUTPUTLIMIT, DEFAULT_WUIDWRITE_LIMIT);
+        if (workunitWriteLimit>DALI_RESULT_OUTPUTMAX)
+            throw MakeActivityException(this, 0, "Dali result outputs are restricted to a maximum of %d MB, the default limit is %d MB. A huge dali result usually indicates the ECL needs altering.", DALI_RESULT_OUTPUTMAX, DEFAULT_WUIDWRITE_LIMIT);
+        assertex(workunitWriteLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
+        workunitWriteLimit *= 0x100000;
+
         if (appendOutput)
         {
             Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();

thorlcr/graph/thgraph.cpp

@@ -2821,7 +2821,7 @@ int CActivityBase::getOptInt(const char *prop, int defVal) const
 {
     int def = queryJob().getOptInt(prop, defVal);
     VStringBuffer path("hint[@name=\"%s\"]/@value", prop);
-    return container.queryXGMML().getPropInt(path.str(), def);
+    return container.queryXGMML().getPropInt(path.toLowerCase().str(), def);
 }
 
 __int64 CActivityBase::getOptInt64(const char *prop, __int64 defVal) const

thorlcr/thorutil/thormisc.hpp

@@ -55,6 +55,7 @@
 #define THOROPT_PARALLEL_FUNNEL       "parallelFunnel"
 #define THOROPT_SORT_MAX_DEVIANCE     "sort_max_deviance"
 #define THOROPT_OUTPUT_FLUSH_THRESHOLD "output_flush_threshold"
+#define THOROPT_OUTPUTLIMIT           "outputLimit"
 
 #define INITIAL_SELFJOIN_MATCH_WARNING_LEVEL 20000  // max of row matches before selfjoin emits warning