HPCC-9742 Allow maxlength to be specified on workunit output

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlgram.y

@@ -3475,6 +3475,12 @@ outputWuFlag
                             $$.setPosition($1);
                         }
     | commonAttribute
+    | MAXSIZE '(' constExpression ')'
+                        {
+                            parser->normalizeExpression($3, type_int, false);
+                            $$.setExpr(createExprAttribute(maxSizeAtom, $3.getExpr()));
+                            $$.setPosition($1);
+                        }
     ;
 
 optCommaTrim

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10947,6 +10947,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx,
     IHqlExpression * dataset = expr->queryChild(0);
     IHqlExpression * seq = querySequence(expr);
     IHqlExpression * name = queryResultName(expr);
+    IHqlExpression * maxsize = expr->queryAttribute(maxSizeAtom);
     int sequence = (int)getIntValue(seq, ResultSequenceInternal);
 
     if (expr->hasAttribute(diskAtom))
@@ -10970,7 +10971,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx,
     Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
 
     StringBuffer graphLabel;
-    bool useImplementationClass = options.minimizeActivityClasses && (sequence == ResultSequenceInternal);
+    bool useImplementationClass = options.minimizeActivityClasses && (sequence == ResultSequenceInternal) && !maxsize;
     Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKworkunitwrite, expr, "WorkUnitWrite");
     if (useImplementationClass)
         instance->setImplementationClass(newWorkUnitWriteArgId);
@@ -10990,6 +10991,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx,
         flags.append("|POFextend");
     if (expr->hasAttribute(groupedAtom))
         flags.append("|POFgrouped");
+    if (maxsize)
+        flags.append("|POFmaxsize");
 
     if (!useImplementationClass)
     {
@@ -11000,6 +11003,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx,
             namectx.addQuotedCompound("virtual const char * queryName()");
             buildReturn(namectx, name, constUnknownVarStringType);
         }
+        if (maxsize)
+            doBuildUnsignedFunction(instance->createctx, "getMaxSize", maxsize->queryChild(0));
 
         IHqlExpression * outputRecord = instance->meta.queryRecord();
         Owned<IWUResult> result = createDatasetResultSchema(seq, name, outputRecord, true, false);

ecl/hthor/hthor.cpp

@@ -5920,10 +5920,13 @@ CHThorWorkUnitWriteActivity::CHThorWorkUnitWriteActivity(IAgentContext &_agent,
 
 void CHThorWorkUnitWriteActivity::execute()
 {
-    grouped = (POFgrouped & helper.getFlags()) != 0;
+    unsigned flags = helper.getFlags();
+    grouped = (POFgrouped & flags) != 0;
     size32_t outputLimit = agent.queryWorkUnit()->getDebugValueInt("outputLimit", defaultWorkUnitWriteLimit);
+    if (flags & POFmaxsize)
+        outputLimit = helper.getMaxSize();
     if (outputLimit>DALI_RESULT_OUTPUTMAX)
-        throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the default limit is %d MB. A huge dali result usually indicates the ECL needs altering.", DALI_RESULT_OUTPUTMAX, defaultWorkUnitWriteLimit);
+        throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", DALI_RESULT_OUTPUTMAX, defaultWorkUnitWriteLimit);
     assertex(outputLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
     outputLimit *= 0x100000;
     MemoryBuffer rowdata;

ecl/regress/issue9742.ecl

@@ -0,0 +1,21 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRecord := { unsigned id; };
+
+largeData := DATASET(1000000, TRANSFORM(idRecord, SELF.id := COUNTER));
+output(largeData, MAXSIZE(1));

rtl/include/eclhelper.hpp

@@ -1822,6 +1822,7 @@ enum
 {
     POFextend           = 0x0001,
     POFgrouped          = 0x0002,
+    POFmaxsize          = 0x0004,
 };
 
 struct IHThorWorkUnitWriteArg : public IHThorArg
@@ -1830,6 +1831,7 @@ struct IHThorWorkUnitWriteArg : public IHThorArg
     virtual void serializeXml(const byte * self, IXmlWriter & out) = 0;
     virtual const char * queryName() = 0;
     virtual unsigned getFlags() = 0;
+    virtual unsigned getMaxSize() = 0; // size in Mb
 };
 
 struct IHThorXmlWorkunitWriteArg : public IHThorWorkUnitWriteArg

rtl/include/eclhelper_base.hpp

@@ -1869,6 +1869,7 @@ class CThorWorkUnitWriteArg : public CThorArg, implements IHThorWorkUnitWriteArg
     virtual const char * queryName()                        { return NULL; }
     virtual unsigned getFlags()                             { return 0; }
     virtual void serializeXml(const byte * self, IXmlWriter & out) { rtlSysFail(1, "serializeXml not implemented"); }
+    virtual unsigned getMaxSize()                           { return 0; }
 };
 
 class CThorXmlWorkunitWriteArg : public CThorArg, implements IHThorXmlWorkunitWriteArg