Merge remote-tracking branch 'origin/candidate-7.10.x' into candidate-7.12.x

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

# Conflicts:
#	version.cmake

ecl/hthor/hthor.cpp

@@ -6300,6 +6300,19 @@ CHThorWorkUnitWriteActivity::CHThorWorkUnitWriteActivity(IAgentContext &_agent,
 {
 }
 
+static void throwWuResultTooLarge(size32_t outputLimit, IHThorWorkUnitWriteArg &helper)
+{
+    StringBuffer errMsg("Dataset too large to output to workunit (limit "); 
+    errMsg.append(outputLimit/0x100000).append(" megabytes), in result (");
+    const char *name = helper.queryName();
+    if (name)
+        errMsg.append("name=").append(name);
+    else
+        errMsg.append("sequence=").append(helper.getSequence());
+    errMsg.append(")");
+    throw MakeStringExceptionDirect(0, errMsg.str());
+}
+
 void CHThorWorkUnitWriteActivity::execute()
 {
     unsigned flags = helper.getFlags();
@@ -6357,18 +6370,8 @@ void CHThorWorkUnitWriteActivity::execute()
                 break;
         }
         size32_t thisSize = inputMeta->getRecordSize(nextrec);
-        if(outputLimit && ((rowdata.length() + thisSize) > outputLimit))
-        {
-            StringBuffer errMsg("Dataset too large to output to workunit (limit "); 
-            errMsg.append(outputLimit/0x100000).append(" megabytes), in result (");
-            const char *name = helper.queryName();
-            if (name)
-                errMsg.append("name=").append(name);
-            else
-                errMsg.append("sequence=").append(helper.getSequence());
-            errMsg.append(")");
-            throw MakeStringExceptionDirect(0, errMsg.str());
-         }
+        if (outputLimit && ((rowdata.length() + thisSize) > outputLimit))
+            throwWuResultTooLarge(outputLimit, helper);
         if (rowSerializer)
         {
             CThorDemoRowSerializer serializerTarget(rowdata);
@@ -6399,7 +6402,12 @@ void CHThorWorkUnitWriteActivity::execute()
     WorkunitUpdate w = agent.updateWorkUnit();
     Owned<IWUResult> result = updateWorkUnitResult(w, helper.queryName(), helper.getSequence());
     if (0 != (POFextend & helper.getFlags()))
+    {
+        __int64 existingSz = result->getResultRawSize(nullptr, nullptr);
+        if (outputLimit && ((rowdata.length() + existingSz) > outputLimit))
+            throwWuResultTooLarge(outputLimit, helper);
         result->addResultRaw(rowdata.length(), rowdata.toByteArray(), ResultFormatRaw);
+    }
     else
         result->setResultRaw(rowdata.length(), rowdata.toByteArray(), ResultFormatRaw);
     result->setResultStatus(ResultStatusCalculated);

esp/src/eclwatch/ActivityWidget.js

@@ -285,6 +285,7 @@ define([
                 }
             });
             this.createStackControllerTooltip(this.id + "AutoRefresh", this.i18n.AutoRefresh + ": " + this.autoRefreshButton.get("checked"));
+            this.resize();
         },
 
         createGrid: function (domID) {

esp/src/eclwatch/DFUQueryWidget.js

@@ -211,6 +211,9 @@ define([
             if (confirm(this.i18n.DeleteSelectedFiles + "\n" + list)) {
                 var context = this;
                 WsDfu.DFUArrayAction(selection, "Delete").then(function (response) {
+                    selection.forEach(item => {
+                        context.listStore.remove(ESPLogicalFile.createID(item.Cluster, item.Name));
+                    });
                     context.refreshGrid(true);
                 });
             }

esp/src/src/ESPLogicalFile.ts

@@ -14,7 +14,7 @@ import * as WsDfu from "./WsDfu";
 
 const _logicalFiles = {};
 
-const createID = function (Cluster, Name) {
+export const createID = function (Cluster, Name) {
     return (Cluster ? Cluster : "") + "--" + Name;
 };
 
@@ -69,6 +69,10 @@ const Store = declare([ESPRequest.Store], {
             });
         }
     },
+    remove(id) {
+        this.inherited(arguments);
+        delete _logicalFiles[id];
+    },
     preProcessRow(item, request, query, options) {
         lang.mixin(item, {
             __hpcc_id: createID(item.NodeGroup, item.Name),

esp/src/src/WsTopology.ts

@@ -66,8 +66,8 @@ const TpLogFileStore = declare([Memory, Evented], {
                                 }
 
                                 tempObj[cleanName] = value;
-                                data.push(tempObj);
                             }
+                            data.push(tempObj);
                         }
                     }, this);
                 }

esp/src/src/react/wuStatus.tsx

@@ -99,7 +99,7 @@ export const WUStatus: React.FunctionComponent<WUStatus> = ({
                         <StepLabel {...labelProps}>{label}</StepLabel>
                     </Step>;
                 })}
-            </Stepper>;
+            </Stepper>
         </MuiThemeProvider>
     );
 };

thorlcr/activities/wuidwrite/thwuidwrite.cpp

@@ -38,12 +38,28 @@ protected:
     StringAttr resultName;
     unsigned resultSeq;
 
+    void throwWuResultTooLarge()
+    {
+        StringBuffer errMsg("Dataset too large to output to workunit (limit is set to ");
+        errMsg.append(workunitWriteLimit/0x100000).append(") megabytes, in result (");
+        if (resultName.length())
+            errMsg.append("name=").append(resultName);
+        else
+            errMsg.append("sequence=").append(resultSeq);
+        errMsg.append(")");
+        throw MakeThorException(TE_WorkUnitWriteLimitExceeded, "%s", errMsg.str());
+    }
     void addResult(rowcount_t resultCount, MemoryBuffer &resultData, bool complete)
     {
         Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
         Owned<IWUResult> result = updateWorkUnitResult(wu, resultName, resultSeq);
         if (appendOutput)
+        {
+            __int64 existingSz = result->getResultRawSize(nullptr, nullptr);
+            if (workunitWriteLimit && (existingSz+resultData.length() > workunitWriteLimit))
+                throwWuResultTooLarge();
             result->addResultRaw(resultData.length(), resultData.toByteArray(), ResultFormatRaw);
+        }
         else
             result->setResultRaw(resultData.length(), resultData.toByteArray(), ResultFormatRaw);
         result->setResultRowCount(resultCount);
@@ -121,17 +137,8 @@ public:
                 unsigned numGot;
                 mb.read(numGot);
                 unsigned l=mb.remaining();
-                if (workunitWriteLimit && totalSize+resultData.length()+l > workunitWriteLimit)
-                {
-                    StringBuffer errMsg("Dataset too large to output to workunit (limit is set to ");
-                    errMsg.append(workunitWriteLimit/0x100000).append(") megabytes, in result (");
-                    if (resultName.length())
-                        errMsg.append("name=").append(resultName);
-                    else
-                        errMsg.append("sequence=").append(resultSeq);
-                    errMsg.append(")");
-                    throw MakeThorException(TE_WorkUnitWriteLimitExceeded, "%s", errMsg.str());
-                }
+                if (workunitWriteLimit && (totalSize+resultData.length()+l > workunitWriteLimit))
+                    throwWuResultTooLarge();
                 resultData.append(l, mb.readDirect(l));
                 mb.clear();
                 numResults += numGot;