Merge remote-tracking branch 'origin/closedown-4.2.x'

Conflicts:
	version.cmake

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/CMakeLists.txt

@@ -16,7 +16,11 @@
 HPCC_ADD_SUBDIRECTORY (deftype)
 HPCC_ADD_SUBDIRECTORY (dllserver)
 HPCC_ADD_SUBDIRECTORY (environment)
-HPCC_ADD_SUBDIRECTORY (fileview2 "PLATFORM")
+if ("${BUILD_LEVEL}" STREQUAL "INTERNAL")
+    HPCC_ADD_SUBDIRECTORY (fileview2)
+else ()
+    HPCC_ADD_SUBDIRECTORY (fileview2 "PLATFORM")
+endif ()
 HPCC_ADD_SUBDIRECTORY (monitoring "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (remote)
 HPCC_ADD_SUBDIRECTORY (roxiecommlib)

common/thorhelper/thorcommon.hpp

@@ -292,9 +292,9 @@ public:
     {
         return ctx->getExpandLogicalName(logicalName);
     }
-    virtual void addWuException(const char * text, unsigned code, unsigned severity)
+    virtual void addWuException(const char * text, unsigned code, unsigned severity, const char *source)
     {
-        ctx->addWuException(text, code, severity);
+        ctx->addWuException(text, code, severity, source);
     }
     virtual void addWuAssertFailure(unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool isAbort)
     {
@@ -424,6 +424,10 @@ public:
     {
         return ctx->getDaliServers();
     }
+    virtual IWorkUnit *updateWorkUnit() const
+    {
+        return ctx->updateWorkUnit();
+    }
 protected:
     ICodeContext * ctx;
 };

dali/base/dadfs.cpp

@@ -1445,6 +1445,7 @@ public:
     }
     void retryActions()
     {
+        clearFiles(); // clear all previously tracked pending file changes, e.g. renames, super file additions/removals
         while (prepared) // unlock for retry
             actions.item(--prepared).retry();
     }
@@ -5721,7 +5722,8 @@ private:
         }
         else
             pos = before?0:subfiles.ordinality();
-        unsigned cmppos = (pos==0)?1:0;
+        if (pos > subfiles.ordinality())
+            throw MakeStringException(-1,"addSubFile: Insert position %d out of range for file %s in superfile %s", pos+1, sub->queryLogicalName(), queryLogicalName());
         addItem(pos,sub.getClear());     // remove if failure TBD?
         setModified();
         updateFileAttrs();

dali/base/dafdesc.cpp

@@ -383,6 +383,10 @@ struct CClusterInfo: public CInterface, implements IClusterInfo
                     {
                         mspec.setDefaultBaseDir(defaultDir);   // MORE - should possibly set up the rest of the mspec info from the group info here
                     }
+                    if (mspec.defaultCopies>1 && mspec.defaultReplicateDir.isEmpty())
+                    {
+                        mspec.setDefaultReplicateDir(queryBaseDirectory(groupType, 1));
+                    }
                     return; // ok
                 }
                 name.clear();

docs/RunningHPCCinaVirtualMachine/RunningHPCCinaVirtualMachine.xml

@@ -137,7 +137,7 @@
 
         <listitem>
           <para>A virtualization software package:
-          VMware<superscript>®</superscript> Player or Server (version 4.0 or
+          VMware<superscript>®</superscript> Player or Server (version 5.0 or
           later) or Oracle VM VirtualBox (version 4.0 or later).</para>
         </listitem>
 

ecl/ecl-bundle/ecl-bundle.cpp

@@ -445,7 +445,7 @@ public:
         VStringBuffer exeFileName(".%c_%s-bundle-selftest", PATHSEPCHAR, cleanName.str());
         VStringBuffer eclOpts("-   --nologfile -o%s", exeFileName.str());
         VStringBuffer bundleCmd("IMPORT %s as B;\n"
-                                "#IF (#ISDEFINED(B.__selftesdft))\n"
+                                "#IF (#ISDEFINED(B.__selftest))\n"
                                 "  EVALUATE(B.__selftest);\n"
                                 "#ELSE\n"
                                 "  FAIL(253, 'No selftests exported');\n"
@@ -463,6 +463,8 @@ public:
         {
             if (retcode != 253)
                 printf("%s selftests returned non-zero\n", cleanName.str());
+            else
+                printf("%s has no selftests\n", cleanName.str());
             return false;
         }
         else

ecl/eclagent/agentctx.hpp

@@ -75,7 +75,7 @@ struct IAgentContext : extends IGlobalCodeContext
     virtual ICodeContext *queryCodeContext() = 0;
 
     virtual IConstWorkUnit *queryWorkUnit() = 0;
-    virtual IWorkUnit *updateWorkUnit() = 0;
+    virtual IWorkUnit *updateWorkUnit() const = 0;
     virtual void unlockWorkUnit() = 0;
     
     virtual ILocalOrDistributedFile *resolveLFN(const char *logicalName, const char *errorTxt=NULL, bool optional=false, bool noteRead=true, bool write=false, StringBuffer * expandedlfn=NULL) = 0;

ecl/eclagent/eclagent.cpp

@@ -705,7 +705,7 @@ const char *EclAgent::loadResource(unsigned id)
     return reinterpret_cast<const char *>(dll->getResource(id));  // stays loaded as long as dll stays loaded
 }
 
-IWorkUnit *EclAgent::updateWorkUnit()
+IWorkUnit *EclAgent::updateWorkUnit() const
 {
     CriticalBlock block(wusect);
     if (!wuWrite)
@@ -1482,11 +1482,6 @@ char * EclAgent::getExpandLogicalName(const char * logicalName)
     return lfn.detach();
 }
 
-void EclAgent::addWuException(const char * text, unsigned code, unsigned severity)
-{
-    addException((WUExceptionSeverity)severity, "user", code, text, NULL, 0, 0, false, false);
-}
-
 void EclAgent::addWuException(const char * text, unsigned code, unsigned severity, char const * source)
 {
     addException((WUExceptionSeverity)severity, source, code, text, NULL, 0, 0, false, false);

ecl/eclagent/eclagent.ipp

@@ -178,7 +178,7 @@ public:
     {
         return ctx->queryWorkUnit();
     }
-    virtual IWorkUnit *updateWorkUnit()
+    virtual IWorkUnit *updateWorkUnit() const
     {
         return ctx->updateWorkUnit();
     }
@@ -351,7 +351,7 @@ private:
     friend class EclAgentWorkflowMachine;
 
     Owned<EclAgentWorkflowMachine> workflow;
-    Owned<IWorkUnit> wuWrite;
+    mutable Owned<IWorkUnit> wuWrite;
     Owned<IConstWorkUnit> wuRead;
     Owned<roxiemem::IRowManager> rowManager;
     StringAttr wuid;
@@ -370,7 +370,7 @@ private:
     Owned<IUserDescriptor> standAloneUDesc;
     outputFmts outputFmt;
     unsigned __int64 stopAfter;
-    CriticalSection wusect;
+    mutable CriticalSection wusect;
     StringArray tempFiles;
     CriticalSection tfsect;
     Array persistReadLocks;
@@ -501,7 +501,6 @@ public:
 
     virtual bool fileExists(const char * filename);
     virtual char * getExpandLogicalName(const char * logicalName);
-    virtual void addWuException(const char * text, unsigned code, unsigned severity);
     virtual void addWuException(const char * text, unsigned code, unsigned severity, char const * source);
     virtual void addWuAssertFailure(unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool isAbort);
     virtual IUserDescriptor *queryUserDescriptor();
@@ -576,7 +575,7 @@ public:
     virtual bool isResult(const char * name, unsigned sequence);
     virtual unsigned getWorkflowId();// { return workflow->queryCurrentWfid(); }
     virtual IConstWorkUnit *queryWorkUnit();  // no link
-    virtual IWorkUnit *updateWorkUnit();        // links
+    virtual IWorkUnit *updateWorkUnit() const; // links
     virtual void unlockWorkUnit();      
     virtual void reloadWorkUnit();
     void addTimings();

ecl/hql/hqlexpr.cpp

@@ -11249,8 +11249,9 @@ static void normalizeCallParameters(HqlExprArray & resolvedActuals, IHqlExpressi
                 else
                 {
                     OwnedHqlExpr actualRecord = getUnadornedRecordOrField(actual->queryRecord());
-                    OwnedHqlExpr formalRecord = getUnadornedRecordOrField(::queryOriginalRecord(type));
-                    if (actualRecord && formalRecord && formalRecord->numChildren() && (formalRecord->queryBody() != actualRecord->queryBody()))
+                    IHqlExpression * formalRecord = ::queryOriginalRecord(type);
+                    OwnedHqlExpr normalFormalRecord = getUnadornedRecordOrField(formalRecord);
+                    if (actualRecord && normalFormalRecord && normalFormalRecord->numChildren() && (normalFormalRecord->queryBody() != actualRecord->queryBody()))
                     {
                         //If the actual dataset is derived from the input dataset, then insert a project so types remain correct
                         //otherwise x+y will change meaning.
@@ -14610,6 +14611,29 @@ bool isPureActivityIgnoringSkip(IHqlExpression * expr)
     return true;
 }
 
+bool assignsContainSkip(IHqlExpression * expr)
+{
+    switch (expr->getOperator())
+    {
+    case no_newtransform:
+    case no_transform:
+    case no_assignall:
+        {
+            ForEachChild(i, expr)
+            {
+                if (assignsContainSkip(expr->queryChild(i)))
+                    return true;
+            }
+            return false;
+        }
+    case no_assign:
+        return containsSkip(expr->queryChild(1));
+    case no_alias_scope:
+        return assignsContainSkip(expr->queryChild(0));
+    default:
+        return false;
+    }
+}
 
 extern HQL_API bool isKnownTransform(IHqlExpression * transform)
 {

ecl/hql/hqlexpr.hpp

@@ -1392,6 +1392,8 @@ extern HQL_API void queryRemoveRows(HqlExprCopyArray & tables, IHqlExpression *
 
 extern HQL_API bool isPureActivity(IHqlExpression * expr);
 extern HQL_API bool isPureActivityIgnoringSkip(IHqlExpression * expr);
+extern HQL_API bool assignsContainSkip(IHqlExpression * expr);
+
 extern HQL_API bool isPureInlineDataset(IHqlExpression * expr);
 extern HQL_API bool transformHasSkipAttr(IHqlExpression * transform);
 extern HQL_API IHqlExpression * queryNewColumnProvider(IHqlExpression * expr);          // what is the transform/newtransform/record?

ecl/hql/hqlopt.cpp

@@ -3053,6 +3053,10 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                     if (!isPureActivityIgnoringSkip(child) || hasUnknownTransform(child))
                         break;
 
+                    IHqlExpression * childTransform = queryNewColumnProvider(child);
+                    if (assignsContainSkip(childTransform))
+                        break;
+
                     IHqlExpression * childCountProject = child->queryAttribute(_countProject_Atom);
                     //Don't merge two count projects - unless we go through and replace counter instances.
                     if (transformedCountProject && childCountProject)

ecl/hqlcpp/hqlcatom.cpp

@@ -106,7 +106,6 @@ IIdAtom * addAggregateRowId;
 IIdAtom * addAllId;
 IIdAtom * addRangeId;
 IIdAtom * addWorkunitAssertFailureId;
-IIdAtom * addWorkunitExceptionId;
 IIdAtom * an2bId;
 IIdAtom * an2fId;
 IIdAtom * an2l4Id;
@@ -743,7 +742,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEID(addAll);
     MAKEID(addRange);
     MAKEID(addWorkunitAssertFailure);
-    MAKEID(addWorkunitException);
     MAKEID(an2b);
     MAKEID(an2f);
     MAKEID(an2l4);

ecl/hqlcpp/hqlcatom.hpp

@@ -106,7 +106,6 @@ extern IIdAtom * addAggregateRowId;
 extern IIdAtom * addAllId;
 extern IIdAtom * addRangeId;
 extern IIdAtom * addWorkunitAssertFailureId;
-extern IIdAtom * addWorkunitExceptionId;
 extern IIdAtom * an2bId;
 extern IIdAtom * an2fId;
 extern IIdAtom * an2l4Id;

ecl/hqlcpp/hqlcppsys.ecl

@@ -686,7 +686,6 @@ const char * cppSystemText[]  = {
     "   doNotifyTarget(const varstring name, const varstring text, const varstring _target) : gctxmethod,entrypoint='doNotify';",
     "   setWorkflowCondition(boolean value) : gctxmethod,entrypoint='setWorkflowCondition';",
     "   returnPersistVersion(const varstring name, unsigned4 eclCRC, unsigned8 allCRC, boolean isFile) : gctxmethod,entrypoint='returnPersistVersion';",
-    "   addWorkunitException(const varstring txt, unsigned code, unsigned severity) : ctxmethod,entrypoint='addWuException'; ",
     "   addWorkunitAssertFailure(unsigned4 errNo, const varstring _msg, const varstring _filename, unsigned4 _lineno, unsigned4 _column, boolean _isAbort) : ctxmethod,entrypoint='addWuAssertFailure'; ",
 
     //

ecl/regress/issue10489.ecl

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+LayoutScoredFetch := RECORD
+  UNSIGNED8 F1;
+END;
+
+ OutputLayout_Base := RECORD
+  BOOLEAN Resolved := FALSE;
+  DATASET(LayoutScoredFetch) Results;
+END;
+
+OutputLayout_Batch := RECORD(OutputLayout_Base)
+  UNSIGNED8 Reference;
+END;
+
+ds := DATASET('x',OutputLayout_Batch,thor);
+
+ScoreSummary(DATASET(OutputLayout_Base) ds0) :=
+        PROJECT(ds0(EXISTS(Results)),TRANSFORM(LayoutScoredFetch,SELF := LEFT.Results[1]));
+
+Stats := ScoreSummary(ds);
+Stats;

esp/eclwatch/ws_XSLT/index.xslt

@@ -830,13 +830,13 @@
                             <col width="250" class="cluster"/>
                         </colgroup>
                         <colgroup>
-                            <col width="200" class="cluster"/>
+                            <col width="300" class="cluster"/>
                         </colgroup>
                         <colgroup>
                             <col width="150" class="cluster"/>
                         </colgroup>
                         <colgroup>
-                            <col width="500" class="cluster"/>
+                            <col width="400" class="cluster"/>
                         </colgroup>
                         <xsl:if test="(position()=1 and $showTitle='1')">
                             <tr>

esp/files/scripts/QuerySetLogicalFilesWidget.js

@@ -27,11 +27,12 @@ define([
     "dgrid/extensions/DijitRegistry",
 
     "hpcc/GridDetailsWidget",
+    "hpcc/LFDetailsWidget",
     "hpcc/WsWorkunits",
     "hpcc/ESPUtil"
 ], function (declare, arrayUtil, lang, on,
                 OnDemandGrid, Keyboard, Selection, selector, ColumnResizer, DijitRegistry,
-                GridDetailsWidget, WsWorkunits, ESPUtil) {
+                GridDetailsWidget, LFDetailsWidget, WsWorkunits, ESPUtil) {
     return declare("QuerySetLogicalFilesWidget", [GridDetailsWidget], {
 
         gridTitle: "Logical Files",
@@ -58,7 +59,7 @@ define([
                 store: this.store,
                 columns: {
                     col1: selector({ width: 27, selectorType: 'checkbox' }),
-                    Name: { label: "Logical Files", width: 108, sortable: false },
+                    Name: {label: "Logical Files", width: 180, sortable: false}
                 }
             }, domID);
 
@@ -71,6 +72,33 @@ define([
             return retVal;
         },
 
+        createDetail: function (id, row, params) {
+            return new LFDetailsWidget.fixCircularDependency({
+                id: id,
+                title: params.Name,
+                closable: true,
+                hpcc: {
+                    params: {
+                        Name: params.Name
+                    }
+                }
+            });
+        },
+
+        _onOpen: function(){
+            var selections = this.grid.getSelected();
+            var firstTab = null;
+            for (var i = selections.length - 1; i >= 0; --i) {
+                var tab = this.ensurePane(this.id + "_" + selections[i].Id, selections[i]);
+                if (i == 0) {
+                    firstTab = tab;
+                }
+            }
+            if (firstTab) {
+                this.selectChild(firstTab);
+            }
+        },
+
         refreshGrid: function (args) {
             if (this.query) {
                 var logicalFiles = [];
@@ -88,4 +116,4 @@ define([
             }
         }
     });
-});
+});

esp/files/scripts/QuerySetQueryWidget.js

@@ -28,7 +28,7 @@ define([
     "dijit/MenuSeparator",
     "dijit/PopupMenuItem",
 
-    "dgrid/OnDemandGrid",
+    "dgrid/Grid",
     "dgrid/Keyboard",
     "dgrid/Selection",
     "dgrid/selector",
@@ -64,7 +64,7 @@ define([
     "dojox/layout/TableContainer"
 ], function (declare, lang, dom, domForm, iframe, arrayUtil, on,
                 registry, Menu, MenuItem, MenuSeparator, PopupMenuItem,
-                OnDemandGrid, Keyboard, Selection, selector, ColumnResizer, DijitRegistry, Pagination,
+                Grid, Keyboard, Selection, selector, ColumnResizer, DijitRegistry, Pagination,
                 _TabContainerWidget, ESPBase, ESPWorkunit, ESPLogicalFile, TargetSelectWidget, QuerySetDetailsWidget, WsWorkunits, ESPQuery, ESPUtil,
                 template) {
     return declare("QuerySetQueryWidget", [_TabContainerWidget], {
@@ -121,7 +121,6 @@ define([
             });
             this.initQuerySetGrid();
             this.selectChild(this.queriesTab, true);
-            this.refreshGrid();
         },
 
         initTab: function () {
@@ -252,10 +251,12 @@ define([
         initQuerySetGrid: function (params) {
             var context = this;
             var store = ESPQuery.CreateQueryStore();
-            this.querySetGrid = declare([OnDemandGrid, Keyboard, Selection, ColumnResizer, DijitRegistry, ESPUtil.GridHelper, Pagination,])({
+            this.querySetGrid = new declare([Grid, Pagination, Selection, ColumnResizer, Keyboard, DijitRegistry, ESPUtil.GridHelper])({
                 allowSelectAll: true,
                 deselectOnRefresh: false,
                 store: store,
+                query: this.getFilter(),
+                sort: [{ attribute: "Id" }],
                 rowsPerPage: 50,
                 pagingLinks: 1,
                 pagingTextBox: true,
@@ -318,7 +319,8 @@ define([
                     },
                     QuerySetId:{
                         width: 180,
-                        label: "Target"
+                        label: "Target",
+                        sortable: false
                     },
                     Wuid: {
                         width: 180,
@@ -330,11 +332,13 @@ define([
                     },
                     Priority: {
                         width: 100,
-                        label: "Priority"
+                        label: "Priority",
+                        sortable: false
                     },
                     IsLibrary: {
                         width: 100,
-                        label: "Is Library"
+                        label: "Is Library",
+                        sortable: false
                     },
                     PublishedBy: {
                         width: 180,

esp/services/ws_smc/ws_smcService.cpp

@@ -173,7 +173,7 @@ struct CActiveWorkunitWrapper: public CActiveWorkunit
         else if(index)
             stateStr.appendf("queued(%d) [%s]", index, state.str());
         else if(location && *location)
-            stateStr.appendf("%s [on %s]", state.str(), location);
+            stateStr.appendf("%s [%s]", state.str(), location);
         else
             stateStr.set(state.str());
         setStateID(wu->getState());

plugins/CMakeLists.txt

@@ -21,10 +21,7 @@ add_subdirectory (parselib)
 add_subdirectory (stringlib)
 add_subdirectory (unicodelib)
 add_subdirectory (workunitservices)
-if ("${BUILD_LEVEL}" STREQUAL "COMMUNITY")
-  add_subdirectory (proxies)
-endif ()
-
+add_subdirectory (proxies)
 add_subdirectory (v8embed)
 add_subdirectory (pyembed)
 add_subdirectory (javaembed)

plugins/fileservices/fileservices.cpp

@@ -168,28 +168,6 @@ static IConstWorkUnit * getWorkunit(ICodeContext * ctx)
     return factory->openWorkUnit(wuid, false);
 }
 
-static IWorkUnit * updateWorkunit(ICodeContext * ctx)
-{
-    // following bit of a kludge, as
-    // 1) eclagent keeps WU locked, and
-    // 2) rtti not available in generated .so's to convert to IAgentContext
-    IAgentContext * actx = dynamic_cast<IAgentContext *>(ctx);
-    if (actx == NULL) { // fall back to pure ICodeContext
-        // the following works for thor only
-        char * platform = ctx->getPlatform();
-        if (strcmp(platform,"thor")==0) {
-            CTXFREE(parentCtx, platform);
-            Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
-            StringAttr wuid;
-            wuid.setown(ctx->getWuid());
-            return factory->updateWorkUnit(wuid);
-        }
-        CTXFREE(parentCtx, platform);
-        return NULL;
-    }
-    return actx->updateWorkUnit();
-}
-
 static IPropertyTree *getEnvironment()
 {
     Owned<IPropertyTree> env;
@@ -285,26 +263,11 @@ StringBuffer & constructLogicalName(ICodeContext * ctx, const char * partialLogi
 
 static void WUmessage(ICodeContext *ctx, WUExceptionSeverity sev, const char *fn, const char *msg)
 {
-    StringBuffer s;
-    s.append("fileservices");
+    StringBuffer s("fileservices");
     if (fn)
         s.append(", ").append(fn);
-    IAgentContext * actx = dynamic_cast<IAgentContext *>(ctx); // doesn't work if called from helper .so (no rtti)
-    if (actx)
-        actx->addWuException(msg,0,sev,s.str());
-    else {
-        Owned<IWorkUnit> wu = updateWorkunit(ctx);
-        if (wu.get()) {
-            Owned<IWUException> we = wu->createException();
-            we->setSeverity(sev);
-            we->setExceptionMessage(msg);
-            we->setExceptionSource(s.str());
-        }
-        else {
-            s.append(" : ").append(msg);
-            ctx->addWuException(s.str(),0,sev); // use plain code context
-        }
-    }
+    ctx->addWuException(msg, 0, sev, s.str()); // use plain code context
+    return;
 }
 
 static void AuditMessage(ICodeContext *ctx,
@@ -549,7 +512,7 @@ static void blockUntilComplete(const char * label, IClientFileSpray &server, ICo
 
     while(true)
     {
-        Owned<IWorkUnit> wu = updateWorkunit(ctx); // may return NULL
+        Owned<IWorkUnit> wu = ctx->updateWorkUnit(); // may return NULL
 
         Owned<IClientGetDFUWorkunit> req = server.createGetDFUWorkunitRequest();
         req->setWuid(wuid);

plugins/logging/logging.cpp

@@ -31,9 +31,9 @@ static const char * compatibleVersions[] = {
 static const char * EclDefinition =
 "export Logging := SERVICE\n"
 "  dbglog(const string src) : c,action,entrypoint='logDbgLog'; \n"
-"  addWorkunitInformation(const varstring txt, unsigned code=0, unsigned severity=0) : ctxmethod,action,entrypoint='addWuException'; \n"
-"  addWorkunitWarning(const varstring txt, unsigned code=0, unsigned severity=1) : ctxmethod,action,entrypoint='addWuException'; \n"
-"  addWorkunitError(const varstring txt, unsigned code=0, unsigned severity=2) : ctxmethod,action,entrypoint='addWuException'; \n"
+"  addWorkunitInformation(const varstring txt, unsigned code=0, unsigned severity=0, const varstring source='user') : ctxmethod,action,entrypoint='addWuException'; \n"
+"  addWorkunitWarning(const varstring txt, unsigned code=0, unsigned severity=1, const varstring source='user') : ctxmethod,action,entrypoint='addWuException'; \n"
+"  addWorkunitError(const varstring txt, unsigned code=0, unsigned severity=2, const varstring source='user') : ctxmethod,action,entrypoint='addWuException'; \n"
 "END;";
 
 LOGGING_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb) 

roxie/ccd/ccdactivities.cpp

@@ -551,7 +551,7 @@ public:
     virtual unsigned __int64 getDatasetHash(const char * name, unsigned __int64 hash)   { throwUnexpected(); return 0; }
 
     virtual char * getExpandLogicalName(const char * logicalName) { throwUnexpected(); }
-    virtual void addWuException(const char * text, unsigned code, unsigned severity) { throwUnexpected(); }
+    virtual void addWuException(const char * text, unsigned code, unsigned severity, const char * source) { throwUnexpected(); }
     virtual void addWuAssertFailure(unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool isAbort) { throwUnexpected(); }
     virtual IUserDescriptor *queryUserDescriptor() { throwUnexpected(); }
     virtual unsigned getNodes() { throwUnexpected(); }
@@ -608,6 +608,7 @@ public:
         return createRowFromXml(rowAllocator, len, utf8, xmlTransformer, stripWhitespace);
     }
     virtual IEngineContext *queryEngineContext() { return NULL; }
+    virtual IWorkUnit *updateWorkUnit() const { throwUnexpected(); }
 };
 
 //================================================================================================

roxie/ccd/ccdcontext.cpp

@@ -1201,7 +1201,7 @@ public:
     virtual void getExternalResultRaw(unsigned & tlen, void * & tgt, const char * wuid, const char * stepname, unsigned sequence, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer) { throwUnexpected(); }
 
     virtual char * getExpandLogicalName(const char * logicalName) { throwUnexpected(); }
-    virtual void addWuException(const char * text, unsigned code, unsigned severity) { throwUnexpected(); }
+    virtual void addWuException(const char * text, unsigned code, unsigned severity, const char * source) { throwUnexpected(); }
     virtual void addWuAssertFailure(unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool isAbort) { throwUnexpected(); }
     virtual IUserDescriptor *queryUserDescriptor() { throwUnexpected(); }
 
@@ -2827,7 +2827,7 @@ public:
         UNIMPLEMENTED;
     }
 
-    virtual void addWuException(const char * text, unsigned code, unsigned _severity)
+    virtual void addWuException(const char * text, unsigned code, unsigned _severity, const char * source)
     {
         WUExceptionSeverity severity = (WUExceptionSeverity) _severity;
         CTXLOG("%s", text);
@@ -2836,7 +2836,7 @@ public:
         if (workUnit)
         {
             WorkunitUpdate wu(&workUnit->lock());
-            addExceptionToWorkunit(wu, severity, "user", code, text, NULL, 0 ,0);
+            addExceptionToWorkunit(wu, severity, source, code, text, NULL, 0 ,0);
         }
     }
     virtual void addWuAssertFailure(unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool isAbort)
@@ -2903,7 +2903,7 @@ public:
     virtual void returnPersistVersion(const char * logicalName, unsigned eclCRC, unsigned __int64 allCRC, bool isFile) { throwUnexpected(); }
     virtual void fail(int code, const char *text)
     {
-        addWuException(text, code, 2);
+        addWuException(text, code, 2, "user");
     }
 
     virtual unsigned getWorkflowId() { return workflow->queryCurrentWfid(); }

rtl/include/eclhelper.hpp

@@ -499,6 +499,7 @@ interface IUserDescriptor;
 interface IHThorArg;
 interface IHThorHashLookupInfo;
 interface IEngineContext;
+interface IWorkUnit;
 
 interface ICodeContext : public IResourceContext
 {
@@ -555,7 +556,7 @@ interface ICodeContext : public IResourceContext
 
     // Exception handling
 
-    virtual void addWuException(const char * text, unsigned code, unsigned severity) = 0; //n.b. this might be better named: it should only be used for adding user-generated exceptions (via the logging plug-in) --- there's a call in IAgentContext which takes a source argument too
+    virtual void addWuException(const char * text, unsigned code, unsigned severity, const char * source) = 0;
     virtual void addWuAssertFailure(unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool isAbort) = 0;
 
     // File resolution etc
@@ -596,6 +597,7 @@ interface ICodeContext : public IResourceContext
     virtual char * queryIndexMetaData(char const * lfn, char const * xpath) = 0;
     virtual IEngineContext *queryEngineContext() = 0;
     virtual char *getDaliServers() = 0;
+    virtual IWorkUnit *updateWorkUnit() const = 0;
 };
 
 

testing/unittests/dalitests.cpp

@@ -466,6 +466,7 @@ class DaliTests : public CppUnit::TestFixture
         CPPUNIT_TEST(testDFSRemoveSuperSub);
 // This test requires access to an external IP with dafilesrv running
 //        CPPUNIT_TEST(testDFSRename3);
+        CPPUNIT_TEST(testDFSAddFailReAdd);
         CPPUNIT_TEST(testDFSHammer);
     CPPUNIT_TEST_SUITE_END();
 
@@ -1585,6 +1586,83 @@ public:
         ASSERT(1 == sfile->numSubFiles() && "regress::clearadd::super1 should contain 1 subfile");
     }
 
+    void testDFSAddFailReAdd()
+    {
+        setupDFS("addreadd");
+
+        Owned<IDistributedFileTransaction> transaction = createDistributedFileTransaction(user); // disabled, auto-commit
+
+        logctx.CTXLOG("Creating super1 and supet2, adding sub1 and sub2 to super1 and sub3 to super2");
+        Owned<IDistributedSuperFile> sfile = dir.createSuperFile("regress::addreadd::super1", user, false, false, transaction);
+        sfile->addSubFile("regress::addreadd::sub1", false, NULL, false, transaction);
+        sfile->addSubFile("regress::addreadd::sub2", false, NULL, false, transaction);
+        sfile.clear();
+        Owned<IDistributedSuperFile> sfile2 = dir.createSuperFile("regress::addreadd::super2", user, false, false, transaction);
+        sfile2->addSubFile("regress::addreadd::sub3", false, NULL, false, transaction);
+        sfile2.clear();
+
+        class CShortLock : implements IThreaded
+        {
+            StringAttr fileName;
+            unsigned secDelay;
+            CThreaded threaded;
+        public:
+            CShortLock(const char *_fileName, unsigned _secDelay) : fileName(_fileName), secDelay(_secDelay), threaded("CShortLock", this) { }
+            virtual void main()
+            {
+                Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(fileName, NULL);
+
+                if (!file)
+                {
+                    PROGLOG("File %s not found", fileName.get());
+                    return;
+                }
+                PROGLOG("Locked file: %s, sleeping (before unlock) for %d secs", fileName.get(), secDelay);
+
+                MilliSleep(secDelay * 1000);
+
+                PROGLOG("Unlocking file: %s", fileName.get());
+            }
+            void start() { threaded.start(); }
+        };
+
+        /* Tests transaction failing, due to lock and retrying after having partial success */
+
+        CShortLock sL("regress::addreadd::sub2", 30); // the 2nd subfile of super1
+        sL.start();
+
+        transaction.setown(createDistributedFileTransaction(user)); // disabled, auto-commit
+        logctx.CTXLOG("Starting transaction");
+        transaction->start();
+
+        logctx.CTXLOG("Adding contents of regress::addreadd::super1 to regress::addreadd::super2, within transaction");
+        sfile.setown(transaction->lookupSuperFile("regress::addreadd::super2"));
+        sfile->addSubFile("regress::addreadd::super1", false, NULL, true, transaction); // add contents of super1 to super2
+        sfile.setown(transaction->lookupSuperFile("regress::addreadd::super1"));
+        sfile->removeSubFile(NULL, false, false, transaction); // clears super1
+        sfile.clear();
+
+        try
+        {
+            transaction->commit();
+        }
+        catch (IException *e)
+        {
+            StringBuffer eStr;
+            e->errorMessage(eStr);
+            CPPUNIT_ASSERT_MESSAGE(eStr.str(), 0);
+            e->Release();
+        }
+        transaction.clear();
+        sfile.setown(dir.lookupSuperFile("regress::addreadd::super2", user));
+        ASSERT(3 == sfile->numSubFiles() && "regress::addreadd::super2 should contain 3 subfiles");
+        ASSERT(NULL != sfile->querySubFileNamed("regress::addreadd::sub1") && "regress::addreadd::sub1, should be a subfile of super2");
+        ASSERT(NULL != sfile->querySubFileNamed("regress::addreadd::sub2") && "regress::addreadd::sub2, should be a subfile of super2");
+        ASSERT(NULL != sfile->querySubFileNamed("regress::addreadd::sub3") && "regress::addreadd::sub3, should be a subfile of super2");
+        sfile.setown(dir.lookupSuperFile("regress::addreadd::super1", user));
+        ASSERT(0 == sfile->numSubFiles() && "regress::addreadd::super1 should contain 0 subfiles");
+    }
+
     void testDFSRename2()
     {
         setupDFS("rename2");

thorlcr/graph/thgraph.hpp

@@ -482,7 +482,7 @@ class graph_decl CGraphBase : public CInterface, implements IEclGraphResults, im
         virtual void getExternalResultRaw(unsigned & tlen, void * & tgt, const char * wuid, const char * stepname, unsigned sequence, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer) { ctx->getExternalResultRaw(tlen, tgt, wuid, stepname, sequence, xmlTransformer, csvTransformer); }
         virtual void executeGraph(const char * graphName, bool realThor, size32_t parentExtractSize, const void * parentExtract) { ctx->executeGraph(graphName, realThor, parentExtractSize, parentExtract); }
         virtual char * getExpandLogicalName(const char * logicalName) { return ctx->getExpandLogicalName(logicalName); }
-        virtual void addWuException(const char * text, unsigned code, unsigned severity) { ctx->addWuException(text, code, severity); }
+        virtual void addWuException(const char * text, unsigned code, unsigned severity, const char * source) { ctx->addWuException(text, code, severity, source); }
         virtual void addWuAssertFailure(unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool isAbort) { ctx->addWuAssertFailure(code, text, filename, lineno, column, isAbort); }
         virtual IUserDescriptor *queryUserDescriptor() { return ctx->queryUserDescriptor(); }
         virtual IThorChildGraph * resolveChildQuery(__int64 activityId, IHThorArg * colocal) { return ctx->resolveChildQuery(activityId, colocal); }
@@ -528,6 +528,7 @@ class graph_decl CGraphBase : public CInterface, implements IEclGraphResults, im
         {
             return ctx->getDaliServers();
         }
+        virtual IWorkUnit *updateWorkUnit() const { return ctx->updateWorkUnit(); }
    } graphCodeContext;
 
 protected:

thorlcr/graph/thgraphmaster.cpp

@@ -750,7 +750,7 @@ class CThorCodeContextMaster : public CThorCodeContextBase
     Linked<IConstWorkUnit> workunit;
     Owned<IDistributedFileTransaction> superfiletransaction;
 
-    IWorkUnit *updateWorkUnit() 
+    virtual IWorkUnit *updateWorkUnit() const
     {
         StringAttr wuid;
         workunit->getWuid(StringAttrAdaptor(wuid));
@@ -1061,7 +1061,7 @@ public:
             throw MakeStringException(TE_FailedToRetrieveWorkunitValue, "Failed to retrieve external data value %s from workunit %s", stepname, wuid);
         }
     }
-    virtual void addWuException(const char * text, unsigned code, unsigned severity)
+    virtual void addWuException(const char * text, unsigned code, unsigned severity, const char * source)
     {
         DBGLOG("%s", text);
         try
@@ -1070,7 +1070,7 @@ public:
             Owned<IWUException> we = w->createException();
             we->setSeverity((WUExceptionSeverity)severity);
             we->setExceptionMessage(text);
-            we->setExceptionSource("user");
+            we->setExceptionSource(source);
             if (code)
                 we->setExceptionCode(code);
         }

thorlcr/graph/thgraphslave.cpp

@@ -953,12 +953,11 @@ public:
 
     virtual void getExternalResultRaw(unsigned & tlen, void * & tgt, const char * wuid, const char * stepname, unsigned sequence, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer) { throwUnexpected(); }
 
-    virtual void addWuException(const char * text, unsigned code, unsigned severity)
+    virtual void addWuException(const char * text, unsigned code, unsigned severity, const char * source)
     {
         DBGLOG("%s", text);
         Owned<IThorException> e = MakeThorException(code, "%s", text);
-        e->setAction(tea_warning);
-        e->setOrigin("user");
+        e->setOrigin(source);
         e->setAction(tea_warning);
         e->setSeverity((WUExceptionSeverity)severity);
         job.fireException(e);

thorlcr/mfilemanager/thmfilemanager.cpp

@@ -510,6 +510,7 @@ public:
                         if (p == partOffset)
                             p += job.querySlaves();
                         IPartDescriptor *partDesc = fileDesc.queryPart(p);
+                        CDateTime createTime, modifiedTime;
                         unsigned c=0;
                         for (; c<partDesc->numCopies(); c++)
                         {
@@ -522,6 +523,15 @@ public:
                                 ensureDirectoryForFile(path.str());
                                 OwnedIFile iFile = createIFile(path.str());
                                 OwnedIFileIO iFileIO = iFile->open(IFOcreate);
+                                iFileIO.clear();
+                                // ensure copies have matching datestamps, as they would do normally (backupnode expects it)
+                                if (partDesc->numCopies() > 1)
+                                {
+                                    if (0 == c)
+                                        iFile->getTime(&createTime, &modifiedTime, NULL);
+                                    else
+                                        iFile->setTime(&createTime, &modifiedTime, NULL);
+                                }
                             }
                             catch (IException *e)
                             {

thorlcr/msort/tsorta.cpp

@@ -77,6 +77,7 @@ void CThorKeyArray::clear()
     totalfilesize = 0;
     filerecsize = 0;
     filerecnum = 0;
+    needFPosExpand = false;
 }
 
 void CThorKeyArray::setSampling(size32_t _maxsamplesize, unsigned _divisor)
@@ -108,6 +109,8 @@ void CThorKeyArray::add(const void *row)
     totalfilesize += recSz;
     if (filerecnum==0)
         filerecsize=recSz;
+    else if (filerecsize!=recSz)
+        needFPosExpand = true;
     filerecnum++;
 
     if (maxsamplesize)
@@ -146,7 +149,7 @@ void CThorKeyArray::add(const void *row)
     }
     if (filepos)
         filepos->append(totalfilesize);
-    else if (filerecsize!=recSz)
+    else if (needFPosExpand)
     {
         expandFPos();
         filepos->append(totalfilesize);

thorlcr/msort/tsorta.hpp

@@ -76,6 +76,7 @@ class CThorKeyArray
     size32_t filerecsize;
     size32_t filerecnum;
     offset_t totalfilesize;
+    bool needFPosExpand;
 
     void split();
     offset_t findLessEqRowPos(const void * row);

thorlcr/thorcodectx/thcodectx.hpp

@@ -118,6 +118,7 @@ public:
     virtual const void * fromXml(IEngineRowAllocator * rowAllocator, size32_t len, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace);
     virtual IEngineContext *queryEngineContext() { return NULL; }
     virtual char *getDaliServers();
+    virtual IWorkUnit *updateWorkUnit() const { throwUnexpected(); }
 };
 
 #endif