Merge branch 'candidate-5.2.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

VERSIONS

@@ -41,21 +41,35 @@ distro for which a pre-packaged binary is not available, then you should build
 the stable branch. Note that stable is rebased when a new MAJOR or MINOR version
 number release is published.
 
-Branches with names starting closedown- contain code that is being prepared for
-the next point release. 
-
 Branches with names starting candidate- contain code that is being prepared for
 release as a stable version. Individual release candidates will be tagged along these
 branches.
 
 When preparing a patch or a GitHub pull request, a new git branch should be created
-based on the appropriate target - if the change is to go into the next 4.0.x build, then
-base it on closedown-4.0.x, for example. All changes that are accepted into candidate-4.0.0
-are normally merged into closedown-4.0.x, and all changes in 4.0.x are normally merged to
-master, without requiring separate pull requests.
+based on the appropriate target - if the change is to go into the 5.2.2 build, then
+base it on candidate-5.2.2, for example. All changes that are accepted into a candidate
+branch are normally upmerged to any later unreleased candidate branches, and to master,
+without requiring separate pull requests. Pull requests against master will go in the
+next major release version.
 
 Tags
 ====
 
 Tags corresponding to the release versions will be applied to points on the
 candidate- branch where binary releases have been built and published.
+
+Rules for major, minor and point releases
+=========================================
+
+It should always be safe to upgrade a system to the latest point release with matching
+major and minor version. If upgrading to a new major or minor version, users should always
+use the latest available point release. It is important to observe the following rules:
+
+1. Sequence number increases on a gold release are for build errors and regressions only.
+   A new sequence release should be considered as rendering earlier sequence releases invalid
+   and unsupported.
+2. NEVER include new functionality in point releases - only bug fixes
+3. If a bug fix introduces incompatibilities that might break existing ECL code, or
+   require it to be recompiled, it should be held until the next minor release.
+4. Any bug fix that is included in a gold release for a give major.minor version must
+   also be available (if relevant) in a gold release of all later released major.minor versions.

common/workunit/workunit.cpp

@@ -572,6 +572,37 @@ class ExtractedStatistic : public CInterfaceOf<IConstWUStatistic>
 public:
     virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const
     {
+        if (!description && createDefault)
+        {
+            switch (kind)
+            {
+            case StTimeElapsed:
+                {
+                    if (scopeType != SSTsubgraph)
+                        break;
+                    //Create a default description for a root subgraph
+                    const char * colon = strchr(scope, ':');
+                    if (!colon)
+                        break;
+
+                    const char * subgraph = colon+1;
+                    //Check for nested subgraph
+                    if (strchr(subgraph, ':'))
+                        break;
+
+                    assertex(strncmp(subgraph, SubGraphScopePrefix, strlen(SubGraphScopePrefix)) == 0);
+                    StringAttr graphname;
+                    graphname.set(scope, colon - scope);
+                    unsigned subId = atoi(subgraph + strlen(SubGraphScopePrefix));
+
+                    StringBuffer desc;
+                    formatGraphTimerLabel(desc, graphname, 0, subId);
+                    str.set(desc);
+                    return str;
+                }
+            }
+        }
+
         str.set(description);
         return str;
     }
@@ -825,11 +856,20 @@ protected:
             IStatisticCollection * curCollection = &collections.tos();
             if (childIterators.ordinality() < collections.ordinality())
             {
-                //Start iterating the children for the current collection
-                childIterators.append(curCollection->getScopes(NULL));
-                if (!childIterators.tos().first())
+                if (!filter || filter->recurseChildScopes(curStat->scopeType, curStat->scope))
+                {
+                    //Start iterating the children for the current collection
+                    childIterators.append(curCollection->getScopes(NULL));
+                    if (!childIterators.tos().first())
+                    {
+                        finishCollection();
+                        continue;
+                    }
+                }
+                else
                 {
-                    finishCollection();
+                    //Don't walk the child scopes
+                    collections.pop();
                     continue;
                 }
             }
@@ -5476,8 +5516,6 @@ IConstWUStatisticIterator& CLocalWorkUnit::getStatistics(const IStatisticsFilter
 
     statistics.load(p,"Statistics/*");
     Owned<IConstWUStatisticIterator> localStats = new WorkUnitStatisticsIterator(statistics, 0, (IConstWorkUnit *) this, filter);
-    if (filter && !filter->queryMergeSources())
-        return *localStats.getClear();
 
     const char * wuid = p->queryName();
     Owned<IConstWUStatisticIterator> graphStats = new CConstGraphProgressStatisticsIterator(wuid, filter);

ecl/eclagent/eclgraph.cpp

@@ -989,19 +989,6 @@ void EclSubGraph::execute(const byte * parentExtract)
     {
         updateProgress();
         cleanupActivities();
-
-        {
-            unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now()-startGraphCycles);
-
-            Owned<IWorkUnit> wu(agent->updateWorkUnit());
-            StringBuffer timerText;
-            formatGraphTimerLabel(timerText, parent.queryGraphName(), seqNo+1, id);
-
-            //graphn: id
-            StringBuffer wuScope;
-            formatGraphTimerScope(wuScope, parent.queryGraphName(), seqNo+1, id);
-            updateWorkunitTimeStat(wu, SSTsubgraph, wuScope, StTimeElapsed, timerText.str(), elapsed);
-        }
     }
     agent->updateWULogfile();//Update workunit logfile name in case of rollover
 }

ecl/hql/hqlexpr.cpp

@@ -948,9 +948,14 @@ void HqlLookupContext::noteExternalLookup(IHqlScope * parentScope, IHqlExpressio
             const char * moduleName = parentScope->queryFullName();
             if (moduleName)
             {
-                IPropertyTree * depend = curAttrTree->addPropTree("Depend", createPTree());
-                depend->setProp("@module", moduleName);
-                depend->setProp("@name", expr->queryName()->str());
+                VStringBuffer xpath("Depend[@module=\"%s\"][@name=\"%s\"]", moduleName, expr->queryName()->str());
+
+                if (!curAttrTree->queryPropTree(xpath.str()))
+                {
+                    IPropertyTree * depend = curAttrTree->addPropTree("Depend", createPTree());
+                    depend->setProp("@module", moduleName);
+                    depend->setProp("@name", expr->queryName()->str());
+                }
             }
         }
     }

ecl/hqlcpp/hqlckey.cpp

@@ -1671,6 +1671,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyPatch(BuildCtx & ctx, IHqlE
 
     //virtual int getSequence() = 0;
     doBuildSequenceFunc(instance->classctx, querySequence(expr), false);
+    HqlExprArray xmlnsAttrs;
+    Owned<IWUResult> result = createDatasetResultSchema(querySequence(expr), NULL, original->queryRecord(), xmlnsAttrs, false, true);
 
     buildExpiryHelper(instance->createctx, expr->queryAttribute(expireAtom));
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5874,7 +5874,7 @@ public:
 
     virtual void report(const char * scope, const char * description, const __int64 totaltime, const __int64 maxtime, const unsigned count)
     {
-        StatisticScopeType scopeType = SSTsection; // MORE?
+        StatisticScopeType scopeType = SSTcompilestage;
         StatisticKind kind = StTimeElapsed;
         wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, description, totaltime, count, maxtime, StatsMergeReplace);
     }

esp/scm/ws_topology.ecm

@@ -583,7 +583,7 @@ ESPresponse [exceptions_inline,encode(0)] TpGetServicePluginsResponse
     ESParray<ESPstruct TpEspServicePlugin, Plugin> Plugins;
 };
 
-ESPservice [noforms, version("1.21"), default_client_version("1.20"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
+ESPservice [noforms, version("1.21"), default_client_version("1.21"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
 {
     ESPuses ESPStruct TpBinding;
     ESPuses ESPstruct TpCluster;

esp/services/ws_ecl/ws_ecl_wuinfo.cpp

@@ -47,6 +47,11 @@ IConstWorkUnit *WsEclWuInfo::ensureWorkUnit()
 
 void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *resultSetFactory, IConstWUResult &var, unsigned hashWebserviceSeq=0)
 {
+    Owned<IResultSetMetaData> meta = resultSetFactory->createResultSetMeta(&var);
+    StringAttr noinput;
+    if (var.getResultFieldOpt("noinput", StringAttrAdaptor(noinput)).length() && strToBool(noinput.length(), noinput.get()))  //developer specified not to show field on form
+        return;
+
     SCMStringBuffer varname;
     var.getResultName(varname);
     int seq = var.getResultSequence();
@@ -56,8 +61,6 @@ void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *r
     SCMStringBuffer eclschema;
     var.getResultEclSchema(eclschema);
 
-    SCMStringBuffer s;
-    Owned<IResultSetMetaData> meta = resultSetFactory->createResultSetMeta(&var);
     StringBuffer width, height, fieldSeq;
     var.getResultFieldOpt("fieldwidth", StringBufferAdaptor(width));
     var.getResultFieldOpt("fieldheight", StringBufferAdaptor(height));
@@ -66,6 +69,7 @@ void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *r
     else
         var.getResultFieldOpt("sequence", StringBufferAdaptor(fieldSeq));
 
+    SCMStringBuffer s;
     Owned<IPropertyTree> part = createPTree("part");
     if (!var.isResultScalar())
     {
@@ -125,23 +129,34 @@ void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *r
     parts.append(*part.getClear());
 }
 
+int orderMatchingSequence(IPropertyTree * left, IPropertyTree * right)
+{
+    if (!right->hasProp("@name"))
+        return -1;
+    if (!left->hasProp("@name"))
+        return 1;
+    return stricmp(left->queryProp("@name"), right->queryProp("@name"));
+}
+
 int orderParts(IInterface * const * pLeft, IInterface * const * pRight)
 {
-    IPropertyTree * right = (IPropertyTree *)*pRight;
     IPropertyTree * left = (IPropertyTree *)*pLeft;
-    bool hasRightSeq = right->hasProp("@sequence");
+    IPropertyTree * right = (IPropertyTree *)*pRight;
     bool hasLeftSeq = left->hasProp("@sequence");
-    if (hasRightSeq && hasLeftSeq)
-        return left->getPropInt("@sequence") - right->getPropInt("@sequence");
+    bool hasRightSeq = right->hasProp("@sequence");
+    if (hasLeftSeq && hasRightSeq)
+    {
+        int rightSeq = right->getPropInt("@sequence");
+        int leftSeq = left->getPropInt("@sequence");
+        if (rightSeq == leftSeq)
+            return orderMatchingSequence(left, right);  //fields with same sequence alphabetical within sequence
+        return leftSeq - rightSeq;
+    }
     if (hasRightSeq)
-        return -1;
-    if (hasLeftSeq)
         return 1;
-    if (!right->hasProp("@name"))
+    if (hasLeftSeq)
         return -1;
-    if (!left->hasProp("@name"))
-        return 1;
-    return stricmp(right->queryProp("@name"), left->queryProp("@name"));  //fields without sequence alphabetical AFTER sequenced fields
+    return orderMatchingSequence(left, right);  //fields without sequence alphabetical AFTER sequenced fields
 }
 
 bool WsEclWuInfo::getWsResource(const char *name, StringBuffer &out)

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -378,7 +378,6 @@ void WsWuInfo::getTimers(IEspECLWorkunit &info, unsigned flags)
 
         IArrayOf<IEspECLTimer> timers;
         StatisticsFilter filter;
-        filter.setMergeSources(false);
         filter.setScopeDepth(1, 2);
         filter.setMeasure(SMeasureTimeNs);
         Owned<IConstWUStatisticIterator> it = &cw->getStatistics(&filter);
@@ -446,7 +445,6 @@ unsigned WsWuInfo::getTimerCount()
     {
         //This filter must match the filter in the function above, otherwise it will be inconsistent
         StatisticsFilter filter;
-        filter.setMergeSources(false);
         filter.setScopeDepth(1, 2);
         filter.setMeasure(SMeasureTimeNs);
         Owned<IConstWUStatisticIterator> it = &cw->getStatistics(&filter);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -4251,8 +4251,6 @@ bool CWsWorkunitsEx::onWUGetStats(IEspContext &context, IEspWUGetStatsRequest &r
             filter.setScopeDepth(req.getMinScopeDepth(), req.getMaxScopeDepth());
         else if (!req.getMinScopeDepth_isNull())
             filter.setScopeDepth(req.getMinScopeDepth());
-        if (!req.getIncludeGraphs_isNull())
-            filter.setMergeSources(req.getIncludeGraphs());
 
         bool createDescriptions = false;
         if (!req.getCreateDescriptions_isNull())

esp/src/Visualization

@@ -1 +1 @@
-Subproject commit 28418fa21158e692a4cd49dde72996d8d04641bf
+Subproject commit da4fdbc3eedec680f916d71c3f2d414730185a8b

esp/src/eclwatch/ESPWorkunit.js

@@ -557,11 +557,12 @@ define([
                                 if (context.timers) {
                                     context.graphs[i].Time = 0;
                                     for (var j = 0; j < context.timers.length; ++j) {
-                                        if (context.timers[j].GraphName == context.graphs[i].Name) {
-                                            context.graphs[i].Time += context.timers[j].Seconds;
+                                        if (context.timers[j].GraphName === context.graphs[i].Name && !context.timers[j].HasSubGraphId) {
+                                            context.graphs[i].Time = context.timers[j].Seconds;
+                                            break;
                                         }
-                                        context.graphs[i].Time = Math.round(context.graphs[i].Time * 1000) / 1000;
                                     }
+                                    context.graphs[i].Time = Math.round(context.graphs[i].Time * 1000) / 1000;
                                 }
                                 if (lang.exists("ApplicationValues.ApplicationValue", context)) {
                                     var idx = context.getApplicationValueIndex("ESPWorkunit.js", context.graphs[i].Name + "_SVG");

esp/src/eclwatch/GraphsWidget.js

@@ -128,8 +128,28 @@ define([
                     },
                     Label: { label: this.i18n.Label, sortable: true },
                     Complete: { label: this.i18n.Completed, width: 72, sortable: true },
+                    WhenStarted: {
+                        label: this.i18n.Started, width: 90,
+                        formatter: function (whenStarted) {
+                            if (whenStarted) {
+                                var dateTime = new Date(whenStarted);
+                                return dateTime.toLocaleTimeString();
+                            }
+                            return "";
+                        }
+                    },
+                    WhenFinished: {
+                        label: this.i18n.Finished, width: 90,
+                        formatter: function (whenFinished, idx) {
+                            if (whenFinished) {
+                                var dateTime = new Date(whenFinished);
+                                return dateTime.toLocaleTimeString();
+                            }
+                            return "";
+                        }
+                    },
                     Time: {
-                        label: this.i18n.Time, width: 90, sortable: true,
+                        label: this.i18n.Duration, width: 90, sortable: true,
                         formatter: function (totalSeconds, idx) {
                             var hours = Math.floor(totalSeconds / 3600);
                             totalSeconds %= 3600;

esp/src/eclwatch/TimingPageWidget.js

@@ -83,6 +83,19 @@ define([
                         SubGraphId: item.SubGraphId
                     });
                 }
+
+                var statsTabID = this.createChildTabID("Stats");
+                var statsTab = new DelayLoadWidget({
+                    id: statsTabID,
+                    title: this.i18n.Stats,
+                    closable: false,
+                    delayWidget: "WUStatsWidget",
+                    hpcc: {
+                        type: "stats",
+                        params: this.params
+                    }
+                });
+                this.addChild(statsTab);
                 this._refreshActionState();
             },
 

esp/src/eclwatch/TopologyDetailsWidget.js

@@ -152,6 +152,31 @@ define([
                         }
                     }
                 }
+                var tpMachine = null;
+                if (this.params.__hpcc_treeItem.__hpcc_type === "TpMachine") {
+                    tpMachine = this.params.__hpcc_treeItem;
+                } else if (this.params.__hpcc_parentNode && this.params.__hpcc_parentNode.__hpcc_treeItem.__hpcc_type === "TpMachine") {
+                    tpMachine = this.params.__hpcc_parentNode.__hpcc_treeItem;
+                };
+                var tpBinding = null;
+                if (this.params.__hpcc_treeItem.__hpcc_type === "TpBinding") {
+                    tpBinding = this.params.__hpcc_treeItem;
+                } else if (this.params.__hpcc_parentNode && this.params.__hpcc_parentNode.__hpcc_treeItem.__hpcc_type === "TpBinding") {
+                    tpBinding = this.params.__hpcc_parentNode.__hpcc_treeItem;
+                };
+                if (tpBinding && tpMachine) {
+                    var tr = domConstruct.create("tr", {}, table);
+                    domConstruct.create("td", {
+                        innerHTML: "<b>URL:&nbsp;&nbsp;</b>"
+                    }, tr);
+                    var td = domConstruct.create("td", {
+                    }, tr);
+                    var url = tpBinding.Protocol + "://" + tpMachine.Netaddress + ":" + tpBinding.Port + "/";
+                    domConstruct.create("a", {
+                        href: url,
+                        innerHTML: url
+                    }, td);
+                }
                 this.details.setContent(table);
             } else if (currSel.id === this.widget._Configuration.id && !this.widget._Configuration.__hpcc_initalized) {
                 this.widget._Configuration.__hpcc_initalized = true;

esp/src/eclwatch/VizWidget.js

@@ -373,7 +373,7 @@ define([
                 });
             } else {
                 if (chartType && this.d3Viz.chart) {
-                    this.d3Viz.chart.chartType(chartType);
+                    this.d3Viz.chart.chart_type(chartType);
                 }
                 deferred.resolve(this.vizType);
             }

esp/src/eclwatch/WUStatsWidget.js

@@ -0,0 +1,347 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+define([
+    "dojo/_base/declare",
+    "dojo/_base/lang",
+    "dojo/i18n",
+    "dojo/i18n!./nls/hpcc",
+    "dojo/on",
+
+    "dijit/registry",
+
+    "hpcc/_Widget",
+    "hpcc/WsWorkunits",
+
+    "dojo/text!../templates/WUStatsWidget.html",
+
+    "dijit/layout/BorderContainer",
+    "dijit/layout/ContentPane",
+    "dijit/form/Button",
+    "dijit/form/Select"
+], function (declare, lang, i18n, nlsHPCC, on,
+            registry,
+            _Widget, WsWorkunits,
+            template) {
+    return declare("WUStatsWidget", [_Widget], {
+        templateString: template,
+        baseClass: "WUStatsWidget",
+        i18n: nlsHPCC,
+
+        buildRendering: function (args) {
+            this.inherited(arguments);
+        },
+
+        postCreate: function (args) {
+            this.inherited(arguments);
+            this.borderContainer = registry.byId(this.id + "BorderContainer");
+        },
+
+        startup: function (args) {
+            this.inherited(arguments);
+        },
+
+        resize: function (args) {
+            this.inherited(arguments);
+            this.borderContainer.resize();
+            if (this.pieCreatorType) this.pieCreatorType.widget.resize().render();
+            if (this.pieScopeType) this.pieScopeType.widget.resize().render();
+            if (this.scopesSurface) this.scopesSurface.resize().render();
+            if (this.bar) this.bar.resize().render();
+        },
+
+        layout: function (args) {
+            this.inherited(arguments);
+        },
+
+        destroy: function (args) {
+            this.inherited(arguments);
+        },
+
+        //  Implementation  ---
+        _onRefresh: function () {
+            this.doRefreshData();
+        },
+
+        _onReset: function () {
+            this.doReset();
+        },
+
+        init: function (params) {
+            if (this.inherited(arguments))
+                return;
+
+            var context = this;
+            require(["src/other/Comms", "src/chart/MultiChartSurface", "src/common/Surface", "src/tree/SunburstPartition", "src/other/Table", "crossfilter/crossfilter"], function (Comms, MultiChartSurface, Surface, SunburstPartition, Table, crossfilterXXX) {
+                function CFGroup(crossfilter, dimensionID, targetID) {
+                    this.targetID = targetID;
+                    this.dimensionID = dimensionID;
+                    this.dimension = crossfilter.dimension(function (d) { return d[dimensionID]; });
+                    this.group = this.dimension.group().reduceCount();
+
+                    this.widget = new MultiChartSurface()
+                        .target(targetID)
+                        .title(dimensionID)
+                        .columns([dimensionID, "Total"])
+                        .show_icon(false)
+                        .menu(["Pie (Google)", "Pie (C3)", "Table"])
+                        .chart_type("GOOGLE_PIE")
+                    ;
+
+                    this.filter = null;
+                    var context = this;
+                    this.widget.click = function (row, column) {
+                        if (context.filter === row[dimensionID]) {
+                            context.filter = null;
+                        } else {
+                            context.filter = row[dimensionID];
+                        }
+                        context.dimension.filter(context.filter);
+                        context.click(row, column);
+                        context.render();
+                    };
+                }
+                CFGroup.prototype.click = function (row, column) {
+                }
+                CFGroup.prototype.resetFilter = function () {
+                    this.filter = null;
+                    this.dimension.filter(null);
+                }
+                CFGroup.prototype.render = function () {
+                    this.widget
+                        .title(this.dimensionID + (this.filter ? " (" + this.filter + ")" : ""))
+                        .data(this.group.all().map(function (row) {
+                            return [row.key, row.value];
+                        }))
+                        .render()
+                    ;
+                }
+
+                context.stats = crossfilter([]);
+                context.summaryByKind = context.stats.dimension(function (d) { return d.Kind; });
+                context.groupByKind = context.summaryByKind.group().reduceSum(function (d) { return d.RawValue; });
+
+                context.select = registry.byId(context.id + "Kind");
+                var prevKind = "";
+                context.select.on("change", function (newValue) {
+                    if (prevKind !== newValue) {
+                        context.summaryByKind.filter(newValue);
+                        context.doRender(context.select);
+                        prevKind = newValue;
+                    }
+                });
+
+                context.pieCreatorType = new CFGroup(context.stats, "CreatorType", context.id + "CreatorType");
+                context.pieCreatorType.click = function (row, column) {
+                    context.doRender(context.pieCreatorType);
+                }
+
+                context.pieScopeType = new CFGroup(context.stats, "ScopeType", context.id + "ScopeType");
+                context.pieScopeType.click = function (row, column) {
+                    context.doRender(context.pieScopeType);
+                }
+
+                context.summaryByScope = context.stats.dimension(function (d) { return d.Scope; });
+                context.groupByScope = context.summaryByScope.group().reduceSum(function (d) { return d.RawValue; });
+
+                context.scopes = new SunburstPartition();
+                context.scopesSurface = new Surface()
+                    .target(context.id + "Scope")
+                    .show_icon(false)
+                    .title("Scope")
+                    .content(context.scopes)
+                ;
+
+                context.prevScope = null;
+                context.scopes.click = function (row, column) {
+                    if (row.id === "") {
+                        context.prevScope = null;
+                        context.summaryByScope.filter(null);
+                    } else if (context.prevScope === row.id) {
+                        context.prevScope = null;
+                        context.summaryByScope.filter(null);
+                    } else {
+                        context.prevScope = row.id;
+                        context.summaryByScope.filter(function (d) {
+                            return d.indexOf(context.prevScope + ":") === 0;
+                        });
+                    }
+                    context.doRender(context.scopes);
+                };
+
+                context.bar = new MultiChartSurface()
+                    .target(context.id + "Stats")
+                    .show_icon(false)
+                    .menu(["Bar (Google)", "Bar (C3)", "Column (Google)", "Column (C3)", "Table"])
+                    .chart_type("GOOGLE_COLUMN")
+                ;
+
+                context.doRefreshData();
+            });
+        },
+
+        formatTree: function (data, label) {
+            var cache = {};
+            var treeDedup = {
+                "": {
+                    parentID: null,
+                    id: "",
+                    label: label,
+                    children: [],
+                    childrenDedup: {}
+                }
+            };
+            data.forEach(function (row, idx) {
+                var i = 1;
+                var scopeParts = row.key.split(":");
+                var scope = "";
+                scopeParts.forEach(function (item, idx) {
+                    var prevScope = scope;
+                    scope += (scope.length ? ":" : "") + item;
+                    if (!treeDedup[scope]) {
+                        var newTreeItem = {
+                            parentID: prevScope,
+                            id: scope,
+                            children: [],
+                            childrenDedup: {}
+                        }
+                        treeDedup[scope] = newTreeItem;
+                        treeDedup[prevScope].children.push(newTreeItem);
+                        treeDedup[prevScope].childrenDedup[scope] = newTreeItem;
+                    }
+                    var scopeItem = treeDedup[scope];
+                    if (idx === scopeParts.length - 1) {
+                        scopeItem.__data = row;
+                        scopeItem.label = row.key;
+                        scopeItem.value = row.value;
+                    };
+                });
+            });
+            function trimTree(node) {
+                var newChildren = [];
+                node.children.forEach(function (childNode) {
+                    trimTree(childNode);
+                    if (childNode.value || childNode.children.length) {
+                        newChildren.push(childNode);
+                    }
+                })
+                node.children = newChildren;
+                return node;
+            }
+            var retVal = trimTree(treeDedup[""]);
+            return retVal;
+        },
+
+        doReset: function () {
+            this.pieCreatorType.resetFilter();
+            this.pieScopeType.resetFilter();
+            this.prevScope = null;
+            this.summaryByScope.filterAll();
+            if (this.select.get("value") !== "TimeElapsed") {
+                this.select.set("value", "TimeElapsed");
+            } else {
+                this.doRender();
+            }
+        },
+
+        doRender: function (source) {
+            if (source !== this.pieCreatorType) this.pieCreatorType.render();
+            if (source !== this.pieScopeType) this.pieScopeType.render();
+
+            if (source !== this.scopes) {
+                var tree = this.formatTree(this.groupByScope.all(), this.params.Wuid);
+                this.scopes
+                    .data(tree)
+                ;
+                this.scopesSurface
+                    .render()
+                ;
+            }
+
+            var scopeData = this.summaryByScope.top(Infinity);
+            var columns = ["Creator", "CreatorType", "Scope", "ScopeType", "Description", "TimeStamp", "Measure", "Kind", "Value", "RawValue", "Count", "Max"];
+            var data = scopeData.map(function (row, idx) {
+                var rowData = [];
+                columns.forEach(function (column) {
+                    rowData.push(row[column]);
+                });
+                return rowData;
+            });
+
+            var statsData = [];
+            if (this.select.get("value")) {
+                statsData = scopeData.map(function (row) {
+                    if (this.prevScope === row.Scope) {
+                        return [row.Scope, row.RawValue];
+                    }
+                    return [(this.prevScope && row.Scope.indexOf(this.prevScope) === 0 ? row.Scope.substring(this.prevScope.length + 1) : row.Scope), row.RawValue];
+                }, this);
+            }
+            this.scopesSurface.title("Scope" + (this.prevScope ? " (" + this.prevScope + ")" : "")).render();
+            var statsLabel = [this.select.get("value"), this.pieCreatorType.filter, this.pieScopeType.filter, this.prevScope].filter(function (item) {
+                return item;
+            }).join(", ") || "Unknown";
+            statsLabel += (scopeData[0] ? " (" + scopeData[0].Measure + ")" : "");
+            this.bar
+                .title(statsLabel)
+                .columns(["Stat", statsLabel])
+                .data(statsData)
+                .render(function (d) {
+                    if (d._content && d._content._chart && d._content._chart.legendShow) {
+                        d._content._chart.legendShow(false);
+                    }
+                })
+            ;
+        },
+
+        doRefreshData: function () {
+            var context = this;
+            this.summaryByKind.filterAll();
+            this.pieCreatorType.dimension.filterAll();
+            this.pieScopeType.dimension.filterAll();
+            this.summaryByScope.filterAll();
+            this.stats.remove();
+
+            WsWorkunits.WUGetStats({
+                request: {
+                    WUID: this.params.Wuid
+                }
+            }).then(function (response) {
+                if (lang.exists("WUGetStatsResponse.Statistics.WUStatisticItem", response)) {
+                    context.stats.add(response.WUGetStatsResponse.Statistics.WUStatisticItem.filter(function (row) {
+                        return row.ScopeType !== "global" && row.Scope !== "Process";
+                    }));
+ 
+                    var kind = context.select.get("value");
+                    context.select.set("options", context.groupByKind.all().map(function (row) {
+                        return { label: row.key, value: row.key, selected: kind === row.key };
+                    }));
+
+                    if (kind) context.summaryByKind.filter(kind);
+                    if (context.pieCreatorType.filter) context.pieCreatorType.dimension.filter(context.pieCreatorType.filter);
+                    if (context.pieScopeType.filter) context.pieScopeType.dimension.filter(context.pieScopeType.filter);
+                    if (context.prevScope) context.summaryByScope.filter(function (d) {
+                        return d.indexOf(context.prevScope + ":") === 0;
+                    });
+                    if (kind === "") {
+                        context.select.set("value", "TimeElapsed");
+                    } else {
+                        context.doRender();
+                    }
+                }
+            });
+        }
+    });
+});

esp/src/eclwatch/WsWorkunits.js

@@ -270,6 +270,10 @@ define([
             });
         },
 
+        WUGetStats: function (params) {
+            return ESPRequest.send("WsWorkunits", "WUGetStats", params);
+        },
+
         //  Stub waiting for HPCC-10308
         visualisations: [
             { value: "DojoD3ScatterChart", label: "Scatter Chart" },

esp/src/eclwatch/dojoConfig.js

@@ -88,6 +88,10 @@ var dojoConfig = (function () {
             location: urlInfo.basePath + "/Visualization/widgets/lib/c3",
             main: "c3"
         }, {
+            name: "crossfilter",
+            location: urlInfo.basePath + "/Visualization/widgets/lib/crossfilter",
+            main: "crossfilter"
+        }, {
             name: "topojson",
             location: urlInfo.basePath + "/Visualization/widgets/lib/topojson"
         }, {

esp/src/eclwatch/nls/hpcc.js

@@ -166,6 +166,7 @@ define({root:
     FileUploadStillInProgress: "File upload still in progress",
     Filter: "Filter",
     Find: "Find",
+    Finished: "Finished",
     FindNext: "Find Next",
     FindPrevious: "Find Previous",
     FirstName: "First Name",
@@ -449,6 +450,7 @@ define({root:
     Started: "Started",
     State: "State",
     Status: "Status",
+    Stats: "Stats",
     Stopped: "Stopped",
     Subgraph: "Subgraph",
     Subgraphs: "Subgraphs",

esp/src/eclwatch/templates/WUStatsWidget.html

@@ -0,0 +1,25 @@
+<div class="${baseClass}">
+    <div id="${id}BorderContainer" class="${baseClass}BorderContainer" style="width: 100%; height: 100%" data-dojo-type="dijit.layout.BorderContainer">
+        <div id="${id}Toolbar" class="topPanel" data-dojo-props="region: 'top'" data-dojo-type="dijit.Toolbar">
+            <div id="${id}Refresh" data-dojo-attach-event="onClick:_onRefresh" data-dojo-props="iconClass:'iconRefresh'" data-dojo-type="dijit.form.Button">${i18n.Refresh}</div>
+            <span data-dojo-type="dijit.ToolbarSeparator"></span>
+            <select id="${id}Kind" data-dojo-props="iconClass:'iconRefresh'" data-dojo-type="dijit.form.Select">
+            </select>
+            <div id="${id}Reset" data-dojo-attach-event="onClick:_onReset" data-dojo-type="dijit.form.Button">${i18n.Reset}</div>
+            <span data-dojo-type="dijit.ToolbarSeparator"></span>
+            <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
+        </div>
+        <div style="padding:0px; overflow:hidden" data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.BorderContainer">
+            <div id="${id}Scope" style="padding:0px; overflow:hidden" data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
+            </div>
+            <div style="width: 50%; padding:0px; overflow:hidden" data-dojo-props="region: 'left'" data-dojo-type="dijit.layout.BorderContainer">
+                <div id="${id}CreatorType" style="padding:0px; overflow:hidden" data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
+                </div>
+                <div id="${id}ScopeType" style="height: 50%; padding:0px; overflow:hidden" data-dojo-props="region: 'bottom'" data-dojo-type="dijit.layout.ContentPane">
+                </div>
+            </div>
+        </div>
+        <div id="${id}Stats" style="height: 50%; padding:0px; overflow:hidden" data-dojo-props="region: 'bottom'" data-dojo-type="dijit.layout.ContentPane">
+        </div>
+    </div>
+</div>

esp/src/eclwatch/viz/DojoD32DChart.js

@@ -38,7 +38,7 @@ define([
 
         renderTo: function (_target) {
             this.chart = new MultiChartSurface()
-                .chartType(this._chartType)
+                .chart_type(this._chartType)
                 .target(_target.domNodeID)
             ;
             if (this.chart.show_title) {

plugins/workunitservices/workunitservices.cpp

@@ -723,7 +723,6 @@ WORKUNITSERVICES_API void wsWorkunitTimings( ICodeContext *ctx, size32_t & __len
     if (wu)
     {
         StatisticsFilter filter;
-        filter.setMergeSources(false);
         filter.setScopeDepth(1, 2);
         filter.setMeasure(SMeasureTimeNs);
 
@@ -823,7 +822,7 @@ WORKUNITSERVICES_API IRowStream * wsWorkunitStatistics( ICodeContext *ctx, IEngi
     //Filter needs to be allocated because the iterator outlasts it.
     Owned<StatisticsFilter> filter = new StatisticsFilter(filterText);
     if (!includeActivities)
-        filter->setMergeSources(false);
+        filter->setScopeDepth(1, 2);
     Owned<IConstWUStatisticIterator> stats = &wu->getStatistics(filter);
     return new StreamedStatistics(wu, allocator, stats);
 }

roxie/ccd/ccd.hpp

@@ -148,10 +148,11 @@ private:
     RoxiePacketHeader(const RoxiePacketHeader &source);
 
 public:
-    unsigned short packetlength;
+    unsigned packetlength;
     unsigned short retries;         // how many retries on this query, the high bits are used as flags, see above
     unsigned short overflowSequence;// Used if more than one packet-worth of data from server - eg keyed join. We don't mind if we wrap...
     unsigned short continueSequence;// Used if more than one chunk-worth of data from slave. We don't mind if we wrap 
+    unsigned short channel;         // multicast family to send on
     unsigned activityId;            // identifies the helper factory to be used (activityId in graph)
     hash64_t queryHash;             // identifies the query
 
@@ -160,7 +161,6 @@ public:
 #ifdef TIME_PACKETS
     unsigned tick;
 #endif
-    unsigned short channel;         // multicast family to send on
 
     inline RoxiePacketHeader(const RemoteActivityId &_remoteId, ruid_t _uid, unsigned _channel, unsigned _overflowSequence)
     {

roxie/ccd/ccdqueue.cpp

@@ -429,7 +429,7 @@ public:
 
 extern IRoxieQueryPacket *createRoxiePacket(void *_data, unsigned _len)
 {
-    if ((unsigned short)_len != _len)
+    if ((unsigned short)_len != _len && !localSlave)
     {
         StringBuffer s;
         RoxiePacketHeader *header = (RoxiePacketHeader *) _data;

system/jlib/jstats.cpp

@@ -1526,6 +1526,16 @@ bool ScopedItemFilter::match(const char * search) const
 }
 
 
+bool ScopedItemFilter::recurseChildScopes(const char * curScope) const
+{
+    if (maxDepth == 0 || !curScope)
+        return true;
+
+    if (queryStatisticsDepth(curScope) >= maxDepth)
+        return false;
+    return true;
+}
+
 void ScopedItemFilter::set(const char * _value)
 {
     if (_value && *_value && !streq(_value, "*") )
@@ -1602,7 +1612,6 @@ StatisticsFilter::StatisticsFilter(StatisticCreatorType _creatorType, const char
 
 void StatisticsFilter::init()
 {
-    mergeSources = true;
     creatorType = SCTall;
     scopeType = SSTall;
     measure = SMeasureAll;
@@ -1626,6 +1635,12 @@ bool StatisticsFilter::matches(StatisticCreatorType curCreatorType, const char *
     return true;
 }
 
+bool StatisticsFilter::recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const
+{
+    return scopeFilter.recurseChildScopes(curScope);
+}
+
+
 void StatisticsFilter::set(const char * creatorTypeText, const char * scopeTypeText, const char * kindText)
 {
     StatisticCreatorType creatorType = queryCreatorType(creatorTypeText);
@@ -1693,11 +1708,6 @@ void StatisticsFilter::setMeasure(StatisticMeasure _measure)
     measure = _measure;
 }
 
-void StatisticsFilter::setMergeSources(bool _value)
-{
-    mergeSources = _value;
-}
-
 void StatisticsFilter::setKind(StatisticKind _kind)
 {
     kind = _kind;

system/jlib/jstats.h

@@ -286,8 +286,7 @@ interface IStatisticsFilter : public IInterface
 {
 public:
     virtual bool matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind) const = 0;
-    //These are a bit arbitrary...
-    virtual bool queryMergeSources() const = 0;
+    virtual bool recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const = 0;
     virtual const char * queryScope() const = 0;
 
 };
@@ -345,6 +344,7 @@ public:
 
     bool match(const char * search) const;
     bool matchDepth(unsigned low, unsigned high) const;
+    bool recurseChildScopes(const char * curScope) const;
 
     const char * queryValue() const { return value ? value.get() : "*"; }
 
@@ -370,7 +370,7 @@ public:
     StatisticsFilter(StatisticCreatorType _creatorType, const char * _creator, StatisticScopeType _scopeType, const char * _scope, StatisticMeasure _measure, StatisticKind _kind);
 
     virtual bool matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind) const;
-    virtual bool queryMergeSources() const { return mergeSources && scopeFilter.matchDepth(2,0); }
+    virtual bool recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const;
     virtual const char * queryScope() const { return scopeFilter.queryValue(); }
 
     void set(const char * _creatorTypeText, const char * _scopeTypeText, const char * _kindText);
@@ -387,7 +387,6 @@ public:
     void setKind(StatisticKind _kind);
     void setKind(const char * _kind);
     void setMeasure(StatisticMeasure _measure);
-    void setMergeSources(bool _value);      // set to false for legacy timing semantics
 
 protected:
     void init();
@@ -399,7 +398,6 @@ protected:
     StatisticKind kind;
     ScopedItemFilter creatorFilter;
     ScopedItemFilter scopeFilter;
-    bool mergeSources;
 };
 
 //---------------------------------------------------------------------------------------------------------------------

testing/regress/ecl/key/keydiff.xml

@@ -1,2 +1 @@
-<Dataset name=''>
-</Dataset>
+

testing/regress/ecl/key/keydiff1.xml

@@ -6,9 +6,7 @@
 </Dataset>
 <Dataset name='Result 4'>
 </Dataset>
-<Dataset name=''>
-</Dataset>
-<Dataset name=''>
+<Dataset name='Result 6'>
 </Dataset>
 <Dataset name='Result 7'>
  <Row><Result_7>0</Result_7></Row>

testing/regress/ecl/key/setup_fetch.xml

@@ -10,3 +10,5 @@
 </Dataset>
 <Dataset name='Result 6'>
 </Dataset>
+<Dataset name='Result 7'>
+</Dataset>

testing/regress/ecl/keydiff.ecl

@@ -22,7 +22,7 @@
 
 //class=file
 //class=index
-//version multiPart=false
+//noversion multiPart=false   // This should be supported but hthor adds meta information into the index that key diff doesn't support
 //version multiPart=true
 //version multiPart=true,useLocal=true
 
@@ -39,6 +39,6 @@ import $.setup;
 Files := setup.Files(multiPart, useLocal, useTranslation);
 
 
-KEYDIFF(Files.DG_FetchIndex1, Files.DG_FetchIndex2, Files.DG_FetchIndexDiffName, OVERWRITE);
+KEYDIFF(Files.DG_KeyDiffIndex1, Files.DG_KeyDiffIndex2, Files.DG_FetchIndexDiffName, OVERWRITE);
 
 

testing/regress/ecl/setup/files.ecl

@@ -58,9 +58,10 @@ EXPORT DG_FetchFileName     := '~REGRESS::' + filePrefix + '::C.DG_FetchFile';
 EXPORT DG_FetchFilePreloadName := '~REGRESS::' + filePrefix + '::C.DG_FetchFilePreload';
 EXPORT DG_FetchFilePreloadIndexedName := '~REGRESS::' + filePrefix + '::C.DG_FetchFilePreloadIndexed';
 EXPORT DG_FetchIndex1Name   := '~REGRESS::' + indexPrefix + '::DG_FetchIndex1';
-EXPORT DG_FetchIndex2Name   := '~REGRESS::' + indexPrefix + '::DG_FetchIndex2';
 EXPORT DG_FetchTransIndexName   := '~REGRESS::' + indexPrefix + '::DG_FetchTransIndex';
 EXPORT DG_FetchIndexDiffName:= '~REGRESS::' + indexPrefix + '::DG_FetchIndexDiff';
+EXPORT DG_KeyDiffIndex1Name   := '~REGRESS::' + indexPrefix + '::DG_KeyDiffIndex1';
+EXPORT DG_KeyDiffIndex2Name   := '~REGRESS::' + indexPrefix + '::DG_KeyDiffIndex2';
 
 EXPORT DG_DsFilename        := '~REGRESS::' + filePrefix + '::SerialLibraryDs';
 EXPORT DG_DictFilename      := '~REGRESS::' + filePrefix + '::SerialLibraryDict';
@@ -84,9 +85,10 @@ EXPORT DG_FetchFilePreloadIndexed := PRELOAD(DATASET(DG_FetchFilePreloadIndexedN
 
 
 EXPORT DG_FetchIndex1 := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname, __filepos},DG_FetchIndex1Name);
-//This version of the index is only used for KEYDIFF
 
-EXPORT DG_FetchIndex2 := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname, __filepos},DG_FetchIndex2Name);
+//These versions of the index are only used for KEYDIFF
+EXPORT DG_KeyDiffIndex1 := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield := fname, __filepos},DG_KeyDiffIndex1Name);
+EXPORT DG_KeyDiffIndex2 := INDEX(DG_KeyDiffIndex1, DG_KeyDiffIndex2Name);
 
 //This version is used for testing reading from a file requiring translation 
 EXPORT DG_FetchTransIndex := INDEX(DG_FetchFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname, __filepos},DG_FetchTransIndexName);

testing/regress/ecl/setup/setup_fetch.ecl

@@ -58,7 +58,13 @@ sortedFile := SORT(Files.DG_FETCHFILE, Lname,Fname,state ,__filepos, LOCAL);
 BUILDINDEX(sortedFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname+lname, __filepos},Files.DG_FetchIndex1Name, OVERWRITE, SORTED);
 
 //This is only used to perform a keydiff.
-BUILDINDEX(sortedFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname+lname, __filepos},Files.DG_FetchIndex2Name, OVERWRITE, SORTED);
+Files.DG_KeyDiffIndex1 createDiffRow(sortedFile l) := TRANSFORM
+    SELF.tfn := TRIM(l.Fname);
+    SELF.blobfield := l.fname+l.lname;
+    SELF := l;
+END;
+BUILDINDEX(Files.DG_KeyDiffIndex1, PROJECT(sortedFile, createDiffRow(LEFT)),OVERWRITE);
+BUILDINDEX(Files.DG_KeyDiffIndex2, PROJECT(sortedFile(lname != 'Doe'),createDiffRow(LEFT)), OVERWRITE);
 
 //A version of the index with LName/FName transposed and x moved to the front.
 BUILDINDEX(sortedFile,{Fname,Lname},{STRING100 blobfield {blob}:= fname+lname, STRING tfn := TRIM(Fname), state, __filepos},Files.DG_FetchTransIndexName, OVERWRITE);
@@ -68,15 +74,18 @@ fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchFilePrel
 
 fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchIndex1Name, '', '', 'view', '1:1', false);
 fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchIndex1Name, '__fileposition__', '__filepos', 'link', '1:1', true);
-fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchIndex2Name, '', '', 'view', '1:1', false);
-fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_FetchIndex2Name, '__fileposition__', '__filepos', 'link', '1:1', true);
+fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_KeyDiffIndex1Name, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_KeyDiffIndex1Name, '__fileposition__', '__filepos', 'link', '1:1', true);
+fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_KeyDiffIndex2Name, '', '', 'view', '1:1', false);
+fileServices.AddFileRelationship( Files.DG_FetchFileName, Files.DG_KeyDiffIndex2Name, '__fileposition__', '__filepos', 'link', '1:1', true);
 
 //Optionally Create local versions of the indexes.
 LocalFiles := $.Files(createMultiPart, TRUE);
 IF (createMultiPart,
     PARALLEL(
         BUILDINDEX(sortedFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname+lname, __filepos},LocalFiles.DG_FetchIndex1Name, OVERWRITE, SORTED, NOROOT);
-        BUILDINDEX(sortedFile,{Lname,Fname},{STRING tfn := TRIM(Fname), state, STRING100 blobfield {blob}:= fname+lname, __filepos},LocalFiles.DG_FetchIndex2Name, OVERWRITE, SORTED, NOROOT);
+        BUILDINDEX(LocalFiles.DG_KeyDiffIndex1, PROJECT(sortedFile, createDiffRow(LEFT)),OVERWRITE,NOROOT);
+        BUILDINDEX(LocalFiles.DG_KeyDiffIndex2, PROJECT(sortedFile(lname != 'Doe'),createDiffRow(LEFT)), OVERWRITE,NOROOT);
         BUILDINDEX(sortedFile,{Fname,Lname},{STRING100 blobfield {blob}:= fname+lname, STRING tfn := TRIM(Fname), state, __filepos},LocalFiles.DG_FetchTransIndexName, OVERWRITE, NOROOT);
    )
 );

testing/regress/ecl/setup/thor/setup_fetch.xml

@@ -16,3 +16,7 @@
 </Dataset>
 <Dataset name='Result 9'>
 </Dataset>
+<Dataset name='Result 10'>
+</Dataset>
+<Dataset name='Result 11'>
+</Dataset>

testing/regress/hpcc/common/shell.py

@@ -55,8 +55,10 @@ class Shell:
             _args, stdout = PIPE, stderr = PIPE, close_fds = True, **kwargs)
         stdout, stderr = process.communicate()
         retCode = process.returncode
-        logging.debug("Shell _run retCode:: %d, stdout:'%s', stderr:'%s'",  retCode,  stdout,  stderr)
-        if retCode or len(stderr) > 0:
+        logging.debug("Shell _run retCode: %d",  retCode)
+        logging.debug("            stdout:'%s'",  stdout)
+        logging.debug("            stderr:'%s'",  stderr)
+        if retCode or ((len(stderr) > 0) and ('Error' in stderr)):
             exception = CalledProcessError(
                 process.returncode, repr(args))
             exception.output = ''.join(filter(None, [stdout, stderr]))

thorlcr/activities/keydiff/thkeydiff.cpp

@@ -159,7 +159,7 @@ public:
     {
         Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
         Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
-        r->setResultStatus(ResultStatusCalculated);
+        //Do not mark the result as calculated - because the patch file isn't a valid result
         r->setResultLogicalName(outputName);
         r.clear();
         wu.clear();

thorlcr/activities/keyedjoin/thkeyedjoin.cpp

@@ -26,9 +26,10 @@
 class CKeyedJoinMaster : public CMasterActivity
 {
     IHThorKeyedJoinArg *helper;
+    Owned<IFileDescriptor> dataFileDesc;
     Owned<CSlavePartMapping> dataFileMapping;
     MemoryBuffer offsetMapMb, initMb;
-    bool localKey;
+    bool localKey, remoteDataFiles;
     unsigned numTags;
     mptag_t tags[4];
     ProgressInfoArray progressInfoArr;
@@ -58,6 +59,7 @@ public:
         numTags = 0;
         tags[0] = tags[1] = tags[2] = tags[3] = TAG_NULL;
         reInit = 0 != (helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename));
+        remoteDataFiles = false;
     }
     ~CKeyedJoinMaster()
     {
@@ -208,7 +210,7 @@ public:
                         {
                             if (superIndex)
                                 throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported");
-                            Owned<IFileDescriptor> dataFileDesc = getConfiguredFileDescriptor(*dataFile);
+                            dataFileDesc.setown(getConfiguredFileDescriptor(*dataFile));
                             void *ekey;
                             size32_t ekeylen;
                             helper->getFileEncryptKey(ekeylen,ekey);
@@ -225,12 +227,26 @@ public:
                             }
                             else if (encrypted)
                                 throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", dataFile->queryLogicalName());
-                            unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0);
-                            if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves())
-                                dataReadWidth = container.queryJob().querySlaves();
-                            Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth);
-                            dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL));
-                            dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear());
+
+                            /* If fetch file is local to cluster, fetches are sent to be processed to local node, each node has info about it's
+                             * local parts only.
+                             * If fetch file is off cluster, fetches are performed by requesting node directly on fetch part, therefore each nodes
+                             * needs all part descriptors.
+                             */
+                            remoteDataFiles = false;
+                            RemoteFilename rfn;
+                            dataFileDesc->queryPart(0)->getFilename(0, rfn);
+                            if (!rfn.queryIP().ipequals(container.queryJob().querySlaveGroup().queryNode(0).endpoint()))
+                                remoteDataFiles = true;
+                            if (!remoteDataFiles) // local to cluster
+                            {
+                                unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0);
+                                if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves())
+                                    dataReadWidth = container.queryJob().querySlaves();
+                                Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth);
+                                dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL));
+                                dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear());
+                            }
                         }
                         else
                             indexFile.clear();
@@ -258,8 +274,19 @@ public:
             IDistributedFile *dataFile = queryReadFile(1);
             if (dataFile)
             {
-                dataFileMapping->serializeMap(slave, dst);
-                dst.append(offsetMapMb);
+                dst.append(remoteDataFiles);
+                if (remoteDataFiles)
+                {
+                    UnsignedArray parts;
+                    parts.append((unsigned)-1); // weird convention meaning all
+                    dst.append(dataFileDesc->numParts());
+                    dataFileDesc->serializeParts(dst, parts);
+                }
+                else
+                {
+                    dataFileMapping->serializeMap(slave, dst);
+                    dst.append(offsetMapMb);
+                }
             }
             else
             {

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -531,7 +531,7 @@ class CKeyedJoinSlave : public CSlaveActivity, public CThorDataLink, implements
     CPartDescriptorArray indexParts, dataParts;
     Owned<IKeyIndexSet> tlkKeySet, partKeySet;
     IThorDataLink *input;
-    bool preserveGroups, preserveOrder, eos, inputStopped, needsDiskRead, atMostProvided, dataRemote;
+    bool preserveGroups, preserveOrder, eos, inputStopped, needsDiskRead, atMostProvided, remoteDataFiles;
     unsigned joinFlags, abortLimit, parallelLookups, freeQSize, filePartTotal;
     size32_t fixedRecordSize;
     CJoinGroupPool *pool;
@@ -545,8 +545,8 @@ class CKeyedJoinSlave : public CSlaveActivity, public CThorDataLink, implements
     OwnedConstThorRow defaultRight;
     unsigned portbase, node;
     IArrayOf<IDelayedFile> fetchFiles;
-    FPosTableEntry *fPosToNodeMap; // maps fpos->node for all parts of logical file
-    FPosTableEntry *fPosToLocalPartMap; // maps fpos->local part #
+    FPosTableEntry *localFPosToNodeMap; // maps fpos->local part #
+    FPosTableEntry *globalFPosToNodeMap; // maps fpos->node for all parts of file. If file is remote, localFPosToNodeMap will have all parts
     unsigned pendingGroups, superWidth;
     Semaphore pendingGroupSem;
     CriticalSection pendingGroupCrit, statCrit, lookupCrit;
@@ -799,13 +799,13 @@ class CKeyedJoinSlave : public CSlaveActivity, public CThorDataLink, implements
                                         if (isLocalFpos(fpos))
                                             localFpos = getLocalFposOffset(fpos);
                                         else
-                                            localFpos = fpos-owner.fPosToLocalPartMap[0].base;
+                                            localFpos = fpos-owner.localFPosToNodeMap[0].base;
                                         break;
                                     }
                                     default:
                                     {
                                         // which of multiple parts this slave is dealing with.
-                                        FPosTableEntry *result = (FPosTableEntry *)bsearch(&fpos, owner.fPosToLocalPartMap, files, sizeof(FPosTableEntry), partLookup);
+                                        FPosTableEntry *result = (FPosTableEntry *)bsearch(&fpos, owner.localFPosToNodeMap, files, sizeof(FPosTableEntry), partLookup);
                                         if (isLocalFpos(fpos))
                                             localFpos = getLocalFposOffset(fpos);
                                         else
@@ -949,17 +949,17 @@ class CKeyedJoinSlave : public CSlaveActivity, public CThorDataLink, implements
                 totalSz = 0;
             }
             unsigned dstNode;
-            if (owner.dataRemote)
+            if (owner.remoteDataFiles)
                 dstNode = owner.node; // JCSMORE - do directly
             else
             {
                 if (1 == owner.filePartTotal)
-                    dstNode = owner.fPosToNodeMap[0].index;
+                    dstNode = owner.globalFPosToNodeMap[0].index;
                 else if (isLocalFpos(fpos))
                     dstNode = getLocalFposPart(fpos);
                 else
                 {
-                    const void *result = bsearch(&fpos, owner.fPosToNodeMap, owner.filePartTotal, sizeof(FPosTableEntry), slaveLookup);
+                    const void *result = bsearch(&fpos, owner.globalFPosToNodeMap, owner.filePartTotal, sizeof(FPosTableEntry), slaveLookup);
                     if (!result)
                         throw MakeThorException(TE_FetchOutOfRange, "FETCH: Offset not found in offset table; fpos=%" I64F "d", fpos);
                     dstNode = ((FPosTableEntry *)result)->index;
@@ -1583,10 +1583,10 @@ public:
         additionalStats = 0;
         lastSeeks = lastScans = 0;
         onFailTransform = localKey = keyHasTlk = false;
-        dataRemote = false;
+        remoteDataFiles = false;
         fetchHandler = NULL;
-        fPosToNodeMap = NULL;
-        fPosToLocalPartMap = NULL;
+        globalFPosToNodeMap = NULL;
+        localFPosToNodeMap = NULL;
 
 #ifdef TRACE_USAGE
         unsigned it=0;
@@ -1599,8 +1599,8 @@ public:
     }
     ~CKeyedJoinSlave()
     {
-        delete [] fPosToNodeMap;
-        delete [] fPosToLocalPartMap;
+        delete [] globalFPosToNodeMap;
+        delete [] localFPosToNodeMap;
         while (doneGroups.ordinality())
         {
             CJoinGroup *jg = doneGroups.dequeue();
@@ -1815,8 +1815,8 @@ public:
         rowLimit = (rowcount_t)helper->getRowLimit();
         additionalStats = 5; // (seeks, scans, accepted, prefiltered, postfiltered)
         needsDiskRead = helper->diskAccessRequired();
-        fPosToNodeMap = NULL;
-        fPosToLocalPartMap = NULL;
+        globalFPosToNodeMap = NULL;
+        localFPosToNodeMap = NULL;
         fetchHandler = NULL;
         filePartTotal = 0;
 
@@ -1920,21 +1920,16 @@ public:
             }
             if (needsDiskRead)
             {
+                data.read(remoteDataFiles); // if true, all fetch parts will be serialized
                 unsigned numDataParts;
                 data.read(numDataParts);
-                size32_t offsetMapSz = 0;
                 if (numDataParts)
                 {
                     deserializePartFileDescriptors(data, dataParts);
-                    RemoteFilename rfn;
-                    dataParts.item(0).getFilename(0, rfn);
-                    if (!rfn.queryIP().ipequals(container.queryJob().queryJobGroup().queryNode(0).endpoint()))
-                        dataRemote = true;
-
-                    fPosToLocalPartMap = new FPosTableEntry[numDataParts];
+                    localFPosToNodeMap = new FPosTableEntry[numDataParts];
                     unsigned f;
                     FPosTableEntry *e;
-                    for (f=0, e=&fPosToLocalPartMap[0]; f<numDataParts; f++, e++)
+                    for (f=0, e=&localFPosToNodeMap[0]; f<numDataParts; f++, e++)
                     {
                         IPartDescriptor &part = dataParts.item(f);
                         e->base = part.queryProperties().getPropInt64("@offset");
@@ -1942,13 +1937,22 @@ public:
                         e->index = f; // NB: index == which local part in dataParts
                     }
                 }
-                data.read(filePartTotal);
-                if (filePartTotal)
+                if (remoteDataFiles) // global offset map not needed if remote and have all fetch parts inc. map (from above)
                 {
-                    data.read(offsetMapSz);
-                    fPosToNodeMap = new FPosTableEntry[filePartTotal];
-                    const void *offsetMapBytes = (FPosTableEntry *)data.readDirect(offsetMapSz);
-                    memcpy(fPosToNodeMap, offsetMapBytes, offsetMapSz);       
+                    if (numDataParts)
+                        filePartTotal = numDataParts;
+                }
+                else
+                {
+                    data.read(filePartTotal);
+                    if (filePartTotal)
+                    {
+                        size32_t offsetMapSz = 0;
+                        data.read(offsetMapSz);
+                        globalFPosToNodeMap = new FPosTableEntry[filePartTotal];
+                        const void *offsetMapBytes = (FPosTableEntry *)data.readDirect(offsetMapSz);
+                        memcpy(globalFPosToNodeMap, offsetMapBytes, offsetMapSz);
+                    }
                 }
                 unsigned encryptedKeyLen;
                 void *encryptedKey;
@@ -1976,6 +1980,8 @@ public:
                 fetchOutputRowIf.setown(createRowInterfaces(fetchOutputMeta,queryActivityId(),queryCodeContext()));
 
                 fetchHandler = new CKeyedFetchHandler(*this);
+
+                FPosTableEntry *fPosToNodeMap = globalFPosToNodeMap ? globalFPosToNodeMap : localFPosToNodeMap;
                 unsigned c;
                 for (c=0; c<filePartTotal; c++)
                 {