Merge branch 'candidate-6.0.6'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlhtcpp.cpp

@@ -393,7 +393,7 @@ public:
                 //MORE: For the moment disable any expressions that are only used conditionally.
                 //Often including conditions improves the code, but sometimes the duplicate evaluation of the
                 //guard conditions in the parent and the child causes excessive code generation.
-                //And forcing it into an alias doesn't help because that isn't currently executed in the parent.
+                //And forcing it into an alias doesn't help becuase that isn't currently executed in the parent.
                 //Uncomment: if (moveTo->guards->guardContainsCandidate(expr))
                 {
                     invalid = true;
@@ -708,7 +708,8 @@ protected:
 
                 if (worthHoisting)
                 {
-                    bool createSubQueryBeforeAll = forceRoot;
+                    //MORE: Remove || true to evaluate the subquery at the latest time.
+                    bool createSubQueryBeforeAll = forceRoot || true;
                     spotter.transformAll(pending, createSubQueryBeforeAll);
                     translator.traceExpressions("spotted child", pending);
                 }

esp/src/eclwatch/DFUQueryWidget.js

@@ -511,6 +511,7 @@ define([
             this.workunitsGrid = new declare([ESPUtil.Grid(true, true)])({
                 deselectOnRefresh: true,
                 store: this.listStore,
+                query: this.getFilter(),
                 columns: {
                     col1: selector({
                         width: 27,

esp/src/eclwatch/DiskUsageWidget.js

@@ -20,6 +20,8 @@ define([
     "dojo/i18n!./nls/hpcc",
     "dojo/on",
 
+    "dijit/registry",
+
     "dgrid/selector",
 
     "hpcc/WsDfu",
@@ -40,6 +42,7 @@ define([
     "dijit/form/TimeTextBox"
 
 ], function (declare, lang, i18n, nlsHPCC, on,
+                registry,
                 selector,
                 WsDfu, ESPUtil, FilterDropDownWidget,
                 template) {
@@ -48,6 +51,11 @@ define([
         baseClass: "DiskUsageWidget",
         i18n: nlsHPCC,
 
+        postCreate: function (args) {
+            this.inherited(arguments);
+            this.filter = registry.byId(this.id + "Filter");
+        },
+
         resize: function (args) {
             this.inherited(arguments);
             this.widget.BorderContainer.resize();
@@ -69,12 +77,12 @@ define([
 
             this.initDiskUsageGrid();
 
-            this.widget.Filter.refreshState();
+            this.filter.refreshState();
             var context = this;
-            this.widget.Filter.on("clear", function (evt) {
+            this.filter.on("clear", function (evt) {
                 context.refreshGrid();
             });
-            this.widget.Filter.on("apply", function (evt) {
+            this.filter.on("apply", function (evt) {
                 context.refreshGrid();
             });
         },
@@ -98,7 +106,7 @@ define([
         },
 
         getFilter: function () {
-            var retVal = this.widget.Filter.toObject();
+            var retVal = this.filter.toObject();
             lang.mixin(retVal, {
                 StartDate: this.getISOString("FromDate", "FromTime"),
                 EndDate: this.getISOString("ToDate", "ToTime")

esp/src/eclwatch/FilterDropDownWidget.js

@@ -106,7 +106,30 @@ define([
             if (this.filterDropDown.get("disabled")) {
                 return {};
             }
-            return domForm.toObject(this.filterForm.id);
+            var retVal = {};
+            arrayUtil.forEach(this.filterForm.getDescendants(), function (item, idx) {
+                var name = item.get("name");
+                if (name) {
+                    var value = item.get("value");
+                    if (value) {
+                        retVal[name] = value;
+                    }
+                }
+            });
+            return retVal;
+        },
+
+        fromObject: function (obj) {
+            arrayUtil.forEach(this.filterForm.getDescendants(), function (item, idx) {
+                var value = obj[item.get("name")];
+                if (value) {
+                    item.set("value", value);
+                    if (item.defaultValue !== undefined) {
+                        item.defaultValue = value;
+                    }
+                }
+            });
+            this.refreshState();
         },
 
         init: function (params) {

esp/src/eclwatch/GetDFUWorkunitsWidget.js

@@ -356,6 +356,7 @@ define([
             var store = this.params.searchResults ? this.params.searchResults : new ESPDFUWorkunit.CreateWUQueryStore();
             this.workunitsGrid = new declare([ESPUtil.Grid(true, true)])({
                 store: store,
+                query: this.filter.toObject(),
                 columns: {
                     col1: selector({
                         width: 27,

esp/src/eclwatch/ResultWidget.js

@@ -19,6 +19,7 @@ define([
     "dojo/_base/array",
     "dojo/i18n",
     "dojo/i18n!./nls/hpcc",
+    "dojo/io-query",
     "dojo/dom",
 
     "dijit/registry",
@@ -48,7 +49,7 @@ define([
     "dijit/Toolbar",
     "dijit/form/Button",
     "dijit/ToolbarSeparator"
-], function (declare, lang, arrayUtil, i18n, nlsHPCC, dom,
+], function (declare, lang, arrayUtil, i18n, nlsHPCC, ioQuery, dom,
                 registry, TextBox,
                 Grid, Keyboard, Selection, selector, ColumnResizer, ColumnHider, CompoundColumns, DijitRegistry, Pagination,
                 _Widget, ESPBase, ESPWorkunit, ESPLogicalFile, FilterDropDownWidget, TableContainer,
@@ -70,6 +71,7 @@ define([
         postCreate: function (args) {
             this.inherited(arguments);
             this.borderContainer = registry.byId(this.id + "BorderContainer");
+            this.filter = registry.byId(this.id + "Filter");
             this.grid = registry.byId(this.id + "Grid");
         },
 
@@ -120,6 +122,7 @@ define([
         },
 
         init: function (params) {
+            this.__filter = params.__filter;
             if (this.inherited(arguments))
                 return;
 
@@ -162,26 +165,21 @@ define([
             if (result) {
                 var context = this;
                 result.fetchStructure(function (structure) {
-                    context.grid = new declare([Grid, Pagination, Keyboard, ColumnResizer, ColumnHider, CompoundColumns, DijitRegistry])({
-                        columns: structure,
-                        rowsPerPage: 50,
-                        pagingLinks: 1,
-                        pagingTextBox: true,
-                        firstLastArrows: true,
-                        pageSizeOptions: [25, 50, 100],
-                        store: result.getStore()
-                    }, context.id + "Grid");
-                    context.grid.startup();
-                    var filterForm = registry.byId(context.widget.Filter.id + "FilterForm");
-                    var origTableContainer = registry.byId(context.widget.Filter.id + "TableContainer");
+                    var filterForm = registry.byId(context.filter.id + "FilterForm");
+                    var origTableContainer = registry.byId(context.filter.id + "TableContainer");
                     var tableContainer = new TableContainer({
                     });
+                    var filterObj = {};
+                    if (lang.exists("__filter", context) && lang.exists("filter.toObject", context)) {
+                        filterObj = ioQuery.queryToObject(context.__filter);
+                    }
                     arrayUtil.forEach(structure, function (item, idx) {
                         if (item.label !== "##") {
                             var textBox = new TextBox({
                                 title: item.label,
                                 label: item.label + (item.__hpcc_keyed ? " (i)" : ""),
                                 name: item.field,
+                                value: filterObj[item.field],
                                 colSpan: 2
                             });
                             tableContainer.addChild(textBox);
@@ -189,13 +187,27 @@ define([
                     });
                     tableContainer.placeAt(origTableContainer.domNode, "replace");
                     origTableContainer.destroyRecursive();
-                    context.widget.Filter.on("clear", function (evt) {
+                    context.filter.on("clear", function (evt) {
                         context.refresh();
                     });
-                    context.widget.Filter.on("apply", function (evt) {
+                    context.filter.on("apply", function (evt) {
                         context.refresh();
                     });
+                    context.filter.refreshState();
 
+                    context.grid = new declare([Grid, Pagination, Keyboard, ColumnResizer, ColumnHider, CompoundColumns, DijitRegistry])({
+                        columns: structure,
+                        rowsPerPage: 50,
+                        pagingLinks: 1,
+                        pagingTextBox: true,
+                        firstLastArrows: true,
+                        pageSizeOptions: [25, 50, 100],
+                        store: result.getStore()
+                    }, context.id + "Grid");
+                    context.grid.set("query", {
+                        FilterBy: context.getFilter()
+                    });
+                    context.grid.startup();
                 });
             } else {
                 this.grid = new declare([Grid, DijitRegistry])({
@@ -212,7 +224,7 @@ define([
         },
 
         getFilter: function () {
-            return this.widget.Filter.toObject();
+            return this.filter.toObject();
         },
 
         refresh: function () {

esp/src/eclwatch/TargetSelectClass.js

@@ -39,7 +39,7 @@ define([
     return {
         i18n: nlsHPCC,
 
-        loading: false,
+        loading: true,
         defaultValue: "",
 
         //  Implementation  ---

esp/src/eclwatch/_Widget.js

@@ -1,6 +1,7 @@
 define([
     "dojo/_base/declare",
     "dojo/_base/array",
+    "dojo/_base/lang",
     "dojo/i18n",
     "dojo/i18n!./nls/hpcc",
     "dojo/io-query",
@@ -14,7 +15,7 @@ define([
     "dijit/_WidgetsInTemplateMixin",
     "dijit/registry"
 
-], function (declare, arrayUtil, i18n, nlsHPCC, ioQuery, dom, domConstruct, domAttr, domStyle,
+], function (declare, arrayUtil, lang, i18n, nlsHPCC, ioQuery, dom, domConstruct, domAttr, domStyle,
     _LayoutWidget, _TemplatedMixin, _WidgetsInTemplateMixin, registry) {
 
     //  IE8 textContent polyfill  ---
@@ -54,9 +55,30 @@ define([
             this.registerChildWidgets(this.domNode);
         },
 
+        getFilterParams: function() {
+            var retVal = null;
+            if (lang.exists("filter.toObject", this)) {
+                var obj = this.filter.toObject();
+                for (var key in obj) {
+                    if (!retVal) {
+                        retVal = {};
+                    }
+                    if (obj[key]) {
+                        retVal[key] = obj[key];
+                    }
+                }
+            }
+            return retVal;
+        },
+
         getURL: function() {
             var baseUrl = document.URL.split("#")[0];
             baseUrl = baseUrl.split("?")[0];
+            delete this.params.__filter;
+            var filterParams = this.getFilterParams();
+            if (filterParams) {
+                this.params.__filter = ioQuery.objectToQuery(filterParams);
+            }
             var paramsString = ioQuery.objectToQuery(this.params);
             return baseUrl + "?" + paramsString;
         },
@@ -76,6 +98,10 @@ define([
             if (!this.params.Widget) {
                 this.params.Widget = this.declaredClass;
             }
+            if (lang.exists("params.__filter", this) && lang.exists("filter.toObject", this)) {
+                var filterObj = ioQuery.queryToObject(this.params.__filter);
+                this.filter.fromObject(filterObj);
+            }
             this.wrapInHRef(this.id + "NewPage", this.getURL());
             return false;
         },

rtl/eclrtl/rtlxml.cpp

@@ -38,6 +38,8 @@
 #include "jmd5.hpp"
 #include "rtlqstr.ipp"
 
+#include "nbcd.hpp"
+
 //---------------------------------------------------------------------------
 
 void outputXmlString(unsigned len, const char *field, const char *fieldname, StringBuffer &out)
@@ -95,15 +97,16 @@ void outputXmlReal(double field, const char *fieldname, StringBuffer &out)
 }
 void outputXmlDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname, StringBuffer &out)
 {
-    char dec[50];
     if (fieldname && *fieldname)
         out.append('<').append(fieldname).append('>');
 
-    BcdCriticalBlock bcdBlock;
     if (DecValid(true, size*2-1, field))
     {
-        DecPushDecimal(field, size, precision);
-        DecPopCString(sizeof(dec), dec);
+        Decimal temp;
+        char dec[50];
+        temp.setDecimal(size, precision, field);
+        temp.getCString(sizeof(dec), dec);
+
         const char *finger = dec;
         while(isspace(*finger)) finger++;
         out.append(finger);
@@ -117,15 +120,16 @@ void outputXmlDecimal(const void *field, unsigned size, unsigned precision, cons
 
 void outputXmlUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname, StringBuffer &out)
 {
-    char dec[50];
     if (fieldname && *fieldname)
         out.append('<').append(fieldname).append('>');
 
-    BcdCriticalBlock bcdBlock;
     if (DecValid(false, size*2, field))
     {
-        DecPushUDecimal(field, size, precision);
-        DecPopCString(sizeof(dec), dec);
+        Decimal temp;
+        char dec[50];
+        temp.setUDecimal(size, precision, field);
+        temp.getCString(sizeof(dec), dec);
+
         const char *finger = dec;
         while(isspace(*finger)) finger++;
         out.append(finger);

testing/regress/ecl/decimalnest.ecl

@@ -0,0 +1,30 @@
+
+
+childRec := RECORD
+     DECIMAL7_2 value;
+END;
+
+mainRec := RECORD
+    UDECIMAL8 id;
+    DECIMAL7_2 total;
+    DATASET(childRec) children;
+END;
+
+
+ds := DATASET([
+        {1, 1.0,[1.1,2.3,4.5,6.7]},
+        {2, 3.45,[-1,10.30]}], mainRec);
+
+
+childRec tc(childRec l) := TRANSFORM,SKIP(l.value = 0)  // Skip forces into a child query
+    SELF.value := l.value * 2D;
+END;
+
+mainRec t(mainRec l) := TRANSFORM
+    p := NOFOLD(PROJECT(l.children, tc(LEFT)));
+    SELF.id := l.id * 2;
+    SELF.total := l.total + COUNT(p) + SUM(p, value);
+    SELF.children := p;
+END;
+
+OUTPUT(PROJECT(ds, t(LEFT)));

testing/regress/ecl/key/decimalnest.xml

@@ -0,0 +1,4 @@
+<Dataset name='Result 1'>
+ <Row><id>2</id><total>34.2</total><children><Row><value>2.2</value></Row><Row><value>4.6</value></Row><Row><value>9</value></Row><Row><value>13.4</value></Row></children></Row>
+ <Row><id>4</id><total>24.05</total><children><Row><value>-2</value></Row><Row><value>20.6</value></Row></children></Row>
+</Dataset>

thorlcr/activities/join/thjoinslave.cpp

@@ -544,7 +544,7 @@ public:
         if (noSortPartitionSide())
         {
             partitionRow.setown(primaryInputStream->ungroupedNextRow());
-            primaryStream.set(new cRowStreamPlus1Adaptor(primaryInputStream, partitionRow));
+            primaryStream.setown(new cRowStreamPlus1Adaptor(primaryInputStream, partitionRow));
         }
         else
         {

thorlcr/activities/nsplitter/thnsplitterslave.cpp

@@ -57,7 +57,7 @@ public:
     virtual void setOutputStream(unsigned index, IEngineRowStream *stream) override;
     virtual IStrandJunction *getOutputStreams(CActivityBase &ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override;
     virtual unsigned __int64 queryTotalCycles() const override { return COutputTiming::queryTotalCycles(); }
-    virtual unsigned __int64 queryEndCycles() const { return COutputTiming::queryEndCycles(); }
+    virtual unsigned __int64 queryEndCycles() const override { return COutputTiming::queryEndCycles(); }
     virtual void debugRequest(MemoryBuffer &mb) override;
 // Stepping methods
     virtual IInputSteppingMeta *querySteppingMeta() { return nullptr; }
@@ -340,12 +340,7 @@ public:
     }
 
 // IEngineRowStream
-    virtual const void *nextRow() override
-    {
-        ActivityTimer t(totalCycles, queryTimeActivities());
-        return inputStream->nextRow();
-    }
-    virtual void stop() override{ inputStream->stop(); }
+    virtual void stop() override{ throwUnexpected(); } // CSplitterOutput deals with stopping inputStream
 
 // IThorDataLink (if single output connected)
     virtual IStrandJunction *getOutputStreams(CActivityBase &ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
@@ -444,12 +439,12 @@ void CSplitterOutput::stop()
 
 const void *CSplitterOutput::nextRow()
 {
+    ActivityTimer t(totalCycles, activity.queryTimeActivities());
     if (rec == max)
     {
         max = activity.writeahead(max, activity.queryAbortSoon(), writeBlockSem, outIdx);
         // NB: if this is sole input that actually started, writeahead will have returned RCMAX and calls to activity.nextRow will go directly to splitter input
     }
-    ActivityTimer t(totalCycles, activity.queryTimeActivities());
     const void *row = activity.nextRow(outIdx); // pass ptr to max if need more
     ++rec;
     if (row)

thorlcr/activities/soapcall/thsoapcallslave.cpp

@@ -129,6 +129,7 @@ class SoapDatasetCallSlaveActivity : public CSlaveActivity, implements IWSCRowPr
     typedef CSlaveActivity PARENT;
 
     bool eof;
+    StringBuffer authToken;
     Owned<IWSCHelper> wscHelper;
     CriticalSection crit;
 
@@ -144,9 +145,7 @@ public:
     // IThorSlaveActivity overloaded methods
     virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
-        StringBuffer authToken;
         buildAuthToken(queryJob().queryUserDescriptor(), authToken);
-        wscHelper.setown(createSoapCallHelper(this, queryRowAllocator(), authToken.str(), SCdataset, NULL, queryDummyContextLogger(),NULL));
     }
     // IThorDataLink methods
     virtual void start() override
@@ -154,10 +153,16 @@ public:
         ActivityTimer s(totalCycles, timeActivities);
         PARENT::start();
         eof = false;
+        wscHelper.setown(createSoapCallHelper(this, queryRowAllocator(), authToken.str(), SCdataset, NULL, queryDummyContextLogger(),NULL));
         wscHelper->start();
     }
     virtual void stop() override
     {
+        if (wscHelper)
+        {
+            wscHelper->waitUntilDone();
+            wscHelper.clear();
+        }
         eof = true;
         PARENT::stop();
     }
@@ -215,7 +220,7 @@ public:
 class SoapRowActionSlaveActivity : public ProcessSlaveActivity, implements IWSCRowProvider
 {
     typedef ProcessSlaveActivity PARENT;
-
+    StringBuffer authToken;
     Owned<IWSCHelper> wscHelper;
 
 public:
@@ -229,17 +234,15 @@ public:
     // IThorSlaveActivity overloaded methods
     virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
-        StringBuffer authToken;
         buildAuthToken(queryJob().queryUserDescriptor(), authToken);
-        if (container.queryLocalOrGrouped() || firstNode())
-            wscHelper.setown(createSoapCallHelper(this, NULL, authToken.str(), SCrow, NULL, queryDummyContextLogger(),NULL));
     }
 
     // IThorSlaveProcess overloaded methods
     virtual void process()
     {
-        if (wscHelper)
+        if (container.queryLocalOrGrouped() || firstNode())
         {
+            wscHelper.setown(createSoapCallHelper(this, NULL, authToken.str(), SCrow, NULL, queryDummyContextLogger(),NULL));
             wscHelper->start();
             wscHelper->waitUntilDone();
             IException *e = wscHelper->getError();
@@ -272,6 +275,7 @@ class SoapDatasetActionSlaveActivity : public ProcessSlaveActivity, implements I
     typedef ProcessSlaveActivity PARENT;
 
     Owned<IWSCHelper> wscHelper;
+    StringBuffer authToken;
     CriticalSection crit;
 
 public:
@@ -285,9 +289,7 @@ public:
     // IThorSlaveActivity overloaded methods
     virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
-        StringBuffer authToken;
         buildAuthToken(queryJob().queryUserDescriptor(), authToken);
-        wscHelper.setown(createSoapCallHelper(this, NULL, authToken.str(), SCdataset, NULL, queryDummyContextLogger(),NULL));
     }
 
     // IThorSlaveProcess overloaded methods
@@ -298,6 +300,7 @@ public:
 
         processed = THORDATALINK_STARTED;
 
+        wscHelper.setown(createSoapCallHelper(this, NULL, authToken.str(), SCdataset, NULL, queryDummyContextLogger(),NULL));
         wscHelper->start();
         wscHelper->waitUntilDone();
         IException *e = wscHelper->getError();

thorlcr/activities/wuidread/thwuidread.cpp

@@ -38,7 +38,7 @@ public:
         else
             queryCodeContext()->getResultRaw(lenData, tempData, helper->queryName(), helper->querySequence(), helper->queryXmlTransformer(), helper->queryCsvTransformer());
         msg.clear();
-        msg.append(lenData, tempData);
+        msg.setBuffer(lenData, tempData, true);
         queryJobChannel().queryJobComm().reply(msg);
     }
 };

thorlcr/activities/wuidread/thwuidreadslave.cpp

@@ -36,7 +36,26 @@ class CWuidReadSlaveActivity : public CSlaveActivity
     bool eogPending;
     mptag_t replyTag;
     CMessageBuffer masterReplyMsg;
+    bool resultFetched;
 
+    void getWuidResult()
+    {
+        if (!resultFetched)
+        {
+            resultFetched = true;
+            CMessageBuffer reqMsg;
+            reqMsg.setReplyTag(replyTag);
+            reqMsg.append(smt_actMsg);
+            reqMsg.append(container.queryOwner().queryGraphId());
+            reqMsg.append(container.queryId());
+
+            if (!queryJobChannel().queryJobComm().sendRecv(reqMsg, 0, container.queryJob().querySlaveMpTag(), LONGTIMEOUT))
+                throwUnexpected();
+
+            masterReplyMsg.swapWith(reqMsg);
+        }
+        masterReplyMsg.reset();
+    }
 public:
     CWuidReadSlaveActivity(CGraphElementBase *_container) 
         : CSlaveActivity(_container)
@@ -48,26 +67,15 @@ public:
         grouped = helper->queryOutputMeta()->isGrouped();
         setRequireInitData(false);
         appendOutputLinked(this);
+        resultFetched = false;
     }
     virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
         PARENT::start();
-
         eogPending = false;
         if (container.queryLocal() || firstNode())
-        {
-            CMessageBuffer reqMsg;
-            reqMsg.setReplyTag(replyTag);
-            reqMsg.append(smt_actMsg);
-            reqMsg.append(container.queryOwner().queryGraphId());
-            reqMsg.append(container.queryId());
-
-            if (!queryJobChannel().queryJobComm().sendRecv(reqMsg, 0, container.queryJob().querySlaveMpTag(), LONGTIMEOUT))
-                throwUnexpected();
-
-            masterReplyMsg.swapWith(reqMsg);
-        }
+            getWuidResult();
     }
     CATCH_NEXTROW()
     {

thorlcr/thorutil/thbuf.cpp

@@ -1078,7 +1078,6 @@ public:
         inMemRows.clear();
     }
 
-
     unsigned anyReaderBehind()
     {
         unsigned reader=0;