Merge branch 'candidate-5.0.4'

Conflicts:
	version.cmake

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/package.cpp

@@ -80,6 +80,11 @@ ISimpleSuperFileEnquiry *CPackageNode::resolveSuperFile(const char *superFileNam
     return NULL;
 }
 
+void CPackageNode::checkPreload()
+{
+    // Only implemented in derived classes
+}
+
 // Load mergedEnvironment from local XML node
 void CPackageNode::loadEnvironment()
 {

common/workunit/package.h

@@ -39,6 +39,7 @@ interface IHpccPackage : extends IInterface
     virtual const char *queryEnv(const char *varname) const = 0;
     virtual bool getEnableFieldTranslation() const = 0;
     virtual bool isCompulsory() const = 0;
+    virtual bool isPreload() const = 0;
     virtual const IPropertyTree *queryTree() const = 0;
     virtual hash64_t queryHash() const = 0;
     virtual const char *queryId() const = 0;

common/workunit/pkgimpl.hpp

@@ -106,6 +106,11 @@ protected:
         return (node) ? node->getPropBool("@compulsory", false) : false;
     }
 
+    virtual bool isPreload() const
+    {
+        return (node) ? node->getPropBool("@preload", false) : false;
+    }
+
     virtual bool resolveLocally() const
     {
         if (isCompulsory())
@@ -162,6 +167,8 @@ public:
         return node;
     }
 
+    virtual void checkPreload();
+
     virtual bool validate(StringArray &warn, StringArray &err) const;
 };
 
@@ -232,6 +239,7 @@ public:
                 }
                 while(baseIterator->next());
             }
+            TYPE::checkPreload();
             baseResolution=basesResolved;
         }
     }

common/workunit/workunit.cpp

@@ -6977,7 +6977,7 @@ IConstWUGraphMetaIterator& CLocalWorkUnit::getGraphsMeta(WUGraphType type) const
             curGraph = &graph;
             SCMStringBuffer graphName;
             curGraph->getName(graphName);
-            if (progressConn->queryRoot())
+            if (progressConn)
                 curGraphProgress.setown(new CConstGraphProgress(wuid, graphName.str(), progressConn->queryRoot()));
         }
     public:

esp/eclwatch/ws_XSLT/index.xslt

@@ -745,7 +745,7 @@
                 <xsl:when test="$warning !=''">
                     <xsl:value-of select="$warning"/>
                 </xsl:when>
-                <xsl:when test="$clusterType = 'DFUserver' or $clusterType = 'ECLCCserver' or $clusterType = 'ECLagent'">
+                <xsl:when test="$clusterType = 'DFUserver' or $clusterType = 'ECLserver' or $clusterType = 'ECLCCserver' or $clusterType = 'ECLagent'">
                     <xsl:choose>
                         <xsl:when test="$queueStatus='paused'"> Queue paused </xsl:when>
                         <xsl:when test="$queueStatus='stopped'"> Queue stopped </xsl:when>

esp/scm/ws_workunits.ecm

@@ -1339,6 +1339,7 @@ ESPrequest WUQueryDetailsRequest
     string QuerySet;
     bool IncludeStateOnClusters(false);
     bool IncludeSuperFiles(false);
+    bool IncludeWsEclAddresses(false);
 };
 
 ESPStruct QuerySuperFile
@@ -1371,6 +1372,7 @@ ESPresponse [exceptions_inline] WUQueryDetailsResponse
     [min_ver("1.46")] int CountGraphs;
     [min_ver("1.46")] ESParray<string> GraphIds;
     [min_ver("1.50")] int ResourceURLCount;
+    [min_ver("1.51")] ESParray<string, Address> WsEclAddresses;
 };
 
 ESPrequest WUMultiQuerySetDetailsRequest

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -1607,6 +1607,61 @@ bool CWsWorkunitsEx::onWUQueryDetails(IEspContext &context, IEspWUQueryDetailsRe
         WsWuInfo winfo(context, wuid);
         resp.setResourceURLCount(winfo.getResourceURLCount());
     }
+    if (req.getIncludeWsEclAddresses())
+    {
+        StringArray wseclAddresses;
+        Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
+        Owned<IConstEnvironment> env = factory->openEnvironment();
+        if (env)
+        {
+            Owned<IPropertyTree> root = &env->getPTree();
+            Owned<IPropertyTreeIterator> services = root->getElements("Software/EspService[Properties/@type='ws_ecl']");
+            StringArray serviceNames;
+            VStringBuffer xpath("Target[@name='%s']", querySet);
+            ForEach(*services)
+            {
+                IPropertyTree &service = services->query();
+                if (!service.hasProp("Target") || service.hasProp(xpath))
+                    serviceNames.append(service.queryProp("@name"));
+            }
+
+            Owned<IPropertyTreeIterator> processes = root->getElements("Software/EspProcess");
+            ForEach(*processes)
+            {
+                StringArray netAddrs;
+                IPropertyTree &process = processes->query();
+                Owned<IPropertyTreeIterator> instances = process.getElements("Instance");
+                ForEach(*instances)
+                {
+                    IPropertyTree &instance = instances->query();
+                    const char *netAddr = instance.queryProp("@netAddress");
+                    if (!netAddr || !*netAddr)
+                        continue;
+                    if (streq(netAddr, "."))
+                        netAddrs.appendUniq(envLocalAddress); //not necessarily local to this server
+                    else
+                        netAddrs.appendUniq(netAddr);
+                }
+                Owned<IPropertyTreeIterator> bindings = process.getElements("EspBinding");
+                ForEach(*bindings)
+                {
+                    IPropertyTree &binding = bindings->query();
+                    const char *srvName = binding.queryProp("@service");
+                    if (!serviceNames.contains(srvName))
+                        continue;
+                    const char *port = binding.queryProp("@port"); //should always be an integer, but we're just concatenating strings
+                    if (!port || !*port)
+                        continue;
+                    ForEachItemIn(i, netAddrs)
+                    {
+                        VStringBuffer wseclAddr("%s:%s", netAddrs.item(i), port);
+                        wseclAddresses.append(wseclAddr);
+                    }
+                }
+            }
+        }
+        resp.setWsEclAddresses(wseclAddresses);
+    }
 
     return true;
 }

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -417,6 +417,17 @@ void CWsWorkunitsEx::init(IPropertyTree *cfg, const char *process, const char *s
     refreshValidClusters();
 
     daliServers.set(cfg->queryProp("Software/EspProcess/@daliServers"));
+    const char *computer = cfg->queryProp("Software/EspProcess/@computer");
+    if (daliServers.isEmpty() || !computer || streq(computer, "localhost")) //otherwise can't assume environment "." netAddresses are the same as my address
+        queryHostIP().getIpText(envLocalAddress);
+    else
+    {
+        //a bit weird, but other netAddresses in the environment are not the same localhost as this server
+        //use the address of the DALI
+        const char *finger = daliServers.get();
+        while (*finger && !strchr(":;,", *finger))
+            envLocalAddress.append(*finger++);
+    }
 
     wuActionTable.setValue("delete", ActionDelete);
     wuActionTable.setValue("abort", ActionAbort);

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -267,6 +267,7 @@ private:
 
     unsigned awusCacheMinutes;
     StringBuffer queryDirectory;
+    StringBuffer envLocalAddress;
     StringAttr daliServers;
     Owned<DataCache> dataCache;
     Owned<ArchivedWuCache> archivedWuCache;

esp/src/eclwatch/ESPUtil.js

@@ -31,11 +31,13 @@ define([
     "dgrid/Keyboard",
     "dgrid/Selection",
     "dgrid/extensions/ColumnResizer",
+    "dgrid/extensions/ColumnHider",
+    "dgrid/extensions/ColumnReorder",
     "dgrid/extensions/DijitRegistry",
     "dgrid/extensions/Pagination"
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, Stateful, query, json,
     registry, Tooltip,
-    Grid, OnDemandGrid, Keyboard, Selection, ColumnResizer, DijitRegistry, Pagination) {
+    Grid, OnDemandGrid, Keyboard, Selection, ColumnResizer, ColumnHider, ColumnReorder, DijitRegistry, Pagination) {
 
     var SingletonData = declare([Stateful], {
         //  Attributes  ---
@@ -259,7 +261,7 @@ define([
             var baseClass = [];
             var params = {};
             if (pagination) {
-                baseClass = [Grid, Pagination, ColumnResizer, Keyboard, DijitRegistry];
+                baseClass = [Grid, Pagination, ColumnResizer, ColumnHider, ColumnReorder, Keyboard, DijitRegistry];
                 lang.mixin(params, {
                     rowsPerPage: 50,
                     pagingLinks: 1,
@@ -268,7 +270,7 @@ define([
                     pageSizeOptions: [25, 50, 100, 1000]
                 });
             } else {
-                baseClass = [OnDemandGrid, ColumnResizer, Keyboard, DijitRegistry];
+                baseClass = [OnDemandGrid, ColumnResizer, ColumnHider, ColumnReorder, Keyboard, DijitRegistry];
             }
             if (selection) {
                 baseClass.push(Selection);

esp/src/eclwatch/GraphPageWidget.js

@@ -347,6 +347,14 @@ define([
             this.xgmmlDialog.show();
         },
 
+        isWorkunit: function () {
+            return lang.exists("params.Wuid", this);
+        },
+
+        isQuery: function () {
+            return lang.exists("params.QueryId", this);
+        },
+
         init: function (params) {
             if (this.inherited(arguments))
                 return;
@@ -367,7 +375,7 @@ define([
                 dotAttrs = dotAttrs.replace("\ngraph[splines=\"line\"];", "\n//graph[splines=\"line\"];");
                 this.global.setDotMetaAttributes(dotAttrs);
             }
-            if (params.Wuid) {
+            if (this.isWorkunit()) {
                 this.graphName = params.GraphName;
                 this.wu = ESPWorkunit.Get(params.Wuid);
 
@@ -390,7 +398,7 @@ define([
                         }
                     });
                 });
-            } else if (params.QueryId) {
+            } else if (this.isQuery()) {
                 this.targetQuery = params.Target;
                 this.queryId = params.QueryId;
                 this.graphName = params.GraphName;
@@ -415,9 +423,9 @@ define([
         },
 
         refreshData: function () {
-            if (lang.exists("params.Wuid", this)) {
+            if (this.isWorkunit()) {
                 this.loadGraphFromWu(this.wu, this.graphName, true);
-            } else if (lang.exists("params.QueryId", this)) {
+            } else if (this.isQuery()) {
                 this.loadGraphFromQuery(this.targetQuery, this.queryId, this.graphName);
             }
         },
@@ -570,7 +578,11 @@ define([
                 },
                 { label: this.i18n.Label, field: "label", width: 150 }
             ];
-            this.verticesStore.appendColumns(columns, ["name"], ["ecl", "definition"]);
+            if (this.isWorkunit()) {
+                this.verticesStore.appendColumns(columns, ["name"], ["ecl", "definition"]);
+            } else if (this.isQuery()) {
+                this.verticesStore.appendColumns(columns, ["localTime", "totalTime", "label", "ecl"], ["definition"]);
+            }
             this.verticesGrid.set("columns", columns);
             this.verticesGrid.refresh();
         },

esp/src/eclwatch/GraphTreeWidget.js

@@ -358,6 +358,14 @@ define([
             this.xgmmlDialog.show();
         },
 
+        isWorkunit: function () {
+            return lang.exists("params.Wuid", this);
+        },
+
+        isQuery: function () {
+            return lang.exists("params.QueryId", this);
+        },
+
         init: function (params) {
             if (this.inherited(arguments))
                 return;
@@ -372,7 +380,7 @@ define([
                 dotAttrs = dotAttrs.replace("\ngraph[splines=\"line\"];", "\n//graph[splines=\"line\"];");
                 this.global.setDotMetaAttributes(dotAttrs);
             }
-            if (params.Wuid) {
+            if (this.isWorkunit()) {
                 this.graphName = params.GraphName;
                 this.wu = ESPWorkunit.Get(params.Wuid);
 
@@ -395,7 +403,7 @@ define([
                         }
                     });
                 });
-            } else if (params.QueryId) {
+            } else if (this.isQuery()) {
                 this.targetQuery = params.Target;
                 this.queryId = params.QueryId;
                 this.graphName = params.GraphName;
@@ -420,9 +428,9 @@ define([
         },
 
         refreshData: function () {
-            if (lang.exists("params.Wuid", this)) {
+            if (this.isWorkunit()) {
                 this.loadGraphFromWu(this.wu, this.graphName, true);
-            } else if (lang.exists("params.QueryId", this)) {
+            } else if (this.isQuery()) {
                 this.loadGraphFromQuery(this.targetQuery, this.queryId, this.graphName);
             }
         },
@@ -557,7 +565,11 @@ define([
                     }
                 })
             ];
-            this.treeStore.appendColumns(columns, ["name"], ["DescendantCount", "ecl", "definition", "SubgraphCount", "ActivityCount", "ChildCount", "Depth"], ["label"]);
+            if (this.isWorkunit()) {
+                this.treeStore.appendColumns(columns, ["name"], ["DescendantCount", "ecl", "definition", "SubgraphCount", "ActivityCount", "ChildCount", "Depth"]);
+            } else if (this.isQuery()) {
+                this.treeStore.appendColumns(columns, ["localTime", "totalTime", "label", "ecl"], ["DescendantCount", "definition", "SubgraphCount", "ActivityCount", "ChildCount", "Depth"]);
+            }
             this.treeGrid.set("query", {id:"0"});
             this.treeGrid.set("columns", columns);
             this.treeGrid.refresh();

esp/src/eclwatch/css/hpcc.css

@@ -712,6 +712,10 @@ margin-left:-20px;
     width: 64em;
 }
 
+.dijitPlaceHolder {
+    pointer-events: none;
+}
+
 .smallForm .dijitValidationTextBoxLabel {
     width: auto;
 }

esp/src/eclwatch/stub.js

@@ -64,12 +64,22 @@ define([
                                 Source: topic.Source
                             }, _item);
 
+                            var clipped = false;
+                            if (item.Message) {
+                                var MAX_LINES = 10;
+                                if (item.Message.length > MAX_LINES * 80) { 
+                                    item.Message = item.Message.substr(0, length = MAX_LINES * 80);
+                                    item.Message += "...";
+                                    clipped = true;
+                                }
+                            }
+
                             if ((item.Source === "WsWorkunits.WUInfo" && item.Code === 20080) ||
                                 (item.Source === "WsWorkunits.WUQuery" && item.Code === 20081) ||
                                 (item.Source === "FileSpray.GetDFUWorkunit" && item.Code === 20080) ||
                                 (item.Source === "WsDfu.DFUInfo" && item.Code === 20038)) {
                             } else {
-                                var message = "<h4>" + entities.encode(item.Source) + "</h4><p>" + entities.encode(item.Message) + "</p>";
+                                var message = "<h4>" + entities.encode(item.Source) + "</h4><p>" + entities.encode(item.Message) + (clipped ? "<br>...  ...  ..." : "") + "</p>";
                                 myToaster.setContent(message, item.Severity, item.Severity === "Error" ? -1 : null);
                                 myToaster.show();
                             }

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -122,7 +122,6 @@
                                     <legend>${i18n.Options}</legend>
                                     <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                                         <select id="${id}xmlsourceFormat" title="${i18n.Format}:" name="sourceFormat" colspan="2" data-dojo-type="dijit.form.Select">
-                                            <option value="1">ASCII</option>
                                             <option value="2">UTF-8</option>
                                             <option value="3">UTF-8N</option>
                                             <option value="4">UTF-16</option>

roxie/ccd/ccdfile.cpp

@@ -1849,7 +1849,7 @@ public:
     {
         return isSuper;
     }
-    inline bool isKey() const
+    virtual bool isKey() const
     {
         return fileType==ROXIE_KEY;
     }
@@ -2011,21 +2011,24 @@ public:
             }
 
             IDefRecordMeta *thisDiskMeta = diskMeta.item(subFile);
-            if (fdesc && thisDiskMeta && activityMeta && !thisDiskMeta->equals(activityMeta))
-                if (allowFieldTranslation)
-                    translators->append(createRecordLayoutTranslator(lfn, thisDiskMeta, activityMeta));
+            if (translators)
+            {
+                if (fdesc && thisDiskMeta && activityMeta && !thisDiskMeta->equals(activityMeta))
+                    if (allowFieldTranslation)
+                        translators->append(createRecordLayoutTranslator(lfn, thisDiskMeta, activityMeta));
+                    else
+                    {
+                        DBGLOG("Key layout mismatch: %s", lfn.get());
+                        StringBuffer q, d;
+                        getRecordMetaAsString(q, activityMeta);
+                        getRecordMetaAsString(d, thisDiskMeta);
+                        DBGLOG("Activity: %s", q.str());
+                        DBGLOG("Disk: %s", d.str());
+                        throw MakeStringException(ROXIE_MISMATCH, "Key layout mismatch detected for index %s", lfn.get());
+                    }
                 else
-                {
-                    DBGLOG("Key layout mismatch: %s", lfn.get());
-                    StringBuffer q, d;
-                    getRecordMetaAsString(q, activityMeta);
-                    getRecordMetaAsString(d, thisDiskMeta);
-                    DBGLOG("Activity: %s", q.str());
-                    DBGLOG("Disk: %s", d.str());
-                    throw MakeStringException(ROXIE_MISMATCH, "Key layout mismatch detected for index %s", lfn.get());
-                }
-            else
-                translators->append(NULL);
+                    translators->append(NULL);
+            }
         }
         CriticalBlock b(lock);
         IKeyArray *ret = keyArrayMap.get(channel);

roxie/ccd/ccdfile.hpp

@@ -111,6 +111,7 @@ interface IResolvedFile : extends ISimpleSuperFileEnquiry
     virtual void remove() = 0;
     virtual bool exists() const = 0;
     virtual bool isSuperFile() const = 0;
+    virtual bool isKey() const = 0;
 };
 
 interface IResolvedFileCreator : extends IResolvedFile

roxie/ccd/ccdstate.cpp

@@ -423,6 +423,9 @@ class CRoxiePackageNode : extends CPackageNode, implements IRoxiePackage
 protected:
     static CResolvedFileCache daliFiles;
     mutable CResolvedFileCache fileCache;
+    IArrayOf<IResolvedFile> files;  // Used when preload set
+    IArrayOf<IKeyArray> keyArrays;  // Used when preload set
+    IArrayOf<IFileIOArray> fileArrays;  // Used when preload set
 
     virtual aindex_t getBaseCount() const = 0;
     virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
@@ -591,6 +594,44 @@ protected:
         return NULL;
     }
 
+    void doPreload(unsigned channel, const IResolvedFile *resolved)
+    {
+        if (resolved->isKey())
+            keyArrays.append(*resolved->getKeyArray(NULL, NULL, false, channel, false));
+        else
+            fileArrays.append(*resolved->getIFileIOArray(false, channel));
+    }
+
+    void checkPreload()
+    {
+        if (isPreload())
+        {
+            // Look through all files and resolve them now
+            Owned<IPropertyTreeIterator> supers = node->getElements("SuperFile");
+            ForEach(*supers)
+            {
+                IPropertyTree &super = supers->query();
+                const char *name = super.queryProp("@id");
+                if (name)
+                {
+                    const IResolvedFile *resolved = lookupFileName(name, false, true, true, NULL);
+                    if (resolved)
+                    {
+                        files.append(*const_cast<IResolvedFile *>(resolved));
+                        doPreload(0, resolved);
+                        Owned<IPropertyTreeIterator> it = ccdChannels->getElements("RoxieSlaveProcess");
+                        ForEach(*it)
+                        {
+                            unsigned channelNo = it->query().getPropInt("@channel", 0);
+                            assertex(channelNo);
+                            doPreload(channelNo, resolved);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     // default constructor for derived class use
     CRoxiePackageNode()
     {
@@ -605,6 +646,9 @@ public:
 
     ~CRoxiePackageNode()
     {
+        keyArrays.kill();
+        fileArrays.kill();
+        files.kill();
         assertex(fileCache.count()==0);
         // If it's possible for cached objects to outlive the cache I think there is a problem...
         // we could set the cache field to null here for any objects still in cache but there would be a race condition
@@ -692,6 +736,10 @@ public:
     {
         return CPackageNode::isCompulsory();
     }
+    virtual bool isPreload() const
+    {
+        return CPackageNode::isPreload();
+    }
     virtual const IPropertyTree *queryTree() const
     {
         return CPackageNode::queryTree();