Merge branch 'candidate-7.12.x' into candidate-8.0.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/getpackagerevisionarch.sh

@@ -85,7 +85,7 @@ elif [ -e /etc/redhat-release ]; then
     OS_GROUP=`/bin/rpm -q --qf "%{NAME}" --whatprovides /etc/redhat-release | sed 's/-release.*//' |  tr '[A-Z]' '[a-z]'`
     REDHAT_VERSION=`/bin/rpm -q --qf "%{VERSION}" --whatprovides /etc/redhat-release`
     case "$OS_GROUP" in
-      "centos" | "fedora")
+      "centos"* | "fedora")
         if [ ${NOARCH} -eq 0 ]; then
             OUTPUT="el${REDHAT_VERSION}.${ARCH}"
         else

dali/base/dasubs.cpp

@@ -130,20 +130,21 @@ public:
 
 static class CDaliPublisher
 {
+protected:
+    std::atomic<bool> running{false};
 public:
     virtual ISubscriptionManager *queryManager(unsigned tag) = 0;
     virtual void stop() = 0;
     virtual ~CDaliPublisher() {}
 } *DaliPublisher;
 
-class CDaliPublisherServer: public IDaliServer, public Thread, implements CDaliPublisher, implements IConnectionMonitor
+class CDaliPublisherServer: public IDaliServer, public Thread, public CDaliPublisher, implements IConnectionMonitor
 {
     ICopyArrayOf<CSubscriptionStub> stubs;
     IArrayOf<ISubscriptionManager> managers;
     UnsignedArray tags;
     CheckedCriticalSection tagsect;
     CheckedCriticalSection stubsect;
-    bool stopped;
     ReadWriteLock processlock;
 public:
     IMPLEMENT_IINTERFACE;
@@ -151,17 +152,17 @@ public:
     CDaliPublisherServer()
         : Thread("CDaliPublisherServer")
     {
-        stopped = true;
     }
 
     ~CDaliPublisherServer()
     {
-        stopped = true;
+        running = false;
         managers.kill();
     }
 
     void start()
     {
+        running = true;
         Thread::start();
     }
     void ready()
@@ -179,8 +180,9 @@ public:
 
     void stop()
     {
-        if (!stopped) {
-            stopped = true;
+        if (running)
+        {
+            running = false;
             queryCoven().cancel(RANK_ALL,MPTAG_DALI_SUBSCRIPTION_REQUEST);
         }
         processlock.unlockWrite();
@@ -192,9 +194,10 @@ public:
         ICoven &coven=queryCoven();
         CMessageHandler<CDaliPublisherServer> handler("CDaliPublisherServer",this,&CDaliPublisherServer::processMessage,NULL, 100);
         CMessageBuffer mb;
-        stopped = false;
-        while (!stopped) {
-            try {
+        while (running)
+        {
+            try
+            {
                 mb.clear();
 #ifdef TRACE_QWAITING
                 unsigned waiting = coven.probe(RANK_ALL,MPTAG_DALI_SUBSCRIPTION_REQUEST,NULL);
@@ -204,7 +207,7 @@ public:
                 if (coven.recv(mb,RANK_ALL,MPTAG_DALI_SUBSCRIPTION_REQUEST,NULL))
                     handler.handleMessage(mb);
                 else
-                    stopped = true;
+                    running = false;
             }
             catch (IException *e)
             {
@@ -218,7 +221,7 @@ public:
     void processMessage(CMessageBuffer &mb)
     {
         ReadLockBlock block(processlock);
-        if (stopped)
+        if (!running)
             return;
         ICoven &coven=queryCoven();
         int fn;
@@ -543,15 +546,13 @@ class CDaliPublisherClient: public Thread, public CDaliPublisher
     CIArrayOf<CDaliSubscriptionManagerStub> managers;
     UnsignedArray tags;
     CheckedCriticalSection tagsect;
-    bool stopped;
-
 
 public:
 
     CDaliPublisherClient()
         :   Thread("CDaliPublisherClient")
     {
-        stopped = true;
+        running = true;
         start();
     }
 
@@ -576,12 +577,12 @@ public:
     {
         ICoven &coven=queryCoven();
         CMessageHandler<CDaliPublisherClient> handler("CDaliPublisherClientMessages",this,&CDaliPublisherClient::processMessage);
-        stopped = false;
         CMessageBuffer mb;
-        stopped = false;
-        while (!stopped) {
+        while (running)
+        {
             mb.clear();
-            try {
+            try
+            {
 #ifdef TRACE_QWAITING
                 unsigned waiting = coven.probe(RANK_ALL,MPTAG_DALI_SUBSCRIPTION_FULFILL,NULL);
                 if ((waiting!=0)&&(waiting%10==0))
@@ -590,12 +591,13 @@ public:
                 if (coven.recv(mb,RANK_ALL,MPTAG_DALI_SUBSCRIPTION_FULFILL,NULL))
                     handler.handleMessage(mb);
                 else
-                    stopped = true;
+                    running = false;
             }
-            catch (IException *e) {
+            catch (IException *e)
+            {
                 EXCLOG(e,"CDaliPublisherClient::run");
                 e->Release();
-                stopped = true;
+                running = false;
             }
         }
         return 0;
@@ -623,16 +625,15 @@ public:
     {
     }
 
-
     void stop()
     {
-        if (!stopped) {
-            stopped = true;
+        if (running)
+        {
+            running = false;
             queryCoven().cancel(RANK_ALL,MPTAG_DALI_SUBSCRIPTION_FULFILL);
         }
         join();
     }
-
 };
 
 

esp/src/eclwatch/LFDetailsWidget.js

@@ -296,7 +296,8 @@ define([
                 } else if (currSel.id === this.dataPatternsWidget.id) {
                     this.dataPatternsWidget.init({
                         NodeGroup: this.logicalFile.NodeGroup,
-                        LogicalName: this.logicalFile.Name
+                        LogicalName: this.logicalFile.Name,
+                        Modified: this.logicalFile.Modified
                     });
                 } else if (currSel.id === this.sourceWidget.id) {
                     this.sourceWidget.init({

esp/src/eclwatch/ResultWidget.js

@@ -113,18 +113,22 @@ define([
         },
 
         _onDataPatterns: function (args) {
+            this.updateDataPatterns();
+        },
+
+        //  Implementation  ---
+        updateDataPatterns() {
             var context = this;
             if (this._logicalFile) {
                 var wuPromise = this.dataPatternsButton.get("checked") ? this._logicalFile.fetchDataPatternsWU() : Promise.resolve(null);
                 wuPromise.then(function (wu) {
-                    return context.gridDPHook.render(wu);
+                    return context.gridDPHook && context.gridDPHook.render(wu);
                 }).then(function () {
                     context.grid.resize();
                 });
             }
         },
 
-        //  Implementation  ---
         onErrorClick: function (line, col) {
         },
 
@@ -275,6 +279,7 @@ define([
                             context.refreshActionState();
                         }
                     }
+                    context.updateDataPatterns();
                 });
             }
         },

esp/src/eclwatch/templates/ResultWidget.html

@@ -12,7 +12,7 @@
             <div id="${id}Filter" data-dojo-type="FilterDropDownWidget">
             </div>
             <span data-dojo-type="dijit.ToolbarSeparator"></span>
-            <div id="${id}DataPatterns" checked=false class="hideNoIcon" data-dojo-attach-event="onClick:_onDataPatterns" data-dojo-props="showIcon:false, showNoIcon:false" data-dojo-type="dijit.form.ToggleButton">${i18n.DataPatterns}</div>
+            <div id="${id}DataPatterns" checked=true class="hideNoIcon" data-dojo-attach-event="onClick:_onDataPatterns" data-dojo-props="showIcon:false, showNoIcon:false" data-dojo-type="dijit.form.ToggleButton">${i18n.DataPatterns}</div>
             <span id="${id}DataPatternsSep" data-dojo-type="dijit.ToolbarSeparator"></span>
             <div class="right" data-dojo-attach-event="onChange:_onMaximize" data-dojo-props="iconClass:'iconMaximize', showLabel:false" checked=false data-dojo-type="dijit.form.ToggleButton">${i18n.MaximizeRestore}</div>
             <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
@@ -22,4 +22,4 @@
             </div>
         </div>
     </div>
-</div>
+</div>

esp/src/src/DataPatterns/DPWorkunit.ts

@@ -41,11 +41,11 @@ export class DPWorkunit {
     private _wu: Workunit;
     private _resultPromise;
 
-    constructor(nodeGroup: string, name: string) {
+    constructor(nodeGroup: string, name: string, modified: string) {
         this._lf = LogicalFile.attach({ baseUrl: "" }, nodeGroup, name);
         this._store = globalKeyValStore();
         this._storeID = `dp-${nodeGroup}-${name}`;
-        this._storeWuidID = `${this._storeID}-wuid`;
+        this._storeWuidID = `${this._storeID}-${modified}-wuid`.split(" ").join("_");
     }
 
     clearCache() {
@@ -58,6 +58,8 @@ export class DPWorkunit {
             if (this._wu && this._wu.Wuid === wuid) {
                 return this._wu;
             }
+            //  Clean out old key (HPCC-25544)  ---
+            this._store.delete(`${this._storeID}-wuid`);
             this.clearCache();
             return wuid && Workunit.attach({ baseUrl: "" }, wuid);
         }).then(wu => {

esp/src/src/DataPatternsWidget.ts

@@ -169,7 +169,7 @@ export class DataPatternsWidget {
         this.optimizeTargetSelect.init({});
         this.optimizeTarget.set("value", params.LogicalName + "::optimized");
 
-        this._dpWu = new DPWorkunit(params.NodeGroup, params.LogicalName);
+        this._dpWu = new DPWorkunit(params.NodeGroup, params.LogicalName, params.Modified);
 
         this.wuStatus.target(this.id + "WUStatus");
         this.dpReport.target(this.id + "DPReport");

esp/src/src/ESPLogicalFile.ts

@@ -203,8 +203,7 @@ const TreeStore = declare(null, {
     }
 });
 
-const LogicalFile = declare([ESPUtil.Singleton], {    // jshint ignore:line
-    _dpWU: DPWorkunit,
+const LogicalFile = declare([ESPUtil.Singleton], {
 
     _FileDetailSetter(FileDetail) {
         this.FileDetail = FileDetail;
@@ -253,7 +252,6 @@ const LogicalFile = declare([ESPUtil.Singleton], {    // jshint ignore:line
             declare.safeMixin(this, args);
         }
         this.logicalFile = this;
-        this._dpWU = new DPWorkunit(this.Cluster, this.Name);
     },
     save(request, args) {
         // WsDfu/DFUInfo?FileName=progguide%3A%3Aexampledata%3A%3Akeys%3A%3Apeople.lastname.firstname&UpdateDescription=true&FileDesc=%C2%A0123&Save+Description=Save+Description
@@ -428,7 +426,8 @@ const LogicalFile = declare([ESPUtil.Singleton], {    // jshint ignore:line
         return this.StateID === 999;
     },
     fetchDataPatternsWU() {
-        return this._dpWU.resolveWU();
+        const dpWU = new DPWorkunit(this.Cluster, this.Name, this.Modified);
+        return dpWU.resolveWU();
     }
 });
 

esp/src/stub.htm

@@ -4,6 +4,7 @@
 <head>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta charset="utf-8">
+    <meta name="robots" content="noindex">
     <title>ECL Watch</title>
     <script src="/esp/files/node_modules/es6-promise/dist/es6-promise.auto.min.js"></script>
     <script>

initfiles/componentfiles/configxml/daliplugin.xsd

@@ -82,14 +82,6 @@
                 </xs:appinfo>
               </xs:annotation>
             </xs:attribute>
-            <xs:attribute name="entrypoint" type="xs:string" use="optional" default="createWorkUnitFactory">
-              <xs:annotation>
-                <xs:appinfo>
-                  <tooltip>Name for this dali server plugin</tooltip>
-                  <required>true</required>
-                </xs:appinfo>
-              </xs:annotation>
-            </xs:attribute>
             <xs:sequence>
                 <xs:element name="Option" maxOccurs="unbounded">
                     <xs:annotation>

roxie/ccd/ccd.hpp

@@ -49,6 +49,8 @@
 #define UDP_QUEUE_SIZE 100
 #define UDP_SEND_QUEUE_SIZE 50
 
+#define NUM_DALI_CRITS 64
+
 #define ROXIE_STATEFILE_VERSION 2
 
 // Have not yet tested impact of new IBYTI handling in non-containerized systems

roxie/ccd/ccdstate.cpp

@@ -20,6 +20,7 @@
 #include "build-config.h"
 
 #include "jisem.hpp"
+#include "jhash.hpp"
 #include "jsort.hpp"
 #include "jregexp.hpp"
 
@@ -423,6 +424,7 @@ class CRoxiePackageNode : extends CPackageNode, implements IRoxiePackage
 {
 protected:
     static CResolvedFileCache daliFiles;
+    static CriticalSection daliLookupCrits[NUM_DALI_CRITS];
     mutable CResolvedFileCache fileCache;
     IArrayOf<IResolvedFile> files;  // Used when preload set
     IArrayOf<IKeyArray> keyArrays;  // Used when preload set
@@ -453,6 +455,8 @@ protected:
     // Use dali to resolve subfile into physical file info
     static IResolvedFile *resolveLFNusingDaliOrLocal(const char *fileName, bool useCache, bool cacheResult, bool writeAccess, bool alwaysCreate, bool resolveLocal, bool isPrivilegedUser)
     {
+        unsigned hash = hashc((const unsigned char *) fileName, strlen(fileName), 0x811C9DC5);
+        CriticalBlock b(daliLookupCrits[hash % NUM_DALI_CRITS]);
         // MORE - look at alwaysCreate... This may be useful to implement earlier locking semantics.
         if (traceLevel > 9)
             DBGLOG("resolveLFNusingDaliOrLocal %s %d %d %d %d", fileName, useCache, cacheResult, writeAccess, alwaysCreate);
@@ -777,6 +781,7 @@ public:
 };
 
 CResolvedFileCache CRoxiePackageNode::daliFiles;
+CriticalSection CRoxiePackageNode::daliLookupCrits[NUM_DALI_CRITS];
 
 typedef CResolvedPackage<CRoxiePackageNode> CRoxiePackage;