Merge branch 'candidate-8.4.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.gitignore

@@ -1,3 +1,5 @@
+build/
+vcpkg/
 externals/
 .project
 .svn/

docs/EN_US/ECLLanguageReference/ECLR_mods/Value-UTF8.xml

@@ -7,15 +7,18 @@
   <para><emphasis role="bold">UTF8<indexterm>
       <primary>UTF8</primary>
     </indexterm>[</emphasis><emphasis>_locale</emphasis><emphasis
+  role="bold">][</emphasis><emphasis>_n</emphasis><emphasis
   role="bold">]<indexterm>
       <primary>UTF8 value type</primary>
     </indexterm></emphasis></para>
 
-  <para>A UTF-8 encoded unicode character string of variable length to the
-  size needed to contain the result of the cast or passed parameter. The
-  optional <emphasis>locale </emphasis>specifies a valid unicode locale code,
-  as specified in ISO standards 639 and 3166 (not needed if LOCALE is
-  specified on the RECORD structure containing the field definition).</para>
+  <para>A UTF-8 encoded unicode character string of <emphasis>n</emphasis>
+  characters, space-padded just as STRING is. If _<emphasis>n</emphasis> is
+  omitted, the string is variable length to the size needed to contain the
+  result of the cast or passed parameter. The optional <emphasis>locale
+  </emphasis>specifies a valid unicode locale code, as specified in ISO
+  standards 639 and 3166 (not needed if LOCALE is specified on the RECORD
+  structure containing the field definition).</para>
 
   <para>Type casting UTF8 to UNICODE, VARUNICODE, STRING, or DATA is allowed,
   while casting to any other type will first implicitly cast to STRING and
@@ -25,9 +28,12 @@
 
   <para>Example:</para>
 
-  <programlisting>UTF8 FirstName := U8'Noël';
-        // utf-8-encoded string
-UTF8_de MyUnicodeString := U8'abcd\353';
-        // becomes 'abcdë' with a German locale
+  <programlisting>UTF8 FirstName := U8'Noël';                 // utf-8-encoded string
+
+UTF8_de MyUnicodeString := U8'abcd\353';    // becomes 'abcdë' with a German locale
+
+UTF8_4 FirstName4 := U8'Noël';              // 4-character utf-8-encoded string
+
+UTF8_de_5 MyUnicodeString5 := U8'abcd\353'; // becomes 'abcdë' with a German locale
 </programlisting>
 </sect1>

esp/src/.eslintrc.js

@@ -4,6 +4,7 @@ module.exports = {
     parser: "@typescript-eslint/parser",
     plugins: [
         "@typescript-eslint",
+        "eclwatch"
     ],
     extends: [
         "eslint:recommended",
@@ -92,6 +93,7 @@ module.exports = {
         ],
         "@typescript-eslint/no-non-null-assertion": "off",
         "@typescript-eslint/no-namespace": "off",
-        "@typescript-eslint/no-var-require": "off"
+        "@typescript-eslint/no-var-require": "off",
+        "eclwatch/no-src-react": 1
     }
 };

esp/src/CMakeLists.txt

@@ -25,21 +25,23 @@ file(REMOVE_RECURSE
 file(COPY
     ${CMAKE_CURRENT_SOURCE_DIR}/dgrid
     ${CMAKE_CURRENT_SOURCE_DIR}/eclwatch
+    ${CMAKE_CURRENT_SOURCE_DIR}/eslint
     ${CMAKE_CURRENT_SOURCE_DIR}/ganglia
-    ${CMAKE_CURRENT_SOURCE_DIR}/index.html
     ${CMAKE_CURRENT_SOURCE_DIR}/loader
-    ${CMAKE_CURRENT_SOURCE_DIR}/lws.config.js
-    ${CMAKE_CURRENT_SOURCE_DIR}/package.json
-    ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
     ${CMAKE_CURRENT_SOURCE_DIR}/put-selector
     ${CMAKE_CURRENT_SOURCE_DIR}/src
     ${CMAKE_CURRENT_SOURCE_DIR}/src-react
-    ${CMAKE_CURRENT_SOURCE_DIR}/stub.htm
-    ${CMAKE_CURRENT_SOURCE_DIR}/Login.html
+    ${CMAKE_CURRENT_SOURCE_DIR}/xstyle
+    ${CMAKE_CURRENT_SOURCE_DIR}/.eslintrc.js
     ${CMAKE_CURRENT_SOURCE_DIR}/GetUserName.html
+    ${CMAKE_CURRENT_SOURCE_DIR}/index.html
+    ${CMAKE_CURRENT_SOURCE_DIR}/Login.html
+    ${CMAKE_CURRENT_SOURCE_DIR}/lws.config.js
+    ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+    ${CMAKE_CURRENT_SOURCE_DIR}/package.json
+    ${CMAKE_CURRENT_SOURCE_DIR}/stub.htm
     ${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.json
     ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
-    ${CMAKE_CURRENT_SOURCE_DIR}/xstyle
     DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
 
 set ( SRCS
@@ -290,9 +292,16 @@ set ( SRCS
     ${CMAKE_CURRENT_SOURCE_DIR}/src-react/util/history.ts
     ${CMAKE_CURRENT_SOURCE_DIR}/src-react/index.tsx
     ${CMAKE_CURRENT_SOURCE_DIR}/src-react/routes.tsx
-    ${CMAKE_CURRENT_SOURCE_DIR}/stub.htm
-    ${CMAKE_CURRENT_SOURCE_DIR}/Login.html
+    ${CMAKE_CURRENT_SOURCE_DIR}/.eslintrc.js
     ${CMAKE_CURRENT_SOURCE_DIR}/GetUserName.html
+    ${CMAKE_CURRENT_SOURCE_DIR}/index.html
+    ${CMAKE_CURRENT_SOURCE_DIR}/Login.html
+    ${CMAKE_CURRENT_SOURCE_DIR}/lws.config.js
+    ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
+    ${CMAKE_CURRENT_SOURCE_DIR}/package.json
+    ${CMAKE_CURRENT_SOURCE_DIR}/stub.htm
+    ${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.json
+    ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
 )
 
 if ( "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" )

esp/src/eslint/index.js

@@ -0,0 +1,22 @@
+// eslint-disable-next-line no-undef
+module.exports = {
+    rules: {
+        "no-src-react": {
+            create: function (context) {
+                return {
+                    ImportDeclaration(node) {
+                        if (node && node.source && node.source.value && node.source.value.indexOf("src-react/") === 0) {
+                            context.report({
+                                node,
+                                message: "Prefer '..' to 'src-react'",
+                                fix: function (fixer) {
+                                    return fixer.replaceText(node.source, node.source.raw.replace("src-react/", "../"));
+                                }
+                            });
+                        }
+                    }
+                };
+            }
+        }
+    }
+};

esp/src/eslint/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "eslint-plugin-eclwatch",
+  "version": "0.0.0",
+  "main": "index.js"
+}

esp/src/package-lock.json

@@ -2885,6 +2885,10 @@
         }
       }
     },
+    "eslint-plugin-eclwatch": {
+      "version": "file:eslint",
+      "dev": true
+    },
     "eslint-plugin-react-hooks": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.2.0.tgz",

esp/src/package.json

@@ -5,7 +5,7 @@
   "scripts": {
     "clean": "rimraf ./build ./lib ./src/nlsHPCCType.ts",
     "lint": "eslint eclwatch/**/*.js src/**/*.ts? src-react/**/*.ts?",
-    "lint-fix": "eslint --fix eclwatch/**/*.js src/**/*.ts",
+    "lint-fix": "eslint --fix eclwatch/**/*.js src/**/*.ts src-react/**/*.ts?",
     "copy-res-es6-promise": "copyfiles -u 3 \"./node_modules/es6-promise/dist/es6-promise.auto.min.js\" ./build/node_modules/es6-promise/dist/",
     "copy-res-eclwatch-img": "copyfiles -u 2 \"./eclwatch/img/**/*.{png,jpg,gif}\" ./build/eclwatch/img/",
     "copy-res-eclwatch-ecl": "copyfiles -u 2 \"./eclwatch/ecl/**/*.*\" ./build/eclwatch/ecl/",
@@ -84,6 +84,7 @@
     "dojo-webpack-plugin": "^3.0.2",
     "eslint": "^7.0.1",
     "eslint-plugin-react-hooks": "^4.2.0",
+    "eslint-plugin-eclwatch": "file:./eslint",
     "file-loader": "^6.2.0",
     "local-web-server": "^5.1.1",
     "npm-run-all": "^4.1.5",

esp/src/src-react/components/forms/Optimize.tsx

@@ -2,11 +2,11 @@ import * as React from "react";
 import { Checkbox, DefaultButton, PrimaryButton, TextField, } from "@fluentui/react";
 import { scopedLogger } from "@hpcc-js/util";
 import { useForm, Controller } from "react-hook-form";
+import { DPWorkunit } from "src/DataPatterns/DPWorkunit";
 import nlsHPCC from "src/nlsHPCC";
 import { MessageBox } from "../../layouts/MessageBox";
+import { pushUrl } from "../../util/history";
 import { TargetClusterTextField } from "./Fields";
-import { DPWorkunit } from "src/DataPatterns/DPWorkunit";
-import { pushUrl } from "src-react/util/history";
 
 const logger = scopedLogger("../components/forms/PublishQuery.tsx");
 

helm/hpcc/templates/_helpers.tpl

@@ -430,6 +430,7 @@ Add sentinel-based probes for a component
 {{- define "hpcc.addSentinelProbes" -}}
 {{- $minStartupTime := .minStartupTime | default 0 }}
 {{- $maxStartupTime := .maxStartupTime | default 300 }}
+{{- $readyProbeName := .readyProbeName | default "" }}
 {{- $numAttempts := div (sub $maxStartupTime $minStartupTime) 10 }}
 startupProbe:
   exec:
@@ -443,7 +444,7 @@ readinessProbe:
   exec:
     command:
     - cat
-    - "/tmp/{{ .name }}.sentinel"
+    - "/tmp/{{ .name }}.sentinel{{ $readyProbeName }}"
   periodSeconds: 10
 {{ end -}}
 

helm/hpcc/templates/roxie.yaml

@@ -277,7 +277,7 @@ spec:
           containerPort: {{ $service.servicePort }}
 {{- end }}
 {{- end }}
-{{ include "hpcc.addSentinelProbes" $roxie | indent 8 }}
+{{ include "hpcc.addSentinelProbes" ( $roxie | merge (dict "readyProbeName" ".ready" )) | indent 8 }}
 {{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
 {{- include "hpcc.addResources" (dict "me" $roxie.serverResources) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
@@ -385,7 +385,7 @@ spec:
           containerPort: {{ $service.servicePort }}
 {{- end }}
 {{- end }}
-{{ include "hpcc.addSentinelProbes" $roxie | indent 8 }}
+{{ include "hpcc.addSentinelProbes" ( $roxie | merge (dict "readyProbeName" ".ready" )) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
 {{- include "hpcc.addResources" (dict "me" $roxie.channelResources) | indent 8 }}

roxie/ccd/ccddali.cpp

@@ -182,10 +182,12 @@ private:
     private:
         CRoxieDaliHelper *owner;
         bool aborted;
+        bool wasConnected;
     public:
         CRoxieDaliConnectWatcher(CRoxieDaliHelper *_owner) : owner(_owner)
         {
             aborted = false;
+            wasConnected = owner->isConnected;
         }
 
         virtual int run()
@@ -194,24 +196,16 @@ private:
             {
                 if (topology && topology->getPropBool("@lockDali", false))
                 {
-                    Sleep(ROXIE_DALI_CONNECT_TIMEOUT);
                 }
-                else if (owner->connect(ROXIE_DALI_CONNECT_TIMEOUT))
+                else
                 {
-                    if (traceLevel)
+                    wasConnected = owner->checkDaliConnectionValid();
+                    bool connected = owner->connect(ROXIE_DALI_CONNECT_TIMEOUT);
+                    if (connected && !wasConnected && traceLevel)
                         DBGLOG("CRoxieDaliConnectWatcher reconnected");
-                    try
-                    {
-                        owner->disconnectSem.wait();
-                        Sleep(5000);   // Don't retry immediately, give Dali a chance to recover.
-                    }
-                    catch (IException *E)
-                    {
-                        if (!aborted)
-                            EXCLOG(E, "roxie: Unexpected exception in CRoxieDaliConnectWatcher");
-                        E->Release();
-                    }
+                    wasConnected = connected;
                 }
+                Sleep(ROXIE_DALI_CONNECT_TIMEOUT);
             }
             return 0;
         }
@@ -230,6 +224,26 @@ private:
         }
     } connectWatcher;
 
+    bool checkDaliConnectionValid()
+    {
+        CriticalBlock b(daliHelperCrit);
+        if (!isConnected)
+            return false;
+        try
+        {
+            Owned<INode> res = querySessionManager().getProcessSessionNode(myProcessSession());
+            if (!res)
+                disconnect();
+        }
+        catch (IException *E)
+        {
+            EXCLOG(E);
+            ::Release(E);
+            disconnect();
+        }
+        return isConnected;
+    }
+
     virtual void beforeDispose()
     {
         CriticalBlock b(daliHelperCrit);

roxie/ccd/ccdmain.cpp

@@ -1266,6 +1266,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         }
 #ifdef _CONTAINERIZED
         initializeTopology(topoValues, myRoles);
+        writeSentinelFile(sentinelFile);
 #endif
         createDelayedReleaser();
         globalPackageSetManager = createRoxiePackageSetManager(standAloneDll.getClear());
@@ -1436,6 +1437,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
                     queryFileCache().loadSavedOsCacheInfo();
                 queryFileCache().startCacheReporter();
 #ifdef _CONTAINERIZED
+                // Roxie indicates readiness when all channels report ready
                 publishTopology(traceLevel, myRoles);
 #else
                 writeSentinelFile(sentinelFile);

roxie/ccd/ccdserver.cpp

@@ -1509,12 +1509,15 @@ public:
             {
                 if (state==STATEstarted || state==STATEstarting)
                 {
-                    if (ctx->queryServerContext()->okToLogStartStopError()) // && traceLevel ?
+                    if (ctx && ctx->queryServerContext() && ctx->queryCodeContext())
                     {
-                        VStringBuffer err("STATE: activity %d reset without stop", activityId);
-                        ctx->queryCodeContext()->addWuException(err.str(), ROXIE_INTERNAL_ERROR, SeverityError, "roxie");
+                        if (ctx->queryServerContext()->okToLogStartStopError()) // && traceLevel ?
+                        {
+                            VStringBuffer err("STATE: activity %d reset without stop", activityId);
+                            ctx->queryCodeContext()->addWuException(err.str(), ROXIE_INTERNAL_ERROR, SeverityError, "roxie");
+                        }
                     }
-                    if (ctx->queryOptions().failOnLeaks)
+                    if (ctx && ctx->queryOptions().failOnLeaks)
                         throw makeStringExceptionV(ROXIE_INTERNAL_ERROR, "STATE: activity %d reset without stop", activityId);
                     try
                     {

roxie/udplib/udpmsgpk.cpp

@@ -624,8 +624,8 @@ void CMessageCollator::collate(DataBuffer *dataBuff)
         mapping.remove(puid);
         queueCrit.enter();
         queue.push(pkSqncr);
-        sem.signal();
         queueCrit.leave();
+        sem.signal();
     }
 }
 

roxie/udplib/udptopo.cpp

@@ -369,7 +369,7 @@ void CTopologyServer::updateStatus() const
         if (rangeStart != numChannels)
             report.appendf("-%u", numChannels);
     }
-    Owned<IFile> sentinelFile = createSentinelTarget();
+    Owned<IFile> sentinelFile = createSentinelTarget(".ready");
     if (unready==0)
     {
         writeSentinelFile(sentinelFile);

system/jlib/jfile.cpp

@@ -6963,11 +6963,14 @@ IFileIOCache* createFileIOCache(unsigned max)
 }
 
 
-extern jlib_decl IFile * createSentinelTarget()
+extern jlib_decl IFile * createSentinelTarget(const char *suffix)
 {
     const char * sentinelFilename = getenv("SENTINEL");
     if (sentinelFilename && *sentinelFilename)
-        return createIFile(sentinelFilename);
+    {
+        VStringBuffer usename("%s%s", sentinelFilename, suffix ? suffix : "");
+        return createIFile(usename);
+    }
     else
         return NULL;
 }
@@ -6998,7 +7001,7 @@ extern jlib_decl void writeSentinelFile(IFile * sentinelFile)
 {
     if ( sentinelFile )
     {
-        DBGLOG("Creating sentinel file %s for rerun from script", sentinelFile->queryFilename());
+        DBGLOG("Creating sentinel file %s", sentinelFile->queryFilename());
         try
         {
             Owned<IFileIO> sentinel = sentinelFile->open(IFOcreate);
@@ -7007,7 +7010,7 @@ extern jlib_decl void writeSentinelFile(IFile * sentinelFile)
         catch(IException *E)
         {
             StringBuffer s;
-            EXCLOG(E, s.appendf("Failed to create sentinel file %s for rerun from script", sentinelFile->queryFilename()).str());
+            EXCLOG(E, s.appendf("Failed to create sentinel file %s", sentinelFile->queryFilename()).str());
             E->Release();
             throw makeOsException(errno, "writeSentinelFile - file not created.");
         }

system/jlib/jfile.hpp

@@ -639,7 +639,7 @@ extern jlib_decl bool containsFileWildcard(const char * path);
 extern jlib_decl bool isDirectory(const char * path);
 extern jlib_decl void removeFileTraceIfFail(const char * filename);
 extern jlib_decl IFileIOCache* createFileIOCache(unsigned max);
-extern jlib_decl IFile * createSentinelTarget();
+extern jlib_decl IFile * createSentinelTarget(const char *suffix = nullptr);
 extern jlib_decl void writeSentinelFile(IFile * file);
 extern jlib_decl void removeSentinelFile(IFile * file);
 extern jlib_decl StringBuffer & appendCurrentDirectory(StringBuffer & target, bool blankIfFails);

system/jlib/jstats.cpp

@@ -2307,7 +2307,9 @@ void CRuntimeStatisticCollection::updateDelta(CRuntimeStatisticCollection & targ
 
 void CRuntimeStatisticCollection::mergeStatistic(StatisticKind kind, unsigned __int64 value)
 {
-    queryStatistic(kind).merge(value, queryMergeMode(kind));
+    CRuntimeStatistic * target = queryOptStatistic(kind);
+    if (target)
+        target->merge(value, queryMergeMode(kind));
 }
 
 void CRuntimeStatisticCollection::sumStatistic(StatisticKind kind, unsigned __int64 value)

system/jlib/jstats.h

@@ -551,6 +551,13 @@ public:
 #endif
         return values[index];
     }
+    inline CRuntimeStatistic * queryOptStatistic(StatisticKind kind)
+    {
+        unsigned index = queryMapping().getIndex(kind);
+        if (index == mapping.numStatistics())
+            return nullptr;
+        return &values[index];
+    }
     inline const CRuntimeStatistic & queryStatistic(StatisticKind kind) const
     {
         unsigned index = queryMapping().getIndex(kind);

thorlcr/activities/thdiskbase.cpp

@@ -126,9 +126,11 @@ void CDiskReadMasterBase::done()
 void CDiskReadMasterBase::deserializeStats(unsigned node, MemoryBuffer &mb)
 {
     CMasterActivity::deserializeStats(node, mb);
-
-    for (auto &stats: subFileStats)
-        stats->deserialize(node, mb);
+    if (mapping && (mapping->queryMapWidth(node)>0)) // there won't be any subfile stats. if worker was sent 0 parts
+    {
+        for (auto &stats: subFileStats)
+            stats->deserialize(node, mb);
+    }
 }
 /////////////////
 

tools/testsocket/testsocket.cpp

@@ -76,6 +76,7 @@ FILE * trace;
 CriticalSection traceCrit;
 
 unsigned queryDelayMS = 0;
+unsigned queryAbsDelayMS = 0;  // ex: -u0 -qd 1000 for 1 q/s ...
 unsigned totalQueryCnt = 0;
 double totalQueryMS = 0.0;
 
@@ -774,7 +775,14 @@ class QueryThread : public Thread
 public:
     QueryThread(const char * _ip, unsigned _port, const char * _base) : ip(_ip),port(_port),base(_base) {}
 
-    virtual int run() { doSendQuery(ip, port, base); done.signal(); okToSend.signal(); return 0; }
+    virtual int run()
+    {
+        doSendQuery(ip, port, base);
+        done.signal();
+        if (multiThreadMax)
+            okToSend.signal();
+        return 0;
+    }
 
 protected:
     StringAttr      ip;
@@ -794,6 +802,10 @@ int sendQuery(const char * ip, unsigned port, const char * base)
     Thread * thread = new QueryThread(ip, port, base);
     thread->start();
     thread->Release();
+
+    if (multiThread && queryAbsDelayMS && !multiThreadMax)
+        Sleep(queryAbsDelayMS);
+
     return 0;
 }
 
@@ -1067,6 +1079,12 @@ int main(int argc, char **argv)
         multiThread = false;
     }
 
+    if (multiThread && queryDelayMS && !multiThreadMax)
+    {
+        queryAbsDelayMS = queryDelayMS;
+        queryDelayMS = 0;
+    }
+
     StringAttr ip;
     unsigned socketPort = (useSSL) ? ROXIE_SSL_SERVER_PORT : ROXIE_SERVER_PORT;
     SplitIpPort(ip, socketPort, argv[1]);