Merge branch 'candidate-7.10.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/dependencies/focal.cmake

@@ -1,5 +1,5 @@
 # Ubuntu 20.04
-SET_DEPENDENCIES ( CPACK_DEBIAN_PACKAGE_DEPENDS g++ openssh-client openssh-server expect rsync libapr1 python psmisc curl )
+SET_DEPENDENCIES ( CPACK_DEBIAN_PACKAGE_DEPENDS g++ openssh-client openssh-server expect rsync libapr1 python2 python3 psmisc curl )
 
 if(SPARK)
     SET_DEPENDENCIES ( CPACK_DEBIAN_PACKAGE_DEPENDS "openjdk-11-jdk" )

dockerfiles/buildall.sh

@@ -33,15 +33,6 @@ USE_CPPUNIT=1
 [[ -n ${INPUT_BUILD_VER} ]] && BUILD_TAG=${INPUT_BUILD_VER}
 [[ -n ${GITHUB_REPOSITORY} ]] && BUILD_USER=${GITHUB_REPOSITORY%/*}
 
-if [[ -n ${INPUT_BUILDTYPE} ]] ; then
-  BUILD_TYPE=$INPUT_BUILDTYPE
-  BUILD_LABEL=${BUILD_TAG}-$INPUT_BUILDTYPE
-else
-  BUILD_LABEL=${BUILD_TAG}
-  BUILD_TYPE=RelWithDebInfo
-  USE_CPPUNIT=0
-fi
-
 if [[ -n ${INPUT_USERNAME} ]] ; then
   echo ${INPUT_PASSWORD} | docker login -u ${INPUT_USERNAME} --password-stdin ${INPUT_REGISTRY}
   PUSH=1
@@ -66,6 +57,15 @@ pushd $DIR 2>&1 > /dev/null
 parse_cmake
 set_tag
 
+if [[ -n ${INPUT_BUILDTYPE} ]] ; then
+  BUILD_TYPE=$INPUT_BUILDTYPE
+  BUILD_LABEL=${HPCC_SHORT_TAG}-$INPUT_BUILDTYPE
+else
+  BUILD_LABEL=${HPCC_SHORT_TAG}
+  BUILD_TYPE=RelWithDebInfo
+  USE_CPPUNIT=0
+fi
+
 if [[ "$HPCC_MATURITY" = "release" ]] && [[ "$INPUT_LATEST" = "1" ]] ; then
   LATEST=1
 fi
@@ -73,14 +73,14 @@ fi
 build_image() {
   local name=$1
   local label=$2
-  [[ -z ${label} ]] && label=$HPCC_SHORT_TAG
+  [[ -z ${label} ]] && label=$BUILD_LABEL
 
   if ! docker pull hpccsystems/${name}:${label} ; then
     docker image build -t hpccsystems/${name}:${label} \
        --build-arg BASE_VER=${BASE_VER} \
        --build-arg DOCKER_REPO=hpccsystems \
        --build-arg BUILD_TAG=${BUILD_TAG} \
-       --build-arg BUILD_LABEL=${label} \
+       --build-arg BUILD_LABEL=${BUILD_LABEL} \
        --build-arg BUILD_USER=${BUILD_USER} \
        --build-arg BUILD_TYPE=${BUILD_TYPE} \
        --build-arg USE_CPPUNIT=${USE_CPPUNIT} \

dockerfiles/platform-core/Dockerfile

@@ -46,6 +46,9 @@ RUN apt-get install -y \
   zip \
   curl
 
+ARG USE_CPPUNIT=1
+RUN if [ ${USE_CPPUNIT} -eq 1 ] ; then apt-get install -y libcppunit-1.15-0 ; fi 
+
 # these are developer tools - we may want to move them elsewhere so that they are only in the incremental builds?
 
 RUN apt-get install -y \

docs/EN_US/HPCCClientTools/CT_Mods/CT_ECL_IDE.xml

@@ -1330,7 +1330,7 @@
                   <entry><emphasis role="bold">Open</emphasis></entry>
 
                   <entry>Provides access to the following Open submenus:
-                  <programlisting role="tab"><emphasis role="bold">Open Builder:</emphasis> Opens a saved query(.ECL file) in a new ECL Builder window. CTRL+O
+                  <programlisting role="tab"><emphasis role="bold">Open Builder:</emphasis> Opens a saved query (.ECL file) in a new ECL Builder window. CTRL+O
 <emphasis role="bold">Open Workunit:</emphasis> opens a dialog to name the Workunit to open. 
                       Enter the Workunit ID (WUID), then press the <emphasis
                         role="bold">OK</emphasis> button.</programlisting></entry>
@@ -1903,6 +1903,69 @@
           <para><emphasis role="bold"> </emphasis></para>
         </sect3>
 
+        <sect3 id="CT_Bookmarks_Tab">
+          <title>Bookmarks Tab</title>
+
+          <para>The <emphasis role="bold">Bookmarks Tab</emphasis> contains
+          the following selections:</para>
+
+          <para><graphic fileref="../../images/CT060.jpg" /></para>
+
+          <informaltable colsep="1" frame="all" rowsep="1">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="122.40pt" />
+
+              <colspec />
+
+              <tbody>
+                <row>
+                  <entry><graphic
+                  fileref="../../images/CT060-0.jpg" /></entry>
+
+                  <entry>Press the <emphasis role="bold">Bookmarks</emphasis>
+                  icon to access the Bookmarks panel.</entry>
+                </row>
+
+                <row>
+                  <entry><graphic
+                  fileref="../../images/CT061-1.jpg" /></entry>
+
+                  <entry>Press the lower half of the Bookmarks icon to access
+                  the submenu:<programlisting role="tab"> <emphasis
+                        role="bold"> Open </emphasis>- Opens the selected bookmark.
+<emphasis role="bold">  Save</emphasis> - Saves the current bookmarks.
+  <emphasis role="bold">Load</emphasis> - Loads the last saved bookmarks.
+  <emphasis role="bold">Load Merge</emphasis> - Merges the last saved bookmarks with the current bookmarks.
+  <emphasis role="bold">Delete</emphasis> - Deletes the selected bookmark.
+</programlisting></entry>
+                </row>
+
+                <row>
+                  <entry><graphic
+                  fileref="../../images/CT060-2.jpg" /></entry>
+
+                  <entry>Next bookmark</entry>
+                </row>
+
+                <row>
+                  <entry><graphic
+                  fileref="../../images/CT060-3.jpg" /></entry>
+
+                  <entry>Previous bookmark</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Clear</emphasis></entry>
+
+                  <entry>Clears bookmarks</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable>
+
+          <para><emphasis role="bold"> </emphasis></para>
+        </sect3>
+
         <sect3 id="CT_ECLIDE_BrowserTab">
           <title>Browser Tab</title>
 
@@ -2940,8 +3003,7 @@ ENDMACRO;
 
                   <entry>Go to Line</entry>
                 </row>
-
-                <row>
+<row>
                   <entry>Alt+0</entry>
 
                   <entry>Fold All - collapses all elements that can be

helm/hpcc/templates/localroxie.yaml

@@ -1,6 +1,6 @@
 {{ range $roxie := $.Values.roxie -}}
 {{- if not $roxie.disabled  -}}
-{{- if $roxie.localSlave -}}
+{{- if $roxie.localAgent -}}
 {{- $name := $roxie.name -}}
 {{- $servername := printf "%s-server" $roxie.name -}}
 
@@ -29,7 +29,7 @@ spec:
                 {{ include "hpcc.configArg" $roxie }},
                 {{ include "hpcc.daliArg" $ }},
                 "--server=true", 
-                "--localSlave=true",
+                "--localAgent=true",
                 "--resolveLocally=false"
               ]
 {{- $local := dict "first" true }}

helm/hpcc/templates/roxie.yaml

@@ -1,6 +1,6 @@
 {{ range $roxie := $.Values.roxie -}}
 {{- if not $roxie.disabled -}}
-{{- if not $roxie.localSlave -}}
+{{- if not $roxie.localAgent -}}
 {{- $toponame := printf "%s-toposerver" $roxie.name -}}
 {{- $numChannels := $roxie.numChannels | int | default 1 -}}
 {{- $topoport := $roxie.topoport | int | default 9004 -}}
@@ -141,7 +141,7 @@ spec:
 {{ end -}}
 {{ range $c, $e := until ($roxie.numChannels|int) -}}
 {{- $channel := add $c 1 -}}
-{{- $name := printf "%s-slave-%d" $roxie.name $channel -}}
+{{- $name := printf "%s-agent-%d" $roxie.name $channel -}}
 
 apiVersion: apps/v1
 kind: Deployment

helm/hpcc/values.yaml

@@ -165,12 +165,12 @@ roxie:
   - name: on-demand
     port: 0
   numChannels: 2
-  ## Set serverReplicas to indicate a separate replicaSet of roxie servers, with slave nodes not acting as servers
+  ## Set serverReplicas to indicate a separate replicaSet of roxie servers, with agent nodes not acting as servers
   serverReplicas: 0
   topoReplicas: 1
   topoport: 9004
-  ## Set localSlave to true for a scalable cluster of "single-node" roxie servers
-  localSlave: false
+  ## Set localAgent to true for a scalable cluster of "single-node" roxie servers
+  localAgent: false
   useAeron: false
 
 thor:

plugins/py3embed/py3embed.cpp

@@ -866,6 +866,7 @@ static void getStringResult(const RtlFieldInfo *field, PyObject *obj, size32_t &
             checkPythonError();
             size_t lenBytes = PyBytes_Size(temp_bytes);
             rtlStrToStrX(chars, result, lenBytes, text);
+            return;
         }
         else if (PyBytes_Check(obj))
         {
@@ -873,6 +874,7 @@ static void getStringResult(const RtlFieldInfo *field, PyObject *obj, size32_t &
             checkPythonError();
             size_t lenBytes = PyBytes_Size(obj);
             rtlStrToStrX(chars, result, lenBytes, text);
+            return;
         }
     }
     typeError("string", field);

testing/regress/hpcc/util/util.py

@@ -142,13 +142,15 @@ def queryWuid(jobname,  taskId):
                     state = val
     return {'wuid':wuid, 'state':state,  'result':result}
 
-def queryEngineProcess(engine):
+def queryEngineProcess(engine,  taskId):
     retVal = []
     myProc = subprocess.Popen(["ps aux | egrep '"+engine+"' | egrep -v 'grep'"],  shell=True,  bufsize=8192,  stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
     result = myProc.stdout.read() + myProc.stderr.read()
     results = result.split('\n')
+    logging.debug("%3d. queryEngineProcess(engine: %s): process(es) :'%s'",  taskId,  engine,  results)
     for line in results:
         line = line.replace('\n','')
+        logging.debug("%3d. queryEngineProcess(engine: %s): line:'%s'",  taskId,  engine,  line)
         if len(line):
             items = line.split()
             if len(items) >= 12:
@@ -156,18 +158,28 @@ def queryEngineProcess(engine):
                     myProc2 = subprocess.Popen(["sudo readlink -f /proc/" + items[1] + "/exe"],  shell=True,  bufsize=8192,  stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
                     result2 = myProc2.stdout.read().replace ('\n', '')
                     binPath = os.path.dirname(result2)
+                    logging.debug("%3d. queryEngineProcess(engine: %s): binary: '%s', binPath:'%s'",  taskId,  engine, result2,  binPath)
                     if 'slavenum' in line:
-                        ind = [items.index(i) for i in items if i.startswith('slavenum')]
-                        slaveNum = items[ind[0]].split('=')[1]
-                        retVal.append({ 'process' : result2, 'name' : os.path.basename(items[10]), 'slaveNum' : slaveNum, 'pid' : items[1], 'binPath': binPath})
+                        ind = [items.index(i) for i in items if i.startswith('--slavenum')]
+                        try:
+                            slaveNum = items[ind[0]].split('=')[1]
+                            retVal.append({ 'process' : result2, 'name' : os.path.basename(items[10]), 'slaveNum' : slaveNum, 'pid' : items[1], 'binPath': binPath})
+                        except Exception as e:
+                            logging.error("%3d. queryEngineProcess(engine: %s): slave number query failed:'%s'",  taskId,  engine, repr(e))
                     else:
                         retVal.append({ 'process' : result2, 'name' : os.path.basename(items[10]), 'slaveNum' : '', 'pid' : items[1], 'binPath': binPath})
     return retVal
 
-def createStackTrace(wuid, proc, taskId):
+def createStackTrace(wuid, proc, taskId, logDir = ""):
+    # Execute this function from CLI:
+    # ~/MyPython/RegressionSuite$ python -c 'import hpcc.util.util as util; p = util.queryEngineProcess("thormaster"); p+= util.queryEngineProcess("thorslave"); print p; [ util.createStackTrace("na", pp, -1, "~/HPCCSystems-regression/log") for pp in p]; '
+
     binPath = proc['process']
     pid = proc['pid']
-    outFile = os.path.expanduser(gConfig.logDir) + '/' + wuid +'-' + proc['name'] + proc['slaveNum'] + '.trace'
+    if logDir == "":
+        outFile = os.path.expanduser(gConfig.logDir) + '/' + wuid +'-' + proc['name'] + proc['slaveNum'] + '.trace'
+    else:
+        outFile = os.path.expanduser(logDir) + '/' + wuid +'-' + proc['name'] + proc['slaveNum'] + '-' + pid + '.trace'
     logging.error("%3d. Create Stack Trace for %s%s (pid:%s) into '%s'" % (taskId, proc['name'], proc['slaveNum'], pid, outFile), extra={'taskId':taskId})
     
     cmd  = 'sudo gdb --batch --quiet -ex "set interactive-mode off" '
@@ -187,15 +199,19 @@ def abortWorkunit(wuid, taskId = -1, engine = None):
     if (gConfig.generateStackTrace and (engine !=  None)):
         if isSudoer():
             if engine.startswith('thor'):
-                hpccProcesses = queryEngineProcess("thormaster")
-                hpccProcesses += queryEngineProcess("thorslave")
+                hpccProcesses = queryEngineProcess("thormaster", taskId)
+                hpccProcesses += queryEngineProcess("thorslave", taskId)
             elif engine.startswith('hthor'):
-                hpccProcesses = queryEngineProcess("eclagent")
+                hpccProcesses = queryEngineProcess("eclagent", taskId)
+                hpccProcesses += queryEngineProcess("hthor", taskId)
             elif engine.startswith('roxie'):
-                hpccProcesses = queryEngineProcess("roxie")
+                hpccProcesses = queryEngineProcess("roxie", taskId)
                 
-            for p in hpccProcesses:
-                createStackTrace(wuid, p, taskId)
+            if len(hpccProcesses) > 0:
+                for p in hpccProcesses:
+                    createStackTrace(wuid, p, taskId)
+            else:
+                logging.debug("%3d. abortWorkunit(wuid:'%s', engine:'%s') related process to generate stack trace not found.", taskId, wuid, str(engine))
             pass
         else:
             err = Error("7100")

thorlcr/graph/thgraphmaster.cpp

@@ -2505,8 +2505,10 @@ void CMasterGraph::executeSubGraph(size32_t parentExtractSz, const byte *parentE
             fatalHandler->clear();
     }
     fatalHandler.clear();
+#ifndef _CONTAINERIZED
     Owned<IWorkUnit> wu = &job.queryWorkUnit().lock();
     queryJobManager().updateWorkUnitLog(*wu);
+#endif
 }
 
 void CMasterGraph::sendGraph()

thorlcr/master/thgraphmanager.cpp

@@ -883,7 +883,9 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName,
     {
         Owned<IWorkUnit> wu = &workunit.lock();
         wu->setTracingValue("ThorBuild", BUILD_TAG);
+#ifndef _CONTAINERIZED
         updateWorkUnitLog(*wu);
+#endif
     }
     workunit.forceReload();
     StringAttr wuid(workunit.queryWuid());