Merge branch 'candidate-7.8.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/go_gold.sh

@@ -52,6 +52,7 @@ done
 update_version_file release $HPCC_POINT $NEW_SEQUENCE
 if [ -e helm/hpcc/Chart.yaml ] ; then
   update_chart_file helm/hpcc/Chart.yaml release $HPCC_POINT $NEW_SEQUENCE
+  doit "git add helm/hpcc/Chart.yaml"
 fi
 
 HPCC_MATURITY=release
@@ -73,17 +74,21 @@ if [ -e helm/hpcc/Chart.yaml ] ; then
   HPCC_DIR="$( pwd )"
   pushd ../helm-chart 2>&1 > /dev/null
   doit "git fetch $REMOTE"
+  doit "git checkout master"
   doit "git merge --ff-only $REMOTE/master"
   doit "git submodule update --init --recursive"
   HPCC_PROJECTS=hpcc-helm
   HPCC_NAME=HPCC
   if [[ "$HPCC_MAJOR" == "7" ]] && [[ "$HPCC_MINOR" == "8" ]] ; then
-    doit rm -rf ./helm
-    doit cp -rf $HPCC_DIR/helm ./helm 
+    doit "rm -rf ./helm"
+    doit "cp -rf $HPCC_DIR/helm ./helm" 
+    doit "rm ./helm/hpcc/*.bak" 
+    doit "git add -A ./helm"
   fi
   cd docs
-  doit helm package ${HPCC_DIR}/helm/hpcc/
-  doit helm repo index . --url https://hpcc-systems.github.io/helm-chart
+  doit "helm package ${HPCC_DIR}/helm/hpcc/"
+  doit "helm repo index . --url https://hpcc-systems.github.io/helm-chart"
+  doit "git add *.tgz"
   
   doit "git commit -a -s -m \"$HPCC_NAME Helm Charts $HPCC_SHORT_TAG Release Candidate $HPCC_SEQUENCE\""
   doit "git push $REMOTE master $FORCE"

cmake_modules/go_rc.sh

@@ -68,7 +68,8 @@ fi
 
 update_version_file rc $NEW_POINT $NEW_SEQUENCE $NEW_MINOR
 if [ -e helm/hpcc/Chart.yaml ] ; then
-  update_chart_file helm/hpcc/Chart.yaml rc $HPCC_POINT $NEW_SEQUENCE
+  update_chart_file helm/hpcc/Chart.yaml rc $NEW_POINT $NEW_SEQUENCE
+  doit "git add helm/hpcc/Chart.yaml"
 fi
 
 HPCC_MATURITY=rc
@@ -91,13 +92,15 @@ if [ -e helm/hpcc/Chart.yaml ] ; then
   HPCC_DIR="$( pwd )"
   pushd ../helm-chart 2>&1 > /dev/null
   doit "git fetch $REMOTE"
+  doit "git checkout master"
   doit "git merge --ff-only $REMOTE/master"
   doit "git submodule update --init --recursive"
   HPCC_PROJECTS=hpcc-helm
   HPCC_NAME=HPCC
   cd docs
-  doit helm package ${HPCC_DIR}/helm/hpcc/
-  doit helm repo index . --url https://hpcc-systems.github.io/helm-chart
+  doit "helm package ${HPCC_DIR}/helm/hpcc/"
+  doit "helm repo index . --url https://hpcc-systems.github.io/helm-chart"
+  doit "git add *.tgz"
   
   doit "git commit -a -s -m \"$HPCC_NAME Helm Charts $HPCC_SHORT_TAG Release Candidate $HPCC_SEQUENCE\""
   doit "git push $REMOTE master $FORCE"

common/thorhelper/roxiehelper.cpp

@@ -2354,9 +2354,9 @@ void FlushingJsonBuffer::startScalar(const char *resultName, unsigned sequence,
     startBlock();
     if (!isBlocked)
     {
+        StringBuffer seqName;
         if (!simpleTag)
         {
-            StringBuffer seqName;
             if (!resultName || !*resultName)
                 resultName = seqName.appendf("Result_%d", sequence+1).str();
             appendJSONName(s, resultName).append('{');

ecl/eclccserver/eclccserver.cpp

@@ -307,6 +307,22 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
         query->getQueryText(eclQuery);
         query->getQueryMainDefinition(mainDefinition);
 
+        bool syntaxCheck = (workunit->getAction()==WUActionCheck);
+        if (syntaxCheck && (mainDefinition.length() == 0))
+        {
+            SCMStringBuffer syntaxCheckAttr;
+            workunit->getApplicationValue("SyntaxCheck", "AttributeName", syntaxCheckAttr);
+            syntaxCheckAttr.s.trim();
+            if (syntaxCheckAttr.length())
+            {
+                workunit->getApplicationValue("SyntaxCheck", "ModuleName", mainDefinition);
+                mainDefinition.s.trim();
+                if (mainDefinition.length())
+                    mainDefinition.s.append('.');
+                mainDefinition.s.append(syntaxCheckAttr.str());
+            }
+        }
+
         StringBuffer eclccProgName;
         splitDirTail(queryCurrentProcessPath(), eclccProgName);
         eclccProgName.append("eclcc");
@@ -318,12 +334,15 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
         if (eclQuery.length())
             eclccCmd.append(" -");
         if (mainDefinition.length())
-            eclccCmd.append(" -main ").append(mainDefinition);
+            eclccCmd.append(" -main \"").append(mainDefinition).append("\"");
         eclccCmd.append(" --timings --xml");
         eclccCmd.append(" --nostdinc");
         eclccCmd.append(" --metacache=");
         VStringBuffer logfile("%s.eclcc.log", workunit->queryWuid());
         eclccCmd.appendf(" --logfile=%s", logfile.str());
+        if (syntaxCheck)
+            eclccCmd.appendf(" -syntax");
+
         if (globals->getPropBool("@enableEclccDali", true))
         {
             const char *daliServers = globals->queryProp("@daliServers");
@@ -917,7 +936,7 @@ int main(int argc, const char *argv[])
                 throw MakeStringException(0, "No queues found to listen on");
 #ifdef _CONTAINERIZED
             bool useChildProcesses = globals->getPropInt("@useChildProcesses", false);
-            unsigned maxThreads = globals->getPropInt("@maxActive", useChildProcesses ? 100 : 4);
+            unsigned maxThreads = globals->getPropInt("@maxActive", 4);
 #else
             // The option has been renamed to avoid confusion with the similarly-named eclcc option, but
             // still accept the old name if the new one is not present.

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1629,7 +1629,8 @@ bool CWsDfuEx::changeFileRestrictions(IEspContext &context, IEspDFUArrayActionRe
     if (changeRestriction == CDFUChangeRestriction_NoChange)
         return true;
 
-    context.ensureFeatureAccess("DFURestrictedAccess", SecAccess_Write, ECLWATCH_DFU_ACCESS_DENIED, "DFURestrictedAccess: Permission denied.");
+    if (changeRestriction == CDFUChangeRestriction_NotRestricted)
+        context.ensureFeatureAccess("CodeSignAccess", SecAccess_Full, ECLWATCH_DFU_ACCESS_DENIED, "CodeSignAccess: Permission denied.");
 
     if (isDetachedFromDali())
         throw makeStringException(ECLWATCH_INVALID_INPUT, "ESP server is detached from Dali. Please try later.");
@@ -2198,7 +2199,8 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
     }
     if (changeRestriction != CDFUChangeRestriction_NoChange)
     {
-        context.ensureFeatureAccess("DFURestrictedAccess", SecAccess_Write, ECLWATCH_DFU_ACCESS_DENIED, "DFURestrictedAccess: Permission denied.");
+        if (changeRestriction == CDFUChangeRestriction_NotRestricted)
+            context.ensureFeatureAccess("CodeSignAccess", SecAccess_Full, ECLWATCH_DFU_ACCESS_DENIED, "CodeSignAccess: Permission denied.");
         df->setRestrictedAccess(changeRestriction==CDFUChangeRestriction_Restrict);
     }
 

esp/src/eclwatch/FilterDropDownWidget.js

@@ -69,8 +69,10 @@ define([
         },
 
         _onFilterApply: function (event) {
-            var formData = this.toObject(event.currentTarget.form);
-            this.initRecentFilter(formData);
+            if (event) {
+                var formData = this.toObject(event.currentTarget.form)
+                this.initRecentFilter(formData);
+            }
             this.filterDropDown.closeDropDown();
             this.emit("apply");
             this.refreshState();

helm/hpcc/values.yaml

@@ -69,7 +69,7 @@ eclagent:
   ## replicas indicates how many eclagent pods should be started
   replicas: 1
   ## maxActive controls how many workunits may be active at once (per replica)
-  maxActive: 100
+  maxActive: 4
   ## prefix may be used to set a filename prefix applied to any relative filenames used by jobs submitted to this queue
   prefix: hthor
   ## Set to false if you want to launch each workunit in its own container, true to run as child processes in eclagent pod
@@ -80,7 +80,8 @@ eclagent:
 - name: roxie
   replicas: 1
   prefix: roxie
-  useChildProcesses: false
+  maxActive: 20
+  useChildProcesses: true
   type: roxie
     
 eclccserver:
@@ -88,6 +89,8 @@ eclccserver:
   replicas: 1
   ## Set to false if you want to launch each workunit compile in its own container, true to run as child processes in eclccserver pod.
   useChildProcesses: false
+  ## maxActive controls how many workunit compiles may be active at once (per replica)
+  maxActive: 4
   ## Specify a list of queues to listen on if you don't want this eclccserver listening on all queues. If empty or missing, listens on all queues
   listen: []
     

initfiles/bash/etc/init.d/distributePKI.in

@@ -76,8 +76,8 @@ if ping -c 1 -w 5 -n \$IP > /dev/null 2>&1; then
   ## ensure certPath exists
   ssh -i $homePath/.ssh/id_rsa -o BatchMode=yes -o LogLevel=QUIET -o StrictHostKeyChecking=no $USER_NAME@\$IP mkdir -p $certPath > /dev/null 2>&1
 
-  ssh -i $homePath/.ssh/id_rsa -o BatchMode=yes -o LogLevel=QUIET -o StrictHostKeyChecking=no $USER_NAME@\$IP rm -f $certPath/key.pem $certPath/certificate.pem > /dev/null 2>&1
-  scp -i $homePath/.ssh/id_rsa $certPath/key.pem $certPath/certificate.pem $USER_NAME@\$IP:$certPath
+  ssh -i $homePath/.ssh/id_rsa -o BatchMode=yes -o LogLevel=QUIET -o StrictHostKeyChecking=no $USER_NAME@\$IP rm -f $certPath/public.key.pem $certPath/key.pem $certPath/certificate.pem > /dev/null 2>&1
+  scp -i $homePath/.ssh/id_rsa $certPath/public.key.pem $certPath/key.pem $certPath/certificate.pem $USER_NAME@\$IP:$certPath
 
   for i in \${complist[@]} ; do
       ## allow for overwrite
@@ -122,7 +122,8 @@ if ping -c 1 -w 5 -n \$IP > /dev/null 2>&1; then
   ## ensure certPath exists
   ssh -i $homePath/.ssh/id_rsa -o BatchMode=yes -o LogLevel=QUIET -o StrictHostKeyChecking=no $USER_NAME@\$IP mkdir -p $certPath > /dev/null 2>&1
 
-  ssh -i $homePath/.ssh/id_rsa -o BatchMode=yes -o LogLevel=QUIET -o StrictHostKeyChecking=no $USER_NAME@\$IP rm -f $certPath/key.pem $certPath/certificate.pem > /dev/null 2>&1
+  ssh -i $homePath/.ssh/id_rsa -o BatchMode=yes -o LogLevel=QUIET -o StrictHostKeyChecking=no $USER_NAME@\$IP rm -f $certPath/public.key.pem $certPath/key.pem $certPath/certificate.pem > /dev/null 2>&1
+  scp -i $homePath/.ssh/id_rsa $USER_NAME@$dali_ip:$certPath/public.key.pem $USER_NAME@\$IP:$certPath
   scp -i $homePath/.ssh/id_rsa $USER_NAME@$dali_ip:$certPath/key.pem $USER_NAME@\$IP:$certPath
   scp -i $homePath/.ssh/id_rsa $USER_NAME@$dali_ip:$certPath/certificate.pem $USER_NAME@\$IP:$certPath
 

initfiles/bash/etc/init.d/setupPKI.in

@@ -38,7 +38,7 @@ envfile="${CONFIG_DIR}/${ENV_XML_FILE}"
 
 OIFS=$IFS
 IFS=$'\n'
-compArray=($(${sbin_path}/configgen -env ${envfile} -listall | awk 'BEGIN { FS=",";} {print $2;}'))
+compArray=($(${sbin_path}/configgen -env ${envfile} -listall | awk 'BEGIN { FS=",";} {print $2;}' | sort | uniq))
 IFS=$OIFS
 
 if [ ${#compArray[@]} -eq 0 ]; then
@@ -73,7 +73,7 @@ fi
 for i in "" ${compArray[@]} ; do
     compName=""
     if [ "$i" != "" ]; then
-        compName=""${i#*_}
+        compName=""${i}
     fi
     if [ "${compName}" != "" ]; then
          compNamePrint=${compName}
@@ -113,6 +113,8 @@ for i in "" ${compArray[@]} ; do
     fi
 done
 
+chown -R ${USER_NAME}:${USER_NAME} ${certPath}
+
 if [ ! -e ${homePath}/.ssh/id_rsa ]; then
     cp ${certPath}/key.pem ${homePath}/.ssh/id_rsa
     ssh-keygen -y -f ${certPath}/key.pem > ${homePath}/.ssh/id_rsa.pub

plugins/proxies/CMakeLists.txt

@@ -17,6 +17,7 @@ set (proxies_out_dir "plugins")
 install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib_saltlib.ecllib DESTINATION ${proxies_out_dir} COMPONENT Runtime)
 install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib_keylib.ecllib DESTINATION  ${proxies_out_dir} COMPONENT Runtime)
 install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib_metaphone.ecllib DESTINATION ${proxies_out_dir} COMPONENT Runtime)
+install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib_metaphone3.ecllib DESTINATION ${proxies_out_dir} COMPONENT Runtime)
 install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib_thorlib.ecllib DESTINATION ${proxies_out_dir} COMPONENT Runtime)
 install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib_word.ecllib DESTINATION ${proxies_out_dir} COMPONENT Runtime)
 

roxie/ccd/ccdcontext.cpp

@@ -1316,7 +1316,8 @@ public:
 
     virtual StringBuffer &getLogPrefix(StringBuffer &ret) const
     {
-        return logctx.getLogPrefix(ret);
+        logctx.getLogPrefix(ret);
+        return ret.append(':').append(factory->queryQueryName());
     }
 
     virtual bool isIntercepted() const
@@ -1480,7 +1481,9 @@ public:
         {
             Owned<IQueryFactory> libraryQuery = factory->lookupLibrary(extra.libraryName, extra.interfaceHash, *this);
             assertex(libraryQuery);
-            return libraryQuery->lookupGraph(this, "graph1", probeManager, *this, parentActivity);
+            IActivityGraph *ret = libraryQuery->lookupGraph(this, "graph1", probeManager, *this, parentActivity);
+            ret->setPrefix(libraryQuery->queryQueryName());
+            return ret;
         }
     }
 

roxie/ccd/ccdquery.hpp

@@ -50,6 +50,7 @@ interface IActivityGraph : extends IInterface
     virtual IRoxieServerChildGraph * createGraphLoopInstance(IRoxieSlaveContext *ctx, unsigned loopCounter, unsigned parentExtractSize, const byte * parentExtract, const IRoxieContextLogger &logctx) = 0;
     virtual const char *queryName() const = 0;
     virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const = 0;
+    virtual void setPrefix(const char *prefix) = 0;
 };
 
 interface IRoxiePackage;

roxie/ccd/ccdqueue.cpp

@@ -603,13 +603,13 @@ void SlaveContextLogger::set(IRoxieQueryPacket *packet)
                 traceInfo += debugLen + sizeof(unsigned short);
                 traceLength -= debugLen + sizeof(unsigned short);
             }
-            // Passing the wuid via the logging context prefix is a bit of a hack...
+            // Passing the wuid via the logging context prefix is a lot of a hack...
             if (loggingFlags & LOGGING_WUID)
             {
                 unsigned wuidLen = 0;
                 while (wuidLen < traceLength)
                 {
-                    if (traceInfo[wuidLen]=='@')
+                    if (traceInfo[wuidLen]=='@'||traceInfo[wuidLen]==':')
                         break;
                     wuidLen++;
                 }

roxie/ccd/ccdserver.cpp

@@ -239,7 +239,7 @@ public:
     {
         ctx->CTXLOGl(log);
     }
-    virtual StringBuffer &getLogPrefix(StringBuffer &ret) const
+    virtual StringBuffer &getLogPrefix(StringBuffer &ret) const override
     {
         return ctx->getLogPrefix(ret);
     }
@@ -27371,6 +27371,10 @@ protected:
         {
             loopCounter = _loopCounter;
         }
+        void setPrefix(const char *name)
+        {
+            prefix.set(name);
+        }
         virtual void noteChildGraph(unsigned id, IActivityGraph *childGraph)
         {
             childGraphs.setValue(id, childGraph);
@@ -27386,7 +27390,9 @@ protected:
         // MORE should really redirect the other log context ones too (though mostly doesn't matter). Really should refactor to have a queryLogContext() method in IRoxieSlaveContext I think
         virtual StringBuffer &getLogPrefix(StringBuffer &ret) const
         {
-            logctx.getLogPrefix(ret);
+            IndirectSlaveContext::getLogPrefix(ret);
+            if (prefix)
+                ret.appendf(":%s", prefix.str());
             if (loopCounter)
                 ret.appendf("{%u}", loopCounter);
             return ret;
@@ -27394,6 +27400,7 @@ protected:
     protected:
         const IRoxieContextLogger &logctx;
         unsigned loopCounter;
+        StringAttr prefix;
         ICodeContext * codeContext;
         MapXToMyClass<unsigned, unsigned, IActivityGraph> childGraphs;
     } graphSlaveContext;
@@ -27508,6 +27515,11 @@ public:
         return graphName.get();
     }
 
+    virtual void setPrefix(const char *name) override
+    {
+        graphSlaveContext.setPrefix(name);
+    }
+
     void createGraph(IRoxieSlaveContext *_ctx)
     {
         if (graphDefinition.isMultiInstance())
@@ -27971,7 +27983,9 @@ public:
         }
         return results.getClear();
     }
-
+    virtual void setPrefix(const char *) override
+    {
+    }
 protected:
     IRoxieSlaveContext *ctx;
     IRoxieServerActivity *parentActivity;
@@ -28144,6 +28158,7 @@ public:
 class CDelayedActivityGraph : implements IActivityGraph, public CInterface
 {
     StringAttr graphName;
+    StringAttr prefix;
     ActivityArray & graphDefinition;
     IProbeManager *probeManager;
     unsigned id;
@@ -28178,11 +28193,17 @@ public:
     { 
         colocalParent = _colocalParent;
     }
+    virtual void setPrefix(const char *pfx) override
+    {
+        prefix.set(pfx);
+    }
 
     virtual IRoxieServerChildGraph * createGraphLoopInstance(IRoxieSlaveContext *ctx, unsigned loopCounter, unsigned parentExtractSize, const byte * parentExtract, const IRoxieContextLogger &logctx) override
     {
         Owned<CIterationActivityGraph> ret = new CIterationActivityGraph(graphName, id, parentActivity, graphDefinition, probeManager, loopCounter, ctx, colocalParent, parentExtractSize, parentExtract, logctx);
         ret->createIterationGraph(ctx);
+        if (prefix)
+            ret->setPrefix(prefix);
         return ret.getClear();
     }
 };