Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -169,6 +169,14 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   endif ()
   if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX)
     execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE CMAKE_CXX_COMPILER_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+  endif()
+  if (CMAKE_COMPILER_IS_CLANGXX)
+    execute_process( COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE clang_full_version_string )
+    if (${clang_full_version_string} MATCHES ".*based on LLVM ([0-9]+\\.[0-9]+).*")
+      string (REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0-9]+).*" "\\1" CLANG_VERSION ${clang_full_version_string})
+    else ()
+      string (REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1" CLANG_VERSION ${clang_full_version_string})
+    endif()
   endif ()
 
   if (WIN32)
@@ -215,7 +223,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     set ( LIBRARY_OUTPUT_PATH "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/libs" )
 
     if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX)
-      message ("Using compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
+      message ("Using compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} ${CLANG_VERSION}")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -frtti -fPIC -fmessage-length=0 -Wformat -Wformat-security -Wformat-nonliteral -pthread -Wuninitialized")
       SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -rdynamic")
       SET (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -g -fno-inline-functions")
@@ -253,7 +261,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=logical-op-parentheses -Werror=bool-conversions -Werror=return-type -Werror=comment")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Werror=bitwise-op-parentheses -Werror=tautological-compare")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Wno-switch-enum -Wno-format-zero-length -Wno-switch")
-      if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.2.1 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 4.2.1)
+      if (CLANG_VERSION VERSION_GREATER 3.6 OR CLANG_VERSION VERSION_EQUAL 3.6)
         SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-undefined-bool-conversion -Wno-pointer-bool-conversion -Wno-tautological-compare")
       endif()
     endif()

common/dllserver/thorplugin.cpp

@@ -29,6 +29,10 @@
 #include <mach-o/getsect.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
+#elif !defined(_WIN32)
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <elf.h>
 #endif
 
 #include "thorplugin.hpp"
@@ -201,7 +205,7 @@ bool HelperDll::getResource(size32_t & len, const void * & data, const char * ty
     if (!data)
     {
         if (trace)
-            printf("Failed to locate symbol %s", symName.str());
+            printf("Failed to locate symbol %s\n", symName.str());
         return false;
     }
     byte bom;
@@ -245,6 +249,67 @@ static void secscan (bfd *file, sec_ptr sec, void *userParam)
 }
 #endif
 
+static bool getResourceFromMappedFile(const char * filename, const byte * start_addr, MemoryBuffer &data, const char * type, unsigned id)
+{
+#if defined(_WIN32) || defined (_USE_BINUTILS)
+    throwUnexpected();
+#elif defined(__APPLE__)
+    VStringBuffer sectname("%s_%u", type, id);
+    // The first bytes are the Mach-O header
+    const struct mach_header_64 *mh = (const struct mach_header_64 *) start_addr;
+    if (mh->magic != MH_MAGIC_64)
+    {
+        DBGLOG("Failed to extract resource %s: Does not appear to be a Mach-O 64-bit binary", filename);
+        return false;
+    }
+
+    unsigned long len = 0;
+    unsigned char *data2 = getsectiondata(mh, "__TEXT", sectname.str(), &len);
+    data.append(len, data2);
+    return true;
+#else
+    // The first bytes are the ELF header
+    const Elf64_Ehdr * hdr = (const Elf64_Ehdr *) start_addr;
+    if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0)
+    {
+        DBGLOG("Failed to extract resource %s: Does not appear to be a ELF binary", filename);
+        return false;
+    }
+    if (hdr->e_ident[EI_CLASS] != ELFCLASS64)
+    {
+        DBGLOG("Failed to extract resource %s: Does not appear to be a ELF 64-bit binary", filename);
+        return false;
+    }
+
+    //Check that there is a symbol table for the sections.
+    if (hdr->e_shstrndx == SHN_UNDEF)
+    {
+        DBGLOG("Failed to extract resource %s: Does not include a section symbol table", filename);
+        return false;
+    }
+
+    //Now walk the sections comparing the section names
+    Elf64_Half numSections = hdr->e_shnum;
+    const Elf64_Shdr * sectionHeaders = reinterpret_cast<const Elf64_Shdr *>(start_addr + hdr->e_shoff);
+    const Elf64_Shdr & symbolTableSection = sectionHeaders[hdr->e_shstrndx];
+    const char * symbolTable = (const char *)start_addr + symbolTableSection.sh_offset;
+    VStringBuffer sectname("%s_%u", type, id);
+    for (unsigned iSect= 0; iSect < numSections; iSect++)
+    {
+        const Elf64_Shdr & section = sectionHeaders[iSect];
+        const char * sectionName = symbolTable + section.sh_name;
+        if (streq(sectionName, sectname))
+        {
+            data.append(section.sh_size, start_addr + section.sh_offset);
+            return true;
+        }
+    }
+
+    DBGLOG("Failed to extract resource %s: Does not include a matching entry", filename);
+    return false;
+#endif
+}
+
 extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id)
 {
 #ifdef _WIN32
@@ -277,8 +342,7 @@ extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const
         bfd_close (file);
    }
    return data.length() != 0;
-#elif defined(__APPLE__)
-    unsigned long len = 0;
+#else
     struct stat stat_buf;
     VStringBuffer sectname("%s_%u", type, id);
     int fd = open(filename, O_RDONLY);
@@ -287,31 +351,21 @@ extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const
         DBGLOG("Failed to load library %s: %d", filename, errno);
         return false;
     }
+
+    bool ok = false;
     __uint64 size = stat_buf.st_size;
-    byte *start_addr = (byte *) mmap(0, size, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
+    const byte *start_addr = (const byte *) mmap(0, size, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
     if (start_addr == MAP_FAILED)
     {
         DBGLOG("Failed to load library %s: %d", filename, errno);
     }
     else
     {
-        // The first bytes are the Mach-O header
-        struct mach_header_64 *mh = (struct mach_header_64 *) start_addr;
-        if (mh->magic != MH_MAGIC_64)
-        {
-            DBGLOG("Failed to load library %s: Does not appear to be a Mach-O 64-bit binary", filename);
-        }
-        else
-        {
-            unsigned char *data2 = getsectiondata(mh, "__TEXT", sectname.str(), &len);
-            data.append(len, data2);
-        }
-        munmap(start_addr, size);
+        ok = getResourceFromMappedFile(filename, start_addr, data, type, id);
+        munmap((void *)start_addr, size);
     }
     close(fd);
-    return len != 0;
-#else
-    UNIMPLEMENTED;
+    return ok;
 #endif
 }
 

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -37,7 +37,7 @@
       example data used in this manual are fictitious. Any similarity to
       actual persons, living or dead, is purely coincidental.</para>
 
-      <para> </para>
+      <para></para>
     </legalnotice>
 
     <xi:include href="common/Version.xml" xpointer="FooterInfo"
@@ -262,7 +262,8 @@
         packages will fail to install if their dependencies are missing from
         the target system.</para>
 
-        <para>Packages are available from the HPCC Systems<superscript>®</superscript> website: <ulink
+        <para>Packages are available from the HPCC
+        Systems<superscript>®</superscript> website: <ulink
         url="http://hpccsystems.com/download/free-community-edition">http://hpccsystems.com/download/free-community-edition</ulink></para>
 
         <para>To install the package, follow the appropriate installation
@@ -773,8 +774,8 @@
 
         <para>This section details reconfiguring a system to use multiple
         nodes. Before you start this section, you must have already downloaded
-        the correct packages for your distro from the HPCC Systems<superscript>®</superscript> website:
-        <ulink
+        the correct packages for your distro from the HPCC
+        Systems<superscript>®</superscript> website: <ulink
         url="http://hpccsystems.com/download/free-community-edition">http://hpccsystems.com/download/free-community-edition</ulink>.</para>
 
         <orderedlist>
@@ -2068,8 +2069,8 @@ OUTPUT(ValidWords)
         </listitem>
       </itemizedlist></para>
 
-    <para>The HPCC Systems<superscript>®</superscript> Portal is also a valuable resource for more
-    information including:</para>
+    <para>The HPCC Systems<superscript>®</superscript> Portal is also a
+    valuable resource for more information including:</para>
 
     <itemizedlist spacing="compact">
       <listitem>
@@ -2133,8 +2134,9 @@ OUTPUT(ValidWords)
         <title>install-cluster.sh</title>
 
         <para><emphasis
-        role="bold">install-cluster.sh</emphasis><emphasis><emphasis> [-k] [-n
-        &lt;value&gt;] &lt;package-name&gt; </emphasis></emphasis></para>
+        role="bold">install-cluster.sh</emphasis><emphasis><emphasis> [-k | -p
+        &lt;directory&gt;] [-n &lt;value&gt;] &lt;package-name&gt;
+        </emphasis></emphasis></para>
 
         <para><informaltable colsep="0" frame="none" rowsep="0">
             <tgroup cols="2">
@@ -2150,13 +2152,26 @@ OUTPUT(ValidWords)
                 </row>
 
                 <row>
-                  <entry>-k</entry>
+                  <entry>-h</entry>
+
+                  <entry>Help. Optional.</entry>
+                </row>
+
+                <row>
+                  <entry>-k, --newkey</entry>
 
                   <entry>When specified, the script generates and distributes
                   ssh keys to all hosts. Optional.</entry>
                 </row>
 
                 <row>
+                  <entry>-p, --pushkeydir</entry>
+
+                  <entry>Push existing ssh key to remote machine. Optional.
+                  Use either -k or -p, not both.</entry>
+                </row>
+
+                <row>
                   <entry>-n, --concurrent</entry>
 
                   <entry>When specified, denotes the number of concurrent
@@ -2199,6 +2214,12 @@ OUTPUT(ValidWords)
           </listitem>
 
           <listitem>
+            <para>optionally, if you specify the -p option it pushes out the
+            existing ssh keys to all nodes specified. Use either the -k or the
+            -p option, but not both.</para>
+          </listitem>
+
+          <listitem>
             <para>optionally, if you specify the -n &lt;value&gt; option it
             spawns that many concurrent executions. Default is 5.</para>
           </listitem>
@@ -2921,8 +2942,8 @@ add1(10);
       <sect2 id="Add_On_Javascript" role="brk">
         <title>JavaScript</title>
 
-        <para>To enable JavaScript support within the HPCC Systems<superscript>®</superscript>
-        Platform:</para>
+        <para>To enable JavaScript support within the HPCC
+        Systems<superscript>®</superscript> Platform:</para>
 
         <orderedlist>
           <listitem>
@@ -3044,7 +3065,8 @@ split_words('Once upon a time');
       <sect2 id="R" role="brk">
         <title>R</title>
 
-        <para>To enable R support within The HPCC Systems<superscript>®</superscript> Platform:</para>
+        <para>To enable R support within The HPCC
+        Systems<superscript>®</superscript> Platform:</para>
 
         <orderedlist>
           <listitem>

ecl/eclccserver/eclccserver.cpp

@@ -390,19 +390,19 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
                 realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
 
                 StringBuffer wuXML;
-                if (getWorkunitXMLFromFile(realdllfilename, wuXML))
+                if (!getWorkunitXMLFromFile(realdllfilename, wuXML))
+                    throw makeStringException(999, "Failed to extract workunit from query dll");
+
+                Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML);
+                queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, true);
+                workunit->setIsClone(false);
+                const char *jobname = embeddedWU->queryJobName();
+                if (jobname && *jobname) //let ECL win naming job during initial compile
+                    workunit->setJobName(jobname);
+                if (!workunit->getDebugValueBool("obfuscateOutput", false))
                 {
-                    Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML);
-                    queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, true);
-                    workunit->setIsClone(false);
-                    const char *jobname = embeddedWU->queryJobName();
-                    if (jobname && *jobname) //let ECL win naming job during initial compile
-                        workunit->setJobName(jobname);
-                    if (!workunit->getDebugValueBool("obfuscateOutput", false))
-                    {
-                        Owned<IWUQuery> query = workunit->updateQuery();
-                        query->setQueryText(eclQuery.s.str());
-                    }
+                    Owned<IWUQuery> query = workunit->updateQuery();
+                    query->setQueryText(eclQuery.s.str());
                 }
 
                 createUNCFilename(realdllfilename.str(), dllurl);

ecl/hql/hqlexpr.cpp

@@ -1935,6 +1935,10 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_assertdistributed:
     case no_extractresult:
         return childdataset_dataset;
+    case no_alias_scope:
+        if (expr->isDataset())
+            return childdataset_dataset_noscope;
+        return childdataset_none;
     case no_keyedlimit:
     case no_preload:
     case no_limit:
@@ -1945,7 +1949,6 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_split:
     case no_spill:
     case no_activerow:
-    case no_alias_scope:
     case no_executewhen:  //second argument is independent of the other arguments
     case no_selectnth:
     case no_readspill:
@@ -2123,9 +2126,12 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_nwayjoin:
     case no_nwaymerge:
         return 0;       //??
+    case no_alias_scope:
+        if (dataset->isDataset())
+            return 1;
+        return 0;
     case no_selfjoin:
     case no_alias_project:
-    case no_alias_scope:
     case no_newaggregate:
     case no_apply:
     case no_cachealias:

ecl/hql/hqlutil.cpp

@@ -8845,6 +8845,8 @@ IHqlExpression * queryTransformAssign(IHqlExpression * transform, IHqlExpression
     ForEachChild(i, transform)
     {
         IHqlExpression * cur = transform->queryChild(i);
+        if (cur->getOperator() == no_alias_scope)
+            cur = cur->queryChild(0);
         switch (cur->getOperator())
         {
         case no_assignall:

ecl/hqlcpp/hqlres.cpp

@@ -544,9 +544,17 @@ bool ResourceManager::flush(StringBuffer &filename, const char *basename, bool f
         const char *type = s.type.str();
         unsigned id = s.id;
         VStringBuffer binfile("%s_%s_%u.bin", filename.str(), type, id);
+        VStringBuffer label("%s_%u_txt_start", type, id);
+#if defined(__APPLE__)
         fprintf(f, " .section __TEXT,%s_%u\n", type, id);
-        fprintf(f, " .global _%s_%u_txt_start\n", type, id);  // For some reason apple needs a leading underbar and linux does not
-        fprintf(f, "_%s_%u_txt_start:\n", type, id);
+        fprintf(f, " .global _%s\n", label.str());  // For some reason apple needs a leading underbar and linux does not
+        fprintf(f, "_%s:\n", label.str());
+#else
+        fprintf(f, " .section %s_%u,\"a\"\n", type, id);
+        fprintf(f, " .global %s\n", label.str());
+        fprintf(f, " .type %s,STT_OBJECT\n", label.str());
+        fprintf(f, "%s:\n", label.str());
+#endif
         fprintf(f, " .incbin \"%s\"\n", binfile.str());
         FILE *bin = fopen(binfile, "wb");
         if (!bin)

initfiles/bash/etc/init.d/hpcc_common.in

@@ -282,7 +282,7 @@ set_componentvars() {
     comp.getByName ${compName}
     compPath=`echo $comp_return | cut -d ' ' -f 2 | cut -d '=' -f 2 `
     compType=`echo $comp_return | cut -d ' ' -f 1 | cut -d '=' -f 2 `
-    PIDPATH=${pid}/${compName}_init.pid
+    PIDPATH=${pid}/init_${compName}.pid
     LOCKPATH=${lock}/$compName/$compName.lock
     COMPPIDPATH=${pid}/${compName}.pid
 }
@@ -298,7 +298,7 @@ configGenCmd() {
 
     # Creating logfiles for component
     logDir=$log/${compName}
-    logFile=$log/${compName}/${compName}_init.log
+    logFile=$log/${compName}/init_${compName}.log
 
     configcmd="${configgen_path}/configgen -env ${envfile} -od ${runtime} -id ${componentFile} -c ${compName}"
     if [ ${DEBUG} != "NO_DEBUG" ]; then
@@ -385,9 +385,8 @@ createRuntime() {
 # cleanup all standard files made during runtime
 cleanupRuntimeEnvironment() {
     unlock ${lock}/${compName}/${compName}.lock
-    removePid ${pid}/${compName}_init.pid
-    removePid ${pid}/${compName}.pid
     removePid ${pid}/init_${compName}.pid
+    removePid ${pid}/${compName}.pid
 }
 
 
@@ -423,7 +422,7 @@ startCmd() {
 
     # Creating logfiles for component
     logDir=$log/${compName}
-    logFile=$log/${compName}/${compName}_init.log
+    logFile=$log/${compName}/init_${compName}.log
 
 
     if [ ${noStatusCheck} -ne 1 ]; then
@@ -470,7 +469,7 @@ startCmd() {
     fi
 
     EXEC_COMMAND="${bin_path}/init_${compType} "
-    startcmd="${START_STOP_DAEMON} -S -p ${pid}/${compName}_init.pid -c ${user}:${group} -d ${compPath} ${UMASK_ARG} -m -x ${EXEC_COMMAND} -b  >>${logFile} 2>&1"
+    startcmd="${START_STOP_DAEMON} -S -p ${pid}/init_${compName}.pid -c ${user}:${group} -d ${compPath} ${UMASK_ARG} -m -x ${EXEC_COMMAND} -b  >>${logFile} 2>&1"
 
     issueTime=`date`
     logCommand="COMMAND:: $startcmd  ::Issued at $issueTime "
@@ -563,7 +562,6 @@ stop_component() {
     eval $stopcmd
 
     unlock ${LOCKPATH}
-    removePid ${PIDPATH}
 
     RESULT=0
     local waittime=30
@@ -609,7 +607,7 @@ start_component() {
 
     # Creating logfiles for component
     logDir=$log/${compName}
-    logFile=$log/${compName}/${compName}_init.log
+    logFile=$log/${compName}/init_${compName}.log
 
     if [ ! -d $logDir ]; then
         mkdir -p $logDir >> tmp.txt 2>&1

initfiles/bin/init_configesp

@@ -16,21 +16,15 @@
 ################################################################################
 
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
-
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
+PID_NAME="$PID/$(basename $PWD).pid"
 
 export SENTINEL="configesp.sentinel"
 rm -f ${SENTINEL}
 
-
-
-SNMPID=$$
-
-killed() {
+killed()
+{
     kill_process ${PID_NAME} configesp 3 ${SENTINEL}
     exit 255
 }
@@ -42,9 +36,9 @@ echo $! > $PID_NAME
 wait
 rm $PID_NAME
 
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     sleep 5
-    if [ -e ${SENTINEL} ]; then
+    if [[ -e ${SENTINEL} ]]; then
         nohup configesp 1>/dev/null 2>/dev/null &
         echo $! > $PID_NAME
         wait

initfiles/bin/init_dafilesrv.in

@@ -22,17 +22,12 @@ if [ -z $1 ]; then
 else
         log=$1
 fi
-
 shift
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
-
-INSTANCE_NAME="`basename $PWD`"
-
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
+PID_NAME="$PID/$(basename $PWD).pid"
+COMP_NAME="$(basename $PWD)"
 
 # this must match jsocket hard limit
 export handlelimit=32768
@@ -40,7 +35,8 @@ export handlelimit=32768
 export SENTINEL="dafilesrv.sentinel"
 rm -f ${SENTINEL}
 
-killed(){
+killed()
+{
     kill_process ${PID_NAME} dafilesrv 3 ${SENTINEL}
     exit 255
 }
@@ -49,15 +45,15 @@ ulimit -c unlimited
 ulimit -n $handlelimit
 
 trap "killed" SIGINT SIGTERM SIGKILL
-dafilesrv -L $log -I ${INSTANCE_NAME} &
+dafilesrv -L $log -I ${COMP_NAME} &
 echo $! > $PID_NAME
 wait
 rm $PID_NAME
 
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     sleep 5
-    if [ -e ${SENTINEL} ]; then
-        dafilesrv -L $log -I ${INSTANCE_NAME} &
+    if [[ -e ${SENTINEL} ]]; then
+        dafilesrv -L $log -I ${COMP_NAME} &
         echo $! > $PID_NAME
         wait
         rm $PID_NAME

initfiles/bin/init_dali

@@ -15,20 +15,15 @@
 #    limitations under the License.
 ################################################################################
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
-
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
+PID_NAME="$PID/$(basename $PWD).pid"
 
 export SENTINEL="daserver.sentinel"
 rm -f ${SENTINEL}
 
-
-
-
-killed(){
+killed()
+{
     dalistop .
     kill_process ${PID_NAME} daserver 3 ${SENTINEL}
     exit 255
@@ -40,9 +35,9 @@ echo $! > $PID_NAME
 wait
 rm $PID_NAME
 
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     sleep 5
-    if [ -e ${SENTINEL} ]; then
+    if [[ -e ${SENTINEL} ]]; then
         daserver 1>/dev/null 2>/dev/null &
         echo $! > $PID_NAME
         wait

initfiles/bin/init_dfuserver

@@ -15,24 +15,16 @@
 #    limitations under the License.
 ################################################################################
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
-
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
+PID_NAME="$PID/$(basename $PWD).pid"
 
 export SENTINEL="dfuserver.sentinel"
 rm -f ${SENTINEL}
 
-
-
-
-
-killed() {
-    dfuserver stop=1
-    sleep 5
-    kill_process ${PID_NAME} dfuserver 3 ${SENTINEL}
+killed()
+{
+    kill_process ${PID_NAME} dfuserver 15 ${SENTINEL}
     exit 255
 }
 
@@ -43,9 +35,9 @@ echo $! > $PID_NAME
 wait
 rm $PID_NAME
 
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     sleep 5
-    if [ -e ${SENTINEL} ]; then
+    if [[ -e ${SENTINEL} ]]; then
         dfuserver 1>/dev/null 2>/dev/null &
         echo $! > $PID_NAME
         wait

initfiles/bin/init_eclagent.in

@@ -17,35 +17,31 @@
 
 ###<REPLACE>###
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
-
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
+PID_NAME="$PID/$(basename $PWD).pid"
 
 export SENTINEL="agentexec.sentinel"
 rm -f ${SENTINEL}
 
 rm -f ${PID_DIR}/hthortemp/*
 
-killed (){
+killed ()
+{
     kill_process ${PID_NAME} agentexec 3 ${SENTINEL}
     exit 255
 }
 
-
 trap "killed" SIGINT SIGTERM SIGKILL
 
-
 agentexec 1>/dev/null 2>/dev/null &
 echo $! > $PID_NAME
 wait
 rm $PID_NAME
 
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     sleep 1
-    if [ -e ${SENTINEL} ]; then
+    if [[ -e ${SENTINEL} ]]; then
         agentexec 1>/dev/null 2>/dev/null &
         echo $! > $PID_NAME
         wait

initfiles/bin/init_eclccserver

@@ -15,19 +15,15 @@
 #    limitations under the License.
 ################################################################################
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
-
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
+PID_NAME="$PID/$(basename $PWD).pid"
 
 export SENTINEL="eclccserver.sentinel"
 rm -f ${SENTINEL}
 
-
-
-killed() {
+killed()
+{
     kill_process ${PID_NAME} eclccserver 3 ${SENTINEL}
     exit 255
 }
@@ -38,9 +34,9 @@ echo $! > $PID_NAME
 wait
 rm $PID_NAME
 
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     sleep 5
-    if [ -e ${SENTINEL} ]; then
+    if [[ -e ${SENTINEL} ]]; then
         eclccserver 1>/dev/null 2>/dev/null & 
         echo $! > $PID_NAME
         wait

initfiles/bin/init_eclscheduler

@@ -15,17 +15,15 @@
 #    limitations under the License.
 ################################################################################
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
-
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
+PID_NAME="$PID/$(basename $PWD).pid"
 
 export SENTINEL="eclscheduler.sentinel"
 rm -f ${SENTINEL}
 
-killed(){
+killed()
+{
     kill_process ${PID_NAME} eclscheduler 3 ${SENTINEL}
     exit 255
 }
@@ -36,9 +34,9 @@ echo $! > $PID_NAME
 wait
 rm $PID_NAME
 
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     sleep 5
-    if [ -e ${SENTINEL} ]; then
+    if [[ -e ${SENTINEL} ]]; then
         eclscheduler 1>/dev/null 2>/dev/null &
         echo $! > $PID_NAME
         wait

initfiles/bin/init_esp

@@ -15,18 +15,13 @@
 #    limitations under the License.
 ################################################################################
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
-
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
+PID_NAME="$PID/$(basename $PWD).pid"
 
 export SENTINEL="esp.sentinel"
 rm -f ${SENTINEL}
 
-
-
 SNMPID=$$
 
 killed() {
@@ -39,9 +34,9 @@ esp snmpid=$SNMPID 1>/dev/null 2>/dev/null &
 echo $! > $PID_NAME
 wait
 rm $PID_NAME
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     sleep 5
-    if [ -e ${SENTINEL} ]; then
+    if [[ -e ${SENTINEL} ]]; then
         esp snmpid=$SNMPID 1>/dev/null 2>/dev/null &
         echo $! > $PID_NAME
         wait

initfiles/bin/init_roxie

@@ -15,12 +15,9 @@
 #    limitations under the License.
 ################################################################################
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
-
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
+PID_NAME="$PID/$(basename $PWD).pid"
 
 export SENTINEL="roxie.sentinel"
 rm -f ${SENTINEL}
@@ -29,11 +26,11 @@ if [ -n "$1" ]; then
     cd $1
 fi
 
-. ./roxievars
+source ./roxievars
 
 if [ -n "$2" ]; then
     logfilename=$2
-else    
+else
     logfilename="`date +%m_%d_%Y_%H_%M_%S`"
 fi
 
@@ -43,8 +40,9 @@ export restarts=0
 ulimit -c unlimited
 ulimit -n $NUM_ROXIE_HANDLES
 
-killed() {
-    if [ -n "$1" ]; then
+killed()
+{
+    if [[ -n "$1" ]]; then
         cd $1
     fi
     kill_process ${PID_NAME} roxie 3 ${SENTINEL}
@@ -58,7 +56,7 @@ wait
 rm $PID_NAME
 
 # Automatically restart roxie when it dies
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     export restarts=$(($restarts+1))
     echo Restarting $restarts >> $logfilename.stderr
     echo Restarting $restarts >> $logfilename.stdout

initfiles/bin/init_roxie_cluster

@@ -15,28 +15,26 @@
 #    limitations under the License.
 ################################################################################
 
-. ./roxievars
-
-killed () {
-    . ./roxievars
+source ./roxievars
 
+killed ()
+{
     echo --------------------------
     echo stopping roxie
-    if [ "$useSSH" = "true" ]; then
+    if [[ "$useSSH" = "true" ]]; then
        frunssh slaves "PATH=$PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH init_stopccd $roxiedir" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries 2>&1 | egrep -v "no process killed"
-    else        
+    else
        frunagent @slaves start "PATH=$PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH init_stopccd $roxiedir"
     fi
-    sleep 10 
+    sleep 10
     exit 255
-
 }
 
 trap "killed" SIGINT SIGTERM SIGKILL
 logfile="`date +%m_%d_%Y_%H_%M_%S`"
-if [ "$useSSH" = "true" ]; then
+if [[ "$useSSH" = "true" ]]; then
   frunssh slaves "/bin/sh -c 'PATH=$PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH init_startccd $roxiedir $logfile 1>/dev/null 2>/dev/null &'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries 2>&1
 else
-  frunagent @slaves start "PATH=$PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH init_startccd $roxiedir $logfile 1>/dev/null 2>/dev/null &"  
+  frunagent @slaves start "PATH=$PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH init_startccd $roxiedir $logfile 1>/dev/null 2>/dev/null &"
 fi
 

initfiles/bin/init_sasha

@@ -15,40 +15,18 @@
 #    limitations under the License.
 ################################################################################
 
-PATH_PRE=`type -path hpcc_setenv`
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE}
-PID_NAME="$PID/`basename $PWD`.pid"
+PID_NAME="$PID/$(basename $PWD).pid"
 
-INSTALL_DIR=`dirname ${PATH_PRE}`/..
+INSTALL_DIR="$(dirname ${PATH_PRE})/.."
 source  ${INSTALL_DIR}/etc/init.d/hpcc_common
 
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
-echo $$ > $INIT_PID_NAME
-
 export SENTINEL="saserver.sentinel"
 rm -f ${SENTINEL}
 
-
-
-
-which_pidof
-
-killed() {
-    pid=`${PIDOF} saserver`
-    if [ -n "$pid" ]; then
-        sasha server=. action=stop
-        I=1
-        while [ -n "$pid" ] && [ $I -lt 30 ]; do
-            echo "Waiting for saserver to stop"
-            sleep 2
-            pid=`${PIDOF} saserver`
-            let I=I+1
-        
-        done
-        if [ -n "$pid" ]; then
-            kill -9 $pid
-        fi
-    fi
+killed()
+{
     kill_process ${PID_NAME} saserver 3 ${SENTINEL}
     exit 255
 }
@@ -59,9 +37,9 @@ echo $! > $PID_NAME
 wait
 rm $PID_NAME
 
-while [ -e ${SENTINEL} ]; do
+while [[ -e ${SENTINEL} ]]; do
     sleep 5
-    if [ -e ${SENTINEL} ]; then
+    if [[ -e ${SENTINEL} ]]; then
         saserver 1>/dev/null 2>/dev/null &
         echo $! > $PID_NAME
         wait

initfiles/componentfiles/configxml/dafilesrv.xsd

@@ -54,6 +54,27 @@
                                 </xs:appinfo>
                             </xs:annotation>
                         </xs:attribute>
+                        <xs:attribute name="parallelRequestLimit" type="xs:nonNegativeInteger" use="optional" default="20">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <tooltip>Defines the maximum number of concurrent dafilesrv requests allowed. Requests that exceed the limit will be delayed. A value of 0 disables throttling. Overrides global settting.</tooltip>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                        <xs:attribute name="throttleDelayMs" type="xs:nonNegativeInteger" use="optional" default="5000">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <tooltip>Defines how many milliseconds delayed requests will be delayed by. Overrides global settting.</tooltip>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                        <xs:attribute name="throttleCPULimit" type="nonNegativeInteger" use="optional" default="75">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <tooltip>If after the initial delay, the CPU % falls below this setting, the transaction will be allowed to continue, i.e. the limit can be exceeded this way. Overrides global settting.</tooltip>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
                     </xs:complexType>
                 </xs:element>
             </xs:sequence>
@@ -93,6 +114,27 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="parallelRequestLimit" type="xs:nonNegativeInteger" use="optional" default="20">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Defines the maximum number of concurrent dafilesrv requests allowed. Requests that exceed the limit will be delayed. A value of 0 disables throttling. This a global setting.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="throttleDelayMs" type="xs:nonNegativeInteger" use="optional" default="5000">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Defines how many milliseconds delayed requests will be delayed by. This a global setting.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="throttleCPULimit" type="nonNegativeInteger" use="optional" default="75">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>If after the initial delay, the CPU % falls below this setting, the transaction will be allowed to continue, i.e. the limit can be exceeded this way. This a global setting.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
         </xs:complexType>
     </xs:element>
 </xs:schema>

roxie/ccd/ccdserver.cpp

@@ -5270,7 +5270,6 @@ class CRoxieServerWorkUnitReadActivity : public CRoxieServerActivity
 {
     IHThorWorkunitReadArg &helper;
     Owned<IWorkUnitRowReader> wuReader; // MORE - can we use IRoxieInput instead?
-
 public:
     CRoxieServerWorkUnitReadActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
         : CRoxieServerActivity(_factory, _probeManager), helper((IHThorWorkunitReadArg &)basehelper)
@@ -5297,6 +5296,7 @@ public:
 
     virtual void reset() 
     {
+        CriticalBlock b(statecrit);
         wuReader.clear();
         CRoxieServerActivity::reset(); 
     };
@@ -5305,7 +5305,10 @@ public:
 
     virtual const void *nextInGroup()
     {
+        CriticalBlock b(statecrit);
         ActivityTimer t(totalCycles, timeActivities);
+        if (!wuReader)
+            return NULL;
         const void *ret = wuReader->nextInGroup();
         if (ret)
             processed++;
@@ -5720,13 +5723,17 @@ public:
 
     virtual void reset() 
     {
+        CriticalBlock b(statecrit);
         iter.clear();
         CRoxieServerActivity::reset(); 
     };
 
     virtual const void *nextInGroup()
     {
+        CriticalBlock b(statecrit);
         ActivityTimer t(totalCycles, timeActivities);
+        if (!iter)
+            return NULL;
         const void * next = iter->nextInGroup();
         if (next)
         {
@@ -6063,6 +6070,7 @@ public:
 
     virtual void reset() 
     {
+        CriticalBlock b(statecrit);
         if (iter)
             iter->reset();
         iter.clear();
@@ -6071,6 +6079,7 @@ public:
 
     virtual const void *nextInGroup()
     {
+        CriticalBlock b(statecrit);
         ActivityTimer t(totalCycles, timeActivities);
         const void * next = iter ? iter->nextInGroup() : NULL;
         if (next)

testing/regress/hpcc/util/ecl/command.py

@@ -44,7 +44,6 @@ class ECLcmd(Shell):
         server = kwargs.pop('server', False)
         if server:
             args.append('--server=' + server)
-            args.append('-XTargetIP=' + server)
 
         username = kwargs.pop('username', False)
         if username:
@@ -79,6 +78,8 @@ class ECLcmd(Shell):
 
             args = args + eclfile.getStoredInputParameters()
 
+            args.append('-XTargetIP=' + server)
+
             args.append(eclfile.getArchive())
 
         data = ""