Merge branch 'candidate-7.12.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.github/workflows/test-eclhelper.yml

@@ -0,0 +1,19 @@
+name: Check that eclhelper interface has not changed
+
+on:
+  pull_request:
+    branches:
+      - "master"
+      - "candidate-*"
+
+jobs:
+  build:
+    name: "Check eclhelper interface"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Check eclhelper
+        run: git diff HEAD^ --exit-code -- rtl/include/eclhelper.hpp

common/thorhelper/thorcommon.hpp

@@ -29,7 +29,6 @@
 #include "thorhelper.hpp"
 #include "thorxmlwrite.hpp"
 
-static unsigned const defaultDaliResultOutputMax = 2000; // MB
 static unsigned const defaultDaliResultLimit = 10; // MB
 static unsigned const defaultMaxCsvRowSize = 10; // MB
 

docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview.xml

@@ -143,19 +143,6 @@
 
       <para>An RPM installation package is provided. <programlisting>sudo yum install &lt;rpm file name&gt;</programlisting></para>
 
-      <para><variablelist>
-          <varlistentry>
-            <term>Note:</term>
-
-            <listitem>
-              <para>If you are using CentOS 6 you may need to run the
-              following command to properly enable the installation.</para>
-            </listitem>
-          </varlistentry>
-        </variablelist></para>
-
-      <para><programlisting>sudo wget http://people.centos.org/tru/devtools-2/devtools-2.repo -O /etc/yum.repos.d/devtools-2.repo</programlisting></para>
-
       <para><emphasis role="bold">Debian-Based Systems
       (Ubuntu):</emphasis></para>
 

docs/EN_US/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml

@@ -128,11 +128,6 @@
       switch, the U or upgrade will perform an upgrade if a previous version
       is already installed. <programlisting>sudo rpm -Uvh &lt;rpm file name&gt;</programlisting></para>
 
-      <para>If you are using CentOS 6 you may need to run the following
-      command to properly enable the installation.</para>
-
-      <para><programlisting>sudo wget http://people.centos.org/tru/devtools-2/devtools-2.repo -O /etc/yum.repos.d/devtools-2.repo</programlisting></para>
-
       <para><emphasis role="bold">Debian-Based Systems
       (Ubuntu):</emphasis></para>
 

docs/EN_US/HPCCMonitoring/HPCCMonitoringAndReporting.xml

@@ -3,7 +3,8 @@
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <book lang="en_US" xml:base="../">
   <bookinfo>
-    <title>HPCC Systems<superscript>®</superscript> Monitoring and Reporting (Technical Preview)</title>
+    <title>HPCC Systems<superscript>®</superscript> Monitoring and Reporting
+    (Technical Preview)</title>
 
     <mediaobject>
       <imageobject>
@@ -69,9 +70,9 @@
 
     <para><emphasis role="bold">Ganglia:</emphasis></para>
 
-    <para>The HPCC Systems monitoring component leverages Ganglia, an open source,
-    scalable, distributed monitoring system to display system information in a
-    graphical manner.</para>
+    <para>The HPCC Systems monitoring component leverages Ganglia, an open
+    source, scalable, distributed monitoring system to display system
+    information in a graphical manner.</para>
 
     <para>With the the graphical monitoring component you can: <itemizedlist>
         <listitem>
@@ -106,13 +107,13 @@
 
     <para><emphasis role="bold">Nagios</emphasis></para>
 
-    <para>The HPCC Systems reporting and alerting component leverages Nagios, a
-    powerful monitoring and notification system, which can help you identify
+    <para>The HPCC Systems reporting and alerting component leverages Nagios,
+    a powerful monitoring and notification system, which can help you identify
     and resolve infrastructure problems before they affect critical
     processes.</para>
 
-    <para>With the HPCC Systems reporting and alerting component you can set up alerts
-    to inform of any changes to:</para>
+    <para>With the HPCC Systems reporting and alerting component you can set
+    up alerts to inform of any changes to:</para>
 
     <para><itemizedlist>
         <listitem>
@@ -156,13 +157,13 @@
   <chapter id="Ganglya_Overview">
     <title>Ganglia</title>
 
-    <para>The HPCC Systems monitoring component leverages Ganglia, an open source,
-    scalable, distributed monitoring system, to produce a graphical view of a
-    Roxie cluster's servers. Ganglia leverages widely accepted technologies
-    for data representation. It provides near real-time monitoring and
-    visualizations for performance metrics. If your enterprise already has a
-    Ganglia monitoring server, you can easily add Roxie clusters to its
-    monitoring.</para>
+    <para>The HPCC Systems monitoring component leverages Ganglia, an open
+    source, scalable, distributed monitoring system, to produce a graphical
+    view of a Roxie cluster's servers. Ganglia leverages widely accepted
+    technologies for data representation. It provides near real-time
+    monitoring and visualizations for performance metrics. If your enterprise
+    already has a Ganglia monitoring server, you can easily add Roxie clusters
+    to its monitoring.</para>
 
     <sect1 id="Ganglia_Overview">
       <title>Ganglia Overview</title>
@@ -236,8 +237,8 @@
         <emphasis>/etc/ganglia/conf.d</emphasis> and
         <emphasis>/etc/ganglia/.pyconf</emphasis> files in place and then add
         the Roxie nodes you wish to monitor. You can do that by installing the
-        Ganglia components and HPCC Systems Monitoring components on to each Roxie
-        node.</para>
+        Ganglia components and HPCC Systems Monitoring components on to each
+        Roxie node.</para>
 
         <para>If you do not have Ganglia, or want to install it, read the
         Ganglia documentation provided at the above link, and install it and
@@ -247,13 +248,13 @@
         <sect3 id="Installing-HPCCGanglia" role="brk">
           <title>Installing the HPCC Systems Monitoring component</title>
 
-          <para>The HPCC Systems Monitoring component is available for download. The
-          HPCC Systems Monitoring components leverage the Ganglia monitoring tools,
-          and would only be needed if you do not already have Ganglia
-          monitoring components on your system.</para>
+          <para>The HPCC Systems Monitoring component is available for
+          download. The HPCC Systems Monitoring components leverage the
+          Ganglia monitoring tools, and would only be needed if you do not
+          already have Ganglia monitoring components on your system.</para>
 
-          <para>To get the HPCC Systems Monitoring components, find the appropriate
-          package for your system.</para>
+          <para>To get the HPCC Systems Monitoring components, find the
+          appropriate package for your system.</para>
 
           <para>Packages are available for download from the HPCC
           Systems<superscript>®</superscript> site:</para>
@@ -269,10 +270,10 @@
           <para>Find and install the appropriate package for your
           system.</para>
 
-          <para>For example, if you have a CentOS 6.x system, get the RPM
+          <para>For example, if you have a CentOS 8.x system, get the RPM
           package.</para>
 
-          <programlisting>hpccsystems-ganglia-monitoring-4.2.0-rc1.el6.x86_64.rpm</programlisting>
+          <programlisting>hpccsystems-ganglia-monitoring-7.12.18-rc1.el8.x86_64.rpm</programlisting>
 
           <para>Install the monitoring package on the system that you want to
           monitor. Optionally, you can look at that installation package
@@ -286,8 +287,8 @@
       <title>The HPCC Systems Ganglia Viewer</title>
 
       <para>A Ganglia viewer comes preinstalled and configured in the 4.2.x
-      (or later) HPCC Systems Virtual Machine. The monitoring provided with the
-      Virtual Machine is set up to monitor Roxie instances on the network.
+      (or later) HPCC Systems Virtual Machine. The monitoring provided with
+      the Virtual Machine is set up to monitor Roxie instances on the network.
       This document introduces the monitoring and describes how to get it
       working on your system. <figure>
           <title>HPCC Systems Monitoring</title>
@@ -331,9 +332,9 @@
       implement them on a larger system, is to examine the metrics in
       action.</para>
 
-      <para>Ganglia integration is built into the current HPCC Systems Virtual Machine
-      images. Download and start up a virtual image and look at how the
-      monitoring component works.</para>
+      <para>Ganglia integration is built into the current HPCC Systems Virtual
+      Machine images. Download and start up a virtual image and look at how
+      the monitoring component works.</para>
 
       <para>This allows you:</para>
 
@@ -391,8 +392,8 @@
         </listitem>
 
         <listitem>
-          <para>Deploy the monitoring daemon (gmond) and the HPCC Systems Monitoring
-          package to each of the nodes you wish to monitor.</para>
+          <para>Deploy the monitoring daemon (gmond) and the HPCC Systems
+          Monitoring package to each of the nodes you wish to monitor.</para>
         </listitem>
       </orderedlist>
 
@@ -400,9 +401,9 @@
       more Roxie nodes installed anywhere on the same network utilizing
       multi-cast.</para>
 
-      <para>To add a new Roxie node, install the HPCC Systems Monitoring package on to
-      each Roxie node to monitor. In most basic configurations you may need to
-      add the node(s) IP address(es) to the
+      <para>To add a new Roxie node, install the HPCC Systems Monitoring
+      package on to each Roxie node to monitor. In most basic configurations
+      you may need to add the node(s) IP address(es) to the
       <emphasis>/etc/ganglia/gmetad.conf</emphasis> file. As long as the new
       Roxie node can communicate with (for example ping) the Monitoring
       component host, the graphs for that will automatically be added to the
@@ -424,10 +425,10 @@
     <sect1 id="GangliaInECLWatch">
       <title>Ganglia in ECL Watch</title>
 
-      <para>With the Ganglia for HPCC Systems Plugin installed. You can view the
-      Ganglia statistics and graphs right through the ECL Watch interface. The
-      out of the box monitoring displays several key statistics by default.
-      You can customize and configure the views.</para>
+      <para>With the Ganglia for HPCC Systems Plugin installed. You can view
+      the Ganglia statistics and graphs right through the ECL Watch interface.
+      The out of the box monitoring displays several key statistics by
+      default. You can customize and configure the views.</para>
 
       <figure>
         <title>Ganglia in ECL Watch</title>
@@ -451,8 +452,8 @@
         <para>In order to get the Ganglia in ECL Watch, You need to have
         Ganglia on your HPCC Systems. <orderedlist>
             <listitem>
-              <para>Install or ensure you have the HPCC Systems Monitoring components
-              on a node where ECL Watch is installed.</para>
+              <para>Install or ensure you have the HPCC Systems Monitoring
+              components on a node where ECL Watch is installed.</para>
             </listitem>
 
             <listitem>
@@ -502,15 +503,16 @@
   <chapter id="HPCC_Nagios_Chapter">
     <title>Nagios</title>
 
-    <para>The HPCC Systems Reporting component leverages Nagios, an open source,
-    system and network infrastructure monitoring application to monitor and
-    alert HPCC Systems Administrators. Nagios leverages established and accepted open
-    source technologies to alert users to changes or potential issues. Nagios
-    provides regular periodic system monitoring and reporting.</para>
+    <para>The HPCC Systems Reporting component leverages Nagios, an open
+    source, system and network infrastructure monitoring application to
+    monitor and alert HPCC Systems Administrators. Nagios leverages
+    established and accepted open source technologies to alert users to
+    changes or potential issues. Nagios provides regular periodic system
+    monitoring and reporting.</para>
 
-    <para>With the HPCC Systems integration, you can generate Nagios configuration
-    files to monitor HPCC Systems server health. Once the Nagios is configured, you
-    can monitor:<itemizedlist>
+    <para>With the HPCC Systems integration, you can generate Nagios
+    configuration files to monitor HPCC Systems server health. Once the Nagios
+    is configured, you can monitor:<itemizedlist>
         <listitem>
           <para>Disk Usage</para>
         </listitem>
@@ -569,13 +571,14 @@
       <title>Nagios Introduction</title>
 
       <para>Nagios is a powerful monitoring and notification system, which can
-      be used with HPCC Systems to help identify and resolve infrastructure problems
-      before they affect critical processes. Nagios hardware notifications can
-      help keep your system highly available and alerts can assist in
-      pre-emptive maintenance for processes which are down or behaving outside
-      expected parameters to ensure system stability, reliability, and uptime.
-      Scripts and tools are provided to extract HPCC Systems platform system metrics
-      and easily integrate that data into Nagios.</para>
+      be used with HPCC Systems to help identify and resolve infrastructure
+      problems before they affect critical processes. Nagios hardware
+      notifications can help keep your system highly available and alerts can
+      assist in pre-emptive maintenance for processes which are down or
+      behaving outside expected parameters to ensure system stability,
+      reliability, and uptime. Scripts and tools are provided to extract HPCC
+      Systems platform system metrics and easily integrate that data into
+      Nagios.</para>
 
       <para>Administrators should note that different platforms may not
       support all plugins. The <emphasis>hpcc-nagios-tools</emphasis> utility
@@ -596,8 +599,8 @@
           </listitem>
 
           <listitem>
-            <para>Review our HPCC Systems Monitoring and Reporting documentation in
-            its entirety.</para>
+            <para>Review our HPCC Systems Monitoring and Reporting
+            documentation in its entirety.</para>
           </listitem>
 
           <listitem>
@@ -623,9 +626,9 @@
       implement it on a larger system, is to examine an established session in
       action.</para>
 
-      <para>Nagios integration is built into the current HPCC Systems Virtual Machine
-      images. Download and start up a virtual image and look at how the
-      monitoring component works.</para>
+      <para>Nagios integration is built into the current HPCC Systems Virtual
+      Machine images. Download and start up a virtual image and look at how
+      the monitoring component works.</para>
 
       <para>The Nagios component for HPCC Systems on the VM allows you:</para>
 
@@ -746,26 +749,27 @@
       <title>Installation of Nagios</title>
 
       <para>The HPCC Systems Nagios package provides tools and utilities for
-      generating Nagios configurations. These configurations check HPCC Systems and
-      perform some of the HPCC Systems specific checks. HPCC Systems Nagios installation is
-      provided on the HPCC Systems<superscript>®</superscript> portal.</para>
+      generating Nagios configurations. These configurations check HPCC
+      Systems and perform some of the HPCC Systems specific checks. HPCC
+      Systems Nagios installation is provided on the HPCC
+      Systems<superscript>®</superscript> portal.</para>
 
       <sect2 id="HPCC_Nagios_Installation">
         <title>HPCC Systems Nagios Installation Package</title>
 
-        <para>To get the HPCC Systems Nagios monitoring on your system you need the
-        Installation package. Download the installation package from the HPCC
-        Systems portal.</para>
+        <para>To get the HPCC Systems Nagios monitoring on your system you
+        need the Installation package. Download the installation package from
+        the HPCC Systems portal.</para>
 
         <para>The HPCC Systems<superscript>®</superscript> web portal is where
-        you can find HPCC Systems resources, downloads, plugins, as well as helpful
-        information.</para>
+        you can find HPCC Systems resources, downloads, plugins, as well as
+        helpful information.</para>
 
         <para><ulink
         url="http://hpccsystems.com/download/free-community-edition/monitoring">http://hpccsystems.com/</ulink></para>
 
-        <para>You can find the HPCC Systems Monitoring and Reporting Installation
-        packages at:</para>
+        <para>You can find the HPCC Systems Monitoring and Reporting
+        Installation packages at:</para>
 
         <para><ulink
         url="http://hpccsystems.com/download/free-community-edition/monitoring">http://hpccsystems.com/download/free-community-edition/monitoring</ulink></para>
@@ -777,8 +781,8 @@
       <sect2 id="HPCC_InstallNagios" role="brk">
         <title>Install Nagios</title>
 
-        <para>To Install Nagios for HPCC Systems, you must have HPCC Systems platform
-        installed and also have the open-source Nagios package
+        <para>To Install Nagios for HPCC Systems, you must have HPCC Systems
+        platform installed and also have the open-source Nagios package
         installed.</para>
 
         <para><orderedlist>
@@ -789,8 +793,8 @@
               Nagios monitoring must have network connectivity to all the
               monitored nodes.</para>
 
-              <para>With the hpcc-nagios tools installed, you have HPCC Systems check
-              utilities in:</para>
+              <para>With the hpcc-nagios tools installed, you have HPCC
+              Systems check utilities in:</para>
 
               <para><programlisting> /usr/lib/nagios/plugins/ </programlisting></para>
             </listitem>
@@ -804,9 +808,9 @@
               configurations. The generated configurations can be modified
               with optional flags to fit the environment that is being
               monitored. The default package also provides some utilities to
-              monitor HPCC Systems processes such as Roxies, ESP Services by node and
-              port, Dali, and dafilesrv. Other processes could also be
-              monitored if a check utility is provided and the generated
+              monitor HPCC Systems processes such as Roxies, ESP Services by
+              node and port, Dali, and dafilesrv. Other processes could also
+              be monitored if a check utility is provided and the generated
               config file is modified (find/replace all would probably
               suffice).</para>
             </listitem>

docs/EN_US/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -157,7 +157,7 @@
           Scripts</link>) to make starting larger multi-node systems
           easier.</para>
 
-          <para>CentOS 6, Ubuntu 14.04, and other System V users see <link
+          <para>System V users please refer to <link
           linkend="hpcc-init">Appendix: hpcc-init</link>.</para>
         </listitem>
 
@@ -293,27 +293,7 @@
           using the -Uvh options), however then you would have to negotiate
           installing any additional dependencies.</para>
 
-          <para><variablelist>
-              <varlistentry>
-                <term>Note:</term>
 
-                <listitem>
-                  <para>If you are using CentOS 6 you may need to run the
-                  following command to properly enable the
-                  installation.</para>
-                </listitem>
-              </varlistentry>
-            </variablelist></para>
-
-          <para><programlisting>sudo wget \
- http://people.centos.org/tru/devtools-2/devtools-2.repo -O /etc/yum.repos.d/devtools-2.repo</programlisting></para>
-
-          <para>This will add the devtools-2.repo file to your system. Verify
-          that the following information is in the devtools-2.repo file for
-          the yum installation to work. <programlisting>[testing-devtools-2-centos-6]
-name=testing 2 devtools for CentOS 6 
-baseurl=http://people.centos.org/tru/devtools-2/6/$basearch/RPMS
-gpgcheck=0</programlisting></para>
         </sect3>
 
         <sect3 id="Installing_Ubuntu-Debian">
@@ -1999,7 +1979,7 @@ init_start_thor--></property></para>
     <sect1 id="hpcc-init">
       <title>hpcc-init</title>
 
-      <para>CentOS 6 and other System V based operating systems do not support
+      <para>Systems utilizing System V based init systems do not support
       the systemd calls utilized by HPCC Systems. We will continue to support
       the old System V style init.d calls.</para>
 
@@ -2404,9 +2384,6 @@ add1(10);
 
             <para><emphasis role="bold">RPM-based systems:</emphasis></para>
 
-            <para>JavaScript support is available for CentOS 6.x or later (not
-            available for CentOS 5.x).</para>
-
             <para>On an RPM-based system (CentOS/Red Hat) install <emphasis
             role="bold">v8embed.</emphasis></para>
 

ecl/eclagent/eclagent.cpp

@@ -1876,6 +1876,20 @@ void EclAgent::doProcess()
                 throw makeStringException(0, "Attempting to execute a workunit that hasn't been compiled");
             if (checkVersion && ((eclccCodeVersion > ACTIVITY_INTERFACE_VERSION) || (eclccCodeVersion < MIN_ACTIVITY_INTERFACE_VERSION)))
                 failv(0, "Workunit was compiled for eclagent interface version %d, this eclagent requires version %d..%d", eclccCodeVersion, MIN_ACTIVITY_INTERFACE_VERSION, ACTIVITY_INTERFACE_VERSION);
+            if (checkVersion && eclccCodeVersion == 652)
+            {
+                // Any workunit compiled using eclcc 7.12.0-7.12.18 is not compatible
+                StringBuffer buildVersion, eclVersion;
+                w->getBuildVersion(StringBufferAdaptor(buildVersion), StringBufferAdaptor(eclVersion));
+                const char *version = strstr(buildVersion, "7.12.");
+                if (version)
+                {
+                    const char *point = version + strlen("7.12.");
+                    unsigned pointVer = atoi(point);
+                    if (pointVer <= 18)
+                        failv(0, "Workunit was compiled by eclcc version %s which is not compatible with this runtime", buildVersion.str());
+                }
+            }
             if(noRetry && (w->getState() == WUStateFailed))
                 throw MakeStringException(0, "Ecl agent started in 'no retry' mode for failed workunit, so failing");
             w->setState(WUStateRunning);

ecl/hqlcpp/hqlcerrors.hpp

@@ -225,6 +225,8 @@
 #define HQLERR_CannotInterpretRecord            4213
 #define HQLERR_NoVirtualAndAlien                4214
 #define HQLERR_CannotFilterLimitInsideActivity  4215
+#define HQLERR_OutputLimitMaxExceeded           4216
+#define HQLERR_OutputLimitFutureMaxExceeded     4217
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -533,6 +535,8 @@
 #define HQLERR_CannotInterpretRecord_Text       "This dataset contains deprecated record formats and virtual fields.  Remove the alien data types, or temporarily add __OPTION__(LEGACY) to the table definition"
 #define HQLERR_NoVirtualAndAlien_Text           "GenericDiskReads: VIRTUAL fields are not currently supported with ALIEN types"
 #define HQLERR_CannotFilterLimitInsideActivity_Text "Cannot filter a non keyed limit within a compound activity"
+#define HQLERR_OutputLimitMaxExceeded_Text      "Dali result outputs are restricted to an absolute maximum of %u MB (%u MB specified by option). A huge dali result usually indicates the ECL needs altering."
+#define HQLERR_OutputLimitFutureMaxExceeded_Text "In the next platform version dali result outputs will be restricted to an absolute maximum of %u MB (%u MB specified by option). A huge dali result usually indicates the ECL needs altering."
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlcpp.cpp

@@ -1539,6 +1539,12 @@ void HqlCppTranslator::checkAbort()
 // Without this restriction it becomes much easier.
 void HqlCppTranslator::cacheOptions()
 {
+    size32_t outputLimit = wu()->getDebugValueInt("outputLimitMb", 0);
+    if (outputLimit > daliResultOutputMax)
+        throwError2(HQLERR_OutputLimitMaxExceeded, daliResultOutputMax, outputLimit);
+    if (outputLimit > futureResultOutputMax)
+        WARNING2(CategoryDeprecated, HQLERR_OutputLimitFutureMaxExceeded, futureResultOutputMax, outputLimit);
+
     SCMStringBuffer targetText;
     wu()->getDebugValue("targetClusterType", targetText);
     ClusterType clusterType = getClusterType(targetText.s.str());

ecl/hthor/hthor.cpp

@@ -6313,8 +6313,8 @@ void CHThorWorkUnitWriteActivity::execute()
     size32_t outputLimit = agent.queryWorkUnit()->getDebugValueInt(OPT_OUTPUTLIMIT, agent.queryWorkUnit()->getDebugValueInt(OPT_OUTPUTLIMIT_LEGACY, defaultDaliResultLimit));
     if (flags & POFmaxsize)
         outputLimit = helper.getMaxSize();
-    if (outputLimit>defaultDaliResultOutputMax)
-        throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", defaultDaliResultOutputMax, defaultDaliResultLimit);
+    if (outputLimit>daliResultOutputMax)
+        throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", daliResultOutputMax, defaultDaliResultLimit);
     assertex(outputLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
     outputLimit *= 0x100000;
     MemoryBuffer rowdata;

esp/bindings/http/client/httpclient.cpp

@@ -402,15 +402,8 @@ HttpClientErrCode CHttpClient::sendRequest(const char* method, const char* conte
     {
         StringBuffer uidpair;
         uidpair.append(m_userid.get()).append(":").append(m_password.get());
-        StringBuffer result;
-        JBASE64_Encode(uidpair.str(), uidpair.length(), result, false);
         StringBuffer authhdr("Basic ");
-
-        //Remove the \n from the end of the encoded string.
-        //Should it even be there??
-        result.setCharAt(result.length() - 1,0);
-
-        authhdr.append(result.str());
+        JBASE64_Encode(uidpair.str(), uidpair.length(), authhdr, false);
         httprequest->addHeader("Authorization", authhdr.str());
     }
     if(m_realm.length() > 0)
@@ -707,11 +700,8 @@ HttpClientErrCode CHttpClient::sendRequest(IProperties *headers, const char* met
     {
         StringBuffer uidpair;
         uidpair.append(m_userid.get()).append(":").append(m_password.get());
-        StringBuffer result;
-        JBASE64_Encode(uidpair.str(), uidpair.length(), result, false);
         StringBuffer authhdr("Basic ");
-
-        authhdr.append(result.str());
+        JBASE64_Encode(uidpair.str(), uidpair.length(), authhdr, false);
         httprequest->addHeader("Authorization", authhdr.str());
     }
     if(m_realm.length() > 0)
@@ -894,15 +884,8 @@ HttpClientErrCode CHttpClient::postRequest(ISoapMessage &req, ISoapMessage& resp
     {
         StringBuffer uidpair;
         uidpair.append(m_userid.get()).append(":").append(m_password.get());
-        StringBuffer result;
-        JBASE64_Encode(uidpair.str(), uidpair.length(), result, false);
         StringBuffer authhdr("Basic ");
-
-        //Remove the \n from the end of the encoded string.
-        //Should it even be there??
-        result.setCharAt(result.length() - 1,0);
-
-        authhdr.append(result.str());
+        JBASE64_Encode(uidpair.str(), uidpair.length(), authhdr, false);
         httprequest->addHeader("Authorization", authhdr.str());
         if(m_proxy.length())
             httprequest->addHeader("Proxy-Authorization", authhdr.str());

esp/services/ws_fs/ws_fsService.cpp

@@ -2367,6 +2367,7 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
         IDFUfileSpec *source = wu->queryUpdateSource();
         IDFUfileSpec *destination = wu->queryUpdateDestination();
         IDFUoptions *options = wu->queryUpdateOptions();
+        bool preserveFileParts = req.getWrap();
 
         source->setLogicalName(srcname);
 
@@ -2379,6 +2380,10 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
 
             StringBuffer destfileWithPath, umask;
             getDropZoneInfoByIP(version, destip, destfile, destfileWithPath, umask);
+            //Ensure the filename is dependent on the file part if parts are being preserved
+            if (preserveFileParts && !strstr(destfileWithPath, "$P$"))
+                destfileWithPath.append("._$P$_of_$N$");
+
             rfn.setPath(ep, destfileWithPath.str());
             if (umask.length())
                 options->setUMask(umask.str());
@@ -2419,7 +2424,7 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
         if(req.getNorecover())
             options->setNoRecover(true);
 
-        if (req.getWrap()) {
+        if (preserveFileParts) {
             options->setPush();             // I think needed for a despray
             destination->setWrap(true);
         }

roxie/ccd/ccdquery.cpp

@@ -1784,6 +1784,20 @@ unsigned checkWorkunitVersionConsistency(const IConstWorkUnit *wu)
         throw makeStringException(ROXIE_MISMATCH, "Attempting to execute a workunit that hasn't been compiled");
     if (wuVersion > ACTIVITY_INTERFACE_VERSION || wuVersion < MIN_ACTIVITY_INTERFACE_VERSION)
         throw MakeStringException(ROXIE_MISMATCH, "Workunit was compiled for eclhelper interface version %d, this roxie requires version %d..%d", wuVersion, MIN_ACTIVITY_INTERFACE_VERSION, ACTIVITY_INTERFACE_VERSION);
+    if (wuVersion == 652)
+    {
+        // Any workunit compiled using eclcc 7.12.0-7.12.18 is not compatible
+        StringBuffer buildVersion, eclVersion;
+        wu->getBuildVersion(StringBufferAdaptor(buildVersion), StringBufferAdaptor(eclVersion));
+        const char *version = strstr(buildVersion, "7.12.");
+        if (version)
+        {
+            const char *point = version + strlen("7.12.");
+            unsigned pointVer = atoi(point);
+            if (pointVer <= 18)
+                throw MakeStringException(ROXIE_MISMATCH, "Workunit was compiled by eclcc version %s which is not compatible with this runtime", buildVersion.str());
+        }
+    }
     return wuVersion;
 }
 

roxie/ccd/ccdserver.cpp

@@ -21357,8 +21357,8 @@ public:
                 // In absense of OPT_OUTPUTLIMIT check pre 5.2 legacy name OPT_OUTPUTLIMIT_LEGACY
                 outputLimit = workunit->getDebugValueInt(OPT_OUTPUTLIMIT, workunit->getDebugValueInt(OPT_OUTPUTLIMIT_LEGACY, defaultDaliResultLimit));
             }
-            if (outputLimit>defaultDaliResultOutputMax)
-                throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", defaultDaliResultOutputMax, defaultDaliResultLimit);
+            if (outputLimit>daliResultOutputMax)
+                throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", daliResultOutputMax, defaultDaliResultLimit);
             assertex(outputLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
             outputLimitBytes = outputLimit * 0x100000;
         }

rtl/eclrtl/rtldynfield.cpp

@@ -41,17 +41,16 @@ extern ECLRTL_API RecordTranslationMode getTranslationMode(const char *val, bool
     else if (strieq(val, "alwaysDisk") || strieq(val, "disk"))
     {
         if (!isLocal)
-            throw makeStringException(0, "alwaysDisk translation mode can only be set via HINT");
+            WARNLOG("alwaysDisk translation mode should only ever be used via a HINT");
         return RecordTranslationMode::AlwaysDisk;
     }
     else if (strieq(val, "alwaysECL") || strieq(val, "ecl"))
     {
-        if (!isLocal)
-            throw makeStringException(0, "alwaysECL translation mode can only be set via HINT");
-        return RecordTranslationMode::AlwaysECL;
+        if (isLocal)
+            return RecordTranslationMode::AlwaysECL;
+        WARNLOG("Unsupported alwaysECL translation mode used globally, translation disabled - use with HINT to set locally.");
     }
-    else
-        return RecordTranslationMode::None;
+    return RecordTranslationMode::None;
 }
 
 extern ECLRTL_API const char *getTranslationModeText(RecordTranslationMode val)

rtl/include/eclhelper.hpp

@@ -173,7 +173,6 @@ interface IRecordSize : public IInterface
 interface IXmlWriter : public IInterface
 {
 public:
-    virtual void flushContent(bool close) = 0;
     virtual void outputQuoted(const char *text) = 0;
     virtual void outputString(unsigned len, const char *field, const char *fieldname) = 0;
     virtual void outputBool(bool field, const char *fieldname) = 0;
@@ -196,6 +195,7 @@ public:
     virtual void outputInlineXml(const char *text) = 0; //for appending raw xml content
     virtual void outputXmlns(const char *name, const char *uri) = 0;
     inline void outputCString(const char *field, const char *fieldname) { outputString((size32_t)strlen(field), field, fieldname); }
+    virtual void flushContent(bool close) = 0;
 };
 
 interface IFieldProcessor : public IInterface
@@ -2966,6 +2966,9 @@ protected:
     Owned<IColumnProvider> cur;
 };
 
+constexpr unsigned daliResultOutputMax = 2000; // MB
+constexpr unsigned futureResultOutputMax = 100; // MB
+
 #ifdef STARTQUERY_EXPORTS
 #define STARTQUERY_API DECL_EXPORT
 #else

thorlcr/activities/wuidwrite/thwuidwrite.cpp

@@ -80,8 +80,8 @@ public:
         CMasterActivity::init();
         // In absense of OPT_OUTPUTLIMIT check pre 5.2 legacy name OPT_OUTPUTLIMIT_LEGACY
         workunitWriteLimit = activityMaxSize ? activityMaxSize : getOptInt(OPT_OUTPUTLIMIT, getOptInt(OPT_OUTPUTLIMIT_LEGACY, defaultDaliResultLimit));
-        if (workunitWriteLimit>defaultDaliResultOutputMax)
-            throw MakeActivityException(this, 0, "Configured max result size, %d MB, exceeds absolute max limit of %d MB. A huge Dali result usually indicates the ECL needs altering.", workunitWriteLimit, defaultDaliResultOutputMax);
+        if (workunitWriteLimit>daliResultOutputMax)
+            throw MakeActivityException(this, 0, "Configured max result size, %d MB, exceeds absolute max limit of %d MB. A huge Dali result usually indicates the ECL needs altering.", workunitWriteLimit, daliResultOutputMax);
         assertex(workunitWriteLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
         workunitWriteLimit *= 0x100000;
     }

thorlcr/master/thgraphmanager.cpp

@@ -429,6 +429,20 @@ bool CJobManager::execute(IConstWorkUnit *workunit, const char *wuid, const char
             throw makeStringException(0, "Attempting to execute a workunit that hasn't been compiled");
         if ((workunit->getCodeVersion() > ACTIVITY_INTERFACE_VERSION) || (workunit->getCodeVersion() < MIN_ACTIVITY_INTERFACE_VERSION))
             throw MakeStringException(0, "Workunit was compiled for eclagent interface version %d, this thor requires version %d..%d", workunit->getCodeVersion(), MIN_ACTIVITY_INTERFACE_VERSION, ACTIVITY_INTERFACE_VERSION);
+        if (workunit->getCodeVersion() == 652)
+        {
+            // Any workunit compiled using eclcc 7.12.0-7.12.18 is not compatible
+            StringBuffer buildVersion, eclVersion;
+            workunit->getBuildVersion(StringBufferAdaptor(buildVersion), StringBufferAdaptor(eclVersion));
+            const char *version = strstr(buildVersion, "7.12.");
+            if (version)
+            {
+                const char *point = version + strlen("7.12.");
+                unsigned pointVer = atoi(point);
+                if (pointVer <= 18)
+                    throw MakeStringException(0, "Workunit was compiled by eclcc version %s which is not compatible with this runtime", buildVersion.str());
+            }
+        }
 
         if (debugListener)
         {