Merge branch 'candidate-6.0.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/FindHIREDIS.cmake

@@ -31,6 +31,27 @@ IF (NOT HIREDIS_FOUND)
   FIND_PATH(HIREDIS_INCLUDE_DIR hiredis/hiredis.h PATHS /usr/include /usr/share/include /usr/local/include PATH_SUFFIXES hiredis)
   FIND_LIBRARY(HIREDIS_LIBRARY NAMES ${libhiredis} PATHS /usr/lib /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64)
 
+  IF(HIREDIS_INCLUDE_DIR)
+    #MAJOR
+    FILE (STRINGS "${HIREDIS_INCLUDE_DIR}/hiredis/hiredis.h" major REGEX "#define HIREDIS_MAJOR")
+    STRING(REGEX REPLACE "#define HIREDIS_MAJOR " "" major "${major}")
+    STRING(REGEX REPLACE "\"" "" major "${major}")
+    #MINOR
+    FILE (STRINGS "${HIREDIS_INCLUDE_DIR}/hiredis/hiredis.h" minor REGEX "#define HIREDIS_MINOR")
+    STRING(REGEX REPLACE "#define HIREDIS_MINOR " "" minor "${minor}")
+    STRING(REGEX REPLACE "\"" "" minor "${minor}")
+    #PATCH
+    FILE (STRINGS "${HIREDIS_INCLUDE_DIR}/hiredis/hiredis.h" patch REGEX "#define HIREDIS_PATCH")
+    STRING(REGEX REPLACE "#define HIREDIS_PATCH " "" patch "${patch}")
+    STRING(REGEX REPLACE "\"" "" patch "${patch}")
+
+    SET(HIREDIS_VERSION_STRING "${major}.${minor}.${patch}")
+    IF ("${HIREDIS_VERSION_STRING}" VERSION_LESS "${HIREDIS_FIND_VERSION}")
+      MESSAGE("WARNING - connection caching not avaliable with libhiredis version '${HIREDIS_VERSION_STRING}' as incompatible with min version>=${HIREDIS_FIND_VERSION}")
+    ENDIF()
+  ENDIF()
+
+
   include(FindPackageHandleStandardArgs)
   find_package_handle_standard_args(hiredis DEFAULT_MSG
     HIREDIS_LIBRARY

cmake_modules/commonSetup.cmake

@@ -397,11 +397,14 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       endif ()
     endif ()
     if (CMAKE_COMPILER_IS_CLANGXX)
-      SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -Werror=logical-op-parentheses -Werror=bool-conversions -Werror=return-type -Werror=comment")
+      SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=logical-op-parentheses -Werror=bool-conversions -Werror=return-type -Werror=comment")
+      if (APPLE)
+        SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+      endif ()
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Werror=bitwise-op-parentheses -Werror=tautological-compare")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Wno-switch-enum -Wno-format-zero-length -Wno-switch")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Qunused-arguments")  # Silence messages about pthread not being used when linking...
-      SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Wno-inconsistent-missing-override")   # Until we fix them all, whcih would be a huge task...      
+      SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Wno-inconsistent-missing-override -Wno-unknown-warning-option")  # Until we fix them all, whcih would be a huge task...
       if (CLANG_VERSION VERSION_GREATER 3.6 OR CLANG_VERSION VERSION_EQUAL 3.6 OR APPLE_CLANG_VERSION VERSION_GREATER 6.0)
         SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-pointer-bool-conversion")
       endif()
@@ -947,7 +950,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     else()
       add_custom_command(
         OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${module}
-        COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/${module} ${CMAKE_CURRENT_BINARY_DIR}/${module}
+        COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/${module} ${CMAKE_CURRENT_BINARY_DIR}/${module}
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
         COMMENT "Adding unsigned ${module} to project"
         VERBATIM

common/environment/environment.hpp

@@ -93,6 +93,11 @@ interface IConstMachineInfo : extends IConstEnvBase
     virtual EnvMachineState getState() const = 0;
 };
 
+interface  IConstMachineInfoIterator : extends IIteratorOf<IConstMachineInfo>
+{
+    virtual unsigned count() const = 0;
+};
+
 
 interface IConstInstanceInfo : extends IConstEnvBase
 {
@@ -111,6 +116,11 @@ interface IConstDropZoneInfo : extends IConstEnvBase
     virtual IStringVal & getUMask(IStringVal & str) const = 0;
 };
 
+interface  IConstDropZoneInfoIterator : extends IIteratorOf<IConstDropZoneInfo>
+{
+    virtual unsigned count() const = 0;
+};
+
 interface IConstEnvironment : extends IConstEnvBase
 {
     virtual IConstDomainInfo * getDomain(const char * name) const = 0;
@@ -125,6 +135,13 @@ interface IConstEnvironment : extends IConstEnvBase
     virtual IEnvironment & lock() const = 0;
     virtual bool isConstEnvironment() const = 0;
     virtual void clearCache() = 0;
+
+    virtual IConstMachineInfoIterator * getMachineIterator() const = 0;
+    virtual IConstDropZoneInfo * getDropZoneByComputer(const char * computer, const char * dzname) const = 0;
+    virtual IConstDropZoneInfoIterator * getDropZoneIteratorByComputer(const char * computer) const = 0;
+    // returns a drop zone that is defined on IP with the shortest path that's a parent of targetPath
+    virtual IConstDropZoneInfo * getDropZoneByAddressPath(const char * netaddress, const char *targetPath) const = 0;
+    virtual IConstDropZoneInfoIterator * getDropZoneIterator() const = 0;
 };
 
 

common/remote/rmtssh.cpp

@@ -565,6 +565,16 @@ public:
         workdir.set(workdirname);
         exec();
     }
+
+    const StringArray &getReplyText() const
+    {
+      return replytext;
+    }
+
+    const UnsignedArray &getReply() const
+    {
+      return reply;
+    }
 };
 
 IFRunSSH *createFRunSSH()

common/remote/rmtssh.hpp

@@ -44,6 +44,8 @@ interface IFRunSSH: extends IInterface
               const IpAddress &ip,
               const char *workdirname,
               bool _background) = 0;
+    virtual const StringArray &getReplyText() const = 0;
+    virtual const UnsignedArray &getReply() const= 0;
 };
 
 

common/workunit/workunit.cpp

@@ -140,17 +140,16 @@ static bool checkWuSecAccess(IConstWorkUnit &cw, ISecManager *secmgr, ISecUser *
 }
 static bool checkWuSecAccess(const char *wuid, ISecManager *secmgr, ISecUser *secuser, int required, const char *action, bool excpt, bool log)
 {
-    StringBuffer wuRoot;
-    Owned<IRemoteConnection> conn = querySDS().connect(getXPath(wuRoot, wuid).str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT);
-    if (conn)
+    if (!secmgr || !secuser)
+        return true;
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid);
+    bool ret=secmgr->authorizeEx(RT_WORKUNIT_SCOPE, *secuser, cw->queryWuScope())>=required;
+    if (!ret && (log || excpt))
     {
-        Owned<IPropertyTree> ptree=conn->getRoot();
-        return checkWuScopeSecAccess(ptree->queryProp("@scope"), secmgr, secuser, required, action, excpt, log);
+        wuAccessError(secuser->getName(), action, cw->queryWuScope(), cw->queryWuid(), excpt, log);
     }
-
-    if (log || excpt)
-        wuAccessError(secuser ? secuser->getName() : NULL, action, "Unknown", NULL, excpt, log);
-    return false;
+    return ret;
 }
 
 void doDescheduleWorkkunit(char const * wuid)

dali/base/dasds.cpp

@@ -1040,6 +1040,12 @@ class CBackupHandler : public CInterface, implements IThreaded
     CTimeMon warningTime;
     unsigned recentTimeThrottled;
     unsigned lastNumWarnItems;
+    IPropertyTree &config;
+
+    const unsigned defaultFreeQueueLimit = 50;
+    const unsigned defaultLargeWarningThreshold = 50;
+    const unsigned defaultSoftQueueLimit = 200;
+    const unsigned defaultSoftQueueLimitDelay = 200;
 
     BackupQueueItem *getFreeItem()
     {
@@ -1159,18 +1165,18 @@ class CBackupHandler : public CInterface, implements IThreaded
     }
 
 public:
-    CBackupHandler() : threaded("CBackupHandler")
+    CBackupHandler(IPropertyTree &_config) : config(_config), threaded("CBackupHandler")
     {
         currentEdition = (unsigned)-1;
         addWaiting = waiting = async = false;
         aborted = true;
         throttleCounter = 0;
-        freeQueueLimit = 10;
-        largeWarningThreshold = 50;
-        softQueueLimit = 200;
-        softQueueLimitDelay = 200;
         recentTimeThrottled = 0;
         lastNumWarnItems = 0;
+        freeQueueLimit = config.getPropInt("@backupFreeQueueLimit", defaultFreeQueueLimit);
+        largeWarningThreshold = config.getPropInt("@backupLargeWarningThreshold", defaultLargeWarningThreshold);
+        softQueueLimit = config.getPropInt("@backupSoftQueueLimit", defaultSoftQueueLimit);
+        softQueueLimitDelay = config.getPropInt("@backupSoftQueueLimitDelay", defaultSoftQueueLimitDelay);
     }
     ~CBackupHandler()
     {
@@ -5720,7 +5726,7 @@ IStoreHelper *createStoreHelper(const char *storeName, const char *location, con
 #endif
 
 CCovenSDSManager::CCovenSDSManager(ICoven &_coven, IPropertyTree &_config, const char *_dataPath) 
-    : coven(_coven), config(_config), server(*this), dataPath(_dataPath)
+    : coven(_coven), config(_config), server(*this), dataPath(_dataPath), backupHandler(_config)
 {
     config.Link();
     restartOnError = config.getPropBool("@restartOnUnhandled");

docs/ECLLanguageReference/ECLR-includer.xml

@@ -781,6 +781,10 @@
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
+    <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-REGEXFINDSET.xml"
+                xpointer="element(/1)"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-REGEXREPLACE.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />

docs/ECLLanguageReference/ECLR_mods/BltInFunc-DENORMALIZE.xml

@@ -241,7 +241,7 @@ OUTPUT(DeNormedRecs);
 /* DeNormedRecs result set is:
  Rec#  Value1 PVal2  CVal2_1  CVal2_2
  1       A      C      Z         T
- 2       B      B      Y         S
+ 2       B      B      S         Y
  3       C      A      X         W
  */
 </programlisting>

docs/ECLLanguageReference/ECLR_mods/BltInFunc-REGEXFIND.xml

@@ -108,5 +108,7 @@ REGEXFIND(searchpattern, search, 2); //returns '14'
 REGEXFIND(searchpattern, search, 3); //returns '1998'
 </programlisting>
 
-  <para>See Also: <link linkend="PARSE">PARSE</link>, <link linkend="REGEXREPLACE">REGEXREPLACE</link></para>
+  <para>See Also: <link linkend="PARSE">PARSE</link>, <link
+  linkend="REGEXFINDSET">REGEXFINDSET</link>, <link
+  linkend="REGEXREPLACE">REGEXREPLACE</link></para>
 </sect1>

docs/ECLLanguageReference/ECLR_mods/BltInFunc-REGEXFINDSET.xml

@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="REGEXFINDSET">
+  <title>REGEXFINDSET</title>
+
+  <para><emphasis role="bold">REGEXFINDSET<indexterm>
+      <primary>REGEXFINDSET</primary>
+    </indexterm><indexterm>
+      <primary>REGEXFINDSET function</primary>
+    </indexterm>(</emphasis><emphasis>regex, text </emphasis><emphasis
+  role="bold"> [, NOCASE<indexterm>
+      <primary>NOCASE</primary>
+    </indexterm>])</emphasis></para>
+
+  <para><informaltable colsep="1" frame="all" rowsep="1">
+      <tgroup cols="2">
+        <colspec colwidth="81.35pt" />
+
+        <colspec />
+
+        <tbody>
+          <row>
+            <entry><emphasis>regex</emphasis></entry>
+
+            <entry>A standard Perl regular expression.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>text</emphasis></entry>
+
+            <entry>The text to parse.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis role="bold">NOCASE</emphasis></entry>
+
+            <entry>Optional. Specifies a case insensitive search.</entry>
+          </row>
+
+          <row>
+            <entry>Return:</entry>
+
+            <entry>REGEXFINDSET returns a set of strings.</entry>
+          </row>
+        </tbody>
+      </tgroup>
+    </informaltable></para>
+
+  <para>The <emphasis role="bold">REGEXFIND </emphasis>function uses the
+  <emphasis>regex<indexterm>
+      <primary>regex</primary>
+    </indexterm></emphasis> to parse through the <emphasis>text</emphasis> and
+  find matches. The <emphasis>regex</emphasis> must be a standard Perl regular
+  expression<indexterm>
+      <primary>Perl regular expression</primary>
+    </indexterm>. We use third-party libraries to support this, so for
+  non-unicode <emphasis>text</emphasis>, see boost docs at <emphasis
+  role="underline">http://www.boost.org/doc/libs/1_39_0/libs/regex/doc/html/index.html</emphasis>.
+  For unicode <emphasis>text</emphasis>, see the ICU docs, the sections
+  ‘Regular Expression Metacharacters' and ‘Regular Expression Operators' at
+  <emphasis
+  role="underline">http://userguide.icu-project.org/strings/regexp</emphasis>
+  and the links from there, in particular the section ‘UnicodeSet patterns' at
+  <emphasis
+  role="underline">http://userguide.icu-project.org/strings/unicodeset</emphasis>.
+  We use version 2.6 which should support all listed features.</para>
+
+  <para>Example:</para>
+
+  <programlisting>sampleStr := 'To: jane@example.com From: john@example.com This is the winter of our discontent.';
+eMails:=REGEXFINDSET('\\w+@[a-zA-Z_]+?\\.[a-zA-Z]{2,3}' , sampleStr);
+OUTPUT(eMails);
+
+UNICODE sampleStr2:= U'To: janë@example.com From john@example.com This is the winter of our discontent.';
+eMails2:= REGEXFINDSET(U'\\w+@[a-zA-Z_]+?\\.[a-zA-Z]{2,3}', sampleStr2);
+OUTPUT(eMails2);
+
+</programlisting>
+
+  <para>See Also: <link linkend="PARSE">PARSE</link>, <link
+  linkend="REGEXREPLACE">REGEXFIND</link>, <link
+  linkend="REGEXREPLACE">REGEXREPLACE</link></para>
+</sect1>

docs/ECLLanguageReference/ECLR_mods/ExpressionsandOperatos.xml

@@ -80,14 +80,17 @@
       </tgroup>
     </informaltable>
 
-    <para>Division by zero<indexterm><primary>Division by zero</primary></indexterm>
-	defaults to generating a zero result (0), rather than reporting a “divide by zero” error.  
-	This avoids invalid or unexpected data aborting a long job.
-	The default behaviour can be changed using </para>
-	<programlisting>#option ('divideByZero', ...);</programlisting> 
-	<para>The #option can take the following values:
-	</para>
-     <informaltable colsep="1" frame="all" rowsep="1">
+    <para>Division by zero<indexterm>
+        <primary>Division by zero</primary>
+      </indexterm> defaults to generating a zero result (0), rather than
+    reporting a “divide by zero” error. This avoids invalid or unexpected data
+    aborting a long job. The default behaviour can be changed using</para>
+
+    <programlisting>#OPTION ('divideByZero', 'zero'); //evaluate to zero</programlisting>
+
+    <para>The divideByZero option can have the following values:</para>
+
+    <informaltable colsep="1" frame="all" rowsep="1">
       <tgroup align="left" cols="2">
         <colspec colwidth="194.80pt" />
 
@@ -99,23 +102,25 @@
 
             <entry>Evaluate to 0 - the default behaviour.</entry>
           </row>
+
           <row>
             <entry>'fail'</entry>
 
             <entry>Stop and report a division by zero error.</entry>
           </row>
+
           <row>
             <entry>'nan'</entry>
 
-            <entry>This is only currently supported for real numbers.  Division by zero
-			creates a quiet NaN, which will propogate through any real expressions it is used in.
-			You can use NOT ISVALID(x) to test if the value is a NaN.
-			Integer and decimal division by zero continue to return 0.</entry>
+            <entry>This is only currently supported for real numbers. Division
+            by zero creates a quiet NaN, which will propagate through any real
+            expressions it is used in. You can use NOT ISVALID(x) to test if
+            the value is a NaN. Integer and decimal division by zero continue
+            to return 0.</entry>
           </row>
         </tbody>
       </tgroup>
     </informaltable>
-
   </sect2>
 
   <sect2 id="Bitwise_Operators">

docs/ECLLanguageReference/ECLR_mods/Templ-OPTION.xml

@@ -477,10 +477,26 @@
 
               <entry>Number of records to limit to</entry>
             </row>
+
+            <row>
+              <entry><emphasis>divideByZero</emphasis></entry>
+
+              <entry>Default zero</entry>
+
+              <entry>'zero' evaluates to 0, the default behavior. 'fail'
+              causes the job to fail and report a division by zero error.
+              'nan' (only currently supported for real numbers) creates a
+              quiet NaN, which will propagate through any real expressions it
+              is used in. You can use NOT ISVALID(x) to test if the value is a
+              NaN. Integer and decimal division by zero continue to return 0.
+              </entry>
+            </row>
           </tbody>
         </tgroup>
       </informaltable></para>
 
+    <para></para>
+
     <para><emphasis role="bold">The following options are all about generating
     Logical graphs in a workunit. </emphasis></para>
 
@@ -953,8 +969,8 @@
 
               <entry>Default: 3 for roxie, else 0</entry>
 
-              <entry>Set the C++ compiler optimization level (optimizations can cause the compiler
-              to take a lot longer).</entry>
+              <entry>Set the C++ compiler optimization level (optimizations
+              can cause the compiler to take a lot longer).</entry>
             </row>
 
             <row>

docs/ECLWatch/ECLWa_mods/ECLWatchQueries.xml

@@ -158,6 +158,146 @@ depending on where they wind up.-->
       checking the check box. You can also open a particular query by double
       clicking on it.</para>
 
+      <sect2 id="Queries_Tab" role="brk">
+        <title>Queries Tab</title>
+
+        <para>When you select the Published Queries hyperlink you open the
+        Queries tab. This tab displays published queries on the system. The
+        Action buttons allow you to perform operations on the published
+        queries selected.</para>
+
+        <para><figure>
+            <title>Published Query Action buttons</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="../../images/ECLWA202.jpg"
+                           vendor="eclwatchSS" />
+              </imageobject>
+            </mediaobject>
+          </figure> <variablelist>
+            <varlistentry>
+              <term>Open</term>
+
+              <listitem>
+                <para>Opens the selected query (or queries).</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>Delete</term>
+
+              <listitem>
+                <para>Deletes the selected query (or queries).</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>Suspend</term>
+
+              <listitem>
+                <para>Suspends the selected active query (or queries).</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>Unsuspend</term>
+
+              <listitem>
+                <para>Unsuspends the selected suspended query (or
+                queries).</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>Activate</term>
+
+              <listitem>
+                <para>Activates the selected query (or queries). This assigns
+                a query to the active alias with the same name as the
+                query.</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>Deactivate</term>
+
+              <listitem>
+                <para>Deactivates the selected active query (or queries) by
+                removing the active query alias from the given
+                queryset.</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <?dbfo keep-together="always"?>
+
+              <term>Filter</term>
+
+              <listitem>
+                <para>Allows you to filter the queries for the criteria you
+                enter. When the Filter is applied the action button displays
+                <emphasis role="bold">Filter Set</emphasis>. This icon
+                indicates that the published queries displayed are
+                filtered.</para>
+
+                <para><graphic fileref="../../images/ECLWA203.jpg" />You can
+                filter for several query attributes. You can filter by:</para>
+
+                <itemizedlist>
+                  <listitem>
+                    <para>ID</para>
+                  </listitem>
+
+                  <listitem>
+                    <para>Name</para>
+                  </listitem>
+
+                  <listitem>
+                    <para>Published by</para>
+                  </listitem>
+
+                  <listitem>
+                    <para>WUID</para>
+                  </listitem>
+
+                  <listitem>
+                    <para>Cluster</para>
+                  </listitem>
+
+                  <listitem>
+                    <para>Logical File Name</para>
+                  </listitem>
+
+                  <listitem>
+                    <para>Libraries Used</para>
+                  </listitem>
+
+                  <listitem>
+                    <para>Suspended queries.</para>
+                  </listitem>
+
+                  <listitem>
+                    <para>Active queries.</para>
+                  </listitem>
+                </itemizedlist>
+
+                <para>The Filter also also supports wild cards. </para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>Options</term>
+
+              <listitem>
+                <para>Provides the option to search/display queries on a
+                single node or all nodes. Using this option can improve
+                performance if you have a large multi-node cluster.</para>
+              </listitem>
+            </varlistentry>
+          </variablelist></para>
+      </sect2>
+
       <sect2 id="QueryDetails" role="brk">
         <title>Query Details</title>
 

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -1547,6 +1547,175 @@ lock=/var/lock/HPCCSystems</programlisting>
         than number of cores. One good rule is to use a formula where the
         number of cores divided by two is the maximum number of Thor clusters
         to use.</para>
+
+        <para></para>
+      </sect2>
+
+      <sect2 id="virtual-thor-slaves">
+        <title><emphasis role="strong">Virtual Thor slaves</emphasis></title>
+
+        <para><indexterm>
+            <primary>Virtual Thor slaves</primary>
+          </indexterm>Beginning in version 6.0.0, Thor clusters can be
+        configured to take full advantage of the resources available per node
+        using Virtual Thor slaves.</para>
+
+        <para>In HPCC versions prior to 6.0.0, cluster configurations were
+        typically set to N number of <emphasis
+        role="bold">slavesPerNode</emphasis><indexterm>
+            <primary>slavesPerNode</primary>
+          </indexterm> <emphasis></emphasis> , where N equalled or approached
+        the number of cores per machine.</para>
+
+        <para>This resulted in N independent slave processes per node, as seen
+        below:</para>
+
+        <para></para>
+
+        <para><graphic fileref="images//SA009.jpg" /></para>
+
+        <para>This had several significant disadvantages:</para>
+
+        <itemizedlist>
+          <listitem>
+            <para>Each slave process in this configuration has an equal fixed
+            split of the physical memory available to the node.</para>
+          </listitem>
+
+          <listitem>
+            <para>Slaves do not share RAM or any other resources.</para>
+          </listitem>
+
+          <listitem>
+            <para>Slaves use message passing via the loopback network
+            interface for communication.</para>
+          </listitem>
+        </itemizedlist>
+
+        <para>Now a new approach is used, allowing virtual slaves to be
+        created with a single slave process, as depicted below:.</para>
+
+        <para><graphic fileref="images//SA010.jpg" /></para>
+
+        <itemizedlist>
+          <listitem>
+            <para>In this configuration, each physical node has a single Thor
+            slave process.</para>
+          </listitem>
+
+          <listitem>
+            <para>Each slave process has N virtual slaves. This is set using a
+            Thor configuration option called <emphasis
+            role="bold">channelsPerSlave</emphasis><indexterm>
+                <primary>channelsPerSlave</primary>
+              </indexterm> <emphasis><emphasis
+            role="strong"></emphasis></emphasis>. Under this architecture,
+            slaves within the same process can communicate directly with one
+            another and share resources.</para>
+          </listitem>
+        </itemizedlist>
+
+        <para>Note: The <emphasis><emphasis
+        role="bold">slavesPerNode</emphasis></emphasis> setting still exists
+        and both may be used in combination if required.</para>
+
+        <para><emphasis role="strong">Key advantages:</emphasis></para>
+
+        <itemizedlist>
+          <listitem>
+            <para>Each virtual slave shares cached resources, such as key
+            index pages, etc.</para>
+          </listitem>
+
+          <listitem>
+            <para>Slaves can request and share all available RAM.</para>
+          </listitem>
+
+          <listitem>
+            <para>Startup and management of the cluster is faster and
+            simpler.</para>
+          </listitem>
+
+          <listitem>
+            <para>Allows for future enhancements to bring better
+            management/coordination of CPU cores.</para>
+          </listitem>
+        </itemizedlist>
+
+        <para>The significance of having access to all available memory
+        becomes very significant for some activities. The clearest example is
+        a SMART or LOOKUP JOIN.</para>
+
+        <sect3 id="smartlookup-join-example">
+          <title>SMART/LOOKUP JOIN example</title>
+
+          <para>A LOOKUP JOIN works approximately as follows:</para>
+
+          <itemizedlist>
+            <listitem>
+              <para>Streams local slave RHS dataset to all other
+              slaves.</para>
+            </listitem>
+
+            <listitem>
+              <para>All slaves gather global RHS into one table.</para>
+            </listitem>
+
+            <listitem>
+              <para>A hash table based on the hard key match fields is
+              built.</para>
+            </listitem>
+
+            <listitem>
+              <para>Once all slaves are done, the LHS is streamed and matched
+              against the hash table to produce the joined results.</para>
+            </listitem>
+          </itemizedlist>
+
+          <para>Note: The complete RHS table and hash table must fit into
+          memory; otherwise the join fails with an out of memory error.</para>
+
+          <para>SMART JOIN is an evolution of LOOKUP JOIN. If it cannot fit
+          the global RHS into memory, it will HASH PARTITION the RHS and HASH
+          DISTRIBUTE the LHS and perform a LOCAL LOOKUP JOIN.</para>
+
+          <para>If it cannot fit the local RHS set into memory on any given
+          node, then it will gather and sort both local datasets and perform a
+          standard JOIN.</para>
+
+          <para>The key advantage of LOOKUP JOIN is speed. If the RHS fits
+          into memory, it can perform a very quick gather and streamed JOIN of
+          a large LHS set, without the need of gathering and sorting
+          anything.</para>
+
+          <para>The advantages of a virtual slave Thor configuration for ECL
+          code using LOOKUP/SMART JOIN is that in effect it will have N times
+          as much memory before it fails or fails over in the SMART JOIN
+          case.</para>
+
+          <para>It is also much quicker; instead of broadcasting the local RHS
+          to N slave process per node; it only has to communicate it to one.
+          That one slave can share the same table and same HT with the other
+          virtual slaves directly.</para>
+
+          <para><emphasis role="strong"><emphasis>Key advantages of
+          LOOKUP/SMART JOIN in a Virtual Slave Thor
+          Setup:</emphasis></emphasis></para>
+
+          <itemizedlist>
+            <listitem>
+              <para>N times as much memory available for RHS. In other words,
+              the RHS can be N times bigger before failing or failing over in
+              SMART JOIN case. (In the illustrated example, the JOIN would
+              have 4 times as much memory available)</para>
+            </listitem>
+
+            <listitem>
+              <para>Significantly less communication of row data – equals
+              significantly faster processing for larger RHS sets.</para>
+            </listitem>
+          </itemizedlist>
+        </sect3>
       </sect2>
 
       <sect2 id="SysAdm_BestPrac_HugePages">

docs/RoxieReference/RoxieRefMods/RoxieCopySettings.xml

@@ -137,7 +137,55 @@
     clusters later.</para>
   </sect3>
 
-  <sect3>
+  <sect3 id="VipsRoxieAlias" role="brk">
+    <title>VIPs and Roxie Alias</title>
+
+    <para>Roxie clusters accessed through a VIP (Virtual IP) usually use
+    separate target names to represent the same set of queries. Each Roxie can
+    also run more than one target at the same time.</para>
+
+    <para>The Topology Alias feature provides a way to specify that any of the
+    targets representing the same set of queries can be used. Allowing targets
+    that represent the same set of queries to have a shared alias allows
+    callers to select the right set of queries when calling through a
+    VIP.</para>
+
+    <para>This setting can be found in the Topology section within the HPCC
+    Configuration Manager.</para>
+
+    <para><figure>
+        <title>Roxie Alias in Configuration Manager</title>
+
+        <mediaobject>
+          <imageobject>
+            <imagedata fileref="../../images/RoxieAlias.jpg" />
+          </imageobject>
+        </mediaobject>
+      </figure></para>
+
+    <para></para>
+
+    <?hard-pagebreak ?>
+
+    <para>The following diagram shows an example of a system using
+    aliases.</para>
+
+    <figure>
+      <title>Roxie Aliases in a Production Environment</title>
+
+      <mediaobject>
+        <imageobject>
+          <imagedata fileref="../../images/RoxieAliasDiagram.png" />
+        </imageobject>
+      </mediaobject>
+    </figure>
+
+    <para></para>
+
+    <para></para>
+  </sect3>
+
+  <sect3 role="brk">
     <title><emphasis role="bold">Command Line support</emphasis></title>
 
     <para>The ecl command line tool offers a means of copying queries from one

docs/RoxieReference/RoxieReference.xml

@@ -5,7 +5,6 @@
   <bookinfo>
     <title>Roxie: The Rapid Data Delivery Engine</title>
 
-
     <mediaobject>
       <imageobject>
         <imagedata fileref="images/redswooshWithLogo3.jpg" />

docs/UsingConfigManager/UsingConfigManager.xml

@@ -1363,6 +1363,75 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
         </sect3>
       </sect2>
 
+      <sect2 id="ESP_Services" role="brk">
+        <title>ESP Services</title>
+
+        <para>ESP Services provide a means to add functionality to an ESP
+        Server.</para>
+
+        <sect3 id="CfgMgrEclWatch">
+          <title>ECL Watch Service</title>
+
+          <para>Ecl Watch allows you to configure options for the ECL Watch
+          utility.</para>
+
+          <para><graphic fileref="images/CM-009.jpg"
+          vendor="configmgrSS" /></para>
+
+          <para>ECL Watch Attribute definitions.</para>
+
+          <!--ECLWa-Include-TBLZ-1-->
+
+          <para><xi:include href="XMLGeneration/xml/espsmcservice.xsd.mod.xml"
+          xpointer="SMC-T01"
+          xmlns:xi="http://www.w3.org/2001/XInclude" /></para>
+
+          <para>ECL Watch Monitoring attributes.</para>
+
+          <!--ECLWa-Include-TBLZ-2-->
+
+          <para><xi:include href="XMLGeneration/xml/espsmcservice.xsd.mod.xml"
+          xpointer="SMC-T02"
+          xmlns:xi="http://www.w3.org/2001/XInclude" /></para>
+        </sect3>
+
+        <sect3 id="cfgmgr_WsECL" role="brk">
+          <title>WsECL Service</title>
+
+          <para>The WsECL service allows you to configure options for the
+          WsECL utility.</para>
+
+          <para><graphic fileref="images/CM-010.jpg"
+          vendor="configmgrSS" /></para>
+
+          <para>The Ws ECL configuration attributes. </para>
+
+          <!--WsECL-Include-ZXX-1-->
+	  <para><xi:include href="XMLGeneration/xml/esp_service_wsecl2.xsd.mod.xml"
+	            xpointer="MyWS2-T01"
+          xmlns:xi="http://www.w3.org/2001/XInclude" /></para>
+
+          <para>Ws ECL VIPS option attributes.</para>
+
+          <!--WsECL-Include-ZXX-2-->
+
+          <para><xi:include
+          href="XMLGeneration/xml/esp_service_wsecl2.xsd.mod.xml"
+          xpointer="MyWS2-T02"
+          xmlns:xi="http://www.w3.org/2001/XInclude" /></para>
+
+          <para>Ws ECL Target Restrictions table. </para>
+
+          <!--WsECL-Include-ZXX-3-->
+	  <para><xi:include href="XMLGeneration/xml/esp_service_wsecl2.xsd.mod.xml"
+	            xpointer="MyWS2-T03"
+          xmlns:xi="http://www.w3.org/2001/XInclude" /></para>
+          
+         
+          
+        </sect3>
+      </sect2>
+
       <sect2 id="FTSlave-Process" role="brk">
         <title>FTSlave Process</title>
 

docs/XMLGeneration/CMakeLists.txt

@@ -24,7 +24,6 @@ FILE(GLOB XSD_TEMPLATE_FILES RELATIVE "${XSD_SOURCE_DIR}" "${XSD_SOURCE_DIR}/*.x
 FILE(GLOB XSD_FILES RELATIVE "${XSD_SOURCE_DIR}" "${XSD_SOURCE_DIR}/*.xsd")
 
 # Skip some files which cause problem
-LIST(REMOVE_ITEM XSD_FILES "esp_service_wsecl2.xsd")
 LIST(REMOVE_ITEM XSD_FILES "ftslave_linux.xsd")
 
 # process and copy *.xsd.in files

ecl/eclccserver/vchooks/git.sh

@@ -51,11 +51,11 @@ function fetch_repo {
     fi
 
     if [ -n "$wu_git_branch" ]; then
-        if [ -z "$git_branch_locked" ]; then
+        if [ -z "$git_branch_locked" -o "$git_branch_locked" == "0" ]; then
+            git_branch=$wu_git_branch
+        else
             echo "GIT: Overriding branch is not allowed" 1>&2
             exit 2
-        else
-            git_branch=$wu_git_branch
         fi
     fi
 
@@ -153,4 +153,4 @@ else
     echo GIT: calling eclcc $GIT_INCLUDE_PATH "$@"  1>&2
   fi
   eclcc $GIT_INCLUDE_PATH "$@"
-fi
+fi

ecl/hql/hqlatoms.cpp

@@ -279,6 +279,7 @@ IAtom * newSetAtom;
 IAtom * _nlpParse_Atom;
 IAtom * noBoundCheckAtom;
 IAtom * noCaseAtom;
+IAtom * noConstAtom;
 IAtom * nofoldAtom;
 IAtom * _noHoist_Atom;
 IAtom * noLocalAtom;
@@ -720,6 +721,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKESYSATOM(nlpParse);
     MAKEATOM(noBoundCheck);
     MAKEATOM(noCase);
+    MAKEATOM(noConst);
     MAKEATOM(nofold);
     MAKESYSATOM(noHoist);
     MAKEATOM(noLocal);

ecl/hql/hqlatoms.hpp

@@ -283,6 +283,7 @@ extern HQL_API IAtom * newSetAtom;
 extern HQL_API IAtom * _nlpParse_Atom;
 extern HQL_API IAtom * noBoundCheckAtom;
 extern HQL_API IAtom * noCaseAtom;
+extern HQL_API IAtom * noConstAtom;
 extern HQL_API IAtom * nofoldAtom;
 extern HQL_API IAtom * _noHoist_Atom;
 extern HQL_API IAtom * noLocalAtom;

ecl/hql/hqlerrors.hpp

@@ -199,7 +199,6 @@
 #define ERR_ASSERT_WRONGSCOPING     2131 /* Incorrect assertion scoping */
 #define ERR_ASSERT_BOOLEXPECTED     2132 /* Assertion must be boolean */
 #define ERR_SCOPE_USEDATASETINEXPR  2133 /* Use dataset in expression without proper context */
-#define ERR_FETCH_NON_DATASET       2134 /* Parameter to fetch isn't a dataset */
 
 #define ERR_MODIFIER_ILLCOMB        2141 /* Illegal combination of modifiers */
 #define ERR_EXPORTSHARECONFLICT     2142 /* EXPORT and SHARED cannot be specified together */

ecl/hql/hqlgram.y

@@ -314,6 +314,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   NOBOUNDCHECK
   NOCASE
   NOCOMBINE
+  NOCONST
   NOFOLD
   NOHOIST
   NOLOCAL
@@ -3873,6 +3874,7 @@ formalQualifiers
 
 formalQualifier
     : TOK_CONST         {   $$.setExpr(createAttribute(constAtom), $1); }
+    | NOCONST           {   $$.setExpr(createAttribute(noConstAtom), $1); }
     | TOK_ASSERT TOK_CONST  
                         {   $$.setExpr(createAttribute(assertConstAtom), $1); }
     | FIELD_REF         {
@@ -8678,12 +8680,6 @@ simpleDataSet
                             IHqlExpression * left = $3.getExpr();
                             IHqlExpression * right = $5.getExpr();
 
-                            node_operator modeOp = no_none;
-                            if (left->getOperator() == no_table)
-                                modeOp = left->queryChild(2)->getOperator();
-                            if ((modeOp != no_thor) && (modeOp != no_flat) && (modeOp != no_csv) && (modeOp != no_xml) && (modeOp != no_json))
-                                parser->reportError(ERR_FETCH_NON_DATASET, $3, "First parameter of FETCH should be a disk file");
-
                             IHqlExpression *join = createDataset(no_fetch, left, createComma(right, $7.getExpr(), $9.getExpr(), createComma($10.getExpr(), $12.getExpr())));
 
                             $$.setExpr(join);
@@ -8696,12 +8692,6 @@ simpleDataSet
                             IHqlExpression * right = $5.getExpr();
                             IHqlExpression * transform = parser->createDefJoinTransform(left, right, $7, $10.queryExpr(),NULL);
 
-                            node_operator modeOp = no_none;
-                            if (left->getOperator() == no_table)
-                                modeOp = left->queryChild(2)->getOperator();
-                            if ((modeOp != no_thor) && (modeOp != no_flat) && (modeOp != no_csv) && (modeOp != no_xml)  && (modeOp != no_json))
-                                parser->reportError(ERR_FETCH_NON_DATASET, $3, "First parameter of FETCH should be a disk file");
-
                             IHqlExpression *join = createDataset(no_fetch, left, createComma(right, $7.getExpr(), transform, createComma($8.getExpr(), $10.getExpr())));
 
                             $$.setExpr(join);

ecl/hql/hqlgram2.cpp

@@ -10682,6 +10682,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case NOBOUNDCHECK: msg.append("NOBOUNDCHECK"); break;
     case NOCASE: msg.append("NOCASE"); break;
     case NOCOMBINE: msg.append("NOCOMBINE"); break;
+    case NOCONST: msg.append("NOCONST"); break;
     case NOFOLD: msg.append("NOFOLD"); break;
     case NOHOIST: msg.append("NOHOIST"); break;
     case NOLOCAL: msg.append("NOLOCAL"); break;

ecl/hql/hqllex.l

@@ -811,6 +811,7 @@ NAMESPACE           { RETURNSYM(NAMESPACE); }
 NOBOUNDCHECK        { RETURNSYM(NOBOUNDCHECK); }
 NOCASE              { RETURNSYM(NOCASE); }
 NOCOMBINE           { RETURNSYM(NOCOMBINE); }
+NOCONST             { RETURNSYM(NOCONST); }
 NOFOLD              { RETURNSYM(NOFOLD); }
 NOHOIST             { RETURNSYM(NOHOIST); }
 NOLOCAL             { RETURNSYM(NOLOCAL); }

ecl/hql/hqlutil.cpp

@@ -5650,7 +5650,9 @@ IHqlExpression * extractCppBodyAttrs(unsigned lenBuffer, const char * buffer)
     unsigned prev = '\n';
     for (unsigned i=0; i < lenBuffer; i++)
     {
-        switch (buffer[i])
+        unsigned next = buffer[i];
+        bool ignore = false;
+        switch (next)
         {
         case '*':
             if ('/' == prev) // Ignore directives in multi-line comments
@@ -5659,6 +5661,7 @@ IHqlExpression * extractCppBodyAttrs(unsigned lenBuffer, const char * buffer)
                 while (i < lenBuffer && ('*' != buffer[i-1] || '/' != buffer[i])) 
                     ++i;
             }
+            next = ' '; // treat as whitespace
             break;
         case '/':
             if ('/' == prev) // Ignore directives in single line comments
@@ -5667,9 +5670,10 @@ IHqlExpression * extractCppBodyAttrs(unsigned lenBuffer, const char * buffer)
                 while (i < lenBuffer && !iseol(buffer[i]))
                     ++i;
             }
+            next = '\n';
             break;
         case ' ': case '\t':
-            // allow whitespace in front of #option
+            ignore = true; // allow whitespace in front of #option
             break;
         case '#':
             if (prev == '\n')
@@ -5709,7 +5713,8 @@ IHqlExpression * extractCppBodyAttrs(unsigned lenBuffer, const char * buffer)
                 }
             }
         }
-        prev = buffer[i];
+        if (!ignore)
+            prev = next;
     }
     return attrs.getClear();
 }
@@ -7381,7 +7386,7 @@ public:
             ITypeInfo *paramType = param->queryType();
 
             bool isOut = param->hasAttribute(outAtom);
-            bool isConst = param->hasAttribute(constAtom);
+            bool isConst = !param->hasAttribute(noConstAtom);
 
             if (isOut)
                 mangled.append("R");
@@ -7629,7 +7634,7 @@ public:
             ITypeInfo *paramType = param->queryType();
 
             bool isOut = param->hasAttribute(outAtom);
-            bool isConst = param->hasAttribute(constAtom);
+            bool isConst = !param->hasAttribute(noConstAtom);
 
             if (isOut)
                 appendRef(mangled, false);

ecl/hql/reservedwords.cpp

@@ -369,6 +369,7 @@ static const char * eclReserved12[] = {//Attributes
     "lookup",
     "lzw",
     "many",
+    "noconst",
     "noroot",
     "noscan",
     "notrim",
@@ -486,6 +487,7 @@ static const char * eclReserved15[] = { //Actions and statements
     "build",
     "buildindex",
     "checkpoint",
+    "critical",
     "cron",
     "define",
     "deprecated",

ecl/hqlcpp/hqlcerrors.hpp

@@ -69,7 +69,7 @@
 #define HQLERR_RowTooLarge                      4043
 #define HQLERR_ShouldHaveBeenHoisted            4044
 #define HQLERR_NoArgumentsInValidator           4045
-
+#define HQLERR_FetchNonDiskfile                 4046
 #define HQLERR_InputMergeNotSorted              4047
 #define HQLERR_TooComplicatedToPreload          4048
 #define HQLERR_KeyedNotKeyed                    4049
@@ -373,6 +373,7 @@
 #define HQLERR_RowTooLarge_Text                 "Row size %u exceeds the maximum specified (%u)"
 #define HQLERR_ShouldHaveBeenHoisted_Text       "Select expression should have been hoisted"
 #define HQLERR_NoArgumentsInValidator_Text      "%s() cannot have a parameter inside a VALIDATE"
+#define HQLERR_FetchNonDiskfile_Text            "First parameter of FETCH should be a disk file"
 #define HQLERR_InputMergeNotSorted_Text         "Input to MERGE does not appear to be sorted"
 #define HQLERR_TooComplicatedToPreload_Text     "Expression is too complicated to preload"
 #define HQLERR_KeyedNotKeyed_Text               "KEYED(%s) could not be looked up in a key."

ecl/hqlcpp/hqlcpp.cpp

@@ -7384,7 +7384,8 @@ bool HqlCppTranslator::ifRequiresAssignment(BuildCtx & ctx, IHqlExpression * exp
         return true;
     if (trueExpr->queryType() != falseExpr->queryType() && isStringType(expr->queryType()))
         return true;
-    if (expr->queryType()->getTypeCode() == type_decimal)
+    type_t tc = expr->queryType()->getTypeCode();
+    if ((tc == type_decimal) || (tc == type_data))
         return true;
     return false;
 }

ecl/hqlcpp/hqlcppsys.ecl

@@ -31,7 +31,6 @@ const char * cppSystemText[]  = {
     
     "export InternalCppService := SERVICE:fold",
     //  searchTableStringN(unsigned4 num, string table, string search) : library='eclrtl';
-    //  memcpy(void * target, void * src, unsigned len);
     //internal code generation helper functions...
     "   unsigned4 an2l4(const string src) : eclrtl,pure,library='eclrtl',entrypoint='rtlStrToUInt4';",
     "   unsigned8 an2l8(const string src) : eclrtl,pure,library='eclrtl',entrypoint='rtlStrToUInt8';",
@@ -57,15 +56,15 @@ const char * cppSystemText[]  = {
 
     "   string _intformat(integer8 value, unsigned4 width, unsigned4 flags) : eclrtl,pure,library='eclrtl',entrypoint='rtlIntFormat';",
     "   string _realformat(real8 value, unsigned4 width, unsigned4 places) : eclrtl,pure,library='eclrtl',entrypoint='rtlRealFormat';",
-    "   l42an(string target, unsigned integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUInt4ToStr';",
-    "   l82an(string target, unsigned integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUInt8ToStr';",
-    "   ls42an(string target, integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt4ToStr';",
-    "   ls82an(string target, integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt8ToStr';",
+    "   l42an(noconst string target, unsigned integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUInt4ToStr';",
+    "   l82an(noconst string target, unsigned integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUInt8ToStr';",
+    "   ls42an(noconst string target, integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt4ToStr';",
+    "   ls82an(noconst string target, integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt8ToStr';",
 
-    "   l42vn(string target, unsigned integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUInt4ToVStr';",
-    "   l82vn(string target, unsigned integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUInt8ToVStr';",
-    "   ls42vn(string target, integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt4ToVStr';",
-    "   ls82vn(string target, integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt8ToVStr';",
+    "   l42vn(noconst string target, unsigned integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUInt4ToVStr';",
+    "   l82vn(noconst string target, unsigned integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUInt8ToVStr';",
+    "   ls42vn(noconst string target, integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt4ToVStr';",
+    "   ls82vn(noconst string target, integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt8ToVStr';",
 
     "   varstring l42vx(unsigned integer4 src) :    eclrtl,pure,library='eclrtl',entrypoint='rtlUInt4ToVStrX';",
     "   varstring l82vx(unsigned integer8 src) :    eclrtl,pure,library='eclrtl',entrypoint='rtlUInt8ToVStrX';",
@@ -77,9 +76,9 @@ const char * cppSystemText[]  = {
     "   string ls42ax(integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt4ToStrX';",
     "   string ls82ax(integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlInt8ToStrX';",
 
-    "   f2an(string target, real8 val) :    eclrtl,pure,library='eclrtl',entrypoint='rtlRealToStr';",
+    "   f2an(noconst string target, real8 val) :    eclrtl,pure,library='eclrtl',entrypoint='rtlRealToStr';",
     "   string _f2ax(real8 val) :   eclrtl,pure,library='eclrtl',entrypoint='rtlRealToStrX';",
-    "   f2vn(string target, real8 val) :    eclrtl,pure,library='eclrtl',entrypoint='rtlRealToVStr';",
+    "   f2vn(noconst string target, real8 val) :    eclrtl,pure,library='eclrtl',entrypoint='rtlRealToVStr';",
     "   varstring f2vx(real8 val) : eclrtl,pure,library='eclrtl',entrypoint='rtlRealToVStrX';",
 
     "   string str2StrX(const data src) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlStrToStrX';",
@@ -89,16 +88,16 @@ const char * cppSystemText[]  = {
     "   string ebcdic2asciiX(const ebcdic string src) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlEStrToStrX';",
     "   ebcdic string ascii2ebcdicX(const string src) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlStrToEStrX';",
 
-    "   strToQStr(qstring tgt, const data src) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlStrToQStr';",
+    "   strToQStr(noconst qstring tgt, const data src) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlStrToQStr';",
     "   qstring strToQStrX(const data src) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlStrToQStrX';",
     "   string qstr2StrX(const qstring src) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToStrX';",
     "   data qstr2DataX(const qstring src) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToDataX';",
     "   boolean qstr2Bool(const qstring src) :  eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToBool';",
-    "   qstr2Data(data tgt, const qstring src) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToData';",
-    "   qstr2Str(string tgt, const qstring src) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToStr';",
-    "   qstrToQStr(qstring tgt, const qstring src) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToQStr';",
+    "   qstr2Data(noconst data tgt, const qstring src) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToData';",
+    "   qstr2Str(noconst string tgt, const qstring src) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToStr';",
+    "   qstrToQStr(noconst qstring tgt, const qstring src) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToQStr';",
     "   qstring qstrToQStrX(const qstring src) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToQStrX';",
-    "   qstr2VStr(string tgt, const qstring src) :  eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToVStr';",
+    "   qstr2VStr(noconst string tgt, const qstring src) :  eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrToVStr';",
     "   unsigned4 qstrLength(unsigned4 sz) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrLength';",
     "   unsigned4 qstrSize(unsigned4 len) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlQStrSize';",
 
@@ -111,35 +110,35 @@ const char * cppSystemText[]  = {
     "   unicode concatUnicode(unicode tgt) :    eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatUnicode';",
     "   varunicode concatVUnicode(varunicode tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatVUnicode';",
     "   utf8 concatUtf8(utf8 tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatUtf8';",
-    "   varstring createQuotedString(string tgt) :  eclrtl,pure,library='eclrtl',entrypoint='rtlCreateQuotedString';",
+    "   varstring createQuotedString(string src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlCreateQuotedString';",
     "   concatExtend(boolean prev, const string src) :  eclrtl,pure,library='eclrtl',include,entrypoint='rtlConcatExtend';",
-    "   concatStrF(data tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatStrF';",
-    "   concatVStrF(string tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatVStrF';",
-    "   concatUnicodeF(unicode tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatUnicodeF';",
-    "   concatVUNicodeF(unicode tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatVUnicodeF';",
+    "   concatStrF(noconst data tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatStrF';",
+    "   concatVStrF(noconst string tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatVStrF';",
+    "   concatUnicodeF(noconst unicode tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatUnicodeF';",
+    "   concatVUNicodeF(noconst unicode tgt) : eclrtl,pure,include,library='eclrtl',entrypoint='rtlConcatVUnicodeF';",
 
     "   string concatStrExtend(const string src) :  eclrtl,extend,pure,library='eclrtl',include,entrypoint='rtlConcatExtend';",
     "   unicode concatUnicodeExtend(const unicode src) :  eclrtl,extend,pure,library='eclrtl',include,entrypoint='rtlConcatUnicodeExtend';",
 
-    "   unsigned4 concatStr2Str(string tgt, unsigned4 idx, const string src) :  eclrtl,library='eclrtl',entrypoint='rtlConcatStrToStr';",
-    "   unsigned4 concatVStr2Str(string tgt, unsigned4 idx, const varstring src) : eclrtl,library='eclrtl',entrypoint='rtlConcatVStrToStr';",
-    "   unsigned4 concatUtf8ToUtf8(utf8 tgt, unsigned4 idx, const utf8 src) : eclrtl,library='eclrtl',entrypoint='rtlConcatUtf8ToUtf8';",
-    "   unsigned4 concatUnicodeToUnicode(unicode tgt, unsigned4 idx, const unicode src) : eclrtl,library='eclrtl',entrypoint='rtlConcatUnicodeToUnicode';",
-    "   unsigned4 concatVUnicodeToUnicode(unicode tgt, unsigned4 idx, const varunicode src) : eclrtl,library='eclrtl',entrypoint='rtlConcatVUnicodeToUnicode';",
-    "   codepage2Unicode(unicode tgt, const string src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlCodepageToUnicode';",
-    "   codepage2VUnicode(unicode tgt, const string src, const varstring codepage) :    eclrtl,pure,library='eclrtl',entrypoint='rtlCodepageToVUnicode';",
-    "   vcodepage2Unicode(unicode tgt, const varstring src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlVCodepageToUnicode';",
-    "   vcodepage2VUnicode(unicode tgt, const varstring src, const varstring codepage) :    eclrtl,pure,library='eclrtl',entrypoint='rtlVCodepageToVUnicode';",
-    "   unicode2Codepage(string tgt, const unicode src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToCodepage';",
-    "   unicode2Data(data tgt, const unicode src) : eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToData';",
-    "   unicode2VCodepage(string tgt, const unicode src, const varstring codepage) :    eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToVCodepage';",
-    "   vunicode2Codepage(string tgt, const varunicode src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToCodepage';",
-    "   vunicode2Data(data tgt, const varunicode src) : eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToData';",
-    "   vunicode2VCodepage(string tgt, const varunicode src, const varstring codepage) :    eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToVCodepage';",
-    "   unicode2Unicode(unicode tgt, const unicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToUnicode';",
-    "   unicode2VUnicode(unicode tgt, const unicode src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToVUnicode';",
-    "   vunicode2Unicode(unicode tgt, const varunicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToUnicode';",
-    "   vunicode2VUnicode(unicode tgt, const varunicode src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToVUnicode';",
+    "   unsigned4 concatStr2Str(noconst string tgt, unsigned4 idx, const string src) :  eclrtl,library='eclrtl',entrypoint='rtlConcatStrToStr';",
+    "   unsigned4 concatVStr2Str(noconst string tgt, unsigned4 idx, const varstring src) : eclrtl,library='eclrtl',entrypoint='rtlConcatVStrToStr';",
+    "   unsigned4 concatUtf8ToUtf8(noconst utf8 tgt, unsigned4 idx, const utf8 src) : eclrtl,library='eclrtl',entrypoint='rtlConcatUtf8ToUtf8';",
+    "   unsigned4 concatUnicodeToUnicode(noconst unicode tgt, unsigned4 idx, const unicode src) : eclrtl,library='eclrtl',entrypoint='rtlConcatUnicodeToUnicode';",
+    "   unsigned4 concatVUnicodeToUnicode(noconst unicode tgt, unsigned4 idx, const varunicode src) : eclrtl,library='eclrtl',entrypoint='rtlConcatVUnicodeToUnicode';",
+    "   codepage2Unicode(noconst unicode tgt, const string src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlCodepageToUnicode';",
+    "   codepage2VUnicode(noconst unicode tgt, const string src, const varstring codepage) :    eclrtl,pure,library='eclrtl',entrypoint='rtlCodepageToVUnicode';",
+    "   vcodepage2Unicode(noconst unicode tgt, const varstring src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlVCodepageToUnicode';",
+    "   vcodepage2VUnicode(noconst unicode tgt, const varstring src, const varstring codepage) :    eclrtl,pure,library='eclrtl',entrypoint='rtlVCodepageToVUnicode';",
+    "   unicode2Codepage(noconst string tgt, const unicode src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToCodepage';",
+    "   unicode2Data(noconst data tgt, const unicode src) : eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToData';",
+    "   unicode2VCodepage(noconst string tgt, const unicode src, const varstring codepage) :    eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToVCodepage';",
+    "   vunicode2Codepage(noconst string tgt, const varunicode src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToCodepage';",
+    "   vunicode2Data(noconst data tgt, const varunicode src) : eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToData';",
+    "   vunicode2VCodepage(noconst string tgt, const varunicode src, const varstring codepage) :    eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToVCodepage';",
+    "   unicode2Unicode(noconst unicode tgt, const unicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToUnicode';",
+    "   unicode2VUnicode(noconst unicode tgt, const unicode src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToVUnicode';",
+    "   vunicode2Unicode(noconst unicode tgt, const varunicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToUnicode';",
+    "   vunicode2VUnicode(noconst unicode tgt, const varunicode src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToVUnicode';",
     "   unicode codepage2UnicodeX(const string src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlCodepageToUnicodeX';",
     "   varunicode codepage2VUnicodeX(const string src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlCodepageToVUnicodeX';",
     "   unicode vcodepage2UnicodeX(const varstring src, const varstring codepage) : eclrtl,pure,library='eclrtl',entrypoint='rtlVCodepageToUnicodeX';",
@@ -153,14 +152,14 @@ const char * cppSystemText[]  = {
     "   varunicode unicode2VUnicodeX(const unicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToVUnicodeX';",
     "   unicode vunicode2UnicodeX(const varunicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToUnicodeX';",
     "   varunicode vunicode2VUnicodeX(const varunicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlVUnicodeToVUnicodeX';",
-    "   unicodeStrcpy(varunicode tgt, const varunicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeStrcpy';",
+    "   unicodeStrcpy(noconst varunicode tgt, const varunicode src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeStrcpy';",
     "   unsigned4 unicodeStrlen(const varunicode src) : eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeStrlen';",
     "   data keyUnicodeX(const unicode src, const varstring loc) :  eclrtl,library='eclrtl',entrypoint='rtlKeyUnicodeX';",
     "   data keyUnicodeStrengthX(const unicode src, const varstring loc, unsigned4 str) :   eclrtl,library='eclrtl',entrypoint='rtlKeyUnicodeStrengthX';",
 
-    "   l2vn4(string target, unsigned integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtl_l2vn4';",
-    "   l2vn8(string target, unsigned integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtl_l2vn8';",
-    "   pack(string target, const string src) : eclrtl,library='eclrtl',entrypoint='rtlPack';",
+    "   l2vn4(noconst string target, unsigned integer4 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtl_l2vn4';",
+    "   l2vn8(noconst string target, unsigned integer8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtl_l2vn8';",
+    "   pack(noconst string target, const string src) : eclrtl,library='eclrtl',entrypoint='rtlPack';",
     "   integer4 searchDataTable(unsigned4 num, unsigned4 elemlen, const string entries, const string search) : eclrtl,pure,library='eclrtl',include='eclrtl.hpp',entrypoint='rtlNewSearchDataTable';",
     "   integer4 searchEStringTable(unsigned4 num, unsigned4 elemlen, const string entries, const string search) : eclrtl,pure,library='eclrtl',include='eclrtl.hpp',entrypoint='rtlNewSearchEStringTable';",
     "   integer4 searchQStringTable(unsigned4 num, unsigned4 elemlen, const string entries, const string search) : eclrtl,pure,library='eclrtl',include='eclrtl.hpp',entrypoint='rtlNewSearchQStringTable';",
@@ -175,9 +174,9 @@ const char * cppSystemText[]  = {
     "   integer4 searchTableUInteger4(set of unsigned4 values, unsigned4 search) : eclrtl,pure,library='eclrtl',include,entrypoint='rtlSearchTableUInteger4',oldSetFormat;",
     "   integer4 searchTableUInteger8(set of unsigned8 values, unsigned8 search) : eclrtl,pure,library='eclrtl',include,entrypoint='rtlSearchTableUInteger8',oldSetFormat;",
 
-    "   unpack(string target, const string src) :   eclrtl,library='eclrtl',entrypoint='rtlUnpack';",
-    "   ebcdic2ascii(string target, const ebcdic string src) :  eclrtl,library='eclrtl',entrypoint='rtlEStrToStr';",
-    "   ascii2ebcdic(ebcdic string target, const string src) :  eclrtl,library='eclrtl',entrypoint='rtlStrToEStr';",
+    "   unpack(noconst string target, const string src) :   eclrtl,library='eclrtl',entrypoint='rtlUnpack';",
+    "   ebcdic2ascii(noconst string target, const ebcdic string src) :  eclrtl,library='eclrtl',entrypoint='rtlEStrToStr';",
+    "   ascii2ebcdic(noconst ebcdic string target, const string src) :  eclrtl,library='eclrtl',entrypoint='rtlStrToEStr';",
 
     "   unsigned4 trimStrLen(const string src): eclrtl,pure,library='eclrtl',entrypoint='rtlTrimStrLen';",
     "   unsigned4 trimUnicodeStrLen(const unicode src): eclrtl,pure,library='eclrtl',entrypoint='rtlTrimUnicodeStrLen';",
@@ -206,13 +205,13 @@ const char * cppSystemText[]  = {
     "   unicode trimVUnicodeAll(const varunicode src) : eclrtl,pure,library='eclrtl',entrypoint='rtlTrimVUnicodeAll';",
     "   data subDataFTX(const data src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubDataFTX';",
     "   data subDataFX(const data src, unsigned4 from) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubDataFX';",
-    "   subDataFT(data tgt, const data src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubDataFT';",
+    "   subDataFT(noconst data tgt, const data src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubDataFT';",
     "   string subStrFTX(const string src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubStrFTX';",
     "   string subStrFX(const string src, unsigned4 from) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubStrFX';",
-    "   subStrFT(string tgt, const string src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubStrFT';",
+    "   subStrFT(noconst string tgt, const string src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubStrFT';",
     "   qstring subQStrFTX(const qstring src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubQStrFTX';",
     "   qstring subQStrFX(const qstring src, unsigned4 from) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubQStrFX';",
-    "   subQStrFT(qstring tgt, const qstring src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubQStrFT';",
+    "   subQStrFT(noconst qstring tgt, const qstring src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlSubQStrFT';",
     "   unicode unicodeSubStrFTX(const unicode src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeSubStrFTX';",
     "   unicode unicodeSubStrFX(const unicode src, unsigned4 from) : eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeSubStrFX';",
     "   unsigned4 rtlMax(unsigned4 idx, unsigned4 len) : eclrtl,pure,entrypoint='rtlMax',include;",
@@ -232,22 +231,22 @@ const char * cppSystemText[]  = {
     "   integer4 prefixDiffStr(const string l, const string r, unsigned4 origin) : eclrtl,pure,library='eclrtl',entrypoint='rtlPrefixDiffStrEx';",
     "   integer4 prefixDiffUnicode(const unicode l, const unicode r, const varstring loc, unsigned4 origin) : eclrtl,pure,library='eclrtl',entrypoint='rtlPrefixDiffUnicodeEx';",
 
-    "   createOrder(data1 tgt, const data1 src, unsigned4 num, unsigned4 width, const data1 compare) : eclrtl,library='eclrtl',entrypoint='rtlCreateOrder';",
+    "   createOrder(noconst data1 tgt, const data1 src, unsigned4 num, unsigned4 width, const data1 compare) : eclrtl,library='eclrtl',entrypoint='rtlCreateOrder';",
     "   unsigned4 rankFromOrder(unsigned4 idx, unsigned4 num, const data1 order) : eclrtl,pure,library='eclrtl',entrypoint='rtlRankFromOrder';",
     "   unsigned4 rankedFromOrder(unsigned4 idx, unsigned4 num, const data1 order) : eclrtl,pure,library='eclrtl',entrypoint='rtlRankedFromOrder';",
         
     "   boolean data2Bool(const data src) : eclrtl,pure,library='eclrtl',entrypoint='rtlDataToBool';",
-    "   bool2Data(data tgt, boolean src) :  eclrtl,library='eclrtl',entrypoint='rtlBoolToData';",
-    "   bool2Str(data tgt, boolean src) :   eclrtl,library='eclrtl',entrypoint='rtlBoolToStr';",
+    "   bool2Data(noconst data tgt, boolean src) :  eclrtl,library='eclrtl',entrypoint='rtlBoolToData';",
+    "   bool2Str(noconst data tgt, boolean src) :   eclrtl,library='eclrtl',entrypoint='rtlBoolToStr';",
     "   bool2VStr(varstring tgt, boolean src) : eclrtl,library='eclrtl',entrypoint='rtlBoolToVStr';",
-    "   str2Data(data tgt, const data src) :    eclrtl,library='eclrtl',entrypoint='rtlStrToData';",
-    "   str2Str(data tgt, const data src) : eclrtl,library='eclrtl',entrypoint='rtlStrToStr';",
-    "   eStr2EStr(data tgt, const data src) :   eclrtl,library='eclrtl',entrypoint='rtlEStrToEStr';",
-    "   eStr2VStr(data tgt, const ebcdic string src) :  eclrtl,library='eclrtl',entrypoint='rtlEStrToVStr';",
-    "   str2VStr(data tgt, const data src) :    eclrtl,library='eclrtl',entrypoint='rtlStrToVStr';",
-    "   vStr2Data(data tgt, const varstring src) :  eclrtl,library='eclrtl',entrypoint='rtlVStrToData';",
-    "   vStr2Str(data tgt, const varstring src) :   eclrtl,library='eclrtl',entrypoint='rtlVStrToStr';",
-    "   vStr2VStr(data tgt, const varstring src) :  eclrtl,library='eclrtl',entrypoint='rtlVStrToVStr';",
+    "   str2Data(noconst data tgt, const data src) :    eclrtl,library='eclrtl',entrypoint='rtlStrToData';",
+    "   str2Str(noconst data tgt, const data src) : eclrtl,library='eclrtl',entrypoint='rtlStrToStr';",
+    "   eStr2EStr(noconst data tgt, const data src) :   eclrtl,library='eclrtl',entrypoint='rtlEStrToEStr';",
+    "   eStr2VStr(noconst data tgt, const ebcdic string src) :  eclrtl,library='eclrtl',entrypoint='rtlEStrToVStr';",
+    "   str2VStr(noconst data tgt, const data src) :    eclrtl,library='eclrtl',entrypoint='rtlStrToVStr';",
+    "   vStr2Data(noconst data tgt, const varstring src) :  eclrtl,library='eclrtl',entrypoint='rtlVStrToData';",
+    "   vStr2Str(noconst data tgt, const varstring src) :   eclrtl,library='eclrtl',entrypoint='rtlVStrToStr';",
+    "   vStr2VStr(noconst data tgt, const varstring src) :  eclrtl,library='eclrtl',entrypoint='rtlVStrToVStr';",
 
     "   boolean csvStr2Bool(const string src) : eclrtl,pure,library='eclrtl',entrypoint='rtlCsvStrToBool';",
     "   string bool2StrX(boolean src) : eclrtl,library='eclrtl',entrypoint='rtlBoolToStrX';",
@@ -261,15 +260,15 @@ const char * cppSystemText[]  = {
     "   unsigned8 readUInt5(const data1 src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlReadUInt5';",
     "   unsigned8 readUInt6(const data1 src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlReadUInt6';",
     "   unsigned8 readUInt7(const data1 src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlReadUInt7';",
-    "   writeInt3(data1 src, unsigned4 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteInt3';",
-    "   writeInt5(data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteInt5';",
-    "   writeInt6(data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteInt6';",
-    "   writeInt7(data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteInt7';",
+    "   writeInt3(noconst data1 src, unsigned4 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteInt3';",
+    "   writeInt5(noconst data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteInt5';",
+    "   writeInt6(noconst data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteInt6';",
+    "   writeInt7(noconst data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteInt7';",
 
     "   integer8 getPackedSigned(const data1 src) : eclrtl,library='eclrtl',entrypoint='rtlGetPackedSigned';",
     "   unsigned8 getPackedUnsigned(const data1 src) :  eclrtl,library='eclrtl',entrypoint='rtlGetPackedUnsigned';",
-    "   setPackedSigned(data1 src, integer8 value) :    eclrtl,library='eclrtl',entrypoint='rtlSetPackedSigned';",
-    "   setPackedUnsigned(data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlSetPackedUnsigned';",
+    "   setPackedSigned(noconst data1 src, integer8 value) :    eclrtl,library='eclrtl',entrypoint='rtlSetPackedSigned';",
+    "   setPackedUnsigned(noconst data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlSetPackedUnsigned';",
     "   unsigned4 getPackedSize(const data1 src) :  eclrtl,library='eclrtl',entrypoint='rtlGetPackedSize';",
 
     #if 0 // Not yet needed....
@@ -281,10 +280,10 @@ const char * cppSystemText[]  = {
     "   unsigned8 readSwapUInt5(const data1 src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlReadSwapUInt5';",
     "   unsigned8 readSwapUInt6(const data1 src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlReadSwapUInt6';",
     "   unsigned8 readSwapUInt7(const data1 src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlReadSwapUInt7';",
-    "   writeSwapInt3(data1 src, unsigned4 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteSwapInt3';",
-    "   writeSwapInt5(data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteSwapInt5';",
-    "   writeSwapInt6(data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteSwapInt6';",
-    "   writeSwapInt7(data1 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteSwapInt7';",
+    "   writeSwapInt3(noconst data3 src, unsigned4 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteSwapInt3';",
+    "   writeSwapInt5(noconst data5 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteSwapInt5';",
+    "   writeSwapInt6(noconst data6 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteSwapInt6';",
+    "   writeSwapInt7(noconst data7 src, unsigned8 value) : eclrtl,library='eclrtl',entrypoint='rtlWriteSwapInt7';",
     #endif
 
     "   integer2 revInt2(const data1 src) : eclrtl,pure,library='eclrtl',include,entrypoint='rtlRevInt2';",
@@ -323,12 +322,12 @@ const char * cppSystemText[]  = {
     "   DecSub() :  eclrtl,library='eclrtl',entrypoint='DecSub';",
     "   DecSubR() : eclrtl,library='eclrtl',entrypoint='DecSubR';",
     "   DecNegate() :   eclrtl,library='eclrtl',entrypoint='DecNegate';",
-    "   unsigned4 DecPopString(string tgt) :    eclrtl,library='eclrtl',entrypoint='DecPopString';",
-    "   DecPopVString(string tgt) : eclrtl,library='eclrtl',entrypoint='DecPopCString';",
+    "   unsigned4 DecPopString(noconst string tgt) :    eclrtl,library='eclrtl',entrypoint='DecPopString';",
+    "   DecPopVString(noconst string tgt) : eclrtl,library='eclrtl',entrypoint='DecPopCString';",
     "   varstring DecPopVStringX() :    eclrtl,library='eclrtl',entrypoint='DecPopCStringX';",
     "   integer8 DecPopInt64() :    eclrtl,library='eclrtl',entrypoint='DecPopInt64';",
-    "   DecPopDecimal(data1 tgt,unsigned1 len,unsigned1 prec) : eclrtl,library='eclrtl',entrypoint='DecPopDecimal';",
-    "   DecPopUDecimal(data1 tgt,unsigned1 len,unsigned1 prec) : eclrtl,library='eclrtl',entrypoint='DecPopUDecimal';",
+    "   DecPopDecimal(noconst data1 tgt,unsigned1 len,unsigned1 prec) : eclrtl,library='eclrtl',entrypoint='DecPopDecimal';",
+    "   DecPopUDecimal(noconst data1 tgt,unsigned1 len,unsigned1 prec) : eclrtl,library='eclrtl',entrypoint='DecPopUDecimal';",
     "   unsigned4 DecPopUlong() :   eclrtl,library='eclrtl',entrypoint='DecPopUlong';",
     "   real8 DecPopReal() :    eclrtl,library='eclrtl',entrypoint='DecPopReal';",
     "   DecPushString(const string tgt) :   eclrtl,library='eclrtl',entrypoint='DecPushString';",
@@ -401,14 +400,14 @@ const char * cppSystemText[]  = {
     "   unsigned8 hash64VUnicode(const varunicode src, unsigned8 initval) : eclrtl,pure,library='eclrtl',entrypoint='rtlHash64VUnicode';",
     "   unsigned8 hash64Utf8(const utf8 src, unsigned8 initval) :   eclrtl,pure,library='eclrtl',entrypoint='rtlHash64Utf8';",
 
-    "   hashMd5Init(data _state) :  eclrtl,entrypoint='rtlHashMd5Init';",
-    "   hashMd5Data(const data _value, data _state) :   eclrtl,entrypoint='rtlHashMd5Data';",
-    "   data16 hashMd5Finish(data _state) : eclrtl,entrypoint='rtlHashMd5Finish';",
+    "   hashMd5Init(noconst data _state) :  eclrtl,entrypoint='rtlHashMd5Init';",
+    "   hashMd5Data(const data _value, noconst data _state) :   eclrtl,entrypoint='rtlHashMd5Data';",
+    "   data16 hashMd5Finish(noconst data _state) : eclrtl,entrypoint='rtlHashMd5Finish';",
 
     "   boolean validReal(const data src) : eclrtl,pure,library='eclrtl',entrypoint='rtlIsValidReal';",
     "   real8 createRealNull() : eclrtl,pure,library='eclrtl',entrypoint='rtlCreateRealNull';",
 
-    "   deserializeRaw(data field, boolean o) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeRaw';",
+    "   deserializeRaw(noconst data field, boolean o) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeRaw';",
     "   data deserializeDataX(boolean o) :  eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeDataX';",
     "   dataset deserializeDatasetX(boolean o) :    eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeDataX';",
     "   _linkcounted_ dictionary deserializeDictionaryX(boolean _deserializer, boolean _input) : eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeDictionaryX';",
@@ -562,10 +561,10 @@ const char * cppSystemText[]  = {
     "   set of unicode regexUStrMatchSet(const unicode _search) : method,pure,entrypoint='getMatchSet';"
 
     //clibrary functions that are called from the code generation
-    "   free(data1 src) : eclrtl,library='eclrtl',entrypoint='rtlFree';",
+    "   free(noconst data1 src) : eclrtl,library='eclrtl',entrypoint='rtlFree';",
     "   integer4 memcmp(const data1 target, const data1 src, unsigned4 len) : sys,pure,entrypoint='memcmp';",
-    "   data1 memcpy(data1 target, const data1 src, unsigned4 len): sys,entrypoint='memcpy';",
-    "   data1 memset(data1 target, integer4 fill, unsigned4 len) :  sys,entrypoint='memset';",
+    "   data1 memcpy(noconst data1 target, const data1 src, unsigned4 len): sys,entrypoint='memcpy';",
+    "   data1 memset(noconst data1 target, integer4 fill, unsigned4 len) :  sys,entrypoint='memset';",
     "   strcpy(const varstring1 target, const varstring1 src) : sys,entrypoint='strcpy';",
     "   unsigned4 strlen(const varstring1 src) :    sys,pure,entrypoint='strlen';",
 
@@ -628,17 +627,17 @@ const char * cppSystemText[]  = {
     "   utf8 utf8SubStrFTX(const utf8 src, unsigned4 from, unsigned4 to) : eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8SubStrFTX';",
     "   utf8 utf8SubStrFX(const utf8 src, unsigned4 from) : eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8SubStrFX';",
 
-    "   utf82Codepage(string tgt, const utf8 src, const varstring codepage) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlUtf8ToCodepage';",
-    "   utf82Data(data tgt, const utf8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToData';",
-    "   utf82Unicode(unicode tgt, const utf8 src) : eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToUnicode';",
-    "   utf8ToUtf8(utf8 tgt, const utf8 src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToUtf8';",
+    "   utf82Codepage(noconst string tgt, const utf8 src, const varstring codepage) :   eclrtl,pure,include,library='eclrtl',entrypoint='rtlUtf8ToCodepage';",
+    "   utf82Data(noconst data tgt, const utf8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToData';",
+    "   utf82Unicode(noconst unicode tgt, const utf8 src) : eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToUnicode';",
+    "   utf8ToUtf8(noconst utf8 tgt, const utf8 src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToUtf8';",
     "   string utf82CodepageX(const utf8 src, const varstring codepage) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToCodepageX';",
     "   data utf82DataX(const utf8 src) :   eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToDataX';",
     "   unicode utf82UnicodeX(const utf8 src) : eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToUnicodeX';",
     "   utf8 utf8ToUtf8X(const utf8 src) :  eclrtl,pure,library='eclrtl',entrypoint='rtlUtf8ToUtf8X';",
-    "   codepageToUtf8(utf8 tgt, const string src, const varstring codepage) :  eclrtl,pure,library='eclrtl',entrypoint='rtlCodepageToUtf8';",
+    "   codepageToUtf8(noconst utf8 tgt, const string src, const varstring codepage) :  eclrtl,pure,library='eclrtl',entrypoint='rtlCodepageToUtf8';",
     "   utf8 codepageToUtf8X(const string src, const varstring codepage) :  eclrtl,pure,library='eclrtl',entrypoint='rtlCodepageToUtf8X';",
-    "   unicodeToUtf8(utf8 tgt, const unicode src) :    eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToUtf8';",
+    "   unicodeToUtf8(noconst utf8 tgt, const unicode src) :    eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToUtf8';",
     "   utf8 unicodeToUtf8X(const unicode src) :    eclrtl,pure,library='eclrtl',entrypoint='rtlUnicodeToUtf8X';",
     "   unsigned4 utf8Size(const data src) : eclrtl,include,pure,library='eclrtl',entrypoint='rtlUtf8Size';", 
     "   unsigned4 utf8Length(const data src) : eclrtl,include,pure,library='eclrtl',entrypoint='rtlUtf8Length';", 
@@ -654,7 +653,7 @@ const char * cppSystemText[]  = {
     "   dataset getResultDataset(const varstring stepname, unsigned4 sequence, boolean xmltransformer, boolean csvtransformer) : ctxmethod,pure,entrypoint='getResultRaw';",    // an alias
     "   qstring getResultQString(const varstring stepname, unsigned4 sequence) : ctxmethod,pure,entrypoint='getResultQString';",
     "   string  getResultString(const varstring stepname, unsigned4 sequence) : ctxmethod,pure,entrypoint='getResultString';",
-    "   getResultStringF(string target, const varstring stepname, unsigned4 sequence) : ctxmethod,pure,entrypoint='getResultStringF';",
+    "   getResultStringF(noconst string target, const varstring stepname, unsigned4 sequence) : ctxmethod,pure,entrypoint='getResultStringF';",
     "   unicode getResultUnicode(const varstring stepname, unsigned4 sequence) : ctxmethod,pure,entrypoint='getResultUnicode';",
     "   varstring   getResultVarString(const varstring stepname, unsigned4 sequence) : ctxmethod,pure,entrypoint='getResultVarString';",
     "   varunicode  getResultVarUnicode(const varstring stepname, unsigned4 sequence) : ctxmethod,pure,entrypoint='getResultVarUnicode';",
@@ -723,23 +722,23 @@ const char * cppSystemText[]  = {
     "   killRange(const data1 lo, const data1 hi) : method,include='rtlkey.hpp',entrypoint='killRange';",
 
     "   boolean columnGetBool(const varstring name) : method,pure,entrypoint='getBool';",
-    "   columnGetData(data result, const varstring name) : method,pure,entrypoint='getData';",
+    "   columnGetData(noconst data result, const varstring name) : method,pure,entrypoint='getData';",
     "   data columnGetDataX(const varstring name) : method,pure,entrypoint='getDataX';",
     "   integer columnGetInt(const varstring name) : method,pure,entrypoint='getInt';",
     "   boolean columnGetSetIsAll(const varstring name) : method,pure,entrypoint='getIsSetAll';",
-    "   columnGetQString(qstring result, const varstring name) : method,pure,entrypoint='getQString';",
-    "   columnGetString(string result, const varstring name) : method,pure,entrypoint='getString';",
+    "   columnGetQString(noconst qstring result, const varstring name) : method,pure,entrypoint='getQString';",
+    "   columnGetString(noconst string result, const varstring name) : method,pure,entrypoint='getString';",
     "   string columnGetStringX(const varstring name) : method,pure,entrypoint='getStringX';",
     "   unicode columnGetUnicodeX(const varstring name) : method,pure,entrypoint='getUnicodeX';",
     "   utf8 columnGetUtf8X(const varstring name) : method,pure,entrypoint='getUtf8X';",
 
     "   boolean columnReadBool(const varstring name, boolean _default) : method,pure,entrypoint='readBool';",
-    "   columnReadData(data result, const varstring name, const data _default) : method,pure,entrypoint='readData';",
+    "   columnReadData(noconst data result, const varstring name, const data _default) : method,pure,entrypoint='readData';",
     "   data columnReadDataX(const varstring name, const data _default) : method,pure,entrypoint='readDataX';",
     "   integer columnReadInt(const varstring name, integer _default) : method,pure,entrypoint='readInt';",
     "   boolean columnReadSetIsAll(const varstring name, boolean _default) : method,pure,entrypoint='readIsSetAll';",
-    "   columnReadQString(qstring result, const varstring name, const qstring _default) : method,pure,entrypoint='readQString';",
-    "   columnReadString(string result, const varstring name, const string _default) : method,pure,entrypoint='readString';",
+    "   columnReadQString(noconst qstring result, const varstring name, const qstring _default) : method,pure,entrypoint='readQString';",
+    "   columnReadString(noconst string result, const varstring name, const string _default) : method,pure,entrypoint='readString';",
     "   string columnReadStringX(const varstring name, const string _default) : method,pure,entrypoint='readStringX';",
     "   unicode columnReadUnicodeX(const varstring name, const unicode _default) : method,pure,entrypoint='readUnicodeX';",
     "   utf8 columnReadUtf8X(const varstring name, const utf8 _default) : method,pure,entrypoint='readUtf8X';",
@@ -814,9 +813,9 @@ const char * cppSystemText[]  = {
     "   row(dummyRecord) deserializerPeek(unsigned4 _maxSize) : omethod,entrypoint='peek';",
     "   unsigned8 deserializerBeginNested() : omethod,entrypoint='beginNested';",
     "   boolean deserializerFinishedNested(unsigned8 pos) : omethod,entrypoint='finishedNested';",
-    "   unsigned4 deserializerReadN(data _target) : omethod,entrypoint='read';",
+    "   unsigned4 deserializerReadN(noconst data _target) : omethod,entrypoint='read';",
     "   unsigned4 deserializerReadSize() : omethod,entrypoint='readSize';",
-    "   unsigned4 deserializerReadPackedInt(data _target) : omethod,entrypoint='readPackedInt';",
+    "   unsigned4 deserializerReadPackedInt(noconst data _target) : omethod,entrypoint='readPackedInt';",
     "   unsigned4 deserializerReadUtf8(boolean rowBuilder, unsigned4 offset, unsigned4 fixedSize, unsigned4 len) : omethod,entrypoint='readUtf8';",
     "   unsigned4 deserializerReadVStr(boolean rowBuilder, unsigned4 offset, unsigned4 fixedSize) : omethod,entrypoint='readVStr';",
     "   unsigned4 deserializerReadVUni(boolean rowBuilder, unsigned4 offset, unsigned4 fixedSize) : omethod,entrypoint='readVUni';",

ecl/hqlcpp/hqlcpputil.cpp

@@ -407,42 +407,6 @@ IHqlExpression * projectCreateSetDataset(IHqlExpression * expr)
     return LINK(expr);
 }
 
-IHqlExpression * mapInternalFunctionParameters(IHqlExpression * expr)
-{
-    switch (expr->getOperator())
-    {
-    case no_sortlist:
-        {
-            HqlExprArray args;
-            ForEachChild(i, expr)
-                args.append(*mapInternalFunctionParameters(expr->queryChild(i)));
-            return cloneOrLink(expr, args);
-        }
-    case no_param:
-        {
-            ITypeInfo * type = expr->queryType();
-            //String parameters need to be passed as c++ const string parameters
-            switch (type->getTypeCode())
-            {
-            case type_string:
-            case type_varstring:
-            case type_data:
-            case type_qstring:
-            case type_unicode:
-            case type_utf8:
-            case type_varunicode:
-                if (!expr->hasAttribute(constAtom))
-                    return appendOwnedOperand(expr, createAttribute(constAtom));
-                break;
-            }
-            break;
-        }
-    }
-
-    return LINK(expr);
-}
-
-
 bool mustInitializeField(IHqlExpression * field)
 {
     if (hasLinkCountedModifier(field))

ecl/hqlcpp/hqlcpputil.hpp

@@ -48,7 +48,6 @@ extern IHqlExpression * ensurePositiveOrZeroInt64(IHqlExpression * expr);
 extern void getOutputLibraryName(SCMStringBuffer & libraryName, IConstWorkUnit * wu);
 extern bool canCreateTemporary(IHqlExpression * expr);
 extern IHqlExpression * projectCreateSetDataset(IHqlExpression * createsetExpr);
-extern IHqlExpression * mapInternalFunctionParameters(IHqlExpression * expr);
 
 extern bool mustInitializeField(IHqlExpression * field);
 extern bool worthGeneratingRowAsSingleActivity(IHqlExpression * expr);

ecl/hqlcpp/hqlsource.cpp

@@ -7251,6 +7251,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityFetch(BuildCtx & ctx, IHqlExpr
 {
     IHqlExpression *fetch = queryFetch(expr);
     IHqlExpression *tableExpr = queryPhysicalRootTable(fetch->queryChild(0));
+    if (!tableExpr)
+        throwError(HQLERR_FetchNonDiskfile);
     FetchBuilder info(*this, tableExpr, tableExpr->queryChild(0), expr);
     info.gatherVirtualFields(false, true);//?needToSerializeRecord(mode)
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -6190,12 +6190,8 @@ IHqlExpression * WorkflowTransformer::transformInternalFunction(IHqlExpression *
         funcname.append("_").append(newFuncDef->queryName()).toLowerCase();
     OwnedHqlExpr funcNameExpr = createConstant(funcname);
 
-    IHqlExpression * formals = newFuncDef->queryChild(1);
-    OwnedHqlExpr newFormals = mapInternalFunctionParameters(formals);
-
     HqlExprArray bodyArgs;
-    bodyArgs.append(*replaceParameters(ecl, formals, newFormals));
-    unwindChildren(bodyArgs, body, 1);
+    unwindChildren(bodyArgs, body, 0);
     bodyArgs.append(*createLocalAttribute());
     bodyArgs.append(*createExprAttribute(entrypointAtom, LINK(funcNameExpr)));
     OwnedHqlExpr newBody = body->clone(bodyArgs);
@@ -6204,8 +6200,7 @@ IHqlExpression * WorkflowTransformer::transformInternalFunction(IHqlExpression *
 
     HqlExprArray funcdefArgs;
     funcdefArgs.append(*LINK(newBody));
-    funcdefArgs.append(*LINK(newFormals));
-    unwindChildren(funcdefArgs, newFuncDef, 2);
+    unwindChildren(funcdefArgs, newFuncDef, 1);
     OwnedHqlExpr namedFuncDef = newFuncDef->clone(funcdefArgs);
     inheritDependencies(namedFuncDef);
 

ecl/hqlcpp/hqlwcpp.cpp

@@ -804,7 +804,7 @@ void HqlCppWriter::generateParamCpp(IHqlExpression * param, IHqlExpression * att
     paramNameText.append(paramName).toLowerCase();
 
     bool isOut = false;
-    bool isConst = false;
+    bool isConst = true;
     unsigned maxAttr = param->numChildren();
     unsigned attrIdx;
     for (attrIdx = 0; attrIdx < maxAttr; attrIdx++)
@@ -812,8 +812,8 @@ void HqlCppWriter::generateParamCpp(IHqlExpression * param, IHqlExpression * att
         IHqlExpression * attr = param->queryChild(attrIdx);
         if (attr->isAttribute())
         {
-            if (attr->queryName() == constAtom)
-                isConst = true;
+            if (attr->queryName() == noConstAtom)
+                isConst = false;
             else if (attr->queryName() == outAtom)
                 isOut = true;
         }
@@ -950,10 +950,13 @@ void HqlCppWriter::generateParamCpp(IHqlExpression * param, IHqlExpression * att
         }
         // Other set types just fall through and will be treated like other types.
     case type_qstring: case type_string: case type_varstring: case type_data:
+    case type_utf8: case type_unicode: case type_varunicode:
+    case type_row:
+        if (isConst)
+            out.append("const ");
+        /* no break */
     default:
         {
-            if (isConst)
-                out.append("const ");
             Owned<ITypeInfo> argType = LINK(paramType);
             if (argType->getTypeCode() == type_function)
                 argType.setown(makePointerType(LINK(argType)));

ecl/regress/cppbody7.ecl

@@ -30,7 +30,7 @@ ENDC++;
 
 integer4 mkRandom3 :=
 BEGINC++
-#option action
+    #option action
 return rtlRandom();
 ENDC++;
 

ecl/regress/fetchinfunc.ecl

@@ -0,0 +1,55 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+import sq;
+sq.declareCommon();
+
+//UseStandardFiles
+
+//Daft test of fetch retrieving a dataset
+myPeople := sqSimplePersonBookDs(surname <> '');
+
+recfp := {unsigned8 rfpos, sqSimplePersonBookDs};
+
+recfp makeRec3(sqSimplePersonBookDs L, myPeople R) := TRANSFORM
+    self.rfpos := R.filepos;
+    self.books := L.books+R.books;
+    self := L;
+END;
+
+// temporary hack to get around codegen optimizing platform(),once call into global (and therefore hthor) context.
+nononcelib :=
+    SERVICE
+varstring platform() : library='graph', include='eclhelper.hpp', ctxmethod, entrypoint='getPlatform';
+    END;
+
+recordof(sqSimplePersonBookDs) removeFp(recfp l) := TRANSFORM
+    SELF := l;
+END;
+sortIt(dataset(recfp) ds) := FUNCTION
+    RETURN IF(nononcelib.platform() = 'thor', PROJECT(SORT(ds, rfpos),removeFp(LEFT)), PROJECT(ds,removeFp(LEFT)));
+END;
+
+fetchit (dataset(sqSimplePersonBookExRec) ds, string sname) := FUNCTION
+   return fetch(ds, ds(surname=sname), right.filepos, makeRec3(left,right));
+END;
+
+fetched := fetchit( sqSimplePersonBookDs, 'Halliday' );
+
+output(sortIt(fetched));

ecl/regress/fetchinfunc2.ecl

@@ -0,0 +1,49 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('targetClusterType', 'roxie');
+
+import sq;
+sq.declareCommon();
+
+//UseStandardFiles
+
+//Daft test of fetch retrieving a dataset
+myPeople := sqSimplePersonBookDs(surname <> '');
+
+recfp := {unsigned8 rfpos, sqSimplePersonBookDs};
+
+recfp makeRec3(sqSimplePersonBookDs L, myPeople R) := TRANSFORM
+    self.rfpos := R.filepos;
+    self.books := L.books+R.books;
+    self := L;
+END;
+
+// temporary hack to get around codegen optimizing platform(),once call into global (and therefore hthor) context.
+nononcelib :=
+    SERVICE
+varstring platform() : library='graph', include='eclhelper.hpp', ctxmethod, entrypoint='getPlatform';
+    END;
+
+fetchit (dataset(sqSimplePersonBookExRec) ds, string sname) := FUNCTION
+   return fetch(ds, ds(surname=sname), right.filepos, makeRec3(left,right));
+END;
+
+ds2 := nofold(dataset(100, transform(recordof(sqSimplePersonBookDs), SELF := [])));
+fetched := fetchit( ds2, 'Halliday' );
+
+output(fetched);

ecl/regress/issue15319.ecl

@@ -0,0 +1,52 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+integer4 mkRandom1() :=
+BEGINC++
+return rtlRandom();
+ENDC++;
+
+integer4 mkRandom2() :=
+BEGINC++
+    #option action
+return rtlRandom();
+ENDC++;
+
+integer4 mkRandom3() :=
+BEGINC++
+    return 1
+    //comment
+*rtlRandom();
+    #option action
+/* ignore me
+*/
+ENDC++;
+
+integer4 mkRandom4() :=
+BEGINC++
+    return 1
+    /*comment
+*/*rtlRandom();
+    #option action
+/* ignore me
+*/
+ENDC++;
+
+output(mkRandom1() * mkRandom1());
+output(mkRandom2() * mkRandom2());
+output(mkRandom3() * mkRandom3());
+output(mkRandom4() * mkRandom4());

esp/services/ws_fs/ws_fsService.cpp

@@ -1921,6 +1921,8 @@ bool CFileSprayEx::onSprayFixed(IEspContext &context, IEspSprayFixed &req, IEspS
         destination->getClusterPartDiskMapSpec(gName.str(), mspec);
         mspec.setDefaultBaseDir(defaultFolder.str());
         mspec.setDefaultReplicateDir(defaultReplicateFolder.str());
+        if (!req.getReplicate())
+            mspec.defaultCopies = DFD_NoCopies;
         destination->setClusterPartDiskMapSpec(gName.str(), mspec);
 
         int repo = req.getReplicateOffset();
@@ -2101,6 +2103,8 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
         destination->getClusterPartDiskMapSpec(gName.str(), mspec);
         mspec.setDefaultBaseDir(defaultFolder.str());
         mspec.setDefaultReplicateDir(defaultReplicateFolder.str());
+        if (!req.getReplicate())
+            mspec.defaultCopies = DFD_NoCopies;
         destination->setClusterPartDiskMapSpec(gName.str(), mspec);
         const char * encryptkey = req.getEncrypt();
         if(req.getCompress()||(encryptkey&&*encryptkey))
@@ -2530,6 +2534,8 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
             wuFSpecDest->getClusterPartDiskMapSpec(destNodeGroup.str(), mspec);
             mspec.setDefaultBaseDir(defaultFolder.str());
             mspec.setDefaultReplicateDir(defaultReplicateFolder.str());
+            if (!req.getReplicate())
+                mspec.defaultCopies = DFD_NoCopies;
             wuFSpecDest->setClusterPartDiskMapSpec(destNodeGroup.str(), mspec);
         }
 

esp/services/ws_machine/ws_machineService.cpp

@@ -1187,7 +1187,18 @@ int Cws_machineEx::runCommand(IEspContext& context, const char* sAddress, const
 
         IFRunSSH * connection = createFRunSSH();
         connection->init(command.str(),NULL,NULL,NULL,m_SSHConnectTimeoutSeconds,0);
+        // executed as single connection
         connection->exec(sAddress,NULL,true);
+        response.append(connection->getReplyText()[0]);
+        exitCode = connection->getReply()[0];
+        int len = response.length();
+        if (len > 0 && response.charAt(--len) == '\n') // strip newline
+          response.setLength(len);
+        if (response.length() && !exitCode)
+          response.insert(0, "Response: ");
+        else if (!exitCode)
+          response.insert(0, "No response recieved.\n");
+
     }
     // CFRunSSH uses a MakeStringExceptionDirect throw to pass code and result string
     catch(IException* e)

esp/src/eclwatch/ActivityWidget.js

@@ -61,6 +61,11 @@ define([
                         context._refreshActionState();
                     });
                 }
+                if (item.ServerType === "ECLserver" || "ECLCCserver") {
+                    setTimeout(function() {
+                        context.refreshGrid();
+                    }, 100);
+                }
             }, this);
         },
 
@@ -72,6 +77,11 @@ define([
                         context._refreshActionState();
                     });
                 }
+                if (item.ServerType === "ECLserver" || "ECLCCserver") {
+                    setTimeout(function() {
+                        context.refreshGrid();
+                    }, 100);
+                }
             }, this);
         },
 

esp/src/eclwatch/ESPGraph.js

@@ -605,6 +605,9 @@ define([
                                 } else if (name.indexOf("Size") === 0) {
                                     retVal["_" + name] = value;
                                     retVal[name] = "" + Utility.espSize2Bytes(value);
+                                } else if (name.indexOf("Skew") === 0) {
+                                    retVal["_" + name] = value;
+                                    retVal[name] = "" + Utility.espSkew2Number(value);
                                 } else {
                                     retVal[name] = value;
                                 }

esp/src/eclwatch/ESPLogicalFile.js

@@ -87,7 +87,9 @@ define([
             lang.mixin(item, {
                 __hpcc_id: createID(item.NodeGroup, item.Name),
                 __hpcc_isDir: false,
-                __hpcc_displayName: item.Name
+                __hpcc_displayName: item.Name,
+                StateID: 0,
+                State: ""
             });
         },
         mayHaveChildren: function (object) {
@@ -219,6 +221,10 @@ define([
                             }, DFUPart));
                         }, this);
                     }
+                    if (idx === 0) {
+                        this.set("CanReplicateFlag" , DFUFilePartsOnCluster.CanReplicate);
+                        this.set("ReplicateFlag", DFUFilePartsOnCluster.Replicate);
+                    }
                 }, this);
             }
             this.set("DFUFileParts", DFUFileParts);
@@ -257,7 +263,13 @@ define([
         doDelete: function (params) {
             var context = this;
             WsDfu.DFUArrayAction([this], "Delete").then(function (response) {
-                context.refresh();
+                if (lang.exists("DFUArrayActionResponse.ActionResults.DFUActionInfo", response) && 
+                        response.DFUArrayActionResponse.ActionResults.DFUActionInfo.length &&
+                        !response.DFUArrayActionResponse.ActionResults.DFUActionInfo[0].Failed) {
+                    context.updateData({ StateID: 999, State: "deleted" })
+                } else {
+                    context.refresh();
+                }
             });
         },
         despray: function (params) {

esp/src/eclwatch/ESPQueue.js

@@ -50,9 +50,11 @@ define([
             var context = this;
             return WsSMC.PauseQueue({
                 request: {
+                    ClusterType: this.ServerType,
                     QueueName: this.QueueName,
                     Cluster: this.ClusterName,
-                    ServerType: this.ServerType
+                    ServerType: this.ServerType,
+                    NetworkAddress: this.NetworkAddress
                 }
             }).then(function (response) {
                 context.refresh();
@@ -63,9 +65,11 @@ define([
             var context = this;
             return WsSMC.ResumeQueue({
                 request: {
+                    ClusterType: this.ServerType,
                     QueueName: this.QueueName,
                     Cluster: this.ClusterName,
-                    ServerType: this.ServerType
+                    ServerType: this.ServerType,
+                    NetworkAddress: this.NetworkAddress
                 }
             }).then(function (response) {
                 context.refresh();
@@ -207,7 +211,8 @@ define([
             return WsSMC.GetStatusServerInfo({
                 request: {
                     ServerName: this.ClusterName,
-                    ServerType: this.ServerType
+                    ServerType: this.ServerType,
+                    NetworkAddress: this.NetworkAddress
                 }
             }).then(function (response) {
                 if (lang.exists("GetStatusServerInfoResponse.StatusServerInfo.TargetClusterInfo", response)) {
@@ -282,7 +287,8 @@ define([
             return WsSMC.GetStatusServerInfo({
                 request: {
                     ServerName: this.ServerName,
-                    ServerType: this.ServerType
+                    ServerType: this.ServerType,
+                    NetworkAddress: this.NetworkAddress
                 }
             }).then(function (response) {
                 if (lang.exists("GetStatusServerInfoResponse.StatusServerInfo.ServerInfo", response)) {

esp/src/eclwatch/ESPResult.js

@@ -335,7 +335,10 @@ define([
                             column = {
                                 label: name,
                                 field: prefix + name,
-                                width: this.extractWidth(type, name) * 9
+                                width: this.extractWidth(type, name) * 9,
+                                formatter: function (cell, row) {
+                                    return cell.replace("\t", "    ");
+                                }
                             };
                         }
                     } else if (children) {

esp/src/eclwatch/FileSpray.js

@@ -196,7 +196,7 @@ define([
             6: "finished",
             7: "monitoring",
             8: "aborting",
-            999: "deleted"
+            999: "not found"
         },
 
         isComplete: function (state) {
@@ -317,6 +317,9 @@ define([
         Despray: function (params) {
             return ESPRequest.send("FileSpray", "Despray", params);
         },
+        Replicate: function (params) {
+          return ESPRequest.send("FileSpray", "Replicate", params);  
+        },
         Copy: function (params) {
             return ESPRequest.send("FileSpray", "Copy", params);
         },
@@ -333,7 +336,7 @@ define([
                                     result: {
                                         Wuid: params.request.Wuid,
                                         State: 999,
-                                        StateMessage: "deleted"
+                                        StateMessage: "not found"
                                     }
                                 }
                             });

esp/src/eclwatch/GraphTreeWidget.js

@@ -664,6 +664,7 @@ define([
             this.widget.ActivityMetric.set("options", arrayUtil.map(arrayUtil.filter(columns, function (col, idx) {
                 return col.label.indexOf("Time") === 0 ||
                         col.label.indexOf("Size") === 0 ||
+                        col.label.indexOf("Skew") === 0 ||
                         col.label.indexOf("Num") === 0;
             }), function (col, idx) {
                 return {
@@ -671,6 +672,13 @@ define([
                     value: col.label,
                     selected: col.label === "TimeMaxLocalExecute"
                 };
+            }).sort(function (l, r) {
+                if (l.label < r.label) {
+                    return -1;
+                } else if (l.label > r.label) {
+                    return 1;
+                }
+                return 0;
             }));
             this.widget.ActivitiesTreeMap.setActivities(vertices, true);
             this.widget.ActivityMetric.set("value", "TimeMaxLocalExecute");

esp/src/eclwatch/GraphWidget.js

@@ -134,7 +134,7 @@ define([
             }, this);
             if (formatTime) {
                 arrayUtil.forEach(target, function (column, idx) {
-                    if (column.label.indexOf("Time") === 0 || column.label.indexOf("Size")) {
+                    if (column.label.indexOf("Time") === 0 || column.label.indexOf("Size") || column.label.indexOf("Skew")) {
                         column.formatter = function (_id, row) {
                             return row["_" + column.field] || "";
                         }

esp/src/eclwatch/HPCCPlatformWidget.js

@@ -301,6 +301,18 @@ define([
             this._openNewTab("http://hpccsystems.com/download");
         },
 
+        _onOpenJira: function (evt) {
+            this._openNewTab("https://track.hpccsystems.com/issues");
+        },
+
+        _onOpenForums: function (evt) {
+            this._openNewTab("https://hpccsystems.com/bb/");
+        },
+
+        _onOpenRedBook: function (evt) {
+            this._openNewTab("https://wiki.hpccsystems.com/x/fYAb");
+        },
+
         _onOpenReleaseNotes: function (evt) {
             this._openNewTab("http://hpccsystems.com/download/free-community-edition-known-limitations#" + this.build.version);
         },

esp/src/eclwatch/LFDetailsWidget.js

@@ -46,6 +46,7 @@ define([
     "hpcc/ESPLogicalFile",
     "hpcc/ESPDFUWorkunit",
     "hpcc/FileBelongsToWidget",
+    "hpcc/FileSpray",
 
     "dojo/text!../templates/LFDetailsWidget.html",
 
@@ -59,7 +60,7 @@ define([
 
 ], function (exports, declare, lang, i18n, nlsHPCC, arrayUtil, dom, domAttr, domClass, domForm, query,
                 BorderContainer, TabContainer, ContentPane, Toolbar, TooltipDialog, Form, SimpleTextarea, TextBox, Button, DropDownButton, TitlePane, registry,
-                _TabContainerWidget, DelayLoadWidget, TargetSelectWidget, TargetComboBoxWidget, ESPLogicalFile, ESPDFUWorkunit, FileBelongsToWidget,
+                _TabContainerWidget, DelayLoadWidget, TargetSelectWidget, TargetComboBoxWidget, ESPLogicalFile, ESPDFUWorkunit, FileBelongsToWidget, FileSpray,
                 template) {
     exports.fixCircularDependency = declare("LFDetailsWidget", [_TabContainerWidget], {
         templateString: template,
@@ -71,6 +72,7 @@ define([
         copyForm: null,
         renameForm: null,
         desprayForm: null,
+        replicateForm: null,
         summaryWidget: null,
         contentWidget: null,
         sourceWidget: null,
@@ -90,6 +92,7 @@ define([
             this.copyForm = registry.byId(this.id + "CopyForm");
             this.renameForm = registry.byId(this.id + "RenameForm");
             this.desprayForm = registry.byId(this.id + "DesprayForm");
+            this.replicateForm = registry.byId(this.id + "ReplicateForm");
             this.summaryWidget = registry.byId(this.id + "_Summary");
             this.contentWidget = registry.byId(this.id + "_Content");
             this.sourceWidget = registry.byId(this.id + "_Source");
@@ -102,6 +105,9 @@ define([
             this.copyTargetSelect = registry.byId(this.id + "CopyTargetSelect");
             this.desprayTargetSelect = registry.byId(this.id + "DesprayTargetSelect");
             this.desprayTooltiopDialog = registry.byId(this.id + "DesprayTooltipDialog");
+            this.replicateTargetSelect = registry.byId(this.id + "ReplicateCluster");
+            this.replicateSourceLogicalFile = registry.byId(this.id + "ReplicateSourceLogicalFile");
+            this.replicateDropDown = registry.byId(this.id + "ReplicateDropDown");
             var context = this;
             var origOnOpen = this.desprayTooltiopDialog.onOpen;
             this.desprayTooltiopDialog.onOpen = function () {
@@ -197,6 +203,19 @@ define([
             }
         },
 
+        _onReplicateOk: function (event) {
+            if (this.replicateForm.validate()) {
+                var context = this;
+                var request = domForm.toObject(this.id + "ReplicateForm");
+                FileSpray.Replicate({
+                    request: request
+                }).then(function (response) {
+                    context._handleResponse("ReplicateResponse.wuid", response);
+                });
+                registry.byId(this.id + "ReplicateDropDown").closeDropDown();
+            }
+        },
+
         //  Implementation  ---
         init: function (params) {
             if (this.inherited(arguments))
@@ -212,6 +231,7 @@ define([
                 this.logicalFile.watch(function (name, oldValue, newValue) {
                     context.updateInput(name, oldValue, newValue);
                 });
+                this.replicateSourceLogicalFile.set("value", params.Name);
                 this.logicalFile.refresh();
             }
             this.copyTargetSelect.init({
@@ -220,6 +240,9 @@ define([
             this.desprayTargetPath.init({
                 DropZoneFolders: true
             });
+            this.replicateTargetSelect.init({
+                Groups: true
+            });
             this.logicalFile.refresh();
         },
 
@@ -340,8 +363,8 @@ define([
                 //  Force Icon to Show (I suspect its not working due to Circular Reference Loading)
                 this.queriesWidget.set("iconClass", "dijitInline dijitIcon dijitTabButtonIcon iconFind");
             } else if (name === "DFUFilePartsOnClusters") {
-            	// Currently only checking first cluster may add loop through clusters and add a tab at a later date
-            	this.updateInput("DFUFilePartsOnClusters", oldValue, newValue.DFUFilePartsOnCluster[0].Replicate);
+                // Currently only checking first cluster may add loop through clusters and add a tab at a later date
+                this.updateInput("DFUFilePartsOnClusters", oldValue, newValue.DFUFilePartsOnCluster[0].Replicate);
             }
         },
 
@@ -378,6 +401,7 @@ define([
             this.setDisabled(this.id + "_Graphs", this.logicalFile.isDeleted() || !this.logicalFile.Graphs);
             this.setDisabled(this.id + "_Workunit", this.logicalFile.isDeleted());
             this.setDisabled(this.id + "_DFUWorkunit", this.logicalFile.isDeleted());
+            this.setDisabled(this.id + "ReplicateDropDown", !this.logicalFile.CanReplicateFlag || this.logicalFile.ReplicateFlag === false);
         }
     });
 });

esp/src/eclwatch/SFDetailsWidget.js

@@ -314,7 +314,7 @@ define([
                     id: id,
                     title: params.Name,
                     closable: true,
-                    delayWidget: "LFDetailsWidget",
+                    delayWidget: params.isSuperfile ? "SFDetailsWidget" : "LFDetailsWidget",
                     _hpccParams: {
                         NodeGroup: params.NodeGroup,
                         Name: params.Name

esp/src/eclwatch/ShowPermissionsWidget.js

@@ -21,19 +21,15 @@ define([
 
     "dijit/registry",
     "dijit/form/CheckBox",
+    "dijit/layout/ContentPane",
 
     "dgrid/editor",
 
     "hpcc/GridDetailsWidget",
     "hpcc/ws_access",
-    "hpcc/ESPUtil",
-
-    "dijit/layout/BorderContainer",
-    "dijit/layout/TabContainer",
-    "dijit/layout/ContentPane"
-
+    "hpcc/ESPUtil"
 ], function (declare, lang, i18n, nlsHPCC,
-                registry, CheckBox,
+                registry, CheckBox, ContentPane,
                 editor,
                 GridDetailsWidget, WsAccess, ESPUtil) {
     return declare("ShowPermissionsWidget", [GridDetailsWidget], {
@@ -229,14 +225,14 @@ define([
         },
 
         createInheritedPermissionsTab: function (something) {
-            var tab = new dijit.layout.ContentPane({
+            var tab = new ContentPane({
                 title:formCount,
                 id:''+formCount,
                 content:cont,
-                class:'tab',
+                'class':'tab',
                 closable: true,
                 onClose:function(){
-                    return confirm('Relly want to remove?');
+                    return confirm(this.i18n.ReallyWantToRemove);
                 }
             });
             dijit.byId('tabContainer').addChild(tab);

esp/src/eclwatch/TargetSelectClass.js

@@ -80,8 +80,9 @@ define([
             } else if (params.UserGroups === true) {
                 this.loadUserGroups();
             } else if (params.DropZoneFolders === true) {
-                this.defaultValue = ".";
-                this.set("value", ".");
+                this.defaultValue = "";
+                this.set("value", "");
+                this.set("placeholder", "/");
                 this.loadDropZoneFolders();
             } else if (params.WUState === true) {
                 this.loadWUState();
@@ -239,7 +240,7 @@ define([
         loadDropZoneFolders: function () {
             var context = this;
             this.getDropZoneFolder = function () {
-                var baseFolder = this._dropZoneTarget.machine.Directory + (this.endsWith(this._dropZoneTarget.machine.Directory, "/") ? "" : "/");
+                var baseFolder = this._dropZoneTarget.machine.Directory;
                 var selectedFolder = this.get("value");
                 return baseFolder + selectedFolder;
             }
@@ -250,7 +251,7 @@ define([
                         data: arrayUtil.map(results, function (_path) {
                             var path = _path.substring(context._dropZoneTarget.machine.Directory.length);
                             return {
-                                name: "." + path,
+                                name: path,
                                 id: _path
                             };
                         })

esp/src/eclwatch/Utility.js

@@ -137,6 +137,27 @@
         }, this);
     }
 
+    function espSkew2Number(skew) {
+        if (!skew) {
+            return 0;
+        }
+        return parseFloat(skew);
+    }
+
+    function espSkew2NumberTests() {
+        var tests = [
+            { str: "", expected: 0 },
+            { str: "1", expected: 1 },
+            { str: "10%", expected: 10 },
+            { str: "-10%", expected: -10 }
+        ];
+        tests.forEach(function (test, idx) {
+            if (espSkew2Number(test.str) !== test.expected) {
+                console.log("espSkew2NumberTests failed with " + test.str + "(" + espSkew2Number(test.str) + ") !== " + test.expected);
+            }
+        }, this);
+    }
+
     /* alphanum.js (C) Brian Huisman
      * Based on the Alphanum Algorithm by David Koelle
      * The Alphanum Algorithm is discussed at http://www.DaveKoelle.com
@@ -245,6 +266,7 @@
     return {
         espTime2Seconds: espTime2Seconds,
         espSize2Bytes: espSize2Bytes,
+        espSkew2Number: espSkew2Number,
         xmlEncode: xmlEncode,
         xmlEncode2: xmlEncode2,
         alphanumSort: alphanumSort

esp/src/eclwatch/WsDfu.js

@@ -181,12 +181,15 @@ define([
                                     FileDetail: {
                                         Name: params.request.Name,
                                         StateID: 999,
-                                        State: "deleted"
+                                        State: "not found"
                                     }
                                 }
                             });
                         }
                     });
+                } else if (lang.exists("DFUInfoResponse.FileDetail", response)) {
+                    response.DFUInfoResponse.FileDetail.StateID = 0;
+                    response.DFUInfoResponse.FileDetail.State = "";
                 }
                 return response;
             });

esp/src/eclwatch/WsWorkunits.js

@@ -52,7 +52,7 @@ define([
             12: "debugging",
             13: "debug_running",
             14: "paused",
-            999: "deleted"
+            999: "not found"
         },
 
         WUCreate: function (params) {
@@ -172,7 +172,7 @@ define([
                                         ECLWorkunit: [{
                                             Wuid: params.request.Wuid,
                                             StateID: 999,
-                                            State: "deleted"
+                                            State: "not found"
                                         }]
                                     }
                                 }
@@ -194,7 +194,7 @@ define([
                                     Workunit: {
                                         Wuid: params.request.Wuid,
                                         StateID: 999,
-                                        State: "deleted"
+                                        State: "not found"
                                     }
                                 }
                             });
@@ -250,7 +250,7 @@ define([
                                 var wu = wuMap[item.Wuid];
                                 if (actionType === "delete" && item.Result === "Success") {
                                     wu.set("StateID", 999);
-                                    wu.set("State", "deleted");
+                                    wu.set("State", "not found");
                                 } else if (wu.refresh) {
                                     wu.refresh();
                                 }

esp/src/eclwatch/nls/hpcc.js

@@ -123,6 +123,7 @@ define({root:
     DOTAttributes: "DOT Attributes",
     Down: "Down",
     Download: "Download",
+    Downloads: "Downloads",
     DropZone: "Drop Zone",
     Duration: "Duration",
     EBCDIC: "EBCDIC",
@@ -184,6 +185,7 @@ define({root:
     Fixed: "Fixed",
     Folder: "Folder",
     Format: "Format",
+    Forums: "Forums",
     Forward: "Forward",
     FromDate: "From Date",
     FromSizes: "From Sizes",
@@ -221,6 +223,7 @@ define({root:
     IPAddress: "IP Address",
     IsLibrary: "Is Library",
     IsReplicated: "Is Replicated",
+    IssueReporting: "Issue Reporting",
     Jobname: "Jobname",
     JobName: "Job Name",
     jsmi: "jsmi*",
@@ -399,12 +402,14 @@ define({root:
     Quote: "Quote",
     QuotedTerminator: "Quoted Terminator",
     RawTextPage: "Raw Text (Current Page)",
+    ReallyWantToRemove: "Really want to remove?",
     RecordCount: "Record Count",
     RecordLength: "Record Length",
     Records: "Records",
     RecordSize: "Record Size",
     RecordStructurePresent: "Record Structure Present",
     Recover: "Recover",
+    RedBook: "Red Book",
     Refresh: "Refresh",
     ReleaseNotes: "Release Notes",
     Reload: "Reload",
@@ -419,6 +424,7 @@ define({root:
     RenderedSVG: "Rendered SVG",
     RenderSVG: "Render SVG",
     Replicate: "Replicate",
+    ReplicateOffset: "Replicate Offset",
     RepresentsASubset: "represent a subset of the total number of matches. Using a correct filter may reduce the number of matches.",
     RequestSchema: "Request Schema",
     RequiredForXML: "Required for spraying XML",
@@ -473,6 +479,7 @@ define({root:
     somefile: "*::somefile*",
     Source: "Source",
     SourceCode: "Source Code",
+    SourceLogicalFile: "Source Logical Name",
     SourcePath: "Source Path (Wildcard Enabled)",
     SourceProcess: "Source Process",
     Spill: "Spill",

esp/src/eclwatch/templates/DFUQueryWidget.html

@@ -130,7 +130,7 @@
                                     <div data-dojo-type="hpcc.TableContainer">
                                         <input id="${id}DesprayTargetSelect" title="${i18n.DropZone}:" name="destGroup" style="width: 100%;" data-dojo-type="TargetSelectWidget" />
                                         <input id="${id}DesprayTargetIPAddress" title="${i18n.IPAddress}:" name="destIP" style="width: 100%;" required="true" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
-                                        <input id="${id}DesprayTargetPath" title="${i18n.Path}:" name="destPath" required="false" style="width: 100%;" data-dojo-props="trim: true, readonly: true" data-dojo-type="TargetComboBoxWidget" />
+                                        <input id="${id}DesprayTargetPath" title="${i18n.Path}:" name="destPath" required="false" style="width: 100%;" data-dojo-props="trim: true, readonly: false" data-dojo-type="TargetComboBoxWidget" />
                                         <input id="${id}DesprayTargetSplitPrefix" title="${i18n.SplitPrefix}:" name="splitprefix" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                     </div>
                                     <div id="${id}DesprayGrid" data-dojo-type="SelectionGridWidget">

esp/src/eclwatch/templates/HPCCPlatformWidget.html

@@ -26,13 +26,21 @@
                         <span data-dojo-type="dijit.MenuSeparator"></span>
                         <div data-dojo-attach-event="onClick:_onOpenReleaseNotes" data-dojo-type="dijit.MenuItem">${i18n.ReleaseNotes}</div>
                         <div data-dojo-attach-event="onClick:_onOpenTransitionGuide" data-dojo-type="dijit.MenuItem">${i18n.TransitionGuide}</div>
-                        <div data-dojo-attach-event="onClick:_onOpenResources" data-dojo-type="dijit.MenuItem">${i18n.AdditionalResources}</div>
+                        <div data-dojo-type="dijit.PopupMenuItem">
+                            <span>${i18n.AdditionalResources}</span>
+                            <div data-dojo-type="dijit.Menu">
+                                <div data-dojo-attach-event="onClick:_onOpenResources" data-dojo-type="dijit.MenuItem">${i18n.Downloads}</div>
+                                <div data-dojo-attach-event="onClick:_onOpenJira" data-dojo-type="dijit.MenuItem">${i18n.IssueReporting}</div>
+                                <div data-dojo-attach-event="onClick:_onOpenForums" data-dojo-type="dijit.MenuItem">${i18n.Forums}</div>
+                                <div data-dojo-attach-event="onClick:_onOpenRedBook" data-dojo-type="dijit.MenuItem">${i18n.RedBook}</div>
+                            </div>
+                        </div>
                         <span data-dojo-type="dijit.MenuSeparator"></span>
                         <div id="${id}Configuration" data-dojo-attach-event="onClick:_onOpenConfiguration" data-dojo-type="dijit.MenuItem">${i18n.Configuration}</div>
                         <div id="${id}About" data-dojo-attach-event="onClick:_onAbout" data-dojo-type="dijit.MenuItem">${i18n.About}</div>
                         <div data-dojo-props="hidden:true" data-dojo-type="dijit.PopupMenuItem">
                             <span>${i18n.Debug}</span>
-                            <div data-dojo-type="dijit.DropDownMenu">
+                            <div data-dojo-type="dijit.Menu">
                                 <div data-dojo-attach-event="onClick:_ondebugLanguageFiles" data-dojo-type="dijit.MenuItem">${i18n.LanguageFiles}</div>
                             </div>
                         </div>

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -68,7 +68,7 @@
                                     <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                                         <input id="${id}DesprayTargetSelect" title="${i18n.DropZone}:" name="destGroup" colspan="2" style="width: 100%;" data-dojo-type="TargetSelectWidget" />
                                         <input id="${id}DesprayTargetIPAddress" title="${i18n.IPAddress}:" name="destIP" colspan="2" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
-                                        <input id="${id}DesprayTargetPath" title="${i18n.Path}:" name="destPath" colspan="2" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="TargetComboBoxWidget" />
+                                        <input id="${id}DesprayTargetPath" title="${i18n.Path}:" name="destPath" colspan="2" required="false" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="TargetComboBoxWidget" />
                                         <input id="${id}DesprayTargetName" title="${i18n.TargetName}:" colspan="2" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input id="${id}DesprayTargetSplitPrefix" title="${i18n.SplitPrefix}:" name="splitprefix" colspan="2" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.TextBox" />
                                     </div>
@@ -87,6 +87,25 @@
                         </div>
                     </div>
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
+                    <div id="${id}ReplicateDropDown" data-dojo-type="dijit.form.DropDownButton">
+                        <span>${i18n.Replicate}</span>
+                        <div id="${id}ReplicateTooltipDialog" data-dojo-type="dijit.TooltipDialog">
+                            <div id="${id}ReplicateForm" style="width: 460px;" onsubmit="return false;" data-dojo-type="dijit.form.Form">
+                                <div data-dojo-type="dijit.Fieldset">
+                                    <legend>${i18n.Replicate}</legend>
+                                    <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
+                                        <input id="${id}ReplicateSourceLogicalFile" title="${i18n.SourceLogicalFile}:" name="sourceLogicalName" colspan="2" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
+                                        <input id="${id}ReplicateOffset" title="${i18n.ReplicateOffset}:" name="replicateOffset" colspan="2" required="false" value="1" style="width: 100%;" data-dojo-type="dijit.form.NumberTextBox" />
+                                        <input id="${id}ReplicateCluster" title="${i18n.Cluster}:" name="cluster" colspan="2" style="width: 100%;" data-dojo-type="TargetSelectWidget" />
+                                    </div>
+                                </div>
+                                <div class="dijitDialogPaneActionBar">
+                                    <button type="submit" data-dojo-attach-event="onClick:_onReplicateOk" data-dojo-type="dijit.form.Button">${i18n.Replicate}</button>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                    <span data-dojo-type="dijit.ToolbarSeparator"></span>
                     <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
                 </div>
                 <div data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
@@ -185,4 +204,4 @@
             </div>
         </div>
     </div>
-</div>
+</div>

initfiles/componentfiles/configxml/dali.xsd

@@ -303,6 +303,27 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="backupLargeWarningThreshold" type="xs:nonNegativeInteger" use="optional" default="50">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Granularity of pending backups to begin issue warnings about backlog</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="backupSoftQueueLimit" type="xs:nonNegativeInteger" use="optional" default="200">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Limit above which the backup queue will start introducing delays until the backup queue decreases below this limit</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="backupSoftQueueLimitDelay" type="xs:nonNegativeInteger" use="optional" default="200">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>The maximum delay to introduce when the backupSoftQueueLimit has been introduced</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
   </xs:attributeGroup>
    <xs:attributeGroup name="DFS">
     <!--DOC-Autobuild-code-->

initfiles/componentfiles/configxml/dali.xsl

@@ -123,7 +123,7 @@
             <xsl:with-param name="val" select="@recoverFromIncErrors"/>
           </xsl:call-template>
         </xsl:attribute>
-        <xsl:copy-of select="@snmpErrorMsgLevel | @msgLevel | @lightweightCoalesce | @keepStores"/>
+        <xsl:copy-of select="@snmpErrorMsgLevel | @msgLevel | @lightweightCoalesce | @keepStores | @backupLargeWarningThreshold | @backupSoftQueueLimit | @backupSoftQueueLimitDelay"/>
         <xsl:attribute name="lCIdlePeriod">
           <xsl:value-of select="@IdlePeriod"/>
         </xsl:attribute>

initfiles/componentfiles/configxml/esp_service_wsecl.xsd

@@ -25,6 +25,12 @@
             <xs:attributeGroup ref="EclServer"/>
             <xs:attributeGroup ref="Roxie"/>
             <xs:attributeGroup ref="Logging"/>
+              <!--DocAutoBuildCode-->
+                <xs:annotation>
+                 <xs:appinfo>
+                   <docid>MyWS_TbL01</docid>
+                 </xs:appinfo>
+                </xs:annotation> 
             <xs:attribute name="build" type="buildType" use="required">
                 <xs:annotation>
                     <xs:appinfo>
@@ -67,9 +73,11 @@
     </xs:element>
     <xs:attributeGroup name="EclServer">
         <xs:annotation>
-            <xs:appinfo>
-                <title>ECL Server</title>
-            </xs:appinfo>
+           <xs:appinfo>
+              <title>ECL Server</title>
+              <!--DocAutoBuildCode-->
+              <docid>MyWS_TbL02</docid>
+	   </xs:appinfo>
         </xs:annotation>
         <xs:attribute name="eclServer" type="eclServerType" use="optional">
             <xs:annotation>
@@ -137,6 +145,12 @@
         </xs:attribute>
     </xs:attributeGroup>
     <xs:attributeGroup name="Roxie">
+            <!--DocAutoBuildCode-->
+            <xs:annotation>
+	      <xs:appinfo>
+	        <docid>MyWS_TbL04</docid>
+	      </xs:appinfo>
+            </xs:annotation> 
         <xs:attribute name="roxieAddress" type="roxieAddress" use="optional">
             <xs:annotation>
                 <xs:appinfo>
@@ -158,6 +172,12 @@
         </xs:attribute>
     </xs:attributeGroup>
     <xs:attributeGroup name="Logging">
+            <!--DocAutoBuildCode-->
+            <xs:annotation>
+               <xs:appinfo>
+                   <docid>MyWS_TbL05</docid>
+               </xs:appinfo>
+            </xs:annotation> 
         <xs:attribute name="loggingServer" type="xs:string" use="optional">
             <xs:annotation>
                 <xs:appinfo>

initfiles/componentfiles/configxml/esp_service_wsecl2.xsd

@@ -30,7 +30,13 @@
                   </xs:appinfo>
                 </xs:annotation>
                 <xs:complexType>
-                  <xs:attribute name="name" type="xs:string" use="required" default="">
+                <!--DocAutoBuildCode-->
+                    <xs:annotation>
+		       <xs:appinfo>
+		          <docid>MyWS2-T01</docid>
+		       </xs:appinfo>
+		    </xs:annotation>
+           <xs:attribute name="name" type="xs:string" use="required" default="">
                     <xs:annotation>
                       <xs:appinfo>
                         <viewType>hidden</viewType>
@@ -76,6 +82,12 @@
                   </xs:appinfo>
                 </xs:annotation>
                 <xs:complexType>
+                <!--DocAutoBuildCode-->
+                 <xs:annotation>
+	           <xs:appinfo>
+                    <docid>MyWS2-T02</docid>
+	           </xs:appinfo>
+                 </xs:annotation>
                   <xs:attribute name="name" type="topologyClusterType" use="required">
 	                <xs:annotation>
 	                  <xs:appinfo>
@@ -86,6 +98,12 @@
                 </xs:complexType>
               </xs:element>
             </xs:sequence>
+            <!--DocAutoBuildCode-->
+                <xs:annotation>
+	          <xs:appinfo>
+                    <docid>MyWS2-T03</docid>
+	          </xs:appinfo>
+	        </xs:annotation>
             <xs:attribute name="build" type="buildType" use="required">
                 <xs:annotation>
                     <xs:appinfo>

initfiles/componentfiles/configxml/espsmcservice.xsd.in

@@ -48,6 +48,12 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
+                <xs:annotation>
+	          <xs:appinfo>
+	          <!--DOCS_Autobuild_Code-->
+          	    <docid>SMC-T01</docid>
+	          </xs:appinfo>
+              </xs:annotation>
             <xs:attribute name="description" type="xs:string" use="optional" default="ESP services for SMC">
                 <xs:annotation>
                     <xs:appinfo>
@@ -150,6 +156,12 @@
         </xs:complexType>
     </xs:element>
     <xs:attributeGroup name="Monitoring">
+         <xs:annotation>
+	    <xs:appinfo>
+            <!--DOCS_Autobuild_Code-->
+              <docid>SMC-T02</docid>
+            </xs:appinfo>
+         </xs:annotation> s	
         <xs:attribute name="monitorDaliFileServer" type="xs:boolean" use="optional" default="false">
             <xs:annotation>
                 <xs:appinfo>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -761,7 +761,7 @@
   </EclSchedulerProcess>
  </Software>
  <EnvSettings>
-  <pid>${PID_DIR}</pid>
+  <pid>${PID_PATH}</pid>
   <sourcedir>${CONFIG_SOURCE_PATH}</sourcedir>
   <mpTraceLevel>0</mpTraceLevel>
   <mpStart>7101</mpStart>

plugins/cassandra/CMakeLists.txt

@@ -132,6 +132,7 @@ if(USE_CASSANDRA)
             install(
                 FILES ${PROJECT_BINARY_DIR}/libuv.so.0.10
                 DESTINATION ${LIB_DIR}
+                PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
                 COMPONENT Runtime)
         endif()
 

plugins/redis/CMakeLists.txt

@@ -25,7 +25,7 @@
 project(redis)
 
 if(REDIS)
-    ADD_PLUGIN(redis PACKAGES HIREDIS)
+    ADD_PLUGIN(redis PACKAGES HIREDIS MINVERSION 0.13.0) #NOTE: If MINVERSION is altered then min versions in redis.hpp must also be altered.
     if(MAKE_REDIS)
         set(
             SRCS

plugins/redis/redis.cpp

@@ -43,8 +43,10 @@ class Connection;
 static const char * REDIS_LOCK_PREFIX = "redis_ecl_lock";
 static __thread Connection * cachedConnection = NULL;
 static __thread Connection * cachedPubConnection = NULL;//database should always = 0
+#if HIREDIS_VERSION_OK
 static __thread ThreadTermFunc threadHookChain = NULL;
 static __thread bool threadHooked = false;
+#endif
 
 static void * allocateAndCopy(const void * src, size_t size)
 {
@@ -146,7 +148,6 @@ protected :
     void parseOptions(ICodeContext * ctx, const char * _options);
     void connect(ICodeContext * ctx, int _database, const char * password);
     void selectDB(ICodeContext * ctx, int _database);
-    void reset(ICodeContext * ctx, const char * password, unsigned _timeout);
     void readReply(Reply * reply);
     void readReplyAndAssert(Reply * reply, const char * msg);
     void readReplyAndAssertWithCmdMsg(Reply * reply, const char * msg, const char * key = NULL);
@@ -158,9 +159,11 @@ protected :
     void assertConnectionWithCmdMsg(const char * cmd, const char * key = NULL);
     void fail(const char * cmd, const char * errmsg, const char * key = NULL);
     void * redisCommand(redisContext * context, const char * format, ...);
+#if HIREDIS_VERSION_OK
     static unsigned hashServerIpPortPassword(ICodeContext * ctx, const char * _options, const char * password);
     bool isSameConnection(ICodeContext * ctx, const char * _options, const char * password) const;
-
+    void reset(ICodeContext * ctx, const char * password, unsigned _timeout);
+#endif
     //-------------------------------LOCKING------------------------------------------------
     void handleLockOnSet(ICodeContext * ctx, const char * key, const char * value, size_t size, unsigned expire);
     void handleLockOnGet(ICodeContext * ctx, const char * key, MemoryAttr * retVal, const char * password, unsigned expire);
@@ -179,6 +182,7 @@ protected :
     int database; //NOTE: redis stores the maximum number of dbs as an 'int'.
 };
 
+#if HIREDIS_VERSION_OK
 static void releaseContext()
 {
     if (cachedConnection)
@@ -206,15 +210,20 @@ public :
     MainThreadCachedConnection() { }
     ~MainThreadCachedConnection() { releaseContext(); }
 } mainThread;
+#endif
+
 Connection::Connection(ICodeContext * ctx, const char * _options, int _database, const char * password, unsigned _timeout)
-  : database(0), timeout(_timeout), port(0), serverIpPortPasswordHash(hashServerIpPortPassword(ctx, _options, password))
+  : context(nullptr), database(0), timeout(_timeout), port(0), serverIpPortPasswordHash(0)
 {
+#if HIREDIS_VERSION_OK
+    serverIpPortPasswordHash = hashServerIpPortPassword(ctx, _options, pass);
+#endif
     options.set(_options, strlen(_options));
     parseOptions(ctx, _options);
     connect(ctx, _database, password);
 }
 Connection::Connection(ICodeContext * ctx, const char * _options, const char * _ip, int _port, unsigned _serverIpPortPasswordHash, int _database, const char * password, unsigned _timeout)
-  : database(0), timeout(_timeout), serverIpPortPasswordHash(_serverIpPortPasswordHash), port(_port)
+  : context(nullptr), database(0), timeout(_timeout), serverIpPortPasswordHash(_serverIpPortPasswordHash), port(_port)
 {
     options.set(_options, strlen(_options));
     ip.set(_ip, strlen(_ip));
@@ -291,6 +300,7 @@ void Connection::redisSetTimeout()
         throwUnexpected();//In case there is a bug in hiredis such that the above err is not reflected in the 'context' (checked in assertConnection) as expected.
     }
 }
+#if HIREDIS_VERSION_OK
 bool Connection::isSameConnection(ICodeContext * ctx, const char * _options, const char * password) const
 {
     return (hashServerIpPortPassword(ctx, _options, password) == serverIpPortPasswordHash);
@@ -299,6 +309,18 @@ unsigned Connection::hashServerIpPortPassword(ICodeContext * ctx, const char * _
 {
     return hashc((const unsigned char*)_options, strlen(_options), hashc((const unsigned char*)password, strlen(password), 0));
 }
+void Connection::reset(ICodeContext * ctx, const char * password, unsigned _timeout)
+{
+    timeout.reset(_timeout);
+    if (context && context->err != REDIS_OK)
+    {
+        redisFree(context);
+        context = NULL;
+        database = 0;
+        connect(ctx, 0, password);
+    }
+}
+#endif
 void Connection::parseOptions(ICodeContext * ctx, const char * _options)
 {
     StringArray optionStrings;
@@ -334,17 +356,6 @@ void Connection::parseOptions(ICodeContext * ctx, const char * _options)
         }
     }
 }
-void Connection::reset(ICodeContext * ctx, const char * password, unsigned _timeout)
-{
-    timeout.reset(_timeout);
-    if (context && context->err != REDIS_OK)
-    {
-        redisFree(context);
-        context = NULL;
-        database = 0;
-        connect(ctx, 0, password);
-    }
-}
 void Connection::readReply(Reply * reply)
 {
     redisReply * nakedReply = NULL;
@@ -364,6 +375,7 @@ void Connection::readReplyAndAssertWithCmdMsg(Reply * reply, const char * msg, c
 }
 Connection * Connection::createConnection(ICodeContext * ctx,  Connection * & _cachedConnection, const char * options, int _database, const char * password, unsigned _timeout)
 {
+#if HIREDIS_VERSION_OK
     if (!_cachedConnection)
     {
         _cachedConnection = new Connection(ctx, options, _database, password, _timeout);
@@ -388,6 +400,8 @@ Connection * Connection::createConnection(ICodeContext * ctx,  Connection * & _c
     _cachedConnection = NULL;
     _cachedConnection = new Connection(ctx, options, _database, password, _timeout);
     return LINK(_cachedConnection);
+#endif
+    return new Connection(ctx, options, _database, password, _timeout);
 }
 void Connection::selectDB(ICodeContext * ctx, int _database)
 {

plugins/redis/redis.hpp

@@ -30,6 +30,12 @@
 #define ECL_REDIS_API
 #endif
 
+//If altered, the MINVERSION in CMakeLists.txt must also be altered
+#define MIN_HIREDIS_MAJOR 0
+#define MIN_HIREDIS_MINOR 13
+#define MIN_HIREDIS_PATCH 0
+#define HIREDIS_VERSION_OK HIREDIS_MAJOR >= MIN_HIREDIS_MAJOR  && HIREDIS_MINOR >= MIN_HIREDIS_MINOR && HIREDIS_PATCH >= MIN_HIREDIS_PATCH
+
 #include "hqlplugins.hpp"
 #include "eclhelper.hpp"
 

roxie/ccd/ccdprotocol.cpp

@@ -1652,8 +1652,19 @@ readAnother:
         }
 
         bool isHTTP = httpHelper.isHttp();
-        TextMarkupFormat mlResponseFmt = isHTTP ? httpHelper.queryResponseMlFormat() : MarkupFmt_XML;
-        TextMarkupFormat mlRequestFmt = isHTTP ? httpHelper.queryRequestMlFormat() : MarkupFmt_XML;
+
+        TextMarkupFormat mlResponseFmt = MarkupFmt_Unknown;
+        TextMarkupFormat mlRequestFmt = MarkupFmt_Unknown;
+        if (!isStatus)
+        {
+            if (!isHTTP)
+                mlResponseFmt = mlRequestFmt = MarkupFmt_XML;
+            else
+            {
+                mlResponseFmt = httpHelper.queryResponseMlFormat();
+                mlRequestFmt = httpHelper.queryRequestMlFormat();
+            }
+        }
 
         bool failed = false;
         bool isRequest = false;

system/jlib/jbuff.cpp

@@ -64,9 +64,16 @@
 #define     CHECKREADPOS(len)  
 #endif
 
-
 //-----------------------------------------------------------------------
 
+static inline size32_t checkMemoryBufferOverflow(size32_t curLen, size32_t inc)
+{
+    size_t newLen = (size_t)curLen + (size_t)inc;
+    if (newLen > MEMBUFFER_MAXLEN)
+        RaiseOutOfMemException(-10, curLen, inc, false, "Exceeded maximum size");
+    return (size32_t)newLen;
+}
+
 jlib_decl void *checked_realloc(void *orig, size_t newlen, size_t origlen,int errcode)
 {
     if (newlen==0) {
@@ -88,21 +95,26 @@ class jlib_thrown_decl COutOfMemException: public CInterface, implements IOutOfM
     size_t got;
     static int recursion;
     bool expected;
+    StringBuffer errorMsg;
 public:
     IMPLEMENT_IINTERFACE;
-    COutOfMemException(int _errcode,size_t _wanted,size_t _got,bool _expected)
+    COutOfMemException(int _errcode, size_t _wanted, size_t _got, bool _expected, const char *errMsg)
     {
         errcode = _errcode;
         wanted = _wanted;
         expected = _expected;
         got = _got;
+        if (nullptr == errMsg)
+            errorMsg.append("Out of Memory");
+        else
+            errorMsg.append(errMsg);
 //      DebugBreak();
 
         if ((recursion++==0)&&!expected) {
 // Bit risky if *very* out of memory so protect against recursion and catch exceptions
             try { 
                 // try to log
-                PROGLOG("Jbuff: Out of Memory (%d,%" I64F "d,%" I64F "dk)",_errcode,(unsigned __int64)wanted,(unsigned __int64) (got/1024));
+                PROGLOG("Jbuff: %s (%d,%" I64F "d,%" I64F "dk)",errorMsg.str(),_errcode,(unsigned __int64)wanted,(unsigned __int64) (got/1024));
                 PrintStackReport();
                 PrintMemoryReport();
             }
@@ -115,7 +127,7 @@ public:
     int             errorCode() const { return errcode; }
     StringBuffer &  errorMessage(StringBuffer &str) const
     { 
-        str.append("Jbuff: Out of Memory (").append((unsigned __int64)wanted);
+        str.append("Jbuff: ").append(errorMsg.str()).append(" (").append((unsigned __int64)wanted);
         if (got) 
             str.append(',').append((unsigned __int64)got);
         return str.append(")");
@@ -125,14 +137,14 @@ public:
 
 int COutOfMemException::recursion=0;
 
-IOutOfMemException *createOutOfMemException(int errcode,size_t wanted,size_t got,bool expected)
+IOutOfMemException *createOutOfMemException(int errcode, size_t wanted, size_t got, bool expected, const char *errMsg)
 {
-    return new COutOfMemException(errcode,wanted,got,expected);
+    return new COutOfMemException(errcode, wanted, got, expected, errMsg);
 }
 
-void RaiseOutOfMemException(int errcode, size_t wanted, size_t got,bool expected)
+void RaiseOutOfMemException(int errcode, size_t wanted, size_t got, bool expected, const char *errMsg)
 {
-    throw createOutOfMemException(errcode, wanted, got,expected);
+    throw createOutOfMemException(errcode, wanted, got, expected, errMsg);
 }
 
 MemoryAttr::MemoryAttr(size_t _len)
@@ -243,13 +255,17 @@ void MemoryBuffer::_realloc(size32_t newLen)
                 newMax = FIRST_CHUNK_SIZE;
             while (newLen > newMax)
             {
-                newMax += newMax;
+                size32_t newMaxTmp = checkMemoryBufferOverflow(newMax, newMax);
+                newMax = newMaxTmp;
             }
         }
         else
+        {
+            size32_t newMaxTmp = checkMemoryBufferOverflow((newLen & ~(ChunkSize-1)), ChunkSize);
             /*** ((Size + 1) + (ChunkSize - 1)) & ~(ChunkSize-1) ***/
-            newMax = (newLen + ChunkSize) & ~(ChunkSize-1);
-        
+            newMax = newMaxTmp;
+        }
+
         buffer =(char *)checked_realloc(buffer, newMax, maxLen, -7);
         maxLen = newMax;
     }
@@ -279,7 +295,7 @@ void MemoryBuffer::init()
 void *MemoryBuffer::insertDirect(unsigned offset, size32_t insertLen)
 {
     assertex(offset<=curLen);
-    unsigned newLen = insertLen + curLen;
+    unsigned newLen = checkMemoryBufferOverflow(curLen, insertLen);
     _realloc(newLen);
     memmove(buffer + offset + insertLen, buffer + offset, curLen - offset);
     curLen += insertLen;
@@ -290,7 +306,10 @@ void *MemoryBuffer::insertDirect(unsigned offset, size32_t insertLen)
 void * MemoryBuffer::ensureCapacity(unsigned max)
 {
     if (maxLen - curLen < max)
-        _realloc(curLen + max);
+    {
+        unsigned newLen = checkMemoryBufferOverflow(curLen, max);
+        _realloc(newLen);
+    }
     return buffer + curLen;
 }
 
@@ -313,7 +332,8 @@ MemoryBuffer & MemoryBuffer::_remove(unsigned start, unsigned len)
 
 void * MemoryBuffer::reserve(unsigned size)
 {
-    _realloc(curLen + size);
+    unsigned newLen = checkMemoryBufferOverflow(curLen, size);
+    _realloc(newLen);
     void * ret = buffer + curLen;
     curLen += size;
     return ret;
@@ -321,8 +341,9 @@ void * MemoryBuffer::reserve(unsigned size)
 
 void * MemoryBuffer::reserveTruncate(unsigned size)
 {
+    unsigned newLen = checkMemoryBufferOverflow(curLen, size);
     curLen += size;
-    _reallocExact(curLen);
+    _reallocExact(newLen);
     truncate();
     return buffer + curLen - size;
 }
@@ -458,7 +479,8 @@ MemoryBuffer::MemoryBuffer(size_t len, const void * newBuffer)
 
 MemoryBuffer & MemoryBuffer::append(char value)
 {
-    _realloc(curLen + 1);
+    unsigned newLen = checkMemoryBufferOverflow(curLen, 1);
+    _realloc(newLen);
     buffer[curLen] = value;
     ++curLen;
     return *this;
@@ -467,7 +489,8 @@ MemoryBuffer & MemoryBuffer::append(char value)
 
 MemoryBuffer & MemoryBuffer::append(unsigned char value)
 {
-    _realloc(curLen + 1);
+    unsigned newLen = checkMemoryBufferOverflow(curLen, 1);
+    _realloc(newLen);
     buffer[curLen] = value;
     ++curLen;
     return *this;
@@ -475,7 +498,8 @@ MemoryBuffer & MemoryBuffer::append(unsigned char value)
 
 MemoryBuffer & MemoryBuffer::append(bool value)
 {
-    _realloc(curLen + 1);
+    unsigned newLen = checkMemoryBufferOverflow(curLen, 1);
+    _realloc(newLen);
     buffer[curLen] = (value==0)?0:1;
     ++curLen;
     return *this;
@@ -496,7 +520,8 @@ MemoryBuffer & MemoryBuffer::append(const unsigned char * value)
 
 MemoryBuffer & MemoryBuffer::append(unsigned len, const void * value)
 {
-    _realloc(curLen + len);
+    unsigned newLen = checkMemoryBufferOverflow(curLen, len);
+    _realloc(newLen);
     memcpy(buffer + curLen, value, len);
     curLen += len;
     return *this;
@@ -569,8 +594,8 @@ MemoryBuffer & MemoryBuffer::appendPacked(unsigned __int64 value)
 MemoryBuffer & MemoryBuffer::append(const MemoryBuffer & value)
 {
     size32_t SourceLen = value.length();
-    
-    _realloc(curLen + SourceLen);
+    unsigned newLen = checkMemoryBufferOverflow(curLen, SourceLen);
+    _realloc(newLen);
     memcpy(buffer + curLen, value.toByteArray(), SourceLen);
     curLen += SourceLen;
     return *this;
@@ -578,7 +603,8 @@ MemoryBuffer & MemoryBuffer::append(const MemoryBuffer & value)
 
 MemoryBuffer & MemoryBuffer::appendBytes(unsigned char value, unsigned count)
 {
-    _realloc(curLen + count);
+    unsigned newLen = checkMemoryBufferOverflow(curLen, count);
+    _realloc(newLen);
     memset(buffer+curLen, value, count);
     curLen+=count;
     return *this;
@@ -586,7 +612,8 @@ MemoryBuffer & MemoryBuffer::appendBytes(unsigned char value, unsigned count)
 
 MemoryBuffer & MemoryBuffer::appendEndian(size32_t len, const void * value)
 {
-    _realloc(curLen + len);
+    unsigned newLen = checkMemoryBufferOverflow(curLen, len);
+    _realloc(newLen);
     
     if (swapEndian)
         _cpyrevn(buffer + curLen, value, len);
@@ -599,7 +626,8 @@ MemoryBuffer & MemoryBuffer::appendEndian(size32_t len, const void * value)
 
 MemoryBuffer & MemoryBuffer::appendSwap(size32_t len, const void * value)
 {
-    _realloc(curLen + len);
+    unsigned newLen = checkMemoryBufferOverflow(curLen, len);
+    _realloc(newLen);
     _cpyrevn(buffer + curLen, value, len);
     curLen += len;
     return *this;

system/jlib/jbuff.hpp

@@ -130,6 +130,8 @@ private:
     CLASS * ptr;
 };
 
+#define MEMBUFFER_MAXLEN UINT_MAX // size32_t
+
 class jlib_decl MemoryBuffer
 {
 public:
@@ -624,8 +626,8 @@ class CLargeMemorySequentialReader
 
 
 interface IOutOfMemException;
-jlib_decl IOutOfMemException *createOutOfMemException(int errcode, size_t wanted, size_t got=0,bool expected=false);
-jlib_decl void RaiseOutOfMemException(int errcode, size_t wanted, size_t got=0,bool expected=false);
+jlib_decl IOutOfMemException *createOutOfMemException(int errcode, size_t wanted, size_t got=0, bool expected=false, const char *errMsg=nullptr);
+jlib_decl void RaiseOutOfMemException(int errcode, size_t wanted, size_t got=0, bool expected=false, const char *errMsg=nullptr);
 
 interface ILargeMemLimitNotify: extends IInterface
 {

system/jlib/jdebug.cpp

@@ -300,7 +300,7 @@ double getCycleToNanoScale()
 
 #else
 
-#if defined(_ARCH_X86_) || defined(_ARCH_X86_64_)
+#if defined(HAS_GOOD_CYCLE_COUNTER)
 static bool useRDTSC = _USE_RDTSC;
 #endif
 static double cycleToNanoScale;
@@ -309,9 +309,6 @@ static double cycleToMilliScale;
 
 void calibrate_timing()
 {
-    cycleToNanoScale = 1.0;
-    cycleToMicroScale = 1.0;
-    cycleToMilliScale = 1.0;
 #if defined(_ARCH_X86_) || defined(_ARCH_X86_64_)
     if (useRDTSC) {
         unsigned long eax;
@@ -334,6 +331,9 @@ void calibrate_timing()
         if ((edx&0x10)==0)
             useRDTSC = false;
     }
+#endif
+
+#if defined(HAS_GOOD_CYCLE_COUNTER)
     if (useRDTSC) {
         unsigned startu = usTick();
         cycle_t start = getTSC();
@@ -358,6 +358,8 @@ void calibrate_timing()
     }
 #endif
     cycleToNanoScale = 1.0;
+    cycleToMicroScale = 1.0;
+    cycleToMilliScale = 1.0;
 }
 
 
@@ -367,7 +369,7 @@ static bool use_gettimeofday=false;
 #endif
 cycle_t jlib_decl get_cycles_now()
 {
-#if defined(_ARCH_X86_) || defined(_ARCH_X86_64_)
+#if defined(HAS_GOOD_CYCLE_COUNTER)
     if (useRDTSC)
         return getTSC();
 #endif
@@ -390,19 +392,19 @@ cycle_t jlib_decl get_cycles_now()
 
 __int64 jlib_decl cycle_to_nanosec(cycle_t cycles)
 {
-#if defined(_ARCH_X86_) || defined(_ARCH_X86_64_)
+#if defined(HAS_GOOD_CYCLE_COUNTER)
     if (useRDTSC)
         return (__int64)((double)cycles * cycleToNanoScale);
+#endif
 #ifdef __APPLE__
     return cycles * (uint64_t) timebase_info.numer / (uint64_t)timebase_info.denom;
 #endif
-#endif
     return cycles;
 }
 
 __int64 jlib_decl cycle_to_microsec(cycle_t cycles)
 {
-#if defined(_ARCH_X86_) || defined(_ARCH_X86_64_)
+#if defined(HAS_GOOD_CYCLE_COUNTER)
     if (useRDTSC)
         return (__int64)((double)cycles * cycleToMicroScale);
 #endif
@@ -411,7 +413,7 @@ __int64 jlib_decl cycle_to_microsec(cycle_t cycles)
 
 __int64 jlib_decl cycle_to_millisec(cycle_t cycles)
 {
-#if defined(_ARCH_X86_) || defined(_ARCH_X86_64_)
+#if defined(HAS_GOOD_CYCLE_COUNTER)
     if (useRDTSC)
         return (__int64)((double)cycles * cycleToMilliScale);
 #endif
@@ -420,7 +422,7 @@ __int64 jlib_decl cycle_to_millisec(cycle_t cycles)
 
 cycle_t nanosec_to_cycle(__int64 ns)
 {
-#if defined(_ARCH_X86_) || defined(_ARCH_X86_64_)
+#if defined(HAS_GOOD_CYCLE_COUNTER)
     if (useRDTSC)
         return (__int64)((double)ns / cycleToNanoScale);
 #endif

system/jlib/jdebug.hpp

@@ -54,6 +54,38 @@ inline __int64 getTSC()
 inline cycle_t getTSC() { return __rdtsc(); }
 #endif // WIN32
 
+#elif defined(_ARCH_PPC)
+
+#define HAS_GOOD_CYCLE_COUNTER
+
+static inline cycle_t getTSC()
+{
+    int64_t result;
+#ifdef _ARCH_PPC64
+    /*
+        This reads timebase in one 64bit go.  Does *not* include a workaround for the cell (see 
+        http://ozlabs.org/pipermail/linuxppc-dev/2006-October/027052.html)
+    */
+    __asm__ volatile(
+        "mftb    %0"
+        : "=r" (result));
+#else
+    /*
+        Read the high 32bits of the timer, then the lower, and repeat if high order has changed in the meantime.  See
+        http://ozlabs.org/pipermail/linuxppc-dev/1999-October/003889.html
+    */
+    unsigned long dummy;
+    __asm__ volatile(
+        "mfspr   %1,269\n\t"  /* mftbu */
+        "mfspr   %L0,268\n\t" /* mftb */
+        "mfspr   %0,269\n\t"  /* mftbu */
+        "cmpw    %0,%1\n\t"   /* check if the high order word has chanegd */
+        "bne     $-16"
+        : "=r" (result), "=r" (dummy));
+#endif
+    return result;
+}
+
 #else
 // ARMFIX: cycle-count is not always available in user mode
 // http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0338g/Bihbeabc.html

system/jlib/jthread.cpp

@@ -634,7 +634,6 @@ void CAsyncFor::For(unsigned num,unsigned maxatonce,bool abortFollowingException
     }
     Mutex errmutex;
     Semaphore ready;
-    Semaphore finished;
     IException *e=NULL;
     Owned<IShuffledIterator> shuffler;
     if (shuffled) {
@@ -665,13 +664,12 @@ void CAsyncFor::For(unsigned num,unsigned maxatonce,bool abortFollowingException
         public:
             Mutex *errmutex;
             Semaphore &ready;
-            Semaphore &finished;
             int timeout;
             IException *&erre;
             unsigned idx;
             CAsyncFor *self;
-            cdothread(CAsyncFor *_self,unsigned _idx,Semaphore &_ready,Semaphore &_finished,Mutex *_errmutex,IException *&_e)
-                : Thread("CAsyncFor"),ready(_ready),finished(_finished),erre(_e)
+            cdothread(CAsyncFor *_self,unsigned _idx,Semaphore &_ready,Mutex *_errmutex,IException *&_e)
+                : Thread("CAsyncFor"),ready(_ready),erre(_e)
             {
                 errmutex =_errmutex;
                 idx = _idx;
@@ -699,7 +697,6 @@ void CAsyncFor::For(unsigned num,unsigned maxatonce,bool abortFollowingException
                 }
 #endif
                 ready.signal();
-                finished.signal();
                 return 0;
             }
         };
@@ -707,14 +704,19 @@ void CAsyncFor::For(unsigned num,unsigned maxatonce,bool abortFollowingException
             maxatonce = num;
         for (i=0;(i<num)&&(i<maxatonce);i++)
             ready.signal();
+        IArrayOf<Thread> started;
+        started.ensure(num);
         for (i=0;i<num;i++) {
             ready.wait();
             if (abortFollowingException && e) break;
-            Thread *thread = new cdothread(this,shuffled?shuffler->lookup(i):i,ready,finished,&errmutex,e);
-            thread->startRelease();
+            Owned<Thread> thread = new cdothread(this,shuffled?shuffler->lookup(i):i,ready,&errmutex,e);
+            thread->start();
+            started.append(*thread.getClear());
+        }
+        ForEachItemIn(idx, started)
+        {
+            started.item(idx).join();
         }
-        while (i--) 
-            finished.wait();
     }
     if (e)
         throw e;

testing/regress/ecl/fetch2.ecl

@@ -52,6 +52,12 @@ fetched := fetch(sq.SimplePersonBookDs, myPeople, right.filepos, makeRec(left, r
 fetched2 := fetch(sq.SimplePersonBookDs, myPeople, right.filepos, makeRec2(left, right));
 fetched3 := fetch(sq.SimplePersonBookDs, myPeople, right.filepos, makeRec3(left, right));
 
+fetchit (dataset(sq.SimplePersonBookExRec) ds, string sname) := FUNCTION
+   return fetch(ds, ds(surname=sname), right.filepos, makeRec3(left,right));
+END;
+
+fetched4 := fetchit( sq.SimplePersonBookDs, 'Halliday' );
+
 recordof(sq.SimplePersonBookDs) removeFp(recfp l) := TRANSFORM
     SELF := l;
 END;
@@ -62,5 +68,6 @@ END;
 sequential(
     output(sortIt(fetched)),
     output(sortIt(fetched2)),
-    output(sortIt(fetched3))
+    output(sortIt(fetched3)),
+    output(sortIt(fetched4))
 );

testing/regress/ecl/platform.ecl

@@ -13,7 +13,7 @@
     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     See the License for the specific language governing permissions and
     limitations under the License.
-############################################################################## */
+##############################################################################*/
 
 import std.system.thorlib;
 

testing/regress/hpcc/regression/suite.py

@@ -27,11 +27,12 @@ from ..common.error import Error
 from ..util.util import checkClusters, getConfig
 
 class Suite:
-    def __init__(self, name, dir_ec, dir_a, dir_ex, dir_r, logDir, args, isSetup=False,  fileList = None):
+    def __init__(self, clusterName, dir_ec, dir_a, dir_ex, dir_r, logDir, args, isSetup=False,  fileList = None):
+        self.clusterName = clusterName
+        self.targetName = clusterName
         if isSetup:
-            self.name = 'setup_'+name
-        else:
-            self.name = name
+            self.targetName = 'setup_'+clusterName
+
         self.args=args
         self.suite = []
         self.dir_ec = dir_ec
@@ -48,7 +49,7 @@ class Suite:
 
         if len(self.exclude):
             curTime = time.strftime("%y-%m-%d-%H-%M-%S")
-            logName = self.name + "-exclusion." + curTime + ".log"
+            logName = self.targetName + "-exclusion." + curTime + ".log"
             self.logName = os.path.join(self.logDir, logName)
             args.exclusionFile=self.logName
             self.log = open(self.logName, "w");
@@ -83,11 +84,11 @@ class Suite:
             if file.endswith(".ecl"):
                 ecl = os.path.join(self.dir_ec, file)
                 eclfile = ECLFile(ecl, self.dir_a, self.dir_ex,
-                                  self.dir_r,  self.args.target,  args)
+                                  self.dir_r,  self.clusterName,   args)
                 if isSetup:
                     skipResult = eclfile.testSkip('setup')
                 else:
-                    skipResult = eclfile.testSkip(self.name)
+                    skipResult = eclfile.testSkip(self.targetName)
 
                 if not skipResult['skip']:
                     exclude=False
@@ -105,7 +106,7 @@ class Suite:
                         exclude = (not included )  or excluded
                         exclusionReason=' class member excluded'
                     if not exclude:
-                        exclude = eclfile.testExclusion(self.name)
+                        exclude = eclfile.testExclusion(self.targetName)
                         exclusionReason=' ECL excluded'
 
                     if not exclude:
@@ -126,7 +127,7 @@ class Suite:
             versions = eclfile.getVersions()
             versionId = 1
             for version in versions:
-                if 'no'+self.name in version:
+                if 'no'+self.targetName in version:
                     # Exclude it from this target
                     pass
                 else:
@@ -144,7 +145,7 @@ class Suite:
             # generate ECLs to suite
             for file in files:
                 generatedEclFile = ECLFile(basename, self.dir_a, self.dir_ex,
-                                 self.dir_r,  self.name, self.args)
+                                 self.dir_r,  self.clusterName, self.args)
 
                 generatedEclFile.setDParameters(file['version'])
                 generatedEclFile.setVersionId(file['id'])
@@ -176,7 +177,7 @@ class Suite:
         return self.endTime-self.startTime
 
     def getSuiteName(self):
-        return self.name
+        return self.targetName
 
     def cleanUp(self):
         # If there are some temporary files left, then remove them

thorlcr/activities/thactivityutil.cpp

@@ -257,9 +257,6 @@ public:
         }
     }
 };
-#ifdef _MSC_VER
-#pragma warning(pop)
-#endif
 
 
 IStartableEngineRowStream *createRowStreamLookAhead(CSlaveActivity *activity, IEngineRowStream *inputStream, IThorRowInterfaces *rowIf, size32_t bufsize, bool allowspill, bool preserveGrouping, rowcount_t maxcount, ILookAheadStopNotify *notify, IDiskUsage *iDiskUsage)

thorlcr/activities/wuidwrite/thwuidwrite.cpp

@@ -319,7 +319,7 @@ public:
             builder.appendOwn(row);
         }
 
-        size32_t usedCount = rtlDictionaryCount(builder.getcount(), builder.queryrows());
+        size32_t usedCount = (size32_t)rtlDictionaryCount(builder.getcount(), builder.queryrows());
         MemoryBuffer rowData;
         CThorDemoRowSerializer out(rowData);
         rtlSerializeDictionary(out, rowIf->queryRowSerializer(), builder.getcount(), builder.queryrows());

thorlcr/graph/thgraphslave.hpp

@@ -219,8 +219,8 @@ public:
 };
 
 
-IEngineRowStream *connectSingleStream(CActivityBase &activity, IThorDataLink *input, unsigned idx, Owned<IStrandJunction> &junction, bool consumerOrdered);
-IEngineRowStream *connectSingleStream(CActivityBase &activity, IThorDataLink *input, unsigned idx, bool consumerOrdered);
+graphslave_decl IEngineRowStream *connectSingleStream(CActivityBase &activity, IThorDataLink *input, unsigned idx, Owned<IStrandJunction> &junction, bool consumerOrdered);
+graphslave_decl IEngineRowStream *connectSingleStream(CActivityBase &activity, IThorDataLink *input, unsigned idx, bool consumerOrdered);
 
 
 #define STRAND_CATCH_NEXTROWX_CATCH \
@@ -242,12 +242,12 @@ IEngineRowStream *connectSingleStream(CActivityBase &activity, IThorDataLink *in
 
 
 class CThorStrandedActivity;
-class CThorStrandProcessor : public CInterfaceOf<IEngineRowStream>, public COutputTiming
+class graphslave_decl CThorStrandProcessor : public CInterfaceOf<IEngineRowStream>, public COutputTiming
 {
 protected:
     CThorStrandedActivity &parent;
     IEngineRowStream *inputStream;
-    unsigned numProcessedLastGroup = 0;
+    rowcount_t numProcessedLastGroup = 0;
     const bool timeActivities;
     bool stopped = false;
     unsigned outputId; // if activity had >1 , this identifies (for tracing purposes) which output this strand belongs to.
@@ -282,7 +282,7 @@ public:
     }
 };
 
-class CThorStrandedActivity : public CSlaveActivity
+class graphslave_decl CThorStrandedActivity : public CSlaveActivity
 {
 protected:
     CThorStrandOptions strandOptions;

thorlcr/thorcodectx/thcodectx.hpp

@@ -50,6 +50,7 @@ protected:
 
 public:
     CThorCodeContextBase(CJobChannel &jobChannel, ILoadedDllEntry &_querySo, IUserDescriptor &_userDesc);
+    IMPLEMENT_IINTERFACE_USING(CSimpleInterfaceOf<ICodeContextExt>) // This is strangely required by visual studio to ensure Release() is resolved
 
 // ICodeContext
     virtual const char *loadResource(unsigned id);