Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -674,7 +674,6 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
         endif()
       else()
         add_definitions (-D_NO_MYSQL_REPOSITORY)
-        set(MYSQL_INCLUDE_DIR "")
       endif(USE_MYSQL_REPOSITORY)
 
       if(USE_APR)

common/remote/sockfile.cpp

@@ -366,6 +366,13 @@ const char *ThrottleStrings[] =
     ThrottleText(ThrottleStd),
     ThrottleText(ThrottleSlow),
 };
+
+// very high upper limits that configure can't exceed
+#define THROTTLE_MAX_LIMIT 1000000
+#define THROTTLE_MAX_DELAYMS 3600000
+#define THROTTLE_MAX_CPUTHRESHOLD 100
+#define THROTTLE_MAX_QUEUELIMIT 10000000
+
 static const char *getThrottleClassText(ThrottleClass throttleClass) { return ThrottleStrings[throttleClass]; }
 
 typedef enum { ACScontinue, ACSdone, ACSerror} AsyncCommandStatus;
@@ -3467,6 +3474,10 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer
         }
         void configure(unsigned _limit, unsigned _delayMs, unsigned _cpuThreshold, unsigned _queueLimit)
         {
+            if (_limit > THROTTLE_MAX_LIMIT || _delayMs > THROTTLE_MAX_DELAYMS || _cpuThreshold > THROTTLE_MAX_CPUTHRESHOLD || _queueLimit > THROTTLE_MAX_QUEUELIMIT)
+                throw MakeStringException(0, "Throttler(%s), rejecting configure command: limit=%u (max permitted=%u), delayMs=%u (max permitted=%u), cpuThreshold=%u (max permitted=%u), queueLimit=%u (max permitted=%u)",
+                                              title.str(), _limit, THROTTLE_MAX_LIMIT, _delayMs, THROTTLE_MAX_DELAYMS, _cpuThreshold,
+                                              THROTTLE_MAX_CPUTHRESHOLD, _queueLimit, THROTTLE_MAX_QUEUELIMIT);
             CriticalBlock b(configureCrit);
             int delta = 0;
             if (_limit)
@@ -3482,7 +3493,7 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer
             }
             else if (0 == disabledLimit)
             {
-                PROGLOG("Throttler(%s): disabled, previous limit: %d", title.get(), limit);
+                PROGLOG("Throttler(%s): disabled, previous limit: %u", title.get(), limit);
                 /* disabling - set limit immediately to let all new transaction through.
                  * NB: the semaphore signals are not consumed in this case, because transactions could be waiting on it.
                  * Instead the existing 'limit' is kept in 'disabledLimit', so that if/when throttling is
@@ -3493,14 +3504,14 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer
             }
             if (delta > 0)
             {
-                PROGLOG("Throttler(%s): Increasing limit from %d to %d", title.get(), limit, _limit);
+                PROGLOG("Throttler(%s): Increasing limit from %u to %u", title.get(), limit, _limit);
                 sem.signal(delta);
                 limit = _limit;
                 // NB: If throttling was off, this doesn't effect transactions in progress, i.e. will only throttle new transactions coming in.
             }
             else if (delta < 0)
             {
-                PROGLOG("Throttler(%s): Reducing limit from %d to %d", title.get(), limit, _limit);
+                PROGLOG("Throttler(%s): Reducing limit from %u to %u", title.get(), limit, _limit);
                 // NB: This is not expected to take long
                 CCycleTimer timer;
                 while (delta < 0)
@@ -3508,7 +3519,7 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer
                     if (sem.wait(1000))
                         ++delta;
                     else
-                        PROGLOG("Throttler(%s): Waited %0.2f seconds so far for up to a maximum of %d (previous limit) transactions to complete, %d completed", title.get(), ((double)timer.elapsedMs())/1000, limit, -delta);
+                        PROGLOG("Throttler(%s): Waited %0.2f seconds so far for up to a maximum of %u (previous limit) transactions to complete, %u completed", title.get(), ((double)timer.elapsedMs())/1000, limit, -delta);
                 }
                 limit = _limit;
                 // NB: doesn't include transactions in progress, i.e. will only throttle new transactions coming in.
@@ -3582,7 +3593,7 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer
                             CriticalBlock b(crit);
                             totalThrottleDelay += ms;
                         }
-                        PROGLOG("Throttler(%s): transaction delayed [cmd=%s] for : %d milliseconds, proceeding as cpu(%u)<throttleCPULimit(%u)", title.get(), getRFCText(cmd), cpu, ms, cpuThreshold);
+                        PROGLOG("Throttler(%s): transaction delayed [cmd=%s] for : %u milliseconds, proceeding as cpu(%u)<throttleCPULimit(%u)", title.get(), getRFCText(cmd), cpu, ms, cpuThreshold);
                         hadSem = false;
                     }
                     else
@@ -3651,7 +3662,7 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer
                 if (ms >= 1000)
                 {
                     if (ms>delayMs)
-                        PROGLOG("Throttler(%s): transaction delayed [cmd=%s] for : %d seconds", title.get(), getRFCText(currentCmd), ms/1000);
+                        PROGLOG("Throttler(%s): transaction delayed [cmd=%s] for : %u seconds", title.get(), getRFCText(currentCmd), ms/1000);
                 }
                 {
                     CriticalBlock b(crit);

docs/ECLLanguageReference/ECLR_mods/Templ-WORKUNIT.xml

@@ -50,8 +50,8 @@
           <row>
             <entry><emphasis>cluster</emphasis></entry>
 
-            <entry>The value parameter specifies the name of the cluster on
-            which the workunit executes.</entry>
+            <entry>The value parameter is a string constant containing the
+            name of the target cluster on which the workunit executes.</entry>
           </row>
 
           <row>
@@ -83,7 +83,7 @@
             <entry>The value parameter is a string constant containing the
             scope value to use to override the workunit's default scope (the
             user ID of the submitting person). This is a Workunit Security
-            feature.</entry>
+            feature and requires a system which is LDAP-enabled. </entry>
           </row>
         </tbody>
       </tgroup>
@@ -91,11 +91,11 @@
 
   <para>Example:</para>
 
-  <programlisting>#WORKUNIT('cluster',400way);   //run the job on the 400-way cluster
-#WORKUNIT('protect',true);     //disallow deletion
+  <programlisting>#WORKUNIT('cluster','400way'); //run the job on the 400-way target cluster
+#WORKUNIT('protect',true);     //disallow deletion or archiving by Sasha 
 #WORKUNIT('name','My Job');    //name it "My Job"
 #WORKUNIT('priority','high');  //run before other lower-priority jobs
 #WORKUNIT('priority',10);      //run before other high-priority jobs
-#WORKUNIT('scope','NewVal');   //override the default scope
+#WORKUNIT('scope','NewVal');   //override the default scope (on an LDAP enabled system)
 </programlisting>
 </sect1>

docs/ECLStandardLibraryReference/SLR-Mods/Cluster.xml

@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE sect1
-  PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
-<sect1 id="Target">
-   <title>
-      <emphasis role="bold">Target</emphasis>
-   </title>
-   <para>
-      <emphasis role="bold">STD.System.Thorlib.Target<indexterm>
-	<primary>STD.System.STD.System.Job.Target</primary>
-</indexterm>
-<indexterm>
-	<primary>System.STD.System.Job.Target</primary>
-</indexterm>
-<indexterm>
-	<primary>STD.System.Job.Target</primary>
-</indexterm>
-<indexterm>
-	<primary>Target</primary>
-</indexterm>
-(</emphasis>
-      <emphasis>
-        </emphasis>
-      <emphasis role="bold">)</emphasis>
-   </para>
-  <informaltable colsep="1" frame="all" rowsep="1">
-    <tgroup cols="2">
-      <colspec colwidth="80.50pt" />
-
-      <colspec />
-
-      <tbody>
-        <row>
-
-    <entry>Return:<emphasis> </emphasis></entry><entry>Target returns a VARSTRING
-        value.</entry>
-
-
-        </row>
-
-      </tbody>
-    </tgroup>
-  </informaltable>
-   <para>The <emphasis role="bold">Target </emphasis>function returns
-        the name of the cluster running the workunit. Not supported on Roxie
-        clusters. This name is used by ECLplus.exe to specify the the target
-        cluster for a workunit.</para>
-   <para>Example:</para>
-   <programlisting format="linespecific">A := STD.System.Thorlib.Target();</programlisting>
-</sect1>

docs/ECLStandardLibraryReference/SLR-Mods/Target.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="Target">
+  <title><emphasis role="bold">Target</emphasis></title>
+
+  <para><emphasis role="bold">STD.System.Job.Target <indexterm>
+      <primary>STD.System.Job.Target</primary>
+    </indexterm> <indexterm>
+      <primary>Target</primary>
+    </indexterm> (</emphasis> <emphasis> </emphasis> <emphasis
+  role="bold">)</emphasis></para>
+
+  <informaltable colsep="1" frame="all" rowsep="1">
+    <tgroup cols="2">
+      <colspec colwidth="80.50pt" />
+
+      <colspec />
+
+      <tbody>
+        <row>
+          <entry>Return:<emphasis> </emphasis></entry>
+
+          <entry>Target returns a VARSTRING value.</entry>
+        </row>
+      </tbody>
+    </tgroup>
+  </informaltable>
+
+  <para>The <emphasis role="bold">Target </emphasis>function returns the name
+  of the cluster running the workunit. Not supported on Roxie clusters. This
+  name is used by #WORKUNIT, the ecl command line utility, or the eclplus
+  command line utility to specify the the target cluster for a
+  workunit.</para>
+
+  <para>Example:</para>
+
+  <programlisting format="linespecific">A := STD.System.Job.Target();</programlisting>
+</sect1>

docs/ECLStandardLibraryReference/SLR-includer.xml

@@ -426,7 +426,7 @@
     <xi:include href="ECLStandardLibraryReference/SLR-Mods/WUID.xml"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
-    <xi:include href="ECLStandardLibraryReference/SLR-Mods/Cluster.xml"
+    <xi:include href="ECLStandardLibraryReference/SLR-Mods/Target.xml"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
     <xi:include href="ECLStandardLibraryReference/SLR-Mods/JobName.xml"

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -60,7 +60,7 @@
     </mediaobject>
   </bookinfo>
 
-  <chapter>
+  <chapter id="HPCC_Systems_Administration">
     <title>Introducing HPCC Systems<superscript>®</superscript>
     Administraton</title>
 
@@ -124,13 +124,13 @@
       <para>The data retrieval process (despraying) places the file back on
       the landing zone.</para>
 
-      <sect2 role="brk">
+      <sect2 id="HPCC_Clusters" role="brk">
         <title>Clusters</title>
 
         <para>HPCC environment contains clusters which you define and use
         according to your needs. The types of clusters used in HPCC:</para>
 
-        <sect3>
+        <sect3 id="SysAdm_Thor_Cluster">
           <title>Thor</title>
 
           <para>Data Refinery (Thor) – Used to process every one of billions
@@ -139,7 +139,7 @@
           inefficient use of the Thor cluster.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_Roxie_Cluster">
           <title>Roxie</title>
 
           <para>Rapid Data Delivery Engine (Roxie) – Used to search quickly
@@ -151,7 +151,7 @@
           data into play.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_Clusters_ECLAgent">
           <title>ECL Agent</title>
 
           <para>The ECL Agent's primary function is to send the job to execute
@@ -171,14 +171,14 @@
         </sect3>
       </sect2>
 
-      <sect2 role="brk">
+      <sect2 id="SysAdm_SystemServers" role="brk">
         <title>System Servers</title>
 
         <para>The System Servers are integral middleware components of an HPCC
         system. They are used to control workflow and intercomponent
         communication.</para>
 
-        <sect3>
+        <sect3 id="SysAdm_Dali">
           <title>Dali</title>
 
           <para>Dali is also known as the system data store. It manages
@@ -190,7 +190,7 @@
           restrictions.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_Sahsa">
           <title>Sasha</title>
 
           <para>The Sasha server is a companion “housekeeping” server to the
@@ -205,7 +205,7 @@
           cached workunits and DFU recovery files.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_DFU">
           <title>DFU Server</title>
 
           <para>DFU server controls the spraying and despraying operations
@@ -227,7 +227,7 @@
             </itemizedlist></para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_ECLCCSvr">
           <title>ECLCC Server</title>
 
           <para>ECLCC Server is the compiler that translates ECL code. When
@@ -251,7 +251,7 @@
           required.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_ECLAgent">
           <title>ECL Agent</title>
 
           <para>ECL Agent (hThor) is a single node process for executing
@@ -262,7 +262,7 @@
           spawned on-demand when you submit a workunit.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_ESPServer">
           <title>ESP Server</title>
 
           <para>ESP (Enterprise Service Platform) Server is the
@@ -291,7 +291,7 @@
           <!--formerly : protocols - HTTP, HTTPS, SOAP, and JSON - -->
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_LDAP">
           <title>LDAP</title>
 
           <para>You can incorporate a Lightweight Directory Access Protocol
@@ -320,13 +320,13 @@
         END COMMENT ***-->
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_ClienInterfaces">
         <title>Client Interfaces</title>
 
         <para>The following Client Interfaces are available to interact with
         the HPCC Platform.</para>
 
-        <sect3>
+        <sect3 id="SysAdm_Eclipse">
           <title>Eclipse</title>
 
           <para>With the ECL plug-in for Eclipse, you can use the Eclipse IDE
@@ -337,7 +337,7 @@
           open-source.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_ECLIDE">
           <title>ECL IDE</title>
 
           <para>ECL IDE is a full-featured GUI providing access to your ECL
@@ -350,7 +350,7 @@
           Once defined, they can be used in succeeding ECL definitions.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_Int_ECLWatch">
           <title>ECL Watch</title>
 
           <para>ECL Watch is a web-based query execution, monitoring, and file
@@ -389,7 +389,7 @@
           details.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_ComLine">
           <title><emphasis role="bold">Command Line Tools</emphasis></title>
 
           <para>Command line tools: <emphasis role="bold">ECL, DFU
@@ -407,7 +407,7 @@
     <!--Inclusion-from-ClientTool-As-Sect1: REMOVED-->
   </chapter>
 
-  <chapter>
+  <chapter id="SysAdm_HW_and_SW-Req">
     <title>Hardware and Software Requirements</title>
 
     <para>This chapter consists of various Hardware and Software requirements
@@ -448,7 +448,7 @@
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
   </chapter>
 
-  <chapter>
+  <chapter id="SysAdm_HWSizing">
     <title>Hardware and Component Sizing</title>
 
     <para>This section provides some insight as to what sort of hardware and
@@ -467,7 +467,7 @@
     to run HPCC processes of a particular type, for example Thor, Roxie, or
     Dali, on a host configured specifically for that type of process.</para>
 
-    <sect1>
+    <sect1 id="SysAdm_ThorHW">
       <title>Thor Hardware</title>
 
       <para>Thor slave nodes require a proper balance of CPU, RAM, network,
@@ -503,7 +503,7 @@
       RAID1, RAID10, RAID5 (preferred), and RAID6.</para>
     </sect1>
 
-    <sect1>
+    <sect1 id="SysAdm_RoxieHW">
       <title>Roxie Hardware Configurations</title>
 
       <para>HPCC Roxie processes require require a proper, yet different (from
@@ -528,7 +528,7 @@
       distribution of file parts from Thor to Roxie.</para>
     </sect1>
 
-    <sect1>
+    <sect1 id="SysAdm_Dali_Sasha">
       <title>Dali and Sasha Hardware Configurations</title>
 
       <para>HPCC Dali processes store cluster metadata in RAM. For optimal
@@ -547,7 +547,7 @@
       the same node.</para>
     </sect1>
 
-    <sect1>
+    <sect1 id="SysAdm_OtherHPCCcomponents">
       <title>Other HPCC Components</title>
 
       <para>ECL Agent, ECLCC Server, DFU Server, the Thor master, and ECL
@@ -606,7 +606,7 @@
       <!--***CAN TIE THIS ALL TOGETHER - as part of routine maint. clean up some data files... archive data... etc. ***TO COME***-->
     </sect1>
 
-    <sect1 role="nobrk">
+    <sect1 id="SysAdm_BackUpData" role="nobrk">
       <title>Back Up Data</title>
 
       <para>An integral part of routine maintenance is the back up of
@@ -615,7 +615,7 @@
       strategy, instead this section supplements it by outlining special
       considerations for HPCC Systems<superscript>®</superscript>.</para>
 
-      <sect2>
+      <sect2 id="SysAdm_BackUpConsider">
         <title>Back Up Considerations</title>
 
         <para>You probably already have some sort of a back up strategy in
@@ -624,7 +624,7 @@
         aware of. The following sections discuss back up considerations for
         the individual HPCC system components.</para>
 
-        <sect3>
+        <sect3 id="SysAdm_BkU_Dali">
           <title>Dali</title>
 
           <para>Dali can be configured to create its own back up, ideally you
@@ -638,7 +638,7 @@
           traditional back up methods.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_Sasha">
           <title>Sasha</title>
 
           <para>Sasha itself generates no original data but archives workunits
@@ -652,14 +652,14 @@
           methods.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_DFUSvr">
           <title>DFU Server</title>
 
           <para>DFU Server has no data. DFU workunits are stored in Dali until
           they are archived by Sasha.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_ECLCCSvr">
           <title>ECLCC Server</title>
 
           <para>ECLCC Server stores no data. ECL workunits are stored in Dali
@@ -670,20 +670,20 @@
           however you must have a C++ compiler to use on your system. </para> -->
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_ECLAgent">
           <title>ECL Agent</title>
 
           <para>ECL Agent stores no data.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_ECLSched">
           <title>ECL Scheduler</title>
 
           <para>ECL Scheduler stores no data. ECL Workunits are stored in
           Dali.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_ESPsvr">
           <title>ESP Server</title>
 
           <para>ESP Server stores no data. If you are using SSL certificates,
@@ -691,7 +691,7 @@
           methods.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_Thor">
           <title>Thor</title>
 
           <para>Thor, the data refinery, as one of the critical components of
@@ -752,7 +752,7 @@
 00000029 2014-02-19 12:01:08 26457 26457 "backupnode finished" </programlisting>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_Roxie">
           <title>Roxie</title>
 
           <para>Roxie data is protected by three forms of redundancy:</para>
@@ -789,7 +789,7 @@
           </itemizedlist>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_LandZone">
           <title>Landing Zone</title>
 
           <para>The Landing Zone is used to host incoming and outgoing files.
@@ -797,7 +797,7 @@
           system level back ups.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BkUp_Misc">
           <title>Misc</title>
 
           <para>Back up of any additional component add-ons, your environment
@@ -807,7 +807,7 @@
       </sect2>
     </sect1>
 
-    <sect1 id="Log_Files">
+    <sect1 id="SysAdm_Log_Files">
       <title>Log Files</title>
 
       <para>You can review system messages and see any error messages as they
@@ -815,7 +815,7 @@
       understanding what is occurring on the system and useful in
       troubleshooting.</para>
 
-      <sect2 id="Component_Logs">
+      <sect2 id="SysAdm_Component_Logs">
         <title>Component Logs</title>
 
         <para>There are log files for each component in directories below
@@ -845,7 +845,7 @@
         and remove the older log files.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_AccessLogFiles">
         <title>Accessing Log Files</title>
 
         <para>You can access and view the log files directly by going to the
@@ -980,7 +980,7 @@
                 xpointer="configuring-a-multi-node-system"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
-    <sect1>
+    <sect1 id="SysAdm_Env.conf">
       <title>Environment.conf</title>
 
       <para>Another component of HPCC system configuration is the
@@ -1050,7 +1050,7 @@ interface=*
 use_epoll=true
 </programlisting></para>
 
-      <sect2>
+      <sect2 id="SysAdm_Paths">
         <title>Path considerations</title>
 
         <para>Most of the directories are defined as absolute paths:</para>
@@ -1075,7 +1075,7 @@ lock=/var/lock/HPCCSystems</programlisting>
         the HPCCSystems path.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_OtherEnv.conf">
         <title>Other Environment.conf items</title>
 
         <para>Some other items used by or referred to in
@@ -1119,7 +1119,8 @@ lock=/var/lock/HPCCSystems</programlisting>
     </sect1>
 
     <!--Inclusions-As-Sect1-->
- <xi:include href="Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml"
+
+    <xi:include href="Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
@@ -1127,7 +1128,7 @@ lock=/var/lock/HPCCSystems</programlisting>
                 xpointer="User_Security_Maint"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
-    <sect1>
+    <sect1 id="SysAdm_WUs_ActiveDir">
       <title>Workunits and Active Directory</title>
 
       <para>The performance of your system can vary depending on how some
@@ -1151,7 +1152,7 @@ lock=/var/lock/HPCCSystems</programlisting>
       appropriate for your environment. Now you can assign users to their
       appropriate group(s).</para>
 
-      <sect2>
+      <sect2 id="SysAdm_AD_and_LDAP">
         <title>Active Directory, and LDAP Commonality</title>
 
         <para>There are components that are common to both Active Directory
@@ -1295,7 +1296,7 @@ lock=/var/lock/HPCCSystems</programlisting>
           </tgroup>
         </informaltable></para>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_Dali">
         <title>Dali</title>
 
         <para>Dali should be run in an active/passive configuration.
@@ -1310,7 +1311,7 @@ lock=/var/lock/HPCCSystems</programlisting>
         passive node and restart the Dali service.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_DFUsvr">
         <title>DFU Server</title>
 
         <para>You can run multiple instances of the DFU Server. You can run
@@ -1320,7 +1321,7 @@ lock=/var/lock/HPCCSystems</programlisting>
         the other(s) will continue to pull new workunits.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_ECLCCSvr_">
         <title>ECLCC Server</title>
 
         <para>You can run multiple active instances of the ECLCC Server for
@@ -1329,7 +1330,7 @@ lock=/var/lock/HPCCSystems</programlisting>
         fail, the other(s) will continue to compile.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_ESP_ECLWatch_WSECL">
         <title>ESP/ECL Watch/WsECL</title>
 
         <para>To establish redundancy, place the ESP Servers in a VIP. For an
@@ -1341,7 +1342,7 @@ lock=/var/lock/HPCCSystems</programlisting>
         server.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_ECLAgent">
         <title>ECL Agent</title>
 
         <para>You can run multiple active instances of the ECL Agent. No need
@@ -1350,7 +1351,7 @@ lock=/var/lock/HPCCSystems</programlisting>
         workunits.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_Sasha">
         <title>Sasha</title>
 
         <para>Sasha should be run in an active/passive configuration.
@@ -1358,7 +1359,7 @@ lock=/var/lock/HPCCSystems</programlisting>
         primary (active), and the other standing by.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_ECLSched">
         <title>ECL Scheduler</title>
 
         <para>No need for a load balancer, runs active/active. Each instance
@@ -1366,7 +1367,7 @@ lock=/var/lock/HPCCSystems</programlisting>
         continue to schdeule workunits.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_ThorMaster">
         <title>Thormaster</title>
 
         <para>Set up Thor in an active/passive configuration. Active/passive
@@ -1377,7 +1378,7 @@ lock=/var/lock/HPCCSystems</programlisting>
         requests.</para>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_DropZone">
         <title>Dropzone</title>
 
         <para>This is just a fileserver that runs the dafilesrv process.
@@ -1421,7 +1422,7 @@ lock=/var/lock/HPCCSystems</programlisting>
           have two of everything.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_HA_ThorUpside">
           <title>The Upside</title>
 
           <para>Almost 100% of the time you can utilize the additional
@@ -1429,7 +1430,7 @@ lock=/var/lock/HPCCSystems</programlisting>
           etc.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_HA_ThorDR">
           <title>Disaster Recovery concerns</title>
 
           <para>The important factor to consider for disaster recovery (DR) is
@@ -1468,7 +1469,7 @@ lock=/var/lock/HPCCSystems</programlisting>
           the costs of preventing against it.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_HA_ThorConclusion">
           <title>Conclusion</title>
 
           <para>Disaster recovery is a calculation. The cost of failure, times
@@ -1512,7 +1513,7 @@ lock=/var/lock/HPCCSystems</programlisting>
       </sect2>
     </sect1>
 
-    <sect1>
+    <sect1 id="SysAdm_BestPrac">
       <title>Best Practice Considerations</title>
 
       <para>There are several other aspects to best practice considerations,
@@ -1525,7 +1526,7 @@ lock=/var/lock/HPCCSystems</programlisting>
 
       <!--/*Further elaboration of both User permissions, and permission settings... also some hardware set up best practices. Suggested***/-->
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_MultiThor">
         <title>Multiple Thors</title>
 
         <para>You can run multiple Thors on the same physical hardware.
@@ -1547,6 +1548,52 @@ lock=/var/lock/HPCCSystems</programlisting>
         number of cores divided by two is the maximum number of Thor clusters
         to use.</para>
       </sect2>
+
+      <sect2 id="SysAdm_BestPrac_HugePages">
+        <title>Huge Pages</title>
+
+        <para>Linux uses pages as its basic units of memory. Your system may
+        run faster and benefit from huge page support. Huge pages of the
+        appropriate type and size need to be allocated from the operating
+        system. Almost all current Linux systems are set up with Transparent
+        Huge Pages (THP) available by default. </para>
+
+        <para>Thor, Roxie, and ECL Agent clusters all have options in the
+        configuration to enable huge page support. The Transparent Huge Pages
+        are enabled for Thor, Roxie, and ECL Agent clusters in the default
+        HPCC environment. Thor clusters can stand to benefit more from huge
+        pages than can Roxie.</para>
+
+        <para>You can check the file
+        /sys/kernel/mm/transparent_hugepage/enabled to see what your OS
+        setting is. With THP you do not have to explicitly set a size. If your
+        system is not configured to use THP, then you may want to implement
+        Huge Pages. </para>
+
+        <sect3 id="SysAdm_BestPrac_SetUpHuge_Pgs">
+          <title>Setting up Huge Pages</title>
+
+          <para>To set up huge page support, consult your OS documentation and
+          determine how to enable huge page support. For example, the
+          administrator can allocate persistent huge pages (for the
+          appropriate OS) on the kernel boot command line by specifying the
+          "hugepages=N" parameter at boot. With huge pages you also need to
+          explicitly allocate the size. </para>
+
+          <para>In HPCC, there are three places in the configuration manager
+          to set the attributes to use Huge Pages. </para>
+
+          <para>There are attributes in each component, in the ECL Agent
+          attributes, in Roxie attributes, and in Thor attributes. In each
+          component there are two values:</para>
+
+          <programlisting>heapUseHugePages  
+heapUseTransparentHugePages</programlisting>
+
+          <para>Enable Huge Pages in your operating system, then configure
+          HPCC for the component(s) you wish.</para>
+        </sect3>
+      </sect2>
     </sect1>
 
     <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
@@ -1578,7 +1625,7 @@ lock=/var/lock/HPCCSystems</programlisting>
       images, source code, documentation, and tutorials.</para>
     </sect1>
 
-    <sect1>
+    <sect1 id="SysAdm_Addl_Resources">
       <title>Additional Resources</title>
 
       <para>Additional help with HPCC and Learning ECL is also available.

docs/HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml

@@ -552,9 +552,9 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
         be about ¼ the size of Thor. This is because the data is compressed
         and the system does not hold any transient data in Roxie. Remember
         that you do not want the number of Roxie nodes to exceed the number of
-        Thor nodes. </para>
+        Thor nodes.</para>
 
-        <sect3>
+        <sect3 id="SysAdm_SmplSiz_HiDataThor">
           <title>High Data Thor sizing considerations</title>
 
           <para>Each Thor node can hold about 2.5 TB of data (MAX), so plan
@@ -568,7 +568,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           copies.</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BestPrac_HiDataRoxie">
           <title>High Data Roxie sizing considerations</title>
 
           <para>Roxie keeps most of its data in memory, so you should allocate
@@ -586,7 +586,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
         </sect3>
       </sect2>
 
-      <sect2>
+      <sect2 id="SysAdm_BestPrac_HevyProc_LowData">
         <title>Sample Sizing for Heavy Processing on Low Data Volume</title>
 
         <para>The following section provides some sample sizing for heavy
@@ -605,7 +605,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           <para>Total = 13 nodes</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BestPrac_1250GB">
           <title>1250 GB of Raw Data</title>
 
           <para>Thor = 6 (slaves) + 2 (management) = 8 Nodes</para>
@@ -618,7 +618,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           <para>Total = 17 nodes</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BestPrac_2000GB">
           <title>2000 GB of Raw Data</title>
 
           <para>Thor = 8 (slaves) + 3 (management) = 11 Nodes</para>
@@ -631,7 +631,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           <para>Total = 20 nodes</para>
         </sect3>
 
-        <sect3>
+        <sect3 id="SysAdm_BestPrac_3500GB">
           <title>3500 GB of Raw Data</title>
 
           <para>Thor = 12 (slaves) + 5 (management) = 17 Nodes</para>

ecl/hqlcpp/hqlcpp.cpp

@@ -11717,7 +11717,7 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
     {
         OwnedHqlExpr concat = createUnbalanced(no_concat, unknownStringType, attrArgs);
         OwnedHqlExpr cast = ensureExprType(concat, unknownVarStringType);
-        OwnedHqlExpr folded = foldHqlExpression(cast);
+        OwnedHqlExpr folded = foldHqlExpression(replaceInlineParameters(funcdef, cast));
         CHqlBoundExpr bound;
         buildExpr(funcctx, folded, bound);
         createParam.append(",");

esp/src/eclwatch/ESPWorkunit.js

@@ -855,8 +855,9 @@ define([
                 request: {
                     Wuid: this.Wuid,
                     GraphName: this.graphs[idx].Name
-                },
-                load: function (response) {
+                }
+            }).then(function (response) {
+                if (lang.exists("WUGetGraphResponse.Graphs.ECLGraphEx", response) && response.WUGetGraphResponse.Graphs.ECLGraphEx.length) {
                     context.graphs[idx].xgmml = response.WUGetGraphResponse.Graphs.ECLGraphEx[0].Graph;
                     onFetchGraphXgmml(context.graphs[idx].xgmml, context.graphs[idx].svg);
                 }

plugins/mysql/mysqlembed.cpp

@@ -194,15 +194,24 @@ public:
         for (int i = 0; i < columns; i++)
         {
             MYSQL_FIELD *col = mysql_fetch_field_direct(res, i);
-            if (col->type == MYSQL_TYPE_DECIMAL || col->type == MYSQL_TYPE_NEWDECIMAL)
+            switch (col->type)
             {
+            case MYSQL_TYPE_DECIMAL:
+            case MYSQL_TYPE_NEWDECIMAL:
                 bindinfo[i].buffer_type = MYSQL_TYPE_STRING;
                 bindinfo[i].buffer_length = 100;  // MORE - is there a better guess?
-            }
-            else
-            {
+                break;
+            case MYSQL_TYPE_TIMESTAMP:
+            case MYSQL_TYPE_DATETIME:
+            case MYSQL_TYPE_TIME:
+            case MYSQL_TYPE_DATE:
+                bindinfo[i].buffer_type = col->type;
+                bindinfo[i].buffer_length = sizeof(MYSQL_TIME);
+                break;
+            default:
                 bindinfo[i].buffer_type = col->type;
                 bindinfo[i].buffer_length = col->length;
+                break;
             }
             bindinfo[i].buffer = rtlMalloc(bindinfo[i].buffer_length);
         }
@@ -332,15 +341,83 @@ static bool isInteger(enum_field_types type)
     case MYSQL_TYPE_LONGLONG:
     case MYSQL_TYPE_INT24:
         return true;
+    default:
+        return false;
+    }
+}
+
+static bool isDateTime(enum_field_types type)
+{
+    switch (type)
+    {
     case MYSQL_TYPE_TIMESTAMP:
+    case MYSQL_TYPE_DATETIME:
     case MYSQL_TYPE_DATE:
     case MYSQL_TYPE_TIME:
-        return false; // Who knows...
+        return true;
+    default:
+        return false;
+    }
+}
+
+static bool isString(enum_field_types type)
+{
+    switch (type)
+    {
+    case MYSQL_TYPE_TINY_BLOB:
+    case MYSQL_TYPE_MEDIUM_BLOB:
+    case MYSQL_TYPE_LONG_BLOB:
+    case MYSQL_TYPE_BLOB:
+    case MYSQL_TYPE_STRING:
+    case MYSQL_TYPE_VAR_STRING:
+        return true;
     default:
         return false;
     }
 }
 
+
+static unsigned __int64 getDateTimeValue(const MYSQL_BIND &bound)
+{
+    const MYSQL_TIME * time = (const MYSQL_TIME *) bound.buffer;
+    switch (bound.buffer_type)
+    {
+    case MYSQL_TYPE_TIMESTAMP:
+    case MYSQL_TYPE_DATETIME:
+        //What format should this be?  Possibly a timestamp_t
+        return (unsigned __int64)((time->year * 10000) + (time->month * 100) + (time->day)) * 1000000 +
+               (time->hour * 10000) + (time->minute * 100) + (time->second);
+    case MYSQL_TYPE_DATE:
+        return (time->year * 10000) + (time->month * 100) + (time->day);
+    case MYSQL_TYPE_TIME:
+        return (time->hour * 10000) + (time->minute * 100) + (time->second);
+    default:
+        throwUnexpected();
+    }
+}
+
+static void getDateTimeText(const MYSQL_BIND &bound, size32_t &chars, char * &result)
+{
+    const MYSQL_TIME * time = (const MYSQL_TIME *) bound.buffer;
+    char temp[20];
+    switch (bound.buffer_type)
+    {
+    case MYSQL_TYPE_TIMESTAMP:
+    case MYSQL_TYPE_DATETIME:
+        _snprintf(temp, sizeof(temp), "%4u-%02u-%02u %02u:%02u:%02u", time->year, time->month, time->day, time->hour, time->minute, time->second);
+        break;
+    case MYSQL_TYPE_DATE:
+        _snprintf(temp, sizeof(temp), "%4u-%02u-%02u", time->year, time->month, time->day);
+        break;
+    case MYSQL_TYPE_TIME:
+        _snprintf(temp, sizeof(temp), "%02u:%02u:%02u", time->hour, time->minute, time->second);
+        break;
+    default:
+        throwUnexpected();
+    }
+    rtlStrToStrX(chars, result, strlen(temp), temp);
+}
+
 static bool getBooleanResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 {
     if (*bound.is_null)
@@ -361,13 +438,7 @@ static void getDataResult(const RtlFieldInfo *field, const MYSQL_BIND &bound, si
         rtlStrToDataX(chars, result, p.resultChars, p.stringResult);
         return;
     }
-    if (bound.buffer_type == MYSQL_TYPE_TINY_BLOB ||
-        bound.buffer_type == MYSQL_TYPE_MEDIUM_BLOB ||
-        bound.buffer_type == MYSQL_TYPE_LONG_BLOB ||
-        bound.buffer_type == MYSQL_TYPE_BLOB ||
-        bound.buffer_type == MYSQL_TYPE_STRING ||
-        bound.buffer_type == MYSQL_TYPE_VAR_STRING
-       )
+    if (isString(bound.buffer_type))
         rtlStrToDataX(chars, result, *bound.length, bound.buffer);   // This feels like it may not work to me - will preallocate rather larger than we want
     else
         typeError("blob", field);
@@ -405,6 +476,8 @@ static __int64 getSignedResult(const RtlFieldInfo *field, const MYSQL_BIND &boun
         NullFieldProcessor p(field);
         return p.intResult;
     }
+    if (isDateTime(bound.buffer_type))
+        return getDateTimeValue(bound);
     if (isInteger(bound.buffer_type))
     {
         if (bound.is_unsigned)
@@ -423,6 +496,8 @@ static unsigned __int64 getUnsignedResult(const RtlFieldInfo *field, const MYSQL
         NullFieldProcessor p(field);
         return p.uintResult;
     }
+    if (isDateTime(bound.buffer_type))
+        return getDateTimeValue(bound);
     if (!isInteger(bound.buffer_type))
         typeError("integer", field);
     if (bound.is_unsigned)
@@ -439,8 +514,15 @@ static void getStringResult(const RtlFieldInfo *field, const MYSQL_BIND &bound,
         rtlStrToStrX(chars, result, p.resultChars, p.stringResult);
         return;
     }
-    if (bound.buffer_type != MYSQL_TYPE_STRING && bound.buffer_type != MYSQL_TYPE_VAR_STRING)
+
+    if (isDateTime(bound.buffer_type))
+    {
+        getDateTimeText(bound, chars, result);
+        return;
+    }
+    if (!isString(bound.buffer_type))
         typeError("string", field);
+
     const char *text = (const char *) bound.buffer;
     unsigned long bytes = *bound.length;
     unsigned numchars = rtlUtf8Length(bytes, text);  // MORE - is it a good assumption that it is utf8 ? Depends how the database is configured I think
@@ -455,8 +537,15 @@ static void getUTF8Result(const RtlFieldInfo *field, const MYSQL_BIND &bound, si
         rtlUtf8ToUtf8X(chars, result, p.resultChars, p.stringResult);
         return;
     }
-    if (bound.buffer_type != MYSQL_TYPE_STRING && bound.buffer_type != MYSQL_TYPE_VAR_STRING)
+
+    if (isDateTime(bound.buffer_type))
+    {
+        getDateTimeText(bound, chars, result);
+        return;
+    }
+    if (!isString(bound.buffer_type))
         typeError("string", field);
+
     const char *text = (const char *) bound.buffer;
     unsigned long bytes = *bound.length;
     unsigned numchars = rtlUtf8Length(bytes, text);  // MORE - is it a good assumption that it is utf8 ? Depends how the database is configured I think

system/include/platform.h

@@ -276,6 +276,7 @@ typedef unsigned long MaxCard;
 #define _llseek     ::lseek
 #define _lseeki64   ::lseek
 #define _vsnprintf  vsnprintf
+#define _snprintf   snprintf
 #define strnicmp    strncasecmp
 #define INFINITE    0xFFFFFFFF
 

testing/regress/ecl/cassandra-simple.ecl

@@ -247,16 +247,10 @@ integer testCassandraCount() := EMBED(cassandra : server(server),user('rchapman'
   SELECT COUNT(*) from tbl1;
 ENDEMBED;
 
-dataset(childrec) testCassandraCountPaged0() := EMBED(cassandra : server(server),user('rchapman'),keyspace('test'),pageSize(0))
+dataset(childrec) testCassandraCountPaged(UNSIGNED ps) := EMBED(cassandra : server(server),user('rchapman'),keyspace('test'),pageSize(ps))
   SELECT name, value, boolval, r8, r4,d,ddd,u1,u2,a,set1,list1,map1 from tbl1;
 ENDEMBED;
 
-dataset(childrec) testCassandraCountPaged101() := EMBED(cassandra : server(server),user('rchapman'),keyspace('test'),pageSize(101))
-  SELECT name, value, boolval, r8, r4,d,ddd,u1,u2,a,set1,list1,map1 from tbl1;
-ENDEMBED;
-dataset(childrec) testCassandraCountPaged100000() := EMBED(cassandra : server(server),user('rchapman'),keyspace('test'),pageSize(100000))
-  SELECT name, value, boolval, r8, r4,d,ddd,u1,u2,a,set1,list1,map1 from tbl1;
-ENDEMBED;
 // Execute the tests
 
 sequential (
@@ -285,8 +279,8 @@ sequential (
   OUTPUT(testCassandraDSParam(PROJECT(init, extractName(LEFT)))),
   testCassandraBulk,
   OUTPUT(testCassandraCount()),
-  OUTPUT(COUNT(testCassandraCountPaged0())),
-  OUTPUT(COUNT(testCassandraCountPaged101())),
-  OUTPUT(COUNT(testCassandraCountPaged100000())),
+  OUTPUT(COUNT(testCassandraCountPaged(0))),
+  OUTPUT(COUNT(testCassandraCountPaged(101))),
+  OUTPUT(COUNT(testCassandraCountPaged(100000))),
   OUTPUT('Done');
 );

testing/regress/ecl/key/mysqlembed.xml

@@ -1,24 +1,24 @@
 <Dataset name='Result 1'>
- <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
- <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
- <Row><name>nulls</name><value>99999</value><boolval>true</boolval><r8>99.98999999999999</r8><r4>999.989990234375</r4><d>393939393939</d><ddd>9.99</ddd><u1>9999 ß</u1><u2>9999 ßßß</u2></Row>
- <Row><name>utf8test</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>1963-11-22 12:30:00</dt></Row>
+ <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2><dt>2015-12-25 01:23:45</dt></Row>
+ <Row><name>nulls</name><value>99999</value><boolval>true</boolval><r8>99.98999999999999</r8><r4>999.989990234375</r4><d>393939393939</d><ddd>9.99</ddd><u1>9999 ß</u1><u2>9999 ßßß</u2><dt>1963-11-22 12:30:00</dt></Row>
+ <Row><name>utf8test</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>2019-02-01 23:59:59</dt></Row>
 </Dataset>
 <Dataset name='Result 2'>
  <Row><Result_2>name1</Result_2></Row>
 </Dataset>
 <Dataset name='Result 3'>
- <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>1963-11-22 12:30:00</dt></Row>
 </Dataset>
 <Dataset name='Result 4'>
  <Row><Result_4>utf8test</Result_4></Row>
 </Dataset>
 <Dataset name='Result 5'>
- <Row><name>utf8test</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
+ <Row><name>utf8test</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>2019-02-01 23:59:59</dt></Row>
 </Dataset>
 <Dataset name='Result 6'>
- <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
- <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2><dt>1963-11-22 12:30:00</dt></Row>
+ <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2><dt>2015-12-25 01:23:45</dt></Row>
 </Dataset>
 <Dataset name='Result 7'>
  <Row><Result_7>2</Result_7></Row>
@@ -41,3 +41,9 @@
 <Dataset name='Result 13'>
  <Row><Result_13>Straße  </Result_13></Row>
 </Dataset>
+<Dataset name='Result 14'>
+ <Row><dt1>19631122123000</dt1><dt2>1963-11-22 12:30:00</dt2></Row>
+ <Row><dt1>20151225012345</dt1><dt2>2015-12-25 01:23:45</dt2></Row>
+ <Row><dt1>0</dt1><dt2>                   </dt2></Row>
+ <Row><dt1>20190201235959</dt1><dt2>2019-02-01 23:59:59</dt2></Row>
+</Dataset>

testing/regress/ecl/mysqlembed.ecl

@@ -32,7 +32,8 @@ childrec := RECORD
    DATA d {default (D'999999')},
    DECIMAL10_2 ddd {default(9.99)},
    UTF8 u1 {default(U'9999 ß')},
-   UNICODE8 u2 {default(U'9999 ßßßß')}
+   UNICODE8 u2 {default(U'9999 ßßßß')},
+   STRING19 dt {default('1963-11-22 12:30:00')},
 END;
 
 stringrec := RECORD
@@ -44,18 +45,18 @@ stringrec extractName(childrec l) := TRANSFORM
 END;
 
 init := DATASET([{'name1', 1, true, 1.2, 3.4, D'aa55aa55', 1234567.89, U'Straße', U'Straße'},
-                 {'name2', 2, false, 5.6, 7.8, D'00', -1234567.89, U'là', U'là'}], childrec);
+                 {'name2', 2, false, 5.6, 7.8, D'00', -1234567.89, U'là', U'là', '2015-12-25 01:23:45' }], childrec);
 
 drop() := EMBED(mysql : user('rchapman'),database('test'))
   DROP TABLE IF EXISTS tbl1;
 ENDEMBED;
 
 create() := EMBED(mysql : user('rchapman'),database('test'))
-  CREATE TABLE tbl1 ( name VARCHAR(20), value INT, boolval TINYINT, r8 DOUBLE, r4 FLOAT, d BLOB, ddd DECIMAL(10,2), u1 VARCHAR(10), u2 VARCHAR(10) );
+  CREATE TABLE tbl1 ( name VARCHAR(20), value INT, boolval TINYINT, r8 DOUBLE, r4 FLOAT, d BLOB, ddd DECIMAL(10,2), u1 VARCHAR(10), u2 VARCHAR(10), dt DATETIME );
 ENDEMBED;
 
 initialize(dataset(childrec) values) := EMBED(mysql : user(myUser),database(myDb))
-  INSERT INTO tbl1 values (?, ?, ?, ?, ?, ?, ?, ?, ?);
+  INSERT INTO tbl1 values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
 ENDEMBED;
 
 initializeNulls() := EMBED(mysql : user('rchapman'),database(myDb))
@@ -63,7 +64,7 @@ initializeNulls() := EMBED(mysql : user('rchapman'),database(myDb))
 ENDEMBED;
 
 initializeUtf8() := EMBED(mysql : user(myUser),database('test'))
-  INSERT INTO tbl1 values ('utf8test', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße');
+  INSERT INTO tbl1 values ('utf8test', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße', '2019-02-01 23:59:59');
 ENDEMBED;
 
 dataset(childrec) testMySQLDS() := EMBED(mysql : user('rchapman'),database('test'))
@@ -126,6 +127,16 @@ UNICODE testMySQLUnicode() := EMBED(mysql : user('rchapman'),database('test'))
   SELECT max(u2) from tbl1;
 ENDEMBED;
 
+datetimerec := RECORD
+   UNSIGNED8 dt1;
+   STRING19 dt2;
+END;
+
+dataset(datetimerec) testMySQLDateTime() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT dt, dt from tbl1;
+ENDEMBED;
+
+
 sequential (
   drop(),
   create(),
@@ -144,5 +155,6 @@ sequential (
   OUTPUT(testMySQLReal4()),
   OUTPUT(testMySQLData()),
   OUTPUT(testMySQLUtf8()),
-  OUTPUT(testMySQLUnicode())
+  OUTPUT(testMySQLUnicode()),
+  OUTPUT(testMySQLDateTime())
 );

thorlcr/master/thgraphmanager.cpp

@@ -795,6 +795,12 @@ void abortThor(IException *e, unsigned errCode, bool abortCurrentJob)
         LOG(MCdebugProgress, thorJob, "aborting any current active job");
         if (jM)
             jM->fireException(e);
+        if (errCode == TEC_Clean)
+        {
+            LOG(MCdebugProgress, thorJob, "Removing sentinel upon normal shutdown");
+            Owned<IFile> sentinelFile = createSentinelTarget();
+            removeSentinelFile(sentinelFile);
+        }
     }
 }