Merge branch 'candidate-6.0.10' into candidate-6.2.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

docs/ECLLanguageReference/ECLR_mods/Value-Unicode.xml

@@ -6,7 +6,7 @@
 
   <para><emphasis role="bold">UNICODE<indexterm>
       <primary>UNICODE</primary>
-    </indexterm>[</emphasis><emphasis>locale</emphasis><emphasis
+    </indexterm>[</emphasis><emphasis>_locale</emphasis><emphasis
   role="bold">][</emphasis><emphasis>n</emphasis><emphasis
   role="bold">]<indexterm>
       <primary>UNICODE value type</primary>
@@ -30,8 +30,8 @@
         // utf-16-encoded string
 UNICODE4 MyUnicodeString := U'abcd';
         // same as: (UNICODE)'abcd'
-UNICODEde5 MyUnicodeString := U'abcd\353';
+UNICODE_de5 MyUnicodeString := U'abcd\353';
         // becomes 'abcdë' with a German locale
-UNICODEde5 MyUnicodeString := U'abcdë';
+UNICODE_de5 MyUnicodeString := U'abcdë';
         // same as previous example</programlisting>
 </sect1>

docs/HPCCClientTools/CT_Mods/CT_ECL_IDE.xml

@@ -210,22 +210,20 @@
             <listitem>
               <para>Specify your Working Folder.</para>
 
-              <para>When you are running your locally, your queries are
+              <para>When you submit to a Local target, your queries are
               compiled and stored in this location.</para>
 
-              <para>To compile a query locally rather than on the thor or
-              hthor of your environment, select <emphasis
-              role="bold">Local</emphasis> as the <emphasis
-              role="bold">Target</emphasis> before pressing <emphasis
-              role="bold">Submit</emphasis> on the <emphasis
+              <para>To compile a query locally rather than on a cluster in
+              your environment, select <emphasis role="bold">Local</emphasis>
+              as the <emphasis role="bold">Target</emphasis> before pressing
+              <emphasis role="bold">Submit</emphasis> on the <emphasis
               role="bold">Builder</emphasis> window.</para>
 
               <!-- -->
 
               <para>If you are running under Windows and want to compile
               locally, install the Microsoft VS 2008 C++ compiler (either
-              Express or Professional edition) and Linux users need
-              GCC.</para>
+              Express or Professional edition). Linux users need GCC.</para>
             </listitem>
 
             <listitem>

docs/HPCCMonitoring/HPCCMonitoringAndReporting.xml

@@ -148,7 +148,6 @@
           <para>System Load</para>
         </listitem>
       </itemizedlist></para>
-
   </chapter>
 
   <chapter id="Ganglya_Overview">
@@ -1191,6 +1190,18 @@
                   </row>
 
                   <row>
+                    <entry>-set_esp_username_pw</entry>
+
+                    <entry>Set specific login credentials for ESP checks. All
+                    fields are required (esp name, user name, password). Can
+                    be specified more than once to support multiple ESP
+                    servers.</entry>
+
+                    <entry><para>&lt;esp name&gt;</para><para>&lt;user
+                    name&gt;</para><para>&lt;password&gt;</para></entry>
+                  </row>
+
+                  <row>
                     <entry>-override_check_all_disks</entry>
 
                     <entry>check_all_disk plugin name</entry>
@@ -1421,11 +1432,8 @@
       </sect2>
     </sect1>
 
-        
-            <xi:include href="HPCCMonitoring/MonRep-Mods/NagiosInECLWa.xml"
-                        xpointer="Nagios_in_ECLWatch"
-                        xmlns:xi="http://www.w3.org/2001/XInclude" />
-        
-
+    <xi:include href="HPCCMonitoring/MonRep-Mods/NagiosInECLWa.xml"
+                xpointer="Nagios_in_ECLWatch"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
   </chapter>
 </book>

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -1109,6 +1109,32 @@ lock=/var/lock/HPCCSystems</programlisting>
             </varlistentry>
           </variablelist></para>
       </sect2>
+
+      <sect2 id="ConfiguringRemoteAccessOverTLS">
+        <title>Remote Access over TLS</title>
+
+        <para>Configuring your system for remote file access over Transport
+        Layer Security (TLS) requires modifying the <emphasis
+        role="bold">dafilesrv</emphasis> setting in the
+        <emphasis>environment.conf</emphasis> file. </para>
+
+        <para>To do this either uncomment (if they are already there), or add
+        the following lines to the <emphasis>environment.conf</emphasis> file.
+        Then set the values as appropriate for your system. </para>
+
+        <para><programlisting>#enable SSL for dafilesrv remote file access
+dfsUseSSL=true
+dfsSSLCertFile=/certfilepath/certfile
+dfsSSLPrivateKeyFile=/keyfilepath/keyfile</programlisting>Set the <emphasis
+        role="blue">dfsUseSSL=true</emphasis> and set the value for the paths
+        to point to the certificate and key file paths on your system. Then
+        deploy the <emphasis>environment.conf</emphasis> file (and cert/key
+        files) to all nodes as appropriate. </para>
+
+        <para>When dafilesrv is enabled for TLS (port 7600), it can still
+        connect over a non-TLS connection (port 7100) to allow legacy clients
+        to work.</para>
+      </sect2>
     </sect1>
 
     <!--Inclusions-As-Sect1-->
@@ -1199,6 +1225,7 @@ lock=/var/lock/HPCCSystems</programlisting>
           </variablelist></para>
       </sect2>
     </sect1>
+
     <xi:include href="HPCCSystemAdmin/SA-Mods/CassandraWUServer.xml"
                 xpointer="CassandraWUStorage"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -1761,12 +1788,11 @@ heapUseTransparentHugePages</programlisting>
       </sect2>
     </sect1>
 
-     <xi:include href="RoxieReference/RoxieRefMods/RoxieCapacityPlanning.xml"
+    <xi:include href="RoxieReference/RoxieRefMods/RoxieCapacityPlanning.xml"
                 xpointer="Capacity_Planning"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
- 
 
-     <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
+    <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
                 xpointer="Sample_Sizings"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
   </chapter>

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml

@@ -297,6 +297,13 @@
     role="bold">initldap</emphasis> utility to create the required default
     HPCC Admin user on your LDAP server.</para>
 
+    <para>If you choose to use LDAP authentication you must enable LDAP
+    security in your HPCC System configuration. With LDAP security enabled on
+    your system you can then choose to enable file scope security. You can
+    choose to use LDAP authentication without enabling file scope security.
+    The following sections describe how to enable LDAP authentication and file
+    scope security for your HPCC system.</para>
+
     <!--***Note: (9/2014) Adding Documentation for initLdap.*** -->
 
     <sect3 id="UsingLDAP_ConnectToConfigMgr">
@@ -606,9 +613,8 @@
       <listitem>
         <?dbfo keep-together="always"?>
 
-        <para>To enable the file scope permissions, configure security for the
-        Dali Server. If you are not interested in file scope permissions you
-        can skip this step.</para>
+        <para>To enable the file scope permissions, configure the file scope
+        security for the Dali Server.</para>
 
         <para>In the Navigator pane, click on the <emphasis role="bold">Dali
         Server – mydali </emphasis><graphic
@@ -628,8 +634,35 @@
             </listitem>
 
             <listitem>
+              <para>Set the <emphasis role="bold">checkScopeScans</emphasis>
+              value to <emphasis>true</emphasis>.</para>
+
+              <para>Only set this value to true when you want file scope
+              security enabled. Security settings can have three
+              states.</para>
+
+              <itemizedlist>
+                <listitem>
+                  <para>None, no authentication and no file scope
+                  security.</para>
+                </listitem>
+
+                <listitem>
+                  <para>LDAP security for authentication only, without
+                  enabling file scope security.</para>
+                </listitem>
+
+                <listitem>
+                  <para>LDAP authentication and file scope security
+                  enabled.</para>
+                </listitem>
+              </itemizedlist>
+            </listitem>
+
+            <listitem>
               <para>Change the LDAP values as appropriate to match the
-              settings in your LDAP server.</para>
+              settings in your LDAP server component in configuration
+              manager.</para>
 
               <para>For example, change the <emphasis
               role="bold">ldapServer</emphasis> to the value you gave your
@@ -643,8 +676,8 @@
               are supplied. This is similar to a guest account, so it should
               be an account with <emphasis role="bold">very</emphasis> limited
               access, if used at all. To disable access without credentials,
-              leave <emphasis role="bold">filesDefaultUser</emphasis> blank.
-              </para>
+              leave <emphasis role="bold">filesDefaultUser</emphasis>
+              blank.</para>
 
               <para>The <emphasis role="bold">filesDefaultPassword</emphasis>
               is the password for that account.</para>
@@ -672,14 +705,17 @@
 
             <listitem>
               <para>Locate the <emphasis role="bold">ldapUser</emphasis> field
-              and verify that there is a "<emphasis>roxie</emphasis>"
+              and verify that there is a valid HPCC user who is a member of
+              the Authenticated Users group on your LDAP server. For example,
+              the "<emphasis>roxie</emphasis>" user assumes that the
+              "<emphasis>roxie</emphasis>" user is a valid HPCC authenticated
               user.</para>
             </listitem>
 
             <listitem>
-              <para>You can add password security for Roxie by adding it to
-              the <emphasis role="bold">ldapPassword</emphasis> field on the
-              same tab.</para>
+              <para>Add the password security for Roxie by adding it to the
+              <emphasis role="bold">ldapPassword</emphasis> field on the same
+              tab.</para>
             </listitem>
           </orderedlist></para>
       </listitem>
@@ -699,12 +735,12 @@
               fileref="../../images/caution.png" /></entry>
 
               <entry><para>In order to run Roxie queries with File Scope
-              security, ensure that the roxie user is created in the list of
+              security, ensure that a Roxie user is created in the list of
               authenticated users.</para>In the following section, <link
               linkend="Adding_Users"><emphasis>Adding and editing
-              users</emphasis></link>, add "<emphasis>roxie</emphasis>" as a
-              user and make sure the password is the same as the one entered
-              in Configuration Manager.</entry>
+              users</emphasis></link>, add the <emphasis>roxie</emphasis> user
+              and make sure that password is the same as the one entered in
+              Configuration Manager.</entry>
             </row>
           </tbody>
         </tgroup>

plugins/javaembed/javaembed.cpp

@@ -33,6 +33,10 @@
 #include "thorxmlwrite.hpp"
 #include "esdl_def.hpp"
 
+#ifndef _WIN32
+ #include <sys/resource.h>
+#endif
+
 #ifdef _WIN32
 #define EXPORT __declspec(dllexport)
 #else
@@ -134,6 +138,21 @@ public:
 
         // Options we know we always want set
         optionStrings.append("-Xrs");
+#ifdef RLIMIT_STACK
+        // JVM has a habit of reducing the stack limit on main thread to 1M - probably dates back to when it was actually an increase...
+        StringBuffer stackOption("-Xss");
+        struct rlimit limit;
+        rlim_t slim = 0;
+        if (getrlimit (RLIMIT_STACK, &limit)==0)
+            slim = limit.rlim_cur;
+        if (!slim)
+            slim = 8*1024*1024;
+        if (slim >= 1*1024*1024)
+        {
+            stackOption.append((__uint64) slim);
+            optionStrings.append(stackOption);
+        }
+#endif
 
         // These may be useful for debugging
 #ifdef _DEBUG

roxie/ccd/ccdserver.cpp

@@ -2121,7 +2121,8 @@ public:
             DBGLOG("RecordPullerThread::stop");
         {
             CriticalBlock c(crit); // stop is called on our consumer's thread. We need to take care calling stop for our input to make sure it is not in mid-nextRow etc etc.
-            inputStream->stop();
+            if (inputStream)
+                inputStream->stop();
         }
         RestartableThread::join();
     }