Merge branch 'candidate-5.2.0' into candidate-5.4.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -76,6 +76,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(DOCS_DRUPAL "Create Drupal HTML Docs" OFF)
   option(DOCS_EPUB "Create EPUB Docs" OFF)
   option(DOCS_MOBI "Create Mobi Docs" OFF)
+  option(DOCS_AUTO "DOCS automation" OFF)
   option(USE_RESOURCE "Use resource download in ECLWatch" OFF)
   option(GENERATE_COVERAGE_INFO "Generate coverage info for gcov" OFF)
   option(USE_SIGNED_CHAR "Build system with default char type is signed" OFF)
@@ -113,6 +114,9 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
 
   if ( MAKE_DOCS_ONLY AND NOT CLIENTTOOLS_ONLY )
       set( MAKE_DOCS ON )
+      if ( USE_DOCS_AUTO )
+        set ( DOCS_AUTO  ON)
+      endif()
   endif()
 
   if ( CLIENTTOOLS_ONLY )
@@ -494,6 +498,12 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     ELSE()
       message(FATAL_ERROR "FOP requested but package not found")
     ENDIF()
+
+    if (DOCS_AUTO)
+       if ("${CONFIGURATOR_DIRECTORY}" STREQUAL "")
+         set(CONFIGURATOR_DIRECTORY ${HPCC_SOURCE_DIR}/../configurator)
+       endif()
+    endif()
   ENDIF(MAKE_DOCS)
 
   IF ( NOT MAKE_DOCS_ONLY )

cmake_modules/docMacros.cmake

@@ -139,3 +139,23 @@ MACRO(FILE_LIST_GENERATOR outxml filename linkname description)
 	message("---- FILE_LIST_GENERATOR: Adding ${filename}")
 	set(${outxml} "${${outxml}}${xmlout}")
 ENDMACRO(FILE_LIST_GENERATOR)
+
+MACRO(XSD_TO_XML _xsd_files _in_dir _out_dir)
+
+        set(_xml_files)
+	ADD_CUSTOM_COMMAND(
+		COMMAND mkdir -p ${_out_dir}
+		OUTPUT ${_out_dir}
+                )
+        foreach(_xsd_file ${_xsd_files})
+            STRING(REGEX REPLACE "(.*).xsd" "\\1.xml" _xml_file "${_xsd_file}")
+	    ADD_CUSTOM_COMMAND(
+		COMMAND ./configurator -doc -use ${_xsd_file} -b ${_in_dir} -t ${_out_dir}
+		OUTPUT ${_out_dir}/${_xml_file}
+                WORKING_DIRECTORY ${CONFIGURATOR_DIRECTORY}
+		DEPENDS ${_out_dir} ${_in_dir}/${_xsd_file} ${_in_dir}/environment.xsd
+		)
+            list(APPEND _xml_files ${_out_dir}/${_xml_file})
+        endforeach()
+	ADD_CUSTOM_TARGET("xsd_to_xml" ALL  DEPENDS ${_out_dir} ${_xml_files} )
+ENDMACRO(XSD_TO_XML)

common/workunit/workunit.cpp

@@ -690,7 +690,8 @@ public:
         }
 
         //Don't lock the statistics while we iterate - any partial updates must not cause problems
-        conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT));
+        if (daliClientActive())
+            conn.setown(querySDS().connect(rootPath.str(), myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT));
 
         if (conn && !singleGraph)
             graphIter.setown(conn->queryRoot()->getElements("*"));

docs/Automation/CMakeLists.txt

@@ -0,0 +1,47 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+SET(XSD_SOURCE_DIR ${HPCC_SOURCE_DIR}/initfiles/componentfiles/configxml)
+SET(XSD_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}/xsd )
+SET(XML_TARGET_DIR ${CMAKE_CURRENT_BINARY_DIR}/xml )
+
+FILE(MAKE_DIRECTORY ${XSD_TARGET_DIR})
+
+# get all *.xsd.in and *.xsd files
+FILE(GLOB XSD_TEMPLATE_FILES RELATIVE "${XSD_SOURCE_DIR}" "${XSD_SOURCE_DIR}/*.xsd.in")
+FILE(GLOB XSD_FILES RELATIVE "${XSD_SOURCE_DIR}" "${XSD_SOURCE_DIR}/*.xsd")
+
+# Skip some files which cause problem
+LIST(REMOVE_ITEM XSD_FILES "esp_service_wsecl2.xsd")
+
+# process and copy *.xsd.in files
+FOREACH( XSD_TEMPLATE_FILE  ${XSD_TEMPLATE_FILES} )
+   STRING(REGEX REPLACE "(.*).in" "\\1" XSD_FILE "${XSD_TEMPLATE_FILE}")
+   CONFIGURE_FILE(${XSD_SOURCE_DIR}/${XSD_TEMPLATE_FILE} ${XSD_TARGET_DIR}/${XSD_FILE})
+ENDFOREACH()
+
+# copy *.xsd.in files
+FOREACH( XSD_FILE  ${XSD_FILES} )
+   CONFIGURE_FILE(${XSD_SOURCE_DIR}/${XSD_FILE} ${XSD_TARGET_DIR}/${XSD_FILE} COPYONLY)
+ENDFOREACH()
+
+# create complete *.xsd file list
+FOREACH( XSD_TEMPLATE_FILE  ${XSD_TEMPLATE_FILES} )
+   STRING(REGEX REPLACE "(.*).in" "\\1" XSD_FILE "${XSD_TEMPLATE_FILE}")
+   SET(XSD_FILES ${XSD_FILES} ${XSD_FILE})
+ENDFOREACH()
+
+# create MakeFiles to generate xsd file to xml file
+XSD_TO_XML("${XSD_FILES}" ${XSD_TARGET_DIR} ${XML_TARGET_DIR})

docs/CMakeLists.txt

@@ -49,6 +49,7 @@ add_subdirectory(resources)
 
 
 # Doc dirs to include.
+add_subdirectory(DynamicESDL)
 add_subdirectory(ECLLanguageReference)
 add_subdirectory(ECLProgrammersGuide)
 add_subdirectory(ECLStandardLibraryReference)
@@ -71,6 +72,11 @@ add_subdirectory(ECLPluginForEclipse)
 add_subdirectory(ECLScheduler)
 add_subdirectory(ECLWatch)
 
+# Docs automation
+if (DOCS_AUTO)
+   add_subdirectory(Automation)
+endif()
+
 
 #WIP -  Docs
 #add_subdirectory(RuningHPCCinAmazonWebServicesEC2)

docs/DynamicESDL/CMakeLists.txt

@@ -0,0 +1,18 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems(r).
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+DOCBOOK_TO_PDF( ${FO_XSL} DynamicESDL_Includer.xml "DynamicESDL" "DESDL-Mods")
+

docs/DynamicESDL/DESDL-Mods/ESDLAttributes.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+
+    <sect1 id="ESDL_Attributes">
+    <title>ESDL Attributes</title>
+
+    <para>You can use ESDL attributes to extend and override the default
+    behavior of an ESDL definition. For example, adding a max_len(n) to a
+    string defines the string will only need to store a certain number of
+    characters.</para>
+
+    <para>Many attributes are treated as hints that may have more effect on
+    some implementations than others. For example, max_len(n) will affect
+    generated ECL code, but is ignored when generating C++.</para>
+    
+    </sect1>

docs/DynamicESDL/DESDL-Mods/ESDLDoc.xml

@@ -0,0 +1,644 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<book lang="en_US">
+  <bookinfo>
+    <author>
+      <surname>DeFabia, Jim (RIS-BCT)</surname>
+    </author>
+
+    <date>2015-01-14</date>
+
+    <corpname>LexisNexis Risk Solutions</corpname>
+  </bookinfo>
+
+  <chapter>
+    <title>ESDL Language Reference</title>
+
+    <sect1 id="ESP_Struct">
+      <title>ESP<emphasis role="bold">struct</emphasis></title>
+
+      <para>ESPstruct is a set of elements grouped together under one name.
+      These elements, known as <emphasis>members</emphasis>, can have
+      different types and different lengths. Structures can be nested and
+      support inheritance.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>
+ESPstruct NameBlock
+
+{
+    string FirstName;
+    string MiddleName;
+    string LastName;
+    int Age;
+};
+   </programlisting>
+    </sect1>
+
+    <sect1 id="ESPrequest">
+      <title><emphasis role="bold">ESPrequest</emphasis></title>
+
+      <para>The request structure for a method.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>
+ESPrequest  MyQueryRequest
+
+{
+    string FirstName;
+    string MiddleName;
+    string LastName;
+    string Sortby;
+ 
+    bool Descending(false);
+};
+   </programlisting>
+    </sect1>
+
+    <sect1 id="ESPresponse">
+      <title><emphasis role="bold">ESPresponse</emphasis></title>
+
+      <para>The response structure for a method.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting> 
+ESPresponse  MyQueryResponse
+ 
+{
+    string FirstName;
+    string MiddleName;
+    string LastName;
+ 
+ 
+};
+   </programlisting>
+    </sect1>
+
+    <sect1 id="ESParray">
+      <title><emphasis role="bold">ESParray</emphasis></title>
+
+      <para>A structure for unbounded arrays. Arrays support inheritance and
+      can be nested.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>
+ESPstruct NameBlock
+  
+{
+    string FirstName;
+    string MiddleName;
+    string LastName;
+ 
+    int Age;
+};
+ 
+ESParray&lt;ESPstruct NameBlock&gt; Names;
+   </programlisting>
+    </sect1>
+
+    <sect1 id="ESPenum">
+      <title><emphasis role="bold">ESPenum</emphasis></title>
+
+      <para>A structure containing an enumerated value.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>
+ESPenum EyeColors : string
+  
+{
+  Brn("Brown"),
+  Blu("Blue"),
+  Grn("Green"),
+};
+ 
+
+ESPstruct Person
+
+{
+  string FirstName;
+  string MiddleName;
+  string LastName;
+
+  ESPenum EyeColors EyeColor;
+};
+   </programlisting>
+    </sect1>
+
+    <sect1 id="ESPinclude">
+      <title><emphasis role="bold">ESPinclude</emphasis></title>
+
+      <para>ESPinclude allows you to include an external ESDLfile. This is
+      similar to the #include ststement.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>
+ESPinclude(ws_topology);
+  </programlisting>
+    </sect1>
+
+    <sect1 id="ESPservice">
+      <title><emphasis role="bold">ESPservice</emphasis></title>
+
+      <para>This defines a Web service. Once defined, you can add a service to
+      an ESP server’s configuration to expose its methods and ESDL interfaces.
+      An ESPservice should contain one or more method definitions.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>
+ESPservice MyService
+
+{
+
+  ESPmethod MyMethod1(MyMethod1Request, MyMethod1Response);
+  ESPmethod MyMethod2(MyMethod2Request, MyMethod2Response);
+
+};
+
+
+   </programlisting>
+    </sect1>
+
+    <sect1 id="ESPmethod">
+      <title><emphasis role="bold">ESPmethod</emphasis></title>
+
+      <para>This defines a method definition you can reference in an
+      ESPservice structure. The method definition should contain references to
+      a previously defined ESPrequest and ESPresponse.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>
+ESPservice MyService
+
+{
+  ESPmethod MyMethod1(MyMethod1Request, MyMethod1Response);
+  ESPmethod MyMethod2(MyMethod2Request, MyMethod2Response);
+};
+
+   </programlisting>
+    </sect1>
+  </chapter>
+
+  <chapter>
+    <title>ESDL Datatypes</title>
+
+    <sect1 id="boolean_bool">
+      <title><emphasis role="bold">boolean</emphasis><emphasis
+      role="bold">bool</emphasis></title>
+
+      <para>A boolean or logical data type having one of two possible values:
+      true (1) or false (0).</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>boolean includeFlag; </programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Data_string">
+      <title><emphasis role="bold">string</emphasis></title>
+
+      <para>A data type consisting of sequence of alphanumeric
+      characters.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>string FirstName;
+</programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Data_int">
+      <title><emphasis role="bold">int</emphasis></title>
+
+      <para>An integer value.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>int Age;   </programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Data_int64">
+      <title><emphasis role="bold">int64</emphasis></title>
+
+      <para>A 64-bit integer value</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>int64 Iterations;
+</programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Data_float">
+      <title><emphasis role="bold">float</emphasis></title>
+
+      <para>A 4-byte floating point or real number.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>float Temperature;
+</programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Data_double">
+      <title><emphasis role="bold">double</emphasis></title>
+
+      <para>An 8-byte floating point or real number.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>float Temperature;   </programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Data_binary">
+      <title><emphasis role="bold">binary</emphasis></title>
+
+      <para>A data type containing binary data, similar to a BLOB .</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>binary RetinaScanSample;   </programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Data_ECLTypeMapping">
+      <title><emphasis role="bold">ESDL, XSD, and ECL Type
+      Mapping</emphasis></title>
+
+      <para><informaltable colsep="1" frame="all" rowsep="1">
+          <tgroup cols="3">
+            <colspec colwidth="86.75pt" />
+
+            <colspec colwidth="76.50pt" />
+
+            <colspec colwidth="126.00pt" />
+
+            <tbody>
+              <row>
+                <entry><emphasis role="bold">ESDL</emphasis></entry>
+
+                <entry><emphasis role="bold">XSD</emphasis></entry>
+
+                <entry><emphasis role="bold">ECL</emphasis></entry>
+              </row>
+
+              <row>
+                <entry>Bool boolean</entry>
+
+                <entry>bool</entry>
+
+                <entry>boolean</entry>
+              </row>
+
+              <row>
+                <entry>Binary</entry>
+
+                <entry>Base64Binary</entry>
+
+                <entry>String (base64 encoded)</entry>
+              </row>
+
+              <row>
+                <entry>Float</entry>
+
+                <entry>float</entry>
+
+                <entry>REAL4</entry>
+              </row>
+
+              <row>
+                <entry>Double</entry>
+
+                <entry>double</entry>
+
+                <entry>REAL8</entry>
+              </row>
+
+              <row>
+                <entry>Int</entry>
+
+                <entry>int</entry>
+
+                <entry>INTEGER</entry>
+              </row>
+
+              <row>
+                <entry>Int64</entry>
+
+                <entry>long</entry>
+
+                <entry>INTEGER8</entry>
+              </row>
+
+              <row>
+                <entry>String</entry>
+
+                <entry>string</entry>
+
+                <entry>String</entry>
+              </row>
+            </tbody>
+          </tgroup>
+        </informaltable></para>
+    </sect1>
+  </chapter>
+
+  <chapter>
+    <title>ESDL Attributes</title>
+
+    <para>You can use ESDL attributes to extend and override the default
+    behavior of an ESDL definition. For example, adding a max_len(n) to a
+    string defines the string will only need to store a certain number of
+    characters.</para>
+
+    <para>Many attributes are treated as hints that may have more effect on
+    some implementations than others. For example, max_len(n) will affect
+    generated ECL code, but is ignored when generating C++.</para>
+
+    <sect1 id="ESDL_Attributes_max_len">
+      <title><emphasis role="bold">max_len (</emphasis><emphasis
+      role="bold">n</emphasis><emphasis role="bold">)</emphasis></title>
+
+      <para>The <emphasis>max_len</emphasis> attribute specifies the field
+      length for ECL string field.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>[max_len(20)] string City;   </programlisting>
+
+      <para>It means that in ECL, City field is a fixed length of 20 chars.
+      For integer type, the max_len means the maximum size in bytes for the
+      integer (8*max_len bits integer).</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>[max_len(3)] int Age;   </programlisting>
+
+      <para>This generates ECL code:</para>
+
+      <programlisting>integer3 Age{xpath('Age')};   </programlisting>
+
+      <para>This attribute works for ESPenum type, too. The ECL type is also
+      string.</para>
+
+      <programlisting>[max_len(2)] ESPenum StateCode State;   </programlisting>
+
+      <para>Here the StateCode is 2-char state code enumeration.</para>
+
+      <para>This attribute can also be used for ESPstruct, ESPrequest,
+      ESPresponse:</para>
+
+      <programlisting>ESPstruct [max_len(1867)] BpsReportIdentitySlim : Identity  
+{
+   ...
+};   </programlisting>
+
+      <para>This generates ECL code:</para>
+
+      <programlisting>export t_MyQuery := record (share.t_Name), MAXLENGTH(1867)
+{
+}; </programlisting>
+
+      <para>The ECL option <emphasis>MAXLENGTH</emphasis> helps ECL engine
+      better manage memory.</para>
+
+      <para>This does not affect the type in the XSD/WSDL. ESP ignores this
+      attribute when generating the schema. The type for a string is
+      xsd:string which has no length limit. Therefore, the schema stays the
+      same if the field length changes in the Roxie query.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_ecl_max_len">
+      <title>ecl_max_len (<emphasis role="bold">n</emphasis><emphasis
+      role="bold">)</emphasis></title>
+
+      <para>This <emphasis>ecl_max_len</emphasis> attribute tells the ECL
+      generator to use ECL <emphasis>maxlength </emphasis>instead of the
+      regular field length.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>[ecl_max_len(50)] string CompanyName;
+[max_len(6)] string Gender;   </programlisting>
+
+      <para>The generated ECL code is:</para>
+
+      <programlisting>string CompanyName { xpath("CompanyName"),maxlength(50) };
+string6 Gender { xpath("Gender") };   </programlisting>
+
+      <para><emphasis role="bold">Note</emphasis>: when both
+      <emphasis>max_len</emphasis> and <emphasis>ecl_max_len</emphasis> are
+      specified, <emphasis>ecl_max_len</emphasis> is used and
+      <emphasis>max_len</emphasis> is ignored.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_ecl_name">
+      <title>ecl_name ("<emphasis role="bold">name</emphasis><emphasis
+      role="bold">")</emphasis></title>
+
+      <para>The <emphasis>ecl_name</emphasis> attribute specifies the field
+      name in generated ECL code. By default, the field name in ECL is the
+      same as the name defined in ECM. However, in some cases, the name could
+      causes issues in ECL. For example keywords in ECL cannot be used as a
+      field name.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>[ecl_name("_export")] string Export;
+[ecl_name("_type")] string Type;   </programlisting>
+
+      <para>Here, both <emphasis role="bold">EXPORT</emphasis> and <emphasis
+      role="bold">TYPE</emphasis> are ECL keywords and cannot be used as ECL
+      field names. We use <emphasis>ecl_name</emphasis> to tell the esdl2ecl
+      process to generate acceptable names.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_counter-count_val">
+      <title>counter and count_val</title>
+
+      <para>These two attributes are used to help ESP calculate the record
+      count of the response.</para>
+
+      <para><emphasis>counter</emphasis> counts the number of children of the
+      nodes. When the node is an array, it is the same as the number of items
+      in the array.</para>
+
+      <para><emphasis>count_val</emphasis> will use the value of the node as
+      record count. Field <emphasis role="bold">RecordCount</emphasis> is
+      implicitly marked as <emphasis>count_val</emphasis>.</para>
+
+      <para>When an response has multiple counter, count_val, the sum of the
+      values is returned as record-count.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>[counter] ESParray&lt;MyRecord, Record&gt; Records;
+[count_val] int TotalFound;   </programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_max_count_var">
+      <title><emphasis role="bold">max_count_var</emphasis></title>
+
+      <para>The max_count_var attribute is used to specify the expected max
+      items in a dataset (ESParray).</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>[max_count_var("iesp.Constants.JD.MaxRecords")] ESParray &lt;ESPstruct MYRecord, Record&gt; Records;   </programlisting>
+
+      <para>Roxie will define the constant iesp.Constants.JD.MaxRecords and
+      change it at will without affecting ESP.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_ecl_null">
+      <title>ecl_null (<emphasis role="bold">n</emphasis><emphasis
+      role="bold"> || </emphasis><emphasis
+      role="bold">string</emphasis><emphasis role="bold">)</emphasis></title>
+
+      <para>The <emphasis>ecl_null </emphasis>attribute tells ESP to remove
+      the field altogether if the field's value is <emphasis>n</emphasis> or
+      <emphasis>string</emphasis>. This provides a means to remove a field
+      completely when there is no data for it.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>[ecl_null(0)] int Age;
+[ecl_null('false')] bool IsMatch;   </programlisting>
+
+      <para>Age 0 means we don’t have data for this person. So if Age is 0,
+      tag &lt;Age&gt; will not be returned.</para>
+
+      <para>Without this attribute, <emphasis
+      role="bold">&lt;Age&gt;0&lt;/Age&gt;</emphasis> will be returned.</para>
+
+      <para>For the second example, a bool value of <emphasis>false</emphasis>
+      (treated as string) is not returned.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_leading_zero">
+      <title>leading_zero(<emphasis role="bold">n</emphasis><emphasis
+      role="bold">)</emphasis></title>
+
+      <para>The <emphasis>leading_zero</emphasis> attribute adds zero(s) to
+      the field value to so that the total length is
+      <emphasis>n</emphasis>.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>ESPstruct Date
+{
+   [leading_zero(4)] Year;
+   [leading_zero(2)] Month;
+   [leading_zero(2)] Day;
+};  </programlisting>
+
+      <para>So the Date will always have a 4-digit Year and a 2-digit Month
+      and a 2-digit Day.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_ecl_hide">
+      <title><emphasis role="bold">ecl_hide</emphasis></title>
+
+      <para>The <emphasis>ecl_hide</emphasis> attribute hides the field from
+      ECL (that is, the field is removed when generating the ECL code). This
+      is used for some special cases.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>ESPstruct Relative
+{
+   [ecl_hide] ESParray&lt;ESPstruct Relative, Relative&gt; Relatives;
+   "
+};   </programlisting>
+
+      <para>In this case, the Relative structure is defined in a recursive
+      manner, and ECL does not support such a construct. Therefore, we use
+      <emphasis>ecl_hide</emphasis> to avoid the recursive definition in
+      ECL.</para>
+
+      <para>Sometimes a field is hidden from ecl for other reasons.</para>
+
+      <para>For example, a field with <emphasis>get_data_from</emphasis>
+      attribute will be automatically hidden. In these cases,
+      <emphasis>ecl_hide</emphasis> is not needed.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_ecl_type">
+      <title>ecl_type ("<emphasis role="bold">type</emphasis><emphasis
+      role="bold">")</emphasis></title>
+
+      <para>The <emphasis>ecl_type</emphasis> attribute defines the field type
+      in ECL.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting> [ecl_type("Decimal10_2")] double RetailPrice;   </programlisting>
+
+      <para>ESDL does not have a monetary type, so we use
+      <emphasis>ecl_type</emphasis> to define it.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_get_data_from">
+      <title>get_data_from ("<emphasis role="bold">field</emphasis><emphasis
+      role="bold">")</emphasis></title>
+
+      <para>For code reuse and simplification, when complex versioning changes
+      have been made.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_ecl_keep">
+      <title>ecl_keep</title>
+
+      <para>The <emphasis>ecl_keep</emphasis> attribute keeps the field in the
+      generated ECL even though this field would have been hidden without this
+      attribute.</para>
+
+      <para>For instance, a field with <emphasis>get_data_from </emphasis>will
+      be hidden, however, adding this attribute will keep this field. This is
+      needed when we use <emphasis>get_data_from </emphasis>on two fields from
+      each other.</para>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_min_ver">
+      <title><emphasis role="bold">min_ver</emphasis></title>
+
+      <para>The min_ver attribute allows you to define the minimum (earliest)
+      version where a field is visible. Requests using a prior version will
+      not have access to the field.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting> [min_ver("1.03")] bool IsValid;   </programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_max_ver">
+      <title><emphasis role="bold">max_ver</emphasis></title>
+
+      <para>The max_ver attribute allows you to define the maximum (latest)
+      version where a field is visible. Requests using a later version will
+      not have access to the field.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting> [max_ver("1.04")] bool IsValid;   </programlisting>
+    </sect1>
+
+    <sect1 id="ESDL_Attributes_depr_ver">
+      <title><emphasis role="bold">depr_ver</emphasis></title>
+
+      <para>The depr_ver attribute allows you to declare a field’s end of life
+      version. The field is deprecated at the specified version number.
+      Requests using that version or any subsequent version will not have
+      access to the field.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting> [depr_ver("1.04")] bool IsValid;   </programlisting>
+    </sect1>
+  </chapter>
+</book>

docs/DynamicESDL/DESDL-Mods/ESDLESParray.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESParray">
+  <title><emphasis role="bold">ESParray</emphasis></title>
+
+  <para>A structure for unbounded arrays. Arrays support inheritance and can
+  be nested.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>
+ ESPstruct NameBlock
+   
+ {
+     string FirstName;
+     string MiddleName;
+     string LastName;
+  
+     int Age;
+ };
+  
+ ESParray&lt;ESPstruct NameBlock, Name&gt; Names;
+
+   </programlisting>
+
+  <para>This results in something like:</para>
+
+  <para><programlisting>&lt;Names&gt;
+  &lt;Name&gt;
+    &lt;FirstName&gt;James&lt;/FirstName&gt;
+    &lt;MiddleName&gt;Joseph&lt;/MiddleName&gt;
+    &lt;LastName&gt;Deerfield&lt;/LastName&gt;
+    &lt;Age&gt;42&lt;/Age&gt;
+  &lt;/Name&gt;
+  &lt;Name&gt;
+    &lt;FirstName&gt;Emily&lt;/FirstName&gt;
+    &lt;MiddleName&gt;Kate&lt;/MiddleName&gt;
+    &lt;LastName&gt;Constance&lt;/LastName&gt;
+    &lt;Age&gt;33&lt;/Age&gt;
+  &lt;/Name&gt;
+&lt;/Names&gt;</programlisting></para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLESPenum.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESPenum">
+  <title>ESPenum</title>
+
+  <para>A structure containing an enumerated value. </para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>
+ESPenum EyeColors : string
+  
+{
+  Brn("Brown"),
+  Blu("Blue"),
+  Grn("Green"),
+};
+ 
+
+ESPstruct Person
+
+{
+  string FirstName;
+  string MiddleName;
+  string LastName;
+
+  ESPenum EyeColors EyeColor("Brown"); //provides a default value
+};
+   </programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLESPinclude.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESPinclude">
+  <title><emphasis role="bold">ESPinclude</emphasis></title>
+
+  <para>ESPinclude allows you to include an external ESDL file. This is
+  similar to the #include statement.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>
+ ESPinclude(commonStructures);
+   </programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLESPmethod.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESPmethod">
+  <title><emphasis role="bold">ESPmethod</emphasis></title>
+
+  <para>This defines a method definition you can reference in an ESPservice
+  structure. The method definition should contain references to a previously
+  defined ESPrequest and ESPresponse.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>
+ ESPservice MyService
+
+ {
+   ESPmethod MyMethod1(MyMethod1Request, MyMethod1Response);
+   ESPmethod 
+    [
+     description("MyMethod Two"),
+     help("This method does everything MyMethod1 does plus a few extra features"),
+     min_ver("1.2")
+    ]
+   MyMethod2(MyMethod2Request, MyMethod2Response);
+ };
+ 
+    </programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLESPrequest.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+   
+
+    <sect2 id="ESPrequest">
+      <title><emphasis role="bold">ESPrequest</emphasis></title>
+
+      <para>The request structure for a method.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>
+ESPrequest  MyQueryRequest
+
+{
+    string FirstName;
+    string MiddleName;
+    string LastName;
+    string Sortby;
+ 
+    bool Descending(false);
+};
+   </programlisting>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLESPresponse.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESPresponse">
+  <title><emphasis role="bold">ESPresponse</emphasis></title>
+
+  <para>The response structure for a method.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>ESPresponse  MyQueryResponse
+ 
+{
+    string FirstName;
+    string MiddleName;
+    string LastName;
+ 
+};  </programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLESPservice.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="EPservice">
+  <title>ESPservice</title>
+
+  <para>This defines an ESP web service interface. Once defined, this
+  interface definition can be assigned (bound) to a Dynamic ESDL-based ESP
+  Service.</para>
+
+  <para>An ESPservice should contain one or more method definitions.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>ESPservice MyService
+{
+  ESPmethod MyMethod1(MyMethod1Request, MyMethod1Response);
+  ESPmethod MyMethod2(MyMethod2Request, MyMethod2Response);
+};
+</programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLESPstruct.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+   
+   <sect2 id="ESP_Struct">
+      <title>ESP<emphasis role="bold">struct</emphasis></title>
+
+      <para>ESPstruct is a set of elements grouped together under one name.
+      These elements, known as <emphasis>members</emphasis>, can have
+      different types and different lengths. Structures can be nested and
+      support inheritance.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>
+ESPstruct NameBlock
+
+{
+    string FirstName;
+    string MiddleName;
+    string LastName;
+    int Age;
+};
+   </programlisting>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLTypeMapping.xml

@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESDL_Data_ECLTypeMapping">
+  <title><emphasis role="bold">ESDL, XSD, and ECL Type
+  Mapping</emphasis></title>
+
+  <para><informaltable colsep="1" frame="all" rowsep="1">
+      <tgroup cols="3">
+        <colspec colwidth="86.75pt" />
+
+        <colspec colwidth="76.50pt" />
+
+        <colspec colwidth="140.00pt" />
+
+        <tbody>
+          <row>
+            <entry><emphasis role="bold">ESDL</emphasis></entry>
+
+            <entry><emphasis role="bold">XSD</emphasis></entry>
+
+            <entry><emphasis role="bold">ECL</emphasis></entry>
+          </row>
+
+          <row>
+            <entry>Bool boolean</entry>
+
+            <entry>bool</entry>
+
+            <entry>boolean</entry>
+          </row>
+
+          <row>
+            <entry>Binary</entry>
+
+            <entry>Base64Binary</entry>
+
+            <entry>String (base64 encoded)</entry>
+          </row>
+
+          <row>
+            <entry>Float</entry>
+
+            <entry>float</entry>
+
+            <entry>REAL4</entry>
+          </row>
+
+          <row>
+            <entry>Double</entry>
+
+            <entry>double</entry>
+
+            <entry>REAL8</entry>
+          </row>
+
+          <row>
+            <entry>Int</entry>
+
+            <entry>int</entry>
+
+            <entry>INTEGER</entry>
+          </row>
+
+          <row>
+            <entry>Int64</entry>
+
+            <entry>long</entry>
+
+            <entry>INTEGER8</entry>
+          </row>
+
+          <row>
+            <entry>String</entry>
+
+            <entry>string</entry>
+
+            <entry>String</entry>
+          </row>
+        </tbody>
+      </tgroup>
+    </informaltable></para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLbinary.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+
+   <sect2 id="ESDL_Data_binary">
+      <title><emphasis role="bold">binary</emphasis></title>
+
+      <para>A data type containing binary data, similar to a BLOB .</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>binary RetinaScanSample;   </programlisting>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLbooleanbool.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="boolean_bool">
+  <title><emphasis role="bold">boolean bool</emphasis></title>
+
+  <para>A boolean or logical data type having one of two possible values: true
+  (1) or false (0).</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>boolean includeFlag; 
+bool includeMore;</programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLcounter_and_count_val.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+   <sect2 id="ESDL_Attributes_counter-count_val">
+      <title>counter and count_val</title>
+
+      <para>These two attributes are used to help ESP calculate the record
+      count of the response.</para>
+
+      <para><emphasis>counter</emphasis> counts the number of children of the
+      nodes. When the node is an array, it is the same as the number of items
+      in the array.</para>
+
+      <para><emphasis>count_val</emphasis> will use the value of the node as
+      record count. Field <emphasis role="bold">RecordCount</emphasis> is
+      implicitly marked as <emphasis>count_val</emphasis>.</para>
+
+      <para>When an response has multiple counter, count_val, the sum of the
+      values is returned as record-count.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>[counter] ESParray&lt;MyRecord, Record&gt; Records;
+[count_val] int TotalFound;   </programlisting>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLdepr_ver.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+   <sect2 id="ESDL_Attributes_depr_ver">
+      <title><emphasis role="bold">depr_ver</emphasis></title>
+
+      <para>The depr_ver attribute allows you to declare a field's end of life
+      version. The field is deprecated at the specified version number.
+      Requests using that version or any subsequent version will not have
+      access to the field.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>[depr_ver("1.04")] bool IsValid;</programlisting>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLdescription.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESDL_Attributes_description">
+  <title><emphasis role="bold">description</emphasis></title>
+
+  <para>The description attribute (valid only for an ESPMethod) allows you to
+  specify some additional text to display on the form that is automatically
+  generated to execute a method.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting> ESPservice MyService
+
+ {
+   ESPmethod MyMethod1(MyMethod1Request, MyMethod1Response);
+   ESPmethod 
+    [
+     description("MyMethod Two"),
+     help("This method does everything MyMethod1 does plus a few extra features"),
+     min_ver("1.2")
+    ]
+   MyMethod2(MyMethod2Request, MyMethod2Response);
+ };</programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLdouble.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESDL_Data_double">
+  <title><emphasis role="bold">double</emphasis></title>
+
+  <para>An 8-byte floating point or real number.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>double Temperature;   </programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLecl_hide.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESDL_Attributes_ecl_hide">
+  <title><emphasis role="bold">ecl_hide</emphasis></title>
+
+  <para>The <emphasis>ecl_hide</emphasis> attribute hides the field from ECL
+  (that is, the field is removed when generating the ECL code). This is used
+  for some special cases.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>ESPstruct Relative
+ {
+    [ecl_hide] ESParray&lt;ESPstruct Relative, Relative&gt; Relatives;
+    "
+ };   </programlisting>
+
+  <para>In this case, the Relative structure is defined in a recursive manner,
+  and ECL does not support such a construct. Therefore, we use
+  <emphasis>ecl_hide</emphasis> to avoid the recursive definition in
+  ECL.</para>
+
+  <para>Sometimes a field is hidden from ECL for other reasons. In these
+  cases, <emphasis>ecl_hide</emphasis> is not needed.</para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLecl_keep.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESDL_Attributes_ecl_keep">
+  <title>ecl_keep</title>
+
+  <para>The <emphasis>ecl_keep</emphasis> attribute keeps the field in the
+  generated ECL even though this field would have been hidden without this
+  attribute.</para>
+
+  <para></para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLecl_max_len.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESDL_Attributes_ecl_max_len">
+  <title>ecl_max_len (<emphasis role="bold">n</emphasis><emphasis
+  role="bold">)</emphasis></title>
+
+  <para>This <emphasis>ecl_max_len</emphasis> attribute tells the ECL
+  generator to use ECL <emphasis>maxlength </emphasis>instead of the regular
+  field length.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>[ecl_max_len(50)] string CompanyName;
+[max_len(6)] string Gender;   </programlisting>
+
+  <para>The generated ECL code is:</para>
+
+  <programlisting>string CompanyName { xpath("CompanyName"),maxlength(50) };
+   string6 Gender { xpath("Gender") };   </programlisting>
+
+  <para><emphasis role="bold">Note</emphasis>: when both
+  <emphasis>max_len</emphasis> and <emphasis>ecl_max_len</emphasis> are
+  specified, <emphasis>ecl_max_len</emphasis> is used and
+  <emphasis>max_len</emphasis> is ignored.</para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLecl_name.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+    <sect2 id="ESDL_Attributes_ecl_name">
+       <title>ecl_name ("<emphasis role="bold">name</emphasis><emphasis
+       role="bold">")</emphasis></title>
+ 
+       <para>The <emphasis>ecl_name</emphasis> attribute specifies the field
+       name in generated ECL code. By default, the field name in ECL is the
+       same as the name defined in ECM. However, in some cases, the name could
+       causes issues in ECL. For example keywords in ECL cannot be used as a
+       field name.</para>
+ 
+       <para><emphasis role="bold">Example:</emphasis></para>
+ 
+       <programlisting>[ecl_name("_export")] string Export;
+ [ecl_name("_type")] string Type;   </programlisting>
+ 
+       <para>Here, both <emphasis role="bold">EXPORT</emphasis> and <emphasis
+       role="bold">TYPE</emphasis> are ECL keywords and cannot be used as ECL
+       field names. We use <emphasis>ecl_name</emphasis> to tell the esdl2ecl
+       process to generate acceptable names.</para>
+     </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLecl_null.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESDL_Attributes_ecl_null">
+  <title>ecl_null (<emphasis role="bold">n</emphasis><emphasis role="bold"> |
+  </emphasis><emphasis role="bold">string</emphasis><emphasis
+  role="bold">)</emphasis></title>
+
+  <para>The <emphasis>ecl_null </emphasis>attribute tells ESP to remove the
+  field altogether if the field's value is <emphasis>n</emphasis> or
+  <emphasis>string</emphasis>. This provides a means to remove a field
+  completely when there is no data for it.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>[ecl_null(0)] int Age;
+[ecl_null('false')] bool IsMatch;   </programlisting>
+
+  <para>Age 0 means there is no Age data for this person. So, if Age is 0, the
+  &lt;Age&gt; tag is not returned. </para>
+
+  <para>Without this attribute, <emphasis
+  role="bold">&lt;Age&gt;0&lt;/Age&gt;</emphasis> would be returned.</para>
+
+  <para>For the second example, a bool value of <emphasis>false</emphasis>,
+  returned as a string, is treated as null and therefore the tag is not
+  returned.</para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLecl_type.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+    <sect2 id="ESDL_Attributes_ecl_type">
+      <title>ecl_type ("<emphasis role="bold">type</emphasis><emphasis
+      role="bold">")</emphasis></title>
+
+      <para>The <emphasis>ecl_type</emphasis> attribute defines the field type
+      in ECL.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting> [ecl_type("Decimal10_2")] double RetailPrice;   </programlisting>
+
+      <para>ESDL does not have a monetary type, so we use
+      <emphasis>ecl_type</emphasis> to define it.</para>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLexceptions_inline.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="ESDL_Attributes_exceptions_inline">
+  <title><emphasis role="bold">exceptions_inline</emphasis></title>
+
+  <para>The exceptions_inline attribute (valid only for an ****ESPMethod)
+  allows you to specify that the service should return app / service level
+  exceptions as part of a response message rather than as a SOAPfault.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>E*****</programlisting>
+</sect1>

docs/DynamicESDL/DESDL-Mods/ESDLfloat.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+
+    <sect2 id="ESDL_Data_float">
+      <title><emphasis role="bold">float</emphasis></title>
+
+      <para>A 4-byte floating point or real number.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>float Temperature;
+</programlisting>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLget_data_from.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+   <sect2 id="ESDL_Attributes_get_data_from">
+      <title>get_data_from ("<emphasis role="bold">field</emphasis><emphasis
+      role="bold">")</emphasis></title>
+
+      <para>For code reuse and simplification, when complex versioning changes
+      have been made.</para>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLhelp.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESDL_Attributes_help">
+  <title><emphasis role="bold">help</emphasis></title>
+
+  <para>The help attribute (valid only for an ESPMethod) allows you to specify
+  some additional text to display on the form that is automatically generated
+  to execute a method. </para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>ESPservice MyService
+
+ {
+   ESPmethod MyMethod1(MyMethod1Request, MyMethod1Response);
+   ESPmethod 
+    [
+     description("MyMethod Two"),
+     help("This method does everything MyMethod1 does plus a few extra features"),
+     min_ver("1.2")
+    ]
+   MyMethod2(MyMethod2Request, MyMethod2Response);
+ };</programlisting>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLint.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+    <sect2 id="ESDL_Data_int">
+      <title><emphasis role="bold">int</emphasis></title>
+
+      <para>An integer value.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>int Age;   </programlisting>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLint64.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+   <sect2 id="ESDL_Data_int64">
+      <title><emphasis role="bold">int64</emphasis></title>
+
+      <para>A 64-bit integer value</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting>int64 Iterations;
+</programlisting>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLleading_zero.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+     <sect2 id="ESDL_Attributes_leading_zero">
+       <title>leading_zero(<emphasis role="bold">n</emphasis><emphasis
+       role="bold">)</emphasis></title>
+ 
+       <para>The <emphasis>leading_zero</emphasis> attribute adds zero(s) to
+       the field value to so that the total length is
+       <emphasis>n</emphasis>.</para>
+ 
+       <para><emphasis role="bold">Example:</emphasis></para>
+ 
+       <programlisting>ESPstruct Date
+ {
+    [leading_zero(4)] Year;
+    [leading_zero(2)] Month;
+    [leading_zero(2)] Day;
+ };  </programlisting>
+ 
+       <para>So the Date will always have a 4-digit Year and a 2-digit Month
+       and a 2-digit Day.</para>
+     </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLmax_count_var.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+     <sect2 id="ESDL_Attributes_max_count_var">
+       <title><emphasis role="bold">max_count_var</emphasis></title>
+ 
+       <para>The max_count_var attribute is used to specify the expected max
+       items in a dataset (ESParray).</para>
+ 
+       <para><emphasis role="bold">Example:</emphasis></para>
+ 
+       <programlisting>[max_count_var("iesp.Constants.JD.MaxRecords")] ESParray &lt;ESPstruct MYRecord, Record&gt; Records;   </programlisting>
+ 
+       <para>Roxie will define the constant iesp.Constants.JD.MaxRecords and
+       change it at will without affecting ESP.</para>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLmax_len.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="ESDL_Attributes_max_len">
+  <title><emphasis role="bold">max_len (</emphasis><emphasis
+  role="bold">n</emphasis><emphasis role="bold">)</emphasis></title>
+
+  <para>The <emphasis>max_len</emphasis> attribute specifies the field length
+  for ECL string field.</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>[max_len(20)] string City;   </programlisting>
+
+  <para>It means that in ECL, City field is a fixed length of 20 chars. For
+  integer type, the max_len means the maximum size in bytes for the integer
+  (8*max_len bits integer).</para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>[max_len(3)] int Age;   </programlisting>
+
+  <para>This generates ECL code:</para>
+
+  <programlisting>integer3 Age{xpath('Age')};   </programlisting>
+
+  <para>This attribute works for ESPenum type, too. The ECL type is also
+  string.</para>
+
+  <programlisting>[max_len(2)] ESPenum StateCode State;   </programlisting>
+
+  <para>Here the StateCode is 2-char state code enumeration.</para>
+
+  <para>This attribute can also be used for ESPstruct, ESPrequest,
+  ESPresponse:</para>
+
+  <programlisting>ESPstruct [max_len(1867)] IdentitySlim : Identity
+  {
+     ...
+  };   
+  </programlisting>
+
+  <para>This generates ECL code:</para>
+
+  <programlisting>export t_MyQuery := record (share.t_Name), MAXLENGTH(1867)
+  {
+  }; </programlisting>
+
+  <para>The ECL option <emphasis>MAXLENGTH</emphasis> helps ECL engine better
+  manage memory.</para>
+
+  <para>This does not affect the type in the XSD/WSDL. ESP ignores this
+  attribute when generating the schema. The type for a string is xsd:string
+  which has no length limit. Therefore, the schema stays the same if the field
+  length changes in the Roxie query.</para>
+</sect2>

docs/DynamicESDL/DESDL-Mods/ESDLmax_ver.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+     <sect2 id="ESDL_Attributes_max_ver">
+       <title><emphasis role="bold">max_ver</emphasis></title>
+ 
+       <para>The max_ver attribute allows you to define the maximum (latest)
+       version where a field is visible. Requests using a later version will
+       not have access to the field.</para>
+ 
+       <para><emphasis role="bold">Example:</emphasis></para>
+ 
+       <programlisting> [max_ver("1.04")] bool IsValid;   </programlisting>
+     </sect2>
+

docs/DynamicESDL/DESDL-Mods/ESDLmin_ver.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+    <sect2 id="ESDL_Attributes_min_ver">
+      <title><emphasis role="bold">min_ver</emphasis></title>
+
+      <para>The min_ver attribute allows you to define the minimum (earliest)
+      version where a field is visible. Requests using a prior version will
+      not have access to the field.</para>
+
+      <para><emphasis role="bold">Example:</emphasis></para>
+
+      <programlisting> [min_ver("1.03")] bool IsValid;   </programlisting>
+    </sect2>

docs/DynamicESDL/DESDL-Mods/ESDLstring.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+ 
+      <sect2 id="ESDL_Data_string">
+        <title><emphasis role="bold">string</emphasis></title>
+  
+        <para>A data type consisting of sequence of alphanumeric
+        characters.</para>
+  
+        <para><emphasis role="bold">Example:</emphasis></para>
+  
+        <programlisting>string FirstName;
+  </programlisting>
+      </sect2>

docs/DynamicESDL/DESDL-Mods/xxESDLnil_remove.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="ESDL_Attributes_nil_remove">
+  <title><emphasis role="bold">nil_remove</emphasis></title>
+
+  <para>The nil_remove attribute (valid for an ESPrequest, ESPStruct, or
+  ESPresponse) specifies to remove any empty tags. This is useful to save
+  bandwidth by returning only relavent tags. </para>
+
+  <para><emphasis role="bold">Example:</emphasis></para>
+
+  <programlisting>ESPresponse [nil_remove] MyQueryResponse
+ 
+{
+    string FirstName;
+    string MiddleName;
+    string LastName;
+ 
+};
+</programlisting>
+</sect1>

docs/DynamicESDL/DynamicESDL_Includer.xml

@@ -0,0 +1,725 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<book lang="en_US" xml:base="../">
+  <bookinfo>
+    <title>Dynamic ESDL</title>
+
+    <mediaobject>
+      <imageobject>
+        <imagedata fileref="images/redswooshWithLogo3.jpg" />
+      </imageobject>
+    </mediaobject>
+
+    <author>
+      <surname>Boca Raton Documentation Team</surname>
+    </author>
+
+    <legalnotice>
+      <para>We welcome your comments and feedback about this document via
+      email to <email>docfeedback@hpccsystems.com</email></para>
+
+      <para>Please include <emphasis role="bold">Documentation
+      Feedback</emphasis> in the subject line and reference the document name,
+      page numbers, and current Version Number in the text of the
+      message.</para>
+
+      <para>LexisNexis and the Knowledge Burst logo are registered trademarks
+      of Reed Elsevier Properties Inc., used under license.</para>
+
+      <para>HPCC Systems<superscript>®</superscript> is a registered trademark
+      of LexisNexis Risk Data Management Inc.</para>
+
+      <para>Other products, logos, and services may be trademarks or
+      registered trademarks of their respective companies.</para>
+
+      <para>All names and example data used in this manual are fictitious. Any
+      similarity to actual persons, living or dead, is purely
+      coincidental.</para>
+
+      <para />
+    </legalnotice>
+
+    <xi:include href="common/Version.xml" xpointer="FooterInfo"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+    <xi:include href="common/Version.xml" xpointer="DateVer"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+    <corpname>HPCC Systems<superscript>®</superscript></corpname>
+
+    <xi:include href="common/Version.xml" xpointer="Copyright"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+    <mediaobject role="logo">
+      <imageobject>
+        <imagedata fileref="images/LN_Rightjustified.jpg" />
+      </imageobject>
+    </mediaobject>
+  </bookinfo>
+
+  <chapter>
+    <title>Dynamic ESDL</title>
+
+    <para>Dynamic ESDL (Enterprise Service Description Language) is a
+    methodology that helps you develop and manage web-based query interfaces
+    quickly and consistently.</para>
+
+    <para>Dynamic ESDL takes an interface-first development approach. It
+    leverages the ESDL Language to create a common interface “contract” that
+    both Roxie Query and Web interface developers will adhere to. It is
+    intended to allow developers to create production web services, with clean
+    interfaces that can evolve and grow over time without breaking existing
+    applications.</para>
+
+    <para>ESDL’s built-in versioning support helps ensure compiled and
+    deployed applications continue to operate while changes are made to the
+    deployed service’s interface for new functionality.</para>
+
+    <para>ESDL’s ability to define and reuse common structures helps maintain
+    consistent interfaces across methods.</para>
+
+    <para>The Dynamic ESDL service is built to scale horizontally, and hooks
+    are provided to add custom logging and security to help create fully
+    “productionalized” web services.</para>
+
+    <para>Once a service is deployed, application developers and end-users can
+    consume the service using REST, JSON, XML, SOAP, or form encoded posts.
+    Dynamic ESDL provides quick and easy access to a WSDL, live forms, sample
+    requests and responses, and testing interfaces to allow developers to test
+    logic changes, data changes, or new features, as well as to interact with
+    the service directly using SOAP, XML, or JSON.</para>
+  </chapter>
+
+  <chapter>
+    <title>Workflow Tutorial</title>
+
+    <sect1 role="nobrk">
+      <title>Overview</title>
+
+      <para>In this section, we will:</para>
+
+      <para />
+
+      <itemizedlist>
+        <listitem>
+          <para>Use Configuration Manager to add a Dynamic ESDL-based ESP
+          Service and bind it to a port on an ESP server.</para>
+
+          <para />
+        </listitem>
+
+        <listitem>
+          <para>Write an ESDL Service Definition.</para>
+
+          <para />
+        </listitem>
+
+        <listitem>
+          <para>Generate ECL from the ESDL Service Definition, copy it to your
+          ECL repository, and the write the ECL query that will deliver the
+          result (the business logic).</para>
+
+          <para />
+        </listitem>
+
+        <listitem>
+          <para>Compile the ECL business logic query and publish it to a Roxie
+          cluster.</para>
+
+          <para>At this point you can test the query using WsECL.</para>
+
+          <para />
+        </listitem>
+
+        <listitem>
+          <para>Publish the Dynamic ESDL definition.</para>
+
+          <para />
+        </listitem>
+
+        <listitem>
+          <para>Bind the service methods to the Roxie queries using a
+          configuration XML file.</para>
+
+          <para />
+        </listitem>
+      </itemizedlist>
+    </sect1>
+
+    <sect1>
+      <title>Configure and Bind a Dynamic ESDL service</title>
+
+      <para>This step adds an ESP service and a service binding that reserves
+      a port for the dynamic ESDL service. This step is independent of the
+      development and publishing of the actual Roxie query, so you can set it
+      up before or after your query is ready.</para>
+
+      <orderedlist>
+        <listitem>
+          <para>If it is running, stop the HPCC system, using this
+          command:</para>
+
+          <para>
+            <emphasis role="bold">Centos/Red Hat</emphasis>
+
+            <programlisting>sudo /sbin/service hpcc-init stop</programlisting>
+          </para>
+
+          <para>
+            <emphasis role="bold">Ubuntu</emphasis>
+          </para>
+
+          <para>
+            <programlisting>sudo service hpcc-init stop</programlisting>
+          </para>
+
+          <para>
+            <emphasis role="bold">Debian 6 (Squeeze)</emphasis>
+          </para>
+
+          <programlisting>sudo /etc/init.d/hpcc-init stop</programlisting>
+
+          <para>
+            <informaltable colsep="1" frame="all" rowsep="1">
+              <?dbfo keep-together="always"?>
+
+              <tgroup cols="2">
+                <colspec colwidth="49.50pt" />
+
+                <colspec />
+
+                <tbody>
+                  <row>
+                    <entry>
+                      <inlinegraphic fileref="images/OSSgr3.png" />
+                    </entry>
+
+                    <entry>You can use this command to confirm HPCC processes
+                    are stopped (on Centos/Red Hat):<para>
+                        <programlisting>sudo /sbin/service hpcc-init status</programlisting>
+
+                        <phrase>For Ubuntu</phrase>
+
+                        <programlisting>sudo service hpcc-init status
+</programlisting>
+
+                        <phrase>For Debian 6 (Squeeze)</phrase>
+
+                        <programlisting>sudo /etc/init.d/hpcc-init status</programlisting>
+                      </para></entry>
+                  </row>
+                </tbody>
+              </tgroup>
+            </informaltable>
+          </para>
+        </listitem>
+
+        <listitem>
+          <para>Start the Configuration Manager service.<emphasis
+          role="bold" /><programlisting>sudo /opt/HPCCSystems/sbin/configmgr
+</programlisting></para>
+        </listitem>
+
+        <listitem>
+          <para>Using a Web browser, go to the Configuration Manager's
+          interface:</para>
+
+          <programlisting>http://&lt;<emphasis>node ip </emphasis>&gt;:8015</programlisting>
+
+          <para>The Configuration Manager startup wizard displays.</para>
+        </listitem>
+
+        <listitem>
+          <para><?dbfo keep-together="always"?>Select <emphasis
+          role="bold">Advanced View</emphasis> and then select the source
+          environment XML file to edit.</para>
+
+          <para>
+            <graphic fileref="images/desdl-openconfig.jpg" />
+          </para>
+        </listitem>
+
+        <listitem>
+          <para>Press the <emphasis role="bold">Next</emphasis> button.</para>
+        </listitem>
+
+        <listitem>
+          <para>Right-click on <emphasis role="bold">Esp Services</emphasis>
+          and select <emphasis role="bold">New Esp Services</emphasis> &gt;
+          <emphasis role="bold">Dynamic ESDL</emphasis>.</para>
+
+          <para />
+
+          <para>
+            <graphic fileref="images/desdl-addDESDL.jpg" />
+          </para>
+        </listitem>
+
+        <listitem>
+          <para>Provide a name for the service.</para>
+
+          <para>
+            <graphic fileref="images/dsdl-NametheServoice.jpg" />
+          </para>
+        </listitem>
+
+        <listitem>
+          <para>Select your ESP , then select the ESP Service Bindings
+          tab.</para>
+
+          <para>
+            <graphic fileref="images/desdl-ESPSVCBinding.jpg" />
+          </para>
+        </listitem>
+
+        <listitem>
+          <para>Right-click in the list of bindings and select <emphasis
+          role="bold">Add</emphasis></para>
+
+          <para>
+            <graphic fileref="images/desdl-AddSVCBinding.jpg" />
+          </para>
+        </listitem>
+
+        <listitem>
+          <para>Provide a name, port, and then select the service. The service
+          definition you added will display in the list of available
+          services.</para>
+
+          <para>
+            <graphic fileref="images/desdl-NameSVCBinding.jpg" />
+          </para>
+        </listitem>
+
+        <listitem>
+          <para>Copy the NewEnvironment.xml file from the source directory to
+          the /etc/HPCCSystems and rename the file to environment.xml</para>
+
+          <programlisting># for example
+sudo cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/environment.xml</programlisting>
+
+          <para>
+            <informaltable colsep="1" frame="all" rowsep="1">
+              <?dbfo keep-together="always"?>
+
+              <tgroup cols="2">
+                <colspec colwidth="49.50pt" />
+
+                <colspec />
+
+                <tbody>
+                  <row>
+                    <entry>
+                      <inlinegraphic fileref="images/caution.png" />
+                    </entry>
+
+                    <entry>Make sure that you have sufficient privileges to
+                    write file(s) to the destination directory before
+                    attempting to copy. If prompted to overwrite the
+                    destination file, you should answer <emphasis
+                    role="bold">yes</emphasis>.</entry>
+                  </row>
+                </tbody>
+              </tgroup>
+            </informaltable>
+          </para>
+        </listitem>
+
+        <listitem>
+          <para>Copy the <emphasis
+          role="bold">/etc/HPCCSystems/environment.xml</emphasis> to <emphasis
+          role="bold">/etc/HPCCSystems/</emphasis> on <emphasis
+          role="bold">every</emphasis> node.</para>
+
+          <para>You may want to create a script to push out the XML file to
+          all nodes. A sample script is provided with HPCC. The following
+          command copies the XML files out to all nodes as required:</para>
+
+          <para>
+            <programlisting>sudo /opt/HPCCSystems/sbin/hpcc-push.sh &lt;sourcefile&gt; &lt;destinationfile&gt;
+</programlisting>
+          </para>
+        </listitem>
+
+        <listitem>
+          <para>Restart the HPCC system on <emphasis
+          role="bold">every</emphasis> node. The following command starts the
+          HPCC system on an individual node:</para>
+
+          <para>
+            <emphasis role="bold">Centos/Red Hat</emphasis>
+
+            <programlisting>sudo /sbin/service hpcc-init start</programlisting>
+          </para>
+
+          <para>
+            <emphasis role="bold">Ubuntu</emphasis>
+          </para>
+
+          <para>
+            <programlisting>sudo service hpcc-init start</programlisting>
+          </para>
+
+          <para>
+            <emphasis role="bold">Debian 6 (Squeeze)</emphasis>
+          </para>
+
+          <programlisting>sudo /etc/init.d/hpcc-init start</programlisting>
+
+          <para />
+
+          <informaltable colsep="1" frame="all" rowsep="1">
+            <?dbfo keep-together="always"?>
+
+            <tgroup cols="2">
+              <colspec colwidth="49.50pt" />
+
+              <colspec />
+
+              <tbody>
+                <row>
+                  <entry>
+                    <inlinegraphic fileref="images/OSSgr3.png" />
+                  </entry>
+
+                  <entry>
+                    <para>You may want to create a script to push this command
+                    out to every node. A sample script is provided with HPCC.
+                    Use the following command to start HPCC on all
+                    nodes:</para>
+
+                    <para>
+                      <programlisting>sudo /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init start</programlisting>
+                    </para>
+                  </entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable>
+
+          <para />
+        </listitem>
+      </orderedlist>
+    </sect1>
+
+    <sect1>
+      <title>Write the ESDL Service Definition</title>
+
+      <para>In this section, we will write a the Service Definitions. The
+      program listing below shows a service called
+      <emphasis>MathService</emphasis>. It contains one method,
+      <emphasis>AddThis</emphasis>, with a simple request and a simple
+      response defined.</para>
+
+      <programlisting>ESPservice MathService
+{
+  ESPmethod AddThis(AddThisRequest, AddThisResponse);
+};
+
+ESPrequest AddThisRequest 
+{
+  int  FirstNumber;
+  int  SecondNumber;
+};
+    
+ESPresponse AddThisResponse 
+{
+  int  Answer;
+};
+
+</programlisting>
+
+      <orderedlist>
+        <listitem>
+          <para>Save the file as MathService.ecm.</para>
+
+          <para />
+        </listitem>
+      </orderedlist>
+    </sect1>
+
+    <sect1>
+      <title>Generate ECL definitions from the ESDL Service Definition</title>
+
+      <para>In this section, we will generate ECL from this ESDL Service
+      Definition file. This will use the esdl executable that is installed
+      with Client Tools.</para>
+
+      <para>You will find this in C:\Program Files
+      (x86)\HPCCSystems\5.2.0\clienttools\bin on a Windows machine</para>
+
+      <para>or /opt/HPCCSystems/bin/ on a Linux machine.</para>
+
+      <para>
+        <orderedlist>
+          <listitem>
+            <para>From the command line, run:</para>
+
+            <para><programlisting>esdl ecl MathService.ecm .</programlisting>This
+            produces a file named MathService.ecl in the current
+            directory</para>
+
+            <programlisting><emphasis role="green">/*** Not to be hand edited (changes will be lost on re-generation) ***/
+/*** ECL Interface generated by esdl2ecl version 1.0 from MathService.xml. ***/
+/*===================================================*/</emphasis>
+
+export MathService := MODULE
+
+export t_AddThisRequest := record
+  integer FirstNumber {xpath('FirstNumber')};
+  integer SecondNumber {xpath('SecondNumber')};
+end;
+
+export t_AddThisResponse := record
+  integer Answer {xpath('Answer')};
+end;
+end;
+
+<emphasis role="green">/*** Not to be hand edited (changes will be lost on re-generation) ***/
+/*** ECL Interface generated by esdl2ecl version 1.0 from MathService.xml. ***/
+/*===================================================*/</emphasis>
+</programlisting>
+
+            <para />
+          </listitem>
+
+          <listitem>
+            <para>Copy the MathService.ecl file to a module in your ECL
+            Repository. For example, myMathService.</para>
+          </listitem>
+
+          <listitem>
+            <para>Write ECL to support the service:</para>
+
+            <programlisting>IMPORT MyMathService;
+rec_in := MyMathService.MathService.t_AddThisRequest;
+
+First_Row := ROW ([], rec_in) : STORED ('AddThisRequest', FEW);
+c:= first_row.FirstNumber + first_row.SecondNumber;
+ds_out := ROW ({c},MyMathService.MathService.t_AddThisResponse);
+OUTPUT(ds_out, NAMED('AddThisResponse')); </programlisting>
+          </listitem>
+
+          <listitem>
+            <para>Compile and Publish to your Roxie cluster.</para>
+          </listitem>
+
+          <listitem>
+            <para>Test the service using WsECL :<programlisting>http://&lt;<emphasis>node ip </emphasis>&gt;:8002 </programlisting></para>
+          </listitem>
+        </orderedlist>
+      </para>
+    </sect1>
+
+    <sect1>
+      <title>Publish the ESDL Service Definitions and Bind the ESDL
+      Service</title>
+
+      <para>In this section, we will publish the ESDL Service definitions to
+      the System Data Store and bind the methods to he published Roxie
+      query.</para>
+
+      <para>
+        <orderedlist>
+          <listitem>
+            <para>Publish the Dynamic ESDL definition</para>
+
+            <programlisting>esdl publish MathService MathService.ecm -s nnn.nnn.nnn.nnn -p 8010 --version 1</programlisting>
+
+            <para>Replace nnn.nnn.nnn.nnn with your ECL Watch ESP server's IP
+            address.</para>
+          </listitem>
+
+          <listitem>
+            <para>Create the configuration XML file</para>
+
+            <programlisting>&lt;Methods&gt;
+    &lt;Method name="AddThis" url="&lt;RoxieIP&gt;:9876" querytype="roxie" queryname="AddThis"/&gt;
+&lt;/Methods&gt;</programlisting>
+
+            <para>Where name is the name of your method(s) and url is the
+            Roxie server's IP and port and queryname is the published name
+            (alias) of the query. For a multi-node Roxie, you can use a range
+            in the form of nnn.nnn.nnn.n-nnn.</para>
+          </listitem>
+
+          <listitem>
+            <para>Save the file as MathServiceConfig.xml</para>
+          </listitem>
+
+          <listitem>
+            <para>Bind the service methods to the queries using a the
+            configuration XML.</para>
+
+            <programlisting>esdl bind-service myesp 8003 MathService.1 MathService --config MathServiceConfig.xml -s nnn.nnn.nnn.nnn -p 8010
+</programlisting>
+
+            <para>Where myesp is the name of your ESP process, 8003 is the
+            port you reserved for your Dynamic ESDL service.</para>
+
+            <para />
+          </listitem>
+
+          <listitem>
+            <para>Test the service using the new interface:<programlisting>http://&lt;<emphasis>node ip </emphasis>&gt;:8003</programlisting></para>
+          </listitem>
+        </orderedlist>
+      </para>
+    </sect1>
+  </chapter>
+
+  <xi:include href="HPCCClientTools/CT_Mods/CT_ESDL_CLI.xml"
+              xpointer="ESDL_CLI" xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+  <chapter>
+    <title>ESDL Language Reference</title>
+
+    <sect1 role="nobrk">
+      <title>ESDL Structures</title>
+
+      <para />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLESPstruct.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLESPrequest.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLESPresponse.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLESParray.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLESPenum.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLESPinclude.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLESPservice.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLESPmethod.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+    </sect1>
+
+    <sect1>
+      <title>ESDL Datatypes</title>
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLbooleanbool.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLstring.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLint.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLint64.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLfloat.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLdouble.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLbinary.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLTypeMapping.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+    </sect1>
+
+    <sect1>
+      <title>ESDL Attributes</title>
+
+      <para>You can use ESDL attributes to extend and override the default
+      behavior of an ESDL definition. For example, adding a max_len(n) to a
+      string defines the string will only need to store a certain number of
+      characters.</para>
+
+      <para>Many attributes are treated as hints that may have more effect on
+      some implementations than others. For example, max_len(n) will affect
+      generated ECL code, but is ignored when generating C++.</para>
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLmax_len.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLecl_max_len.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLecl_name.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLcounter_and_count_val.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLmax_count_var.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLecl_null.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLleading_zero.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLecl_hide.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLecl_type.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLecl_keep.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLmin_ver.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLmax_ver.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLdepr_ver.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLhelp.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <xi:include href="DynamicESDL/DESDL-Mods/ESDLdescription.xml"
+                  xpointer="element(/1)"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+    </sect1>
+  </chapter>
+</book>

docs/ECLWatch/TheECLWatchMan.xml

@@ -667,6 +667,97 @@
     </sect1>
   </chapter>
 
+  <chapter id="Plug-Ins_for_ECLWatch">
+    <title>Plug-ins</title>
+
+    <para>You can add functionality to ECL Watch with plug-ins. You can
+    integrate certain plug-ins into the ECL Watch interface. After you install
+    an approved plug-in, the plug-in icon displays in the navigation bar at
+    the top of the ECL Watch page. Click on the plug-in icon to view the
+    plug-ins page in ECL Watch. <figure>
+        <title>ECL Watch Plug-in icon</title>
+
+        <mediaobject>
+          <imageobject>
+            <imagedata fileref="images/ECLWaPlug.jpg" vendor="VM_welcome" />
+          </imageobject>
+        </mediaobject>
+      </figure></para>
+
+    <sect1 id="Ganglia_in_ECLWatch">
+      <title>Ganglia in ECL Watch</title>
+
+      <para>With the HPCC Systems<superscript>®</superscript>
+      Ganglia-monitoring plug-in installed, you can view the Ganglia
+      statistics and graphs through the ECL Watch interface. The default
+      monitoring displays several key statistics, but you can customize and
+      configure the views.</para>
+
+      <figure>
+        <title>Ganglia in ECL Watch</title>
+
+        <mediaobject>
+          <imageobject>
+            <imagedata fileref="images/GAN051.jpg" vendor="VM_welcome" />
+          </imageobject>
+        </mediaobject>
+      </figure>
+
+      <sect2 id="Customize_Monitoring" role="brk">
+        <title>Customize Monitoring</title>
+
+        <para>The default Ganglia page has a tab for Custom Monitoring where
+        you can easily add custom monitoring components. <figure>
+            <title>Ganglia Custom Monitoring </title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="images/ECLWa481.jpg" vendor="VM_welcome" />
+              </imageobject>
+            </mediaobject>
+          </figure></para>
+
+        <para>To customize the monitoring page;</para>
+
+        <para><orderedlist>
+            <listitem>
+              <para>Select the <emphasis role="bold">Custom
+              Monitoring</emphasis> tab.</para>
+            </listitem>
+
+            <listitem>
+              <para>Press the <emphasis role="bold">Metrics</emphasis>
+              button.</para>
+            </listitem>
+
+            <listitem>
+              <para>Use the drop menus to display the various graphing
+              utilities.</para>
+            </listitem>
+          </orderedlist></para>
+
+        <para><figure>
+            <title>Customize the Metrics</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="images/ECLWa482.jpg" vendor="VM_welcome" />
+              </imageobject>
+            </mediaobject>
+          </figure> </para>
+      </sect2>
+
+      <sect2>
+        <title>Installing Ganglia in ECL Watch</title>
+
+        <para>In order to use Ganglia in ECL Watch, you need to have Ganglia
+        installed on your HPCC System. For details on installing Ganglia for
+        ECL Watch, refer to the <emphasis>HPCC Monitoring and
+        Reporting</emphasis> manual. </para>
+      </sect2>
+    </sect1>
+  </chapter>
+
   <chapter id="Account_Info">
     <title>Advanced Menu</title>
 

docs/HPCCClientTools/CT_Mods/CT_Overview.xml

@@ -54,6 +54,12 @@
           facilitate automation of data file spray, despray, and other common
           file handling tasks.</entry>
         </row>
+
+        <row>
+          <entry><emphasis role="bold">ESDL</emphasis></entry>
+
+          <entry>Command line ESDL management tool.</entry>
+        </row>
       </tbody>
     </tgroup>
   </informaltable>

docs/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml

@@ -40,6 +40,12 @@
           facilitate automation of data file spray, despray, and other common
           file handling tasks.</entry>
         </row>
+
+        <row>
+          <entry><emphasis role="bold">ESDL</emphasis></entry>
+
+          <entry>Command line ESDL management tool.</entry>
+        </row>
       </tbody>
     </tgroup>
   </informaltable>
@@ -201,7 +207,7 @@
       the one you want to use last. Copy to a different folder or Rename the
       client tools found in /opt/HPCCSystems after installing the older
       version and before installing the newer version. This is to prevent the
-      newer client tools from overwriting the older one. </para>
+      newer client tools from overwriting the older one.</para>
 
       <para>To use the Client tools for the various version number(s)
       explicitly call the client tool you wish to use, or set up an alias to
@@ -223,8 +229,8 @@
 
       <para>Client tools for Windows installs in a directory such as:
       C:\Program Files (x86)\HPCCSystems\4.2.0\clientools where the number
-      (4.2.0 for example) corresponds to the version of the client tools.
-      </para>
+      (4.2.0 for example) corresponds to the version of the client
+      tools.</para>
 
       <para>The Windows installer will prompt you to delete the previous
       version during installation. If you want to keep both, decline the offer

docs/HPCCClientTools/ClientTools.xml

@@ -75,4 +75,8 @@
   <xi:include href="HPCCClientTools/CT_Mods/CT_Comm_Line_DFU.xml"
               xpointer="Command_Line_DFU"
               xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+  <xi:include href="HPCCClientTools/CT_Mods/CT_ESDL_CLI.xml"
+              xpointer="ESDL_CLI" xmlns:xi="http://www.w3.org/2001/XInclude" />
+              
 </book>

docs/HPCCClientTools/TheECLIDEandHPCCClientTools.xml

@@ -79,4 +79,6 @@
   <xi:include href="HPCCClientTools/CT_Mods/CT_Comm_Line_DFU.xml"
               xpointer="Command_Line_DFU"
               xmlns:xi="http://www.w3.org/2001/XInclude" />
+  <xi:include href="HPCCClientTools/CT_Mods/CT_ESDL_CLI.xml"
+              xpointer="ESDL_CLI" xmlns:xi="http://www.w3.org/2001/XInclude" />
 </book>

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -1195,7 +1195,55 @@ lock=/var/lock/HPCCSystems</programlisting>
       Tutorial</emphasis> documents.</para>
     </sect1>
 
-    <!--add-certify-->
+    <sect1 id="Redefining_Thor_Nodes">
+      <title>Redefining nodes in a Thor Cluster</title>
+
+      <para>To reconfigure a Thor cluster where you replace existing nodes
+      (with new IP's) or add or remove nodes, you must take an additional step
+      to restructure the group. Dali will not automatically restructure an
+      existing group.</para>
+
+      <para>This is because existing published files reference the previous
+      cluster group state by name and therefore changing its structure would
+      invalidate those files and make the physical files inaccessible.</para>
+
+      <para>There are a couple of scenarios where you would want to redefine
+      your Thor cluster. </para>
+
+      <sect2 id="Replace_Faulty_Node" role="nobrk">
+        <title>Replacing faulty node(s)</title>
+
+        <para>If data files are replicated, replacing a node and forcing the
+        new group to be used by existing files may be desirable. In this
+        scenario, reading an existing file will failover to finding a part on
+        the replicate node, when it tries to find a physical file on the new
+        replacement node.</para>
+
+        <para>To force the new group to be used, use the following
+        command:</para>
+
+        <para><programlisting>updtdalienv &lt;environment_file&gt; -f</programlisting>
+        In cases where there is no replication, data loss may be unavoidable
+        and forcing the new group may still be the best option.</para>
+      </sect2>
+
+      <sect2 id="SysAdmin_Resizing_The_Cluster" role="nobrk">
+        <title>Resizing the cluster </title>
+
+        <para>If you are adding or removing Thor cluster nodes but
+        <emphasis>all previous nodes remain part of the environment and
+        accessible</emphasis>, you can <emphasis role="bold">rename</emphasis>
+        the group that is associated with the Thor cluster (or the Cluster
+        name if there is no group name).</para>
+
+        <para>This will ensure all previously existing files, continue to use
+        the old group structure, while new files use the new group
+        structure.</para>
+
+        <para>In summary, if the Thor cluster changes it must be updated in
+        the Dali.</para>
+      </sect2>
+    </sect1>
   </chapter>
 
   <chapter id="Best_Practices_Chapter">

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/Hardware.xml

@@ -400,6 +400,10 @@
             </listitem>
 
             <listitem>
+              <para>64-bit CentOS 7</para>
+            </listitem>
+
+            <listitem>
               <para>64-bit RedHat Enterprise 5</para>
             </listitem>
 

docs/RDDERef/RDDERef.xml

@@ -111,7 +111,7 @@
     <para>Typically, Roxie results are returned to the requester rather than
     writing a result to a file. However, Roxie can write data files, although
     you generally would not want to write a file when a query is not
-    workunit-based. </para>
+    workunit-based.</para>
 
     <sect1 id="Roxie_Overview">
       <title>Roxie Overview</title>
@@ -537,72 +537,17 @@
   <chapter id="Deploying-Data-to-a-Cluster-using-DFU">
     <title>Deploying Data to a Roxie Cluster using DFU</title>
 
-    <para>You can use the Distributed File Utility (DFU) in ECL Watch to
-    spray, copy, or remote copy data files to a Roxie cluster. This allows you
-    to copy large files to a cluster in advance of publishing a query. If data
-    files are copied in advance, a query which requires those files will use
-    the ones already in place. If you have large data sets, this allows you to
+    <para>You can use the Distributed File Utility (DFU) in ECL Watch to copy,
+    or remote copy data files to a Roxie cluster. This allows you to copy
+    large files to a cluster in advance of publishing a query. If data files
+    are copied in advance, a query which requires those files will use the
+    ones already in place. If you have large data sets, this allows you to
     prepare in advance of query deployment.</para>
 
     <para><emphasis role="bold">Note:</emphasis> To use this feature, the
     FTSlave utility must be installed to each node in the cluster. This is
     done automatically by the Configuration Manager Wizard.</para>
 
-    <sect1 id="LZ_Spray">
-      <title>Spray Files</title>
-
-      <para>You can spray files to your Roxie clusters from the Landing Zone
-      page. <orderedlist>
-          <listitem>
-            <para>In ECL Watch, select the file from your drop zone by
-            checking the box next to it.</para>
-          </listitem>
-
-          <listitem>
-            <para>Select the appropriate drop menu option for the type of
-            spray you want.</para>
-
-            <para>For example, to spray a delimited file, select the <emphasis
-            role="bold">Delimited</emphasis> drop menu option.</para>
-
-            <para><figure>
-                <title>Landing Zone Spray</title>
-
-                <mediaobject>
-                  <imageobject>
-                    <imagedata fileref="images/RM441.jpg" />
-                  </imageobject>
-                </mediaobject>
-              </figure></para>
-          </listitem>
-
-          <listitem>
-            <para>Fill in the values as appropriate for the spray.</para>
-          </listitem>
-
-          <listitem>
-            <para>Press the <emphasis role="bold">Spray</emphasis> button to
-            spray the file.</para>
-          </listitem>
-        </orderedlist></para>
-
-      <sect2 id="multi-fileSpray">
-        <title>Spray multiple files</title>
-
-        <para>You can choose to spray multiple files with the multi-file spray
-        feature. This feature is useful for spraying a number of files of the
-        same type and the same scope using the same spray options.</para>
-
-        <para>Fixed (length) files can have different record lengths and XML
-        files can have different row tags which must be specified individually
-        for each file. To specify these differences select the files you want
-        to spray and the spray type. You will then see the files listed. Enter
-        the record length or row tag information for each file if using the
-        Fixed or XML spray types, then check all other applicable options and
-        Spray.</para>
-      </sect2>
-    </sect1>
-
     <sect1 id="Copy">
       <title>DFU Copy</title>
 

ecl/eclcc/eclcc.cpp

@@ -819,15 +819,18 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
                 }
             }
         }
-        catch (IException * e)
+        catch (IException * _e)
         {
-            if (e->errorCode() != HQLERR_ErrorAlreadyReported)
+            Owned<IException> e = _e;
+            unsigned errCode = e->errorCode();
+            if (errCode != HQLERR_ErrorAlreadyReported)
             {
                 StringBuffer exceptionText;
                 e->errorMessage(exceptionText);
-                errorProcessor.reportError(ERR_INTERNALEXCEPTION, exceptionText.toCharArray(), queryFullName, 1, 0, 0);
+                if (errCode == 0)
+                    errCode = ERR_INTERNALEXCEPTION;
+                errorProcessor.reportError(errCode, exceptionText.toCharArray(), queryFullName, 1, 0, 0);
             }
-            e->Release();
         }
 
         try

ecl/hql/hqlerrors.hpp

@@ -477,6 +477,7 @@
 #define HQLERR_AtmostLegacyMismatch             3134
 #define HQLERR_PropertyArgumentNotConstant      3135
 #define HQLERR_InvalidErrorCategory             3136
+#define HQLERR_MultipleHashWebserviceCalls      3137
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contains an illegal cycle/recursive definition %s"
@@ -515,6 +516,7 @@
 #define HQLERR_AtmostLegacyMismatch_Text        "Legacy JOIN condition on field[1..*] should be included in the optional fields"
 #define HQLERR_PropertyArgumentNotConstant_Text "The argument to attribute '%s' must be a constant"
 #define HQLERR_InvalidErrorCategory_Text        "Unrecognised ONWARNING category '%s'"
+#define HQLERR_MultipleHashWebserviceCalls_Text "#webservice can only be called once"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */

ecl/hqlcpp/hqlttcpp.cpp

@@ -284,7 +284,11 @@ void NewThorStoredReplacer::doAnalyseBody(IHqlExpression * expr)
         IAtom * kind = expr->queryChild(0)->queryName();
         if (kind == webserviceAtom)
         {
-            Owned<IWUWebServicesInfo> wsi = wu->updateWebServicesInfo(true);
+            Owned<IWUWebServicesInfo> wsi = wu->updateWebServicesInfo(false);
+            if (wsi)
+                throwError(HQLERR_MultipleHashWebserviceCalls);
+            wsi.setown(wu->updateWebServicesInfo(true));
+
             IHqlExpression *wsExpr = expr->queryChild(0);
             ForEachChild(i, wsExpr)
             {

ecl/regress/errwebservice.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+string descrText := 'only the listed fields should show in form, xsd, and wsdl<br/>and in the given order<br/><br/>';
+string helpText := 'Enter some values and hit submit';
+
+#webservice(fields('u1', 'i1'), help(helpText), description(descrText));
+#webservice(help('can not use #webservice more than once'), description('multiple #webservice error'));
+
+integer1 i1 := 0 : stored('i1');
+unsigned1 u1 := 0 : stored('u1');
+
+output (i1, named('i1'));
+output (u1, named('u1'));
+

plugins/memcached/lib_memcached.ecllib

@@ -16,14 +16,14 @@
 ############################################################################## */
 
 export memcached := SERVICE : plugin('memcached'), namespace('MemCachedPlugin')
-  SetUnicode(CONST VARSTRING key, CONST UNICODE value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetString(CONST VARSTRING key, CONST STRING value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetUtf8(CONST VARSTRING key, CONST UTF8 value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSetUtf8';
-  SetBoolean(CONST VARSTRING key, BOOLEAN value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetReal(CONST VARSTRING key, REAL value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetInteger(CONST VARSTRING key, INTEGER value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetUnsigned(CONST VARSTRING key, UNSIGNED value, CONST VARSTRING options, CONST VARSTRING partitionKey = '',  UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetData(CONST VARSTRING key, CONST DATA value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSetData';
+  SetUnicode(CONST VARSTRING key, CONST UNICODE value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetString(CONST VARSTRING key, CONST STRING value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetUtf8(CONST VARSTRING key, CONST UTF8 value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0) : cpp,action,context,entrypoint='MSetUtf8';
+  SetBoolean(CONST VARSTRING key, BOOLEAN value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetReal(CONST VARSTRING key, REAL value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetInteger(CONST VARSTRING key, INTEGER value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetUnsigned(CONST VARSTRING key, UNSIGNED value, CONST VARSTRING options, CONST VARSTRING partitionKey = '',  UNSIGNED expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetData(CONST VARSTRING key, CONST DATA value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0) : cpp,action,context,entrypoint='MSetData';
 
   INTEGER8 GetInteger(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetInt8';
   UNSIGNED8 GetUnsigned(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUint8';

plugins/memcached/memcachedplugin.cpp

@@ -41,7 +41,6 @@ ECL_MEMCACHED_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
 }
 
 namespace MemCachedPlugin {
-static const unsigned unitExpire = 86400;//1 day (secs)
 
 enum eclDataType {
     ECL_BOOLEAN,
@@ -89,8 +88,8 @@ public :
     ~MCached();
 
     //set
-    template <class type> void set(ICodeContext * ctx, const char * partitionKey, const char * key, type value, unsigned expire, eclDataType eclType);
-    template <class type> void set(ICodeContext * ctx, const char * partitionKey, const char * key, size32_t valueLength, const type * value, unsigned expire, eclDataType eclType);
+    template <class type> void set(ICodeContext * ctx, const char * partitionKey, const char * key, type value, unsigned __int64 expire, eclDataType eclType);
+    template <class type> void set(ICodeContext * ctx, const char * partitionKey, const char * key, size32_t valueLength, const type * value, unsigned __int64 expire, eclDataType eclType);
     //get
     template <class type> void get(ICodeContext * ctx, const char * partitionKey, const char * key, type & value, eclDataType eclType);
     template <class type> void get(ICodeContext * ctx, const char * partitionKey, const char * key, size_t & valueLength, type * & value, eclDataType eclType);
@@ -131,6 +130,7 @@ static const unsigned MAX_TYPEMISMATCHCOUNT = 10;
 
 static __thread MCached * cachedConnection;
 static __thread ThreadTermFunc threadHookChain;
+static __thread bool threadHooked;
 
 //The following class is here to ensure destruction of the cachedConnection within the main thread
 //as this is not handled by the thread hook mechanism.
@@ -141,26 +141,37 @@ public :
     ~mainThreadCachedConnection()
     {
         if (cachedConnection)
+        {
             cachedConnection->Release();
+            cachedConnection = NULL;
+        }
     }
 } mainThread;
 
 static void releaseContext()
 {
     if (cachedConnection)
+    {
         cachedConnection->Release();
+        cachedConnection = NULL;
+    }
     if (threadHookChain)
     {
         (*threadHookChain)();
         threadHookChain = NULL;
     }
+    threadHooked = false;
 }
 MCached * createConnection(ICodeContext * ctx, const char * options)
 {
     if (!cachedConnection)
     {
         cachedConnection = new MemCachedPlugin::MCached(ctx, options);
-        threadHookChain = addThreadTermFunc(releaseContext);
+        if (!threadHooked)
+        {
+            threadHookChain = addThreadTermFunc(releaseContext);
+            threadHooked = true;
+        }
         return LINK(cachedConnection);
     }
 
@@ -168,18 +179,19 @@ MCached * createConnection(ICodeContext * ctx, const char * options)
         return LINK(cachedConnection);
 
     cachedConnection->Release();
+    cachedConnection = NULL;
     cachedConnection = new MemCachedPlugin::MCached(ctx, options);
     return LINK(cachedConnection);
 }
 
 //-------------------------------------------SET-----------------------------------------
-template<class type> void MSet(ICodeContext * ctx, const char * _options, const char * partitionKey, const char * key, type value, unsigned expire, eclDataType eclType)
+template<class type> void MSet(ICodeContext * ctx, const char * _options, const char * partitionKey, const char * key, type value, unsigned __int64 expire, eclDataType eclType)
 {
     OwnedMCached serverPool = createConnection(ctx, _options);
     serverPool->set(ctx, partitionKey, key, value, expire, eclType);
 }
 //Set pointer types
-template<class type> void MSet(ICodeContext * ctx, const char * _options, const char * partitionKey, const char * key, size32_t valueLength, const type * value, unsigned expire, eclDataType eclType)
+template<class type> void MSet(ICodeContext * ctx, const char * _options, const char * partitionKey, const char * key, size32_t valueLength, const type * value, unsigned __int64 expire, eclDataType eclType)
 {
     OwnedMCached serverPool = createConnection(ctx, _options);
     serverPool->set(ctx, partitionKey, key, valueLength, value, expire, eclType);
@@ -202,25 +214,25 @@ void MGetVoidPtrLenPair(ICodeContext * ctx, const char * options, const char * p
 }
 
 //----------------------------------SET----------------------------------------
-template<class type> void MemCachedPlugin::MCached::set(ICodeContext * ctx, const char * partitionKey, const char * key, type value, unsigned expire, eclDataType eclType)
+template<class type> void MemCachedPlugin::MCached::set(ICodeContext * ctx, const char * partitionKey, const char * key, type value, unsigned __int64 expire, eclDataType eclType)
 {
     const char * _value = reinterpret_cast<const char *>(&value);//Do this even for char * to prevent compiler complaining
     size_t partitionKeyLength = strlen(partitionKey);
     const char * msg = "'Set' request failed - ";
     if (partitionKeyLength)
-        assertOnError(memcached_set_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), _value, sizeof(value), (time_t)(expire*unitExpire), (uint32_t)eclType), msg);
+        assertOnError(memcached_set_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), _value, sizeof(value), (time_t)expire, (uint32_t)eclType), msg);
     else
-        assertOnError(memcached_set(connection, key, strlen(key), _value, sizeof(value), (time_t)(expire*unitExpire), (uint32_t)eclType), msg);
+        assertOnError(memcached_set(connection, key, strlen(key), _value, sizeof(value), (time_t)expire, (uint32_t)eclType), msg);
 }
-template<class type> void MemCachedPlugin::MCached::set(ICodeContext * ctx, const char * partitionKey, const char * key, size32_t valueLength, const type * value, unsigned expire, eclDataType eclType)
+template<class type> void MemCachedPlugin::MCached::set(ICodeContext * ctx, const char * partitionKey, const char * key, size32_t valueLength, const type * value, unsigned __int64 expire, eclDataType eclType)
 {
     const char * _value = reinterpret_cast<const char *>(value);//Do this even for char * to prevent compiler complaining
     size_t partitionKeyLength = strlen(partitionKey);
     const char * msg = "'Set' request failed - ";
     if (partitionKeyLength)
-        assertOnError(memcached_set_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), _value, (size_t)(valueLength), (time_t)(expire*unitExpire), (uint32_t)eclType), msg);
+        assertOnError(memcached_set_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), _value, (size_t)(valueLength), (time_t)expire, (uint32_t)eclType), msg);
     else
-        assertOnError(memcached_set(connection, key, strlen(key), _value, (size_t)(valueLength), (time_t)(expire*unitExpire), (uint32_t)eclType), msg);
+        assertOnError(memcached_set(connection, key, strlen(key), _value, (size_t)(valueLength), (time_t)expire, (uint32_t)eclType), msg);
 }
 //----------------------------------GET----------------------------------------
 template<class type> void MemCachedPlugin::MCached::get(ICodeContext * ctx, const char * partitionKey, const char * key, type & returnValue, eclDataType eclType)
@@ -563,7 +575,9 @@ void MemCachedPlugin::MCached::setPoolSettings()
     assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS, 1), msg);
 #endif
     assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_NO_BLOCK, 0), msg);
-    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT, 100), msg);//units of ms MORE: What should I set this to or get from?
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT, 1000), msg);//units of ms.
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_SND_TIMEOUT, 1000000), msg);//units of mu-s.
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_RCV_TIMEOUT, 1000000), msg);//units of mu-s.
     assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 0), msg);// Buffering does not work with the ecl runtime paradigm
 }
 
@@ -628,35 +642,35 @@ ECL_MEMCACHED_API const char * ECL_MEMCACHED_CALL MKeyType(ICodeContext * ctx, c
 }
 //-----------------------------------SET------------------------------------------
 //NOTE: These were all overloaded by 'value' type, however; this caused problems since ecl implicitly casts and doesn't type check.
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned __int64 expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, valueLength, value, expire, MemCachedPlugin::ECL_STRING);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, size32_t valueLength, const UChar * value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, size32_t valueLength, const UChar * value, const char * options, const char * partitionKey, unsigned __int64 expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, (valueLength)*sizeof(UChar), value, expire, MemCachedPlugin::ECL_UNICODE);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, signed __int64 value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, signed __int64 value, const char * options, const char * partitionKey, unsigned __int64 expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, value, expire, MemCachedPlugin::ECL_INTEGER);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, unsigned __int64 value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, unsigned __int64 value, const char * options, const char * partitionKey, unsigned __int64 expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, value, expire, MemCachedPlugin::ECL_UNSIGNED);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, double value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, double value, const char * options, const char * partitionKey, unsigned __int64 expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, value, expire, MemCachedPlugin::ECL_REAL);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, bool value, const char * options, const char * partitionKey, unsigned expire)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, bool value, const char * options, const char * partitionKey, unsigned __int64 expire)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, value, expire, MemCachedPlugin::ECL_BOOLEAN);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetData(ICodeContext * ctx, const char * key, size32_t valueLength, const void * value, const char * options, const char * partitionKey, unsigned expire)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetData(ICodeContext * ctx, const char * key, size32_t valueLength, const void * value, const char * options, const char * partitionKey, unsigned __int64 expire)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, valueLength, value, expire, MemCachedPlugin::ECL_DATA);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned __int64 expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, rtlUtf8Size(valueLength, value), value, expire, MemCachedPlugin::ECL_UTF8);
 }

plugins/memcached/memcachedplugin.hpp

@@ -44,14 +44,14 @@ extern "C++"
 {
 namespace MemCachedPlugin {
     //--------------------------SET----------------------------------------
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, bool value, const char * options, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, signed __int64 value, const char * options, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, unsigned __int64 value, const char * options, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, double value, const char * options, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * _ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, size32_t valueLength, const UChar * value, const char * options, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetData(ICodeContext * _ctx, const char * key, size32_t valueLength, const void * value, const char * options, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, bool value, const char * options, const char * partitionKey, unsigned __int64 expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, signed __int64 value, const char * options, const char * partitionKey, unsigned __int64 expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, unsigned __int64 value, const char * options, const char * partitionKey, unsigned __int64 expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, double value, const char * options, const char * partitionKey, unsigned __int64 expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * _ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned __int64 expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned __int64 expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, size32_t valueLength, const UChar * value, const char * options, const char * partitionKey, unsigned __int64 expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetData(ICodeContext * _ctx, const char * key, size32_t valueLength, const void * value, const char * options, const char * partitionKey, unsigned __int64 expire);
     //--------------------------GET----------------------------------------
     ECL_MEMCACHED_API bool             ECL_MEMCACHED_CALL MGetBool  (ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
     ECL_MEMCACHED_API signed __int64   ECL_MEMCACHED_CALL MGetInt8  (ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);

plugins/redis/redis.cpp

@@ -43,6 +43,7 @@ class Connection;
 static const char * REDIS_LOCK_PREFIX = "redis_ecl_lock";
 static __thread Connection * cachedConnection;
 static __thread ThreadTermFunc threadHookChain;
+static __thread bool threadHooked;
 
 static void * allocateAndCopy(const void * src, size_t size)
 {
@@ -158,7 +159,10 @@ public :
     ~MainThreadCachedConnection()
     {
         if (cachedConnection)
+        {
             cachedConnection->Release();
+            cachedConnection = NULL;
+        }
     }
 } mainThread;
 
@@ -174,6 +178,7 @@ static void releaseContext()
         (*threadHookChain)();
         threadHookChain = NULL;
     }
+    threadHooked = false;
 }
 Connection::Connection(ICodeContext * ctx, const char * _options, unsigned __int64 _database, const char * password, unsigned __int64 _timeout)
   : database(0), timeout(_timeout), port(0), serverIpPortPasswordHash(hashServerIpPortPassword(ctx, _options, password))
@@ -290,7 +295,11 @@ Connection * Connection::createConnection(ICodeContext * ctx, const char * optio
     if (!cachedConnection)
     {
         cachedConnection = new Connection(ctx, options, _database, password, _timeout);
-        threadHookChain = addThreadTermFunc(releaseContext);
+        if (!threadHooked)
+        {
+            threadHookChain = addThreadTermFunc(releaseContext);
+            threadHooked = true;
+        }
         return LINK(cachedConnection);
     }
 

testing/regress/ecl/key/memcachedtest.xml

@@ -37,3 +37,12 @@
 <Dataset name='Result 13'>
  <Row><Result_13>Nonexistent</Result_13></Row>
 </Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>true</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>true</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>false</Result_16></Row>
+</Dataset>

testing/regress/ecl/memcachedtest.ecl

@@ -16,46 +16,77 @@
 ############################################################################## */
 
 IMPORT memcached FROM lib_memcached;
+IMPORT Std;
 
 STRING servers := '--SERVER=127.0.0.1:11211';
-memcached.Clear(servers);
+SEQUENTIAL(
+    memcached.Clear(servers);
 
-memcached.SetBoolean('b', TRUE, servers);
-memcached.GetBoolean('b', servers);
+    memcached.SetBoolean('b', TRUE, servers);
+    memcached.GetBoolean('b', servers);
+    );
 
 REAL pi := 3.14159265359;
-memcached.SetReal('pi', pi, servers);
-memcached.GetReal('pi', servers);
+SEQUENTIAL(
+    memcached.SetReal('pi', pi, servers);
+    memcached.GetReal('pi', servers);
+    );
 
 INTEGER i := 123456789;
-memcached.SetInteger('i', i, servers);
-memcached.GetInteger('i', servers);
+SEQUENTIAL(
+    memcached.SetInteger('i', i, servers);
+    memcached.GetInteger('i', servers);
+    );
 
 UNSIGNED u := 7;
-memcached.SetUnsigned('u', u, servers);
-memcached.GetUnsigned('u', servers);
+SEQUENTIAL(
+    memcached.SetUnsigned('u', u, servers);
+    memcached.GetUnsigned('u', servers);
+    );
 
 STRING str  := 'supercalifragilisticexpialidocious';
-memcached.SetString('str', str, servers);
-memcached.GetString('str', servers);
+SEQUENTIAL(
+    memcached.SetString('str', str, servers);
+    memcached.GetString('str', servers);
+    );
 
 UNICODE uni := U'אבגדהוזחטיךכלםמןנסעףפץצקרשת';
-memcached.SetUnicode('uni', uni, servers);
-memcached.GetUnicode('uni', servers);
+SEQUENTIAL(
+    memcached.SetUnicode('uni', uni, servers);
+    memcached.GetUnicode('uni', servers);
+    );
 
 UTF8 utf := U'אבגדהוזחטיךכלםמןנסעףפץצקרשת';
-memcached.SetUtf8('utf8', utf, servers);
-memcached.GetUtf8('utf8', servers);
+SEQUENTIAL(
+    memcached.SetUtf8('utf8', utf, servers);
+    memcached.GetUtf8('utf8', servers);
+    );
 
 DATA mydata := x'd790d791d792d793d794d795d796d798d799d79ad79bd79cd79dd79dd79ed79fd7a0d7a1d7a2d7a3d7a4d7a5d7a6d7a7d7a8d7a9d7aa';
-memcached.SetData('data', mydata, servers);
-memcached.GetData('data', servers);
+SEQUENTIAL(
+    memcached.SetData('data', mydata, servers);
+    memcached.GetData('data', servers);
+    );
 
-memcached.Exists('utf8', servers);
-memcached.KeyType('utf8', servers);
+SEQUENTIAL(
+    memcached.Exists('utf8', servers);
+    memcached.KeyType('utf8', servers);
+    );
 
 //The following test some exceptions
-memcached.GetInteger('pi', servers);
-memcached.Clear(servers);
-memcached.Exists('utf8', servers);
-memcached.KeyType('utf8', servers);
+SEQUENTIAL(
+    memcached.GetInteger('pi', servers);
+    memcached.Clear(servers);
+    memcached.Exists('utf8', servers);
+    memcached.KeyType('utf8', servers);
+    );
+
+SEQUENTIAL(
+    memcached.Clear(servers);
+    memcached.SetString('testExpire', 'foobar', servers,, 10);
+    memcached.Exists('testExpire', servers);
+    Std.System.Debug.Sleep(9 * 1000);
+    memcached.Exists('testExpire', servers);
+    Std.System.Debug.Sleep(1 * 1000);
+    memcached.Exists('testExpire', servers);
+    );

testing/regress/ecl/redislockingtest.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//nothor
+
 IMPORT redisServer FROM lib_redis;
 IMPORT Std;
 

testing/regress/ecl/redissynctest.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//nothor
+
 IMPORT redis FROM lib_redis;
 IMPORT Std;
 

testing/regress/ecl/sendlimit.ecl

@@ -15,11 +15,17 @@
     limitations under the License.
 ############################################################################## */
 
-//This is primarily here to ensure that roxie fails gracefully when a packet that is too large is sent from the master to the slave
+// This is here to ensure that roxie fails gracefully when a packet that is too large
+// is sent from the master to the slave. Note that the limit is not applied when localSlave is set,
+// so the test is meaningless - in order to ensure that the results still match on such systems,
+// we simply fake the expected output.
+
 //nothor
 //nothorlcr
 //nohthor
 
+import Std.Str;
+
 string s10 := (string10)'' : stored ('s10');
 string s100 := s10 + s10 + s10 + s10 + s10 + s10 + s10 + s10 + s10 + s10 : stored ('s100');
 string s1000 := s100 + s100 + s100 + s100 + s100 + s100 + s100 + s100 + s100 + s100 : stored ('s1000');
@@ -30,6 +36,8 @@ string myStoredString := 'x' + s100000 + 'x';
 
 rec := { string x{maxlength(100)}, unsigned value };
 
+expected := DATASET([{'Packet length', 1446}], rec);
+
 ds := dataset([{'a',1},{'b',2},{'c',3},{'d',4}], rec);
 
 
@@ -42,5 +50,7 @@ END;
 
 caught := catch(allnodes(f(ds)), onfail(FailTransform));
 
-output(caught);
+string localSlave := getEnv('control:localSlave');
+
+output(IF(localSlave='1' OR Str.toLowerCase(localSlave)='true', expected, caught));
 

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -438,18 +438,18 @@ protected:
                         rawSz = 0;
                         msg.clear();
                     }
-                    owner.decTotal(writerTotalSz);
-                    writerTotalSz = 0;
                     // see if others to process
                     // NB: this will never start processing a bucket for a destination which already has an active writer.
                     CriticalBlock b(owner.activeWritersLock);
+                    owner.decTotal(writerTotalSz);
                     target->decActiveWriters();
                     sendBucket.setown(owner.getAnotherBucket(nextPending));
-                    if (!sendBucket)
+                    if (!sendBucket) // 0 pending buckets to any target OR those that are pending have enough handlers (>targetWriterLimit)
                     {
                         target = NULL; // will be reinitialized to new target in init(), when thread pool thread is reused
                         break;
                     }
+                    writerTotalSz = 0; // now reset for new bucket to send
                     dest = sendBucket->queryDestination();
                     target = owner.targets.item(dest);
                     target->incActiveWriters();
@@ -765,7 +765,7 @@ protected:
                         loop
                         {
                             if (timer.elapsedCycles() >= queryOneSecCycles()*10)
-                                owner.ActPrintLog("HD sender, waiting for space, active writers = %d", queryInactiveWriters());
+                                owner.ActPrintLog("HD sender, waiting for space, inactive writers = %d, totalSz = %d", queryInactiveWriters(), queryTotalSz());
                             timer.reset();
 
                             if (senderFullSem.wait(10000))

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -1815,7 +1815,12 @@ protected:
                 if (!success)
                 {
                     ActPrintLog("Out of memory trying to allocate [LOCAL] tables for a SMART join (%" RIPF "d rows), will now failover to a std hash join", rhs.ordinality());
-                    Owned<IThorRowCollector> collector = createThorRowCollector(*this, queryRowInterfaces(rightITDL), NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_LOOKUPJOIN);
+                    Owned<IThorRowCollector> collector;
+                    if (isSmart())
+                        collector.setown(createThorRowCollector(*this, queryRowInterfaces(rightITDL), NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_LOOKUPJOIN));
+                    else
+                        collector.setown(createThorRowCollector(*this, queryRowInterfaces(rightITDL), NULL, stableSort_none, rc_allMem, SPILL_PRIORITY_DISABLE));
+
                     collector->setOptions(rcflag_noAllInMemSort); // If fits into memory, don't want it resorted
                     collector->transferRowsIn(rhs); // can spill after this
                     rightStream.setown(collector->getStream());

tools/esdlcmd/esdl2ecl.cpp

@@ -188,8 +188,14 @@ public:
 class Esdl2EclCmd : public EsdlConvertCmd
 {
 public:
-    Esdl2EclCmd() : optGenerateAllIncludes(false), optOutputExpandedXML(false), optHPCCCompFilesDir(COMPONENTFILES_DIR)
-    {}
+    Esdl2EclCmd() : optGenerateAllIncludes(false), optOutputExpandedXML(false)
+    {
+        StringBuffer componentsfolder;
+        if (getComponentFilesRelPathFromBin(componentsfolder))
+            optHPCCCompFilesDir.set(componentsfolder.str());
+        else
+            optHPCCCompFilesDir.set(COMPONENTFILES_DIR);
+    }
 
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {

tools/esdlcmd/esdlcmd_common.hpp

@@ -27,6 +27,9 @@
 #include "ws_esdlconfig_esp.ipp"
 #include "esdl2xml.hpp"
 
+#define COMPONENTS_DIR_NAME "componentfiles"
+#define HIGHER_DIR_RELATIVE ".."
+
 //=========================================================================================
 
 interface IEsdlCommand : extends IInterface
@@ -300,4 +303,27 @@ public:
     virtual void setTransformParams(IProperties *params )=0;
 };
 
+static bool getCurrentFolder(StringBuffer & path)
+{
+    StringBuffer folder;
+    splitDirTail(queryCurrentProcessPath(), folder);
+    removeTrailingPathSepChar(folder);
+    if (folder.length())
+    {
+        path = folder;
+        return true;
+    }
+    return false;
+}
+
+static bool getComponentFilesRelPathFromBin(StringBuffer & path)
+{
+    if (getCurrentFolder(path))
+    {
+        path.appendf("%c%s%c%s", PATHSEPCHAR, HIGHER_DIR_RELATIVE,PATHSEPCHAR,COMPONENTS_DIR_NAME);
+        return true;
+    }
+
+    return false;
+}
 #endif

tools/esdlcmd/esdlcmd_core.cpp

@@ -145,7 +145,13 @@ public:
         }
 
         if (optXsltPath.isEmpty())
-            optXsltPath.set(COMPONENTFILES_DIR);
+        {
+            StringBuffer tmp;
+            if (getComponentFilesRelPathFromBin(tmp))
+                optXsltPath.set(tmp.str());
+            else
+                optXsltPath.set(COMPONENTFILES_DIR);
+        }
 
         fullxsltpath.set(optXsltPath);
         fullxsltpath.append("/xslt/esxdl2xsd.xslt");