Merge remote-tracking branch 'origin/candidate-4.0.0' into closedown-4.0.x

Conflicts:
	version.cmake

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

build_utils/cleanDeb.sh

@@ -25,6 +25,7 @@ DNAME=$(basename ${FNAME} .deb)
 mkdir -p ${DNAME}/DEBIAN
 fakeroot dpkg-deb -x ${FNAME} ${DNAME}
 fakeroot dpkg-deb -e ${FNAME} ${DNAME}/DEBIAN
+fakeroot chmod 0644 ${DNAME}/DEBIAN/md5sums
 rm ${FNAME}
 fakeroot dpkg-deb -b ${DNAME} ${FNAME}
 rm -rf ${DNAME}

cmake_modules/FindJNI.cmake

@@ -163,6 +163,7 @@ JAVA_APPEND_LIBRARY_DIRECTORIES(JAVA_AWT_LIBRARY_DIRECTORIES
   /usr/lib/jvm/java-1.5.0-sun/jre/lib/{libarch}
   /usr/lib/jvm/java-6-sun-1.6.0.00/jre/lib/{libarch}       # can this one be removed according to #8821 ? Alex
   /usr/lib/jvm/java-6-openjdk-{libarch}/jre/lib/{libarch}  # Ubuntu 12.10 location
+  /usr/lib/jvm/java-7-openjdk-{libarch}/jre/lib/{libarch}  # Ubuntu 13.04 location
   /usr/lib/jvm/java-6-openjdk/jre/lib/{libarch}
   /usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0/jre/lib/{libarch}        # fedora
   # Debian specific paths for default JVM
@@ -182,10 +183,7 @@ FOREACH(dir ${JAVA_AWT_LIBRARY_DIRECTORIES})
 ENDFOREACH(dir)
 
 
-SET(JAVA_AWT_INCLUDE_DIRECTORIES
-  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.4;JavaHome]/include"
-  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.3;JavaHome]/include"
-  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\${java_install_version};JavaHome]/include"
+JAVA_APPEND_LIBRARY_DIRECTORIES(JAVA_AWT_EXPANDED_INCLUDE_DIRECTORIES
   ${_JAVA_HOME}/include
   /usr/include
   /usr/local/include
@@ -195,7 +193,8 @@ SET(JAVA_AWT_INCLUDE_DIRECTORIES
   /usr/lib/jvm/java-6-sun/include
   /usr/lib/jvm/java-1.5.0-sun/include
   /usr/lib/jvm/java-6-sun-1.6.0.00/include       # can this one be removed according to #8821 ? Alex
-  /usr/lib/jvm/java-6-openjdk-amd64/jre/lib/{libarch}  # Ubuntu 12.10 location
+  /usr/lib/jvm/java-6-openjdk-{libarch}/jre/lib/{libarch}  # Ubuntu 12.10 location
+  /usr/lib/jvm/java-7-openjdk-{libarch}/include  # Ubuntu 13.04 location
   /usr/lib/jvm/java-6-openjdk/include
   /usr/local/share/java/include
   /usr/lib/j2sdk1.4-sun/include
@@ -205,6 +204,13 @@ SET(JAVA_AWT_INCLUDE_DIRECTORIES
   /usr/lib/jvm/default-java/include
   )
 
+SET (JAVA_AWT_INCLUDE_DIRECTORIES
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.4;JavaHome]/include"
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.3;JavaHome]/include"
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\${java_install_version};JavaHome]/include"
+  ${JAVA_AWT_EXPANDED_INCLUDE_DIRECTORIES}
+  )
+
 FOREACH(JAVA_PROG "${JAVA_RUNTIME}" "${JAVA_COMPILE}" "${JAVA_ARCHIVE}")
   GET_FILENAME_COMPONENT(jpath "${JAVA_PROG}" PATH)
   FOREACH(JAVA_INC_PATH ../include ../java/include ../share/java/include)

cmake_modules/FindLIBARCHIVE.cmake

@@ -23,7 +23,7 @@
 
 IF (NOT LIBARCHIVE_FOUND)
   IF (WIN32)
-    SET (libarchive_lib "libarchive")
+    SET (libarchive_lib "archive_static")
   ELSE()
     SET (libarchive_lib "archive")
   ENDIF()

cmake_modules/commonSetup.cmake

@@ -53,10 +53,11 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(USE_OPENSSL "Configure use of OpenSSL" ON)
   option(USE_ZLIB "Configure use of zlib" ON)
   if (WIN32)
-    option(USE_LIBARCHIVE "Configure use of libarchive" OFF)   # libarchive rather less standard on windows systems
+    option(USE_GIT "Configure use of GIT (Hooks)" OFF)
   else()
-    option(USE_LIBARCHIVE "Configure use of libarchive" ON)
+    option(USE_GIT "Configure use of GIT (Hooks)" ON)
   endif()
+  option(USE_LIBARCHIVE "Configure use of libarchive" ON)
   option(USE_URIPARSER "Configure use of uriparser" OFF)
   option(USE_NATIVE_LIBRARIES "Search standard OS locations for thirdparty libraries" ON)
   option(USE_GIT_DESCRIBE "Use git describe to generate build tag" ON)
@@ -539,6 +540,17 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       endif(USE_ZLIB)
 
       if(USE_LIBARCHIVE)
+        if (WIN32)
+          if(NOT USE_ZLIB)
+            message(FATAL ERROR "LIBARCHIVE requires ZLIB")
+          endif(NOT USE_ZLIB)
+          find_package(BZip2)
+          if (BZIP2_FOUND)
+            add_definitions (-D_USE_BZIP2)
+          else()
+            message(FATAL_ERROR "LIBARCHIVE requires BZIP2 but package not found")
+          endif()
+        endif (WIN32)
         find_package(LIBARCHIVE)
         if (LIBARCHIVE_FOUND)
           add_definitions (-D_USE_LIBARCHIVE)

common/remote/CMakeLists.txt

@@ -24,7 +24,7 @@
 
 project( remote ) 
 
-HPCC_ADD_SUBDIRECTORY(hooks "PLATFORM")
+HPCC_ADD_SUBDIRECTORY(hooks)
 
 set (    SRCS 
          rmtfile.cpp 

common/remote/hooks/CMakeLists.txt

@@ -24,7 +24,9 @@
 
 project( hooks )
 
-add_subdirectory(git)
+IF (USE_GIT)
+  add_subdirectory(git)
+ENDIF()
 IF (USE_LIBARCHIVE)
   add_subdirectory(libarchive)
 ENDIF()

common/remote/hooks/libarchive/CMakeLists.txt

@@ -37,11 +37,21 @@ include_directories (
 include_directories ( ${LIBARCHIVE_INCLUDE_DIR} )
 
 ADD_DEFINITIONS( -D_USRDLL -DARCHIVEFILE_EXPORTS )
+if ( WIN32 )
+  ADD_DEFINITIONS( -DLIBARCHIVE_STATIC )
+endif ( WIN32 )
 
 HPCC_ADD_LIBRARY( archivefile SHARED ${SRCS}  )
-install ( TARGETS archivefile DESTINATION filehooks )
+install ( TARGETS archivefile RUNTIME DESTINATION filehooks LIBRARY DESTINATION filehooks )
 
 target_link_libraries ( archivefile
     jlib
-    archive
+    ${LIBARCHIVE_LIBRARIES}
     )
+
+if (WIN32)
+target_link_libraries ( archivefile
+    ${ZLIB_LIBRARIES}
+    ${BZIP2_LIBRARIES}
+    )
+endif (WIN32)

common/remote/hooks/libarchive/archive.cpp

@@ -17,6 +17,10 @@
 
 #include "platform.h"
 
+#ifdef _WIN32
+#define S_ISDIR(m) (((m)&_S_IFDIR)!=0)
+#endif
+
 #include "jlib.hpp"
 #include "jio.hpp"
 
@@ -35,7 +39,11 @@
  * Installs hooks into createIFile, spotting filenames of the form /my/directory/myfile.zip/{password}/path/within/archive
  */
 
-#define ARCHIVE_SIGNATURE "[.]{zip|tar|tar[.]gz|tgz}{$|"PATHSEPSTR"}"
+#ifdef _WIN32
+#define ARCHIVE_SIGNATURE "[.]{zip|tar|tar[.]gz|tgz}{$|/|\\\\}"
+#else
+#define ARCHIVE_SIGNATURE "[.]{zip|tar|tar[.]gz|tgz}{$|/}"
+#endif
 
 static RegExpr *signature;
 static SpinLock *lock;
@@ -170,8 +178,14 @@ public:
         curBuffSize = 0;
         curBuff = NULL;
         archive = archive_read_new();
+#ifdef _WIN32
+        archive_read_support_format_zip(archive);
+        archive_read_support_format_tar(archive);
+        archive_read_support_compression_bzip2(archive);
+#else
         archive_read_support_format_all(archive);
         archive_read_support_compression_all(archive);
+#endif
         int retcode = archive_read_open_filename(archive, container, 10240);
         if (retcode == ARCHIVE_OK)
         {
@@ -261,6 +275,9 @@ protected:
 #if ARCHIVE_VERSION_NUMBER < 3000000
     off_t curPos;
 #else
+#if defined(_WIN32)
+#define	int64_t	__int64
+#endif
     int64_t curPos;
 #endif
     offset_t lastPos;
@@ -463,8 +480,14 @@ public:
         entries.kill();
         curIndex = 0;
         struct archive *archive = archive_read_new();
+#ifdef _WIN32
+        archive_read_support_format_zip(archive);
+        archive_read_support_format_tar(archive);
+        archive_read_support_compression_bzip2(archive);
+#else
         archive_read_support_format_all(archive);
         archive_read_support_compression_all(archive);
+#endif
         int retcode = archive_read_open_filename(archive, container, 10240);
         if (retcode == ARCHIVE_OK)
         {
@@ -606,5 +629,6 @@ MODULE_EXIT()
     delete signature;
     delete lock;
     lock = NULL;
+    signature = NULL;
     ::Release(archiveFileHook);
 }

common/remote/rmtfile.cpp

@@ -851,10 +851,11 @@ static void installFileHook(const char *hookFile)
         }
         else if (file->isFile() == foundYes)
         {
+            HookInstallFunction *hookInstall;
             SharedObject *so = new SharedObject(); // MORE - this leaks! Kind-of deliberate right now...
-            if (so->load(file->queryFilename(), false))
+            if (so->load(file->queryFilename(), false) &&
+                (hookInstall = (HookInstallFunction *) GetSharedProcedure(so->getInstanceHandle(), "installFileHook")) != NULL)
             {
-                HookInstallFunction *hookInstall = (HookInstallFunction *) GetSharedProcedure(so->getInstanceHandle(), "installFileHook");
                 hookInstall();
                 hookDlls->append(so);
             }

docs/ECLLanguageReference/ECLR_mods/BltInFunc-EVALUATE.xml

@@ -15,6 +15,11 @@
       </indexterm>(</emphasis><emphasis>expression</emphasis><emphasis
     role="bold">) ;</emphasis></para>
 
+    <para><emphasis>[</emphasis><emphasis>attrname</emphasis> :=<emphasis
+    role="bold"> ] EVALUATE(</emphasis><emphasis>module </emphasis><emphasis
+    role="bold">[</emphasis><emphasis>, defname </emphasis><emphasis
+    role="bold"> ] ) ;</emphasis></para>
+
     <para><informaltable colsep="1" frame="all" rowsep="1">
         <tgroup cols="2">
           <colspec colwidth="80.60pt" />
@@ -25,28 +30,51 @@
             <row>
               <entry><emphasis>attrname</emphasis></entry>
 
-              <entry>Optional. The action name, which turns the action into an
-              attribute definition, therefore not executed until the
+              <entry>Optional. The action name, which turns the action into a
+              definition, therefore not executed until the
               <emphasis>attrname</emphasis> is used as an action.</entry>
             </row>
 
             <row>
-              <entry>expression</entry>
+              <entry><emphasis>expression</emphasis></entry>
 
               <entry>The function to call in an action context.</entry>
             </row>
+
+            <row>
+              <entry><emphasis>module</emphasis></entry>
+
+              <entry>The module to evaluate.</entry>
+            </row>
+
+            <row>
+              <entry><emphasis>defname</emphasis></entry>
+
+              <entry>Optional. The name of a specific defintion within the
+              <emphasis>module</emphasis> to evaluate. If omitted, all
+              definitions in the <emphasis>module</emphasis> are
+              evaluated.</entry>
+            </row>
           </tbody>
         </tgroup>
       </informaltable></para>
 
-    <para>The <emphasis role="bold">EVALUATE</emphasis> action names an
-    <emphasis>expression</emphasis> (typically a function call) to execute in
-    an action context. This is mainly useful for calling functions that have
-    side-effects, where you don't care about the return value.</para>
+    <para>The first form of the <emphasis role="bold">EVALUATE</emphasis>
+    action names an <emphasis>expression</emphasis> (typically a function
+    call) to execute in an action context. This is mainly useful for calling
+    functions that have side-effects, where you don't care about the return
+    value.</para>
+
+    <para>The second form of the <emphasis role="bold">EVALUATE</emphasis>
+    action recursively expands the exported definitions of the
+    <emphasis>module</emphasis> and evaluates them. If a
+    <emphasis>defname</emphasis> is specified, then only that definition is
+    evaluated.</para>
 
     <para>Example:</para>
 
-    <programlisting>myService := SERVICE
+    <programlisting>//Form 1 example:
+myService := SERVICE
  UNSIGNED4 doSomething(STRING text);
 END;
 
@@ -55,7 +83,23 @@ ds := DATASET('MyFile', {STRING20 text} , THOR);
 APPLY(ds, EVALUATE(doSomething(ds.text)));
  //calls the doSomething function once for each record in the ds
  // dataset, ignoring the returned values from the function
-</programlisting>
+ 
+//Form 2 example:
+M := MODULE
+  EXPORT a := 10;
+  EXPORT b := OUTPUT('Hello');
+END;
+ 
+M2 := MODULE
+  EXPORT mx := M;
+  EXPORT d := OUTPUT('Richard');
+END;
+ 
+EVALUATE(M2); 
+//produces three results:
+//  Result_1: 10
+//  Result_2: Hello
+//  Result_3: Richard</programlisting>
 
     <para>See Also: APPLY, SERVICE</para>
   </sect2>

docs/ECLLanguageReference/ECLR_mods/BltInFunc-LIMIT.xml

@@ -130,7 +130,7 @@ X := LIMIT(SomeFile,10, FAIL(99,'error!'));
 //single record output
 Y := LIMIT(SomeFile,10,
       ONFAIL(TRANSFORM(RecStruct,
-        SELF := ROW({0,''},RecStruct)));
+        SELF := ROW({0,''},RecStruct))));
 //no exception, just no record
 Z := LIMIT(SomeFile,10,SKIP);
 </programlisting>

docs/ECLLanguageReference/ECLR_mods/BltInFunc-SET.xml

@@ -43,17 +43,49 @@
     </informaltable></para>
 
   <para>The <emphasis role="bold">SET </emphasis>function returns a SET for
-  use in any set operation (such as the IN operator).</para>
+  use in any set operation (such as the IN operator), similar to a sub-select
+  in SQL when used with the IN operator. It does not remove duplicate elements
+  and does not order the set.</para>
+
+  <para>One common problem is the use of the SET function in a filter
+  condition, like this:</para>
+
+  <programlisting>MyDS := myDataset(myField IN SET(anotherDataset, someField));</programlisting>
+
+  <para>The code generated for this is inefficient if "anotherDataset"
+  contains a large number of elements, and may also cause a "Dataset too large
+  to output to workunit" error. A better way to recode the expression would be
+  this:</para>
+
+  <programlisting>MyDS := JOIN(myDataset, anotherDataset, LEFT.myField = RIGHT.someField, TRANSFORM(LEFT), LOOKUP) ;</programlisting>
+
+  <para>The end result is the same, the set of "myDataset" records where the
+  "myField" value is one of the "someField" values from "anotherDataset," but
+  the code is much more efficient in execution.</para>
 
   <para>Example:</para>
 
-  <programlisting>r := {STRING1 Letter};
-SomeFile := DATASET([{'A'},{'B'},{'C'},{'D'},{'E'},
-                     {'F'},{'G'},{'H'},{'I'},{'J'}],r);
-x := SET(SomeFile(Letter &gt; 'C'),Letter);
-y := 'A' IN x;  //results in FALSE
-z := 'D' IN x;  //results in TRUE
-</programlisting>
+  <programlisting>ds := DATASET([{'X',1},{'B',3},{'C',2},{'B',5},
+               {'C',4},{'D',6},{'E',2}],
+              {STRING1 Ltr, INTEGER1 Val});
+
+//a SET of just the Ltr field values:
+s1 := SET(ds,Ltr);
+COUNT(s1);  //results in 7
+s1;         //results in ['X','B','C','B','C','D','E']
+
+//a simple way to get just the unique elements
+//is to use a crosstab TABLE:
+t := TABLE(ds,{Ltr},Ltr); //order indeterminant
+
+s2 := SET(t,Ltr);
+COUNT(s2);  //results in 5
+s2;         //results in   ['D','X','C','E','B']
+
+//sorted unique elements
+s3 := SET(SORT(t,Ltr),Ltr);
+COUNT(s3);  //results in 5
+s3;         //results in ['B','C','D','E','X']</programlisting>
 
   <para>See Also: <link linkend="Recordset_Filtering">Sets and Filters</link>,
   <link linkend="SET_OF">SET OF</link>, <link linkend="Set_Operators">Set

docs/ECLLanguageReference/ECLR_mods/Expr-InOps.xml

@@ -30,8 +30,8 @@
         <row>
           <entry><emphasis>value_set</emphasis></entry>
 
-          <entry>A set of values. This may be a set expression or a
-          DICTIONARY.</entry>
+          <entry>A set of values. This may be a set expression, the SET
+          function, or a DICTIONARY.</entry>
         </row>
       </tbody>
     </tgroup>
@@ -56,6 +56,14 @@ Trades_ABCstat := Trades(IsABC(rate));
     // Trades_ABCstat is a record set definition of all those
     // trades with a trade status of A, B, or C
 
+//SET function examples
+r := {STRING1 Letter};
+SomeFile := DATASET([{'A'},{'B'},{'C'},{'D'},{'E'},
+                     {'F'},{'G'},{'H'},{'I'},{'J'}],r);
+x := SET(SomeFile(Letter &gt; 'C'),Letter);
+y := 'A' IN x;  //results in FALSE
+z := 'D' IN x;  //results in TRUE
+
 //DICTIONARY examples:
 rec := {STRING color,UNSIGNED1 code};
 ColorCodes := DATASET([{'Black' ,0 },
@@ -75,6 +83,8 @@ OUTPUT(ROW({'Red',2},rec) IN ColorCodesDCT);
   (<link linkend="Set_Attributes">Set Definitions</link>), <link
   linkend="Logical_Operators">Logical Operators</link>, <link
   linkend="PARSE_Pattern_Value_Types">PATTERN</link>, <link
-  linkend="DICTIONARY">DICTIONARY</link>, <link
-  linkend="ROW">ROW</link></para>
+  linkend="DICTIONARY">DICTIONARY</link>, <link linkend="ROW">ROW</link>,
+  <link linkend="SET">SET</link>, <link linkend="Recordset_Filtering">Sets and
+  Filters</link>, <link linkend="SET_OF">SET OF</link>, <link
+  linkend="Set_Operators">Set Operators</link></para>
 </sect1>

docs/ECLStandardLibraryReference/SLR-Mods/WordCount.xml

@@ -62,7 +62,7 @@ str1 := 'a word a day keeps the doctor away';
 output(LENGTH(TRIM(Str1,LEFT,RIGHT)) - LENGTH(TRIM(Str1,ALL)) + 1);
                                  //finds eight words by removing spaces
 
-STD.STr.CountWords(str1);    //finds eight words based on space delimiter
+STD.Str.WordCount(str1);         //finds eight words based on space delimiter
 </programlisting>
 
   <para></para>

docs/ECLWatch/ECLWTechPrev.xml

@@ -58,7 +58,7 @@
 
       <para>ECL Watch provides an interface to the HPCC system and allows you
       to view information and interrogate nodes to confirm all expected
-      processes are running. This provides a simple centric view into the
+      processes are running. This provides a simple Web-based view into the
       system and a means to perform Workunit and data files
       maintenance.</para>
 
@@ -76,16 +76,14 @@
         </figure></para>
 
       <para>On the ECL Watch page, there are links in the Navigator panel
-      underneath the <emphasis role="bold">Tech Preview</emphasis> heading.
-      The links below the header open the respective pages for the new
-      ECLWatch. The pages available through the Tech Preview are ECL
-      Workunits, DFU Workunits, and Logical Files.</para>
+      under the <emphasis role="bold">Tech Preview</emphasis> heading. The
+      links below the header open the respective pages for the new ECL Watch.
+      There are pages available through the Tech Preview that allow you to
+      access ECL Workunits, DFU Workunits, and Logical Files.</para>
 
       <para>For the preview period, these pages are only available through the
-      Tech Preview links. These pages will later replace the ECL Browse
-      Workunits, Browse DFU Workunits, and the Browse Logical Files pages. In
-      addition each page has filter capabilities, so the search pages will
-      also be replaced. </para>
+      Tech Preview link. These redesigned pages with the improved
+      functionality will later replace ECL Watch.</para>
     </sect1>
 
     <xi:include href="ECLWatch/ECLWa_mods/ECLWatchSrc.xml"
@@ -99,10 +97,17 @@
     <xi:include href="ECLWatch/ECLWa_mods/ECLWatchSrc.xml"
                 xpointer="LogicalFiles"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
-                
+
+    <xi:include href="ECLWatch/ECLWa_mods/ECLWatchSrc.xml"
+                xpointer="Landing_Zones"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+    <xi:include href="ECLWatch/ECLWa_mods/ECLWatchSrc.xml"
+                xpointer="Pub_Query"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
     <xi:include href="ECLWatch/ECLWa_mods/ECLWatchSrc.xml"
-		xpointer="Landing_Zones"
+                xpointer="Operations"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
-                
   </chapter>
 </book>

docs/ECLWatch/ECLWa_mods/ECLWatchSrc.xml

@@ -65,14 +65,13 @@
 
       <para>ECL Watch provides an interface to the HPCC system and allows you
       to view information and interrogate nodes to confirm all expected
-      processes are running. This provides a simple centric view into the
-      system and a means to perform Work Unit and data files
-      maintenance.***</para>
+      processes are running. This provides a simple view into the system and a
+      means to perform Work Unit and data files maintenance.***</para>
 
       <para>Using ECL watch assumes your HPCC system is up and running, able
       to process jobs, and workunits. You would use the ECL Watch interface to
       both check the status of your jobs, and perform operations on those
-      jobs.</para>
+      jobs.***</para>
     </sect1>
 
     <sect1 id="ECLWorkunits" role="brk">
@@ -83,8 +82,19 @@
       can also perform actions on the selected workunit using the Workunit
       Action buttons.</para>
 
-      <para>Using the Navigator panel on the left, click on the <emphasis
-      role="bold">Browse Workunits </emphasis>link.</para>
+      <para>To access the workunits page click on the<emphasis role="bold">
+      ECL</emphasis> icon, then click the <emphasis
+      role="bold">Workunits</emphasis> link from the navigation sub-menu.
+      <figure>
+          <title>ECL Files</title>
+
+          <mediaobject>
+            <imageobject>
+              <imagedata fileref="../../images/ECLWA00A.jpg"
+                         vendor="eclwatchSS" />
+            </imageobject>
+          </mediaobject>
+        </figure></para>
 
       <para>The page displays the ECL workunits on your system. Choose
       the<emphasis role="bold"> Workunits</emphasis> Navigation tab to display
@@ -102,8 +112,8 @@
         </figure></para>
 
       <para>To further examine a workunit or to perform some action on it, you
-      must select it. You can select the workunit by checking the check
-      box.<figure>
+      must select it. You can select the workunit by checking the check box.
+      You can also open a work unit by double-clicking on the workunit.<figure>
           <title>Select ECL Workunit</title>
 
           <mediaobject>
@@ -175,7 +185,7 @@
         </listitem>
       </itemizedlist>
 
-      <sect2>
+      <sect2 role="brk">
         <title>Filter Options</title>
 
         <para>You can filter the workunits displayed on the Workunits tab by
@@ -395,12 +405,21 @@
       <para>The DFU Workunits page contains a list of all the DFU workunits on
       your system. It provides access to more details about the workunits. You
       can also perform actions on the selected workunit using the Workunit
-      Action buttons.</para>
+      Action buttons. <figure>
+          <title>Files Link</title>
+
+          <mediaobject>
+            <imageobject>
+              <imagedata fileref="../../images/ECLWA00D.jpg" />
+            </imageobject>
+          </mediaobject>
+        </figure></para>
 
-      <para>In the Navigator panel of the ECL Watch, click on the<emphasis
-      role="bold"> Browse </emphasis>link under <emphasis role="bold">DFU
-      Workunits</emphasis> header. This opens the DFU Workunits page. The page
-      displays the DFU Workunits on your system.</para>
+      <para>To access the DFU workunits page, click on the<emphasis
+      role="bold"> Files </emphasis>link on the navigation pane, then click
+      the <emphasis role="bold">Workunits</emphasis> link from the navigation
+      sub-menu. This action opens the DFU Workunits page. The page displays
+      the DFU Workunits on your system.</para>
 
       <para><figure>
           <title>Browse DFU Workunits</title>
@@ -477,7 +496,7 @@
       You can also double-click on a particular workunit to open the details
       tab.</para>
 
-      <sect2>
+      <sect2 role="brk">
         <title>DFU Filter Options</title>
 
         <para>You can filter the workunits displayed on the Workunits tab by
@@ -555,7 +574,7 @@
         </sect3>
       </sect2>
 
-      <sect2>
+      <sect2 role="brk">
         <title>DFU Workunit Details Page</title>
 
         <para>When you open the selected workunit(s) you will see the workunit
@@ -619,17 +638,17 @@
       also perform actions on selected files using the Workunit Action
       buttons.</para>
 
-      <para>In the Navigator panel of ECL Watch, click on the <emphasis
-      role="bold">Browse Logical Files </emphasis>link under the <emphasis
-      role="bold">DFU Files</emphasis> header. This will open the Logical
-      Files page.</para>
+      <para>To access the Logical Files page click on the <emphasis
+      role="bold">Files </emphasis>icon, then click the <emphasis
+      role="bold">Logical Files</emphasis> link from the navigation
+      sub-menu.</para>
 
       <para><figure>
-          <title>Browse Logical Files</title>
+          <title>Logical Files</title>
 
           <mediaobject>
             <imageobject>
-              <imagedata fileref="../../images/CSimg14.jpg"
+              <imagedata fileref="../../images/ECLWA430.jpg"
                          vendor="eclwatchSS" />
             </imageobject>
           </mediaobject>
@@ -727,7 +746,7 @@
       <para>You can press the <emphasis role="bold">Open</emphasis> button to
       open a tab with the details for each selected file.</para>
 
-      <sect2>
+      <sect2 role="brk">
         <title>Logical Files Filter Options</title>
 
         <para>You can filter the logIcal files displayed on the Logical Files
@@ -930,9 +949,8 @@
     <sect1 id="Landing_Zones" vendor="eclwatchSS">
       <title>Landing Zones</title>
 
-      <para>The Landing Zone link displays the Landing Zones page. The Landing
-      Zone Page shows you each landing zone you have configured for your
-      cluster and its contents. <figure>
+      <para>The Landing Zone Page shows you each landing zone you have
+      configured for your cluster and its contents. <figure>
           <title>Landing Zone Page</title>
 
           <mediaobject>
@@ -1044,7 +1062,7 @@
           </orderedlist></para>
       </sect2>
 
-      <sect2>
+      <sect2 role="brk">
         <title>Hex Preview</title>
 
         <para>You can view a Hexadecimal representation of files on the
@@ -1118,5 +1136,83 @@
           </orderedlist></para>
       </sect2>
     </sect1>
+
+    <sect1 id="Pub_Query">
+      <title>Published Queries</title>
+
+      <para>The Queries page provides access to published Queries on all
+      clusters in an environment. To access the Queries page click the
+      Published Queries link from the Navigation pane in ECL Watch.</para>
+
+      <sect2>
+        <title>Queries</title>
+
+        <para>Click on any link from the <emphasis
+        role="bold">Targets</emphasis> page to displays that target page. The
+        Target page has two tabs, one for all target queries, and another tab
+        for only active queries. From this page you can get more information
+        about the specific published queries.</para>
+
+        <para><figure>
+            <title>Target Queries</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="../../images/ECLW006.jpg"
+                           vendor="eclwatchSS" />
+              </imageobject>
+            </mediaobject>
+          </figure>Select the queries by clicking on them or shift+click to
+        select multiple queries. Press the buttons for the desired
+        activity.</para>
+
+        <para><variablelist>
+            <varlistentry>
+              <term>Delete</term>
+
+              <listitem>
+                <para>Deletes the selected query (or queries).</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>Toggle Suspend</term>
+
+              <listitem>
+                <para>Toggles between suspending or unsuspending the selected
+                query (or queries).</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>Activate</term>
+
+              <listitem>
+                <para>Activates the selected query (or queries).</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>Query Details</term>
+
+              <listitem>
+                <para>Links to a page with the details for the selected
+                query.</para>
+              </listitem>
+            </varlistentry>
+          </variablelist></para>
+      </sect2>
+    </sect1>
+
+    <sect1 id="Operations">
+      <title>Operations</title>
+
+      <para>The Operations page provides access to several system
+      administrator tasks you can perform on your HPCC system. To access the
+      Operations page click the Operations link from the Navigation pane in
+      ECL Watch.</para>
+
+      <para>More to come...</para>
+    </sect1>
   </chapter>
 </book>

docs/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml

@@ -2391,6 +2391,365 @@ ecl packagemap validate roxie --active</programlisting>
                 </row>
               </tbody>
             </tgroup>
+          </informaltable><parameter></parameter></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl bundle depends</title>
+
+        <para><emphasis role="bold">ecl bundle depends
+        &lt;bundleName&gt;</emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl bundle depends mybundle
+ecl bundle depends mybundle --version=2</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl bundle depends</entry>
+
+                  <entry>Shows the dependencies of a bundle</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>&lt;bundleName&gt;</entry>
+
+                  <entry>The name of a bundle file or installed bundle</entry>
+                </row>
+
+                <row>
+                  <entry>--recurse</entry>
+
+                  <entry>Displays indirect dependencies</entry>
+                </row>
+
+                <row>
+                  <entry>--version</entry>
+
+                  <entry>Specify a version of the bundle</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl bundle info</title>
+
+        <para><emphasis role="bold">ecl bundle info
+        &lt;bundleName&gt;</emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl bundle info mybundle
+ecl bundle info mybundle --version=2</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl bundle info</entry>
+
+                  <entry>Lists information about a bundle</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>&lt;bundleName&gt;</entry>
+
+                  <entry>The name of a bundle file or installed bundle</entry>
+                </row>
+
+                <row>
+                  <entry>--version</entry>
+
+                  <entry>Specify a version of the bundle</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl bundle install</title>
+
+        <para><emphasis role="bold">ecl bundle install
+        &lt;bundleName&gt;</emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl bundle install mybundle
+ecl bundle install mybundle --dryrun
+ecl bundle install mybundle --update
+ecl bundle install mybundle --keepprior</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl bundle install</entry>
+
+                  <entry>Installs a bundle</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>&lt;bundleName&gt;</entry>
+
+                  <entry>The name of a bundle file</entry>
+                </row>
+
+                <row>
+                  <entry>--dryrun</entry>
+
+                  <entry>List what would be installed, but do not copy</entry>
+                </row>
+
+                <row>
+                  <entry>--force</entry>
+
+                  <entry>Install even if required dependencies missing</entry>
+                </row>
+
+                <row>
+                  <entry>--keepprior</entry>
+
+                  <entry>Do not remove any previous versions of the
+                  bundle</entry>
+                </row>
+
+                <row>
+                  <entry>--update</entry>
+
+                  <entry>Update an existing installed bundle</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl bundle uninstall</title>
+
+        <para><emphasis role="bold">ecl bundle uninstall
+        &lt;bundleName&gt;</emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl bundle uninstall mybundle
+ecl bundle install mybundle --dryrun
+ecl bundle install mybundle --update
+ecl bundle install mybundle --keepprior</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl bundle install</entry>
+
+                  <entry>Installs a bundle</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>&lt;bundleName&gt;</entry>
+
+                  <entry>The name of a bundle file</entry>
+                </row>
+
+                <row>
+                  <entry>--dryrun</entry>
+
+                  <entry>List what would be removed, but do not remove
+                  them</entry>
+                </row>
+
+                <row>
+                  <entry>--force</entry>
+
+                  <entry>Uninstall even if other bundles are dependent on
+                  this</entry>
+                </row>
+
+                <row>
+                  <entry>--version</entry>
+
+                  <entry>Specify a version of the bundle</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl bundle list</title>
+
+        <para><emphasis role="bold">ecl bundle list
+        &lt;pattern&gt;</emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl bundle list
+ecl bundle list myb*
+</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl bundle list</entry>
+
+                  <entry>Lists bundles matching specified pattern</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>&lt;pattern&gt;</entry>
+
+                  <entry>A pattern specifying bundles to list. If omitted, all
+                  bundles are listed</entry>
+                </row>
+
+                <row>
+                  <entry>--details</entry>
+
+                  <entry>Report details of each installed bundle</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl bundle use</title>
+
+        <para><emphasis role="bold">ecl bundle use &lt;bundleName&gt;
+        &lt;version&gt;</emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl bundle use myBundle
+ecl bundle use myBundle 2
+</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl bundle use</entry>
+
+                  <entry>Makes a specified version of a bundle active</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>&lt;bundleName&gt;</entry>
+
+                  <entry>The name of a bundle file</entry>
+                </row>
+
+                <row>
+                  <entry>--version</entry>
+
+                  <entry>The version of the bundle to make active, or
+                  "none"</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+              </tbody>
+            </tgroup>
           </informaltable></para>
       </sect2>
 

docs/RDDERef/RDDERef.xml

@@ -513,7 +513,8 @@
 
   <xi:include href="RDDERef/RDDE_Mods/Packages.xml" xpointer="element(/1)"
               xmlns:xi="http://www.w3.org/2001/XInclude" />
-
+  <xi:include href="RDDERef/RDDE_Mods/directAccessToRoxie.xml" xpointer="element(/1)"
+              xmlns:xi="http://www.w3.org/2001/XInclude" />
   <chapter id="Deploying-Data-to-a-Cluster-using-DFU">
     <title>Deploying Data to a Cluster using DFU</title>
 

docs/RDDERef/RDDE_Mods/directAccessToRoxie.xml

@@ -0,0 +1,278 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<chapter>
+  <title>Direct Access to Roxie</title>
+
+  <sect1 role="nobrk">
+    <title>Roxie Query Access Overview</title>
+
+    <para>WsECL provides quick and easy access to your published queries, but
+    is not optimized for the rapid throughput Roxie provides. To take full
+    advantage of this capability, you should access Roxie directly.</para>
+
+    <para>WsECL still plays an important role when using this technique. It
+    provides an immediate means of testing, provides a web interface for
+    one-off query execution, and provides the WSDL and schema which can be
+    used for automated code generators for SOAP access. It also provides the
+    HTTP-GET and form encodes support. Finally, it also provides a JSON
+    (Javascript Simple Object Notation) interface which is similar to Roxie’s
+    direct interface.</para>
+
+    <para></para>
+
+    <sect2>
+      <title>Web Services</title>
+
+      <para>A Web service is a standards-based software component accessible
+      over the Internet. The service can be simple or complex.</para>
+
+      <para>For example, a Web service could request a number of stock quotes
+      and return them in an XML result set.</para>
+
+      <para>Web Services are available to any platform, object model, or
+      programming language. This provides access to users over the Internet,
+      Intranet, or Extranet regardless of the user’s platform.</para>
+
+      <para>It also simplifies distributed systems development. The use of
+      standards-based components protects development investment regardless of
+      future trends. Web Service Technologies are based upon HTTP.</para>
+
+      <sect3>
+        <title><emphasis role="bold">Simple Object Access Protocol
+        (SOAP)</emphasis></title>
+
+        <para>SOAP is the most common XML-based protocol for sending requests
+        to and receiving responses from Web Services. Basically, it is a
+        protocol for communication between applications. It is designed to
+        communicate over the Internet and is platform independent and language
+        independent. Based on XML, SOAP is simple and extensible.</para>
+      </sect3>
+
+      <sect3>
+        <title>JavaScript Object Notation<emphasis role="bold">
+        (JSON)</emphasis></title>
+
+        <para><emphasis role="bold">JSON</emphasis> is a lightweight
+        data-interchange format. It is easy for humans to read and write. It
+        is easy for machines to parse and generate. It is based on a subset of
+        the JavaScript Programming Language, it is completely language
+        independent.</para>
+
+        <para></para>
+      </sect3>
+    </sect2>
+  </sect1>
+
+  <sect1>
+    <title>Accessing your Roxie Queries</title>
+
+    <para>Once you have developed, compiled, and published your queries, you
+    need to provide access to users.</para>
+
+    <sect2>
+      <title><emphasis role="bold">ESP and WsECL</emphasis></title>
+
+      <para>A standard installation of the HPCC platform includes an
+      Enterprise Services Platform (ESP) with the WsECL service configured
+      through a service binding to port 8002 (configurable).</para>
+
+      <para>WsECL provides a means to access all of your compiled and
+      published queries using a Web interface or by using SOAP or JSON.</para>
+
+      <para><emphasis role="bold">Tip: </emphasis>You can provide a JobName in
+      the URL using this form:</para>
+
+      <programlisting>http://ip:port/WsEcl/submit/query/<emphasis role="bold">&lt;targetCluster&gt;</emphasis>/<emphasis
+          role="bold">&lt;query&gt;</emphasis>?_jobname=<emphasis role="bold">&lt;jobname&gt;</emphasis></programlisting>
+
+      <para>This is useful for testing queries. It is also suitable for
+      providing real-time access to a small set of users; however, it is not
+      intended to scale to larger audiences.</para>
+    </sect2>
+
+    <sect2>
+      <title><emphasis role="bold">Access WsECL VIA SOAP</emphasis></title>
+
+      <sect3>
+        <title><emphasis role="bold">WSDL</emphasis></title>
+
+        <para>The Simple Object Access Protocol (SOAP) sets a standard for
+        communication between processes using a common xml based format. SOAP
+        libraries are readily available for many languages and development
+        platforms, including Microsoft .NET (accessible using C#, VB.NET,
+        ASP.NET, and other CLR languages), Java (e.g.,. JAX and Apache AXIS),
+        PERL, and many others.</para>
+
+        <para>Web Service Description Language (WSDL) is used to provide a
+        structured description of a Web service interface.</para>
+
+        <para>Many of the libraries available for allowing applications to use
+        the SOAP protocol also provide tools for automatically generating
+        service specific APIs from a WSDL document.</para>
+
+        <para>The WSDL description is automatically available for any
+        published query.</para>
+
+        <para>The following URL can be used to retrieve the WSDL description
+        of a published query:</para>
+
+        <para>http://<emphasis>nnn.nnn.nnn.nnn</emphasis>:8002/WsEcl/definitions/query/roxie/&lt;queryName&gt;/main/&lt;queryName&gt;.wsdl</para>
+
+        <para>where <emphasis>nnn.nnn.nnn.nnn</emphasis> is the IP address (or
+        DNS Name) of that ESP server with the binding to WsECL.</para>
+
+        <para>8002 is the default port. If you have a modified the port
+        setting, use the port you have selected for that purpose.</para>
+
+        <para>The syntax and functionality of the SOAP protocol itself is also
+        beyond the scope of this document. The following list of external
+        resources can help in your understanding of standards-based
+        technologies used.</para>
+      </sect3>
+
+      <sect3>
+        <title>SOAP</title>
+
+        <para>http://www.w3.org/TR/soap12-part0</para>
+
+        <para>http://www.w3.org/TR/soap12-part1</para>
+
+        <para>http://www.w3.org/TR/soap12-part2</para>
+      </sect3>
+
+      <sect3>
+        <title><emphasis role="bold">Web Service Definition Language (WSDL)
+        1.1</emphasis></title>
+
+        <para>http://www.w3.org/TR/wsdl</para>
+      </sect3>
+    </sect2>
+
+    <sect2>
+      <title><emphasis role="bold">Direct Access VIA SOAP</emphasis></title>
+
+      <orderedlist numeration="arabic">
+        <listitem>
+          <para>Create and publish your query to a target cluster
+          (Roxie).</para>
+        </listitem>
+
+        <listitem>
+          <para>Test and validate using WsECL to access the query.</para>
+
+          <para>Use the default interface provided to test:</para>
+
+          <para>http://nnn.nnn.nnn.nnn:8002/</para>
+
+          <para>where nnn.nnn.nnn.nnn is the IP address (or DNS Name) of that
+          ESP server with the binding to WsECL.</para>
+
+          <para>8002 is the default port. If you have a modified the port
+          setting, use the port you have selected for that purpose.</para>
+        </listitem>
+
+        <listitem>
+          <para>Begin development of the application that will consume the web
+          service by importing the WSDL from:</para>
+
+          <para>http://<emphasis>nnn.nnn.nnn.nnn</emphasis>:8002/WsEcl/definitions/query/roxie/&lt;queryName&gt;/main/&lt;queryName&gt;.wsdl</para>
+
+          <para>where nnn.nnn.nnn.nnn is the IP address (or DNS Name) of that
+          ESP server with the binding to WsECL.</para>
+
+          <para>8002 is the default port. If you have a modified the port
+          setting, use the port you have selected for that purpose.</para>
+        </listitem>
+
+        <listitem>
+          <para>When testing is complete, change the base URL to</para>
+
+          <para>http://nnn.nnn.nnn.nnn:9876/</para>
+
+          <para>where nnn.nnn.nnn.nnn is the IP address (or DNS Name) of a
+          Roxie Server.</para>
+
+          <para>9876 is the default port. If you have a modified the port
+          setting, use the port you have selected for that purpose.</para>
+        </listitem>
+      </orderedlist>
+
+      <para></para>
+    </sect2>
+
+    <sect2>
+      <title><emphasis role="bold">Direct Access VIA JSON</emphasis></title>
+
+      <orderedlist numeration="arabic">
+        <listitem>
+          <para>Create and publish your query to a target cluster
+          (Roxie).</para>
+        </listitem>
+
+        <listitem>
+          <para>Test and validate using WsECL to access the query.</para>
+
+          <para>Use the default interface provided to test:</para>
+
+          <para>http://nnn.nnn.nnn.nnn:8002/</para>
+
+          <para>where nnn.nnn.nnn.nnn is the IP address (or DNS Name) of that
+          ESP server with the binding to WsECL.</para>
+
+          <para>8002 is the default port. If you have a modified the port
+          setting, use the port you have selected for that purpose.</para>
+        </listitem>
+
+        <listitem>
+          <para>Begin development of the application that will consume the web
+          service using the JSON definitions available from the form in
+          WsECL.</para>
+        </listitem>
+
+        <listitem>
+          <para>From the drop list, select JSON Test, then press the submit
+          button.</para>
+
+          <para><graphic fileref="../../images/json1.jpg" /></para>
+        </listitem>
+
+        <listitem>
+          <para>Edit the query criteria on the left, then press the <emphasis
+          role="bold">Send Request</emphasis> button.</para>
+        </listitem>
+
+        <listitem>
+          <para>The results display on the right side.</para>
+        </listitem>
+
+        <listitem>
+          <para>Edit and resubmit (by pressing the <emphasis role="bold">Send
+          Request</emphasis> button ) to see responses for different
+          criteria.</para>
+
+          <para><graphic fileref="../../images/json2.jpg" /></para>
+        </listitem>
+
+        <listitem>
+          <para>Set the base URL in your application to</para>
+
+          <para>http://nnn.nnn.nnn.nnn:9876/</para>
+
+          <para>where nnn.nnn.nnn.nnn is the IP address (or DNS Name) of a
+          Roxie Server.</para>
+
+          <para>9876 is the default port. If you have a modified the port
+          setting, use the port you have selected for that purpose.</para>
+        </listitem>
+      </orderedlist>
+
+      <para><emphasis role="bold">Note:</emphasis> The WsECL service
+      automatically distributes the load by sending queries to Roxie Servers
+      in a round-robin fashion. To utilize all servers, your application
+      should implement a similar load distribution scheme. Alternatively, you
+      can use a load balancer and submit the query to the Virtual IP created
+      by the load balancer.</para>
+    </sect2>
+  </sect1>
+</chapter>

ecl/ecl-bundle/CMakeLists.txt

@@ -66,4 +66,4 @@ target_link_libraries ( ecl-bundle
 if ( UNIX )
     install ( PROGRAMS ecl-bundle.install DESTINATION etc/init.d/install COMPONENT Runtime )
     install ( PROGRAMS ecl-bundle.uninstall DESTINATION etc/init.d/uninstall COMPONENT Runtime )
-endif ( UNIX )
+endif ( UNIX )

ecl/ecl-bundle/ecl-bundle.cpp

@@ -629,13 +629,16 @@ private:
     {
         const IBundleInfo &goer = bundles.item(idx);
         const char *installedDir = goer.queryInstalledPath();
-        Owned<IFile> versionDir = createIFile(installedDir);
-        recursiveRemoveDirectory(versionDir, isDryRun);
+        if (installedDir)
+        {
+            Owned<IFile> versionDir = createIFile(installedDir);
+            recursiveRemoveDirectory(versionDir, isDryRun);
+        }
         if (!idx)
             active = false;
         if (!isDryRun)
             bundles.remove(idx);
-        if (bundles.length() == (isDryRun ? 1 : 0))
+        if (installedDir && bundles.length() == (isDryRun ? 1 : 0))
         {
             Owned<IFile> versionsDir = createIFile(path);
             recursiveRemoveDirectory(versionsDir, isDryRun);
@@ -1049,10 +1052,12 @@ public:
             const char *version = bundle->queryVersion();
             printf("Installing bundle %s version %s\n", bundle->queryBundleName(), version);
 
+            StringBuffer thisBundlePath(bundlePath);
+            addPathSepChar(thisBundlePath).append(VERSION_SUBDIR).append(PATHSEPCHAR).append(bundle->queryCleanName());
             CBundleCollection allBundles(bundlePath);
             IBundleInfoSet *bundleSet = allBundles.queryBundleSet(bundle->queryCleanName());
             if (!bundleSet)
-                bundleSet = new CBundleInfoSet(bundle->queryCleanName(), bundlePath, bundle);
+                bundleSet = new CBundleInfoSet(thisBundlePath, bundlePath, bundle);
             else
             {
                 const IBundleInfo *active = bundleSet->queryVersion(version);
@@ -1099,9 +1104,8 @@ public:
                 throw MakeStringException(0, "Cannot create bundle directory %s", bundlePath.str());
 
             // Copy the bundle contents
-            StringBuffer versionPath(bundlePath);
-            addPathSepChar(versionPath).append(VERSION_SUBDIR).append(PATHSEPCHAR);
-            versionPath.append(bundle->queryCleanName()).append(PATHSEPCHAR).append(bundle->queryCleanVersion());
+            StringBuffer versionPath(thisBundlePath);
+            versionPath.append(PATHSEPCHAR).append(bundle->queryCleanVersion());
             if (!optDryRun && !recursiveCreateDirectory(versionPath))
                 throw MakeStringException(0, "Cannot create bundle version directory %s", versionPath.str());
             if (bundleFile->isDirectory() == foundYes) // could also be an archive, acting as a directory

ecl/eclcc/eclcc.cpp

@@ -25,6 +25,10 @@
 #include "build-config.h"
 #include "workunit.hpp"
 
+#ifndef _WIN32
+#include <pwd.h>
+#endif
+
 #include "hqlecl.hpp"
 #include "hqlir.hpp"
 #include "hqlerrors.hpp"
@@ -162,6 +166,31 @@ static bool getPackageFolder(StringBuffer & path)
     return false;
 }
 
+static bool getHomeFolder(StringBuffer & homepath)
+{
+#ifdef _WIN32
+    const char *home = getenv("APPDATA");
+    if (!home)
+        return false;
+    // Not the 'official' way - which changes with every windows version
+    // but should work well enough for us (and avoids sorting out windows include mess)
+    homepath.append(home);
+    addPathSepChar(homepath).append(DIR_NAME);
+#else
+    const char *home = getenv("HOME");
+    if (!home)
+    {
+        struct passwd *pw = getpwuid(getuid());
+        home = pw->pw_dir;
+        if (!home)
+            return false;
+    }
+    homepath.append(home);
+    addPathSepChar(homepath).append(".").append(DIR_NAME);
+#endif
+    return true;
+}
+
 struct EclCompileInstance
 {
 public:
@@ -500,8 +529,8 @@ void EclCC::loadOptions()
     if (globals->hasProp("targetGcc"))
         optTargetCompiler = globals->getPropBool("targetGcc") ? GccCppCompiler : Vs6CppCompiler;
 
-    StringBuffer syspath;
-    if (getPackageFolder(syspath))
+    StringBuffer syspath, homepath;
+    if (getPackageFolder(syspath) && getHomeFolder(homepath))
     {
 #if _WIN32
         extractOption(compilerPath, globals, "CL_PATH", "compilerPath", syspath, "componentfiles\\cl");
@@ -515,7 +544,7 @@ void EclCC::loadOptions()
         extractOption(hooksPath, globals, "HPCC_FILEHOOKS_PATH", "filehooks", syspath, "filehooks");
         extractOption(templatePath, globals, "ECLCC_TPL_PATH", "templatePath", syspath, "componentfiles");
         extractOption(eclLibraryPath, globals, "ECLCC_ECLLIBRARY_PATH", "eclLibrariesPath", syspath, "share/ecllibrary/");
-        extractOption(eclBundlePath, globals, "ECLCC_ECLBUNDLE_PATH", "eclBundlesPath", syspath, "share/bundles/");
+        extractOption(eclBundlePath, globals, "ECLCC_ECLBUNDLE_PATH", "eclBundlesPath", homepath, "/bundles/");
     }
     extractOption(stdIncludeLibraryPath, globals, "ECLCC_ECLINCLUDE_PATH", "eclIncludePath", ".", NULL);
 
@@ -1478,7 +1507,6 @@ bool EclCC::processFiles()
     }
     if (optShowPaths)
     {
-        loadOptions();
         printf("CL_PATH=%s\n", compilerPath.str());
         printf("ECLCC_ECLBUNDLE_PATH=%s\n", eclBundlePath.str());
         printf("ECLCC_ECLINCLUDE_PATH=%s\n", stdIncludeLibraryPath.str());

ecl/hql/hqlcollect.cpp

@@ -464,17 +464,6 @@ void FileSystemEclCollection::processFilePath(IErrorReceiver * errs, const char
             {
                 root.addFile(*file, allowPlugins);
             }
-            else
-            {
-                if (errs)
-                {
-                    StringBuffer msg;
-                    msg.appendf("Explicit source file %s not found", searchPattern.str());
-                    errs->reportWarning(10, msg.str(), NULL, 0, 0, 0);
-                }
-                else
-                    printf("Explicit source file %s not found\n", searchPattern.str());
-            }
         }
         else
         {

plugins/Rembed/Rembed.cpp

@@ -53,10 +53,15 @@ extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
     return true;
 }
 
+#ifdef _WIN32
+    EXTERN_C IMAGE_DOS_HEADER __ImageBase;
+#endif
+
 namespace Rembed
 {
 
 // Use a global object to ensure that the R instance is initialized only once
+// Because of R's dodgy stack checks, we also have to do so on main thread
 
 static class RGlobalState
 {
@@ -65,7 +70,47 @@ public:
     {
         const char *args[] = {"R", "--slave" };
         R = new RInside(2, args, true, false, false);
-
+        // The R code for checking stack limits assumes that all calls are on the same thread
+        // as the original context was created on - this will not always be true in ECL (and hardly
+        // ever true in Roxie
+        // Setting the stack limit to -1 disables this check
+        R_CStackLimit = -1;
+// Make sure we are never unloaded (as R does not support it)
+// we do this by doing a dynamic load of the Rembed library
+#ifdef _WIN32
+        char path[_MAX_PATH];
+        ::GetModuleFileName((HINSTANCE)&__ImageBase, path, _MAX_PATH);
+        if (strstr(path, "Rembed"))
+        {
+            HINSTANCE h = LoadSharedObject(path, false, false);
+            DBGLOG("LoadSharedObject returned %p", h);
+        }
+#else
+        FILE *diskfp = fopen("/proc/self/maps", "r");
+        if (diskfp)
+        {
+            char ln[_MAX_PATH];
+            while (fgets(ln, sizeof(ln), diskfp))
+            {
+                if (strstr(ln, "libRembed"))
+                {
+                    const char *fullName = strchr(ln, '/');
+                    if (fullName)
+                    {
+                        char *tail = (char *) strstr(fullName, SharedObjectExtension);
+                        if (tail)
+                        {
+                            tail[strlen(SharedObjectExtension)] = 0;
+                            HINSTANCE h = LoadSharedObject(fullName, false, false);
+                            DBGLOG("LoadSharedObject %s returned %p", fullName, h);
+                            break;
+                        }
+                    }
+                }
+            }
+            fclose(diskfp);
+        }
+#endif
     }
     ~RGlobalState()
     {
@@ -94,11 +139,13 @@ extern void unload()
 
 MODULE_INIT(INIT_PRIORITY_STANDARD)
 {
+    queryGlobalState(); // make sure gets loaded by main thread
     return true;
 }
 MODULE_EXIT()
 {
-    unload();
+// Don't unload, because R seems to have problems with being reloaded, i.e. crashes on next use
+//    unload();
 }
 
 // Each call to a R function will use a new REmbedFunctionContext object