Merge lates changes from candidate-3.6.x to master

Ensure that fixes commited directly to 3.6.x branch also appear in 3.8 and
later versions

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/FindFOP.cmake

@@ -23,7 +23,7 @@
 
 if (NOT FOP_FOUND)
   IF (WIN32)
-    SET (fop_n "fop.exe")
+    SET (fop_n "fop.bat")
   ELSE()
     SET (fop_n "fop")
   ENDIF()

cmake_modules/dependencies/precise.cmake

@@ -0,0 +1 @@
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.46.1, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect")

common/commonext/thorport.cpp

@@ -36,7 +36,8 @@
 #include "portlist.h"
 #include "thorport.hpp"
 
-#define WATCHDOGINC        1
+#define MPPORT       0
+#define WATCHDOGPORT 1
 
 static CriticalSection *portallocsection;
 static IBitSet *portmap;
@@ -44,8 +45,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
 {
     portallocsection = new CriticalSection;
     portmap = createBitSet();
-    portmap->set(THOR_MP_INC, true);
-    portmap->set(WATCHDOGINC, true);
+    portmap->set(MPPORT, true);
+    portmap->set(WATCHDOGPORT, true);
     return true;
 }
 MODULE_EXIT()
@@ -74,9 +75,9 @@ unsigned short getExternalFixedPort(unsigned short masterBase, unsigned short ma
     if (!machineBase) machineBase = THOR_BASESLAVE_PORT;
     switch (category) {
     case TPORT_watchdog:
-        return machineBase+WATCHDOGINC;
+        return machineBase+WATCHDOGPORT;
     case TPORT_mp:
-        return machineBase+THOR_MP_INC; 
+        return machineBase+MPPORT; 
     }
     LOG(MCerror,unknownJob,"getFixedPort: Unknown Port Kind!");
     return 0;

dali/server/daserver.cpp

@@ -213,7 +213,6 @@ int main(int argc, char* argv[])
 
                     try
                     {
-                        recursiveCreateDirectory(mirrorPath);
                         StringBuffer backupURL;
                         if (mirrorPath.length()<=2 || !isPathSepChar(mirrorPath.charAt(0)) || !isPathSepChar(mirrorPath.charAt(1)))
                         { // local machine path, convert to url
@@ -231,6 +230,7 @@ int main(int argc, char* argv[])
                         }
                         else
                             backupURL.append(mirrorPath);
+                        recursiveCreateDirectory(backupURL.str());
                         addPathSepChar(backupURL);
                         serverConfig->setProp("SDS/@remoteBackupLocation", backupURL.str());
                         PROGLOG("Backup URL = %s", backupURL.str());

docs/BuildTools/fo.xsl

@@ -324,6 +324,11 @@
        </fo:inline>
 </xsl:template>
 
+  <xsl:template match="emphasis[@role='underline']">
+      <fo:inline text-decoration="underline">
+         <xsl:apply-templates/>
+      </fo:inline>
+  </xsl:template>
 
 <xsl:template match="ulink">
      <fo:inline color="blue" text-decoration="underline">

docs/BuildTools/fo.xsl.in

@@ -324,6 +324,12 @@
        </fo:inline>
 </xsl:template>
 
+  <xsl:template match="emphasis[@role='underline']">
+      <fo:inline text-decoration="underline">
+         <xsl:apply-templates/>
+      </fo:inline>
+  </xsl:template>
+
 
 <xsl:template match="ulink">
      <fo:inline color="blue" text-decoration="underline">

docs/HPCCDataHandling/DataHandling.xml

@@ -32,7 +32,7 @@
       <para></para>
     </legalnotice>
 
-        <xi:include href="common/Version.xml" xpointer="FooterInfo"
+    <xi:include href="common/Version.xml" xpointer="FooterInfo"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
     <xi:include href="common/Version.xml" xpointer="DateVer"
@@ -263,8 +263,8 @@
 
         <para><orderedlist>
             <listitem>
-              <para>Open the WinSCP tool, and login to your Virtual Machine's
-              IP address using the username and password given.</para>
+              <para>Open the WinSCP tool, and login to your Landing Zone node
+              using the username and password given.</para>
 
               <para><informaltable colsep="1" rowsep="1">
                   <tgroup cols="2">
@@ -391,7 +391,7 @@
             </listitem>
 
             <listitem>
-              <para> Provide <emphasis role="bold">Destination</emphasis>
+              <para>Provide <emphasis role="bold">Destination</emphasis>
               information.</para>
 
               <para><informaltable colsep="0" frame="none" rowsep="0">
@@ -1311,4 +1311,201 @@
       </sect2>
     </sect1>
   </chapter>
+
+  <chapter>
+    <title>HPCC Data Backups</title>
+
+    <sect1 id="Introduction2" role="nobrk">
+      <title>Introduction</title>
+
+      <para>This section covers critical system data that requires regular
+      backup procedures to prevent data loss. </para>
+
+      <para>There are </para>
+
+      <itemizedlist>
+        <listitem>
+          <para>The System Data Store (Dali data)</para>
+        </listitem>
+
+        <listitem>
+          <para>Environment Configuration files</para>
+        </listitem>
+
+        <listitem>
+          <para>Data Refinery (Thor) data files</para>
+        </listitem>
+
+        <listitem>
+          <para>Rapid Data Delivery Engine (Roxie) data files</para>
+        </listitem>
+
+        <listitem>
+          <para>Attribute Repositories</para>
+        </listitem>
+
+        <listitem>
+          <para>Landing Zone files</para>
+        </listitem>
+      </itemizedlist>
+    </sect1>
+
+    <sect1>
+      <title>Dali data</title>
+
+      <para>The Dali Server data is typically mirrored to its backup node.
+      This location is specified in the environment configuration file using
+      the Configuration Manager. </para>
+
+      <para>Since the data is written simultaneously to both nodes, there is
+      no need for a manual backup procedure. </para>
+    </sect1>
+
+    <sect1>
+      <title>Environment Configuration files</title>
+
+      <para>There is only one active environment file, but you may have many
+      alternative configurations. </para>
+
+      <para>Configuration manager only works on files in the
+      /etc/HPCCSystems/source/ folder. To make a configuration active, it is
+      copied to /etc/HPCCSystems/environment.xml on all nodes. </para>
+
+      <para>Configuration Manager automatically creates backup copies in the
+      /etc/HPCCSystems/source/backup/ folder.</para>
+    </sect1>
+
+    <sect1>
+      <title>Thor data files</title>
+
+      <para>Thor clusters are normally configured to automatically replicate
+      data to a secondary location known as the mirror location. Usually, this
+      is on the second drive of the subsequent node. </para>
+
+      <para>If the data is not found at the primary location (for example, due
+      to drive failure or because a node has been swapped out), it looks in
+      the mirror directory to read the data. Any writes go to the primary and
+      then to the mirror. This provides continual redundancy and a quick means
+      to restore a system after a node swap.</para>
+
+      <para>A Thor data backup should be performed on a regularly scheduled
+      basis and on-demand after a node swap.</para>
+
+      <sect2>
+        <title>Manual backup</title>
+
+        <para>To run a backup manually, follow these steps:</para>
+
+        <orderedlist>
+          <listitem>
+            <para>Login to the Thor Master node.</para>
+
+            <para>If you don't know which node is your Thor Master node, you
+            can look it up using ECL Watch.</para>
+          </listitem>
+
+          <listitem>
+            <para>Run this command:</para>
+
+            <programlisting>sudo su hpcc
+/opt/HPCCSystems/bin/start_backupnode &lt;thor_cluster_name&gt; </programlisting>
+
+            <para>This starts the backup process.</para>
+
+            <para></para>
+
+            <graphic fileref="images/backupnode.jpg" />
+
+            <para>Wait until completion. It will say "backupnode finished" as
+            shown above.</para>
+          </listitem>
+
+          <listitem>
+            <para>Run the XREF utility in ECL Watch to verify that there are
+            no orphan files or lost files.</para>
+          </listitem>
+        </orderedlist>
+      </sect2>
+
+      <sect2>
+        <title>Scheduled backup</title>
+
+        <para>The easiest way to schedule the backup process is to create a
+        cron job. Cron is a daemon that serves as a task scheduler. </para>
+
+        <para>Cron tab (short for CRON TABle) is a text file that contains a
+        the task list. To edit with the default editor, use the
+        command:</para>
+
+        <programlisting>sudo crontab -e</programlisting>
+
+        <para>Here is a sample cron tab entry:</para>
+
+        <para><programlisting>30 23 * * * /opt/HPCCSystems/bin/start_backupnode mythor 
+</programlisting>30 represents the minute of the hour. </para>
+
+        <para>23 represents the hour of the day </para>
+
+        <para>The asterisks (*) represent every day, month, and
+        weekday.</para>
+
+        <para>mythor is the clustername</para>
+
+        <para>To list the tasks scheduled, use the command:</para>
+
+        <programlisting>sudo crontab -l</programlisting>
+
+        <para></para>
+      </sect2>
+    </sect1>
+
+    <sect1 id="Roxie-Data-Backup">
+      <title>Roxie data files</title>
+
+      <para>Roxie data is protected by three forms of redundancy:</para>
+
+      <itemizedlist mark="bullet">
+        <listitem>
+          <para>Original Source Data File Retention: When a query is deployed,
+          the data is typically copied from a Thor cluster's hard drives.
+          Therefore, the Thor data can serve as backup, provided it is not
+          removed or altered on Thor. Thor data is typically retained for a
+          period of time sufficient to serve as a backup copy.</para>
+        </listitem>
+
+        <listitem>
+          <para>Peer-Node Redundancy: Each Slave node typically has one or
+          more peer nodes within its cluster. Each peer stores a copy of data
+          files it will read.</para>
+        </listitem>
+
+        <listitem>
+          <para>Sibling Cluster Redundancy: Although not required, Roxie
+          deployments may run multiple identically-configured Roxie clusters.
+          When two clusters are deployed for Production each node has an
+          identical twin in terms of data and queries stored on the node in
+          the other cluster.</para>
+        </listitem>
+      </itemizedlist>
+
+      <para>This provides multiple redundant copies of data files.</para>
+    </sect1>
+
+    <sect1>
+      <title>Attribute Repositories</title>
+
+      <para>Attribute repositories are stored on ECL developer's local hard
+      drives. They can contain a significant number of hours of work and
+      therefore should be regularly backed up. In addition, we suggest using
+      some form of source version control, too. </para>
+    </sect1>
+
+    <sect1>
+      <title>Landing Zone files</title>
+
+      <para>Landing Zones contain raw data for input. They can also contain
+      output files. Depending on the size or complexity of these files, you
+      may want to retain copies for redundancy.</para>
+    </sect1>
+  </chapter>
 </book>

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -1668,6 +1668,13 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
         </orderedlist>
       </sect2>
     </sect1>
+    
+     <xi:include href="Installing_and_RunningTheHPCCPlatform/hpcc_ldap.xml" xpointer="element(/1)"
+		                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+     <xi:include href="Installing_and_RunningTheHPCCPlatform/user_Sect.xml" xpointer="element(/1)"
+		                xmlns:xi="http://www.w3.org/2001/XInclude" />
+    
   </chapter>
 
   <chapter>

docs/wip/hpcc_ldap.xml

@@ -1,11 +1,11 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
-<sect1>
+<sect1 id="ldap_config">
   <title>Configuring HPCC to use LDAP Authentication</title>
 
   <para>This section details the steps to connect your HPCC platform to an
-  existing LDAP Server.</para>
+  existing LDAP Server to enable user security.</para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <?dbfo keep-together="always"?>
@@ -20,8 +20,8 @@
           <entry><inlinegraphic fileref="images/caution.png" /></entry>
 
           <entry><emphasis role="bold">You should not attempt this until you
-          have already deployed, configured, and preflighted the environment
-          you will use.</emphasis></entry>
+          have already deployed, configured, and certified the environment you
+          will use.</emphasis></entry>
         </row>
       </tbody>
     </tgroup>
@@ -30,8 +30,8 @@
   <sect2>
     <title>Connect to Configuration Manager</title>
 
-    <para>In order to change the configuration for any HPCC components, we
-    need to connect to the Configuration Manager.</para>
+    <para>In order to change the configuration for HPCC components, connect to
+    the Configuration Manager.</para>
 
     <orderedlist numeration="arabic">
       <listitem>
@@ -53,17 +53,18 @@
         <para>Connect to the Configuration Manager web interface.</para>
 
         <para>(using the url of
-        http://<emphasis>&lt;ESP_IP_Address&gt;</emphasis>:8015, where
-        <emphasis>&lt;ESP_IP_Address&gt;</emphasis> is the IP address of your
-        ESP Server)</para>
+        http://<emphasis>&lt;configmgr_IP_Address&gt;</emphasis>:8015, where
+        <emphasis>&lt;configmgr_IP_Address&gt;</emphasis> is the IP address of
+        the node running Configuration Manager)</para>
       </listitem>
 
       <listitem>
-        <para>Select the Advanced View radio button.</para>
+        <para>Select the <emphasis role="bold">Advanced View</emphasis> radio
+        button.</para>
       </listitem>
 
       <listitem>
-        <para>Use the drop list to select an XML file for editing.</para>
+        <para>Use the drop list to select the XML configuration file.</para>
       </listitem>
     </orderedlist>
 
@@ -85,13 +86,13 @@
       </listitem>
 
       <listitem>
-        <para>From the Navigator pane, select <emphasis
-        role="bold">Hardware</emphasis>.</para>
+        <para>From the <emphasis role="bold">Navigator</emphasis> pane, select
+        <emphasis role="bold">Hardware</emphasis>.</para>
       </listitem>
 
       <listitem>
         <para>Select the <emphasis role="bold">Computers</emphasis> tab from
-        the page on the right.</para>
+        the panel on the right.</para>
       </listitem>
 
       <listitem>
@@ -100,7 +101,7 @@
         <para>Rt-click on the table below computers and select <emphasis
         role="bold">New </emphasis>from the pop up menu.</para>
 
-        <para><graphic fileref="images/LDAP_001.jpg" /></para>
+        <para><graphic fileref="../images/LDAP_001.jpg" /></para>
 
         <para>The <emphasis role="bold">Add New Computers</emphasis> dialog
         displays.</para>
@@ -110,34 +111,36 @@
         <para>Fill in the values for the <emphasis role="bold">Computer
         Attributes</emphasis></para>
 
-        <para><graphic fileref="images/LDAP_002.jpg" /></para>
+        <para><graphic fileref="../images/LDAP_002.jpg" /></para>
       </listitem>
     </orderedlist>
 
     <orderedlist numeration="loweralpha">
       <listitem>
         <para>Provide a <emphasis role="bold">Name Prefix</emphasis>, for
-        example: <emphasis role="bluel">ldap</emphasis>. This helps you to
-        identify it among the list of computers.</para>
+        example: <emphasis role="blue">ldap</emphasis>.</para>
+
+        <para>This helps you to identify it in the list of computers.</para>
       </listitem>
 
       <listitem>
-        <para>Fill in Domain and type with the values of your domain name, as
-        well as the types of machines you are using, In the example above,
-        <emphasis role="bold">Domain</emphasis> is <emphasis
-        role="blue">localdomain</emphasis>, and the <emphasis
+        <para>Fill in <emphasis role="bold">Domain</emphasis> and <emphasis
+        role="bold">Type</emphasis> with the values of your domain name, as
+        well as the types of machines you are using.</para>
+
+        <para>In the example above, <emphasis role="bold">Domain</emphasis> is
+        <emphasis role="blue">localdomain</emphasis>, and the <emphasis
         role="bold">Type</emphasis> is <emphasis
         role="blue">linuxmachine</emphasis>. These should correspond to your
         domain and type.</para>
 
         <para>If you need to add a new domain or machine type to your system
-        to be able to define an existing LDAP or MySQL server, you should set
-        these up first in the other two tabs in the hardware section.</para>
+        to be able to define an existing LDAP server, you should set these up
+        first in the other two tabs in the hardware section.</para>
       </listitem>
 
       <listitem>
-        <para>Add the IP address (or IP address range) as appropriate for the
-        LDAP server.</para>
+        <para>Add the IP address as appropriate for the LDAP server.</para>
       </listitem>
 
       <listitem>
@@ -153,16 +156,15 @@
   <sect2>
     <title>Adding the ldapServer component</title>
 
-    <para>After the LDAP Server component has been added to the appropriate
-    Hardware configuration, you need to configure the Software component
-    definition.</para>
+    <para>After the LDAP Server node has been added to the Hardware
+    configuration, configure the Software LDAP server definition.</para>
 
     <orderedlist numeration="arabic">
       <listitem>
-        <para>Rt-click on Navigator Pane and choose <emphasis role="bold">New
-        Components</emphasis> from the pop up menu, then choose <emphasis
-        role="bold">ldapServer</emphasis> from the pop up menu. <graphic
-        fileref="images/LDAP_003.jpg" /></para>
+        <para>Rt-click on <emphasis role="bold">Navigator</emphasis> Pane and
+        choose <emphasis role="bold">New Components</emphasis> from the pop up
+        menu, then choose <emphasis role="bold">ldapServer</emphasis> from the
+        pop-up menu. <graphic fileref="../images/LDAP_003.jpg" /></para>
 
         <para><emphasis role="bold">Note</emphasis>: The ldapServer component
         is merely a definition that specifies an existing LDAP server. It does
@@ -175,7 +177,7 @@
         <para>Fill in the <emphasis role="bold">LDAP Server Process</emphasis>
         properties:</para>
 
-        <para><graphic fileref="images/LDAP_004.jpg" /><orderedlist
+        <para><graphic fileref="../images/LDAP_004.jpg" /><orderedlist
             numeration="loweralpha">
             <listitem>
               <para>On the <emphasis role="bold">Instances</emphasis> tab,
@@ -187,8 +189,10 @@
             </listitem>
 
             <listitem>
-              <para>Select the computer to use by checking the box next to it.
-              This is the computer you added in the <emphasis
+              <para>Select the computer to use by checking the box next to
+              it.</para>
+
+              <para>This is the computer you added in the <emphasis
               role="bold">Hardware</emphasis> / <emphasis role="bold">Add New
               Computers</emphasis> portion earlier.</para>
             </listitem>
@@ -208,6 +212,14 @@
               <para>Click on the disk icon to save.</para>
             </listitem>
           </orderedlist></para>
+
+        <para><emphasis role="bold">Note</emphasis>: The <emphasis
+        role="bold">cacheTimeout </emphasis>value is the number of minutes
+        that permissions are cached in ESP. If you change any permissions in
+        LDAP, the new settings will not take effect until ESP and Dali refresh
+        the permissions. This could take as long as the cacheTimeout. Setting
+        this to 0 means no cache, but this has performance overhead so it
+        should not be used in production.</para>
       </listitem>
     </orderedlist>
 
@@ -225,7 +237,7 @@
         right hand side, select the <emphasis
         role="bold">Authentication</emphasis> tab.</para>
 
-        <para><graphic fileref="images/LDAP_005.jpg" /></para>
+        <para><graphic fileref="../images/LDAP_005.jpg" /></para>
 
         <para>Fill in the appropriate values:</para>
 
@@ -271,7 +283,7 @@
 
         <para>In the Navigator pane, click on the <emphasis role="bold">Dali
         Server – mydali </emphasis><graphic
-        fileref="images/LDAP_006.jpg" /></para>
+        fileref="../images/LDAP_006.jpg" /></para>
 
         <para>Fill in the values as appropriate:</para>
 
@@ -303,12 +315,56 @@
       </listitem>
 
       <listitem>
-        <para>Deploy your environment settings (see <link
-        linkend="Multi-Node-System">Configuring a Multi-Node System</link>
-        section)</para>
+        <?dbfo keep-together="always"?>
+
+        <para>In the Navigator pane, click on the <emphasis role="bold">Roxie
+        Clusrer – myroxie </emphasis><graphic
+        fileref="../images/LDAP_007.jpg" /></para>
+
+        <para><orderedlist continuation="restarts" numeration="loweralpha">
+            <listitem>
+              <para>On the <emphasis role="bold">RoxieCluster</emphasis> page
+              on the right hand side, select the <emphasis
+              role="bold">Options</emphasis> tab.</para>
+            </listitem>
+
+            <listitem>
+              <para>Scroll down to the <emphasis
+              role="bold">ldapUser</emphasis> field and verify that there is a
+              "<emphasis>roxie</emphasis>" user.</para>
+            </listitem>
+
+            <listitem>
+              <para>You can add password security for Roxie by adding it to
+              the <emphasis role="bold">ldapPassword</emphasis> field on the
+              same tab.</para>
+            </listitem>
+          </orderedlist></para>
       </listitem>
     </orderedlist>
 
-    <para></para>
+    <para><informaltable colsep="1" frame="all" rowsep="1">
+        <?dbfo keep-together="always"?>
+
+        <tgroup cols="2">
+          <colspec colwidth="49.50pt" />
+
+          <colspec />
+
+          <tbody>
+            <row>
+              <entry><inlinegraphic fileref="images/caution.png" /></entry>
+
+              <entry><para>In order to run Roxie queries with File Scope
+              security, ensure that the roxie user is created in the list of
+              authenticated users.</para>In the following section, <link
+              linkend="Adding_Users"><emphasis>Adding and editing
+              users</emphasis></link>, add "<emphasis>roxie</emphasis>" as a
+              user and make sure the password is the same as the one entered
+              in Configuration Manager.</entry>
+            </row>
+          </tbody>
+        </tgroup>
+      </informaltable></para>
   </sect2>
 </sect1>

docs/wip/user_Sect.xml

@@ -10,17 +10,17 @@
   <sect2>
     <title>Introduction</title>
 
-    <para>HPCC systems are designed to maintain security in a number of ways.
-    A secure set of users are created and maintained and access is controlled
-    using LDAP (Lightweight Directory Access Protocol) authentication.
-    Security rights are stored in the LDAP database and maintained by
-    Microsoft’s Active Directory on a Windows system or OpenLDAP on Linux
+    <para>HPCC systems maintain security in a number of ways. HPCC Systems can
+    be configured to manage users' security rights by pointing either at
+    Microsoft’s Active Directory on a Windows system, or OpenLDAP on Linux
     systems.</para>
 
     <para>Using the Permissions interface in ECL Watch, administrators can
     control access to features in ECL IDE, ECL Watch, ECL Plus, DFU Plus, and
-    the ECL modules within the Attribute Repository. Dali server also enforces
-    access to data file folders and workunits.</para>
+    the ECL modules within the Attribute Repository. Additional "file access
+    control" can be implemented over data files by configuring the Dali server
+    to point to the Active Directory/LDAP server. This is what is known as
+    enabling file security.</para>
 
     <para>Permissions are established by group or by user and are defined as
     they are associated with a particular feature of the HPCC System. Only one
@@ -88,9 +88,8 @@
   <sect2>
     <title>Security Administration using ECL Watch</title>
 
-    <para>Administrator access is controlled by your system administrators and
-    you should contact them if you require this level of access. Once you have
-    administrator access rights, open ECL Watch in your browser using the
+    <para>Administrator rights are needed to manager permissions. Once you
+    have administrator access rights, open ECL Watch in your browser using the
     following URL:</para>
 
     <itemizedlist>
@@ -148,7 +147,7 @@
         </listitem>
 
         <listitem>
-          <para>Delete a user who no longer requires access</para>
+          <para>Delete a user</para>
         </listitem>
 
         <listitem>
@@ -163,39 +162,46 @@
           <para>Modify the details/permissions of an individual user</para>
         </listitem>
       </itemizedlist>
+    </sect3>
 
-      <sect4>
-        <title>Adding and editing users</title>
+    <sect3 id="Adding_Users">
+      <title>Adding and editing users</title>
 
-        <para>In ECL Watch, go to the <emphasis
-        role="bold">Users/Permissions</emphasis> menu item and click <emphasis
-        role="bold">Users<emphasis>:</emphasis></emphasis></para>
+      <para>In ECL Watch, go to the <emphasis
+      role="bold">Users/Permissions</emphasis> menu item and click <emphasis
+      role="bold">Users<emphasis>:</emphasis></emphasis></para>
 
-        <graphic fileref="../images/Permissions001.jpg" />
+      <graphic fileref="../images/Permissions001.jpg" />
 
-        <para>All current users are identified in the list by their UserID and
-        full name.</para>
+      <para>All current users are identified in the list by their UserID and
+      full name.</para>
 
-        <para>To add a new user to the list of authenticated users:</para>
+      <sect4>
+        <title>To add a new user to the list of authenticated users:</title>
 
         <orderedlist>
           <listitem>
-            <para>Press the <emphasis role="bold">Add</emphasis> button. The
-            <emphasis role="bold">Add User</emphasis> window is
+            <para>Press the <emphasis role="bold">Add</emphasis>
+            button.</para>
+
+            <para>The <emphasis role="bold">Add User</emphasis> window is
             displayed.</para>
           </listitem>
 
           <listitem>
-            <para>Enter a <emphasis role="bold">UserID</emphasis>. This is the
-            login name for using ECL Watch, ECL IDE, WSECL etc.</para>
+            <para>Enter a <emphasis role="bold">UserID</emphasis>.</para>
+
+            <para>This is the login name for using ECL Watch, ECL IDE, WSECL
+            etc.</para>
           </listitem>
 
           <listitem>
             <para>Enter the <emphasis role="bold">First Name</emphasis> and
-            <emphasis role="bold">Last Name</emphasis> of the user. This
-            information helps to easily identify the user and is displayed in
-            the <emphasis role="bold">Full Name</emphasis> field on the main
-            <emphasis role="bold">Users</emphasis> window.</para>
+            <emphasis role="bold">Last Name</emphasis> of the user.</para>
+
+            <para>This information helps to easily identify the user and is
+            displayed in the <emphasis role="bold">Full Name</emphasis> field
+            on the main <emphasis role="bold">Users</emphasis> window.</para>
           </listitem>
 
           <listitem>
@@ -205,49 +211,60 @@
           </listitem>
 
           <listitem>
-            <para>Press <emphasis role="bold">Submit</emphasis>. Confirmation
-            of the request is shown.</para>
+            <para>Press <emphasis role="bold">Submit</emphasis>.</para>
+
+            <para>Confirmation of the request is shown.</para>
           </listitem>
         </orderedlist>
 
         <para>Once added, the user is displayed in the list and you can modify
         the user's details and set permissions as required.</para>
+      </sect4>
 
-        <para>To modifiy a user's personal details:</para>
+      <sect4>
+        <title>To modifiy a user's personal details:</title>
 
         <orderedlist>
           <listitem>
-            <para>Click the <emphasis role="bold">Edit</emphasis> link. The
-            <emphasis role="bold">User Info Edit</emphasis> window is
-            displayed.</para>
+            <para>Click the <emphasis role="bold">Edit</emphasis> link.</para>
+
+            <para>The <emphasis role="bold">User Info Edit</emphasis> window
+            is displayed.</para>
           </listitem>
 
           <listitem>
             <para>Change the <emphasis role="bold">First Name</emphasis> and
-            <emphasis role="bold">Last Name</emphasis> as required. (The
-            <emphasis role="bold">User Name</emphasis> cannot be
-            changed.)</para>
+            <emphasis role="bold">Last Name</emphasis> as required.</para>
+
+            <para><emphasis role="bold">Note</emphasis>: The <emphasis
+            role="bold">User Name</emphasis> cannot be changed.</para>
           </listitem>
 
           <listitem>
-            <para>Press <emphasis role="bold">Submit</emphasis>. Confirmation
-            of the request is shown.</para>
+            <para>Press <emphasis role="bold">Submit</emphasis>.</para>
+
+            <para>Confirmation of the request is shown.</para>
           </listitem>
         </orderedlist>
+      </sect4>
 
-        <para>To add the user to a group:</para>
+      <sect4>
+        <title>To add the user to a group:</title>
 
         <orderedlist>
           <listitem>
             <para>Click on the <emphasis role="bold">Member of</emphasis>
-            link. The list of groups the user is already associated with is
+            link.</para>
+
+            <para>The list of groups the user is already associated with is
             displayed.</para>
           </listitem>
 
           <listitem>
             <para>To add the user to a group press <emphasis
-            role="bold">ADD</emphasis>. The list of available groups is
-            displayed.</para>
+            role="bold">ADD</emphasis>.</para>
+
+            <para>The list of available groups is displayed.</para>
           </listitem>
 
           <listitem>
@@ -257,15 +274,22 @@
           </listitem>
 
           <listitem>
-            <para>Click OK to confirm. Confirmation of the request is
-            shown.</para>
+            <para>Click OK to confirm.</para>
+
+            <para>Confirmation of the request is shown.</para>
           </listitem>
         </orderedlist>
+      </sect4>
 
-        <para>To delete the user from a group:<orderedlist>
+      <sect4>
+        <title>To delete the user from a group:</title>
+
+        <para><orderedlist>
             <listitem>
               <para>Click on the <emphasis role="bold">Member of</emphasis>
-              link. The list of groups the user is already associated with is
+              link.</para>
+
+              <para>The list of groups the user is already associated with is
               displayed.</para>
             </listitem>
 
@@ -276,58 +300,76 @@
             </listitem>
 
             <listitem>
-              <para>Click OK to confirm. Confirmation of the request is
-              shown.</para>
+              <para>Click OK to confirm.</para>
+
+              <para>Confirmation of the request is shown.</para>
             </listitem>
           </orderedlist></para>
+      </sect4>
+
+      <sect4>
+        <title>To change a user's password:</title>
 
-        <para>To change a user's password:<orderedlist>
+        <para><orderedlist>
             <listitem>
               <para>Click on the <emphasis role="bold">Password</emphasis>
-              link. The <emphasis role="bold">Reset Password</emphasis> window
+              link.</para>
+
+              <para>The <emphasis role="bold">Reset Password</emphasis> window
               is displayed.</para>
             </listitem>
 
             <listitem>
               <para>Complete the <emphasis role="bold">New Password</emphasis>
               and <emphasis role="bold">Retype New Password</emphasis> fields
-              as required. Press <emphasis role="bold">Clear</emphasis> to
-              empty these fields and start again.</para>
+              as required.</para>
+
+              <para>Press <emphasis role="bold">Clear</emphasis> to empty
+              these fields and start again.</para>
             </listitem>
 
             <listitem>
-              <para>Click <emphasis role="bold">Submit</emphasis>.
-              Confirmation of the request is shown.</para>
+              <para>Click <emphasis role="bold">Submit</emphasis>.</para>
+
+              <para>Confirmation of the request is shown.</para>
             </listitem>
           </orderedlist></para>
+      </sect4>
 
-        <para>To delete a user from the list of authenticated
-        users:<orderedlist>
+      <sect4>
+        <title>To delete a user from the list of authenticated users:</title>
+
+        <para><orderedlist>
             <listitem>
               <para>Check the checkbox to the left of the user(s) you want to
-              remove. (These users will no longer have access to ECL
-              Watch.)</para>
+              remove.</para>
+
+              <para><emphasis role="bold">Note:</emphasis> These users will no
+              longer have access to ECL Watch.</para>
             </listitem>
 
             <listitem>
-              <para>Click <emphasis role="bold">Delete</emphasis>.
-              Confirmation of the request is shown.</para>
+              <para>Click <emphasis role="bold">Delete</emphasis>.</para>
+
+              <para>Confirmation of the request is shown.</para>
             </listitem>
           </orderedlist></para>
       </sect4>
+    </sect3>
 
-      <sect4>
-        <title>Setting permissions for an individual user</title>
+    <sect3>
+      <title>Setting permissions for an individual user</title>
 
-        <para>There may be occasions when you need to modify the permissions
-        for individual users. For example, users may have individual security
-        needs that are not completely covered in any group or, there may be
-        occasions when a user requires temporary access to an HPCC feature.
-        Permissions set in this area of ECL Watch only affect the user you
-        choose and any permissions you set here overwrite those set in any
-        group to which the user belongs.</para>
+      <para>There may be occasions when you need to modify the permissions for
+      individual users. For example, users may have individual security needs
+      that are not completely covered in any group or, there may be occasions
+      when a user requires temporary access to an HPCC feature. Permissions
+      set in this area of ECL Watch only affect the user you choose and any
+      permissions you set here overwrite those set in any group to which the
+      user belongs.</para>
 
-        <para>To set new permissions for an individual user:</para>
+      <sect4>
+        <title>To set new permissions for an individual user:</title>
 
         <orderedlist>
           <listitem>
@@ -336,20 +378,24 @@
           </listitem>
 
           <listitem>
-            <para>Locate the user in the list of authenticated users and click
-            on the <emphasis role="bold">Permissions</emphasis> link in the
-            <emphasis role="bold">Operations</emphasis> column. The list of
-            permissions currently set for this user are displayed and the
-            groups from which the user has inherited permissions are also
-            listed. <graphic fileref="../images/Permissions002.jpg" /></para>
+            <para><?dbfo keep-together="always"?>Locate the user in the list
+            of authenticated users and click on the <emphasis
+            role="bold">Permissions</emphasis> link in the <emphasis
+            role="bold">Operations</emphasis> column.</para>
+
+            <para>The list of permissions currently set for this user are
+            displayed and the groups from which the user has inherited
+            permissions are also listed. <graphic
+            fileref="../images/Permissions002.jpg" /></para>
           </listitem>
 
           <listitem>
             <para>A drop down list showing the 6 feature areas of the HPCC is
             provided. Select the feature area you are interested in and press
-            <emphasis role="bold">Add</emphasis>. The <emphasis
-            role="bold">Add Permissions for Authenticated Users</emphasis>
-            page is displayed.</para>
+            <emphasis role="bold">Add</emphasis>.</para>
+
+            <para>The <emphasis role="bold">Add Permissions for Authenticated
+            Users</emphasis> page is displayed.</para>
           </listitem>
 
           <listitem>
@@ -365,12 +411,17 @@
           </listitem>
 
           <listitem>
-            <para>Click <emphasis role="bold">Add</emphasis>. Confirmation of
-            the request is shown.</para>
+            <para>Click <emphasis role="bold">Add</emphasis>.</para>
+
+            <para>Confirmation of the request is shown.</para>
           </listitem>
         </orderedlist>
+      </sect4>
+
+      <sect4>
+        <title>To modify permissions for an individual user:</title>
 
-        <para>To modify permissions for an individual user:<orderedlist>
+        <para><orderedlist>
             <listitem>
               <para>Click the <emphasis role="bold">Users</emphasis> menu item
               in ECL Watch.</para>
@@ -396,8 +447,9 @@
             </listitem>
 
             <listitem>
-              <para>Press OK to confirm. Confirmation of the request is
-              shown.</para>
+              <para>Press OK to confirm.</para>
+
+              <para>Confirmation of the request is shown.</para>
             </listitem>
           </orderedlist></para>
       </sect4>
@@ -407,11 +459,13 @@
       <title>Setting and modifying group permissions</title>
 
       <para>Setting up groups ensures that all users with the same permission
-      needs have the same permission settings, giving them the access they
-      require to the feature areas of HPPC they need to use. There is no limit
-      to the number of groups you can create, so you can create as many groups
-      as you need to control access for all your users regardless of their
-      tasks. Use the <emphasis role="bold">Group</emphasis> menu item
+      needs have the same permission settings. You can give users the access
+      they require to the feature areas of HPCC they need. There is no limit
+      to the number of groups you can create. You can create as many groups as
+      you need to control access for all your users regardless of their
+      tasks.</para>
+
+      <para>Use the <emphasis role="bold">Group</emphasis> menu item
       to:<itemizedlist>
           <listitem>
             <para>Add a new group</para>
@@ -434,25 +488,30 @@
         <title>Adding and editing groups</title>
 
         <para>When adding or changing the permissions for a group, all members
-        of that group are given those settings. So it important to be sure
-        that you are giving or denying access to features appropriately for
-        the members of that group. If you need to make a change for a single
-        users (or small number of users), it is probably better to make that
-        change for each individual user as demonstrated earlier. Since
-        individual permission settings take precedence over the group
-        settings, you can safely change the individual settings for a user
-        without affecting the rest of the group(s) to which they belong.
-        <graphic fileref="../images/Permissions003.jpg" />In ECL Watch, go to
-        the <emphasis role="bold">Users/Permissions</emphasis> menu item and
-        click <emphasis role="bold">Groups</emphasis>:</para>
-
-        <para>To add a new group:</para>
+        of that group are given those permission settings. So it is important
+        to be sure that you are giving or denying access to features
+        appropriate for the members of that group. If you need to make a
+        change for a single user (or small number of users), it is probably
+        better to make that change for each individual user as illustrated in
+        the previous sections. Since individual permission settings take
+        precedence over the group settings, you can safely change the
+        individual settings for a user without affecting the rest of the
+        group(s) to which they belong. <graphic
+        fileref="../images/Permissions003.jpg" />In ECL Watch, go to the
+        <emphasis role="bold">Users/Permissions</emphasis> menu item and click
+        <emphasis role="bold">Groups</emphasis>:</para>
+      </sect4>
+
+      <sect4>
+        <title>To add a new group:</title>
 
         <para><orderedlist>
             <listitem>
-              <para>Press the <emphasis role="bold">Add</emphasis> button. The
-              <emphasis role="bold">Add Group</emphasis> window is displayed.
-              </para>
+              <para>Press the <emphasis role="bold">Add</emphasis>
+              button.</para>
+
+              <para>The <emphasis role="bold">Add Group</emphasis> window is
+              displayed.</para>
             </listitem>
 
             <listitem>
@@ -461,14 +520,19 @@
             </listitem>
 
             <listitem>
-              <para>Press <emphasis role="bold">Submit</emphasis>.
-              Confirmation of the request is shown. <emphasis
+              <para>Press <emphasis role="bold">Submit</emphasis>.</para>
+
+              <para>Confirmation of the request is shown. <emphasis
               role="bold">Permissions</emphasis> may now be set for this new
               group.</para>
             </listitem>
           </orderedlist></para>
+      </sect4>
+
+      <sect4>
+        <title>To delete a group:</title>
 
-        <para>To delete a group:<orderedlist>
+        <para><orderedlist>
             <listitem>
               <para>Locate the group in the list and check the checkbox to the
               left.</para>
@@ -480,22 +544,30 @@
             </listitem>
 
             <listitem>
-              <para>Press <emphasis role="bold">OK</emphasis> to confirm.
-              Confirmation of the request is shown.</para>
+              <para>Press <emphasis role="bold">OK</emphasis> to
+              confirm.</para>
+
+              <para>Confirmation of the request is shown.</para>
             </listitem>
           </orderedlist></para>
+      </sect4>
+
+      <sect4>
+        <title>To add new members to a group:</title>
 
-        <para>To add new members to a group:<orderedlist>
+        <para><orderedlist>
             <listitem>
               <para>Locate the group in the list and click on the <emphasis
               role="bold">Members</emphasis> link in the <emphasis
-              role="bold">Operations</emphasis> column. All current members of
-              the group are listed.</para>
+              role="bold">Operations</emphasis> column.</para>
+
+              <para>All current members of the group are listed.</para>
             </listitem>
 
             <listitem>
-              <para>Press <emphasis role="bold">Add</emphasis>. All
-              authenticated users are listed.</para>
+              <para>Press <emphasis role="bold">Add</emphasis>.</para>
+
+              <para>All authenticated users are listed.</para>
             </listitem>
 
             <listitem>
@@ -505,12 +577,18 @@
             </listitem>
 
             <listitem>
-              <para>Press <emphasis role="bold">OK</emphasis> to confirm.
-              Confirmation of the request is shown.</para>
+              <para>Press <emphasis role="bold">OK</emphasis> to
+              confirm.</para>
+
+              <para>Confirmation of the request is shown.</para>
             </listitem>
           </orderedlist></para>
+      </sect4>
+
+      <sect4>
+        <title>To delete members from a group:</title>
 
-        <para>To delete members from a group:<orderedlist>
+        <para><orderedlist>
             <listitem>
               <para>Locate the group in the list and click on the <emphasis
               role="bold">Members</emphasis> link in the <emphasis
@@ -524,45 +602,64 @@
             </listitem>
 
             <listitem>
-              <para>Press <emphasis role="bold">OK</emphasis> to confirm.
-              Confirmation of the request is shown.</para>
+              <para>Press <emphasis role="bold">OK</emphasis> to
+              confirm.</para>
+
+              <para>Confirmation of the request is shown.</para>
             </listitem>
           </orderedlist></para>
       </sect4>
+    </sect3>
 
-      <sect4>
-        <title>Setting permissions for a group</title>
+    <sect3>
+      <title>Setting permissions for a group</title>
+
+      <para>By default, all users are members of the <emphasis
+      role="bold">Authenticated Users </emphasis>group. The <emphasis
+      role="bold">Authenticated users</emphasis> group has access rights to
+      almost all controls.</para>
+
+      <para>If you intend to restrict permissions for some users, you must
+      remove <emphasis role="bold">Authenticated Users</emphasis> from the
+      sections you wish to limit. You can then create groups with only those
+      access rights you wish to grant. This approach allows the most
+      flexibility since a single User ID can have multiple group
+      memberships.</para>
 
-        <para>All users are a member of the <emphasis
-        role="bold">Authenticated Users </emphasis>group, which is
-        automatically updated as you add or remove users. Those who need to be
-        able to carry out security administration must have full access to be
-        able to gain access to the <emphasis
-        role="bold">User/Permissions</emphasis> area of ECL Watch
-        (Administrators).</para>
+      <para>As a best practice, you should use <emphasis
+      role="bold">Allow</emphasis> instead of <emphasis
+      role="bold">Deny</emphasis> to control access. Denies should be used
+      only as an exception.</para>
 
-        <para>To set new permissions for a group:</para>
+      <para></para>
+
+      <sect4>
+        <title>To set new permissions for a group:</title>
 
         <orderedlist>
           <listitem>
-            <para> Click the <emphasis role="bold">Groups</emphasis> menu item
+            <para>Click the <emphasis role="bold">Groups</emphasis> menu item
             in ECL Watch.</para>
           </listitem>
 
           <listitem>
-            <para>Locate the group in the list and click on the <emphasis
-            role="bold">Permissions</emphasis> link in the <emphasis
-            role="bold">Operations</emphasis> column. The list of permissions
-            currently set for this group are displayed.<graphic
+            <para><?dbfo keep-together="always"?>Locate the group in the list
+            and click on the <emphasis role="bold">Permissions</emphasis> link
+            in the <emphasis role="bold">Operations</emphasis> column.</para>
+
+            <para>The list of permissions currently set for this group are
+            displayed.<graphic
             fileref="../images/Permissions004.jpg" /></para>
           </listitem>
 
           <listitem>
             <para>A drop down list showing the 6 feature areas of the HPCC is
             shown. Select the feature area you want and press <emphasis
-            role="bold">Add</emphasis>. The <emphasis role="bold">Add
-            Permissions for </emphasis><emphasis
-            role="bold">&lt;GroupName&gt;</emphasis> page is displayed.</para>
+            role="bold">Add</emphasis>.</para>
+
+            <para>The <emphasis role="bold">Add Permissions for
+            </emphasis><emphasis role="bold">&lt;GroupName&gt;</emphasis> page
+            is displayed.</para>
           </listitem>
 
           <listitem>
@@ -578,12 +675,15 @@
           </listitem>
 
           <listitem>
-            <para>Click <emphasis role="bold">Add</emphasis>. MORE NEEDED WHEN
-            FIXED</para>
+            <para>Click <emphasis role="bold">Add</emphasis>.</para>
           </listitem>
         </orderedlist>
+      </sect4>
+
+      <sect4>
+        <title>To modify permissions for a group:</title>
 
-        <para>To modify permissions for a group:<orderedlist>
+        <para><orderedlist>
             <listitem>
               <para>Click the <emphasis role="bold">Groups</emphasis> menu
               item in ECL Watch.</para>
@@ -609,8 +709,9 @@
             </listitem>
 
             <listitem>
-              <para>Press OK to confirm. Confirmation of the request is
-              shown.</para>
+              <para>Press OK to confirm.</para>
+
+              <para>Confirmation of the request is shown.</para>
             </listitem>
           </orderedlist></para>
       </sect4>
@@ -651,89 +752,96 @@
           <para>Delete a resource</para>
         </listitem>
       </itemizedlist>
+    </sect3>
 
-      <sect4>
-        <title>Adding and editing feature permissions</title>
+    <sect3>
+      <title>Adding and editing feature permissions</title>
+
+      <para>Each feature contains a list of resources which are used to
+      control access to an HPCC feature or folders containing files or
+      workunits. The main HPCC feature permission setting are controlled using
+      the <emphasis role="bold">ESP Features for SMC</emphasis> setting. When
+      new features are added to the HPCC System, the release notes inform you
+      that new permissions may be set. This is also true for <emphasis
+      role="bold">ESP Features for ECLDirectAccess</emphasis> and <emphasis
+      role="bold">Esp Features for WsEclAccess</emphasis>. Generally, all the
+      permissions you require to control access to these features are already
+      included.</para>
+
+      <para>However, to control access to file or workunit scopes, you must
+      add the location as a resource before you can set permissions.</para>
+    </sect3>
 
-        <para>Each feature contains a list of resources which are used to
-        control access to an HPCC feature or folders containing files or
-        workunits. The main HPCC feature permission setting are controlled
-        using the <emphasis role="bold">ESP Features for SMC</emphasis>
-        setting. When new features are added to the HPCC System, the release
-        notes inform you that new permissions may be set. This is also true
-        for <emphasis role="bold">ESP Features for ECLDirectAccess</emphasis>
-        and <emphasis role="bold">Esp Features for WsEclAccess</emphasis>.
-        Generally, all the permissions you require to control access to these
-        features are already included.</para>
+    <sect3>
+      <title>To add a scope:</title>
 
-        <para>However, to control access to file or workunit scopes, you must
-        add the location as a resource before you can set permissions.</para>
+      <orderedlist>
+        <listitem>
+          <para>Click the <emphasis role="bold">Permissions</emphasis> menu
+          item in ECL Watch, locate the feature you want and click <emphasis
+          role="bold">Edit</emphasis>.</para>
 
-        <para>To add a scope:</para>
+          <para>The resources for that feature are listed.</para>
+        </listitem>
 
-        <orderedlist>
-          <listitem>
-            <para>Click the <emphasis role="bold">Permissions</emphasis> menu
-            item in ECL Watch, locate the feature you want and click <emphasis
-            role="bold">Edit</emphasis>. The resources for that feature are
-            listed.</para>
-          </listitem>
+        <listitem>
+          <para>Press the <emphasis role="bold">Add</emphasis> button.</para>
+        </listitem>
 
-          <listitem>
-            <para>Press the <emphasis role="bold">Add</emphasis>
-            button.</para>
-          </listitem>
+        <listitem>
+          <para>Enter the exact name of the scope you want to add (for example
+          a file or workunit scope) in the <emphasis
+          role="bold">Name</emphasis> field and also a short <emphasis
+          role="bold">Description<emphasis>.</emphasis></emphasis></para>
+        </listitem>
 
-          <listitem>
-            <para>Enter the exact name of the scope you want to add (for
-            example a file or workunit scope) in the <emphasis
-            role="bold">Name</emphasis> field and also a short <emphasis
-            role="bold">Description<emphasis>.</emphasis></emphasis></para>
-          </listitem>
+        <listitem>
+          <para>Click <emphasis role="bold">Submit</emphasis>.</para>
 
-          <listitem>
-            <para>Click <emphasis role="bold">Submit</emphasis>. Confirmation
-            of your request is shown.</para>
-          </listitem>
+          <para>Confirmation of your request is shown.</para>
+        </listitem>
 
-          <listitem>
-            <para>Go back to the features page using the link provided.</para>
-          </listitem>
+        <listitem>
+          <para>Go back to the features page using the link provided.</para>
+        </listitem>
 
-          <listitem>
-            <para>Locate your new scope in the list and click <emphasis
-            role="bold">Permissions</emphasis>. The <emphasis
-            role="bold">Administrator</emphasis> and <emphasis
-            role="bold">Authenticated Users</emphasis> groups are shown
-            showing the default permission settings which you can update as
-            appropriate.</para>
-          </listitem>
+        <listitem>
+          <para>Locate your new scope in the list and click <emphasis
+          role="bold">Permissions</emphasis>.</para>
 
-          <listitem>
-            <para>To add more users and groups and set permissions for this
-            scope, click <emphasis role="bold">Add</emphasis>. The <emphasis
-            role="bold">Add Permissions</emphasis> window is displayed.</para>
-          </listitem>
+          <para>The <emphasis role="bold">Administrator</emphasis> and
+          <emphasis role="bold">Authenticated Users</emphasis> groups are
+          shown showing the default permission settings which you can update
+          as appropriate.</para>
+        </listitem>
 
-          <listitem>
-            <para>Select a <emphasis role="bold">User</emphasis> or <emphasis
-            role="bold">Group</emphasis> from the dropdown lists provided and
-            check the checkboxes for <emphasis role="bold">allow</emphasis>
-            and <emphasis role="bold">deny</emphasis> as appropriate.</para>
-          </listitem>
+        <listitem>
+          <para>To add more users and groups and set permissions for this
+          scope, click <emphasis role="bold">Add</emphasis>.</para>
 
-          <listitem>
-            <para>Click <emphasis role="bold">Add</emphasis>. Confirmation of
-            your request is shown.</para>
+          <para>The <emphasis role="bold">Add Permissions</emphasis> window is
+          displayed.</para>
+        </listitem>
 
-            <para><emphasis role="bold">Note:</emphasis>This description shows
-            how to add a file or workunit scope, However if you do need to add
-            a new resource any other feature area, the process is the
-            same.</para>
-          </listitem>
-        </orderedlist>
+        <listitem>
+          <para>Select a <emphasis role="bold">User</emphasis> or <emphasis
+          role="bold">Group</emphasis> from the dropdown lists provided and
+          check the checkboxes for <emphasis role="bold">allow</emphasis> and
+          <emphasis role="bold">deny</emphasis> as appropriate.</para>
+        </listitem>
 
-        <para>To edit the permissions for a feature resource:</para>
+        <listitem>
+          <para>Click <emphasis role="bold">Add</emphasis>. Confirmation of
+          your request is shown.</para>
+
+          <para><emphasis role="bold">Note:</emphasis>This description shows
+          how to add a file or workunit scope, However if you do need to add a
+          new resource any other feature area, the process is the same.</para>
+        </listitem>
+      </orderedlist>
+
+      <sect4>
+        <title>To edit the permissions for a feature resource:</title>
 
         <orderedlist>
           <listitem>
@@ -768,8 +876,12 @@
             process for each user or group separately.</para>
           </listitem>
         </orderedlist>
+      </sect4>
+
+      <sect4>
+        <title>To delete a resource from a feature list:</title>
 
-        <para>To delete a resource from a feature list:<orderedlist>
+        <para><orderedlist>
             <listitem>
               <para>Click the <emphasis role="bold">Permissions</emphasis>
               menu item in ECL Watch, locate the feature you want and click
@@ -792,20 +904,26 @@
               shown.<emphasis role="bold"></emphasis></para>
             </listitem>
           </orderedlist></para>
+      </sect4>
 
-        <para>To delete the resource permission settings for a user or group:
-        <orderedlist>
+      <sect4>
+        <title>To delete the resource permission settings for a user or
+        group:</title>
+
+        <para><orderedlist>
             <listitem>
               <para>Click the <emphasis role="bold">Permissions</emphasis>
               menu item in ECL Watch, locate the feature you want and click
-              <emphasis role="bold">Edit</emphasis>. The resources for that
-              feature are listed.</para>
+              <emphasis role="bold">Edit</emphasis>.</para>
+
+              <para>The resources for that feature are listed.</para>
             </listitem>
 
             <listitem>
               <para>Locate the resource you want to remove and click <emphasis
-              role="bold">Permissions</emphasis>. The users and groups are
-              displayed.</para>
+              role="bold">Permissions</emphasis>.</para>
+
+              <para>The users and groups are displayed.</para>
             </listitem>
 
             <listitem>
@@ -814,8 +932,10 @@
             </listitem>
 
             <listitem>
-              <para>Click OK to confirm. Confirmation of your request is
-              shown.<emphasis role="bold"></emphasis></para>
+              <para>Click OK to confirm.</para>
+
+              <para>Confirmation of your request is shown.<emphasis
+              role="bold"></emphasis></para>
             </listitem>
           </orderedlist></para>
       </sect4>
@@ -1364,7 +1484,7 @@
 
         <mediaobject>
           <imageobject>
-            <imagedata fileref="images/US003.jpg" />
+            <imagedata fileref="../images/US003.jpg" />
           </imageobject>
         </mediaobject>
       </figure></para>
@@ -1413,13 +1533,11 @@
     <para>Attempting to access a file in a folder for which access is not
     granted will result in one of the following errors:</para>
 
-    <para>DFS Exception: 4 Create access denied for scope
-    &lt;filepath&gt;</para>
+    <programlisting>DFS Exception: 4 Create access denied for scope &lt;filepath&gt;</programlisting>
 
-    <para></para>
+    <para>or</para>
 
-    <para>DFS Exception: 3 Lookup access denied for scope
-    &lt;filepath&gt;</para>
+    <programlisting>DFS Exception: 3 Lookup access denied for scope &lt;filepath&gt;</programlisting>
 
     <para></para>
 
@@ -1493,7 +1611,7 @@
   <sect2>
     <title>Workunit Access Control</title>
 
-    <para>There are 2 aspects of workunit security:</para>
+    <para>There are 2 aspects of workunit (WU) security:</para>
 
     <itemizedlist>
       <listitem>

ecl/hql/hqlexpr.cpp

@@ -11062,8 +11062,6 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
     case no_newrow:
     case no_assert_ds:
     case no_spillgraphresult:
-    case no_loop:
-    case no_graphloop:
     case no_cluster:
     case no_forcenolocal:
     case no_thisnode:
@@ -11079,6 +11077,18 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
     case no_dataset_alias:
         type.set(datasetType);
         break;
+    case no_loop:
+        {
+            IHqlExpression & body = parms.item(4);
+            type.setown(getTypeIntersection(datasetType, body.queryChild(0)->queryType()));
+            break;
+        }
+    case no_graphloop:
+        {
+            IHqlExpression & body = parms.item(2);
+            type.setown(getTypeIntersection(datasetType, body.queryChild(0)->queryType()));
+            break;
+        }
     case no_serialize:
         type.setown(getSerializedForm(datasetType));
         break;

initfiles/componentfiles/configxml/@temp/esp_service_WsSMC.xsl

@@ -502,7 +502,14 @@ This is required by its binding with ESP service '<xsl:value-of select="$espServ
         
         <EspService name="{$serviceName}" type="{$serviceType}" plugin="{$servicePlugin}">
             <xsl:variable name="ldapservername" select="$bindingNode/../Authentication/@ldapServer"/>
-            <Files basedn="{../DaliServerProcess/@filesBasedn}"/>
+            <xsl:choose>
+                <xsl:when test="string(@filesBasedn) != ''">
+                    <Files basedn="{@filesBasedn}"/>
+                </xsl:when>
+                <xsl:otherwise>
+                    <Files basedn="ou=Files,ou=ecl"/>
+                </xsl:otherwise>
+            </xsl:choose>
             <Resources>
                 <xsl:for-each select="../EspProcess[Authentication/@ldapServer=$ldapservername]/EspBinding">
                     <Binding name="{@name}" service="{@service}" port="{@port}" basedn="{@resourcesBasedn}" workunitsBasedn="{@workunitsBasedn}"/>

initfiles/componentfiles/configxml/espsmcservice.xsd.in

@@ -63,6 +63,13 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="filesBasedn" type="xs:string" use="optional" default="ou=Files,ou=ecl">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>base location for ldap file scopes</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
             <xs:attribute name="pluginsPath" type="relativePath" use="optional" default="${PLUGINS_PATH}">
                 <xs:annotation>
                     <xs:appinfo>

initfiles/componentfiles/configxml/setvars_linux.xsl

@@ -60,7 +60,7 @@ export THORMASTER=<xsl:call-template name="getNetAddress">
 export THORMASTERPORT=<xsl:value-of select="@masterport"/>
     </xsl:when>
     <xsl:otherwise>
-export THORMASTERPORT=6500
+export THORMASTERPORT=20000
     </xsl:otherwise>
 </xsl:choose>
 <xsl:choose>
@@ -68,7 +68,7 @@ export THORMASTERPORT=6500
 export THORSLAVEPORT=<xsl:value-of select="@slaveport"/>
     </xsl:when>
     <xsl:otherwise>
-export THORSLAVEPORT=6600
+export THORSLAVEPORT=20100
     </xsl:otherwise>
 </xsl:choose>
 <xsl:if test="string(@localThorPortInc) != ''">

initfiles/etc/DIR_NAME/environment.xml.in

@@ -29,7 +29,7 @@
   <Switch name="Switch"/>
  </Hardware>
  <Programs>
-  <Build name="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}" url="${INSTALL_DIR}">
+  <Build name="${projname}_${version}-${stagever}" url="${INSTALL_DIR}">
    <BuildSet installSet="deploy_map.xml"
              name="dafilesrv"
              path="componentfiles/dafilesrv"
@@ -371,7 +371,7 @@
   </Build>
  </Programs>
  <Software>
-  <DafilesrvProcess build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+  <DafilesrvProcess build="${projname}_${version}-${stagever}"
                     buildSet="dafilesrv"
                     description="DaFileSrv process"
                     name="mydafilesrv"
@@ -381,7 +381,7 @@
              name="s1"
              netAddress="."/>
   </DafilesrvProcess>
-  <DaliServerProcess build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+  <DaliServerProcess build="${projname}_${version}-${stagever}"
                      buildSet="dali"
                      environment="${CONFIG_DIR}/${ENV_XML_FILE}"
                      name="mydali"
@@ -392,7 +392,7 @@
              netAddress="."
              port="7070"/>
   </DaliServerProcess>
-  <DfuServerProcess build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+  <DfuServerProcess build="${projname}_${version}-${stagever}"
                     buildSet="dfuserver"
                     daliServers="mydali"
                     description="DFU Server"
@@ -423,14 +423,14 @@
    <Category dir="${EXEC_PREFIX}/lib/[NAME]/queries/[INST]" name="query"/>
    <Category dir="${EXEC_PREFIX}/lock/[NAME]/[INST]" name="lock"/>
   </Directories>
-  <DropZone build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+  <DropZone build="${projname}_${version}-${stagever}"
             buildSet="DropZone"
             computer="localhost"
             description="DropZone process"
             directory="${RUNTIME_PATH}/mydropzone"
             name="mydropzone"/>
   <EclAgentProcess allowedPipePrograms="*"
-                   build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+                   build="${projname}_${version}-${stagever}"
                    buildSet="eclagent"
                    daliServers="mydali"
                    defaultMemoryLimitMB="300"
@@ -445,7 +445,7 @@
              name="s1"
              netAddress="."/>
   </EclAgentProcess>
-   <EclCCServerProcess build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+   <EclCCServerProcess build="${projname}_${version}-${stagever}"
                     buildSet="eclccserver"
                     daliServers="mydali"
                     description="EclCCServer process"
@@ -458,7 +458,7 @@
              name="s1"
              netAddress="."/>
   </EclCCServerProcess>
-   <EclSchedulerProcess build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+   <EclSchedulerProcess build="${projname}_${version}-${stagever}"
                 buildSet="eclscheduler"
                 daliServers="mydali"
                 description="EclScheduler process"
@@ -468,7 +468,7 @@
                name="s1"
                netAddress="."/>
   </EclSchedulerProcess>
-   <EspProcess build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+   <EspProcess build="${projname}_${version}-${stagever}"
               buildSet="esp"
               componentfilesDir="${COMPONENTFILES_PATH}"
               daliServers="mydali"
@@ -667,7 +667,7 @@
   </EspProcess>
   <EspService allowNewRoxieOnDemandQuery="false"
               AWUsCacheTimeout="15"
-              build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+              build="${projname}_${version}-${stagever}"
               buildSet="espsmc"
               description="ESP services for SMC"
               disableUppercaseTranslation="false"
@@ -677,6 +677,7 @@
               name="EclWatch"
               pluginsPath="${PLUGINS_PATH}"
               syntaxCheckQueue=""
+              filesBasedn="ou=Files,ou=ecl"
               viewTimeout="1000"
               warnIfCpuLoadOver="95"
               warnIfFreeMemoryUnder="5"
@@ -863,7 +864,7 @@
     </ProcessFilters>
    </Properties>
   </EspService>
-  <EspService build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+  <EspService build="${projname}_${version}-${stagever}"
               buildSet="ws_ecl"
               description="WS ECL Service"
               name="ws_ecl">
@@ -884,7 +885,7 @@
                          service="ws_ecl"/>
    </Properties>
   </EspService>
-  <EspService build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+  <EspService build="${projname}_${version}-${stagever}"
               buildSet="ecldirect"
               clusterName="hthor"
               description="ESP service for running raw ECL queries"
@@ -906,7 +907,7 @@
                          service="ecldirect"/>
    </Properties>
   </EspService>
-  <FTSlaveProcess build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+  <FTSlaveProcess build="${projname}_${version}-${stagever}"
                   buildSet="ftslave"
                   description="FTSlave process"
                   name="myftslave"
@@ -921,7 +922,7 @@
                 baseDataDir="${RUNTIME_PATH}/hpcc-data/roxie"
                 blindLogging="false"
                 blobCacheMem="0"
-                build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+                build="${projname}_${version}-${stagever}"
                 buildSet="roxie"
                 callbackRetries="3"
                 callbackTimeout="500"
@@ -1101,7 +1102,7 @@
                       netAddress="."/>
   </RoxieCluster>
   <SashaServerProcess autoRestartInterval="0"
-                      build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+                      build="${projname}_${version}-${stagever}"
                       buildSet="sasha"
                       cachedWUat="* * * * *"
                       cachedWUinterval="24"
@@ -1154,7 +1155,7 @@
              port="8877"/>
   </SashaServerProcess>
   <ThorCluster autoCopyBackup="false"
-               build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}"
+               build="${projname}_${version}-${stagever}"
                buildSet="thor"
                computer="localhost"
                daliServers="mydali"
@@ -1179,7 +1180,7 @@
    <ThorMasterProcess computer="localhost" name="m1"/>
    <ThorSlaveProcess computer="localhost" name="s1"/>
   </ThorCluster>
-  <Topology build="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}" buildSet="topology" name="topology">
+  <Topology build="${projname}_${version}-${stagever}" buildSet="topology" name="topology">
    <Cluster name="hthor" prefix="hthor">
     <EclCCServerProcess process="myeclccserver"/>
     <EclSchedulerProcess process="myeclscheduler"/>

initfiles/etc/DIR_NAME/version.in

@@ -1 +1 @@
-${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}
+${projname}_${version}-${stagever}

plugins/datastream/CMakeLists.txt

@@ -3,6 +3,7 @@ cmake_minimum_required (VERSION 2.6)
 
 set ( HPCC_DATASTREAM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set ( HPCC_SOURCE_DIR ${HPCC_DATASTREAM_SOURCE_DIR}/../../)
+include(${HPCC_SOURCE_DIR}/version.cmake)
 
 set ( CMAKE_MODULE_PATH "${HPCC_SOURCE_DIR}/cmake_modules")
 set ( EXECUTABLE_OUTPUT_PATH "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bin" )
@@ -31,31 +32,25 @@ message ("-- 64bit architecture is ${ARCH64BIT}")
 
 set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_DEBUG -DDEBUG")
 
-###
-## Version Information keep in synch with top level hpcc versioning
-###
-set ( HPCC_PROJECT "community" )
-set ( HPCC_MAJOR 3 )
-set ( HPCC_MINOR 6 )
-set ( HPCC_POINT 2 )
-set ( HPCC_MATURITY "rc" )
-set ( HPCC_SEQUENCE 3 )
-###
-
 include(${HPCC_SOURCE_DIR}/cmake_modules/optionDefaults.cmake)
 include(${HPCC_SOURCE_DIR}/cmake_modules/commonSetup.cmake)
 
+INCLUDE(InstallRequiredSystemLibraries)
+SET(CPACK_PACKAGE_VERSION_MAJOR ${majorver})
+SET(CPACK_PACKAGE_VERSION_MINOR ${minorver})
+SET(CPACK_PACKAGE_VERSION_PATCH ${point}${stagever})
 set ( CPACK_PACKAGE_CONTACT "HPCCSystems <ossdevelopment@lexisnexis.com>" )
-set ( CPACK_SOURCE_GENERATOR TGZ )
-set ( CPACK_RPM_PACKAGE_VERSION "${projname}" )
-
+set( CPACK_SOURCE_GENERATOR TGZ )
+set ( CPACK_RPM_PACKAGE_VERSION "${projname}")
+SET(CPACK_RPM_PACKAGE_RELEASE "${version}${stagever}")
 if ( ${ARCH64BIT} EQUAL 1 )
     set ( CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
 else( ${ARCH64BIT} EQUAL 1 )
     set ( CPACK_RPM_PACKAGE_ARCHITECTURE "i386")
 endif ( ${ARCH64BIT} EQUAL 1 )
-
 set(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CPACK_RPM_PACKAGE_ARCHITECTURE}")
+
+
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
     set(CPACK_STRIP_FILES TRUE)
 endif()

plugins/datastream/hdfsstream/CMakeLists.txt

@@ -12,7 +12,10 @@ add_subdirectory (ecl)
 	#generate config for script.
 	#add script processor for vars.
 
-	configure_file("hdfsstreamconfig.in" "hdfsstreamconfig")
+	configure_file("hdfsstream.conf.in" "hdfsstream.conf")
+
+	set(HDFSCONFIG "${CMAKE_INSTALL_PREFIX}/${OSSDIR}${CONFIG_DIR}")
+	configure_file("hdfspipe.in" "hdfspipe" @ONLY )
 
 	find_package(JNI REQUIRED)
 	find_package(LIBHDFS REQUIRED)
@@ -31,9 +34,8 @@ add_subdirectory (ecl)
 
 	set ( INSTALLDIR "${OSSDIR}/bin")
 	Install ( TARGETS hdfsstream DESTINATION ${INSTALLDIR} COMPONENT Runtime)
-	Install ( PROGRAMS hdfspipe DESTINATION ${INSTALLDIR} COMPONENT Runtime )
-	Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/hdfsstreamconfig DESTINATION ${INSTALLDIR} COMPONENT Runtime )
-
+	Install ( PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/hdfspipe DESTINATION ${INSTALLDIR} COMPONENT Runtime )
+	Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/hdfsstream.conf DESTINATION ${HDFSCONFIG} COMPONENT Runtime )
 	target_link_libraries ( hdfsstream
 					${JAVA_JVM_LIBRARY}
 					${LIBHDFS_LIBRARIES})

plugins/datastream/hdfsstream/ecl/CMakeLists.txt

@@ -1 +1 @@
-Install ( FILES HDFSPipe.ecl DESTINATION "${OSSDIR}/share/My Files/DataStream" COMPONENT Runtime )
+Install ( FILES HDFSPipe.ecl DESTINATION "${OSSDIR}/share/DataStream" COMPONENT Runtime )

plugins/datastream/hdfsstream/hdfspipe

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-source hdfsstreamconfig 
+
+source @HDFSCONFIG@/hdfsstream.conf 
 
 CLASSPATH=$CLASSPATH:$HADOOP_LOCATION/conf
 

plugins/dbconnectors/CMakeLists.txt

@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 2.8)
+project (hpccsystems-dbconnectors)
+add_subdirectory (ecljdbc)

plugins/dbconnectors/ecljdbc/CMakeLists.txt

@@ -0,0 +1,20 @@
+cmake_minimum_required(VERSION 2.8)
+project(ecljdbc Java)
+option(USE_ECLJDBC "Configure use of DB Connectors")
+if ( USE_ECLJDBC )
+	add_subdirectory (src/main/java/com/hpccsystems/ecljdbc)
+
+        find_package(JNI REQUIRED)
+        find_package(Java REQUIRED)
+
+        include_directories (
+                                        ${CMAKE_BINARY_DIR}
+                                        ${CMAKE_BINARY_DIR}/oss
+                                        ${JNI_INCLUDE_DIRS}
+                                        ${JAVA_INCLUDE_PATH}
+                                        ${JAVA_INCLUDE_PATH2}
+                                        )
+
+        set ( INSTALLDIR "${OSSDIR}/bin")
+
+endif( USE_ECLJDBC)

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/CMakeLists.txt

@@ -0,0 +1,50 @@
+cmake_minimum_required(VERSION 2.8)
+
+#configure_file("EclVersionTracker.java.in" "EclVersionTracker.java")
+
+set ( HPCC_ECLJDBC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set ( HPCC_SOURCE_DIR ${HPCC_ECLJDBC_SOURCE_DIR}/../../../../../../../../../..)
+include(${HPCC_SOURCE_DIR}/version.cmake)
+
+set ( CMAKE_MODULE_PATH "${HPCC_SOURCE_DIR}/cmake_modules")
+set ( EXECUTABLE_OUTPUT_PATH "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bin" )
+set ( PRODUCT_PREFIX "hpccsystems" )
+
+#include(${HPCC_SOURCE_DIR}/cmake_modules/optionDefaults.cmake)
+#include(${HPCC_SOURCE_DIR}/cmake_modules/commonSetup.cmake)
+
+SET (CLASS_DIR "class")
+#SET (JAR_DIR "jar")
+FILE(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${CLASS_DIR})
+#FILE(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${JAR_DIR})
+#SET (JAR_FILE ecljdbc.jar)
+SET (JAR_FILE "${PRODUCT_PREFIX}-ecljdbc_${HPCC_PROJECT}-${HPCC_MAJOR}.${HPCC_MINOR}.${HPCC_POINT}.jar")
+SET (JAVA_FILES
+EclDatabaseMetaData.java EclQuery.java SqlExpression.java DFUFile.java EclDriver.java EclResultSet.java SqlOperator.java EclColumn.java EclEngine.java EclResultSetMetadata.java SqlParser.java EclColumnMetaData.java EclPreparedStatement.java EclStatement.java SqlWhereClause.java EclConnection.java EclQueries.java SqlColumn.java Utils.java 
+#EclDatabaseMetaData.java EclQuery.java SqlExpression.java DFUFile.java EclDriver.java EclResultSet.java SqlOperator.java EclColumn.java EclEngine.java EclResultSetMetadata.java SqlParser.java EclColumnMetaData.java EclPreparedStatement.java EclStatement.java SqlWhereClause.java EclConnection.java EclQueries.java SqlColumn.java Utils.java ${CMAKE_CURRENT_BINARY_DIR}/EclVersionTracker.java
+)
+
+# compile all .java files with javac to .class
+ADD_CUSTOM_COMMAND(
+OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${JAVA_FILES}.class
+COMMAND ${CMAKE_Java_COMPILER}
+#ARGS -d ${CMAKE_CURRENT_SOURCE_DIR}/${CLASS_DIR}
+ARGS -d ${CMAKE_CURRENT_BINARY_DIR}/${CLASS_DIR}
+${CMAKE_CURRENT_SOURCE_DIR}/*.java
+)
+
+# build .jar file from .class files
+ADD_CUSTOM_COMMAND(
+OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${JAR_FILE}
+DEPENDS
+${CMAKE_CURRENT_BINARY_DIR}/${JAVA_FILES}.class
+COMMAND ${CMAKE_COMMAND}
+ARGS -E chdir ${CMAKE_CURRENT_BINARY_DIR}/${CLASS_DIR}
+#${CMAKE_Java_ARCHIVE} -cfv ${JAR_DIR}/${JAR_FILE} ${CLASS_DIR}/com/hpccsystems/ecljdbc
+${CMAKE_Java_ARCHIVE} -cfv ${CMAKE_CURRENT_BINARY_DIR}/${JAR_FILE} com/hpccsystems/ecljdbc
+)
+
+# the target
+ADD_CUSTOM_TARGET(
+${JAR_FILE}
+ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${JAR_FILE})

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/DFUFile.java

@@ -0,0 +1,771 @@
+package com.hpccsystems.ecljdbc;
+
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.StringTokenizer;
+
+public class DFUFile
+{
+	private String Prefix;
+	private String ClusterName;
+	private String Directory;
+	private String Description;
+	private int Parts;
+	private String FullyQualifiedName;
+	private String FileName;
+	private String Owner;
+	private long TotalSize;
+	private long RecordCount;
+	private String Modified;
+	private long LongSize;
+	private long LongRecordCount;
+	private boolean isSuperFile;
+	private boolean isZipFile;
+	private boolean isDirectory;
+	private int Replicate;
+	private int IntSize;
+	private int IntRecordCount;
+	private boolean FromRoxieCluster;
+	private boolean BrowseData;
+	private boolean IsKeyFile;
+	private Properties Fields;
+	private String Format;
+	private String CsvSeparate;
+	private String CsvTerminate;
+	private String CsvQuote;
+	private String Ecl;
+	private Properties KeyedColumns;
+	private Properties NonKeyedColumns;
+	private List<String> relatedIndexes;
+	private String IdxFilePosField;
+	private boolean HasPayLoad;
+
+	private final static String RELATEDINDEXKEYWORD = "XDBC:RelIndexes";
+	private final static String IDXFILEPOSFIELDTAG = "XDBC:PosField";
+
+	public DFUFile(String prefix, String clusterName, String filename) {
+		super();
+		Prefix = prefix;
+		ClusterName = clusterName;
+		FileName = filename;
+		FullyQualifiedName = filename;
+
+		Directory = null;
+		Description = null;
+		Parts = -1;
+		Owner = null;
+		TotalSize = -1;
+		RecordCount = -1;
+		Modified = null;
+		LongSize = -1;
+		LongRecordCount = -1;
+		isSuperFile = false;
+		isZipFile = false;
+		isDirectory = false;
+		Replicate = -1;
+		IntSize = -1;
+		IntRecordCount = -1;
+		FromRoxieCluster = false;
+		BrowseData = false;
+		IsKeyFile = false;
+		Format = "FLAT";
+		CsvSeparate = null;
+		CsvTerminate = null;
+		CsvQuote = null;
+		Ecl = null;
+		Fields = new Properties();
+		KeyedColumns = new Properties();
+		NonKeyedColumns = new Properties();
+		relatedIndexes = null;
+		IdxFilePosField = null;
+		HasPayLoad = false;
+	}
+
+	public DFUFile(String prefix, String clusterName, String directory,
+			String description, int parts, String filename, String fullyqualifiedname, String owner,
+			long totalSize, long recordCount, String modified, long longSize,
+			long longRecordCount, boolean isSuperFile, boolean isZipFile,
+			boolean isDirectory, int replicate, int intSize,
+			int intRecordCount, boolean fromRoxieCluster, boolean browseData,
+			boolean isKeyFile, String format, String csvseparate, String csvterminate, String csvquote, String ecl)
+	{
+		Prefix = prefix;
+		ClusterName = clusterName;
+		Directory = directory;
+		Description = description;
+		Parts = parts;
+		FileName = filename;
+		FullyQualifiedName = fullyqualifiedname;
+		Owner = owner;
+		TotalSize = totalSize;
+		RecordCount = recordCount;
+		Modified = modified;
+		LongSize = longSize;
+		LongRecordCount = longRecordCount;
+		this.isSuperFile = isSuperFile;
+		this.isZipFile = isZipFile;
+		this.isDirectory = isDirectory;
+		Replicate = replicate;
+		IntSize = intSize;
+		IntRecordCount = intRecordCount;
+		FromRoxieCluster = fromRoxieCluster;
+		BrowseData = browseData;
+		IsKeyFile = isKeyFile;
+		Fields = new Properties();
+		Format = format;
+		CsvSeparate = csvseparate;
+		CsvTerminate = csvterminate;
+		CsvQuote = csvquote;
+		KeyedColumns = new Properties();
+		NonKeyedColumns = new Properties();
+
+		if (ecl != null && ecl.length()>0)
+		{
+			Ecl = ecl;
+			setFileRecDef(ecl);
+		}
+		relatedIndexes = null;
+		IdxFilePosField = null;
+		HasPayLoad = false;
+	}
+
+	public DFUFile()
+	{
+		Prefix = null;
+		ClusterName = null;
+		Directory = null;
+		Description = null;
+		Parts = -1;
+		FullyQualifiedName = null;
+		FileName = null;
+		Owner = null;
+		TotalSize = -1;
+		RecordCount = -1;
+		Modified = null;
+		LongSize = -1;
+		LongRecordCount = -1;
+		this.isSuperFile = false;
+		this.isZipFile = false;
+		this.isDirectory = false;
+		Replicate = -1;
+		IntSize = -1;
+		IntRecordCount = -1;
+		FromRoxieCluster = false;
+		BrowseData = false;
+		IsKeyFile = false;
+		Fields = new Properties();
+		Format = "FLAT";
+		CsvSeparate = null;
+		CsvTerminate = null;
+		CsvQuote = null;
+		Ecl = null;
+		KeyedColumns = new Properties();
+		NonKeyedColumns = new Properties();
+		relatedIndexes = null;
+		IdxFilePosField = null;
+		HasPayLoad = false;
+	}
+
+	public String getFileName() {
+		return FileName;
+	}
+
+	public void setFileName(String fileName) {
+		FileName = fileName;
+	}
+	public String getPrefix() {
+		return Prefix;
+	}
+
+	public void setPrefix(String prefix) {
+		Prefix = prefix;
+	}
+
+	public String getClusterName() {
+		return ClusterName;
+	}
+
+	public void setClusterName(String clusterName) {
+		ClusterName = clusterName;
+	}
+
+	public String getDirectory() {
+		return Directory;
+	}
+
+	public void setDirectory(String directory) {
+		Directory = directory;
+	}
+
+	public String getDescription() {
+		return Description;
+	}
+
+	public void setDescription(String description)
+	{
+		Description = description;
+
+		if (description.contains(RELATEDINDEXKEYWORD))
+			setRelatedIndexes(description.substring(description.indexOf(RELATEDINDEXKEYWORD)));
+
+		if (description.contains(IDXFILEPOSFIELDTAG))
+			setIdxFilePosField(description.substring(description.indexOf(IDXFILEPOSFIELDTAG)));
+	}
+
+	private void setIdxFilePosField(String str)
+	{
+		IdxFilePosField = str.substring(IDXFILEPOSFIELDTAG.length()+1+1, str.indexOf(']'));
+	}
+
+	public String getIdxFilePosField()
+	{
+		return IdxFilePosField != null ? IdxFilePosField : getLastNonKeyedNumericField();
+	}
+
+	public boolean hasValidIdxFilePosField()
+	{
+		String tmp = getIdxFilePosField();
+		return tmp != null && tmp.length() > 0 ? true : false;
+	}
+
+	private String getLastNonKeyedNumericField()
+	{
+		//TODO get numeric field
+		return (String)NonKeyedColumns.get(NonKeyedColumns.size());
+	}
+
+	private void setRelatedIndexes(String str)
+	{
+		String indexes = str.substring(RELATEDINDEXKEYWORD.length()+1+1, str.indexOf(']'));
+		StringTokenizer indexeToks = new StringTokenizer(indexes,";");
+
+		while (indexeToks.hasMoreTokens())
+		{
+			addRelatedIndex(indexeToks.nextToken().trim());
+		}
+	}
+
+	public int getParts() {
+		return Parts;
+	}
+
+	public void setParts(int parts) {
+		Parts = parts;
+	}
+
+	public String getFullyQualifiedName() {
+		return FullyQualifiedName;
+	}
+
+	public void setFullyQualifiedName(String name) {
+		FullyQualifiedName = name;
+	}
+
+	public String getOwner() {
+		return Owner;
+	}
+
+	public void setOwner(String owner) {
+		Owner = owner;
+	}
+
+	public long getTotalSize() {
+		return TotalSize;
+	}
+
+	public void setTotalSize(long totalSize) {
+		TotalSize = totalSize;
+	}
+
+	public long getRecordCount() {
+		return RecordCount;
+	}
+
+	public void setRecordCount(long recordCount) {
+		RecordCount = recordCount;
+	}
+
+	public String getModified() {
+		return Modified;
+	}
+
+	public void setModified(String modified) {
+		Modified = modified;
+	}
+
+	public long getLongSize() {
+		return LongSize;
+	}
+
+	public void setLongSize(long longSize) {
+		LongSize = longSize;
+	}
+
+	public long getLongRecordCount() {
+		return LongRecordCount;
+	}
+
+	public void setLongRecordCount(long longRecordCount) {
+		LongRecordCount = longRecordCount;
+	}
+
+	public boolean isSuperFile() {
+		return isSuperFile;
+	}
+
+	public void setSuperFile(boolean isSuperFile) {
+		this.isSuperFile = isSuperFile;
+	}
+
+	public boolean isZipFile() {
+		return isZipFile;
+	}
+
+	public void setZipFile(boolean isZipFile) {
+		this.isZipFile = isZipFile;
+	}
+
+	public boolean isDirectory() {
+		return isDirectory;
+	}
+
+	public void setDirectory(boolean isDirectory) {
+		this.isDirectory = isDirectory;
+	}
+
+	public int getReplicate() {
+		return Replicate;
+	}
+
+	public void setReplicate(int replicate) {
+		Replicate = replicate;
+	}
+
+	public int getIntSize() {
+		return IntSize;
+	}
+
+	public void setIntSize(int intSize) {
+		IntSize = intSize;
+	}
+
+	public int getIntRecordCount() {
+		return IntRecordCount;
+	}
+
+	public void setIntRecordCount(int intRecordCount) {
+		IntRecordCount = intRecordCount;
+	}
+
+	public boolean isFromRoxieCluster() {
+		return FromRoxieCluster;
+	}
+
+	public void setFromRoxieCluster(boolean fromRoxieCluster) {
+		FromRoxieCluster = fromRoxieCluster;
+	}
+
+	public boolean isBrowseData() {
+		return BrowseData;
+	}
+
+	public void setBrowseData(boolean browseData) {
+		BrowseData = browseData;
+	}
+
+	public boolean isKeyFile() {
+		return IsKeyFile;
+	}
+
+	public void setIsKeyFile(boolean isKeyFile) {
+		IsKeyFile = isKeyFile;
+	}
+
+	public void setFileFields(String eclString)
+	{
+		//Ecl = "filerecstruct := RECORD ";
+		Ecl = "";
+		if (eclString != null && eclString.length()>0)
+		{
+			try
+			{
+				StringTokenizer comatokens = null;
+				//ECL RECORD can be defined as { type name,...,type name}; or RECORD type name;...;type name;END;
+				//TODO we should handle nested file types
+				if (eclString.toUpperCase().contains("RECORD"))
+				{
+					String tmp = eclString.substring(eclString.indexOf("RECORD")+6, eclString.indexOf("END"));
+					comatokens = new StringTokenizer(tmp, ";");
+
+				}
+				else if (eclString.contains("{"))
+				{
+					String tmp = eclString.substring(eclString.indexOf('{')+1, eclString.indexOf('}'));
+					comatokens = new StringTokenizer(tmp, ",");
+
+				}
+
+				if (comatokens != null)
+				{
+					int index = 0;
+					while (comatokens.hasMoreTokens())
+					{
+						StringTokenizer spacetokens = new StringTokenizer(comatokens.nextToken()," \n");
+						if(spacetokens.hasMoreTokens())
+						{
+							String type = spacetokens.nextToken();
+							String name = spacetokens.nextToken();
+							EclColumnMetaData columnmeta = new EclColumnMetaData(name, index, EclDatabaseMetaData.convertECLtype2SQLtype(type.toUpperCase()));
+							columnmeta.setEclType(type);
+							//columnmeta.setTableName(this.FileName);
+							columnmeta.setTableName(this.FullyQualifiedName);
+
+							Ecl += type + " " + name + "; ";
+							//Fields.add(columnmeta);
+							Fields.put(name.toUpperCase(), columnmeta);
+						}
+						index++;
+					}
+				}
+			}
+			catch (Exception e)
+			{
+				System.out.println("Invalid ECL Record definition found in " + this.getFullyQualifiedName() + " details.");
+				return;
+			}
+			/*finally
+			{
+				Ecl += "END; ";
+			}*/
+		}
+	}
+
+	public Enumeration<Object> getAllFields()
+	{
+		return Fields.elements();
+	}
+
+	public Properties getAllFieldsProps()
+	{
+		return Fields;
+	}
+	public boolean containsField(String fieldName)
+	{
+		return Fields.containsKey(fieldName.toUpperCase());
+	}
+
+	public EclColumnMetaData getFieldMetaData(String fieldName)
+	{
+		return (EclColumnMetaData) Fields.get(fieldName.toUpperCase());
+	}
+
+	public String[] getAllTableFieldsStringArray()
+	{
+		String [] fields = new String [Fields.size()];
+		Enumeration<Object> it = Fields.elements();
+		while(it.hasMoreElements())
+		{
+			EclColumnMetaData col = ((EclColumnMetaData)it.nextElement());
+			fields[col.getIndex()] = col.getColumnName();
+		}
+		return fields;
+	}
+
+
+	public String getFormat() {
+		return Format;
+	}
+
+	public String getCsvSeparate() {
+		return CsvSeparate;
+	}
+
+	public String getCsvTerminate() {
+		return CsvTerminate;
+	}
+
+	public void setFormat(String format)
+	{
+		if (format != null && format.length()>0)
+		{
+			if (format.equalsIgnoreCase("THOR") || format.equalsIgnoreCase("CSV") || format.equalsIgnoreCase("XML"))
+			{
+				Format = format;
+				return;
+			}
+		}
+		//TODO Default = FLAT... is this safe to assume???
+		Format = "FLAT";
+	}
+
+	public void setCsvSeparate(String csvSeparate)
+	{
+		CsvSeparate = csvSeparate;
+	}
+
+	public void setCsvTerminate(String csvTerminate)
+	{
+		CsvTerminate = csvTerminate;
+	}
+
+	public void setCsvQuote(String csvQuote)
+	{
+		CsvQuote = csvQuote;
+	}
+
+	public String getCsvQuote()
+	{
+		return CsvQuote;
+	}
+
+	public boolean hasFileRecDef()
+	{
+		return Ecl == null || Ecl.length() <= 0 ? false : true;
+	}
+
+	public String getFileRecDef(String structname)
+	{
+		return Ecl == null ? null : structname + " := RECORD " + Ecl + "END; ";
+	}
+
+	public void setFileRecDef(String ecl) {
+		if (ecl != null && ecl.length()>0)
+			setFileFields(ecl);
+	}
+
+	public EclColumnMetaData getCompatibleField(String keyName, String keyType) {
+		//EclColumnMetaData field = (EclColumnMetaData) Fields.get(keyName.toUpperCase());
+		//want to match up name and type, for now just check name.
+		return (EclColumnMetaData) Fields.get(keyName.toUpperCase());
+	}
+
+	public void  setKeyedColumns (Properties KeyFields)
+	{
+		KeyedColumns = KeyFields;
+	}
+
+	public void  setNonKeyedColumns (Properties NonKeyFields)
+	{
+		NonKeyedColumns = NonKeyFields;
+		if (NonKeyFields != null && NonKeyFields.size()>0)
+			HasPayLoad = true;
+	}
+
+	public void  addKeyedColumnInOrder(String KeyLabel)
+	{
+		KeyedColumns.put(KeyedColumns.size()+1, KeyLabel);
+	}
+
+	public void  addKeyedColumn (int KeyIndex, String KeyLabel)
+	{
+		if (!KeyedColumns.containsKey(KeyIndex))
+			KeyedColumns.put(KeyIndex, KeyLabel);
+	}
+
+	public void  addNonKeyedColumnInOrder(String KeyLabel)
+	{
+		if (KeyLabel.startsWith("__internal_fpos__"))
+		{
+			//IndexPositionField = KeyLabel;
+			return;
+		}
+		NonKeyedColumns.put(NonKeyedColumns.size()+1, KeyLabel);
+	}
+
+	public void  addNonKeyedColumn (int NonKeyIndex, String NonKeyLabel)
+	{
+		if (!NonKeyedColumns.containsKey(NonKeyIndex))
+		{
+			NonKeyedColumns.put(NonKeyIndex, NonKeyLabel);
+			HasPayLoad = true;
+		}
+	}
+
+	public boolean hasPayLoad()
+	{
+		return HasPayLoad;
+	}
+
+	public Properties getKeyedColumns()
+	{
+		return KeyedColumns;
+	}
+
+	public Properties getNonKeyedColumns()
+	{
+		return NonKeyedColumns;
+	}
+
+	public int getNonKeyColumnIndex(String ColumnName)
+	{
+		int cols = NonKeyedColumns.size();
+		for (int i = 1; i <= cols; i++)
+		{
+			if(NonKeyedColumns.get(i).equals(ColumnName))
+				return i;
+		}
+
+		return -1;
+	}
+
+	public int getKeyColumnIndex(String ColumnName)
+	{
+		int cols = KeyedColumns.size();
+		for (int i = 1; i <= cols; i++)
+		{
+			if(KeyedColumns.get(i).equals(ColumnName))
+				return i;
+		}
+		return -1;
+	}
+
+	public void addRelatedIndex(String IndexName)
+	{
+		if (relatedIndexes == null)
+			relatedIndexes = new ArrayList<String>();
+
+		relatedIndexes.add(IndexName);
+	}
+
+	public void setRelatedIndexes(List<String> indexes)
+	{
+		relatedIndexes = indexes;
+	}
+
+	public boolean isRelatedIndex(String indexname)
+	{
+		if (relatedIndexes != null)
+			return relatedIndexes.contains(indexname);
+		return false;
+	}
+
+	public Iterator<String> getRelatedIndexes()
+	{
+		return relatedIndexes == null ? null : relatedIndexes.iterator();
+	}
+
+	public String[] getRelatedIndexesAsArray()
+	{
+		String [] indexes = new String [relatedIndexes.size()];
+		for (int i = 0; i < indexes.length; i++)
+		{
+			indexes[i] = relatedIndexes.get(i);
+		}
+		return indexes;
+	}
+
+	public List<String> getRelatedIndexesList()
+	{
+		return relatedIndexes;
+	}
+
+	public int getRelatedIndexesCount()
+	{
+		return relatedIndexes == null ? 0 : relatedIndexes.size();
+	}
+
+	public Object getFileRecDefwithIndexpos(EclColumnMetaData fieldMetaData, String structname)
+	{
+		if(fieldMetaData != null)
+			//return "filerecstruct := RECORD " + Ecl + fieldMetaData.getEclType() + " " + fieldMetaData.getColumnName() + " {virtual(fileposition)}; END; ";
+			return structname + " := RECORD " + Ecl + fieldMetaData.getEclType() + " " + fieldMetaData.getColumnName() + " {virtual(fileposition)}; END; ";
+
+		return structname + " := RECORD " + Ecl +  " END; "; //might need to throw exception instead
+	}
+
+	public String getKeyedFieldsAsDelmitedString(char c, String prefixtoappend)
+	{
+		StringBuilder fields = new StringBuilder();
+		int colscount = KeyedColumns.size();
+
+		for (int i = 1; i <= colscount; i++)
+		{
+			if (prefixtoappend != null)
+				fields.append(prefixtoappend).append('.');
+			fields.append(KeyedColumns.get(i));
+			if (i < colscount)
+				fields.append(c).append(" ");
+		}
+		return fields.toString();
+	}
+
+	public String getNonKeyedFieldsAsDelmitedString(char c, String prefixtoappend)
+	{
+		StringBuilder fields = new StringBuilder();
+		int colscount = NonKeyedColumns.size();
+
+		for (int i = 1; i <= colscount; i++)
+		{
+			if (prefixtoappend != null)
+				fields.append(prefixtoappend).append('.');
+			fields.append(NonKeyedColumns.get(i));
+			if (i < colscount)
+				fields.append(c).append(" ");
+		}
+		return fields.toString();
+	}
+	public String getAllFieldsAsDelimitedString(char c)
+	{
+		String [] cols = getAllTableFieldsStringArray();
+		StringBuilder fields = new StringBuilder();
+		if (cols.length <= 0)
+		{
+			for (int index = 0; index < cols.length; index++ )
+			{
+				fields.append(cols[index]);
+
+				if (index < cols.length-1)
+					fields.append(c).append(" ");
+			}
+		}
+
+		return fields.toString();
+	}
+
+	public String getIndexNameByIndex(int i)
+	{
+		return relatedIndexes.get(i);
+	}
+
+	public boolean hasRelatedIndexes()
+	{
+		return relatedIndexes == null || relatedIndexes.size() <= 0 ? false : true;
+	}
+
+	public boolean containsField(EclColumnMetaData fieldMetaData, boolean verifyEclType)
+	{
+		String fieldName = fieldMetaData.getColumnName().toUpperCase();
+		if (Fields.containsKey(fieldName))
+			if (!verifyEclType || ((EclColumnMetaData)Fields.get(fieldName)).getEclType().equals(fieldMetaData.getEclType()))
+				return true;
+		return false;
+	}
+
+	public boolean containsAllFieldsNames(String[] columnNames)
+	{
+		if(columnNames != null)
+		{
+			for (int i = 0; i < columnNames.length; i++)
+			{
+				if(!this.containsField(columnNames[i]))
+					return false;
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	public int getNonKeyedColumnsCount()
+	{
+		return NonKeyedColumns == null ? 0 : NonKeyedColumns.size();
+	}
+
+	public boolean containsField(EclColumnMetaData column)
+	{
+		return this.containsField(column.getColumnName());
+	}
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclColumn.java

@@ -0,0 +1,26 @@
+package com.hpccsystems.ecljdbc;
+
+/**
+ * @author ChalaAX
+ */
+public class EclColumn
+{
+    private String name;
+    private String value;
+
+    public EclColumn(String name, String value)
+    {
+        this.name = name;
+        this.value = value;
+    }
+
+    public String getName()
+    {
+        return name;
+    }
+
+    public String getValue()
+    {
+        return value;
+    }
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclColumnMetaData.java

@@ -0,0 +1,263 @@
+package com.hpccsystems.ecljdbc;
+
+import java.util.List;
+
+/**
+ * @author rpastrana
+ */
+
+public class EclColumnMetaData {
+	private String columnName;
+	private int index;
+	private int sqlType;
+
+	private String eclType;
+	private String tableName;
+	private int columnSize;
+	private int decimalDigits;
+	private int radix;
+	private String nullable;
+	private String remarks;
+	private String columnDefault;
+	// private Properties restrictions;
+	private int paramType;
+	private String javaClassName;
+	private String constantValue;
+	private int columnType;
+	private String alias;
+	private List<EclColumnMetaData> funccols;
+
+
+	final static int COLUMN_TYPE_DATA 		= 1;
+	final static int COLUMN_TYPE_CONSTANT 	= 2;
+	final static int COLUMN_TYPE_FNCTION	= 3;
+
+	public EclColumnMetaData(String columnName, int index, int sqlType,	String constant, String eclType)
+	{
+		constantValue = constant;
+		this.columnName = columnName;
+		this.index = index;
+		this.sqlType = sqlType;
+		this.paramType = EclDatabaseMetaData.procedureColumnUnknown;
+		javaClassName = EclDatabaseMetaData.convertSQLtype2JavaClassName(this.sqlType);
+		this.eclType = eclType;
+		columnType = COLUMN_TYPE_CONSTANT;
+		funccols = null;
+		alias = null;
+	}
+
+	public EclColumnMetaData(String columnName, int index, int sqlType)
+	{
+		this.columnName = columnName;
+		this.index = index;
+		this.sqlType = sqlType;
+		this.paramType = EclDatabaseMetaData.procedureColumnUnknown;
+		javaClassName = EclDatabaseMetaData.convertSQLtype2JavaClassName(this.sqlType);
+		constantValue = null;
+		columnType = COLUMN_TYPE_DATA;
+		funccols = null;
+		alias = null;
+	}
+
+	public EclColumnMetaData(String columnName, int index, List<EclColumnMetaData> columns )
+	{
+		this.columnName = columnName;
+		this.index = index;
+		this.sqlType = java.sql.Types.OTHER;
+		this.paramType = EclDatabaseMetaData.procedureColumnUnknown;
+		javaClassName = EclDatabaseMetaData.convertSQLtype2JavaClassName(this.sqlType);
+		constantValue = null;
+		columnType = COLUMN_TYPE_FNCTION;
+		funccols = columns;
+		alias = null;
+	}
+
+	public void setConstantValue(String value)
+	{
+		constantValue = value;
+	}
+
+	public boolean isConstant()
+	{
+		return constantValue == null ? false : true;
+	}
+
+	public String getConstantValue()
+	{
+		return constantValue;
+	}
+
+	public int getSQLType()
+	{
+		return sqlType;
+	}
+
+	public void setSQLType(int type)
+	{
+		this.sqlType = type;
+	}
+
+	public String getColumnName()
+	{
+		return columnName;
+	}
+
+	public void setColumnName(String columnName)
+	{
+		this.columnName = columnName;
+	}
+
+	public int getIndex()
+	{
+		return index;
+	}
+
+	public void setIndex(int index)
+	{
+		this.index = index;
+	}
+
+	public int getParamType()
+	{
+		return paramType;
+	}
+
+	public void setParamType(int paramType)
+	{
+		this.paramType = paramType;
+	}
+
+	// public Properties getRestrictions() {
+	// return restrictions;
+	// }
+	//
+	// public void addRestriction(String restriction, String value)
+	// {
+	// if (restrictions == null)
+	// restrictions = new Properties();
+	//
+	// this.restrictions.put(restriction, value);
+	// }
+	//
+	// public String getRestrictionStringValue(String restriction)
+	// {
+	// return (String)this.restrictions.get(restriction);
+	// }
+
+	public int getSqlType()
+	{
+		return sqlType;
+	}
+
+	public String getEclType()
+	{
+		return eclType;
+	}
+
+	public void setEclType(String eclType)
+	{
+		this.eclType = eclType;
+		this.sqlType = EclDatabaseMetaData.convertECLtype2SQLtype(eclType.toUpperCase());
+	}
+
+	public String getTableName()
+	{
+		return tableName;
+	}
+
+	public void setTableName(String tableName)
+	{
+		this.tableName = tableName;
+	}
+
+	public int getColumnSize()
+	{
+		return columnSize;
+	}
+
+	public void setColumnSize(int columnSize)
+	{
+		this.columnSize = columnSize;
+	}
+
+	public int getDecimalDigits()
+	{
+		return decimalDigits;
+	}
+
+	public void setDecimalDigits(int decimalDigits)
+	{
+		this.decimalDigits = decimalDigits;
+	}
+
+	public int getRadix()
+	{
+		return radix;
+	}
+
+	public void setRadix(int radix)
+	{
+		this.radix = radix;
+	}
+
+	public String getNullable()
+	{
+		return nullable;
+	}
+
+	public void setNullable(String nullable)
+	{
+		this.nullable = nullable;
+	}
+
+	public String getRemarks()
+	{
+		return remarks;
+	}
+
+	public void setRemarks(String remarks)
+	{
+		this.remarks = remarks;
+	}
+
+	public String getColumnDefault()
+	{
+		return columnDefault;
+	}
+
+	public void setColumnDefault(String columnDefault)
+	{
+		this.columnDefault = columnDefault;
+	}
+
+	public String getJavaClassName()
+	{
+		return javaClassName;
+	}
+
+	public int getColumnType() {
+		return columnType;
+	}
+
+	public void setColumnType(int columnType) {
+		this.columnType = columnType;
+	}
+
+	public List<EclColumnMetaData> getFunccols() {
+		return funccols;
+	}
+
+	public String getAlias() {
+		return alias;
+	}
+
+	public void setAlias(String alias) {
+		this.alias = alias;
+	}
+
+	@Override
+	public String toString()
+	{
+		return "Name: " + this.columnName + " SQL Type: " + sqlType	+ " ECL Type: " + eclType;
+	}
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclConnection.java

@@ -0,0 +1,368 @@
+package com.hpccsystems.ecljdbc;
+
+import java.sql.Array;
+import java.sql.Blob;
+import java.sql.CallableStatement;
+import java.sql.Clob;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.NClob;
+import java.sql.PreparedStatement;
+import java.sql.SQLClientInfoException;
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.sql.SQLXML;
+import java.sql.Savepoint;
+import java.sql.Statement;
+import java.sql.Struct;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ *
+ * @author rpastrana
+ */
+
+public class EclConnection implements Connection {
+    private boolean closed;
+    private EclDatabaseMetaData metadata;
+    private Properties props;
+    private String serverAddress;
+    private String cluster;
+    private Properties clientInfo;
+
+    public EclConnection(Properties props)
+    {
+       closed = false;
+
+       this.serverAddress = props.getProperty("ServerAddress","localhost");
+       this.cluster = props.getProperty("Cluster","myroxie");
+       this.props = props;
+
+       if (!this.props.containsKey("WsECLWatchPort"))
+    	   this.props.setProperty("WsECLWatchPort", "8010");
+
+       if (!this.props.containsKey("WsECLPort"))
+    	   this.props.setProperty("WsECLPort", "8002");
+
+       //TODO soon wsecldirect will be part of wseclwatch
+       //I will continue to support this property but, by
+       //default we should use the wseclwatch port
+       if (!this.props.containsKey("WsECLDirectPort"))
+    	   this.props.setProperty("WsECLDirectPort", "8008");
+
+       if (!this.props.containsKey("username"))
+    	   this.props.setProperty("username", "");
+
+       if (!this.props.containsKey("password"))
+    	   this.props.setProperty("password", "");
+
+       if (!this.props.containsKey("EclLimit"))
+    	   this.props.setProperty("EclLimit", "100");
+
+       //basicAuth
+       String userPassword = this.props.getProperty("username") + ":" + props.getProperty("password");
+       //String basicAuth = "Basic " + new String(new Base64().encode(userPassword.getBytes()));
+
+       String basicAuth = "Basic " + Utils.Base64Encode(userPassword.getBytes(), false);
+
+       //System.out.println(Utils.Base64Decode(Utils.Base64Encode(userPassword.getBytes(), false).getBytes()));
+       this.props.put("BasicAuth", basicAuth);
+       metadata = new EclDatabaseMetaData(props);
+
+       //TODO not doing anything w/ this yet, just exposing it to comply w/ API definition...
+       clientInfo = new Properties();
+
+       System.out.println("EclConnection initialized - server: " + this.serverAddress + " cluster: " + this.cluster);
+
+    }
+
+    public Properties getProperties()
+    {
+    	return props;
+    }
+
+    public String getProperty(String propname)
+    {
+    	return props.getProperty(propname, "");
+    }
+    public String getCluster() {
+        return cluster;
+    }
+
+    public void setCluster(String cluster) {
+        this.cluster = cluster;
+    }
+
+    public String getServerAddress() {
+        return serverAddress;
+    }
+
+    public void setServerAddress(String serverAddress) {
+        this.serverAddress = serverAddress;
+    }
+
+    public EclDatabaseMetaData getDatabaseMetaData() {
+        return metadata;
+    }
+
+    public void setMetadata(EclDatabaseMetaData metadata) {
+        this.metadata = metadata;
+    }
+
+
+    public Statement createStatement() throws SQLException {
+    	System.out.println("##Statement EclConnection::createStatement()##");
+        return new EclPreparedStatement(this, null);
+    }
+
+
+    public PreparedStatement prepareStatement(String query) throws SQLException {
+    	System.out.println("##PreparedStatement EclConnection::createStatement("+ query +")##");
+        return new EclPreparedStatement(this, query);
+    }
+
+
+    public CallableStatement prepareCall(String sql) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: prepareCall(string sql) Not supported yet.");
+    }
+
+
+    public String nativeSQL(String sql) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: nativeSQL(string sql) Not supported yet.");
+    }
+
+
+    public void setAutoCommit(boolean autoCommit) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: setAutoCommit(boolean autoCommit) Not supported yet.");
+    }
+
+
+    public boolean getAutoCommit() throws SQLException {
+        return true;
+    }
+
+
+    public void commit() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: commit Not supported yet.");
+    }
+
+
+    public void rollback() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: rollback Not supported yet.");
+    }
+
+
+    public void close() throws SQLException {
+        closed = true;
+    }
+
+
+    public boolean isClosed() throws SQLException {
+        return closed;
+    }
+
+
+    public DatabaseMetaData getMetaData() throws SQLException {
+        return metadata;
+    }
+
+
+    public void setReadOnly(boolean readOnly) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: setReadOnly Not supported yet.");
+    }
+
+
+    public boolean isReadOnly() throws SQLException {
+        return true;
+    }
+
+
+    public void setCatalog(String catalog) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: setCatalog Not supported yet.");
+    }
+
+
+    public String getCatalog() throws SQLException
+    {
+    //    throw new UnsupportedOperationException("EclConnection: getCatalog Not supported yet.");
+    	return "myroxie";
+    }
+
+
+    public void setTransactionIsolation(int level) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: settransactionisolation Not supported yet.");
+    }
+
+
+    public int getTransactionIsolation() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: getTransactionIsolation Not supported yet.");
+    }
+
+
+    public SQLWarning getWarnings() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: getWarnings Not supported yet.");
+    }
+
+
+    public void clearWarnings() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: clearWarnings Not supported yet.");
+    }
+
+
+    public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException {
+    	System.out.println("##Statement EclConnection::createStatement(resulttype, resultsetcon)##");
+        return new EclPreparedStatement(this, null);
+    }
+
+
+    public PreparedStatement prepareStatement(String query, int resultSetType, int resultSetConcurrency) throws SQLException {
+    	System.out.println("##EclConnection::createStatement("+ query +", resultsetype, resultsetcon)##");
+        return new EclPreparedStatement(this, query);
+    }
+
+
+    public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: prepareCall(String sql, int resultSetType, int resultSetConcurrency) Not supported yet.");
+    }
+
+
+    public Map<String, Class<?>> getTypeMap() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: getTypeMap Not supported yet.");
+    }
+
+
+    public void setTypeMap(Map<String, Class<?>> map) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: setTypeMap Not supported yet.");
+    }
+
+
+    public void setHoldability(int holdability) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: setHoldability Not supported yet.");
+    }
+
+
+    public int getHoldability() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: getHoldability Not supported yet.");
+    }
+
+
+    public Savepoint setSavepoint() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: setSavepoint Not supported yet.");
+    }
+
+
+    public Savepoint setSavepoint(String name) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: setSavepoint Not supported yet.");
+    }
+
+
+    public void rollback(Savepoint savepoint) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: rollback Not supported yet.");
+    }
+
+
+    public void releaseSavepoint(Savepoint savepoint) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: releaseSavepoint Not supported yet.");
+    }
+
+
+    public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: createStatement Not supported yet.");
+    }
+
+
+    public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) Not supported yet.");
+    }
+
+
+    public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) Not supported yet.");
+    }
+
+
+    public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: prepareStatement(String sql, int autoGeneratedKeys) Not supported yet.");
+    }
+
+
+    public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: prepareStatement(String sql, int[] columnIndexes) Not supported yet.");
+    }
+
+
+    public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection:  prepareStatement(String sql, String[] columnNames) Not supported yet.");
+    }
+
+
+    public Clob createClob() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: createClob Not supported yet.");
+    }
+
+
+    public Blob createBlob() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: createBlob Not supported yet.");
+    }
+
+
+    public NClob createNClob() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: createNClob Not supported yet.");
+    }
+
+
+    public SQLXML createSQLXML() throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: createSQLXML Not supported yet.");
+    }
+
+
+    public boolean isValid(int timeout) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: isValid Not supported yet.");
+    }
+
+
+    public void setClientInfo(String name, String value) throws SQLClientInfoException {
+    	System.out.println("ECLCONNECTION SETCLIENTINFO");
+    	clientInfo.put(name, value);
+    }
+
+
+    public void setClientInfo(Properties properties) throws SQLClientInfoException {
+    	System.out.println("ECLCONNECTION SETCLIENTINFO");
+    	clientInfo = properties;
+    }
+
+
+    public String getClientInfo(String name) throws SQLException {
+    	System.out.println("ECLCONNECTION GETCLIENTINFO");
+    	return (String)clientInfo.getProperty(name);
+    }
+
+
+    public Properties getClientInfo() throws SQLException {
+    	System.out.println("ECLCONNECTION GETCLIENTINFO");
+    	return clientInfo;
+    }
+
+
+    public Array createArrayOf(String typeName, Object[] elements) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: createArrayOf(String typeName, Object[] elements) Not supported yet.");
+    }
+
+
+    public Struct createStruct(String typeName, Object[] attributes) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: createStruct(String typeName, Object[] attributes)Not supported yet.");
+    }
+
+
+    public <T> T unwrap(Class<T> iface) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: unwrap(Class<T> iface) Not supported yet.");
+    }
+
+
+    public boolean isWrapperFor(Class<?> iface) throws SQLException {
+        throw new UnsupportedOperationException("EclConnection: isWrapperFor(Class<?> iface) sNot supported yet.");
+    }
+
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclDriver.java

@@ -0,0 +1,318 @@
+package com.hpccsystems.ecljdbc;
+
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.DriverManager;
+import java.sql.DriverPropertyInfo;
+import java.sql.PreparedStatement;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.Properties;
+import java.util.StringTokenizer;
+
+public class EclDriver implements Driver
+{
+	static
+	{
+		try
+		{
+			EclDriver driver = new EclDriver();
+			DriverManager.registerDriver(driver);
+			System.out.println("EclDriver initialized");
+		}
+		catch (SQLException ex)
+		{
+			ex.printStackTrace();
+		}
+	}
+
+	static final int ECLJDBCMajorVersion = 0;
+	static final int ECLJDBCMinorVersion = 1;
+	static final int ECLJDBCPatchVersion = 0;
+
+	public EclDriver()
+	{
+	}
+
+
+	public Connection connect(String url, Properties info) throws SQLException
+	{
+		String serverAddress = info.getProperty("ServerAddress");
+		String cluster = info.getProperty("Cluster");
+
+		try
+		{
+			StringTokenizer urltokens = new StringTokenizer(url,";");
+			while (urltokens.hasMoreTokens())
+			{
+				String token = urltokens.nextToken();
+				if (token.contains("="))
+				{
+					StringTokenizer keyvalues = new StringTokenizer(token, "=");
+					while (keyvalues.hasMoreTokens())
+					{
+						String key = keyvalues.nextToken();
+						String value = keyvalues.nextToken();
+						if (!info.containsKey(key))
+							info.put(key, value);
+					}
+				}
+			}
+		}
+		catch (Exception e)
+		{
+			System.out.println("Issue parsing URL! \"" + url +"\"" );
+		}
+
+
+		System.out.println("EclDriver::connect" + serverAddress + ":" + cluster);
+		return new EclConnection(info);
+	}
+
+
+	public boolean acceptsURL(String url) throws SQLException {
+		return true;
+	}
+
+
+	public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException
+	{
+		DriverPropertyInfo[] infoArray = new DriverPropertyInfo[1];
+		infoArray[0] = new DriverPropertyInfo("ip", "IP Address");
+		return infoArray;
+	}
+
+
+	public int getMajorVersion()
+	{
+		System.out.println("ECLDRIVER GETMAJORVERSION");
+		return ECLJDBCMajorVersion;
+	}
+
+
+	public int getMinorVersion() {
+		System.out.println("ECLDRIVER GETMINORVERSION");
+		return ECLJDBCMinorVersion;
+	}
+
+
+	public boolean jdbcCompliant() {
+		return true;
+	}
+
+	public static void main(String[] args) {
+		EclDriver d = new EclDriver();
+		EclConnection conn;
+		try
+		{
+			Properties info = new Properties();
+			info.put("ServerAddress", "192.168.124.128"); //Mine
+			//info.put("ServerAddress", "10.239.20.80"); //clo
+			//info.put("ServerAddress", "10.239.219.10"); //fishbeck
+			//info.put("ServerAddress", "172.25.237.145"); //arjuna
+
+			info.put("Cluster", "thor"); //analagous to DB server instance
+			info.put("DefaultQuery", "fetchpeoplebyzipservice"); //analagous to default DB name
+			info.put("WsECLWatchPort", "8010");
+			//info.put("EclLimit", "100");
+			info.put("WsECLPort", "8002");
+			info.put("WsECLDirectPort", "8008");
+			info.put("username", "_rpastrana");
+			info.put("password", "ch@ng3m3");
+
+			//conn = (EclConnection) d.connect("url:jdbc:ecl;ServerAddress=192.168.124.128;Cluster=myroxie",info);
+			//conn = (EclConnection) d.connect("url:jdbc:ecl;ServerAddress=10.239.20.80;Cluster=thor;EclLimit=8",info);
+			conn = (EclConnection) d.connect("url:jdbc:ecl;ServerAddress=10.239.219.10;Cluster=thor;EclLimit=8",info);
+
+
+			//PreparedStatement p = conn.prepareStatement("SELECT 2");
+			//PreparedStatement p = conn.prepareStatement("select  * from thor::full_test_distributed_index where lname = BRYANT , fname = whoknows group by zips order by birth_month DESC");
+			//PreparedStatement p = conn.prepareStatement("select  * from thor::full_test_distributed_index where lname = BRYANT AND fname = SILVA ");
+
+			//PreparedStatement p = conn.prepareStatement("select  x, 'firstname', 1  from tutorial::rp::tutorialperson where  firstname >= ? and firstname >= ? limit 1000");
+			//PreparedStatement p = conn.prepareStatement("select  count(*) from tutorial::rp::tutorialperson where  firstname = 'A' or firstname = 'Z' limit 1000");
+			//PreparedStatement p = conn.prepareStatement("select  firstname from tutorial::rp::tutorialperson where  middlename =' ' ");
+			//PreparedStatement p = conn.prepareStatement("select  * from tutorial::rp::tutorialperson where zip = ? and  middlename>='W'    city='DELRAY BEACH' limit 1000" );
+			//PreparedStatement p = conn.prepareStatement("select * from tutorial::rp::tutorialperson where firstname = 'MARICHELLE'  order by lastname ASC, firstname DESC limit 1000");
+			//PreparedStatement p = conn.prepareStatement("select count(*) from tutorial::rp::tutorialperson");
+			//PreparedStatement p = conn.prepareStatement("select tutorial::rp::peoplebyzipindex.zip from \n tutorial::rp::peoplebyzipindex order by zip");
+			//PreparedStatement p = conn.prepareStatement("select zip, count( * ) from tutorial::rp::tutorialperson  group by zip");
+			//PreparedStatement p = conn.prepareStatement("select city, zip, count(*) from tutorial::rp::tutorialperson where zip ='33445' limit 1000");
+			//PreparedStatement p = conn.prepareStatement("select city from tutorial::rp::tutorialperson USE INDEX(tutorial::rp::peoplebyzipindex2) where zip = ? ");
+			//PreparedStatement p = conn.prepareStatement("select count(*) from tutorial::rp::tutorialperson USE INDEX(0) where zip > ?");
+			//PreparedStatement p = conn.prepareStatement("select count(city)  from tutorial::rp::tutorialperson where zip = '33445'");//where zip = '33445'");
+			//PreparedStatement p = conn.prepareStatement("select * from enron::final where tos = 'randy.young@enron.com' limit 1000");
+			PreparedStatement p = conn.prepareStatement("select count(*), zip from tutorial::rp::tutorialperson ");
+
+
+
+			//PreparedStatement p = conn.prepareStatement("select tbl.* from progguide::exampledata::peopleaccts tbl");
+			//PreparedStatement p = conn.prepareStatement("select firstname, lastname, middlename, city, street, state, zip from tutorial::rp::tutorialperson where firstname = VIMA LIMIT 1000");
+		//PreparedStatement p = conn.prepareStatement("select tbl.* from progguide::exampledata::people tbl");
+
+			//PreparedStatement p = conn.prepareStatement("select * from certification::full_test_distributed limit 100");
+			//PreparedStatement p = conn.prepareStatement("select * from certification::full_test_distributed where birth_state = FL LIMIT 1000");
+			//PreparedStatement p = conn.prepareStatement("select * from customer::customer");
+			//PreparedStatement p = conn.prepareStatement("select count(*) from tutorial::rp::tutorialperson");
+			//PreparedStatement p = conn.prepareStatement("select * from tutorial::rp::tutorialperson");
+
+
+			//PreparedStatement p = conn.prepareStatement("select tbl.* from .::xdbcsample tbl");
+
+			//PreparedStatement p = conn.prepareStatement("select tbl.* from fetchpeoplebyzipservice tbl where zipvalue=33445  order by fname DESC group by zip limit 10");
+			//PreparedStatement p = conn.prepareStatement("select tbl.* from fetchpeoplebyzipservice tbl where zipvalue=33445 group by zip, fname order by fname DESC, lname, zip ASC limit 10");
+			//PreparedStatement p = conn.prepareStatement("call fetchpeoplebyzipservice(?)");
+			//PreparedStatement p = conn.prepareStatement("select tbl.* from bestdemo tbl ");
+			//PreparedStatement p = conn.prepareStatement("select * fname from fetchpeoplebyzipservice");
+
+
+			//PreparedStatement p = conn.prepareStatement("select * from 'Result 1' where zipvalue=33445");
+
+			//PreparedStatement p = conn.prepareStatement("select * from Timeline_Total_Property_Sales");
+			//PreparedStatement p = conn.prepareStatement("select * from motiondemo");
+
+			p.clearParameters();
+			p.setObject(1, "'33445'");
+			//p.setObject(1, "'A'");
+			//p.setObject(2, "'D'");
+			EclResultSet qrs = (EclResultSet)((EclPreparedStatement)p).executeQuery();
+
+			ResultSetMetaData meta = qrs.getMetaData();
+			System.out.println();
+
+			for (int i = 1; i <= meta.getColumnCount(); i++)
+			{
+				System.out.print("[*****" + meta.getColumnName(i) + "******]");
+			}
+			System.out.println();
+			for (int i = 1; i <= meta.getColumnCount(); i++)
+			{
+				System.out.print("[*****" + EclDatabaseMetaData.convertSQLtype2JavaClassName(meta.getColumnType(i)) + "******]");
+			}
+
+			while (qrs.next())
+			{
+				System.out.println();
+				for (int i = 1; i <= meta.getColumnCount(); i++)
+				{
+					System.out.print("[ " + qrs.getObject(i) + " ]");
+				}
+			}
+
+			System.out.println("\nTotal Records found: " + qrs.getRowCount());
+
+//			p.clearParameters();
+//			//p.setObject(1, "'33445'");
+//			p.setObject(1, "'ROBERT'");
+//			p.setObject(2, "'LO'");
+//			qrs = (EclResultSet)((EclPreparedStatement)p).executeQuery();
+//
+//
+//			for (int i = 1; i <= meta.getColumnCount(); i++)
+//			{
+//				System.out.print("[*****" + meta.getColumnName(i) + "******]");
+//			}
+//
+//			while (qrs.next())
+//			{
+//				System.out.println();
+//				for (int i = 1; i <= meta.getColumnCount(); i++)
+//				{
+//					System.out.print("[ " + qrs.getObject(i) + " ]");
+//				}
+//			}
+//
+//			System.out.println("\nTotal Records found: " + qrs.getRowCount());
+
+			//String tabname = meta.getTableName(1);
+			//int count = meta.getColumnCount();
+			//String label = meta.getColumnLabel(2);
+			//String typename = meta.getColumnTypeName(1);
+
+			//System.out.println("Table name: " + tabname + "tablecolumncount: " + count);
+			/* while (qrs.next())
+			 {
+				 String fnvalue = qrs.getString("firstname");
+				 String lnvalue = qrs.getString("lastname");
+				 //String fnvalue = qrs.getString(1);
+				 //String lnvalue = qrs.getString(2);
+				 System.out.println("name: " + fnvalue + " " + lnvalue);
+			 }*/
+
+			//String hpccname = conn.getMetaData().getDatabaseProductVersion();
+			//int major = conn.getMetaData().getDatabaseMajorVersion();
+			//int minor = conn.getMetaData().getDatabaseMinorVersion();
+
+			//System.out.println("Connected to : " + hpccname +" version:" + major +"."+ minor);
+			//ECLDATABASEMETADATA GETCOLUMNS catalog: myroxie, schemaPattern: null, tableNameP
+			//attern: tutorial::rp::tutorialperson, columnNamePattern: null
+			// ResultSet columns = conn.getMetaData().getColumns("myroxie", null,"tutorial::rp::tutorialperson", null);
+
+			/*ResultSet typeinfo = conn.getMetaData().getTypeInfo();
+			 while (typeinfo.next())
+			 {
+				 System.out.print("col 1: " + typeinfo.getObject(1) + "\t");
+				 System.out.print("col 2: " + typeinfo.getObject(2) + "\t");
+				 System.out.print("col 3: " + typeinfo.getObject(3) + "\t");
+				 System.out.print("col 4: " + typeinfo.getObject(4) + "\t");
+				 System.out.print("col 5: " + typeinfo.getObject(5) + "\t");
+				 System.out.print("col 6: " + typeinfo.getObject(6) + "\t");
+				 System.out.print("col 7: " + typeinfo.getObject(7) + "\t");
+				 System.out.print("col 8: " + typeinfo.getObject(8) + "\t");
+
+					System.out.println("   " + typeinfo.getString("TYPE_NAME")+"("+typeinfo.getString("LOCAL_TYPE_NAME") + ")");
+			}*/
+
+			/*ResultSet columns = conn.getMetaData().getColumns(null, "",	"", "%");
+
+			 while (columns.next())
+			 {
+				 columns.getMetaData().getColumnType(1);
+				 columns.getMetaData().getColumnTypeName(1);
+				 System.out.print("col 1: " + columns.getObject(1) + "\t");
+				 System.out.print("col 2: " + columns.getObject(2) + "\t");
+				 System.out.print("col 3: " + columns.getObject(3) + "\t");
+				 System.out.print("col 4: " + columns.getObject(4) + "\t");
+				 System.out.print("col 5: " + columns.getObject(5) + "\t");
+				 System.out.print("col 6: " + columns.getObject(6) + "\t");
+				 System.out.print("col 7: " + columns.getObject(7) + "\t");
+				 System.out.print("col 8: " + columns.getObject(8) + "\t");
+
+					System.out.println("   " + columns.getString("TABLE_NAME")+"."+columns.getString("COLUMN_NAME")	+ "(" + columns.getString("TYPE_CAT") +")");
+			}
+			*/
+
+			//ECLDATABASEMETADATA GETTABLES catalog: myroxie, schemaPattern: null, tableNamePattern: %
+			//ResultSet tables = conn.getMetaData().getTables(null,null,null,	new String[] {""});
+			//ResultSet tables = conn.getMetaData().getTables("myroxie", null, "%",new String[] {""} );
+
+			//System.out.println("Tables found: ");
+			//while (tables.next()) {
+			//	System.out.println("   " + tables.getString("TABLE_NAME") + " Remarks: \'" + tables.getString("REMARKS")+"\'");
+		//	}
+
+
+			/*
+			ResultSet procs = conn.getMetaData().getProcedures(null, null, null);
+
+			System.out.println("procs found: ");
+			while (procs.next()) {
+				System.out.println("   " + procs.getString("PROCEDURE_NAME"));
+			}
+			*/
+			/*
+			ResultSet proccols = conn.getMetaData().getProcedureColumns(null, null, null, null);
+
+			System.out.println("procs cols found: ");
+			while (proccols.next()) {
+				System.out.println("   " + proccols.getString("COLUMN_NAME"));
+			}
+			*/
+			//getProcedureColumns catalog: null, schemaPattern: null, procedureNamePattern: fetchpeoplebyzipservice columnanmepat: null
+
+		}
+		catch (SQLException e) {
+			e.printStackTrace();
+		}
+	}
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclEngine.java

@@ -0,0 +1,932 @@
+package com.hpccsystems.ecljdbc;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.net.UnknownHostException;
+import java.sql.SQLWarning;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Vector;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class EclEngine
+{
+
+	private  		String		urlString;
+	private final 	String 		basicAuth;
+	private 	 	String 		eclqueryname;
+	private final 	String 		datasetname;
+	private 		NodeList 	resultschema;
+	private final 	Properties 	props;
+	private 		ArrayList<SQLWarning> warnings;
+	private 		SqlParser 	parser;
+	private			EclDatabaseMetaData dbMetadata;
+	private 		String 		indexToUseName;
+	private 		HashMap<String, String> 	eclEnteties;
+
+	public EclEngine(String eclqueryname, String datasetname, Properties props)
+	{
+		this.props = props;
+		this.datasetname = datasetname;
+		this.eclqueryname = eclqueryname;
+		basicAuth = props.getProperty("BasicAuth");
+		resultschema = null;
+		eclEnteties = new HashMap<String, String>();
+	}
+
+	public EclEngine(SqlParser parser, EclDatabaseMetaData dbmetadata, Properties props, String indextouse)
+	{
+		this.props = props;
+		this.dbMetadata = dbmetadata;
+		this.parser = parser;
+		basicAuth = props.getProperty("BasicAuth");
+		this.indexToUseName = indextouse;
+		datasetname = null;
+		resultschema = null;
+		eclqueryname = null;
+		eclEnteties = new HashMap<String, String>();
+	}
+
+	public ArrayList executeSelectConstant(String eclstring)
+	{
+		try
+		{
+			urlString = "http://" + props.getProperty("ServerAddress") + ":" + props.getProperty("WsECLDirectPort") + "/EclDirect/RunEcl?Submit&eclText=";
+			urlString +=  URLEncoder.encode(eclstring, "UTF-8");
+
+			System.out.println("WSECL:executeSelect: " + urlString);
+
+			// Send data
+			long startTime = System.currentTimeMillis();
+
+			URL url = new URL(urlString);
+			HttpURLConnection conn = (HttpURLConnection)url.openConnection();
+			conn.setInstanceFollowRedirects(false);
+			conn.setRequestProperty("Authorization", basicAuth);
+			conn.setRequestMethod("GET");
+			conn.setDoOutput(true);
+			conn.setDoInput(true);
+
+			return parse(conn.getInputStream(),startTime);
+		}
+		catch (Exception e)
+		{
+			throw new RuntimeException(e);
+		}
+	}
+
+	public  ArrayList execute() throws Exception
+	{
+		int sqlreqtype = parser.getSqlType();
+
+		List<EclColumnMetaData> expectedretcolumns = new ArrayList();
+		ArrayList<EclColumnMetaData> storeProcInParams = new ArrayList();
+		String hpccfilename = "";
+		String indexposfield = null;
+		StringBuilder keyedandwild = new StringBuilder();
+		DFUFile indexfiletouse = null;
+		StringBuilder eclcode = new StringBuilder("");
+		int totalparamcount = 0;
+		boolean isPayloadIndex = false;
+
+		switch (sqlreqtype)
+		{
+			case SqlParser.SQL_TYPE_SELECT:
+			{
+				hpccfilename = Utils.handleQuotedString(parser.getTableName());
+				if(!dbMetadata.tableExists("", hpccfilename))
+					throw new Exception("Invalid table found: " + hpccfilename);
+
+				DFUFile dfufile = dbMetadata.getDFUFile(hpccfilename);
+
+				//in future this might need to be a container of dfufile(s)
+				parser.verifySelectColumns(dfufile);
+
+				expectedretcolumns = parser.getSelectColumns();
+				totalparamcount = parser.getWhereClauseExpressionsCount();
+
+				if (indexToUseName != null)
+				{
+					indexfiletouse = dbMetadata.getDFUFile(indexToUseName);
+					indexposfield = indexfiletouse.getIdxFilePosField();
+
+					isPayloadIndex = processIndex(indexfiletouse, keyedandwild);
+
+					eclEnteties.put("KEYEDWILD", keyedandwild.toString());
+					if (isPayloadIndex)
+						eclEnteties.put("PAYLOADINDEX", "true");
+				}
+
+				eclEnteties.put("PARAMCOUNT", Integer.toString(totalparamcount));
+
+				if(dfufile.hasFileRecDef())
+				{
+					if (indexfiletouse != null && indexposfield != null)
+						eclcode.append(dfufile.getFileRecDefwithIndexpos(indexfiletouse.getFieldMetaData(indexposfield), "filerecstruct"));
+					else
+						eclcode.append(dfufile.getFileRecDef("filerecstruct"));
+				}
+				else
+					throw new Exception("Target HPCC file ("+hpccfilename+") does not contain ECL record definition");
+
+				if(!dfufile.isKeyFile())
+					eclcode.append("fileds := DATASET(\'~").append(dfufile.getFullyQualifiedName()).append("\', filerecstruct,").append(dfufile.getFormat()).append("); ");
+				else
+				{
+					eclcode.append("fileds := INDEX( ");
+					eclcode.append('{');
+					eclcode.append(dfufile.getKeyedFieldsAsDelmitedString(',', "filerecstruct"));
+					eclcode.append("},{");
+					eclcode.append(dfufile.getNonKeyedFieldsAsDelmitedString(',', "filerecstruct"));
+					eclcode.append("},");
+					eclcode.append("\'~").append(dfufile.getFullyQualifiedName()).append("\');");
+				}
+
+				StringBuilder selectstruct = new StringBuilder(" selectstruct:=RECORD ");
+				String datasource = indexToUseName == null || isPayloadIndex ? "fileds" : "fetchedds";
+
+				//boolean usescalar = expectedretcolumns.size() == 1;
+				for (int i = 0; i < expectedretcolumns.size(); i++)
+				{
+					EclColumnMetaData col = expectedretcolumns.get(i);
+					if (col.getColumnType() == EclColumnMetaData.COLUMN_TYPE_CONSTANT)
+					{
+						selectstruct.append(col.getEclType()).append(" ").append(col.getColumnName()).append(" := ").append(col.getConstantValue()).append("; ");
+						if (i == 0 && expectedretcolumns.size() == 1)
+							eclEnteties.put("SCALAROUTNAME", col.getColumnName());
+					}
+
+					else if (col.getColumnType() == EclColumnMetaData.COLUMN_TYPE_FNCTION)
+					{
+						if (col.getColumnName().equalsIgnoreCase("COUNT"))
+						{
+							eclEnteties.put("COUNTFN", "TRUE");
+							selectstruct.append("countout := ");
+							if (parser.hasGroupByColumns())
+							{
+								selectstruct.append(col.getColumnName().toUpperCase()).append("( GROUP");
+								List<EclColumnMetaData> funccols = col.getFunccols();
+
+								if (funccols.size() > 0)
+								{
+									String paramname = funccols.get(0).getColumnName();
+									if (!paramname.equals("*") && funccols.get(0).getColumnType() != EclColumnMetaData.COLUMN_TYPE_CONSTANT)
+									{
+										selectstruct.append(", ");
+										selectstruct.append(datasource);
+										selectstruct.append(".");
+										selectstruct.append(paramname);
+										selectstruct.append("<> \'\'");
+										//if (eclEnteties.size() > 0)
+										//	addFilterClause(selectstruct, eclEnteties);
+									}
+								}
+								selectstruct.append(" );");
+							}
+							else
+							{
+								selectstruct.append(" scalarout;");
+								if (expectedretcolumns.size() == 1)
+									eclEnteties.put("SCALAROUTNAME", col.getColumnName());
+							}
+
+							col.setSQLType(java.sql.Types.NUMERIC);
+						}
+						else if (col.getColumnName().equalsIgnoreCase("MAX"))
+						{
+							eclEnteties.put("MAXFN", "TRUE");
+							selectstruct.append("maxout :=  ");
+
+							if (parser.hasGroupByColumns())
+							{
+								selectstruct.append("MAX( GROUP ");
+							}
+							else
+							{
+								selectstruct.append("MAX( ").append(datasource);
+								if (eclEnteties.size() > 0)
+									addFilterClause(selectstruct, eclEnteties);
+							}
+
+							List<EclColumnMetaData> funccols = col.getFunccols();
+							if (funccols.size() > 0)
+							{
+								String paramname = funccols.get(0).getColumnName();
+								eclEnteties.put("FNCOLS", paramname);
+								if (!paramname.equals("*") && funccols.get(0).getColumnType() != EclColumnMetaData.COLUMN_TYPE_CONSTANT)
+								{
+									selectstruct.append(", ");
+									selectstruct.append(datasource);
+									selectstruct.append(".");
+									selectstruct.append(paramname);
+								}
+							}
+							selectstruct.append(" );");
+
+							/*
+							if (parser.hasGroupByColumns())
+							{
+								selectstruct.append(col.getColumnName().toUpperCase()).append("( GROUP");
+								List<EclColumnMetaData> funccols = col.getFunccols();
+
+								if (funccols.size() > 0)
+								{
+									String paramname = funccols.get(0).getColumnName();
+									eclEnteties.put("FNCOLS", paramname);
+									if (!paramname.equals("*") && funccols.get(0).getColumnType() != EclColumnMetaData.COLUMN_TYPE_CONSTANT)
+									{
+										selectstruct.append(", ");
+										selectstruct.append(datasource);
+										selectstruct.append(".");
+										selectstruct.append(paramname);
+									}
+								}
+								selectstruct.append(" );");
+							}
+							else
+							{
+								selectstruct.append(" totalmax;");
+								//if (expectedretcolumns.size() == 1)
+								//	eclEnteties.put("SCALAROUTNAME", col.getColumnName());
+							}
+							*/
+							/*
+
+							selectstruct.append(col.getColumnName().toUpperCase()).append(" ( ");
+							if (parser.hasGroupByColumns())
+								selectstruct.append("GROUP");
+							else
+								selectstruct.append(datasource);
+
+							List<EclColumnMetaData> funccols = col.getFunccols();
+
+							if (funccols.size() > 0)
+							{
+								String paramname = funccols.get(0).getColumnName();
+								if (!paramname.equals("*") && funccols.get(0).getColumnType() != EclColumnMetaData.COLUMN_TYPE_CONSTANT)
+								{
+									selectstruct.append(", ");
+									selectstruct.append(datasource);
+									selectstruct.append(".");
+									selectstruct.append(paramname);
+								}
+							}
+							selectstruct.append(" );");*/
+						}
+					}
+					else
+						selectstruct.append(col.getEclType()).append(" ").append(col.getColumnName()).append(" := ").append(datasource).append(".").append(col.getColumnName()).append("; ");
+
+					//if (i == 0 && expectedretcolumns.size() == 1 &&  col.getColumnType() != EclColumnMetaData.COLUMN_TYPE_DATA )
+					//	eclEnteties.put("SCALAROUTNAME", col.getColumnName());
+				}
+				selectstruct.append("END; ");
+
+				eclEnteties.put("SELECTSTRUCT", selectstruct.toString());
+
+				if(parser.hasOrderByColumns())
+					eclEnteties.put("ORDERBY",parser.getOrderByString());
+				if (parser.hasGroupByColumns())
+					eclEnteties.put("GROUPBY",parser.getGroupByString());
+				if (parser.hasLimitBy())
+					eclEnteties.put("LIMIT",Integer.toString(parser.getLimit()));
+
+				return executeSelect(eclcode.toString(), eclEnteties, indexfiletouse);
+			}
+			case SqlParser.SQL_TYPE_SELECTCONST:
+			{
+				System.out.println("Processing test_query...");
+				//ArrayList<EclColumnMetaData> columns = new ArrayList();
+				eclcode.append("selectstruct:=RECORD ");
+				expectedretcolumns = parser.getSelectColumns();
+				//defaultEclQueryReturnDatasetName = "ConstECLQueryResult";
+				StringBuilder ecloutput = new StringBuilder(" OUTPUT(DATASET([{ ");
+				for (int i = 1;  i <= expectedretcolumns.size(); i++)
+				{
+					EclColumnMetaData col = expectedretcolumns.get(i-1);
+					eclcode.append(col.getEclType()).append(" ").append(col.getColumnName()).append("; ");
+					ecloutput.append(col.getConstantValue());
+					if (i < expectedretcolumns.size())
+						ecloutput.append(", ");
+				}
+				ecloutput.append("}],selectstruct), NAMED(\'");
+				//ecloutput.append(defaultEclQueryReturnDatasetName);
+				ecloutput.append("ConstECLQueryResult");
+				ecloutput.append("\'));");
+
+				eclcode.append(" END; ");
+				eclcode.append(ecloutput.toString());
+
+				return executeSelectConstant(eclcode.toString());
+			}
+			case SqlParser.SQL_TYPE_CALL:
+			{
+				eclqueryname = Utils.handleQuotedString(parser.getStoredProcName());
+				if(!dbMetadata.eclQueryExists("", eclqueryname))
+					throw new Exception("Invalid store procedure found");
+
+				return executeCall(null);
+			}
+			default:
+
+				break;
+		}
+
+		return null;
+	}
+
+	public boolean processIndex(DFUFile indexfiletouse, StringBuilder keyedandwild)
+	{
+		boolean isPayloadIndex = containsPayload(indexfiletouse.getAllFieldsProps(), parser.getSelectColumns().iterator());
+
+		Vector<String> keyed = new Vector<String>();
+		Vector<String> wild = new Vector<String>();
+
+		//Create keyed and wild string
+		Properties keyedcols = indexfiletouse.getKeyedColumns();
+		for (int i = 1; i <= keyedcols.size(); i++)
+		{
+			String keyedcolname = (String)keyedcols.get(i);
+			if(parser.whereClauseContainsKey(keyedcolname))
+				keyed.add(" " + parser.getExpressionFromName(keyedcolname).toString() + " ");
+			else if (keyed.isEmpty())
+				wild.add(" " + keyedcolname + " ");
+		}
+
+		if(isPayloadIndex)
+		{
+			if (keyed.size()>0)
+			{
+				keyedandwild.append("KEYED( ");
+				for (int i = 0 ; i < keyed.size(); i++)
+				{
+					keyedandwild.append(keyed.get(i));
+					if (i < keyed.size()-1)
+						keyedandwild.append(" AND ");
+				}
+				keyedandwild.append(" )");
+			}
+			if (wild.size()>0)
+			{
+				//TODO should I bother making sure there's a KEYED entry ?
+				for (int i = 0 ; i < wild.size(); i++)
+				{
+					keyedandwild.append(" and WILD( ");
+					keyedandwild.append(wild.get(i));
+					keyedandwild.append(" )");
+				}
+			}
+
+			keyedandwild.append(" and (").append(parser.getWhereClauseString()).append(" )");
+		}
+		else //non-payload just AND the keyed expressions
+		{
+			keyedandwild.append("( ");
+			keyedandwild.append(parser.getWhereClauseString());
+			keyedandwild.append(" )");
+		}
+
+		return isPayloadIndex;
+	}
+
+	private boolean containsPayload(Properties indexfields,	Iterator<EclColumnMetaData> selectcolsit)
+	{
+		while(selectcolsit.hasNext())
+		{
+			EclColumnMetaData currentselectcol = selectcolsit.next();
+			String colname = currentselectcol.getColumnName();
+			int type = currentselectcol.getColumnType();
+			if (type == EclColumnMetaData.COLUMN_TYPE_DATA && !indexfields.containsKey(colname.toUpperCase()))
+				return false;
+			else if (type == EclColumnMetaData.COLUMN_TYPE_FNCTION && !containsPayload(indexfields, currentselectcol.getFunccols().iterator()))
+				return false;
+		}
+		return true;
+	}
+
+	public ArrayList executeSelect(String eclcode, HashMap parameters, DFUFile indexfile)
+	{
+		int responseCode = -1;
+		try
+		{
+			urlString = "http://" + props.getProperty("ServerAddress") + ":" + props.getProperty("WsECLDirectPort") + "/EclDirect/RunEcl?Submit"; 			//&eclText=
+
+			StringBuilder sb = new StringBuilder("&eclText=");
+			sb.append(eclcode);
+
+			if (indexfile == null) //no indexfile read...
+			{
+				if (!parameters.containsKey("GROUPBY"))
+				{
+					if (parameters.containsKey("COUNTFN"))
+					{
+						sb.append("scalarout := COUNT(fileds");
+						if (parameters.size() > 0)
+							addFilterClause(sb, parameters);
+						sb.append(");");
+					}
+
+					if (parameters.containsKey("MAXFN"))
+					{
+						sb.append("scalarout := MAX(fileds");
+						if (parameters.size() > 0)
+							addFilterClause(sb, parameters);
+
+						sb.append(" , fileds.");
+						sb.append(parameters.get("FNCOLS"));
+						sb.append(");");
+					}
+				}
+
+				if (parameters.containsKey("SCALAROUTNAME"))
+				{
+					sb.append("OUTPUT(scalarout ,NAMED(\'");
+					sb.append(parameters.get("SCALAROUTNAME"));
+					sb.append("\'));");
+				}
+				else
+				{
+					sb.append(parameters.get("SELECTSTRUCT"));
+					sb.append("OUTPUT(CHOOSEN(");
+
+					if (parameters.containsKey("ORDERBY"))
+						sb.append("SORT( ");
+
+					sb.append("TABLE(fileds");
+
+					if (parameters.size() > 0)
+						addFilterClause(sb, parameters);
+
+					sb.append(", selectstruct");
+					if (parameters.containsKey("GROUPBY"))
+					{
+						sb.append(",");
+						sb.append(parameters.get("GROUPBY"));
+					}
+
+					sb.append(")");
+
+					if (parameters.containsKey("ORDERBY"))
+					{
+						sb.append(",");
+						sb.append(parameters.get("ORDERBY"));
+						sb.append(")");
+					}
+				}
+			}
+			else // use index
+			{
+				sb.append("IDX := INDEX(fileds, {")
+				.append(indexfile.getKeyedFieldsAsDelmitedString(',', null))
+				.append("}");
+
+				if(indexfile.getNonKeyedColumnsCount()>0)
+					sb.append(",{ ").append(indexfile.getNonKeyedFieldsAsDelmitedString(',', null)).append(" }");
+
+				sb.append(",\'~").append(indexfile.getFullyQualifiedName()).append("\');");
+
+				if( parameters.containsKey("PAYLOADINDEX"))
+				{
+					sb.append(" OUTPUT(CHOOSEN(");
+					if (parameters.containsKey("ORDERBY"))
+						sb.append("SORT( ");
+
+					sb.append("IDX(")
+					.append(parameters.get("KEYEDWILD"))
+					.append(")");
+					if (parameters.containsKey("ORDERBY"))
+					{
+						sb.append(",");
+						sb.append(parameters.get("ORDERBY"));
+						sb.append(")");
+					}
+				}
+				else
+				{
+					sb.append("fetchedds := FETCH(fileds, IDX( ");
+
+					sb.append(parameters.get("KEYEDWILD"));
+					sb.append("), RIGHT.");
+					sb.append(indexfile.getIdxFilePosField()).append(");");
+
+
+					if (!parameters.containsKey("GROUPBY"))
+					{
+						if (parameters.containsKey("COUNTFN"))
+							sb.append("scalarout := COUNT(fetchedds);");
+						if (parameters.containsKey("MAXFN"))
+							sb.append("scalarout := MAX(fetchedds, fileds.zip);");
+					}
+
+					if (parameters.containsKey("SCALAROUTNAME"))
+					{
+						sb.append("OUTPUT(scalarout ,NAMED(\'");
+						sb.append(parameters.get("SCALAROUTNAME"));
+						sb.append("\'));");
+					}
+					else
+					{
+
+					sb.append(parameters.get("SELECTSTRUCT"));
+
+					sb.append(" IDXTABLE := TABLE(fetchedds , selectstruct ");
+
+					if (parameters.containsKey("GROUPBY"))
+					{
+						sb.append(", ");
+						sb.append(parameters.get("GROUPBY"));
+					}
+
+					sb.append("); ");
+
+					sb.append(" OUTPUT(CHOOSEN(");
+					sb.append(" IDXTABLE ");
+					}
+
+					/*{
+						sb.append(" OUTPUT(CHOOSEN(");
+						if (parameters.containsKey("ORDERBY"))
+							sb.append("SORT( ");
+
+						sb.append("fetchedds");
+						if (parameters.size() > 0)
+							addFilterClause(sb, parameters);
+
+						if (parameters.containsKey("ORDERBY"))
+						{
+							sb.append(",");
+							sb.append(parameters.get("ORDERBY"));
+							sb.append(")");
+						}
+					}*/
+				}
+			}
+
+			if (!parameters.containsKey("SCALAROUTNAME"))
+			{
+				sb.append(",");
+				if (parameters.containsKey("LIMIT"))
+					sb.append(parameters.get("LIMIT"));
+				else
+					sb.append( props.getProperty("EclLimit"));
+				sb.append("),NAMED(\'ECLJDBCSelectOutput\'));");
+			}
+
+			System.out.println("WSECL:executeSelect: " + urlString + sb.toString());
+
+			// Send data
+			long startTime = System.currentTimeMillis();
+
+			URL url = new URL(urlString);
+			HttpURLConnection conn = (HttpURLConnection)url.openConnection();
+			conn.setInstanceFollowRedirects(false);
+			conn.setRequestProperty("Authorization", basicAuth);
+			conn.setRequestMethod("GET");
+			conn.setDoOutput(true);
+			conn.setDoInput(true);
+
+			OutputStreamWriter wr = new OutputStreamWriter(	conn.getOutputStream());
+			wr.write(sb.toString());
+			wr.flush();
+
+			responseCode = conn.getResponseCode();
+
+			return parse(conn.getInputStream(),startTime);
+		}
+		catch (Exception e)
+		{
+			if (responseCode != 200)
+			{
+				throw new RuntimeException("HTTP Connection Response code: " + responseCode + " verify access to WsECLDirect", e);
+			}
+			else
+				throw new RuntimeException(e);
+		}
+	}
+
+	private void addFilterClause(StringBuilder sb, HashMap parameters)
+	{
+		String whereclause = parser.getWhereClauseString();
+		if (whereclause != null && whereclause.length() > 0)
+		{
+			sb.append("( ");
+			sb.append(whereclause);
+			sb.append(" )");
+		}
+	}
+
+	public ArrayList executeCall(Map parameters)
+	{
+		try
+		{
+			urlString = "http://" + props.getProperty("ServerAddress") + ":" + props.getProperty("WsECLPort") + "/WsEcl/submit/query/" + props.getProperty("Cluster") + "/" + eclqueryname + "/expanded";
+			System.out.println("WSECL:executeCall: " + urlString);
+
+			// Construct data
+			StringBuilder sb = new StringBuilder();
+			sb.append(URLEncoder.encode("submit_type_=xml", "UTF-8"));
+			sb.append("&").append(URLEncoder.encode("S1=Submit", "UTF-8"));
+
+			ArrayList<EclColumnMetaData> storeProcInParams = dbMetadata.getStoredProcInColumns("",eclqueryname);
+			String[] procInParamValues = parser.getStoredProcInParamVals();
+
+			for (int i = 0; i < procInParamValues.length; i++)
+			{
+				String key = storeProcInParams.get(i).getColumnName();
+				{
+					sb.append("&").append(key).append("=").append(procInParamValues[i]);
+				}
+			}
+
+			//if (parameters.size() > 0)
+			//{
+			//	for (Object key : parameters.keySet())
+			//	{
+			//		Object value = parameters.get(key);
+			//		sb.append("&").append(key).append("=").append(value);
+			//	}
+			//}
+
+			long startTime = System.currentTimeMillis();
+			// Send data
+			URL url = new URL(urlString);
+			HttpURLConnection conn = (HttpURLConnection)url.openConnection();
+			conn.setInstanceFollowRedirects(false);
+			conn.setRequestProperty("Authorization", basicAuth);
+			conn.setRequestMethod("GET");
+			conn.setDoOutput(true);
+			conn.setDoInput(true);
+
+			OutputStreamWriter wr = new OutputStreamWriter(	conn.getOutputStream());
+			wr.write(sb.toString());
+			wr.flush();
+
+			System.out.println("WsEcl Execute: " + urlString + " : " + sb.toString());
+
+			// Get the response
+			return parse(conn.getInputStream(), startTime);
+		}
+		catch (Exception e)
+		{
+			throw new RuntimeException(e);
+		}
+	}
+
+	public ArrayList parse(InputStream xml, long startTime) throws Exception
+	{
+		ArrayList results = null;
+
+		try
+		{
+			DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+			DocumentBuilder db = dbf.newDocumentBuilder();
+
+			Document dom = db.parse(xml);
+
+			long elapsedTime = System.currentTimeMillis() - startTime;
+		    System.out.println("Total elapsed http request/response time in milliseconds: " + elapsedTime);
+
+
+			Element docElement = dom.getDocumentElement();
+
+			NodeList dsList = docElement.getElementsByTagName("Dataset");
+
+			if (dsList != null && dsList.getLength() > 0)
+			{ // The dataset element is encapsulated within a Result element
+
+				//need to fetch appropriate resulst dataset ie map eclqueryname to dataset name
+				ArrayList dsArray = new ArrayList();
+
+				results = dsArray;
+
+				for (int i = 0; i < dsList.getLength(); i++)
+				{
+					Element ds = (Element) dsList.item(i);
+					String currentdatsetname = ds.getAttribute("name");
+					if (datasetname == null || datasetname.length() == 0 || currentdatsetname.equalsIgnoreCase(datasetname))
+					{
+						NodeList rowList = ds.getElementsByTagName("Row");
+
+						if (rowList != null && rowList.getLength() > 0)
+						{
+
+							ArrayList rowArray = new ArrayList();
+
+							dsArray.add(rowArray);
+
+							for (int j = 0; j < rowList.getLength(); j++)
+							{
+								Element row = (Element) rowList.item(j);
+
+								NodeList columnList = row.getChildNodes();
+
+								ArrayList columnsArray = new ArrayList();
+								rowArray.add(columnsArray);
+
+								for (int k = 0; k < columnList.getLength(); k++)
+								{
+									columnsArray.add(new EclColumn(columnList.item(k).getNodeName(), columnList.item(k).getTextContent()));
+								}
+							}
+						}
+						break;
+					}
+
+				}
+			}
+			else if (docElement.getElementsByTagName("Exception").getLength()>0)
+			{
+				Exception xmlexception = new Exception();
+				NodeList exceptionlist = docElement.getElementsByTagName("Exception");
+				//for (int i = 0; i < exceptionlist.getLength(); i++)
+				//{
+				if (exceptionlist.getLength()>0)
+				{
+					Exception resexception = null;
+					NodeList currexceptionelements = exceptionlist.item(0).getChildNodes();
+
+					for (int j = 0; j < currexceptionelements.getLength(); j++)
+					{
+						Node exceptionelement = currexceptionelements.item(j);
+						if (exceptionelement.getNodeName().equals("Message"))
+						{
+							resexception = new Exception("ECLJDBC error in response: \'" + exceptionelement.getTextContent()+"\'");
+						}
+					}
+					if (dsList == null || dsList.getLength() <= 0)
+						throw resexception;
+				}
+			}
+			else
+			{
+				// The root element is itself the Dataset element
+				if (dsList.getLength() == 0)
+				{
+					ArrayList dsArray = new ArrayList();
+					results = dsArray;
+					NodeList rowList = docElement.getElementsByTagName("Row");
+
+					if (rowList != null && rowList.getLength() > 0)
+					{
+						ArrayList rowArray = new ArrayList();
+						dsArray.add(rowArray);
+
+						for (int j = 0; j < rowList.getLength(); j++)
+						{
+							Element row = (Element) rowList.item(j);
+							NodeList columnList = row.getChildNodes();
+
+							ArrayList columnsArray = new ArrayList();
+							rowArray.add(columnsArray);
+
+							for (int k = 0; k < columnList.getLength(); k++)
+							{
+								columnsArray.add(new EclColumn(columnList.item(k).getNodeName(), columnList.item(k).getTextContent()));
+							}
+						}
+					}
+				}
+			}
+
+			/*not being used right not
+			 *
+			 * setResultschema( docElement.getElementsByTagName("XmlSchema"));
+			 */
+
+			System.out.println("Parsing results...");
+			Iterator itr = results.iterator();
+			System.out.println("Results datsets found: " + results.size());
+			while (itr.hasNext())
+			{
+				ArrayList rows = (ArrayList) itr.next();
+				System.out.println("Results rows found: " + rows.size());
+				Iterator itr2 = rows.iterator();
+				while (itr2.hasNext())
+				{
+					ArrayList cols = (ArrayList) itr2.next();
+					Iterator itr3 = cols.iterator();
+					while (itr3.hasNext())
+					{
+						EclColumn element = (EclColumn) itr3.next();
+					}
+				}
+			}
+			System.out.println("Finished Parsing results.");
+		}
+		catch (Exception e)
+		{
+			throw new Exception("Invalid response received, verify serveraddress and cluster name and HPCC query/file name:\n" +e.getMessage() );
+		}
+
+		return results;
+	}
+
+	public String convertInputStreamToString(InputStream ists)	throws IOException
+	{
+		if (ists != null)
+		{
+			StringBuilder sb = new StringBuilder();
+			String line;
+
+			try
+			{
+				BufferedReader r1 = new BufferedReader(new InputStreamReader(ists, "UTF-8"));
+				while ((line = r1.readLine()) != null)
+				{
+					sb.append(line).append("\n");
+				}
+			}
+			finally
+			{
+				ists.close();
+			}
+			return sb.toString();
+		}
+		else
+		{
+			return "";
+		}
+	}
+
+	public boolean hasResultSchema()
+	{
+		return (this.resultschema != null && this.resultschema.getLength() > 0);
+	}
+
+	public void setResultschema(NodeList resultschema)
+	{
+		this.resultschema = resultschema;
+
+		if (this.resultschema != null && this.resultschema.getLength() > 0)
+		{
+			System.out.println("contains resultschema");
+		}
+	}
+
+	public NodeList getResultschema()
+	{
+		return resultschema;
+	}
+
+
+	public static void main(String[] args)
+	{
+		// WsEcl wsEcl = new WsEcl("localhost", "myroxie", "person_query.1");
+		//WsEcl wsEcl = new WsEcl("192.168.92.130", "myroxie",
+
+		Properties info = new Properties();
+		// info.put("ServerAddress",
+		// "192.168.92.130:8002/WsEcl/definitions/query");
+		//info.put("ServerAddress", "192.168.124.128");
+		info.put("ServerAddress", "10.239.20.80");
+
+		info.put("Cluster", "myroxie");
+		info.put("WsECLWatchPort", "8010");
+		info.put("WsECLPort", "8002");
+		info.put("username", "_rpastrana");
+		info.put("password", "ch@ng3m3");
+
+		EclEngine wsEcl = new EclEngine("fetchpeoplebyzipservice.2","" ,info);
+		// http://192.168.56.102:8002/
+		HashMap params = new HashMap();
+		params.put("zipvalue", "30005");
+		ArrayList dataset = wsEcl.executeCall(params);
+
+		Iterator itr = dataset.iterator();
+		while (itr.hasNext())
+		{
+			ArrayList rows = (ArrayList) itr.next();
+			Iterator itr2 = rows.iterator();
+			while (itr2.hasNext())
+			{
+				ArrayList cols = (ArrayList) itr2.next();
+				Iterator itr3 = cols.iterator();
+				while (itr3.hasNext())
+				{
+					EclColumn element = (EclColumn) itr3.next();
+					System.out.print(element.getName() + " : "	+ element.getValue());
+				}
+			}
+		}
+		System.out.println();
+	}
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclPreparedStatement.java

@@ -0,0 +1,580 @@
+package com.hpccsystems.ecljdbc;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.math.BigDecimal;
+import java.net.URL;
+import java.sql.Array;
+import java.sql.Blob;
+import java.sql.Clob;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.NClob;
+import java.sql.ParameterMetaData;
+import java.sql.PreparedStatement;
+import java.sql.Ref;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.RowId;
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.sql.SQLXML;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.HashMap;
+/**
+ *
+ * @author rpastrana
+ */
+public class EclPreparedStatement implements PreparedStatement {
+    private boolean closed = false;
+    private int maxRows = 100;
+    private String query;
+    private Connection connection;
+    private HashMap<Integer, Object> parameters = new HashMap<Integer, Object>();
+    private ArrayList<SQLWarning> warnings;
+    private String indexToUse;
+
+    private EclResultSet result;
+
+    public EclPreparedStatement(Connection connection, String query)
+    {
+    	System.out.println("#####ECLPreparedStatement::ECLPreparedStatement: " + query + "###");
+        this.query = query;
+        this.connection = connection;
+        warnings = new ArrayList<SQLWarning>();
+        indexToUse = null;
+    }
+
+    public boolean isIndexSet()
+    {
+    	return indexToUse == null ? false : true;
+    }
+
+    public void setIndexToUse(String indexfilename)
+    {
+    	indexToUse = indexfilename;
+    }
+
+    public String getIndexToUse()
+    {
+    	return indexToUse;
+    }
+
+
+    public ResultSet executeQuery() throws SQLException
+    {
+    	System.out.println("#####ECLPreparedStatement::executeQuery: " + query + "###");
+    	return new EclResultSet(this, query, parameters);
+    }
+
+
+    public int executeUpdate() throws SQLException
+    {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT: executeUpdate Not supported yet.");
+    }
+
+
+    public void setNull(int parameterIndex, int sqlType) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), sqlType);
+    }
+
+
+    public void setBoolean(int parameterIndex, boolean x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setByte(int parameterIndex, byte x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setShort(int parameterIndex, short x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setInt(int parameterIndex, int x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setLong(int parameterIndex, long x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setFloat(int parameterIndex, float x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setDouble(int parameterIndex, double x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setString(int parameterIndex, String x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setBytes(int parameterIndex, byte[] x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setDate(int parameterIndex, Date x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setTime(int parameterIndex, Time x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT: setAsciiStream Not supported yet.");
+    }
+
+
+    public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT: setUnicodeStream Not supported yet.");
+    }
+
+
+    public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT: setBinaryStream Not supported yet.");
+    }
+
+
+    public void clearParameters() throws SQLException {
+        parameters.clear();
+    }
+
+
+    public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException {
+    	System.out.println("ECLPrepStamt::setObject Parameter Index = " + parameterIndex + ", Object = " + x + ", targetSqlType = " + targetSqlType);
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setObject(int parameterIndex, Object x) throws SQLException {
+        System.out.println("ECLPrepStamt::setObject Parameter Index = " + parameterIndex + ", Object = " + x);
+        parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public boolean execute() throws SQLException
+    {
+    	result = new EclResultSet(this, query, parameters);
+    	System.out.println("ECLPreparedStatement: execute rowcount: " + result.getRowCount());
+    	return result.getRowCount() > 0 ? true : false;
+    }
+
+
+    public void addBatch() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT: addBatch Not supported yet.");
+    }
+
+
+    public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setCharacterStream Not supported yet.");
+    }
+
+
+    public void setRef(int parameterIndex, Ref x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setBlob(int parameterIndex, Blob x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setClob(int parameterIndex, Clob x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setArray(int parameterIndex, Array x) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public ResultSetMetaData getMetaData() throws SQLException
+    {
+    	System.out.println("ECLPreparedStatement: getMetaData");
+    	return result.getMetaData();
+    }
+
+
+    public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setDate Not supported yet.");
+    }
+
+
+    public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setTime Not supported yet.");
+    }
+
+
+    public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setTimestamp Not supported yet.");
+    }
+
+
+    public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setNull Not supported yet.");
+    }
+
+
+    public void setURL(int parameterIndex, URL x) throws SQLException
+    {
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public ParameterMetaData getParameterMetaData() throws SQLException
+    {
+    	System.out.println("ECLPREPSTATEMENT getParameterMetaData");
+    	throw new UnsupportedOperationException("ECLPREPSTATEMNT:getParameterMetaData Not supported yet.");
+    }
+
+
+    public void setRowId(int parameterIndex, RowId x) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setRowId Not supported yet.");
+    }
+
+
+    public void setNString(int parameterIndex, String value) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), value);
+    }
+
+
+    public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setNCharacterStream Not supported yet.");
+    }
+
+
+    public void setNClob(int parameterIndex, NClob value) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), value);
+    }
+
+
+    public void setClob(int parameterIndex, Reader reader, long length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setClob Not supported yet.");
+    }
+
+
+    public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setBlob Not supported yet.");
+    }
+
+
+    public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setNClob Not supported yet.");
+    }
+
+
+    public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException {
+    	parameters.put(new Integer(parameterIndex), xmlObject);
+    }
+
+
+    public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException {
+    	System.out.println("ECLPrepStamt::setObject Parameter Index = " + parameterIndex + ", Object = " + x + ", targetSqlType = " + targetSqlType + ", scaleOrLength = " + scaleOrLength );
+    	parameters.put(new Integer(parameterIndex), x);
+    }
+
+
+    public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setAsciiStream Not supported yet.");
+    }
+
+
+    public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setBinaryStream Not supported yet.");
+    }
+
+
+    public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setCharacterStream Not supported yet.");
+    }
+
+
+    public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setAsciiStream Not supported yet.");
+    }
+
+
+    public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setBinaryStream Not supported yet.");
+    }
+
+
+    public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setCharacterStream Not supported yet.");
+    }
+
+
+    public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setNCharacterStream Not supported yet.");
+    }
+
+
+    public void setClob(int parameterIndex, Reader reader) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setClob Not supported yet.");
+    }
+
+
+    public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setBlob Not supported yet.");
+    }
+
+
+    public void setNClob(int parameterIndex, Reader reader) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setNClob Not supported yet.");
+    }
+
+
+    public ResultSet executeQuery(String query) throws SQLException
+    {
+    	System.out.println("ECLPreparedStatement: executeQuery(" + query +")");
+    	result = new EclResultSet(this, query, parameters);
+    	System.out.println("ECLPreparedStatement: executeQuery returned " + result.getRowCount() + " rows, and " + result.getMetaData().getColumnCount() + " columns");
+    	System.out.println("results object id: " + (Object)result.hashCode());
+
+        return result;
+    }
+
+
+    public int executeUpdate(String sql) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:executeUpdate Not supported yet.");
+    }
+
+
+    public void close() throws SQLException
+    {
+        connection.close();
+        query = "";
+        result = null;
+        closed = true;
+    }
+
+
+    public int getMaxFieldSize() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT: getMaxFieldSizeNot supported yet.");
+    }
+
+
+    public void setMaxFieldSize(int max) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setMaxFieldSize Not supported yet.");
+    }
+
+
+    public int getMaxRows() throws SQLException {
+        return maxRows;
+    }
+
+
+    public void setMaxRows(int max) throws SQLException {
+        maxRows = max;
+    }
+
+
+    public void setEscapeProcessing(boolean enable) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setEscapeProcessing Not supported yet.");
+    }
+
+
+    public int getQueryTimeout() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:getQueryTimeout Not supported yet.");
+    }
+
+
+    public void setQueryTimeout(int seconds) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setQueryTimeout Not supported yet.");
+    }
+
+
+    public void cancel() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:cancel Not supported yet.");
+    }
+
+
+    public SQLWarning getWarnings() throws SQLException {
+        return (warnings.size()<=0) ? null : warnings.get(1);
+    }
+
+
+    public void clearWarnings() throws SQLException {
+        warnings.clear();
+    }
+
+
+    public void setCursorName(String name) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setCursorName Not supported yet.");
+    }
+
+
+    public boolean execute(String sql) throws SQLException {
+        //throw new UnsupportedOperationException("ECLPREPSTATEMNT:execute Not supported yet.");
+    	System.out.println("ECLPREPSTMT execute(" + sql +")");
+    	query = sql;
+    	result = new EclResultSet(this, query, parameters);
+    	System.out.println("ECLPreparedStatement: execute rowcount: " + result.getRowCount());
+    	return result.getRowCount() > 0 ? true : false;
+    }
+
+
+    public ResultSet getResultSet() throws SQLException {
+        return result;
+    }
+
+
+    public int getUpdateCount() throws SQLException {
+    	return 0;
+    }
+
+
+    public boolean getMoreResults() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:getMoreResults Not supported yet.");
+    }
+
+
+    public void setFetchDirection(int direction) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setFetchDirection Not supported yet.");
+    }
+
+
+    public int getFetchDirection() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:getFetchDirection Not supported yet.");
+    }
+
+
+    public void setFetchSize(int rows) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setFetchSize Not supported yet.");
+    }
+
+
+    public int getFetchSize() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:getFetchSize Not supported yet.");
+    }
+
+
+    public int getResultSetConcurrency() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:getResultSetConcurrency Not supported yet.");
+    }
+
+
+    public int getResultSetType() throws SQLException {
+        return ResultSet.TYPE_FORWARD_ONLY;
+    }
+
+
+    public void addBatch(String sql) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:addBatch Not supported yet.");
+    }
+
+
+    public void clearBatch() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:clearBatch Not supported yet.");
+    }
+
+
+    public int[] executeBatch() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:executeBatch Not supported yet.");
+    }
+
+
+    public Connection getConnection() throws SQLException {
+        return connection;
+    }
+
+
+    public boolean getMoreResults(int current) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:getMoreResults Not supported yet.");
+    }
+
+
+    public ResultSet getGeneratedKeys() throws SQLException {
+        return null;
+    }
+
+
+    public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:executeUpdate Not supported yet.");
+    }
+
+
+    public int executeUpdate(String sql, int[] columnIndexes) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:executeUpdate Not supported yet.");
+    }
+
+
+    public int executeUpdate(String sql, String[] columnNames) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:executeUpdate Not supported yet.");
+    }
+
+
+    public boolean execute(String sql, int autoGeneratedKeys) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:execute(String sql, int autoGeneratedKeys) Not supported yet.");
+    }
+
+
+    public boolean execute(String sql, int[] columnIndexes) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:execute(String sql, int[] columnIndexes) Not supported yet.");
+    }
+
+
+    public boolean execute(String sql, String[] columnNames) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:execute(String sql, String[] columnNames) Not supported yet.");
+    }
+
+
+    public int getResultSetHoldability() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:getResultSetHoldability Not supported yet.");
+    }
+
+
+    public boolean isClosed() throws SQLException {
+        return closed;
+    }
+
+
+    public void setPoolable(boolean poolable) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:setPoolable Not supported yet.");
+    }
+
+
+    public boolean isPoolable() throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:isPoolable Not supported yet.");
+    }
+
+
+    public <T> T unwrap(Class<T> iface) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:unwrap Not supported yet.");
+    }
+
+
+    public boolean isWrapperFor(Class<?> iface) throws SQLException {
+        throw new UnsupportedOperationException("ECLPREPSTATEMNT:isWrapperFor Not supported yet.");
+    }
+
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclQueries.java

@@ -0,0 +1,35 @@
+package com.hpccsystems.ecljdbc;
+
+import java.util.Enumeration;
+import java.util.Properties;
+
+public class EclQueries {
+
+	private Properties queries;
+	private String clusterName;
+
+	public EclQueries(String cluster)
+	{
+		clusterName = cluster;
+		queries = new Properties();
+	}
+
+	public void put(String name, EclQuery query)
+	{
+		queries.put(name, query);
+	}
+
+	public String getClusterName()
+	{
+		return clusterName;
+	}
+	public Enumeration<Object> getQueries()
+	{
+		return queries.elements();
+	}
+
+	public EclQuery getQuery(String eclqueryname)
+	{
+		return (EclQuery)queries.get(eclqueryname);
+	}
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclQuery.java

@@ -0,0 +1,253 @@
+package com.hpccsystems.ecljdbc;
+
+import java.sql.DatabaseMetaData;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class EclQuery
+{
+	private String ID;
+	private String Name;
+	private String WUID;
+	private String DLL;
+	private String Alias;
+
+	private boolean Suspended;
+	private List<String> ResultDatasets;
+	private List<EclColumnMetaData> schema;
+	private String defaulttable;
+	private List<EclColumnMetaData> defaultfields;
+
+	private final static String DEFAULTDATASETNAME = "BIOUTPUT";
+
+	public EclQuery()
+	{
+		ID = "";
+		Name = "";
+		WUID = "";
+		Suspended = false;
+		Alias = "";
+		ResultDatasets = new ArrayList<String>();
+		schema = new ArrayList<EclColumnMetaData>();
+		defaulttable = null;
+		defaultfields = new ArrayList<EclColumnMetaData>();;
+	}
+
+	public String getAlias() {
+		return Alias;
+	}
+
+	public void setAlias(String alias) {
+		Alias = alias;
+	}
+
+	public String [] getAllTableFieldsStringArray(String tablename)
+	{
+		//ArrayList<String> fields = new ArrayList<String>();
+
+		//Iterator<EclColumnMetaData> it = schema.iterator();
+		String fields [] = new String [defaultfields.size()];
+		int i = 0;
+		Iterator<EclColumnMetaData> it = defaultfields.iterator();
+		while (it.hasNext())
+		{
+			EclColumnMetaData field = (EclColumnMetaData)it.next();
+			//if (field.getTableName().equals(tablename))
+				//fields.add(field.getColumnName());
+				fields[i++] = field.getColumnName();
+		}
+
+
+	    /*String f [] = new String [fields.size()];
+	    for (String string : fields)
+	    {
+	    	f[i++] = string;
+		}*/
+
+		return fields;
+	}
+
+	public void addResultElement(EclColumnMetaData elem) throws Exception
+	{
+		if (defaulttable != null)
+		{
+			if (elem.getTableName().equalsIgnoreCase(defaulttable) || elem.getParamType() == DatabaseMetaData.procedureColumnIn)
+			{
+				defaultfields.add(elem);
+			}
+
+			schema.add(elem);
+		}
+		else
+			throw new Exception("Cannot add Result Elements before adding Result Dataset");
+
+	}
+
+	public void addResultDataset(String ds)
+	{
+		if (defaulttable == null || ds.equalsIgnoreCase(DEFAULTDATASETNAME))
+		{
+			defaulttable = ds;
+			defaultfields.clear();
+		}
+		ResultDatasets.add(ds);
+	}
+
+	/*Doesn't make since if we're mapping one ECL query to one DB table*/
+	public List<String> getAllTables()
+	{
+		return ResultDatasets;
+	}
+
+
+	public List<EclColumnMetaData> getAllFields()
+	{
+		return defaultfields;
+	}
+
+	public String getID()
+	{
+		return ID;
+	}
+
+	public void setID(String iD)
+	{
+		ID = iD;
+	}
+
+	public String getName() {
+		return Name;
+	}
+
+	public void setName(String name) {
+		Name = name;
+	}
+
+	public String getWUID() {
+		return WUID;
+	}
+
+	public void setWUID(String wUID) {
+		WUID = wUID;
+	}
+
+	public String getDLL() {
+		return DLL;
+	}
+
+	public void setDLL(String dLL) {
+		DLL = dLL;
+	}
+
+	public boolean isSuspended() {
+		return Suspended;
+	}
+
+	public void setSuspended(boolean suspended) {
+		Suspended = suspended;
+	}
+
+	@Override
+	public String toString()
+	{
+		String tmp = Name + " " + WUID;
+
+		tmp += "tables: {";
+		Iterator<String> iterator = ResultDatasets.iterator();
+		while (iterator.hasNext()) {
+			tmp += " " + iterator.next();
+		}
+
+		tmp += "}";
+
+		tmp += "elements: {";
+		Iterator<EclColumnMetaData> iterator2 = schema.iterator();
+		while (iterator2.hasNext()) {
+			tmp += " " + iterator2.next();
+		}
+
+		tmp += "}";
+
+		return tmp;
+	}
+
+	public boolean containsField(String tablename, String fieldname)
+	{
+		//Iterator<EclColumnMetaData> it = schema.iterator();
+		Iterator<EclColumnMetaData> it = defaultfields.iterator();
+		while (it.hasNext())
+		{
+			EclColumnMetaData field = (EclColumnMetaData)it.next();
+			if (field.getTableName().equalsIgnoreCase(tablename) && field.getColumnName().equalsIgnoreCase(fieldname))
+				return true;
+		}
+
+		return false;
+	}
+
+	public EclColumnMetaData getFieldMetaData(String fieldname)
+	{
+		Iterator<EclColumnMetaData> it = schema.iterator();
+		while (it.hasNext())
+		{
+			EclColumnMetaData field = (EclColumnMetaData)it.next();
+			if (field.getColumnName().equalsIgnoreCase(fieldname))
+				return field;
+		}
+
+		return null;
+	}
+
+	public Iterator<EclColumnMetaData> getColumnsMetaDataIterator()
+	{
+		return schema.iterator();
+	}
+
+	public String getDefaultTableName()
+	{
+		return defaulttable;
+	}
+
+	public boolean containsField(String fieldname) {
+		Iterator<EclColumnMetaData> it = defaultfields.iterator();
+		while (it.hasNext())
+		{
+			EclColumnMetaData field = (EclColumnMetaData)it.next();
+			if (field.getColumnName().equalsIgnoreCase(fieldname))
+				return true;
+		}
+
+		return false;
+	}
+
+	public ArrayList<EclColumnMetaData> getAllNonInFields()
+	{
+		ArrayList<EclColumnMetaData> expectedretcolumns = new ArrayList();
+
+		Iterator<EclColumnMetaData> it = defaultfields.iterator();
+		while (it.hasNext())
+		{
+			EclColumnMetaData field = (EclColumnMetaData)it.next();
+			if (field.getParamType() != EclDatabaseMetaData.procedureColumnIn)
+				expectedretcolumns.add(field);
+		}
+
+		return expectedretcolumns;
+	}
+
+	public ArrayList<EclColumnMetaData> getAllInFields()
+	{
+		ArrayList<EclColumnMetaData> inparams = new ArrayList();
+
+		Iterator<EclColumnMetaData> it = defaultfields.iterator();
+		while (it.hasNext())
+		{
+			EclColumnMetaData field = (EclColumnMetaData)it.next();
+			if (field.getParamType() == EclDatabaseMetaData.procedureColumnIn)
+				inparams.add(field);
+		}
+
+		return inparams;
+	}
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclResultSetMetadata.java

@@ -0,0 +1,214 @@
+package com.hpccsystems.ecljdbc;
+
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class EclResultSetMetadata implements ResultSetMetaData{
+
+    private List <EclColumnMetaData> columnList;
+    private String tableName;
+    private String schemaName 	= "HPCC";
+    private String catalogName 	= "roxie";
+    //private EclDatabaseMetaData dbmetadata;
+
+    public EclResultSetMetadata(List <EclColumnMetaData> columnList, String tableName)
+    {
+        this.columnList = columnList;
+        this.tableName = tableName;
+    }
+
+    @SuppressWarnings("rawtypes")
+	public ArrayList createDefaultResultRow()
+    {
+    	ArrayList rowValues = new ArrayList();
+
+    	for(int i = 0; i < columnList.size(); i++)
+    	{
+    		//some columns might not be nullable!
+    		//rowValues.add(i, new Object());
+    		if (columnList.get(i).isConstant())
+    			rowValues.add(i,columnList.get(i).getConstantValue());
+    		else
+    			rowValues.add(i, null);
+    	}
+    	return rowValues;
+    }
+
+
+    public int getColumnCount() throws SQLException {
+        return columnList.size();
+    }
+
+
+    public boolean isAutoIncrement(int column) throws SQLException {
+        return false;
+    }
+
+
+    public boolean isCaseSensitive(int column) throws SQLException {
+        return false;
+    }
+
+
+    public boolean isSearchable(int column) throws SQLException {
+        return false;
+    }
+
+
+    public boolean isCurrency(int column) throws SQLException {
+        return false;
+    }
+
+
+    public int isNullable(int column) throws SQLException {
+        return 1;
+    }
+
+
+    public boolean isSigned(int column) throws SQLException {
+        return false;
+    }
+
+
+    public int getColumnDisplaySize(int column) throws SQLException {
+        return 255;
+    }
+
+
+    public String getColumnLabel(int column) throws SQLException {
+        if (column >= 1 && column <= columnList.size())
+           return columnList.get(column-1).getColumnName();
+        else
+           throw new SQLException("Invalid Column Index = " + column);
+    }
+
+
+    public String getColumnName(int column) throws SQLException {
+        if (column >= 1 && column <= columnList.size())
+           return columnList.get(column-1).getColumnName();
+        else
+           throw new SQLException("Invalid Column Index = column");
+    }
+
+
+    public String getSchemaName(int column) throws SQLException {
+        return schemaName;
+    }
+
+
+    public int getPrecision(int column) throws SQLException {
+        return 0;
+    }
+
+
+    public int getScale(int column) throws SQLException {
+        return 0;
+    }
+
+
+    public String getTableName(int column) throws SQLException {
+        return tableName;
+    }
+
+
+    public String getCatalogName(int column) throws SQLException {
+        return catalogName;
+    }
+
+
+    public int getColumnType(int column) throws SQLException
+    {
+		if (column >= 1 && column <= columnList.size())
+			return columnList.get(column - 1).getSQLType();
+		else
+			throw new SQLException("Invalid Column Index = " + column);
+    }
+
+
+    public String getColumnTypeName(int column) throws SQLException
+    {
+		if (column >= 1 && column <= columnList.size())
+			return EclDatabaseMetaData.getFieldName(columnList.get(column - 1).getSQLType());
+		else
+			throw new SQLException("Invalid Column Index = " + column);
+    }
+
+
+    public boolean isReadOnly(int column) throws SQLException
+    {
+    	return true;
+    }
+
+
+    public boolean isWritable(int column) throws SQLException
+    {
+    	return false;
+    }
+
+
+    public boolean isDefinitelyWritable(int column) throws SQLException
+    {
+        return false;
+    }
+
+
+    public String getColumnClassName(int column) throws SQLException
+    {
+    	if (column >= 1 && column <= columnList.size())
+    	{
+    		return EclDatabaseMetaData.convertSQLtype2JavaClassName(columnList.get(column - 1).getSQLType());
+		}
+		else
+		{
+			throw new SQLException("Invalid Column Index = " + column);
+		}
+    }
+
+
+    public <T> T unwrap(Class<T> iface) throws SQLException {
+        throw new UnsupportedOperationException("Not supported yet.");
+    }
+
+
+    public boolean isWrapperFor(Class<?> iface) throws SQLException {
+        throw new UnsupportedOperationException("Not supported yet.");
+    }
+
+    public int getColumnIndex(String columnLabel)
+	{
+			int colindex = -1;
+			try
+			{
+				int columCount = this.getColumnCount();
+				for (int col = 1; col <= columCount; col++)
+				{
+					if (this.getColumnLabel(col).equalsIgnoreCase(columnLabel))
+					{
+						colindex = col;
+						break;
+					}
+				}
+			}
+			catch (SQLException e){}
+
+			return colindex;
+	}
+
+	public String[] getInParamNames()
+	{
+		String [] inparams = new String [this.columnList.size()];
+		for(int i =0; i<columnList.size(); i++)
+		{
+			inparams[i] = columnList.get(i).getColumnName();
+		}
+
+		return inparams;
+	}
+
+	/*public void parseResultSchema(Node schema)
+	{
+		EclDatabaseMetaData.registerSchemaElements(schema, resultElements, "");
+	}*/
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclStatement.java

@@ -0,0 +1,225 @@
+package com.hpccsystems.ecljdbc;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.sql.Statement;
+
+/**
+ *
+ * @author ChalaAX
+ */
+public class EclStatement implements Statement {
+
+    public ResultSet executeQuery(String sql) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: executeQuery(String sql) Not supported yet.");
+    }
+
+
+    public int executeUpdate(String sql) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: executeUpdate(String sql) Not supported yet.");
+    }
+
+
+    public void close() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: close() Not supported yet.");
+    }
+
+
+    public int getMaxFieldSize() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getMaxFieldSize() Not supported yet.");
+    }
+
+
+    public void setMaxFieldSize(int max) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: setMaxFieldSize(int max) Not supported yet.");
+    }
+
+
+    public int getMaxRows() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getMaxRows() Not supported yet.");
+    }
+
+
+    public void setMaxRows(int max) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: setMaxRows(int max) Not supported yet.");
+    }
+
+
+    public void setEscapeProcessing(boolean enable) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: setEscapeProcessing(boolean enable) Not supported yet.");
+    }
+
+
+    public int getQueryTimeout() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement:  getQueryTimeout() Not supported yet.");
+    }
+
+
+    public void setQueryTimeout(int seconds) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: setQueryTimeout(int seconds) Not supported yet.");
+    }
+
+
+    public void cancel() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: cancel()  Not supported yet.");
+    }
+
+
+    public SQLWarning getWarnings() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getWarnings Not supported yet.");
+    }
+
+
+    public void clearWarnings() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: clearWarnings() Not supported yet.");
+    }
+
+
+    public void setCursorName(String name) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement:  setCursorName(String name) Not supported yet.");
+    }
+
+
+    public boolean execute(String sql) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: execute(String sql) Not supported yet.");
+    }
+
+
+    public ResultSet getResultSet() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getResultSet Not supported yet.");
+    }
+
+
+    public int getUpdateCount() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement:getUpdateCount  Not supported yet.");
+    }
+
+
+    public boolean getMoreResults() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getMoreResults Not supported yet.");
+    }
+
+
+    public void setFetchDirection(int direction) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: only ResultSet.FETCH_FORWARD supported");
+    }
+
+
+    public int getFetchDirection() throws SQLException {
+        //throw new UnsupportedOperationException("EclStatement: getFetchDirection() Not supported yet.");
+    	return ResultSet.FETCH_FORWARD;
+    }
+
+
+    public void setFetchSize(int rows) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: setFetchSize(int rows) Not supported yet.");
+    }
+
+
+    public int getFetchSize() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getFetchSize() Not supported yet.");
+    }
+
+
+    public int getResultSetConcurrency() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getResultSetConcurrency() Not supported yet.");
+    }
+
+
+    public int getResultSetType() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement:  getResultSetType() Not supported yet.");
+    }
+
+
+    public void addBatch(String sql) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: addBatch(String sql) Not supported yet.");
+    }
+
+
+    public void clearBatch() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: clearBatch() Not supported yet.");
+    }
+
+
+    public int[] executeBatch() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: executeBatch() Not supported yet.");
+    }
+
+
+    public Connection getConnection() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getConnection() Not supported yet.");
+    }
+
+
+    public boolean getMoreResults(int current) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getMoreResults(int current) Not supported yet.");
+    }
+
+
+    public ResultSet getGeneratedKeys() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getGeneratedKeys() Not supported yet.");
+    }
+
+
+    public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: executeUpdate(String sql, int autoGeneratedKeys) Not supported yet.");
+    }
+
+
+    public int executeUpdate(String sql, int[] columnIndexes) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement:  executeUpdate(String sql, int[] columnIndexes) Not supported yet.");
+    }
+
+
+    public int executeUpdate(String sql, String[] columnNames) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: executeUpdate(String sql, String[] columnNames) Not supported yet.");
+    }
+
+
+    public boolean execute(String sql, int autoGeneratedKeys) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: execute(String sql, int autoGeneratedKeys) Not supported yet.");
+    }
+
+
+    public boolean execute(String sql, int[] columnIndexes) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement:  execute(String sql, int[] columnIndexes) Not supported yet.");
+    }
+
+
+    public boolean execute(String sql, String[] columnNames) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: execute(String sql, String[] columnNames) Not supported yet.");
+    }
+
+
+    public int getResultSetHoldability() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: getResultSetHoldability Not supported yet.");
+    }
+
+
+    public boolean isClosed() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: isClosed() Not supported yet.");
+    }
+
+
+    public void setPoolable(boolean poolable) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: Not supported yet.");
+    }
+
+
+    public boolean isPoolable() throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: Not supported yet.");
+    }
+
+
+    public <T> T unwrap(Class<T> iface) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: Not supported yet.");
+    }
+
+
+    public boolean isWrapperFor(Class<?> iface) throws SQLException {
+        throw new UnsupportedOperationException("EclStatement: Not supported yet.");
+    }
+
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/EclVersionTracker.java.in

@@ -0,0 +1,8 @@
+package org.hpccsystems.ecljdbc;
+
+public class EclVersionTracker 
+{
+	static final int ECLJDBCMajorVersion = ${majorver};
+	static final int ECLJDBCMinorVersion = ${minorver};
+	static final int ECLJDBCPatchVersion = ${point};
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/SqlColumn.java

@@ -0,0 +1,155 @@
+package com.hpccsystems.ecljdbc;
+
+import java.util.List;
+
+public class SqlColumn
+{
+	private String name;
+	private String alias;
+	private String parenttable;
+	private int type;
+	private List<SqlColumn> functionParams;
+	private String constVal;
+	private int position;
+
+
+	final static int UNKNOWN_COLUMN_TYPE = -1;
+	final static int CONST_COLUMN_TYPE = 1;
+	final static int FIELDNAME_COLUMN_TYPE = 2;
+	final static int FUNCTION_COLUMN_TYPE = 3;
+
+	public SqlColumn()
+	{
+		name = null;
+		alias = null;
+		parenttable = null;
+		type = UNKNOWN_COLUMN_TYPE;
+		functionParams = null;
+		constVal = null;
+		position = -1;
+	}
+
+	public SqlColumn(String fullname, int coltype, int position)
+	{
+		String [] nameparts = name.split("\\.");
+		if (nameparts.length > 1)
+		{
+			parenttable = nameparts[0];
+		}
+		else
+			parenttable = null;
+
+		this.name = nameparts[nameparts.length-1];
+		alias = null;
+		functionParams = null;
+		type = coltype;
+		constVal = null;
+		this.position = position;
+	}
+
+	public SqlColumn(String fullname, int position)
+	{
+
+		String [] nameparts = fullname.split("\\.");
+		if (nameparts.length > 1)
+		{
+			parenttable = nameparts[0];
+		}
+		else
+			parenttable = null;
+
+		this.name = nameparts[nameparts.length-1];
+		alias = null;
+		type = UNKNOWN_COLUMN_TYPE;
+		functionParams = null;
+		constVal = null;
+		this.position = position;
+	}
+
+	public SqlColumn(String parentname, String columnname, int position)
+	{
+		parenttable = parentname;
+		this.name = columnname;
+		alias = null;
+		type = FIELDNAME_COLUMN_TYPE;
+		functionParams = null;
+		constVal = null;
+		this.position = position;
+	}
+
+	public SqlColumn(String columnname, List<SqlColumn> funcols, int position)
+	{
+		this.name = columnname;
+		alias = null;
+		type = FUNCTION_COLUMN_TYPE;
+		functionParams = funcols;
+		constVal = null;
+		this.position = position;
+	}
+
+	public SqlColumn(String columnname, String constVal, String alias, int position)
+	{
+		this.name = columnname;
+		this.alias = alias;
+		type = CONST_COLUMN_TYPE;
+		functionParams = null;
+		this.constVal = constVal;
+		this.position = position;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public String getAlias() {
+		return alias;
+	}
+
+	public void setAlias(String alias) {
+		this.alias = alias;
+	}
+
+	public String getParenttable() {
+		return parenttable;
+	}
+
+	public void setParenttable(String parenttable) {
+		this.parenttable = parenttable;
+	}
+
+	public int getType() {
+		return type;
+	}
+
+	public void setType(int type) {
+		this.type = type;
+	}
+
+	public List<SqlColumn> getFunctionParams() {
+		return functionParams;
+	}
+
+	public void setFunctionParams(List<SqlColumn> functionParams) {
+		this.functionParams = functionParams;
+	}
+
+	public String getConstVal() {
+		return constVal;
+	}
+
+	public void setConstVal(String constVal) {
+		this.constVal = constVal;
+	}
+	public int getPosition() {
+		return position;
+	}
+
+	public void setPosition(int position) {
+		this.position = position;
+	}
+
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/SqlExpression.java

@@ -0,0 +1,90 @@
+package com.hpccsystems.ecljdbc;
+
+public class SqlExpression
+{
+	public static final int LOGICAL_OPERATOR_TYPE = 1;
+	public static final int LOGICAL_EXPRESSION_TYPE = 2;
+
+	private String name;
+	private SqlOperator operator;
+	private String value;
+	private int type;
+	private boolean isParametrized;
+
+	public SqlExpression(String name, String operator, String value)
+	{
+		this.name = name;
+		this.operator = new SqlOperator(operator);
+		this.value = value;
+		type = LOGICAL_EXPRESSION_TYPE;
+	}
+
+	public SqlExpression(String operator)
+	{
+		name = null;
+		this.operator = new SqlOperator(operator);
+		value = null;
+		type = LOGICAL_OPERATOR_TYPE;
+		isParametrized = false;
+	}
+
+	public SqlExpression(int type)
+	{
+		name = null;
+		operator = null;
+		value = null;
+		this.type = type;
+		isParametrized = false;
+	}
+
+	public int getType()
+	{
+		return type;
+	}
+
+	public String getName() {
+		return name;
+	}
+	public void setName(String name) {
+		this.name = name;
+	}
+	public SqlOperator getOperator() {
+		return operator;
+	}
+	public void setOperator(SqlOperator operator) {
+		this.operator = operator;
+	}
+
+	public void setOperator(String operator) {
+		this.operator = new SqlOperator(operator);
+	}
+
+	public String getValue() {
+		return value;
+	}
+	public void setValue(String value)
+	{
+		this.value = value;
+
+		if (value != null && value.length() > 0 && (value.contains("${") || value.equals("?")))
+			isParametrized = true;
+		else
+			isParametrized = false;
+	}
+
+	public boolean isParametrized(String param)
+	{
+		return isParametrized;
+	}
+
+	public boolean isParametrized()
+	{
+		return isParametrized;
+	}
+
+	@Override
+	public String toString()
+	{
+		return (type == LOGICAL_EXPRESSION_TYPE ? name + " "  + operator.toString() + " " + value : " " + operator.toString() + " ");
+	}
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/SqlOperator.java

@@ -0,0 +1,68 @@
+package com.hpccsystems.ecljdbc;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SqlOperator
+{
+	private static List<String> validOps;
+	public static final String eq = new String("=");
+	public static final String neq = new String("<>");
+	public static final String isNull = new String("IS NULL");
+	public static final String isNotNull = new String("IS NOT NULL");
+	public static final String gt = new String(">");
+	public static final String lt = new String("<");
+	public static final String gte = new String(">=");
+	public static final String lte = new String("<=");
+	public static final String and = new String("AND");
+	public static final String or = new String("OR");
+	public static final String not = new String("NOT");
+	public static final String exists = new String("EXISTS");
+	public static final String like = new String("LIKE");
+	public static final String in = new String("IN");
+
+	private final String value;
+
+	public String getValue()
+	{
+		return value;
+	}
+
+	public boolean isValid()
+	{
+		return value == null ? false : true;
+	}
+
+	public SqlOperator(String operator)
+	{
+		if (validOps == null)
+		{
+			validOps = new ArrayList<String>();
+			validOps.add(eq);
+			validOps.add(neq);
+			validOps.add(isNull);
+			validOps.add(isNotNull);
+			validOps.add(gt);
+			validOps.add(lt);
+			validOps.add(gte);
+			validOps.add(lte);
+			validOps.add(and);
+			validOps.add(or);
+			validOps.add(not);
+			validOps.add(exists);
+			validOps.add(like);
+			validOps.add(in);
+		}
+
+		if( validOps.contains(operator.toUpperCase()))
+				value = operator.toUpperCase();
+		else
+				value = null;
+	}
+
+	@Override
+	public String toString()
+	{
+		return value == null ? "" : value;
+	}
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/SqlWhereClause.java

@@ -0,0 +1,129 @@
+package com.hpccsystems.ecljdbc;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class SqlWhereClause
+{
+	private List<SqlExpression> expressions;
+	private List<String> expressionUniqueColumnNames;
+	private int expressionsCount;
+	private int operatorsCount;
+	private boolean orOperatorUsed;
+
+	public SqlWhereClause()
+	{
+		expressions = new ArrayList<SqlExpression>();
+		expressionUniqueColumnNames = new ArrayList<String>();
+		expressionsCount = 0;
+		operatorsCount = 0;
+		orOperatorUsed = false;
+	}
+
+	public void addExpression(SqlExpression expression)
+	{
+		expressions.add(expression);
+		if (expression.getType() == SqlExpression.LOGICAL_EXPRESSION_TYPE)
+		{
+			expressionsCount++;
+			if (!expressionUniqueColumnNames.contains(expression.getName()))
+					expressionUniqueColumnNames.add(expression.getName());
+		}
+		else
+		{
+			operatorsCount++;
+			if (expression.getOperator().getValue().equals(SqlOperator.or))
+				orOperatorUsed = true;
+		}
+	}
+
+	public Iterator<SqlExpression> getExpressions()
+	{
+		return expressions.iterator();
+	}
+
+	public int getExpressionsCount() {
+		return expressionsCount;
+	}
+
+	public int getOperatorsCount() {
+		return operatorsCount;
+	}
+
+	@Override
+	public String toString()
+	{
+		String clause = new String("");
+		Iterator<SqlExpression> it = expressions.iterator();
+		while (it.hasNext())
+		{
+			clause += ((SqlExpression)it.next()).toString();
+		}
+		return clause;
+	}
+
+	public String [] getExpressionNames()
+	{
+		String [] names = new String[getExpressionsCount()];
+		Iterator<SqlExpression> it = expressions.iterator();
+		int i = 0;
+		while (it.hasNext())
+		{
+			SqlExpression exp = it.next();
+			if (exp.getType() == SqlExpression.LOGICAL_EXPRESSION_TYPE)
+			{
+				names[i++] = exp.getName();
+			}
+		}
+		return names;
+	}
+
+	public SqlExpression getExpressionFromName(String name)
+	{
+		Iterator<SqlExpression> it = expressions.iterator();
+		while (it.hasNext())
+		{
+			SqlExpression exp = it.next();
+			if (exp.getType() == SqlExpression.LOGICAL_EXPRESSION_TYPE && exp.getName().equals(name))
+			{
+				return exp;
+			}
+		}
+		return null;
+	}
+
+	public boolean containsKey(String name)
+	{
+		Iterator<SqlExpression> it = expressions.iterator();
+		while (it.hasNext())
+		{
+			SqlExpression exp = it.next();
+			if (exp.getType() == SqlExpression.LOGICAL_EXPRESSION_TYPE && exp.getName().equals(name))
+				return true;
+		}
+		return false;
+	}
+
+	public int getUniqueColumnCount()
+	{
+		return expressionUniqueColumnNames.size();
+	}
+
+	public String[] getUniqueExpressionNames()
+	{
+		String [] names = new String[getUniqueColumnCount()];
+		Iterator<String> it = expressionUniqueColumnNames.iterator();
+		int i = 0;
+		while (it.hasNext())
+		{
+			names[i++] = it.next();
+		}
+		return names;
+	}
+
+	public boolean isOrOperatorUsed()
+	{
+		return orOperatorUsed;
+	}
+}

plugins/dbconnectors/ecljdbc/src/main/java/com/hpccsystems/ecljdbc/Utils.java

@@ -0,0 +1,198 @@
+package com.hpccsystems.ecljdbc;
+
+import java.text.NumberFormat;
+import java.text.ParseException;
+import java.util.Locale;
+
+public class Utils
+{
+	static final char pad = '=';
+	static final char BASE64_enc[] =  {'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z',
+            						  'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',
+			                          '0','1','2','3','4','5','6','7','8','9','+','"'};
+
+	static final char BASE64_dec[] =
+	{
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x3e, (char)0x00, (char)0x00, (char)0x00, (char)0x3f,
+		(char)0x34, (char)0x35, (char)0x36, (char)0x37, (char)0x38, (char)0x39, (char)0x3a, (char)0x3b, (char)0x3c, (char)0x3d, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x01, (char)0x02, (char)0x03, (char)0x04, (char)0x05, (char)0x06, (char)0x07, (char)0x08, (char)0x09, (char)0x0a, (char)0x0b, (char)0x0c, (char)0x0d, (char)0x0e,
+		(char)0x0f, (char)0x10, (char)0x11, (char)0x12, (char)0x13, (char)0x14, (char)0x15, (char)0x16, (char)0x17, (char)0x18, (char)0x19, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x1a, (char)0x1b, (char)0x1c, (char)0x1d, (char)0x1e, (char)0x1f, (char)0x20, (char)0x21, (char)0x22, (char)0x23, (char)0x24, (char)0x25, (char)0x26, (char)0x27, (char)0x28,
+		(char)0x29, (char)0x2a, (char)0x2b, (char)0x2c, (char)0x2d, (char)0x2e, (char)0x2f, (char)0x30, (char)0x31, (char)0x32, (char)0x33, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00,
+		(char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00, (char)0x00
+	};
+
+//	public static String Base64Decode(byte [] input)
+//	{
+//		StringBuilder out = new StringBuilder("");
+//
+//		char c1, c2, c3, c4;
+//	    char d1, d2, d3, d4;
+//
+//	    for(int i = 0;;)
+//	    {
+//	        c1 = (char)input[i++];
+//			if (c1 == 0)
+//				break;
+//			//else if (!isspace(c1))
+//			else if (!String.valueOf(c1).equals(" "))
+//			{
+//				c2 = (char)input[i++];
+//				c3 = (char)input[i++];
+//				c4 = (char)input[i++];
+//				d1 = BASE64_dec[c1];
+//				d2 = BASE64_dec[c2];
+//				d3 = BASE64_dec[c3];
+//				d4 = BASE64_dec[c4];
+//
+//				out.append((char)((d1 << 2) | (d2 >> 4)));
+//
+//				if(c3 == pad)
+//					break;
+//
+//				out.append((char)((d2 << 4) | (d3 >> 2)));
+//
+//				if(c4 == pad)
+//					break;
+//
+//				out.append((char)((d3 << 6) | d4));
+//			}
+//	    }
+//
+//	    return out.toString();
+//	}
+
+	public static String Base64Encode(byte [] input, boolean addLineBreaks)
+	{
+		int length = input.length;
+		StringBuilder out = new StringBuilder("");
+		char one;
+		char two;
+		char three;
+
+		int i;
+		for (i = 0; i < length &&  length -i >= 3;)
+		{
+			one = (char)input[i++];
+			two = (char)input[i++];
+			three = (char)input[i++];
+
+			out.append((char)BASE64_enc[one >> 2]);
+			out.append((char)BASE64_enc[((one << 4) & 0x30 | (two >> 4))]);
+			out.append((char)BASE64_enc[((two << 2) & 0x3c | (three >> 6))]);
+			out.append((char)BASE64_enc[three & 0x3f]);
+
+			if(addLineBreaks && (i % 54 ==0))
+				out.append("\n");
+
+			switch( length - i)
+			{
+				case 2:
+					one = (char)input[i++];
+					two = (char)input[i++];
+
+					out.append((char)BASE64_enc[one >> 2]);
+					out.append((char)BASE64_enc[((one << 4) & 0x30 | (two >> 4))]);
+					out.append((char)BASE64_enc[((two << 2) & 0x3c)]);
+					out.append(pad);
+					break;
+
+				case 1:
+					one = (char)input[i++];
+
+					out.append((char)BASE64_enc[one >> 2]);
+					out.append((char)BASE64_enc[((one << 4) & 0x30)]);
+					out.append(pad);
+					out.append(pad);
+					break;
+			}
+
+		}
+		return out.toString();
+	}
+
+	public static String removeAllNewLines(String str)
+	{
+		return str.replaceAll("\\r\\n|\\r|\\n", " ");
+	}
+
+	public static boolean isLiteralString(String str)
+	{
+		String trimmed  = str.trim();
+		if( trimmed.charAt(0) != '\'' && trimmed.charAt(0) != '\"')
+			return false;
+		if( trimmed.charAt(trimmed.length()-1) != '\'' && trimmed.charAt(trimmed.length()-1) != '\"')
+			return false;
+
+		return true;
+	}
+
+	public static boolean isNumeric(String str)
+	{
+		NumberFormat format = NumberFormat.getInstance(Locale.US);
+		try
+		{
+			format.parse(str);
+		}
+		catch (ParseException e)
+		{
+			return false;
+		}
+
+		return true;
+	}
+	public static String replaceAll(String input, String forReplace, String replaceWith)
+	{
+		if (input == null)
+			return "null";
+
+		StringBuffer result = new StringBuffer();
+		boolean hasMore = true;
+		while (hasMore)
+		{
+			int start = input.indexOf(forReplace);
+			int end = start + forReplace.length();
+			if (start != -1)
+			{
+				result.append(input.substring(0, start) + replaceWith);
+				input = input.substring(end);
+			}
+			else
+			{
+				hasMore = false;
+				result.append(input);
+			}
+		}
+		if (result.toString().equals(""))
+			return input; // nothing is changed
+		else
+			return result.toString();
+	}
+
+	public static String handleQuotedString(String quotedString)
+	{
+		if (quotedString == null)
+			return "null";
+		String retVal = quotedString;
+		if ((retVal.startsWith("'") && retVal.endsWith("'")))
+		{
+			if (!retVal.equals("''"))
+			{
+				retVal = retVal.substring(retVal.indexOf("'") + 1,
+						retVal.lastIndexOf("'"));
+			}
+			else
+				retVal = "";
+		}
+		return retVal;
+	}
+}

system/include/portlist.h

@@ -49,9 +49,6 @@
 #define SLAVE_CONNECT_PORT              6400 //..6439    [ uses numSlaveKinds * NUM_SLAVE_CONNECT_PORT ]
 #define NUM_SLAVE_CONNECT_PORT          20   // 20 for dfu, 20 for dkc
 
-#define THOR_BASE_PORT                  6500 //..6599
-#define THOR_BASESLAVE_PORT             6600 //..6999
-
 #define WUJOBQ_BASE_PORT                6700 // ..6999  // overlapped with Thor slaves
 #define WUJOBQ_PORT_NUM                 300
 
@@ -59,10 +56,7 @@
 #define DATA_TRANSFER_PORT              7080
 #define DAFILESRV_PORT                  7100 /// aka daliservix
 #define MP_BASE_PORT                    7101 //..7999 (for Dali too)
-
-#define THOR_MP_INC                     0
-#define THOR_MP_BASE_MASTER_PORT        (THOR_BASE_PORT+THOR_MP_INC)            // 7500+
-#define THOR_MP_BASE_SLAVE_PORT         (THOR_BASE_SLAVE_PORT+THOR_MP_INC)      // 7600+
+#define MP_PORT_RANGE                   400
 
 //ESP SERVICES
 //INSECURE
@@ -128,4 +122,7 @@
 #define WS_MOXIE_SECURE_PORT            18999
 #define WS_DISTRIX_SECURE_PORT          18999
 
+#define THOR_BASE_PORT                  20000 //..~20099
+#define THOR_BASESLAVE_PORT             20100 //..~20199
+
 #endif

system/mp/mpcomm.cpp

@@ -26,6 +26,7 @@
 */
 
 #include "platform.h"
+#include "portlist.h"
 #include "jlib.hpp"
 #include <limits.h>
 
@@ -59,10 +60,6 @@
 
 #define CANCELTIMEOUT       1000        // 1 sec
 
-#define MP_BASE_PORT    7101 //..7999 (for Dali too) -- Thor reserves 7500+
-#define MP_PORT_RANGE    400 //
-
-
 #define CONNECT_TIMEOUT         (5*60*1000) // 5 mins
 #define CONNECT_READ_TIMEOUT    (3*60*1000) // 3 min    
 #define CONFIRM_TIMEOUT         (CONNECT_READ_TIMEOUT/2)    

thorlcr/graph/thgraph.cpp

@@ -1950,6 +1950,7 @@ void CGraphTempHandler::registerFile(const char *name, graph_id graphId, unsigne
 {
     assertex(temp);
     LOG(MCdebugProgress, thorJob, "registerTmpFile name=%s, usageCount=%d", name, usageCount);
+    CriticalBlock b(crit);
     if (tmpFiles.find(name))
         throw MakeThorException(TE_FileAlreadyUsedAsTempFile, "File already used as temp file (%s)", name);
     tmpFiles.replace(* new CFileUsageEntry(name, graphId, fileKind, usageCount));
@@ -1958,6 +1959,7 @@ void CGraphTempHandler::registerFile(const char *name, graph_id graphId, unsigne
 void CGraphTempHandler::deregisterFile(const char *name, bool kept)
 {
     LOG(MCdebugProgress, thorJob, "deregisterTmpFile name=%s", name);
+    CriticalBlock b(crit);
     CFileUsageEntry *fileUsage = tmpFiles.find(name);
     if (!fileUsage)
         throw MakeThorException(TE_FileNotFound, "File not found (%s) deregistering tmp file", name);
@@ -1979,6 +1981,7 @@ void CGraphTempHandler::deregisterFile(const char *name, bool kept)
 
 void CGraphTempHandler::clearTemps()
 {
+    CriticalBlock b(crit);
     Owned<IFileUsageIterator> iter = getIterator();
     ForEach(*iter)
     {

thorlcr/graph/thgraph.hpp

@@ -378,6 +378,7 @@ class graph_decl CGraphTempHandler : public CInterface, implements IGraphTempHan
 protected:
     CFileUsageTable tmpFiles;
     CJobBase &job;
+    mutable CriticalSection crit;
 public:
     IMPLEMENT_IINTERFACE;
 

thorlcr/master/thmastermain.cpp

@@ -116,7 +116,6 @@ class CRegistryServer : public CSimpleInterface
                     return;
                 rank_t sender = queryClusterGroup().rank(senderNode);
                 SocketEndpoint ep = senderNode->endpoint();
-                ep.port -= THOR_MP_INC;
                 StringBuffer url;
                 ep.getUrlStr(url);
                 if (RANK_NULL == sender)
@@ -184,7 +183,6 @@ public:
     void registerNode(unsigned slave)
     {
         SocketEndpoint ep = queryClusterGroup().queryNode(slave).endpoint();
-        ep.port -= THOR_MP_INC;
         StringBuffer url;
         ep.getUrlStr(url);
         if (status->test(slave-1))
@@ -337,7 +335,6 @@ public:
             {
                 status->set(i, false);
                 SocketEndpoint ep = queryClusterGroup().queryNode(i+1).endpoint();
-                ep.port -= THOR_MP_INC;
                 if (watchdog)
                     watchdog->removeSlave(ep);
                 CMessageBuffer msg;

thorlcr/slave/slwatchdog.cpp

@@ -55,7 +55,6 @@ public:
     CGraphProgressHandler() : threaded("CGraphProgressHandler")
     {
         self = queryMyNode()->endpoint();
-        self.port -= THOR_MP_INC;
         stopped = true;
 
         StringBuffer ipStr;

thorlcr/slave/thslavemain.cpp

@@ -78,7 +78,6 @@ void mergeCmdParams(IPropertyTree *props)
 static void replyError(const char *errorMsg)
 {
     SocketEndpoint myEp = queryMyNode()->endpoint();
-    myEp.port -= THOR_MP_INC;
     StringBuffer str("Node '");
     myEp.getUrlStr(str);
     str.append("' exception: ").append(errorMsg);
@@ -114,7 +113,6 @@ static bool RegisterSelf(SocketEndpoint &masterEp)
         setClusterGroup(group);
 
         SocketEndpoint myEp = queryMyNode()->endpoint();
-        myEp.port -= THOR_MP_INC;
         if (RANK_NULL == group->rank(queryMyNode()))
         {
             replyError("Node not part of thorgroup");

version.cmake

@@ -7,4 +7,4 @@ set ( HPCC_MINOR 7 )
 set ( HPCC_POINT 0 )
 set ( HPCC_MATURITY "trunk" )
 set ( HPCC_SEQUENCE 1 )
-###
+###