HPCC-18380 WsSQL migration into platform

Signed-off-by: Michael Gardner <michael.gardner@lexisnexisrisk.com>

CMakeLists.txt

@@ -390,6 +390,11 @@ if(TOP_LEVEL_PROJECT)
             ###
             ## CPack instruction required for RPM
             ###
+            set(CPACK_RPM_SPEC_MORE_DEFINE
+                "%define _use_internal_dependency_generator 0
+                %define __getdeps()while read file; do /usr/lib/rpm/rpmdeps -%{1} ${file} | %{__grep} -v libantlr3c.so ; done | /bin/sort -u
+                %define __find_provides /bin/sh -c '%{__getdeps P}'
+                %define __find_requires /bin/sh -c '%{__getdeps R}'")
             message("-- Will build RPM package")
             message("-- Packing BASH installation files")
             if(CLIENTTOOLS_ONLY)

cmake_modules/FindANTLR.cmake

@@ -0,0 +1,118 @@
+################################################################################
+# Copyright (C) 2013 HPCC Systems.
+#
+# All rights reserved. This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+# - Attempt to find the ANTLR jar needed to build and run ANTLR Lexers and Parsers
+# Once done this will define
+#
+# ANTLR_FOUND - ANTLR found in local system
+# ANTLR_BUILDTIME_JAR - The jar needed to build/generate ANTLR Lexers and Parsers
+# ANTLR_RUNTIME_JAR - The jar needed to build/generate ANTLR Lexers and Parsers
+################################################################################
+
+include(UseJava)
+
+option(ANTLR_VER "ANTLR runtime and buildtime version.")
+if(NOT ANTLR_VER)
+    set(ANTLR_VER 3.4)
+endif()
+
+option(ANTLR_BUILDTIME_DEP "ANTLR buildtime jar file name.")
+if(NOT ANTLR_BUILDTIME_DEP)
+    set(ANTLR_BUILDTIME_DEP "antlr-${ANTLR_VER}-complete")
+    message(STATUS "Option ANTLR_BUILDTIME_DEP not set, setting to: ${ANTLR_BUILDTIME_DEP}")
+    message(STATUS "The jar can be downloaded directly from:\n\thttps://github.com/antlr/website-antlr3/raw/gh-pages/download/antlr-3.4-complete.jar")
+endif()
+
+option(ANTLR_RUNTIME_DEP "ANTLR runtime jar file name.")
+if(NOT ANTLR_RUNTIME_DEP)
+    set(ANTLR_RUNTIME_DEP "antlr-runtime-${ANTLR_VER}")
+    message(STATUS "Option ANTLR_RUNTIME_DEP not set, setting to: ${ANTLR_RUNTIME_DEP}")
+    message(STATUS "The jar can be downloaded directly from:\n\thttps://github.com/antlr/website-antlr3/raw/gh-pages/download/antlr-runtime-3.4.jar")
+endif()
+
+option(ANTLR_PATH "Location of ANTLR runtime and buildtime jar files.")
+if(NOT ANTLR_PATH)
+    set(ANTLR_PATH "/usr/local/ANTLR/${ANTLR_VER}")
+    message(STATUS "Option ANTLR_PATH not set, setting to: ${ANTLR_PATH}")
+endif()
+
+find_jar(ANTLR_RUNTIME_JAR ${ANTLR_RUNTIME_DEP} PATHS ${ANTLR_PATH})
+find_jar(ANTLR_BUILDTIME_JAR ${ANTLR_BUILDTIME_DEP} PATHS ${ANTLR_PATH})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+    ANTLR DEFAULT_MSG
+    ANTLR_BUILDTIME_JAR
+    ANTLR_RUNTIME_JAR
+    )
+mark_as_advanced(ANTLR_BUILDTIME_JAR ANTLR_RUNTIME_JAR)
+
+function(ANTLR_TARGET)
+    # argument setup
+    set(options "")
+    set(oneValueArgs GRAMMAR_PREFIX DESTINATION)
+    set(multiValueArgs GRAMMAR_FILES)
+    cmake_parse_arguments(antlr "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+    list(GET ARGN 0 target_name)
+
+    if(antlr_DESTINATION)
+        # normalize path
+        string(REGEX REPLACE "[/]$" "" antlr_DESTINATION "${antlr_DESTINATION}")
+        set(antlr_options ${antlr_options} -o ${antlr_DESTINATION})
+    else()
+        set(antlr_options ${antlr_options} -o ${CMAKE_CURRENT_BINARY_DIR})
+    endif()
+
+    set(generated_sources
+        ${antlr_DESTINATION}/${antlr_GRAMMAR_PREFIX}Lexer.c
+        ${antlr_DESTINATION}/${antlr_GRAMMAR_PREFIX}Parser.c
+        )
+    set(generated_headers
+        ${antlr_DESTINATION}/${antlr_GRAMMAR_PREFIX}Lexer.h
+        ${antlr_DESTINATION}/${antlr_GRAMMAR_PREFIX}Parser.h
+        )
+    set(generated_misc
+        ${antlr_DESTINATION}/${antlr_GRAMMAR_PREFIX}.tokens
+        )
+
+    add_custom_command(OUTPUT ${generated_sources} ${generated_headers}
+        COMMAND ${Java_JAVA_EXECUTABLE} -jar ${ANTLR_BUILDTIME_JAR} ${antlr_GRAMMAR_FILES} ${antlr_options}
+        COMMENT "Generated ANTLR3 Lexer and Parser from grammars"
+        DEPENDS ${antlr_GRAMMAR_FILES}
+        VERBATIM
+        )
+    add_custom_target("${target_name}"
+        DEPENDS ${generated_sources} ${generated_headers}
+        )
+    set_source_files_properties(
+        ${generated_sources}
+        ${generated_headers}
+        PROPERTIES GENERATED TRUE
+        )
+    if(antlr_DESTINATION)
+        set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES ${antlr_DESTINATION})
+    endif()
+    set_property(DIRECTORY PROPERTY
+        ADDITIONAL_MAKE_CLEAN_FILES
+            ${generated_sources}
+            ${generated_headers}
+            ${generated_misc}
+        )
+    set(ANTLR_${target_name}_OUTPUTS ${generated_sources} ${generated_headers} ${generated_misc} PARENT_SCOPE)
+    set(ANTLR_${target_name}_SOURCES ${generated_sources} PARENT_SCOPE)
+    set(ANTLR_${target_name}_HEADERS ${generated_headers} PARENT_SCOPE) 
+    set(ANTLR_${target_name}_MISC    ${generated_misc}    PARENT_SCOPE) 
+endfunction(ANTLR_TARGET)

cmake_modules/commonSetup.cmake

@@ -920,7 +920,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   endif()
   set (CMAKE_SKIP_BUILD_RPATH  FALSE)
   set (CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
-  set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${LIB_DIR};${CMAKE_INSTALL_PREFIX}/${PLUGINS_DIR}")
+  set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${LIB_DIR};${CMAKE_INSTALL_PREFIX}/${PLUGINS_DIR};${CMAKE_INSTALL_PREFIX}/${LIB_DIR}/external")
   set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
   MACRO (FETCH_GIT_TAG workdir edition result)
       execute_process(COMMAND "${GIT_COMMAND}" describe --tags --dirty --abbrev=6 --match ${edition}*

esp/scm/additional.cmake

@@ -27,6 +27,7 @@ set ( ESPSCM_SRCS
       ws_config.ecm
       ws_fileio.ecm
       ws_account.ecm
+      ws_sql.ecm
       ##### LIST FOR ESPECL
       WsDeploy.ecm
     )

esp/scm/ws_sql.ecm

@@ -0,0 +1,343 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+EspInclude(ws_workunits);
+EspInclude(ws_topology);
+
+ESPStruct HPCCColumn
+{
+   string Name;
+   string Type;
+};
+
+ESPStruct HPCCTable
+{
+    string Name;
+    ESParray<ESPstruct HPCCColumn, Column> Columns;
+    string ECL;
+    string Format;
+    string ContentType;
+    string Description;
+    boolean IsKeyed;
+    //string IndexFilePosFieldName;
+    boolean IsSuper;
+    string CsvQuote;
+    string CsvSeparate;
+    string CsvTerminate;
+    string Group;
+    integer MaxRecordSize;
+    string Modified;
+    integer NumParts;
+    string Owner;
+};
+
+ESPstruct OutputDataset
+{
+    string name;
+    ESParray<ESPstruct HPCCColumn, OutParam> OutParams;
+};
+ESPstruct QuerySignature
+{
+    ESParray<ESPstruct HPCCColumn, InParam> InParams;
+    ESParray<ESPstruct OutputDataset, ResultSet> ResultSets;
+};
+
+ESPstruct PublishedQuery
+{
+    string Name;
+    string Id;
+    string Wuid;
+    boolean Suspended;
+    ESPstruct QuerySignature Signature;
+};
+
+ESPStruct QuerySetAliasMap
+{
+    string Id;
+    string Name;
+};
+
+ESPStruct HPCCQuerySet
+{
+    string Name;
+    ESParray<ESPstruct PublishedQuery, QuerySetQuery> QuerySetQueries;
+    ESParray<ESPstruct QuerySetAliasMap, QuerySetAlias> QuerySetAliases;
+};
+
+ESPStruct HPCCTargetCluster
+{
+    string Name;
+    ESParray<ESPstruct PublishedQuery, QuerySetQuery> QuerySetQueries;
+    ESParray<ESPstruct QuerySetAliasMap, QuerySetAlias> QuerySetAliases;
+};
+
+ESPrequest PrepareSQLRequest
+{
+    [rows(28), cols(80)] string SqlText;
+    string TargetCluster; //where should this be compiled
+    [min_ver("3.04")] ESParray<string, AlternateCluster> AlternateClusters; //Although this appears to be a runtime option, in ECL this is a compiletime option.
+    string TargetQuerySet; //pertaining to Stored Proc calls
+    int Wait(-1);
+    //not so sure about this setting result limits in the prepare
+    //int resultLimit(0);
+};
+
+ESPresponse [exceptions_inline] PrepareSQLResponse
+{
+    ESPstruct ECLWorkunit Workunit;
+    string Result;
+};
+
+ESPrequest ExecuteSQLRequest
+{
+    [rows(28), cols(80)] string SqlText;
+    string UserName;
+    string TargetCluster; //where should this compile and execute
+    [min_ver("3.04")] ESParray<string, AlternateCluster> AlternateClusters;
+    string TargetQuerySet; //pertaining to Stored Proc calls
+    boolean SuppressResults(false);
+    boolean SuppressXmlSchema(true);
+    int Wait(-1);
+    int resultLimit(0);
+    int ResultWindowStart(0);
+    int ResultWindowCount(0);
+    [min_ver(3.05)] boolean IgnoreCache(false);
+};
+
+ESPresponse [exceptions_inline] ExecuteSQLResponse
+{
+    string ParentWuId;
+    string Result;
+    ESPstruct ECLWorkunit Workunit;
+    int resultLimit;
+    unsigned ResultWindowStart;
+    unsigned ResultWindowCount;
+};
+
+ESPrequest ExecutePreparedSQLRequest
+{
+    string WuId;
+    string UserName;
+    string TargetCluster; //where should this execute
+    boolean SuppressResults(false);
+    boolean SuppressXmlSchema(true);
+    int Wait(-1);
+    //int resultLimit(0); underlying WU code doesn't allow setting result limits on cloned wu
+    int ResultWindowStart(0);
+    int ResultWindowCount(0);
+    ESParray<ESPstruct NamedValue> Variables;
+};
+
+ESPresponse [exceptions_inline] ExecutePreparedSQLResponse
+{
+    string ParentWuId;
+    string Result;
+    //int resultLimit;
+    unsigned ResultWindowStart;
+    unsigned ResultWindowCount;
+    ESPstruct ECLWorkunit Workunit;
+};
+
+ESPrequest GetDBSystemInfoRequest
+{
+    boolean IncludeAll(true);
+};
+
+ESPresponse [exceptions_inline] GetDBSystemInfoResponse
+{
+    string Name;
+    string FullVersion;
+    string Major;
+    string Minor;
+    string Point;
+    string Project;
+    string Maturity;
+
+    [min_ver("3.01")] string WsSQLFullVersion;
+    [min_ver("3.01")] string WsSQLMajor;
+    [min_ver("3.01")] string WsSQLMinor;
+    [min_ver("3.01")] string WsSQLPoint;
+    [min_ver("3.01")] string WsSQLProject;
+    [min_ver("3.01")] string WsSQLMaturity;
+};
+
+ESPrequest GetDBMetaDataRequest
+{
+    boolean IncludeTables;
+    string  TableFilter;
+    boolean IncludeStoredProcedures;
+    string  QuerySet;
+    boolean IncludeTargetClusters;
+    string  ClusterType; //ALLCLUSTERS | ROOT
+};
+
+ESPresponse [exceptions_inline] GetDBMetaDataResponse
+{
+    ESParray<ESPstruct HPCCTable, Table> Tables;
+    integer TableCount;
+    ESParray<ESPstruct HPCCQuerySet, QuerySet> QuerySets;
+    ESParray<string, ClusterName> ClusterNames;
+};
+
+ESPrequest GetResultsRequest
+{
+    string  WuId;
+    boolean SuppressXmlSchema(true);
+    int     ResultWindowStart(0);
+    int     ResultWindowCount(0);
+};
+
+ESPresponse [exceptions_inline] GetResultsResponse
+{
+    string Result;
+    unsigned ResultWindowStart;
+    unsigned ResultWindowCount;
+    ESPstruct ECLWorkunit Workunit;
+};
+
+ESPrequest EchoRequest
+{
+    string Request;
+};
+
+ESPresponse EchoResponse
+{
+    string Response;
+};
+
+ESPStruct RelatedIndexSet
+{
+    String FileName;
+    ESParray<String, Index> Indexes;
+};
+
+ESPrequest GetRelatedIndexesRequest
+{
+    ESParray<String, FileName> FileNames;
+};
+
+ESPresponse GetRelatedIndexesResponse
+{
+    ESParray<ESPstruct RelatedIndexSet, RelatedIndexSet> RelatedIndexSets;
+};
+
+ESPrequest SetRelatedIndexesRequest
+{
+    ESParray<ESPstruct RelatedIndexSet, RelatedIndexSet> RelatedIndexSets;
+};
+
+ESPresponse SetRelatedIndexesResponse
+{
+    ESParray<ESPstruct RelatedIndexSet, RelatedIndexSet> RelatedIndexSets;
+};
+
+ESPenum HPCCFieldType : string
+{
+    BOOLEAN("BOOLEAN"),
+    INTEGER("INTEGER"),
+    xUNSIGNED("UNSIGNED"),
+    REAL("REAL"),
+    DECIMAL("DECIMAL"),
+    UDECIMAL("UDECIMAL"),
+    xSTRING("STRING"),
+    QSTRING("QSTRING"),
+    UNICODE("UNICODE"),
+    DATA("DATA"),
+    VARSTRING("VARSTRING"),
+    VARUNICODE("VARUNICODE"),
+};
+
+ESPStruct EclFieldType
+{
+    ESPenum HPCCFieldType Type;
+    String Locale;
+    Integer Length;
+    Integer Precision;
+};
+
+ESPStruct EclFieldDeclaration
+{
+    String FieldName;
+    ESPStruct EclFieldType EclFieldType;
+};
+
+ESPstruct DataTypeParam
+{
+    String Name;
+    ESParray<String, Value> Values;
+};
+
+ESPenum HPCCFileType : string
+{
+    FLAT("FLAT"),
+    CSV("CSV"),
+    JSON("JSON"),
+    XML("XML"),
+};
+
+ESPStruct DataType
+{
+    ESPenum HPCCFileType Type;
+    ESParray<ESPstruct DataTypeParam, Param> Params;
+};
+
+ESPStruct DataSourceInfo
+{
+    String SprayedFileName;
+    String LandingZoneIP;
+    String LandingZonePath;
+    String LandingZoneFileName;
+};
+
+ESPrequest CreateTableAndLoadRequest
+{
+    String  TableName;
+    String  TableDescription;
+    Boolean Overwrite;
+    ESParray<ESPstruct EclFieldDeclaration, EclField> EclFields;
+    String  TargetCluster;
+    String  Owner;
+    ESPStruct DataSourceInfo DataSource;
+    ESPStruct DataType DataSourceType;
+    int Wait(-1);
+};
+
+ESPresponse CreateTableAndLoadResponse
+{
+    String TableName;
+    Boolean Success;
+    String EclRecordDefinition;
+    ESPstruct ECLWorkunit Workunit;
+};
+
+ESPservice [version("3.05"), default_client_version("3.05"), auth_feature("DEFERRED"), exceptions_inline("./smc_xslt/exceptions.xslt")] wssql
+{
+    ESPmethod [min_ver("3.03")] CreateTableAndLoad(CreateTableAndLoadRequest, CreateTableAndLoadResponse);
+    ESPmethod [min_ver("3.02")] SetRelatedIndexes(SetRelatedIndexesRequest, SetRelatedIndexesResponse);
+    ESPmethod [min_ver("3.02")] GetRelatedIndexes(GetRelatedIndexesRequest, GetRelatedIndexesResponse);
+    ESPmethod [] PrepareSQL(PrepareSQLRequest, PrepareSQLResponse);
+    ESPmethod [] ExecuteSQL(ExecuteSQLRequest, ExecuteSQLResponse);
+    ESPmethod [] ExecutePreparedSQL(ExecutePreparedSQLRequest, ExecutePreparedSQLResponse);
+    ESPmethod [] GetResults(GetResultsRequest, GetResultsResponse);
+    ESPmethod [] GetDBSystemInfo(GetDBSystemInfoRequest, GetDBSystemInfoResponse);
+    ESPmethod [] GetDBMetaData(GetDBMetaDataRequest, GetDBMetaDataResponse);
+    ESPmethod Echo(EchoRequest, EchoResponse);
+};
+
+SCMexportdef(wssql);
+
+SCMapi(wssql) IClientwssql *createwssqlClient();

esp/services/CMakeLists.txt

@@ -31,6 +31,7 @@ HPCC_ADD_SUBDIRECTORY (WsDeploy "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_packageprocess "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ws_esdlconfig "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (esdl_svc_engine "PLATFORM")
+HPCC_ADD_SUBDIRECTORY (ws_sql "PLATFORM")
 if (LOGGING_SERVICE)
 HPCC_ADD_SUBDIRECTORY (ws_loggingservice "PLATFORM")
 endif()

esp/services/ws_sql/CMakeLists.txt

@@ -0,0 +1,115 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+# Component: ws_sql
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for ws_sql
+#####################################################
+
+project(ws_sql)
+
+find_package(Java REQUIRED)
+find_package(ANTLR REQUIRED)
+
+include(${HPCC_SOURCE_DIR}/esp/scm/additional.cmake)
+include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
+
+include(antlr3c.cmake)
+
+antlr_target(sql2ecl
+    GRAMMAR_PREFIX HPCCSQL
+    DESTINATION "${CMAKE_CURRENT_BINARY_DIR}"
+    GRAMMAR_FILES "${CMAKE_CURRENT_SOURCE_DIR}/SQL2ECL/ANTLR3c/HPCCSQL.g"
+    )
+
+set(SRCS
+    ${CMAKE_CURRENT_SOURCE_DIR}/ws_sqlPlugin.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/ws_sqlService.cpp
+    ${ESPSCM_GENERATED_DIR}/common_esp.cpp
+    ${ESPSCM_GENERATED_DIR}/ws_sql_esp.cpp
+    ${ESPSCM_GENERATED_DIR}/ws_workunits_esp.cpp
+    ${ESPSCM_GENERATED_DIR}/ws_topology_esp.cpp
+    ${HPCC_SOURCE_DIR}/esp/services/ws_ecl/ws_ecl_wuinfo.cpp
+    ${HPCC_SOURCE_DIR}/esp/services/ws_workunits/ws_workunitsHelpers.cpp
+    SQL2ECL/HPCCFileCache.cpp
+    SQL2ECL/HPCCFile.cpp
+    SQL2ECL/ECLFunction.cpp
+    SQL2ECL/ECLEngine.cpp
+    SQL2ECL/SQLColumn.cpp
+    SQL2ECL/SQLTable.hpp
+    SQL2ECL/SQLExpression.cpp
+    SQL2ECL/SQLJoin.cpp
+    SQL2ECL/HPCCSQLTreeWalker.cpp
+    ${ANTLR_sql2ecl_SOURCES}
+    )
+
+include_directories(
+    ##added for build-config.h
+    ${CMAKE_BINARY_DIR}
+    ##added for build-config.h
+    ${CMAKE_BINARY_DIR}/oss
+    ${HPCC_WSSQL_SOURCE_DIR}/esp/services/ws_sql/SQL2ECL
+    ${HPCC_SOURCE_DIR}/esp/bindings
+    ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
+    ${HPCC_SOURCE_DIR}/esp/smc/SMCLib
+    ${HPCC_SOURCE_DIR}/esp/platform
+    ${HPCC_SOURCE_DIR}/esp/services
+    ${HPCC_SOURCE_DIR}/esp/services/ws_ecl
+    ${HPCC_SOURCE_DIR}/esp/services/ws_workunits
+    ${HPCC_SOURCE_DIR}/esp/services/ws_dfu
+    ${HPCC_SOURCE_DIR}/system/xmllib
+    ${HPCC_SOURCE_DIR}/system/include
+    ${HPCC_SOURCE_DIR}/system/jlib
+    ${HPCC_SOURCE_DIR}/system/security/shared
+    ${HPCC_SOURCE_DIR}/system/security/securesocket
+    ${HPCC_SOURCE_DIR}/system/security/zcrypt
+    ${HPCC_SOURCE_DIR}/system/security/LdapSecurity
+    ${HPCC_SOURCE_DIR}/system/mp
+    ${HPCC_SOURCE_DIR}/dali/dfu
+    ${HPCC_SOURCE_DIR}/dali/base/
+    ${HPCC_SOURCE_DIR}/common/workunit
+    ${HPCC_SOURCE_DIR}/common/remote
+    ${HPCC_SOURCE_DIR}/common/environment
+    ${HPCC_SOURCE_DIR}/common/wuwebview
+    ${HPCC_SOURCE_DIR}/common/fileview2
+    ${HPCC_SOURCE_DIR}/common/dllserver
+    ${HPCC_SOURCE_DIR}/common/deftype
+    ${HPCC_SOURCE_DIR}/ecl/hql
+    ${CMAKE_CURRENT_BINARY_DIR}/antlr3c/include
+    ${ESPSCM_GENERATED_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}/SQL2ECL
+    )
+
+add_definitions(-D_USRDLL)
+HPCC_ADD_LIBRARY(ws_sql SHARED ${SRCS})
+add_dependencies(ws_sql espscm sql2ecl)
+target_link_libraries(
+    ws_sql
+    ws_dfu
+    jlib
+    xmllib
+    esphttp
+    securesocket
+    dalibase
+    libantlr3c
+    wuwebview
+    )
+install(TARGETS ws_sql
+    RUNTIME DESTINATION ${EXEC_DIR}
+    LIBRARY DESTINATION ${LIB_DIR}
+    COMPONENT Runtime
+    )

esp/services/ws_sql/SQL2ECL/ANTLR3c/HPCCSQL.g

@@ -0,0 +1,997 @@
+grammar HPCCSQL;
+
+options
+{
+  language=C;
+  output=AST;
+  backtrack=true;
+  ASTLabelType = pANTLR3_BASE_TREE;
+}
+
+tokens
+{
+    TOKEN_ROOT;
+    TOKEN_SELECT_STATEMENT;
+    TOKEN_CALL_STATEMENT;
+    TOKEN_CREATE_LOAD_TABLE_STATEMENT;
+    TOKEN_CREATE_INDEX_STATEMENT;
+    TOKEN_CREATE_TABLE;
+    TOKEN_DONOT_OVERWRITE;
+    TOKEN_OVERWRITE;
+    TOKEN_LOAD_TABLE;
+    TOKEN_FROM_LIST;
+    TOKEN_FROM_TABLE;
+    TOKEN_PROC_NAME;
+    TOKEN_PROC_PARAMS;
+    TOKEN_ALIAS;
+    TOKEN_INNER_JOIN;
+    TOKEN_OUTTER_JOIN;
+    TOKEN_INDEX_HINT;
+    TOKEN_AVOID_INDEX;
+    TOKEN_COLUMN;
+    TOKEN_LISTEXP;
+    TOKEN_FUNCEXP;
+    TOKEN_PARAMPLACEHOLDER;
+    TOKEN_COLUMNWILDCARD;
+    TOKEN_TABLE_SCHEMA;
+    TOKEN_COLUMN_DEF_LIST;
+    TOKEN_COLUMN_DEF;
+    TOKEN_LANDING_ZONE;
+    TOKEN_VARIABLE_FILE;
+    TOKEN_VAR_SEPERATOR;
+    TOKEN_VAR_TERMINATOR;
+    TOKEN_VAR_ENCLOSED;
+    TOKEN_VAR_ESCAPED;
+}
+
+@header
+{
+}
+@members
+{
+}
+@lexer::includes
+{
+}
+@lexer::apifuncs
+{
+}
+@lexer::header
+{
+}
+@lexer::members
+{
+}
+@parser::apifuncs
+{
+}
+
+fragment Ai :  'a' | 'A';
+fragment Bi :  'b' | 'B';
+fragment Ci :  'c' | 'C';
+fragment Di :  'd' | 'D';
+fragment Ei :  'e' | 'E';
+fragment Fi :  'f' | 'F';
+fragment Gi :  'g' | 'G';
+fragment Hi :  'h' | 'H';
+fragment Ii :  'i' | 'I';
+fragment Ji :  'j' | 'J';
+fragment Ki :  'k' | 'K';
+fragment Li :  'l' | 'L';
+fragment Mi :  'm' | 'M';
+fragment Ni :  'n' | 'N';
+fragment Oi :  'o' | 'O';
+fragment Pi :  'p' | 'P';
+fragment Qi :  'q' | 'Q';
+fragment Ri :  'r' | 'R';
+fragment Si :  's' | 'S';
+fragment Ti :  't' | 'T';
+fragment Ui :  'u' | 'U';
+fragment Vi :  'v' | 'V';
+fragment Wi :  'w' | 'W';
+fragment Xi :  'x' | 'X';
+fragment Yi :  'y' | 'Y';
+fragment Zi :  'z' | 'Z';
+
+ADD_SYM     : Ai Di Di  ;
+ALL         : Ai Li Li  ;
+ANY         : Ai Ni Yi ;
+AS_SYM      : Ai Si  ;
+ASC         : Ai Si Ci  ;
+ASCII_SYM   : Ai Si Ii Ii ;
+AT_SYM      : Ai Ti  ;
+AVG         : Ai Vi Gi ;
+BETWEEN     : Bi Ei Ti Wi Ei Ei Ni  ;
+BINARY_SYM  : Bi Ii Ni Ai Ri Yi ;
+BIT_SYM     : Bi Ii Ti ;
+BOOL_SYM    : Bi Oi Oi Li  ;
+BOOLEAN_SYM : Bi Oi Oi Li Ei Ai Ni  ;
+BY_SYM      : Bi Yi ;
+CALL_SYM    : Ci Ai Li Li  ;
+CREATE_SYM  : Ci Ri Ei Ai Ti Ei ;
+COLUMN_SYM  : Ci Oi Li Ui Mi Ni  ;
+COMMENT_SYM : Ci Oi Mi Mi Ei Ni Ti ;
+CONTAINS_SYM : Ci Oi Ni Ti Ai Ii Ni Si  ;
+COUNT       : Ci Oi Ui Ni Ti  ;
+DATA_SYM    : Di Ai Ti Ai ;
+DESC        : Di Ei Si Ci  ;
+DISTINCT    : Di Ii Si Ti Ii Ni Ci Ti ;
+EXISTS_SYM  : Ei Xi Ii Si Ti Si ;
+FALSE_SYM   : Fi Ai Li Si Ei ;
+FOR_SYM     : Fi Oi Ri ;
+FROM        : Fi Ri Oi Mi  ;
+GROUP_SYM   : Gi Ri Oi Ui Pi  ;
+HAVING      : Hi Ai Vi Ii Ni Gi  ;
+IF_SYM      : Ii Fi ;
+IGNORE_SYM  : Ii Gi Ni Oi Ri Ei ;
+IN_SYM      : Ii Ni ;
+INFILE_SYM  : Ii Ni Fi Ii Li Ei ;
+INTO_SYM    : Ii Ni Ti Oi ;
+INT_SYM     : Ii Ni Ti ;
+INTEGER_SYM : Ii Ni Ti Ei Gi Ei Ri ;
+INDEX_SYM   : Ii Ni Di Ei Xi  ;
+INNER_SYM   : Ii Ni Ni Ei Ri  ;
+IS_SYM      : Ii Si  ;
+JOIN_SYM    : Ji Oi Ii Ni  ;
+KEY_SYM     : Ki Ei Yi  ;
+KEYS        : Ki Ei Yi Si  ;
+LAST_SYM    : Li Ai Si Ti  ;
+LEFT        : Li Ei Fi Ti  ;
+LIKE_SYM    : Li Ii Ki Ei  ;
+LIMIT       : Li Ii Mi Ii Ti  ;
+LOAD_SYM    : Li Oi Ai Di ;
+LOCAL_SYM   : Li Oi Ci Ai Li ;
+LOWER       : (Li Oi Wi Ei Ri) | (Li Ci Ai Si Ei) ;
+MAX_SYM     : Mi Ai Xi  ;
+MEDIUMINT   : Mi Ei Di Ii Ui Mi Ii Ni Ti ;
+MIN_SYM     : Mi Ii Ni  ;
+MOD         : Mi Oi Di  ;
+NOT_SYM     : (UNDERSCORE Ni Oi Ti) | (Ni Oi Ti) | ('!') ;
+NULL_SYM    : Ni Ui Li Li  ;
+OFFSET_SYM  : Oi Fi Fi Si Ei Ti  ;
+ON          : Oi Ni  ;
+ORDER_SYM   : Oi Ri Di Ei Ri  ;
+OUT_SYM     : Oi Ui Ti  ;
+OUTER       : Oi Ui Ti Ei Ri  ;
+POWER       : Pi Oi Wi Ei Ri  ;
+REPLACE_SYM : Ri Ei Pi Li Ai Ci Ei ;
+SELECT      : Si Ei Li Ei Ci Ti ;
+SMALLINT    : Si Mi Ai Li Li Ii Ni Ti ;
+SUM         : Si Ui Mi  ;
+TABLE_SYM   : Ti Ai Bi Li Ei ;
+TINYINT     : Ti Ii Ni Yi Ii Ni Ti ;
+TRUE_SYM    : Ti Ri Ui Ei ;
+TYPE_SYM    : Ti Yi Pi Ei ;
+UPPER       : Ui Pi Pi Ei Ri ;
+USE_SYM     : Ui Si Ei  ;
+UTF8_SYM    : Ui Ti Fi '8' ;
+UNSIGNED_SYM : Ui Ni Si Ii Gi Ni Ei Di ;
+WHERE       : Wi Hi Ei Ri Ei  ;
+XOR         : Xi Oi Ri  ;
+
+BIGINT_SYM  : Bi Ii Gi Ii Ni Ti ;
+REAL_SYM    : Ri Ei Ai Li ;
+DOUBLE_SYM  : Di Oi Ui Bi Li Ei ;
+FLOAT_SYM   : Fi Li Oi Ai Ti ;
+DECIMAL_SYM : Di Ei Ci Ii Mi Ai Li ;
+NUMERIC_SYM : Ni Ui Mi Ei Ri Ii Ci ;
+DATE_SYM    : Di Ai Ti Ei ;
+TIME_SYM    : Ti Ii Mi Ei ;
+TIMESTAMP_SYM  : Ti Ii Mi Ei Si Ti Ai Mi Pi ;
+DATETIME_SYM   : Di Ai Ti Ei Ti Ii Mi Ei ;
+YEAR_SYM       : Yi Ei Ai Ri ;
+CHAR_SYM       : Ci Hi Ai Ri ;
+VARCHAR_SYM    : Vi Ai Ri Ci Hi Ai Ri ;
+VARBINARY_SYM  : Vi Ai Ri Bi Ii Ni Ai Ri Yi ;
+TINYBLOB_SYM   : Ti Ii Ni Yi Bi Li Oi Bi ;
+BLOB_SYM       : Bi Li Oi Bi ;
+MEDIUMBLOB_SYM : Mi Ei Di Ii Ui Mi Bi Li Oi Bi ;
+LONGBLOB_SYM   : Li Oi Ni Gi Bi Li Oi Bi ;
+TINYTEXT_SYM   : Ti Ii Ni Yi Ti Ei Xi Ti ;
+TEXT_SYM       : Ti Ei Xi Ti ;
+MEDIUMTEXT_SYM : Mi Ei Di Ii Ui Mi Ti Ei Xi Ti ;
+LONGTEXT_SYM   : Li Oi Ni Gi Ti Ei Xi Ti ;
+ENUM_SYM       : Ei Ni Ui Mi ;
+SET_SYM        : Si Ei Ti ;
+FLAT_SYM       : Fi Li Ai Ti ;
+XML_SYM        : Xi Mi Li ;
+CSV_SYM        : Ci Si Vi ;
+JSON_SYM       : Ji Si Oi Ni ;
+CONNECTION_SYM : Ci Oi Ni Ni Ei Ci Ti Ii Oi Ni ;
+DIRECTORY_SYM  : Di Ii Ri Ei Ci Ti Oi Ri Yi ;
+ENCLOSED_SYM   : Ei Ni Ci Li Oi Si Ei Di ;
+LINES_SYM      : Li Ii Ni Ei Si ;
+ESCAPED_SYM    : Ei Si Ci Ai Pi Ei Di ;
+TERMINATED_SYM : Ti Ei Ri Mi Ii Ni Ai Ti Ei Di ;
+OPTIONALLY_SYM : Oi Pi Ti Ii Oi Ni Ai Li Li Yi ;
+EBCDIC_SYM     : Ei Bi Ci Di Ii Ci ;
+FIELDS_SYM     : Fi Ii Ei Li Di Si ;
+COLUMNS_SYM    : Ci Oi Li Ui Mi Ni Si ;
+
+CHARACTER_SET : ('CHARACTER SET' | 'character set');
+IFNOTEXISTS : ('IF NOT EXISTS' | 'if not exists');
+ISNOTNULL   : ('IS NOT NULL' | 'is not null');
+ISNULL      : ('IS NULL' | 'is null');
+NOT_IN      : ('NOT IN' | 'not in');
+NOT_LIKE    : ('NOT LIKE' | 'not like');
+
+DIVIDE      : (  Di Ii Vi ) | '/' ;
+MOD_SYM     : (  Mi Oi Di ) | '%' ;
+OR_SYM      : (  Oi Ri ) | '||';
+AND_SYM     : (  Ai Ni Di ) | '&&';
+
+ARROW       : '=>' ;
+EQ_SYM      : '=' | '<=>' ;
+NOT_EQ      : '<>' | '!=' | '~='| '^=';
+LET         : '<=' ;
+GET         : '>=' ;
+SET_VAR     : ':=' ;
+SHIFT_LEFT  : '<<' ;
+SHIFT_RIGHT : '>>' ;
+//ALL_FIELDS  : '.*' ;
+SQUOTE      : '\'' ;
+//DQUOTE      : '\"' ;
+DQUOTE      : '"' ;
+DOLLAR      : '$' ;
+QUESTION    : '?' ;
+SEMI        : ';' ;
+COLON       : ':' ;
+DOT         : '.' ;
+COMMA       : ',' ;
+ASTERISK    : '*' ;
+RPAREN      : ')' ;
+LPAREN      : '(' ;
+RBRACK      : ']' ;
+LBRACK      : '[' ;
+LCURLY      : '{' ;
+RCURLY      : '}' ;
+PLUS        : '+' ;
+MINUS       : '-' ;
+NEGATION    : '~' ;
+VERTBAR     : '|' ;
+BITAND      : '&' ;
+POWER_OP    : '^' ;
+GTH         : '>' ;
+LTH         : '<' ;
+
+fragment UNDERSCORE  : '_' ;
+
+fragment DCOLON      : '::' ;
+
+fragment DEFSCOPE    : DOT DCOLON ;
+
+fragment LETTER_FRAGMENT
+:
+    ( 'A'..'Z' | 'a'..'z')
+;
+
+INTEGER_NUM
+:
+    DIGIT_FRAGMENT+
+;
+
+fragment DIGIT_FRAGMENT
+:
+    ( '0'..'9')
+;
+
+fragment HEX_DIGIT_FRAGMENT
+:
+   ( 'a'..'f' | 'A'..'F' | DIGIT_FRAGMENT )
+;
+
+HEX_DIGIT
+:
+  (  '0x'     (HEX_DIGIT_FRAGMENT)+  )
+  |
+  (  'X' '\'' (HEX_DIGIT_FRAGMENT)+ '\''  )
+;
+
+BIT_NUM
+:
+  (  '0b'    ('0'|'1')+  )
+  |
+  (  Bi '\'' ('0'|'1')+ '\''  )
+;
+
+REAL_NUMBER
+:
+  (  INTEGER_NUM DOT INTEGER_NUM | INTEGER_NUM DOT | DOT INTEGER_NUM | INTEGER_NUM  )
+  (  (Ei) ( PLUS | MINUS )? INTEGER_NUM  )?
+;
+
+TEXT_STRING
+:
+    SQUOTE
+    (
+        (SQUOTE SQUOTE)
+        | ('\\''\'')
+        | ~('\'')
+    )*
+    SQUOTE
+;
+
+quoted_id
+:
+    DQUOTE! ID DQUOTE!
+;
+
+quoted_table_id
+:
+    DQUOTE! (ABSOLUTE_FILE_ID | ID) DQUOTE!
+;
+
+//Cannot rewrite lexer rules see above parser rule
+//QUOTED_ID
+//:
+//  DQUOTE! ID DQUOTE!
+//;
+
+fragment ABSOLUTE_FILE_ID_PREFIX
+:
+    NEGATION |
+    DEFSCOPE |
+    NEGATION DEFSCOPE
+;
+
+ABSOLUTE_FILE_ID
+:
+    ABSOLUTE_FILE_ID_PREFIX ID_FRAGMENT+
+;
+
+ID
+:
+    LETTER_FRAGMENT ( ID_FRAGMENT )*
+;
+
+fragment ID_FRAGMENT
+:
+    LETTER_FRAGMENT | UNDERSCORE | INTEGER_NUM | DCOLON
+;
+
+WHITE_SPACE
+:
+  ( ' '|'\r'|'\t'|'\n' ) {$channel=HIDDEN;}
+;
+
+relational_op
+:
+  EQ_SYM
+  | LTH
+  | GTH
+  | NOT_EQ
+  | LET
+  | GET
+;
+
+strcomp_op
+:
+  LIKE_SYM
+  | NOT_LIKE
+;
+
+list_op
+:
+    IN_SYM
+    | NOT_IN
+;
+
+string_literal
+:
+   TEXT_STRING
+;
+number_literal
+:
+   (PLUS | MINUS)? (INTEGER_NUM | REAL_NUMBER)
+;
+
+hex_literal
+:
+  HEX_DIGIT
+;
+
+boolean_literal
+:
+  TRUE_SYM
+  | FALSE_SYM
+;
+
+bit_literal
+:
+    BIT_NUM
+;
+
+literal_value
+:
+  (
+      string_literal
+      | number_literal
+      | hex_literal
+      | boolean_literal
+      | bit_literal
+  )
+;
+
+functionList
+:
+  group_functions
+  | char_functions
+;
+
+char_functions
+:
+  LOWER
+  | UPPER
+;
+
+group_functions
+:
+  AVG
+  | COUNT
+  | MAX_SYM
+  | MIN_SYM
+  | SUM
+;
+
+query_set_name      : ID ;
+schema_name         : ID ;
+table_name          : ABSOLUTE_FILE_ID | ID ;
+quoted_table_name   : quoted_table_id;
+engine_name         : ID ;
+column_name         : ID ;
+quoted_column_name  : quoted_id ;
+index_name          : ID ;
+user_name           : ID ;
+function_name       : ID ;
+procedure_name      : ID ;
+
+alias
+:
+  ( AS_SYM )? ID -> ^( TOKEN_ALIAS ID)
+  | ( AS_SYM )? quoted_id -> ^( TOKEN_ALIAS quoted_id)
+;
+
+column_spec
+:
+    ( quoted_table_name DOT )? quoted_column_name -> ^(TOKEN_COLUMN quoted_column_name quoted_table_name? )
+    |( table_name DOT )? column_name -> ^(TOKEN_COLUMN column_name table_name? )
+;
+
+expression_list
+:
+    LPAREN expression ( COMMA expression )* RPAREN
+;
+
+expression
+:
+    orExpression
+;
+
+orExpression
+:
+    andExpression
+    (
+      OR_SYM^ andExpression
+    )*
+;
+
+andExpression
+:
+    relationalExpression
+    (
+      AND_SYM^ relationalExpression
+    )*
+;
+
+relationalExpression
+:
+    stringCompExpression
+    (
+      relational_op^
+      stringCompExpression
+    )*
+;
+
+stringCompExpression
+:
+  additionExpression
+  (
+    strcomp_op^
+    additionExpression
+  )*
+;
+
+additionExpression
+:
+    multiplyExpression
+    (
+      (PLUS|MINUS)^ multiplyExpression
+    )?
+;
+
+multiplyExpression
+:
+    listExpression
+    (
+      (ASTERISK
+      |DIVIDE
+      |MOD_SYM
+      |POWER_OP)^
+      listExpression
+    )?
+;
+
+listExpression
+    :
+    unaryExpression
+    (
+      list_op^
+      literalOrPlaceholderExpressionList
+    )?
+;
+
+unaryExpression
+:
+    (NEGATION | NOT_SYM )^ simpleExpression
+    |
+    simpleExpression (ISNOTNULL | ISNULL)^
+    |
+    simpleExpression
+;
+
+simpleExpression
+:
+    column_spec
+    | literal_value
+    | function_call
+    | parenExpression
+    | parameterPlaceHolder
+    | literalExpressionList
+;
+
+parenExpression
+:
+  LPAREN^ orExpression RPAREN!
+;
+
+literalExpressionList
+:
+    LPAREN literal_value( COMMA literal_value )* RPAREN -> {$COMMA != NULL}? ^(TOKEN_LISTEXP literal_value+)
+    -> ^(TOKEN_LISTEXP literal_value)
+;
+
+literalOrPlaceholderExpressionList
+:
+    LPAREN literalOrPlaceholderValue( COMMA literalOrPlaceholderValue )* RPAREN -> {$COMMA != NULL}? ^(TOKEN_LISTEXP literalOrPlaceholderValue+)
+    -> ^(TOKEN_LISTEXP literalOrPlaceholderValue)
+;
+
+literalOrPlaceholderValue
+:
+    literal_value | parameterPlaceHolder
+;
+
+function_call
+:
+    (
+      functionList ( LPAREN (functionParam (COMMA functionParam)*)? RPAREN ) ?
+    ) -> {$COMMA != NULL}? ^(TOKEN_FUNCEXP functionList functionParam+)
+      -> ^(TOKEN_FUNCEXP functionList functionParam)
+;
+
+functionParam
+:
+    (DISTINCT)?
+    (
+      literal_value
+      | column_spec
+      | parameterPlaceHolder
+      | ASTERISK
+    )
+;
+
+parameterPlaceHolder
+:
+  (
+    QUESTION
+    | DOLLAR LCURLY ID? RCURLY
+    | userVariable
+  )
+  -> ^(TOKEN_PARAMPLACEHOLDER)
+;
+
+userVariable
+:
+  '@' ID
+;
+
+table_references
+:
+  table_reference ( COMMA! table_reference )*
+;
+
+table_reference
+:
+  table_atom |
+  table_atom (jointable)+
+;
+
+table_list:
+  table_atom ( COMMA! table_atom )*
+  ;
+
+jointable
+:
+    INNER_SYM JOIN_SYM table_atom join_condition ->  ^( TOKEN_INNER_JOIN table_atom join_condition )
+    |  OUTER JOIN_SYM table_atom join_condition  ->  ^( TOKEN_OUTTER_JOIN table_atom join_condition )
+;
+
+table_factor1
+:
+  table_factor2 (  INNER_SYM JOIN_SYM table_atom join_condition )?
+    -> {$JOIN_SYM != NULL}? ^( table_factor2 TOKEN_INNER_JOIN table_atom join_condition )
+    -> table_factor2
+;
+
+table_factor2
+:
+  table_factor3 ( OUTER JOIN_SYM table_atom join_condition)?
+    -> {$JOIN_SYM != NULL}? ^( table_factor3 TOKEN_OUTTER_JOIN table_atom join_condition )
+    -> table_atom
+;
+
+table_factor3
+:
+  table_atom
+;
+
+table_atom
+:
+  table_spec^ (alias)? (index_hint)?
+;
+
+join_condition
+:
+    ON^ expression
+;
+
+index_hint
+:
+  USE_SYM INDEX_SYM LPAREN ( index_name | u = 'NONE' | l = 'none') RPAREN -> {$u != NULL}? ^(TOKEN_AVOID_INDEX index_name)
+                                                                          -> {$l != NULL}? ^(TOKEN_AVOID_INDEX index_name)
+                                                                          -> ^(TOKEN_INDEX_HINT index_name)
+;
+
+root_statement
+@init
+{
+}
+:
+  ( data_manipulation_statements)(SEMI)? -> ^(TOKEN_ROOT data_manipulation_statements)
+;
+
+data_manipulation_statements
+:
+    select_statement -> ^( TOKEN_SELECT_STATEMENT select_statement)
+  | call_statement -> ^( TOKEN_CALL_STATEMENT call_statement)
+  | create_load_table_statement -> ^( TOKEN_CREATE_LOAD_TABLE_STATEMENT create_load_table_statement)
+  //| create_index_statement -> ^( TOKEN_CREATE_INDEX_STATEMENT create_index_statement)
+;
+
+create_index_statement
+:
+    CREATE_SYM INDEX_SYM index_name ON table_spec column_list
+;
+
+create_load_table_statement
+:
+    create_table_statement
+    (SEMI?)!
+    load_table_statement
+    -> create_table_statement load_table_statement
+;
+
+table_options
+:
+    table_option (( COMMA )? table_option)*
+;
+
+table_option:
+     (  COMMENT_SYM^ (EQ_SYM?)! string_literal  )
+;
+
+create_table_statement
+:
+    CREATE_SYM TABLE_SYM noov=IFNOTEXISTS? table_name create_table_columns_definition
+    (table_options)?
+    -> {$noov != NULL}? ^( TOKEN_CREATE_TABLE table_name TOKEN_DONOT_OVERWRITE table_options? create_table_columns_definition)
+    -> ^( TOKEN_CREATE_TABLE table_name TOKEN_OVERWRITE table_options? create_table_columns_definition)
+;
+
+create_table_columns_definition
+:
+    LPAREN create_definition (COMMA create_definition)* RPAREN -> {$COMMA != NULL}? ^(TOKEN_COLUMN_DEF_LIST create_definition+)
+                                                               -> ^(TOKEN_COLUMN_DEF_LIST create_definition)
+;
+
+create_definition
+:
+       column_name column_definition -> ^(TOKEN_COLUMN_DEF column_name column_definition)
+;
+
+length_and_or_precision_definition
+:
+    LPAREN length ( COMMA number_literal)? RPAREN -> ^(length number_literal?)
+;
+
+length_and_precision_definition
+:
+    LPAREN length COMMA number_literal RPAREN -> ^(length number_literal)
+;
+
+length_definition
+:
+    LPAREN length RPAREN -> ^(length)
+;
+
+text_params
+:
+    LPAREN string_literal (COMMA string_literal)* RPAREN -> ^( TOKEN_PROC_PARAMS string_literal+ )
+;
+column_definition
+:
+   BIT_SYM       length_definition?                                -> ^(BIT_SYM)
+   | TINYINT     length_definition? UNSIGNED_SYM?                  -> ^(TINYINT UNSIGNED_SYM? length_definition?)
+   | SMALLINT    length_definition? UNSIGNED_SYM?                  -> ^(SMALLINT UNSIGNED_SYM? length_definition?)
+   | MEDIUMINT   length_definition? UNSIGNED_SYM?                  -> ^(MEDIUMINT UNSIGNED_SYM? length_definition?)
+   | INT_SYM     length_definition? UNSIGNED_SYM?                  -> ^(INTEGER_SYM UNSIGNED_SYM? length_definition?)
+   | INTEGER_SYM length_definition? UNSIGNED_SYM?                  -> ^(INTEGER_SYM UNSIGNED_SYM? length_definition?)
+   | BIGINT_SYM  length_definition? UNSIGNED_SYM?                  -> ^(BIGINT_SYM UNSIGNED_SYM? length_definition?)
+   | REAL_SYM    length_and_precision_definition?    UNSIGNED_SYM? -> ^(REAL_SYM UNSIGNED_SYM? length_and_precision_definition?)
+   | DOUBLE_SYM  length_and_precision_definition?    UNSIGNED_SYM? -> ^(DOUBLE_SYM UNSIGNED_SYM? length_and_precision_definition?)
+   | FLOAT_SYM   length_and_precision_definition?    UNSIGNED_SYM? -> ^(FLOAT_SYM UNSIGNED_SYM? length_and_precision_definition?)
+   | DECIMAL_SYM length_and_or_precision_definition? UNSIGNED_SYM? -> ^(DECIMAL_SYM UNSIGNED_SYM? length_and_or_precision_definition?)
+   | NUMERIC_SYM length_and_or_precision_definition? UNSIGNED_SYM? -> ^(NUMERIC_SYM UNSIGNED_SYM? length_and_or_precision_definition?)
+   | DATE_SYM
+   | TIME_SYM
+   | TIMESTAMP_SYM
+   | DATETIME_SYM
+   | YEAR_SYM
+   | CHAR_SYM      length_definition? charset_declaration?  -> ^(CHAR_SYM length_definition? charset_declaration?)
+   | VARCHAR_SYM   length_definition  charset_declaration?  -> ^(VARCHAR_SYM length_definition charset_declaration?)
+   | BINARY_SYM    length_definition?                       -> ^(BINARY_SYM length_definition?)
+   | VARBINARY_SYM length_definition                        -> ^(VARBINARY_SYM length_definition)
+   | TINYBLOB_SYM
+   | BLOB_SYM
+   | MEDIUMBLOB_SYM
+   | LONGBLOB_SYM
+   | TINYTEXT_SYM   BINARY_SYM?                      -> ^(TINYTEXT_SYM BINARY_SYM?)
+   | TEXT_SYM       BINARY_SYM? charset_declaration? -> ^(TEXT_SYM BINARY_SYM? charset_declaration?)
+   | MEDIUMTEXT_SYM BINARY_SYM? charset_declaration? -> ^(MEDIUMTEXT_SYM BINARY_SYM? charset_declaration?)
+   | LONGTEXT_SYM   BINARY_SYM? charset_declaration? -> ^(LONGTEXT_SYM BINARY_SYM? charset_declaration?)
+   | ENUM_SYM       text_params charset_declaration? -> ^(ENUM_SYM text_params charset_declaration?)
+   | SET_SYM        text_params charset_declaration? -> ^(SET_SYM text_params charset_declaration?)
+;
+
+charset_declaration
+:
+    CHARACTER_SET^ charset_name
+;
+charset_name
+:
+    ASCII_SYM
+    | UTF8_SYM
+;
+
+file_data_format_type_options
+:
+    LPAREN! (ID EQ_SYM! string_literal) (COMMA! (ID EQ_SYM! string_literal))* RPAREN!
+;
+
+file_data_format_type
+:
+    FLAT_SYM
+    | XML_SYM
+    | CSV_SYM
+    | JSON_SYM
+;
+
+file_data_format_declaration
+:
+    TYPE_SYM file_data_format_type file_data_format_type_options? -> ^(TYPE_SYM file_data_format_type file_data_format_type_options?)
+;
+
+variable_data_line_terminator
+:
+    TERMINATED_SYM BY_SYM t=TEXT_STRING -> ^(TOKEN_VAR_TERMINATOR $t)
+;
+
+variable_data_field_terminator
+:
+    TERMINATED_SYM BY_SYM t=TEXT_STRING -> ^(TOKEN_VAR_SEPERATOR $t)
+;
+
+variable_data_escaped
+:
+    ESCAPED_SYM BY_SYM t=TEXT_STRING -> ^(TOKEN_VAR_ESCAPED $t)
+;
+
+variable_data_enclosed
+:
+    (OPTIONALLY_SYM)? ENCLOSED_SYM BY_SYM t=TEXT_STRING -> ^(TOKEN_VAR_ENCLOSED $t)
+;
+
+variable_data_format_declaration_field
+:
+    (FIELDS_SYM | COLUMNS_SYM)
+    variable_data_field_terminator?
+    variable_data_enclosed?
+    variable_data_escaped?
+
+    -> ^(TOKEN_VARIABLE_FILE variable_data_field_terminator? variable_data_enclosed? variable_data_escaped?)
+;
+
+variable_data_format_declaration_line
+:
+    LINES_SYM
+    //(STARTING BY_SYM TEXT_STRING)?
+    variable_data_line_terminator?
+    -> ^(TOKEN_VARIABLE_FILE variable_data_line_terminator?)
+;
+
+landing_zone_information
+:
+    CONNECTION_SYM EQ_SYM? conn=string_literal
+    DIRECTORY_SYM EQ_SYM? dir=string_literal
+    -> ^(TOKEN_LANDING_ZONE $conn $dir)
+;
+
+load_table_statement
+:
+    LOAD_SYM DATA_SYM
+    landing_zone_information?
+    INFILE_SYM ff=string_literal landing_zone_information? file_data_format_declaration?
+    INTO_SYM TABLE_SYM table_spec
+    variable_data_format_declaration_field?
+    variable_data_format_declaration_line?
+    -> ^( TOKEN_LOAD_TABLE table_spec $ff landing_zone_information? file_data_format_declaration? variable_data_format_declaration_field? variable_data_format_declaration_line?)
+;
+
+select_statement
+:
+        select_portion
+        ( from_portion
+          ( where_clause )?
+          ( groupby_clause )?
+          ( having_clause )?
+        )?
+
+        ( orderby_clause )?
+        ( limit_clause )?
+;
+
+select_portion
+:
+    SELECT^ ( DISTINCT )? select_list
+;
+
+from_portion
+:
+    FROM^ table_references
+;
+
+where_clause
+:
+  WHERE^ expression
+;
+
+groupby_clause
+:
+  GROUP_SYM^ BY_SYM! column_spec (COMMA! column_spec)*
+;
+
+having_clause
+:
+  HAVING^ expression
+;
+
+orderby_clause
+:
+  ORDER_SYM^ BY_SYM! orderby_item (COMMA! orderby_item)*
+;
+
+orderby_item
+:
+  column_spec^ (ASC! | DESC)?
+;
+
+limit_clause
+:
+  LIMIT^ row_count (OFFSET_SYM! offset)?
+;
+
+length
+:
+  INTEGER_NUM
+;
+
+offset
+:
+  INTEGER_NUM
+;
+
+row_count
+:
+  INTEGER_NUM;
+
+select_list
+:
+    select_item
+    (COMMA! select_item)*
+;
+
+select_item
+:
+    column_spec^ (alias)?
+    | string_literal^ (alias)?
+    | number_literal^ (alias)?
+    | hex_literal^ (alias)?
+    | boolean_literal^ (alias)?
+    | function_call^ (alias)?
+    | column_wildcard
+;
+
+column_list
+:
+  LPAREN column_spec (COMMA column_spec)* RPAREN
+;
+
+subquery
+:
+  LPAREN select_statement RPAREN
+;
+
+table_spec
+:
+  schema_spec? table_fork ->  ^(table_fork  schema_spec?)
+;
+
+table_fork
+:
+    (table_name | quoted_table_name)^
+;
+
+schema_spec
+:
+    (ID|quoted_id) DOT -> ^(TOKEN_TABLE_SCHEMA ID? quoted_id?)
+;
+
+column_wildcard
+:
+    (ID DOT)? ASTERISK -> ^(TOKEN_COLUMNWILDCARD ID?)
+;
+
+callParam
+:
+    string_literal
+    | number_literal
+    | hex_literal
+    | boolean_literal
+    | parameterPlaceHolder
+;
+
+call_procedure_name_part
+:
+     (query_set_name DOT)? procedure_name -> ^( TOKEN_PROC_NAME procedure_name query_set_name? )
+;
+
+call_procedure_params
+:
+    callParam ( COMMA callParam)* -> ^( TOKEN_PROC_PARAMS callParam+ )
+;
+
+call_statement
+:
+    CALL_SYM! call_procedure_name_part LPAREN! call_procedure_params? RPAREN!
+;
+

esp/services/ws_sql/SQL2ECL/ECLEngine.cpp

@@ -0,0 +1,941 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#include "ECLEngine.hpp"
+#include <limits>       // std::numeric_limits
+
+const char * ECLEngine::SELECTOUTPUTNAME = "WSSQLSelectQueryResult";
+
+ECLEngine::ECLEngine(){}
+
+ECLEngine::~ECLEngine(){}
+
+void ECLEngine::generateECL(HPCCSQLTreeWalker * sqlobj, StringBuffer & out)
+{
+    if (sqlobj)
+    {
+        switch (sqlobj->getSqlType())
+        {
+            case SQLTypeSelect:
+                generateSelectECL(sqlobj, out);
+                break;
+            case SQLTypeCall:
+                break;
+            case SQLTypeCreateAndLoad:
+                generateCreateAndLoad(sqlobj, out);
+            break;
+            case SQLTypeUnknown:
+            default:
+                break;
+        }
+    }
+}
+
+void ECLEngine::generateIndexSetupAndFetch(HPCCFilePtr file, SQLTable * table, int tableindex, HPCCSQLTreeWalker * selectsqlobj, IProperties* eclEntities)
+{
+    bool isPayloadIndex = false;
+    bool avoidindex = false;
+    StringBuffer indexname;
+
+    if (!table)
+        return;
+
+    const char * tname = table->getName();
+    StringBuffer indexHintFromSQL;
+    if (table->hasIndexHint())
+    {
+        indexHintFromSQL.set(table->getIndexhint());
+        if (strncmp(indexHintFromSQL.trim().str(), "0", 1)==0)
+        {
+            avoidindex = true;
+            WARNLOG("Will not use any index.");
+            return;
+        }
+        else
+            WARNLOG("Empty index hint found!");
+    }
+
+    findAppropriateIndex(file, indexHintFromSQL.str(), selectsqlobj, indexname);
+    if (indexHintFromSQL.length() > 0 && indexname.length() == 0)
+        WARNLOG("Unusable index hint detected.");
+
+    if (indexname.length()>0)
+    {
+        HPCCFilePtr indexfile = dynamic_cast<HPCCFile *>(selectsqlobj->queryHPCCFileCache()->getHpccFileByName(indexname));
+        if (indexfile && file)
+        {
+            StringBuffer idxsetupstr;
+            StringBuffer idxrecdefname;
+
+            idxrecdefname.set("TblDS").append(tableindex).append("RecDef");
+
+            StringBuffer indexPosField;
+            indexPosField.set(indexfile->getIdxFilePosField());
+            HPCCColumnMetaData * poscol = indexfile->getColumn(indexPosField);
+
+            file->getFileRecDefwithIndexpos(poscol, idxsetupstr, idxrecdefname.str());
+            eclEntities->appendProp("INDEXFILERECDEF", idxsetupstr.str());
+
+            StringBuffer keyedAndWild;
+            isPayloadIndex = processIndex(indexfile, keyedAndWild, selectsqlobj);
+
+            if (keyedAndWild.length() > 0)
+                eclEntities->appendProp("KEYEDWILD", keyedAndWild.str());
+
+            if (isPayloadIndex)
+                eclEntities->appendProp("PAYLOADINDEX", "true");
+
+            idxsetupstr.clear();
+            idxsetupstr.appendf("Idx%d := INDEX(TblDS%d, {", tableindex, tableindex);
+            indexfile->getKeyedFieldsAsDelimitedString(',', "", idxsetupstr);
+            idxsetupstr.append("}");
+
+            if (indexfile->getNonKeyedColumnsCount() > 0)
+            {
+                idxsetupstr.append(",{ ");
+                indexfile->getNonKeyedFieldsAsDelmitedString(',', "", idxsetupstr);
+                idxsetupstr.append(" }");
+            }
+
+            //Note, currently '~' is not valid char, if it is ever allowed, we'd have verify that the file name does not lead off with ~
+            idxsetupstr.appendf(",\'~%s\');\n",indexfile->getFullname());
+
+            eclEntities->appendProp("IndexDef", idxsetupstr.str());
+
+            idxsetupstr.clear();
+
+            if (isPayloadIndex)
+            {
+                WARNLOG(" as PAYLOAD");
+                idxsetupstr.appendf("IdxDS%d := Idx%d(%s", tableindex, tableindex, keyedAndWild.str());
+            }
+            else
+            {
+                WARNLOG(" Not as PAYLOAD");
+                idxsetupstr.appendf("IdxDS%d := FETCH(TblDS%d, Idx%d( %s ), RIGHT.%s", tableindex, tableindex, tableindex, keyedAndWild.str(), indexfile->getIdxFilePosField());
+            }
+            idxsetupstr.append(");\n");
+
+
+            eclEntities->appendProp("IndexRead", idxsetupstr.str());
+        }
+        else
+            WARNLOG("NOT USING INDEX!");
+    }
+}
+
+void ECLEngine::generateCreateAndLoad(HPCCSQLTreeWalker * sqlobj, StringBuffer & out)
+{
+        const char * targetTableName = sqlobj->getTableName();
+        if (!targetTableName || !*targetTableName)
+            throw MakeStringException(-1, "Error: TableName cannot be empty.");
+
+        if (!HPCCFile::validateFileName(targetTableName))
+            throw MakeStringException(-1, "Error: Target TableName is invalid: %s.", targetTableName);
+
+        StringBuffer sourceFileName;
+        sourceFileName.set(sqlobj->getSourceDataTableName()).trim();
+
+        StringBuffer landingZoneIP = sqlobj->getLandingZoneIp();
+        if (landingZoneIP.length())
+        {
+            StringBuffer landingZonePath = sqlobj->getLandingZonePath();
+            if (landingZonePath.length())
+            {
+                addPathSepChar(landingZonePath);
+
+                RemoteFilename rfn;
+                SocketEndpoint ep(landingZoneIP);
+
+                rfn.setPath(ep, landingZonePath.append(sourceFileName.str()).str());
+
+                CDfsLogicalFileName dlfn;
+                dlfn.setExternal(rfn);
+                dlfn.get(sourceFileName.clear(), false, false);
+            }
+        }
+
+        out.appendf("import std;\nTABLERECORDDEF := RECORD\n%s\nEND;\n", sqlobj->getRecordDefinition());
+        out.appendf("FILEDATASET := DATASET('~%s', TABLERECORDDEF, %s);\n",sourceFileName.str(), sqlobj->getSourceDataType());
+        out.appendf("OUTPUT(true, NAMED(\'%s\'));\n",  SELECTOUTPUTNAME); //THOR= WU results written to file
+        out.appendf("OUTPUT(COUNT(FILEDATASET), NAMED(\'%sCount\'));\n", SELECTOUTPUTNAME);
+        out.appendf("OUTPUT(FILEDATASET, ,'~%s'%s);\n", targetTableName, sqlobj->isOverwrite() ? ", OVERWRITE" : "");
+
+        const char * description = sqlobj->getComment();
+        if (description && * description)
+            out.appendf("Std.file.setfiledescription('~%s','%s')\n", targetTableName, description);
+}
+
+void ECLEngine::generateSelectECL(HPCCSQLTreeWalker * selectsqlobj, StringBuffer & out)
+{
+    StringBuffer latestDS = "TblDS0";
+
+    Owned<IProperties> eclEntities = createProperties(true);
+    Owned<IProperties> eclDSSourceMapping = createProperties(true);
+    Owned<IProperties> translator = createProperties(true);
+
+    out.clear();
+    out.append("import std;\n"); /* ALL Generated ECL will import std, even if std lib not used */
+
+    //Prepared statement parameters are handled by ECL STORED service workflow statements
+    if (selectsqlobj->hasWhereClause())
+        selectsqlobj->getWhereClause()->eclDeclarePlaceHolders(out, 0,0);
+
+    if (selectsqlobj->hasHavingClause())
+        selectsqlobj->getHavingClause()->eclDeclarePlaceHolders(out, 0,0);
+
+    const IArrayOf<SQLTable> * tables = selectsqlobj->getTableList();
+
+    ForEachItemIn(tableidx, *tables)
+    {
+        SQLTable table = tables->item(tableidx);
+        const char * tname = table.getName();
+
+        HPCCFilePtr file = dynamic_cast<HPCCFile *>(selectsqlobj->queryHPCCFileCache()->getHpccFileByName(tname));
+        if (file)
+        {
+            translator->setProp(tname, "LEFT");
+
+            StringBuffer currntTblDS("TblDS");
+            currntTblDS.append(tableidx);
+
+            StringBuffer currntTblRecDef(currntTblDS);
+            currntTblRecDef.append("RecDef");
+
+            StringBuffer currntJoin("JndDS");
+            currntJoin.append(tableidx);
+
+            out.append("\n");
+            if (tableidx == 0)
+            {
+                //Currently only utilizing index fetch/read for single table queries
+                if ((table.hasIndexHint() || file->getRelatedIndexCount()) && tables->length() == 1 && !file->isFileKeyed())
+                    generateIndexSetupAndFetch(file, &table, tableidx, selectsqlobj, eclEntities);
+
+                if (eclEntities->hasProp("INDEXFILERECDEF"))
+                {
+                    eclDSSourceMapping->appendProp(tname, "IdxDS0");
+                    eclEntities->getProp("INDEXFILERECDEF", out);
+                    latestDS.set("IdxDS0");
+                }
+                else
+                    file->getFileRecDef(out, currntTblRecDef);
+            }
+            else
+                file->getFileRecDef(out, currntTblRecDef);
+            out.append("\n");
+
+            if (!file->isFileKeyed())
+            {
+                out.appendf("%s := DATASET(\'~%s\', %s, %s);\n", currntTblDS.str(), file->getFullname(), currntTblRecDef.str(), file->getFormat());
+            }
+            else
+            {
+                out.appendf("%s := INDEX( {", currntTblDS.str());
+                file->getKeyedFieldsAsDelimitedString(',', currntTblRecDef.str(), out);
+                out.append("},{");
+                file->getNonKeyedFieldsAsDelmitedString(',', currntTblRecDef.str(), out);
+                //Note, currently '~' is not valid char, if it is ever allowed, we'd have verify that the file name does not lead off with ~
+                out.appendf("},\'~%s\');\n",file->getFullname());
+            }
+
+            if (tableidx > 0)
+            {
+                out.append("\n").append(currntJoin).append(" := JOIN( ");
+                translator->setProp(tname, "RIGHT");
+
+                if (tableidx == 1)
+                {
+                    //First Join, previous DS is TblDS0
+                    out.append("TblDS0");
+                    latestDS.set("JndDS1");
+                }
+                else
+                {
+                    //Nth Join, previous DS is JndDS(N-1)
+                    out.appendf("JndDS%d",tableidx-1);
+                    latestDS.setf("JndDS%d",tableidx);
+                }
+
+                StringBuffer translatedAndFilteredOnClause;
+                SQLJoin * tablejoin = table.getJoin();
+                if (tablejoin && tablejoin->doesHaveOnclause())
+                {
+                    tablejoin->getOnClause()->toECLStringTranslateSource(translatedAndFilteredOnClause, translator,true, false, false, false);
+                    if (selectsqlobj->hasWhereClause())
+                    {
+                        translatedAndFilteredOnClause.append(" AND ");
+                        selectsqlobj->getWhereClause()->toECLStringTranslateSource(translatedAndFilteredOnClause, translator, true, true, false, false);
+                    }
+                }
+                else if ( tablejoin && tablejoin->getType() == SQLJoinTypeImplicit && selectsqlobj->hasWhereClause())
+                {
+                    if (translatedAndFilteredOnClause.length() > 0)
+                        translatedAndFilteredOnClause.append(" AND ");
+                    selectsqlobj->getWhereClause()->toECLStringTranslateSource(translatedAndFilteredOnClause, translator, true, true, false, false);
+                }
+                else
+                    throw MakeStringException(-1,"No join condition between tables %s, and earlier table", tname);
+
+                if (translatedAndFilteredOnClause.length() <= 0)
+                    throw MakeStringException(-1,"Join condition does not contain proper join condition between tables %s, and earlier table", tname);
+
+                out.appendf(", %s, %s, ", currntTblDS.str(), translatedAndFilteredOnClause.length() > 0 ? translatedAndFilteredOnClause.str() : "TRUE");
+                tablejoin->getECLTypeStr(out);
+
+                if (tablejoin->getOnClause() != NULL && !tablejoin->getOnClause()->containsEqualityCondition(translator, "LEFT", "RIGHT"))
+                {
+                    WARNLOG("Warning: No Join EQUALITY CONDITION detected!, using ECL ALL option");
+                    out.append(", ALL");
+                }
+
+                out.append(" );\n");
+
+                //move this file to LEFT for possible next iteration
+                translator->setProp(tname, "LEFT");
+            }
+            eclEntities->setProp("JoinQuery", "1");
+        }
+    }
+
+    int limit=selectsqlobj->getLimit();
+    int offset=selectsqlobj->getOffset();
+
+    if (!eclEntities->hasProp("IndexDef"))
+    {
+        //Create filtered DS if there's a where clause, and no join clause,
+        //because filtering is applied while performing join.
+        //if (sqlParser.getWhereClause() != null && !eclEntities.containsKey("JoinQuery"))
+        if (selectsqlobj->hasWhereClause())
+        {
+            out.appendf("%sFiltered := %s",latestDS.str(), latestDS.str());
+            addFilterClause(selectsqlobj, out);
+            out.append(";\n");
+            latestDS.append("Filtered");
+        }
+
+        generateSelectStruct(selectsqlobj, eclEntities.getLink(), *selectsqlobj->getSelectList(),latestDS.str());
+        out.append(eclEntities->queryProp("SELECTSTRUCT"));
+
+        if (tables->length() > 0)
+        {
+            out.append(latestDS).append("Table").append(" := TABLE( ");
+            out.append(latestDS);
+
+            out.append(", SelectStruct ");
+
+            // If group by contains HAVING clause, use ECL 'HAVING' function,
+            // otherwise group can be done implicitly in table step.
+            // since the implicit approach has better performance.
+            if (selectsqlobj->hasGroupByColumns() && !selectsqlobj->hasHavingClause())
+            {
+                out.append(", ");
+                selectsqlobj->getGroupByString(out);
+                out.append(" /*grouped by this field*/");
+            }
+
+            out.append(");\n");
+            latestDS.append("Table");
+
+            if (selectsqlobj->hasGroupByColumns() && selectsqlobj->hasHavingClause())
+            {
+                out.appendf("%sGrouped := GROUP( %s, ", latestDS.str(), latestDS.str());
+                selectsqlobj->getGroupByString(out);
+                out.append(", ALL);\n");
+
+                latestDS.append("Grouped");
+
+                if (appendTranslatedHavingClause(selectsqlobj, out, latestDS.str()))
+                   latestDS.append("Having");
+            }
+        }
+        else
+        {
+            generateConstSelectDataset(selectsqlobj, eclEntities.getLink(), *selectsqlobj->getSelectList(),latestDS.str());
+            out.append(latestDS).append(" := ").append(eclEntities->queryProp("CONSTDATASETSTRUCT")).append(";\n");
+        }
+    }
+    else //PROCESSING FOR INDEX BASED FETCH
+    {
+        eclEntities->getProp("IndexDef",out);
+        eclEntities->getProp("IndexRead",out);
+
+        // If group by contains HAVING clause, use ECL 'HAVING' function,
+        // otherwise group can be done implicitly in table step.
+        // since the implicit approach has better performance.
+        if (selectsqlobj->hasGroupByColumns() && selectsqlobj->hasHavingClause())
+        {
+            out.appendf("%sGrouped := GROUP( %s, ", latestDS.str(), latestDS.str());
+            selectsqlobj->getGroupByString(out);
+            out.append(", ALL);\n");
+
+            latestDS.append("Grouped");
+
+            if (appendTranslatedHavingClause(selectsqlobj, out, latestDS.str()))
+                latestDS.append("Having");
+        }
+
+        generateSelectStruct(selectsqlobj, eclEntities.get(), *selectsqlobj->getSelectList(),latestDS.str());
+
+        out.append(eclEntities->queryProp("SELECTSTRUCT"));
+        out.appendf("%sTable := TABLE(%s",latestDS.str(), latestDS.str());
+
+        //Filtering all non-payload index, because the original filter applied at fetch could
+        //be incomplete due to non-keyed fields
+        if (!eclEntities->hasProp("PAYLOADINDEX"))
+        {
+            addFilterClause(selectsqlobj, out);
+        }
+
+        out.append(", SelectStruct ");
+
+        if (selectsqlobj->hasGroupByColumns() && !selectsqlobj->hasHavingClause())
+        {
+            out.append(", ");
+            selectsqlobj->getGroupByString(out);
+            out.append(" /*grouped by this field*/");
+        }
+        out.append(");\n");
+        latestDS.append("Table");
+    }
+
+    if (selectsqlobj->isSelectDistinct())
+    {
+        out.appendf("%sDeduped := Dedup( %s, HASH);\n",latestDS.str(),latestDS.str());
+        latestDS.append("Deduped");
+    }
+
+    out.appendf("%sOut := CHOOSEN(", latestDS.str());
+
+    if (selectsqlobj->hasOrderByColumns())
+        out.append("SORT(");
+
+    out.append(latestDS);
+    if (selectsqlobj->hasOrderByColumns())
+    {
+        out.append(",");
+        selectsqlobj->getOrderByString(out);
+        out.append(")");
+    }
+    latestDS.append("Out");
+
+    if (!eclEntities->hasProp("NONSCALAREXPECTED") && !selectsqlobj->hasGroupByColumns())
+    {
+        out.append(", 1);\n");
+    }
+    else
+    {
+        out.append(",");
+        if (limit>0)
+        {
+            out.append(limit);
+            if (offset>0)
+            {
+                out.append(",");
+                out.append(offset);
+            }
+        }
+        else
+            out.append("ALL");
+
+        out.append(");\n");
+    }
+
+    out.appendf("OUTPUT(%s, NAMED(\'%s\'), THOR);\n", latestDS.str(), SELECTOUTPUTNAME); //THOR= WU results written to file
+    out.appendf("OUTPUT( COUNT(%s), NAMED(\'%sCount\'));\n", latestDS.str(), SELECTOUTPUTNAME);
+}
+
+void ECLEngine::generateConstSelectDataset(HPCCSQLTreeWalker * selectsqlobj, IProperties* eclEntities,  const IArrayOf<ISQLExpression> & expectedcolumns, const char * datasource)
+{
+    StringBuffer datasetStructSB = "DATASET([{ ";
+
+    ForEachItemIn(i, expectedcolumns)
+    {
+        ISQLExpression * col = &expectedcolumns.item(i);
+        col->toString(datasetStructSB, true);
+        if (i < expectedcolumns.length()-1)
+            datasetStructSB.append(", ");
+    }
+
+    datasetStructSB.append("}],SelectStruct)");
+
+    eclEntities->setProp("CONSTDATASETSTRUCT", datasetStructSB.str());
+}
+
+void ECLEngine::generateSelectStruct(HPCCSQLTreeWalker * selectsqlobj, IProperties* eclEntities,  const IArrayOf<ISQLExpression> & expectedcolumns, const char * datasource)
+{
+    StringBuffer selectStructSB = "SelectStruct := RECORD\n";
+
+    ForEachItemIn(i, expectedcolumns)
+    {
+        selectStructSB.append(" ");
+        ISQLExpression * col = &expectedcolumns.item(i);
+
+        if (col->getExpType() == Value_ExpressionType)
+        {
+            const char * alias = col->getAlias();
+            if (alias && *alias)
+                selectStructSB.appendf("%s %s := ", col->getECLType(), alias);
+            else
+                selectStructSB.appendf("%s %s%d := ", col->getECLType(), col->getName(), i);
+            col->toString(selectStructSB, false);
+            selectStructSB.append("; ");
+
+            if (i == 0 && expectedcolumns.length() == 1)
+                eclEntities->setProp("SCALAROUTNAME", col->getNameOrAlias());
+        }
+        else if (col->getExpType() == Function_ExpressionType)
+        {
+            SQLFunctionExpression * funcexp = dynamic_cast<SQLFunctionExpression *>(col);
+            IArrayOf<ISQLExpression> * funccols = funcexp->getParams();
+
+            ECLFunctionDefCfg func = ECLFunctions::getEclFuntionDef(funcexp->getName());
+
+            if (func.functionType == CONTENT_MODIFIER_FUNCTION_TYPE )
+            {
+                if (funccols->length() > 0)
+                {
+                    ISQLExpression * param = &funccols->item(0);
+                    int paramtype = param->getExpType();
+
+                    const char * alias = col->getAlias();
+                    if (alias && *alias)
+                        selectStructSB.append(alias);
+                    else
+                        selectStructSB.append(param->getName());
+                    selectStructSB.append(" := ");
+                    selectStructSB.append(func.eclFunctionName).append("( ");
+                    if (paramtype == FieldValue_ExpressionType)
+                    {
+                        eclEntities->setProp("NONSCALAREXPECTED", "TRUE");
+
+                        selectStructSB.append(datasource);
+                        selectStructSB.append(".");
+                        selectStructSB.append(param->getNameOrAlias());
+                    }
+                    else
+                        param->toString(selectStructSB, false);
+                }
+            }
+            else
+            {
+                const char * alias = col->getAlias();
+                if (alias && *alias)
+                    selectStructSB.append(alias);
+                else
+                {
+                    selectStructSB.append(col->getName());
+                    selectStructSB.append("out");
+                    selectStructSB.append(i+1);
+                }
+                selectStructSB.append(" := ");
+
+                selectStructSB.append(func.eclFunctionName).append("( ");
+
+                if (selectsqlobj->hasGroupByColumns())
+                {
+                    selectStructSB.append("GROUP ");
+                }
+                else
+                {
+                    if (funcexp->isDistinct())
+                    {
+                        selectStructSB.append("DEDUP( ");
+                        selectStructSB.append(datasource);
+                        addFilterClause(selectsqlobj, selectStructSB);
+
+                        for (int j = 0; j < funccols->length(); j++)
+                        {
+                            StringBuffer paramname = funccols->item(j).getName();
+                            selectStructSB.append(", ");
+                            selectStructSB.append(paramname);
+                        }
+                        selectStructSB.append(", HASH)");
+                    }
+                    else
+                    {
+                        selectStructSB.append(datasource);
+                        addFilterClause(selectsqlobj, selectStructSB);
+                    }
+                }
+
+                if ((strcmp(func.name,"COUNT"))!=0  && funccols->length() > 0)
+                {
+
+                    ISQLExpression &funccol = funccols->item(0);
+                    const char * paramname = funccol.getName();
+                    if (paramname && paramname[0]!='*')
+                    {
+                        selectStructSB.append(", ");
+                        if (funccol.getExpType() != Value_ExpressionType)
+                        {
+                            selectStructSB.append(datasource);
+                            selectStructSB.append(".");
+                            selectStructSB.append(paramname);
+                        }
+                        else
+                        {
+                            funccol.toString(selectStructSB, false);
+                        }
+                    }
+                }
+            }
+
+            //AS OF community_3.8.6-4 this is causing error:
+            // (0,0): error C3000: assert(!cond) failed - file: /var/jenkins/workspace/<build number>/HPCC-Platform/ecl/hqlcpp/hqlhtcpp.cpp, line XXXXX
+            //Bug reported: https://track.hpccsystems.com/browse/HPCC-8268
+            //Leaving this code out until fix is produced.
+            //UPDATE: Issue has been resolved as of 3.10.0
+
+            //RODRIGO below if condition not completed yet
+            //if (eclEntities.containsKey("PAYLOADINDEX") && !sqlParser.hasGroupByColumns() && !col.isDistinct())
+            if (false && !selectsqlobj->hasGroupByColumns() && !funcexp->isDistinct())
+            {
+                    selectStructSB.append(", KEYED");
+            }
+
+            selectStructSB.append(" );");
+        }
+        else
+        {
+            eclEntities->setProp("NONSCALAREXPECTED", "TRUE");
+            selectStructSB.appendf("%s %s := %s.%s;", col->getECLType(), col->getNameOrAlias(), datasource, col->getName());
+        }
+
+        selectStructSB.append("\n");
+    }
+    selectStructSB.append("END;\n");
+
+    eclEntities->setProp("SELECTSTRUCT", selectStructSB.str());
+}
+
+bool containsPayload(const HPCCFile * indexfiletotest, const HPCCSQLTreeWalker * selectsqlobj)
+{
+    if (selectsqlobj)
+    {
+        const IArrayOf <ISQLExpression> * selectlist = selectsqlobj->getSelectList();
+
+        for (int j = 0; j < selectlist->length(); j++)
+        {
+            ISQLExpression * exp = &selectlist->item(j);
+            if (exp->getExpType() == FieldValue_ExpressionType)
+            {
+                SQLFieldValueExpression * currentselectcol = dynamic_cast<SQLFieldValueExpression *>(exp);
+                if (!indexfiletotest->containsField(currentselectcol->queryField(), true))
+                    return false;
+            }
+            else if (exp->getExpType() == Function_ExpressionType)
+            {
+                SQLFunctionExpression * currentfunccol = dynamic_cast<SQLFunctionExpression *>(exp);
+
+                IArrayOf<ISQLExpression> * funcparams = currentfunccol->getParams();
+                ForEachItemIn(paramidx, *funcparams)
+                {
+                    ISQLExpression * param = &(funcparams->item(paramidx));
+                    if (param->getExpType() == FieldValue_ExpressionType)
+                    {
+                        SQLFieldValueExpression * currentselectcol = dynamic_cast<SQLFieldValueExpression *>(param);
+                        if (!indexfiletotest->containsField(currentselectcol->queryField(), true))
+                            return false;
+                    }
+                }
+            }
+        }
+        return true;
+    }
+    return false;
+}
+
+bool ECLEngine::processIndex(HPCCFile * indexfiletouse, StringBuffer & keyedandwild, HPCCSQLTreeWalker * selectsqlobj)
+{
+    ISQLExpression * whereclause = selectsqlobj->getWhereClause();
+
+    if (!whereclause)
+        return false;
+
+    bool isPayloadIndex = containsPayload(indexfiletouse, selectsqlobj);
+
+    StringArray keyed;
+    StringArray wild;
+
+    StringArray filterclauseuniquenames;
+    whereclause->getUniqueExpressionColumnNames(filterclauseuniquenames);
+    bool allfieldsinfilterexistinindexfile = true;
+
+    // Create keyed and wild string
+    IArrayOf<HPCCColumnMetaData> * indexfilecolumns = indexfiletouse->getColumns();
+
+    //We need to know if the filter clause contains only fields which exist in the index file
+    ForEachItemIn(uniquenamesidx, filterclauseuniquenames)
+    {
+        bool currfilterfieldexistsinindexfile = false;
+        for (int indexfilecolumnindex = 0; indexfilecolumnindex < indexfilecolumns->length(); indexfilecolumnindex++)
+        {
+            HPCCColumnMetaData currcol = indexfilecolumns->item(indexfilecolumnindex);
+            const char * currindexfilecolname = currcol.getColumnName();
+            if(stricmp( filterclauseuniquenames.item(uniquenamesidx), currindexfilecolname)==0)
+            {
+                currfilterfieldexistsinindexfile = true;
+                break;
+            }
+        }
+
+        if (!currfilterfieldexistsinindexfile)
+        {
+            allfieldsinfilterexistinindexfile = false;
+            break;
+        }
+    }
+
+    if (allfieldsinfilterexistinindexfile)
+    {
+        for (int indexfilecolumnindex = 0; indexfilecolumnindex < indexfilecolumns->length(); indexfilecolumnindex++)
+        {
+            HPCCColumnMetaData currcol = indexfilecolumns->item(indexfilecolumnindex);
+            const char * currindexfilecolname = currcol.getColumnName();
+            if (currcol.isKeyedField())
+            {
+                StringBuffer keyedorwild;
+                whereclause->getExpressionFromColumnName(currindexfilecolname, keyedorwild);
+                if (whereclause->containsKey(currindexfilecolname) && keyedorwild.length())
+                {
+                    keyed.append(keyedorwild.str());
+                }
+                else
+                {
+                    keyedorwild.setf(" %s ", currindexfilecolname);
+                    wild.append(keyedorwild.str());
+                }
+            }
+        }
+
+        int keyedlen = keyed.length();
+        if (keyedlen > 0)
+        {
+            for (int keyedi = 0; keyedi < keyedlen; keyedi++)
+            {
+                keyedandwild.append(" KEYED( ");
+                keyedandwild.append(keyed.item(keyedi));
+
+                keyedandwild.append(" )");
+                if (keyedi < keyedlen - 1)
+                    keyedandwild.append(", ");
+            }
+            if (wild.length() > 0)
+            {
+                for (int wildi = 0; wildi < wild.length(); wildi++)
+                {
+                    if (keyedlen || wildi > 0)
+                        keyedandwild.append(" AND ");
+                    keyedandwild.appendf("WILD( %s )", wild.item(wildi));
+                }
+            }
+
+            keyedandwild.append(" AND ");
+        }
+        keyedandwild.append(" (");
+        whereclause->toString(keyedandwild, false);
+        keyedandwild.append(") ");
+    }
+    /*
+     * Even though filtering at the fetch is preferable,
+     * the filter can only reference keyed/non-keyed fields from the index file
+     * it is not always feasible to extract the algebraic condition based only on those fields
+     * else {}
+     *
+     */
+
+    //if the filter condition contains field not in the payload index file, we cannot do a payload read
+    return isPayloadIndex && allfieldsinfilterexistinindexfile;
+}
+
+void ECLEngine::findAppropriateIndex(HPCCFilePtr file, const char * indexhint, HPCCSQLTreeWalker * selectsqlobj, StringBuffer & indexname)
+{
+    StringArray indexhints;
+    if (indexhint && *indexhint)
+        indexhints.append(indexhint);
+
+    if (file)
+        file->getRelatedIndexes(indexhints);
+
+    findAppropriateIndex(&indexhints, selectsqlobj, indexname);
+}
+
+void ECLEngine::findAppropriateIndex(StringArray * relindexes, HPCCSQLTreeWalker * selectsqlobj, StringBuffer & indexname)
+{
+    StringArray uniquenames;
+    ISQLExpression * whereclause = selectsqlobj->getWhereClause();
+
+    if (whereclause)
+        whereclause->getUniqueExpressionColumnNames(uniquenames);
+    else
+        return;
+
+    int totalparamcount = uniquenames.length();
+
+    if (relindexes->length() <= 0 || totalparamcount <= 0)
+        return;
+
+    bool payloadIdxWithAtLeast1KeyedFieldFound = false;
+
+    IntArray scores;
+    for (int indexcounter = 0; indexcounter < relindexes->length(); indexcounter++)
+    {
+        scores.add(std::numeric_limits<int>::min(), indexcounter);
+
+        const char * indexname = relindexes->item(indexcounter);
+        if (!selectsqlobj->queryHPCCFileCache()->isHpccFileCached(indexname))
+            selectsqlobj->queryHPCCFileCache()->cacheHpccFileByName(indexname);
+
+        HPCCFilePtr indexfile = dynamic_cast<HPCCFile *>(selectsqlobj->queryHPCCFileCache()->getHpccFileByName(indexname));
+
+        if (indexfile)
+        {
+            const IArrayOf<ISQLExpression> * expectedretcolumns =selectsqlobj->getSelectList();
+            if (indexfile && indexfile->isFileKeyed() && indexfile->hasValidIdxFilePosField())
+            {
+                //The more fields this index has in common with the select columns higher score
+                int commonparamscount = 0;
+                for (int j = 0; j < expectedretcolumns->length(); j++)
+                {
+                    ISQLExpression * exp = &expectedretcolumns->item(j);
+                    if (exp->getExpType() == FieldValue_ExpressionType)
+                    {
+                        SQLFieldValueExpression * fieldexp = dynamic_cast<SQLFieldValueExpression *>(exp);
+                        if (indexfile->containsField(fieldexp->queryField(), true))
+                            commonparamscount++;
+                    }
+                    else if (exp->getExpType() == Function_ExpressionType)
+                    {
+                        SQLFunctionExpression * currentfunccol = dynamic_cast<SQLFunctionExpression *>(exp);
+
+                        IArrayOf<ISQLExpression> * funcparams = currentfunccol->getParams();
+                        ForEachItemIn(paramidx, *funcparams)
+                        {
+                            ISQLExpression * param = &(funcparams->item(paramidx));
+                            if (param->getExpType() == FieldValue_ExpressionType)
+                            {
+                                SQLFieldValueExpression * currentselectcol = dynamic_cast<SQLFieldValueExpression *>(param);
+                                if (indexfile->containsField(currentselectcol->queryField(), true))
+                                    commonparamscount++;
+                            }
+                        }
+                    }
+                }
+                int commonparamsscore = commonparamscount * NumberOfCommonParamInThisIndex_WEIGHT;
+                scores.replace(commonparamsscore, indexcounter);
+
+                if (payloadIdxWithAtLeast1KeyedFieldFound && commonparamscount == 0)
+                    break; // Don't bother with this index
+
+                //The more keyed fields this index has in common with the where clause, the higher score
+                //int localleftmostindex = -1;
+                int keycolscount = 0;
+                IArrayOf<HPCCColumnMetaData> * columns = indexfile->getColumns();
+                ForEachItemIn(colidx, *columns)
+                {
+                    HPCCColumnMetaData currcol = columns->item(colidx);
+                    if (currcol.isKeyedField())
+                    {
+                        ForEachItemIn(uniqueidx, uniquenames)
+                        {
+                            if(strcmp( uniquenames.item(uniqueidx), currcol.getColumnName())==0)
+                                keycolscount++;
+                        }
+                    }
+                }
+
+                if (keycolscount == 0)
+                {
+                    scores.replace(std::numeric_limits<int>::min(), indexcounter);
+                    continue;
+                }
+
+                int keycolsscore = keycolscount * NumberofColsKeyedInThisIndex_WEIGHT;
+                scores.replace(keycolsscore + scores.item(indexcounter), indexcounter);
+                if (commonparamscount == expectedretcolumns->length() && keycolscount > 0)
+                        payloadIdxWithAtLeast1KeyedFieldFound = true; // during scoring, give this priority
+            }
+        }
+    }
+
+    int highscore = std::numeric_limits<int>::min();
+    int highscoreidx = -1;
+    for (int i = 0; i < scores.length(); i++)
+    {
+        if (highscore < scores.item(i))
+        {
+           highscore = scores.item(i);
+           highscoreidx = i;
+        }
+    }
+    if (highscoreidx != -1 && highscoreidx < relindexes->length())
+        indexname.set(relindexes->item(highscoreidx));
+}
+
+void ECLEngine::addFilterClause(HPCCSQLTreeWalker * sqlobj, StringBuffer & sb)
+{
+    if (sqlobj->hasWhereClause())
+    {
+        StringBuffer where;
+        sqlobj->getWhereClauseString(where);
+        if (where.length()>0)
+        {
+            sb.append("( ").append(where.str()).append(" )");
+        }
+    }
+}
+
+void ECLEngine::addHavingCluse(HPCCSQLTreeWalker * sqlobj, StringBuffer & sb)
+{
+    StringBuffer having;
+    sqlobj->getHavingClauseString(having);
+    if (having.length()>0)
+    {
+        sb.append("( ").append(having.str()).append(" )");
+    }
+}
+
+bool ECLEngine::appendTranslatedHavingClause(HPCCSQLTreeWalker * sqlobj, StringBuffer & sb, const char * latesDSName)
+{
+    bool success = false;
+    if (sqlobj)
+    {
+        if (sqlobj->hasHavingClause())
+        {
+            Owned<IProperties> translator = createProperties(true);
+
+            const IArrayOf<SQLTable> * tables = sqlobj->getTableList();
+            ForEachItemIn(tableidx, *tables)
+            {
+                SQLTable table = tables->item(tableidx);
+                translator->appendProp(table.getName(), "LEFT");
+            }
+
+            ISQLExpression * having = sqlobj->getHavingClause();
+            StringBuffer havingclause;
+            having->toECLStringTranslateSource(havingclause, translator, false, true, false, false);
+
+            if (havingclause.length() > 0)
+            {
+                sb.appendf("%sHaving := HAVING( %s, %s );\n", latesDSName, latesDSName, havingclause.str());
+            }
+            success = true;
+        }
+    }
+    return success;
+}

esp/services/ws_sql/SQL2ECL/ECLEngine.hpp

@@ -0,0 +1,56 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#ifndef ECLENGINE_HPP_
+#define ECLENGINE_HPP_
+
+#include "HPCCSQLTreeWalker.hpp"
+#include "HPCCFile.hpp"
+#include "SQLColumn.hpp"
+#include "dautils.hpp"
+
+#define NumberOfCommonParamInThisIndex_WEIGHT    5
+#define NumberofColsKeyedInThisIndex_WEIGHT      2
+
+class ECLEngine
+{
+public:
+    ECLEngine();
+    virtual ~ECLEngine();
+
+    static void generateECL(HPCCSQLTreeWalker * sqlobj, StringBuffer & out);
+
+private:
+    static void generateSelectECL(HPCCSQLTreeWalker * selectsqlobj, StringBuffer & out);
+    static void generateCreateAndLoad(HPCCSQLTreeWalker * sqlobj, StringBuffer & out);
+    //static void generateCallECL(HPCCSQLTreeWalker * callsqlobj, StringBuffer & out);
+
+    static void findAppropriateIndex(HPCCFilePtr file, const char * indexhint, HPCCSQLTreeWalker * selectsqlobj, StringBuffer & indexname);
+    static void findAppropriateIndex(StringArray * relindexes, HPCCSQLTreeWalker * selectsqlobj, StringBuffer & indexname);
+    static bool processIndex(HPCCFile * indexfiletouse, StringBuffer & keyedandwild, HPCCSQLTreeWalker * selectsqlobj);
+
+    static void generateConstSelectDataset(HPCCSQLTreeWalker * selectsqlobj, IProperties* eclEntities,  const IArrayOf<ISQLExpression> & expectedcolumns, const char * datasource);
+    static void generateSelectStruct(HPCCSQLTreeWalker * selectsqlobj, IProperties* eclEntities, const IArrayOf<ISQLExpression> & expectedcolumns, const char * datasource);
+    static void addFilterClause(HPCCSQLTreeWalker * sqlobj, StringBuffer & sb);
+    static void addHavingCluse(HPCCSQLTreeWalker * sqlobj, StringBuffer & sb);
+    static bool appendTranslatedHavingClause(HPCCSQLTreeWalker * sqlobj, StringBuffer & sb, const char * latesDSName);
+    static void generateIndexSetupAndFetch(HPCCFilePtr file, SQLTable * table, int tableindex, HPCCSQLTreeWalker * selectsqlobj, IProperties* eclEntities);
+
+    static const char * SELECTOUTPUTNAME;
+};
+
+#endif /* ECLENGINE_HPP_ */

esp/services/ws_sql/SQL2ECL/ECLFunction.cpp

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#include "ECLFunction.hpp"
+
+bool ECLFunctions::funcsinited = false;
+std::map<std::string,ECLFunctionDefCfg> ECLFunctions::eclfuncstable;
+
+void ECLFunctions::init()
+{
+    if (!funcsinited)
+    {
+        eclfuncstable.insert(std::pair<std::string,ECLFunctionDefCfg>(COUNT_FUNCTION, ECLFunctionDefCfg( COUNT_FUNCTION, true,  "INTEGER", true,  true,  AGGREGATE_FUNCTION_TYPE,        COUNT_FUNCTION, false)));
+        eclfuncstable.insert(std::pair<std::string,ECLFunctionDefCfg>(MAX_FUNCTION,   ECLFunctionDefCfg(MAX_FUNCTION, true,  "NUMERIC",   false, false, AGGREGATE_FUNCTION_TYPE,                                            MAX_FUNCTION, true)));
+        eclfuncstable.insert(std::pair<std::string,ECLFunctionDefCfg>( MIN_FUNCTION,  ECLFunctionDefCfg(MIN_FUNCTION, true,  "NUMERIC",   false, false, AGGREGATE_FUNCTION_TYPE,        MIN_FUNCTION, true)));
+        eclfuncstable.insert(std::pair<std::string,ECLFunctionDefCfg>(SUM_FUNCTION,   ECLFunctionDefCfg(SUM_FUNCTION, true,  "NUMERIC",   false, false, AGGREGATE_FUNCTION_TYPE,        SUM_FUNCTION, true)));
+        eclfuncstable.insert(std::pair<std::string,ECLFunctionDefCfg>(AVG_FUNCTION,   ECLFunctionDefCfg(AVG_FUNCTION, true,  "DECIMAL",   false, false, AGGREGATE_FUNCTION_TYPE,        "AVE", false)));
+        eclfuncstable.insert(std::pair<std::string,ECLFunctionDefCfg>(UPPER_FUNCTION, ECLFunctionDefCfg(UPPER_FUNCTION, false, "STRING",            false, false, CONTENT_MODIFIER_FUNCTION_TYPE, "Std.Str.ToUpperCase", true)));
+        eclfuncstable.insert(std::pair<std::string,ECLFunctionDefCfg>(LOWER_FUNCTION, ECLFunctionDefCfg(LOWER_FUNCTION, false, "STRING",            false, false, CONTENT_MODIFIER_FUNCTION_TYPE, "Std.Str.ToLowerCase", true)));
+
+        funcsinited = true;
+    }
+}

esp/services/ws_sql/SQL2ECL/ECLFunction.hpp

@@ -0,0 +1,118 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#ifndef ECLFUNCTION_HPP_
+#define ECLFUNCTION_HPP_
+
+#include "ws_sql.hpp"
+#include "ws_sql_esp.ipp"
+
+#define AGGREGATE_FUNCTION_TYPE 1
+#define CONTENT_MODIFIER_FUNCTION_TYPE 2
+
+#define COUNT_FUNCTION "COUNT"
+#define MAX_FUNCTION "MAX"
+#define MIN_FUNCTION "MIN"
+#define SUM_FUNCTION "SUM"
+#define AVG_FUNCTION "AVG"
+#define UPPER_FUNCTION "UPPER"
+#define LOWER_FUNCTION "LOWER"
+
+struct ECLFunctionDefCfg
+{
+    const char * name;
+    bool acceptsWildCard;
+    const char * returnType;
+    bool acceptsMultipleInputs;
+    bool acceptsEmptyInput;
+    int functionType;
+    const char * eclFunctionName;
+    bool returnsSameAsArgumentType;
+
+    ECLFunctionDefCfg()
+    {
+        name = "";
+        acceptsWildCard = false;
+        returnType = "";
+        acceptsMultipleInputs = false;
+        acceptsEmptyInput = false;
+        functionType = -1;
+        eclFunctionName = "";
+        returnsSameAsArgumentType = false;
+    }
+    ECLFunctionDefCfg( const char * n, bool awc, const char * rt, bool ami, bool aei, int ft, const char * efn, bool rsaat)
+    {
+        name = n;
+        acceptsWildCard = awc;
+        returnType = rt;
+        acceptsMultipleInputs = ami;
+        acceptsEmptyInput = aei;
+        functionType = ft;
+        eclFunctionName = efn;
+        returnsSameAsArgumentType = rsaat;
+    }
+
+};
+
+/*
+typedef enum ECLFunctionDefs_
+{
+    UnknownECLFunc  = -1,
+    CountECLFunc    = 0,
+    MaxECLFunc      = 1,
+    MinECLFunc      = 2,
+    SumECLFunc      = 3,
+    AvgECLFunc      = 4,
+    UpperECLFunc    = 5,
+    LowerECLFunc    = 6,
+    ECLFuncMax      = 7
+} EclFunctionDef;
+
+*/
+
+class ECLFunctions
+{
+
+public:
+    ECLFunctions();
+    virtual ~ECLFunctions();
+
+    static void init();
+
+    static ECLFunctionDefCfg getEclFuntionDef(const char * funcname)
+    {
+        if (!funcsinited)
+        {
+            init();
+        }
+
+        if (funcname && strlen(funcname)>0)
+        {
+            StringBuffer fnnameupper = funcname;
+
+            if (eclfuncstable.count(fnnameupper.toUpperCase().str()))
+                return  eclfuncstable.find(fnnameupper.toUpperCase().str())->second;
+        }
+
+        throw MakeStringException(-1, "Invalid ECL function: %s", funcname);
+    }
+
+    static bool funcsinited;
+    static std::map<std::string,ECLFunctionDefCfg> eclfuncstable;
+};
+
+#endif /* ECLFUNCTION_HPP_ */

esp/services/ws_sql/SQL2ECL/HPCCFile.cpp

@@ -0,0 +1,284 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#include "HPCCFile.hpp"
+
+HPCCFileFormat HPCCFile::DEFAULTFORMAT = HPCCFileFormatFlat;
+
+HPCCFile * HPCCFile::createHPCCFile()
+{
+    return new HPCCFile();
+}
+
+HPCCFile::HPCCFile()
+{
+    iskeyfile=false;
+    issuperfile=false;
+    keyedCount = -1;
+    nonKeyedCount = -1;
+    hasNestedColumns = false;
+}
+
+HPCCFile::~HPCCFile()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "\nLeaving HPCCFile\n\t");
+#endif
+
+    columns.kill(false);
+}
+
+bool HPCCFile::validateFileName(const char * fullname)
+{
+    if (!fullname || !*fullname)
+        return false;
+
+    int len = strlen(fullname);
+    for (int strindex = 0; strindex < len; strindex++)
+    {
+        switch (fullname[strindex])
+        {
+            case '\'':
+            case '\"':
+                return false;
+            case '~':
+                if (strindex > 0)
+                    return false;
+                break;
+            default:
+                break;
+        }
+    }
+
+    return true;
+}
+
+bool HPCCFile::getFileRecDef(StringBuffer & out, const char * structname, const char * linedelimiter, const char * recordindent)
+{
+    if (this->columns.length() > 0)
+    {
+        out.append(structname);
+        out.append(" := RECORD ");
+        out.append(linedelimiter);
+
+        ForEachItemIn(rowindex, this->columns)
+        {
+           out.append(recordindent);
+           out.append(this->columns.item(rowindex).toEclRecString().str());
+           out.append(";");
+           out.append(linedelimiter);
+        }
+
+        out.append("END;");
+    }
+    else
+        return false;
+
+    return true;
+}
+
+void HPCCFile::setKeyedColumn(const char * name)
+{
+    ForEachItemIn(colidx, columns)
+    {
+        if (strcmp(columns.item(colidx).getColumnName(), name)==0)
+        {
+            columns.item(colidx).setKeyedField(true);
+            break;
+        }
+    }
+}
+
+bool HPCCFile::getFileRecDefwithIndexpos(HPCCColumnMetaData * fieldMetaData, StringBuffer & out, const char * structname)
+{
+    if (fieldMetaData)
+    {
+        out.append(structname);
+        out.append(" := RECORD ");
+        out.append("\n");
+
+        ForEachItemIn(rowindex, this->columns)
+        {
+           out.append("\t");
+           out.append(this->columns.item(rowindex).toEclRecString().str());
+           out.append(";");
+           out.append("\n");
+        }
+
+        out.append("\t")
+           .append(fieldMetaData->getColumnType())
+           .append(" ")
+           .append(fieldMetaData->getColumnName())
+           .append(" {virtual(fileposition)};\n");
+
+        out.append("END;");
+        return true;
+    }
+
+    return false;
+}
+
+bool HPCCFile::setFileColumns(const char * eclString)
+{
+    StringBuffer text = eclString;
+
+    MultiErrorReceiver errs;
+    OwnedHqlExpr record = parseQuery(text.str(), &errs);
+    if (errs.errCount())
+    {
+       StringBuffer errtext;
+       IError *first = errs.firstError();
+       first->toString(errtext);
+       ESPLOG(LogNormal, "Could not set HPCC file columns: %s", errtext.str());
+       return false;
+    }
+
+    if(!record)
+        return false;
+
+    Owned<IPropertyTree> rectree = createPTree("Table", ipt_caseInsensitive);
+
+    exportData(rectree, record);
+
+    StringBuffer ecltype;
+    StringBuffer colname;
+    int colsize;
+    int colindex;
+
+    Owned<IPropertyTreeIterator> fields = rectree->getElements("Field");
+    ForEach(*fields)
+    {
+      fields->query().getProp("@ecltype", ecltype.clear());
+      if (strncmp(ecltype, "table of", 8)==0)
+          setHasNestedColumns(true);
+
+      fields->query().getProp("@name", colname.clear());
+      colsize = fields->query().getPropInt("@size", -1);
+      colindex = fields->query().getPropInt("@position", -1);
+
+      Owned<HPCCColumnMetaData> col = HPCCColumnMetaData::createHPCCColumnMetaData(colname.str());
+      col->setColumnType(ecltype.str());
+      col->setIndex(colindex);
+      col->setTableName(this->fullname.str());
+
+      columns.append(*LINK(col));
+    }
+
+    return true;
+}
+
+HPCCColumnMetaData * HPCCFile::getColumn(const char * colname)
+{
+    ForEachItemIn(colidx, columns)
+    {
+        if (strcmp(columns.item(colidx).getColumnName(), colname)==0)
+        {
+            return &(columns.item(colidx));
+        }
+    }
+    return NULL;
+}
+
+void HPCCFile::getKeyedFieldsAsDelimitedString(char delim, const char * prefix, StringBuffer & out)
+{
+    getFieldsAsDelmitedString(delim, prefix, out, true);
+}
+
+void HPCCFile::getNonKeyedFieldsAsDelmitedString(char delim, const char * prefix, StringBuffer & out)
+{
+    getFieldsAsDelmitedString(delim, prefix, out, false);
+}
+void HPCCFile::getFieldsAsDelmitedString(char delim, const char * prefix, StringBuffer & out, bool onlykeyed)
+{
+    bool isFirst = true;
+    ForEachItemIn(colidx, columns)
+    {
+        HPCCColumnMetaData currcol = columns.item(colidx);
+        if ((onlykeyed && currcol.isKeyedField()) || (!onlykeyed && !currcol.isKeyedField()))
+        {
+            if (!isFirst)
+            {
+                out.append(delim);
+                out.append(" ");
+            }
+
+            if (prefix && *prefix)
+            {
+                out.append(prefix);
+                out.append('.');
+            }
+            out.append(currcol.getColumnName());
+            isFirst = false;
+        }
+    }
+}
+
+bool HPCCFile::containsField(SQLColumn * field, bool verifyEclType) const
+{
+    const char *  fieldName = field->getName();
+
+    for (int i = 0; i < columns.length(); i++)
+    {
+        const char * name = columns.item(i).getColumnName();
+        if (stricmp(name, fieldName)==0)
+        {
+            if (!verifyEclType ||  strcmp(columns.item(i).getColumnType(), field->getColumnType())==0)
+                return true;
+            return false;
+        }
+    }
+    return false;
+}
+void HPCCFile::setKeyCounts()
+{
+    nonKeyedCount = 0;
+    keyedCount = 0;
+    ForEachItemIn(colidx, columns)
+    {
+        HPCCColumnMetaData currcol = columns.item(colidx);
+        if (!currcol.isKeyedField())
+            nonKeyedCount++;
+        else
+            keyedCount++;
+    }
+}
+
+int HPCCFile::getNonKeyedColumnsCount()
+{
+    if (nonKeyedCount == -1)
+        setKeyCounts();
+
+    return nonKeyedCount;
+}
+int HPCCFile::getKeyedColumnsCount()
+{
+    if (keyedCount == -1)
+        setKeyCounts();
+
+    return keyedCount;
+}
+
+int main()
+{
+    HPCCFile * file = new HPCCFile();
+    file->setCluster("cluster");
+    file->setName("name");
+
+    delete file;
+
+    return 0;
+}

esp/services/ws_sql/SQL2ECL/HPCCFile.hpp

@@ -0,0 +1,389 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#ifndef HPCCFILE_HPP_
+#define HPCCFILE_HPP_
+
+#include "ws_sql.hpp"
+#include "SQLColumn.hpp"
+
+#include "ws_sql_esp.ipp"
+#include "hqlerror.hpp"
+#include "hqlexpr.hpp"
+
+/* undef SOCKET definitions to avoid collision in Antlrdefs.h*/
+#ifdef INVALID_SOCKET
+    //#pragma message( "UNDEFINING INVALID_SOCKET - Will be redefined by ANTLRDEFS.h" )
+    #undef INVALID_SOCKET
+#endif
+#ifdef SOCKET
+    //#pragma message( "UNDEFINING SOCKET - Will be redefined by ANTLRDEFS.h" )
+    #undef SOCKET
+#endif
+
+#include "HPCCSQLLexer.h"
+#include "HPCCSQLParser.h"
+
+typedef enum _HPCCFileFormat
+{
+    HPCCFileFormatUnknown=-1,
+    HPCCFileFormatFlat,
+    HPCCFileFormatCSV,
+    HPCCFileFormatXML,
+    HPCCFileFormatKey,
+    HPCCFileFormatJSON,
+} HPCCFileFormat;
+
+
+class HPCCFile : public CInterface, public IInterface
+{
+public:
+    static HPCCFileFormat DEFAULTFORMAT;
+
+    IMPLEMENT_IINTERFACE;
+    HPCCFile();
+    virtual ~HPCCFile();
+
+    const char * getCluster() const
+    {
+        return cluster.str();
+    }
+
+    void setCluster(const char * cluster)
+    {
+        this->cluster.set(cluster);
+    }
+
+    HPCCColumnMetaData * getColumn(const char * colname);
+
+    IArrayOf<HPCCColumnMetaData> * getColumns()
+    {
+        return &columns;
+    }
+
+    const char * getEcl() const
+    {
+        return ecl;
+    }
+
+    bool setEcl(const char * ecl)
+    {
+        if (setFileColumns(ecl))
+            this->ecl = ecl;
+        else
+            return false;
+
+        return true;
+    }
+
+    static const char * formatToString(HPCCFileFormat format)
+    {
+        switch(format)
+        {
+            case HPCCFileFormatFlat:
+                return "FLAT";
+            case HPCCFileFormatCSV:
+                return "CSV";
+            case HPCCFileFormatXML:
+                return "XML";
+            case HPCCFileFormatKey:
+                return "KEYED";
+            case HPCCFileFormatJSON:
+                return "JSON";
+            case HPCCFileFormatUnknown:
+            default:
+                return "UNKNOWN";
+        }
+    }
+
+    const char * getFormat() const
+    {
+        return formatToString(formatEnum);
+    }
+
+    static HPCCFileFormat formatStringToEnum(const char * formatstr)
+    {
+        if (!formatstr || !*formatstr)
+            return HPCCFileFormatUnknown;
+        else
+        {
+            StringBuffer toUpper = formatstr;
+            toUpper.trim().toUpperCase();
+
+            if (strcmp(toUpper.str(), "FLAT")==0)
+                return HPCCFileFormatFlat;
+            else if (strcmp(toUpper.str(), "UTF8N")==0)
+                return HPCCFileFormatCSV;
+            else if (strcmp(toUpper.str(), "CSV")==0)
+                return HPCCFileFormatCSV;
+            else if (strcmp(toUpper.str(), "XML")==0)
+                return HPCCFileFormatXML;
+            else if (strcmp(toUpper.str(), "KEY")==0)
+                return HPCCFileFormatKey;
+            else if (strcmp(toUpper.str(), "JSON")==0)
+                return HPCCFileFormatJSON;
+            else
+                return HPCCFileFormatUnknown;
+        }
+    }
+
+    void setFormat(const char * format)
+    {
+        HPCCFileFormat formatenum = formatStringToEnum(format);
+        if (formatenum == HPCCFileFormatUnknown)
+            this->formatEnum = DEFAULTFORMAT;
+        else
+            this->formatEnum = formatenum;
+    }
+
+    const char * getFullname() const
+    {
+        return fullname.str();
+    }
+
+    void setFullname(const char * fullname)
+    {
+        this->fullname.set(fullname);
+    }
+
+    bool isFileKeyed() const
+    {
+        return iskeyfile;
+    }
+
+    void setIsKeyedFile(bool iskeyfile)
+    {
+        this->iskeyfile = iskeyfile;
+    }
+
+    bool isFileSuper() const
+    {
+        return issuperfile;
+    }
+
+    void setIsSuperfile(bool issuperfile)
+    {
+        this->issuperfile = issuperfile;
+    }
+
+    const char * getName() const
+    {
+        return name.str();
+    }
+
+    void setName(const char * name)
+    {
+        this->name.set(name);
+    }
+
+    static bool validateFileName(const char * fullname);
+    void setKeyedColumn(const char * name);
+    bool getFileRecDefwithIndexpos(HPCCColumnMetaData * fieldMetaData, StringBuffer & out, const char * structname);
+    bool getFileRecDef(StringBuffer & out, const char * structname, const char * linedelimiter  = "\n", const char * recordindent  = "\t");
+    int getNonKeyedColumnsCount();
+    int getKeyedColumnsCount();
+    void getKeyedFieldsAsDelimitedString(char delim, const char * prefix, StringBuffer & out);
+    void getNonKeyedFieldsAsDelmitedString(char delim, const char * prefix, StringBuffer & out);
+    const char * getIdxFilePosField()
+    {
+       return idxFilePosField.length() > 0 ? idxFilePosField.str() : getLastNonKeyedNumericField();
+    }
+
+    bool hasValidIdxFilePosField()
+    {
+        const char * posfield = getIdxFilePosField();
+        return (posfield && *posfield);
+    }
+
+    static HPCCFile * createHPCCFile();
+
+    bool containsField(SQLColumn * field, bool verifyEclType) const;
+
+    const char * getOwner() const
+    {
+        return owner.str();
+    }
+
+    void setOwner(const char * owner)
+    {
+        this->owner.set(owner);
+    }
+
+    const char* getDescription() const
+    {
+        return description.str();
+    }
+
+    void setDescription(const char* description)
+    {
+        if (description && *description)
+        {
+            this->description.set(description);
+            const char * pos = strstr(description, "XDBC:RelIndexes");
+            if (pos)
+            {
+                pos = pos + 15;//advance to end of "XDBC:RelIndexes"
+                while(pos && *pos) //find the = char
+                {
+                    if (!isspace(*pos))
+                    {
+                        if (*pos == '=' )
+                        {
+                            pos++;
+                            while(pos && *pos) //find the beginning bracket
+                            {
+                                if (!isspace(*pos))
+                                {
+                                    if (*pos == '[' )
+                                    {
+                                        pos++;
+                                        break;
+                                    }
+                                    else
+                                        return;//found invalid char before [
+                                }
+                                pos++;
+                            }
+                            break;
+                        }
+                        else
+                            return;//found invalid char before = char
+                    }
+                    pos++;
+                }
+
+                if ( pos && *pos) //found keyword
+                    setRelatedIndexes(pos);
+            }
+        }
+    }
+
+    static bool parseOutRelatedIndexes(StringBuffer & description, StringBuffer & releatedIndexes)
+    {
+        if (description.length() > 0)
+        {
+            const char * head = strstr(description.str(), "XDBC:RelIndexes");
+            if (head && *head)
+            {
+                const char * tail = strchr(head, ']');
+                if (tail && *tail)
+                {
+                    description.remove(head-description.str(), tail-description.str()).trim();
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    void getRelatedIndexes(StringArray & indexes)
+    {
+        ForEachItemIn(c, relIndexFiles)
+        {
+            indexes.append(relIndexFiles.item(c));
+        }
+    }
+
+    const char * getRelatedIndex(int relindexpos)
+    {
+        if (relindexpos > -1 && relindexpos < relIndexFiles.length())
+            return relIndexFiles.item(relindexpos);
+        else
+            return NULL;
+    }
+
+    int getRelatedIndexCount()
+    {
+        return relIndexFiles.length();
+    }
+  /*
+    tutorial::yn::peoplebyzipindex;
+    Tutorial.IDX_PeopleByName;
+    Tutorial.IDX_PeopleByPhonetic]
+*/
+    void setRelatedIndexes(const char * str)
+    {
+        StringBuffer index;
+        while (str && *str)
+        {
+            if (!isspace(*str))
+            {
+                if  (*str == ';' || *str == ']')
+                {
+                    relIndexFiles.append(index.str());
+                    if (*str == ']')
+                        break;
+                    else
+                    index.clear();
+                }
+                else
+                    index.append(*str);
+            }
+            str++;
+        }
+    }
+
+    void setIdxFilePosField(const char* idxfileposfieldname)
+    {
+        idxFilePosField.set(idxfileposfieldname);
+    }
+
+    bool containsNestedColumns() const
+    {
+        return hasNestedColumns;
+    }
+
+    void setHasNestedColumns(bool hasNestedColumns)
+    {
+        this->hasNestedColumns = hasNestedColumns;
+    }
+
+private:
+    void getFieldsAsDelmitedString(char delim, const char* prefix, StringBuffer& out, bool onlykeyed);
+    bool setFileColumns(const char* eclString);
+    void setKeyCounts();
+
+    const char* getLastNonKeyedNumericField()
+    {
+        for (int i = columns.length() - 1;i >= 0;i--)
+        {
+            if (!columns.item(i).isKeyedField())
+                return columns.item(i).getColumnName();
+        }
+        return NULL;
+    }
+
+private:
+    HPCCFileFormat formatEnum;
+    StringBuffer name;
+    StringBuffer fullname;
+    StringBuffer cluster;
+    StringBuffer idxFilePosField;
+    bool iskeyfile;
+    bool issuperfile;
+    StringBuffer ecl;
+    IArrayOf<HPCCColumnMetaData> columns;
+    int keyedCount;
+    int nonKeyedCount;
+    bool hasNestedColumns;
+    StringBuffer description;
+    StringBuffer owner;
+    StringArray relIndexFiles;
+}
+;
+
+#endif /* HPCCFILE_HPP_ */

esp/services/ws_sql/SQL2ECL/HPCCFileCache.cpp

@@ -0,0 +1,352 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#include "HPCCFileCache.hpp"
+
+
+HPCCFileCache * HPCCFileCache::createFileCache(const char * username, const char * passwd)
+{
+    ESPLOG(LogMax, "WsSQL: Creating new HPCC FILE CACHE");
+    return new HPCCFileCache(username,passwd);
+}
+
+bool HPCCFileCache::populateTablesResponse(IEspGetDBMetaDataResponse & tablesrespstruct, const char * filterby)
+{
+    bool success = false;
+
+    cacheAllHpccFiles(filterby);
+    IArrayOf<IEspHPCCTable> tables;
+
+    HashIterator iterHash(cache);
+    ForEach(iterHash)
+    {
+       const char* key = (const char*)iterHash.query().getKey();
+       HPCCFilePtr file = dynamic_cast<HPCCFile *>(*cache.getValue(key));
+       if ( file )
+       {
+           Owned<IEspHPCCTable> pTable = createHPCCTable();
+           pTable->setName(file->getFullname());
+           pTable->setFormat(file->getFormat());
+           const char * ecl = file->getEcl();
+
+           if (!ecl || !*ecl)
+               continue;
+           else
+           {
+               pTable->setDescription(file->getDescription());
+               pTable->setIsKeyed(file->isFileKeyed());
+               pTable->setIsSuper(file->isFileSuper());
+               pTable->setOwner(file->getOwner());
+
+               IArrayOf<IEspHPCCColumn> pColumns;
+
+               IArrayOf<HPCCColumnMetaData> * cols = file->getColumns();
+               for (int i = 0; i < cols->length(); i++)
+               {
+                   Owned<IEspHPCCColumn> pCol = createHPCCColumn();
+                   HPCCColumnMetaData currcol = cols->item(i);
+                   pCol->setName(currcol.getColumnName());
+                   pCol->setType(currcol.getColumnType());
+                   pColumns.append(*pCol.getLink());
+               }
+               pTable->setColumns(pColumns);
+               tables.append(*pTable.getLink());
+           }
+        }
+     }
+
+     tablesrespstruct.setTables(tables);
+     return success;
+ }
+
+bool HPCCFileCache::fetchHpccFilesByTableName(IArrayOf<SQLTable> * sqltables)
+{
+    bool allFound = true;
+
+    ForEachItemIn(tableindex, *sqltables)
+    {
+       SQLTable table = sqltables->item(tableindex);
+       const char * cachedKey = cacheHpccFileByName(table.getName());
+       allFound &= (cachedKey && *cachedKey);
+    }
+
+    return allFound;
+}
+
+bool HPCCFileCache::cacheAllHpccFiles(const char * filterby)
+{
+    bool success = false;
+    StringBuffer filter;
+    if(filterby && *filterby)
+        filter.append(filterby);
+    else
+        filter.append("*");
+
+    Owned<IDFAttributesIterator> fi = queryDistributedFileDirectory().getDFAttributesIterator(filter, userdesc.get(), true, true, NULL);
+    if(!fi)
+        throw MakeStringException(-1,"Cannot get information from file system.");
+
+    success = true;
+    ForEach(*fi)
+    {
+       IPropertyTree &attr=fi->query();
+
+#if defined(_DEBUG)
+    StringBuffer toxml;
+    toXML(&attr, toxml);
+    fprintf(stderr, "%s", toxml.str());
+#endif
+
+       StringBuffer name(attr.queryProp("@name"));
+
+       if (name.length()>0 && HPCCFile::validateFileName(name.str()))
+       //if (name.length()>0)
+       {
+           const char * cachedKey = cacheHpccFileByName(name.str(), true);
+           success &= (cachedKey && *cachedKey);
+       }
+    }
+
+    return success;
+}
+bool HPCCFileCache::updateHpccFileDescription(const char * filename, const char * user, const char * pass, const char * description)
+{
+    Owned<IUserDescriptor> userdesc;
+    try
+    {
+        userdesc.setown(createUserDescriptor());
+        userdesc->set(user, pass);
+
+        //queryDistributedFileDirectory returns singleton
+        IDistributedFileDirectory & dfd = queryDistributedFileDirectory();
+        Owned<IDistributedFile> df = dfd.lookup(filename, userdesc);
+
+        if(!df)
+            return false;
+
+        DistributedFilePropertyLock lock(df);
+        lock.queryAttributes().setProp("@description",description);
+    }
+    catch (...)
+    {
+        return false;
+    }
+
+    return true;
+}
+
+
+HPCCFile * HPCCFileCache::fetchHpccFileByName(IUserDescriptor *user, const char * filename, bool namevalidated, bool acceptrawfiles)
+{
+    StringBuffer username;
+    user->getUserName(username);
+    StringBuffer password;
+    user->getPassword(password);
+
+    return HPCCFileCache::fetchHpccFileByName(filename, username.str(), password.str(), namevalidated, acceptrawfiles);
+}
+
+HPCCFile * HPCCFileCache::fetchHpccFileByName(const char * filename, const char * user, const char * pass, bool namevalidated, bool acceptrawfiles)
+{
+    Owned<HPCCFile> file;
+    Owned<IUserDescriptor> userdesc;
+    try
+    {
+        userdesc.setown(createUserDescriptor());
+        userdesc->set(user, pass);
+
+        //queryDistributedFileDirectory returns singleton
+        IDistributedFileDirectory & dfd = queryDistributedFileDirectory();
+        Owned<IDistributedFile> df = dfd.lookup(filename, userdesc);
+
+        if(!df)
+            throw MakeStringException(-1,"Cannot find file %s.",filename);
+
+        const char* lname=df->queryLogicalName();
+        if (lname && *lname)
+        {
+            const char* fname=strrchr(lname,':');
+            if (!namevalidated && !HPCCFile::validateFileName(lname))
+                throw MakeStringException(-1,"Invalid SQL file name detected %s.", fname);
+            file.setown(HPCCFile::createHPCCFile());
+            file->setFullname(lname);
+            file->setName(fname ? fname+1 : lname);
+        }
+        else
+            throw MakeStringException(-1,"Cannot find file %s.",filename);
+
+        file->setDescription(df->queryAttributes().queryProp("@description"));
+
+        //Do we care about the clusters??
+        //StringArray clusters;
+        //if (cluster && *cluster)
+        //{
+        //df->getClusterNames(clusters);
+        //if(!FindInStringArray(clusters, cluster))
+            //throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Cannot find file %s.",fname);
+        //}
+
+    #if defined(_DEBUG)
+        StringBuffer atttree;
+        toXML(&df->queryAttributes(), atttree, 0);
+        fprintf(stderr, "%s", atttree.str());
+    #endif
+
+        IPropertyTree & properties = df->queryAttributes();
+
+        if(properties.hasProp("ECL"))
+            file->setEcl(properties.queryProp("ECL"));
+        else if (!acceptrawfiles)
+            throw MakeStringException(-1,"File %s does not contain required ECL record layout.",filename);
+
+        file->setOwner(properties.queryProp("@owner"));
+        IDistributedSuperFile *sf = df->querySuperFile();
+        if(sf)
+        {
+            file->setIsSuperfile(true);
+        }
+
+        //unfortunately @format sometimes holds the file format, sometimes @kind does
+        const char * kind = properties.queryProp("@kind");
+        if (kind)
+        {
+            file->setFormat(kind);
+            if ((stricmp(kind, "key") == 0))
+            {
+                file->setIsKeyedFile(true);
+
+                ISecManager *secmgr = NULL;
+                ISecUser *secuser = NULL;
+                StringBuffer username;
+                StringBuffer passwd;
+                userdesc->getUserName(username);
+                userdesc->getPassword(passwd);
+
+                Owned<IResultSetFactory> resultSetFactory = getSecResultSetFactory(secmgr, secuser, username.str(), passwd.str());
+                Owned<INewResultSet> result;
+                try
+                {
+                    result.setown(resultSetFactory->createNewFileResultSet(filename, NULL));
+
+                    if (result)
+                    {
+                        Owned<IResultSetCursor> cursor = result->createCursor();
+                        const IResultSetMetaData & meta = cursor->queryResultSet()->getMetaData();
+                        int columnCount = meta.getColumnCount();
+                        int keyedColumnCount = meta.getNumKeyedColumns();
+
+                        for (int i = 0; i < keyedColumnCount; i++)
+                        {
+                            SCMStringBuffer columnLabel;
+                            if (meta.hasSetTranslation(i))
+                            {
+                                meta.getNaturalColumnLabel(columnLabel, i);
+                            }
+
+                            if (columnLabel.length() < 1)
+                            {
+                                meta.getColumnLabel(columnLabel, i);
+                            }
+
+                            file->setKeyedColumn(columnLabel.str());
+                        }
+                    }
+                }
+                catch (IException * se)
+                {
+                    StringBuffer s;
+                    se->errorMessage(s);
+                    DBGLOG("Error fetching keyed file %s info: %s", filename, s.str());
+                    se->Release();
+                    if (file)
+                        file.clear();
+                }
+            }
+        }
+        else
+        {
+            //@format - what format the file is (if not fixed width)
+            const char* format = properties.queryProp("@format");
+            file->setFormat(format);
+        }
+
+        if (file && ( file->containsNestedColumns() || strncmp(file->getFormat(), "XML", 3)==0))
+            throw MakeStringException(-1,"Nested data files not supported: %s.",filename);
+    }
+
+    catch (IException * se)
+    {
+        StringBuffer s;
+        se->errorMessage(s);
+        DBGLOG("Error fetching file %s info: %s", filename, s.str());
+        se->Release();
+        if (file)
+            file.clear();
+    }
+
+    catch (...)
+    {
+        if (file)
+            file.clear();
+    }
+
+    if (file)
+        return file.getLink();
+    else
+        return NULL;
+}
+
+const char * HPCCFileCache::cacheHpccFileByName(const char * filename, bool namevalidated)
+{
+    if(cache.getValue(filename))
+        return filename;
+
+    Owned<HPCCFile> file;
+
+    try
+    {
+        file.setown(HPCCFileCache::fetchHpccFileByName(userdesc, filename, namevalidated, false));
+    }
+    catch (...)
+    {
+        if (file)
+            file.clear();
+    }
+
+    if (file)
+    {
+        cache.setValue(file->getFullname(), file.getLink());
+        return file->getFullname();
+    }
+
+    return "";
+}
+
+HPCCFilePtr HPCCFileCache::getHpccFileByName(const char * filename)
+{
+    HPCCFilePtr * hpccfile = cache.getValue(filename);
+
+    if (hpccfile)
+        return *hpccfile;
+
+    return NULL;
+}
+
+bool HPCCFileCache::isHpccFileCached(const char * filename)
+{
+    return cache.getValue(filename) != NULL;
+}

esp/services/ws_sql/SQL2ECL/HPCCFileCache.hpp

@@ -0,0 +1,74 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#ifndef HPCCFILECACHE_HPP_
+#define HPCCFILECACHE_HPP_
+
+#include "ws_sql.hpp"
+#include "ws_sql_esp.ipp"
+#include "dadfs.hpp"
+#include "dasess.hpp"
+#include "HPCCFile.hpp"
+#include "SQLTable.hpp"
+#include "fileview.hpp"
+#include "fvrelate.hpp"
+
+typedef HPCCFile * HPCCFilePtr;
+typedef MapStringTo<HPCCFilePtr> HpccFiles;
+
+class HPCCFileCache : public CInterface, public IInterface
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    static HPCCFileCache * createFileCache(const char * username, const char * passwd);
+    bool cacheAllHpccFiles(const char * filterby);
+    bool fetchHpccFilesByTableName(IArrayOf<SQLTable> * sqltables);
+    static bool updateHpccFileDescription(const char * filename, const char * user, const char * pass, const char * description);
+    HPCCFile * fetchHpccFileByName(IUserDescriptor *user, const char * filename, bool namevalidated, bool acceptrawfiles=false);
+    static HPCCFile * fetchHpccFileByName(const char * filename, const char * user, const char * pass, bool namevalidated, bool acceptrawfiles);
+    const char * cacheHpccFileByName(const char * filename, bool namevalidated = false);
+    bool isHpccFileCached(const char * filename);
+    HPCCFilePtr getHpccFileByName(const char * filename);
+    bool populateTablesResponse(IEspGetDBMetaDataResponse & tablesrespstruct, const char * filterby);
+
+    HPCCFileCache(const char * username, const char * passwd)
+    {
+        userdesc.setown(createUserDescriptor());
+        userdesc->set(username, passwd);
+    }
+
+    virtual ~HPCCFileCache()
+    {
+        HashIterator sIter(cache);
+        for(sIter.first();sIter.isValid();sIter.next())
+        {
+            HPCCFilePtr a = *cache.mapToValue(&sIter.query());
+            delete a;
+        }
+#ifdef _DEBUG
+        fprintf(stderr, "Leaving filecache");
+#endif
+    }
+
+private:
+
+    Owned<IUserDescriptor> userdesc;
+    HpccFiles cache;
+};
+
+#endif /* HPCCFILECACHE_HPP_ */

esp/services/ws_sql/SQL2ECL/HPCCSQLTreeWalker.hpp

@@ -0,0 +1,353 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#ifndef HPCCSQLTREEWALKER_HPP_
+#define HPCCSQLTREEWALKER_HPP_
+#include <stdlib.h>
+#include "ws_sql.hpp"
+
+/* undef SOCKET definitions to avoid collision in Antlrdefs.h*/
+#ifdef INVALID_SOCKET
+    //#pragma message("UNDEFINING INVALID_SOCKET - Will be redefined by ANTLRDEFS.h" )
+    #undef INVALID_SOCKET
+#endif
+#ifdef SOCKET
+    //#pragma message( "UNDEFINING SOCKET - Will be redefined by ANTLRDEFS.h" )
+    #undef SOCKET
+#endif
+/* undef SOCKET definitions to avoid collision in Antlrdefs.h*/
+
+#include "HPCCSQLLexer.h"
+#include "HPCCSQLParser.h"
+
+
+#include "SQLColumn.hpp"
+#include "SQLTable.hpp"
+#include "SQLExpression.hpp"
+#include "HPCCFile.hpp"
+#include "HPCCFileCache.hpp"
+#include "ECLFunction.hpp"
+#include "SQLJoin.hpp"
+
+typedef enum _SQLQueryType
+{
+    SQLTypeUnknown=-1,
+    SQLTypeSelect,
+    SQLTypeCall,
+    SQLTypeCreateAndLoad
+} SQLQueryType;
+
+class HPCCSQLTreeWalker: public CInterface, public IInterface
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+private:
+    SQLQueryType sqlType;
+    IArrayOf<ISQLExpression> selectList;
+    IArrayOf<ISQLExpression> groupbyList;
+    IArrayOf<ISQLExpression> orderbyList;
+    IArrayOf<SQLTable>       tableList;
+
+    int limit;
+    int offset;
+    string indexhint;
+    Owned<ISQLExpression> whereClause;
+    Owned<ISQLExpression> havingClause;
+    Owned<HPCCFileCache>  tmpHPCCFileCache;
+
+    bool selectDistinct;
+
+    StringBuffer procedureName;
+    StringBuffer querySetName;
+    IArrayOf<ISQLExpression> paramList;
+
+    bool overwrite;
+    StringBuffer tableName;
+    StringBuffer sourceDataTableName;
+    StringBuffer comment;
+    StringBuffer sourceDataType;
+    StringBuffer landingZoneIP;
+    StringBuffer landingZonePath;
+    StringBuffer recordDefinition;
+
+    void sqlTreeWalker(pANTLR3_BASE_TREE sqlAST);
+    void selectStatementTreeWalker(pANTLR3_BASE_TREE selectsqlAST);
+    void callStatementTreeWalker(pANTLR3_BASE_TREE callsqlAST);
+    void createAndLoadStatementTreeWalker(pANTLR3_BASE_TREE clsqlAST);
+    void columnListTreeWalker(pANTLR3_BASE_TREE columnsAST, IArrayOf<SQLColumn>& collist);
+    ISQLExpression* expressionTreeWalker(pANTLR3_BASE_TREE exprAST, pANTLR3_BASE_TREE parent);
+    void fromTreeWalker(pANTLR3_BASE_TREE fromsqlAST);
+    void limitTreeWalker(pANTLR3_BASE_TREE limitAST);
+    void processAllColumns(HpccFiles *  availableFiles);
+    void verifyColumn(SQLFieldValueExpression * col );
+    void verifyColAndDisambiguateName();
+    void verifyAndDisambiguateNameFromList(IArrayOf<ISQLExpression> * explist);
+    void assignParameterIndexes();
+    int parameterizedCount;
+
+    StringBuffer normalizedSQL;
+    bool parameterizeStaticValues;
+
+public:
+
+    bool isParameterizedCall();
+    void setQuerySetName(const char * qsname)
+    {
+        querySetName.set(qsname);
+    }
+
+    const char * getQuerySetName()
+    {
+        return querySetName.str();
+    }
+
+    const char * getStoredProcName()
+    {
+        return procedureName.str();
+    }
+
+    int getStoredProcParamListCount()
+    {
+        return paramList.length();
+    }
+
+    IArrayOf<ISQLExpression>* getStoredProcParamList()
+    {
+        return &paramList;
+    }
+
+    void getWhereClauseString(StringBuffer & str);
+    void getHavingClauseString(StringBuffer & str);
+    ISQLExpression * getHavingClause();
+
+    void expandWildCardColumn();
+    HPCCSQLTreeWalker();
+    HPCCSQLTreeWalker(pANTLR3_BASE_TREE t, IEspContext &context, bool attemptParameterization = true);
+
+    virtual ~HPCCSQLTreeWalker();
+
+    int getParameterizedCount() const
+    {
+        return parameterizedCount;
+    }
+
+    const IArrayOf<SQLTable>* getTableList() const
+    {
+        return const_cast <const IArrayOf<SQLTable>*> (&tableList);
+    }
+
+    const IArrayOf<ISQLExpression>* getSelectList() const
+    {
+        return const_cast <const IArrayOf<ISQLExpression>*> (&selectList);
+    }
+
+    const IArrayOf<ISQLExpression>* getParamList() const
+    {
+        return const_cast <const IArrayOf<ISQLExpression>*> (&paramList);
+    }
+
+    int getLimit() const
+    {
+        return limit;
+    }
+
+    void setLimit(int limit)
+    {
+        this->limit = limit;
+    }
+
+    int getOffset() const
+    {
+        return offset;
+    }
+
+    void setOffset(int _offset)
+    {
+        offset = _offset;
+    }
+
+    void setSqlType(SQLQueryType type)
+    {
+        sqlType = type;
+    }
+
+    SQLQueryType getSqlType()
+    {
+        return sqlType;
+    }
+
+    bool hasGroupByColumns()
+    {
+        return !groupbyList.empty();
+    }
+
+    bool hasOrderByColumns()
+    {
+        return !orderbyList.empty();
+    }
+
+    bool hasHavingClause()
+    {
+        return havingClause != NULL;
+    }
+
+    bool hasWhereClause()
+    {
+        return whereClause != NULL;
+    }
+
+    bool isSelectDistinct()
+    {
+        return selectDistinct;
+    }
+
+    void getOrderByString(StringBuffer & str)
+    {
+        getOrderByString(',', str);
+    }
+
+    void getOrderByString(char delimiter, StringBuffer & str)
+    {
+        int orderbycount = orderbyList.length();
+        for (int i = 0; i < orderbycount; i++)
+        {
+            ISQLExpression* ordercol = &orderbyList.item(i);
+            SQLFieldValueExpression* colexp = dynamic_cast<SQLFieldValueExpression*>(ordercol);
+            str.append(colexp->isAscending() ? "" : "-");
+            str.append(colexp->getNameOrAlias());
+            if (i != orderbycount - 1)
+                str.append(delimiter);
+        }
+    }
+
+    void getGroupByString(StringBuffer & str)
+    {
+        getGroupByString(',', str);
+    }
+
+    void getGroupByString(char delimiter, StringBuffer & str)
+    {
+        int groupbycount = groupbyList.length();
+        for (int i = 0; i < groupbycount; i++)
+        {
+            ISQLExpression * ordercol =  &groupbyList.item(i);
+            SQLFieldValueExpression * colexp = dynamic_cast<SQLFieldValueExpression *>(ordercol);
+            str.append(colexp->getName());
+            if (i != groupbycount - 1)
+                str.append(delimiter);
+        }
+    }
+
+    ISQLExpression * getWhereClause()
+    {
+        return whereClause.get();
+    }
+
+    HPCCFileCache * queryHPCCFileCache()
+    {
+        return tmpHPCCFileCache.get();
+    }
+
+    const char* getNormalizedSQL();
+
+    const char * getRecordDefinition()
+    {
+        return recordDefinition.str();
+    }
+
+    void setRecordDefinition(const char * _recordDefinition)
+    {
+        recordDefinition.set(_recordDefinition);
+    }
+
+    const char *  getComment() const
+    {
+        return comment.str();
+    }
+
+    void setComment(const char * _comment)
+    {
+        comment.set(_comment);
+    }
+
+    const char * getLandingZoneIp() const
+    {
+        return landingZoneIP.str();
+    }
+
+    void setLandingZoneIp(const char * _landingZoneIp)
+    {
+        landingZoneIP.set(_landingZoneIp);
+    }
+
+    const char * getLandingZonePath() const
+    {
+        return landingZonePath.str();
+    }
+
+    void setLandingZonePath(const char * _landingZonePath)
+    {
+        landingZonePath.set(_landingZonePath);
+    }
+
+    const char * getSourceDataType() const
+    {
+        return sourceDataType.str();
+    }
+
+    void setSourceDataType(const char * _sourceDataType)
+    {
+        sourceDataType.set(_sourceDataType);
+    }
+
+    bool isOverwrite() const
+    {
+        return overwrite;
+    }
+
+    void setOverwrite(bool _overwrite)
+    {
+        overwrite = _overwrite;
+    }
+
+    const char * getSourceDataTableName() const
+    {
+        return sourceDataTableName.str();
+    }
+
+    void setSourceDataTableName(const char * _sourceDataTableName)
+    {
+        sourceDataTableName.set(_sourceDataTableName);
+    }
+
+    const char * getTableName() const
+    {
+        return tableName.str();
+    }
+
+    void setTableName(const char * _tableName)
+    {
+        tableName.set(_tableName);
+    }
+
+private:
+
+    bool normalizeSQL();
+};
+
+#endif /* HPCCSQLTREEWALKER_HPP_ */

esp/services/ws_sql/SQL2ECL/SQLColumn.cpp

@@ -0,0 +1,65 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#include "SQLColumn.hpp"
+
+SQLColumn::SQLColumn() {}
+SQLColumn::SQLColumn(const char * parentname, const char * columnname, const char * alias, int position)
+{
+    this->parenttable.clear();
+    this->name.clear();
+    this->alias.clear();
+
+    if (parentname)
+        this->parenttable.append(parentname);
+    if (columnname)
+        this->name.append(columnname);
+    if (alias)
+        this->alias.append(alias);
+
+    this->position = position;
+
+    setAscending(true);
+}
+
+
+SQLColumn::~SQLColumn(){}
+
+bool SQLColumn::isFieldNameOrAalias( const char * possiblenameoralias)
+{
+    if (
+        (name.length() > 0 && strcmp(name.str(), possiblenameoralias)==0)
+        ||
+        (alias.length() > 0 && strcmp(alias.str(), possiblenameoralias)==0)
+       )
+        return true;
+    else
+        return false;
+}
+
+void SQLColumn::toString(StringBuffer & str, bool fullOutput)
+{
+    if (fullOutput)
+    {
+        if (parenttable.length() > 0)
+        {
+            str.append(parenttable.str());
+            str.append(".");
+        }
+    }
+    str.append(name.str());
+}

esp/services/ws_sql/SQL2ECL/SQLColumn.hpp

@@ -0,0 +1,222 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#ifndef SQLCOLUMN_HPP_
+#define SQLCOLUMN_HPP_
+
+#include "ws_sql.hpp"
+
+class  SQLColumn : public CInterface, public IInterface
+{
+private:
+    StringBuffer name;
+    StringBuffer alias;
+    StringBuffer parenttable;
+    int position;
+    bool ascending;
+    StringBuffer columnType;
+
+public:
+    IMPLEMENT_IINTERFACE;
+    SQLColumn();
+    SQLColumn(const char* parentname, const char* columnname, const char* alias, int position);
+    virtual ~SQLColumn();
+
+    bool isAscending() const
+    {
+        return ascending;
+    }
+
+    void setAscending(bool ascending)
+    {
+        this->ascending = ascending;
+    }
+
+    const char * getAlias() const
+    {
+        return alias.str();
+    }
+
+    void setAlias(const char * alias)
+    {
+        this->alias.set(alias);
+    }
+
+    const char * getName()
+    {
+        return name.str();
+    }
+
+    void setName(const char * name)
+    {
+        this->name.set(name);
+    }
+
+    const char *  getParenttable() const
+    {
+        return parenttable.str();
+    }
+
+    void setParenttable(const char * parenttable)
+    {
+        this->parenttable.set(parenttable);
+    }
+
+    int getPosition() const
+    {
+        return position;
+    }
+
+    void setPosition(int position)
+    {
+        this->position = position;
+    }
+
+    bool isFieldNameOrAalias(const char* possiblenameoralias);
+    void toString(StringBuffer & str, bool fullOutput);
+
+    const char * getColumnNameOrAlias()
+    {
+        if (alias.length() > 0)
+            return alias.str();
+        else
+            return name.str();
+    }
+
+    const char * getColumnType() const
+    {
+       return columnType.str();
+    }
+
+    void setColumnType(const char* columnType)
+    {
+       this->columnType.set(columnType);
+    }
+};
+
+
+#define DEFAULTDECIMALCHARS     32;
+#define DEFAULTINTBYTES         8;
+#define DEFAULTREALBYTES        8;
+#define DEFAULTCOLCHARS         0;
+#define DEFAULTDECDIGITS        0;
+
+class HPCCColumnMetaData : public CInterface, public IInterface
+{
+private:
+    StringBuffer columnName;
+    StringBuffer tableName;
+    int index;
+    int columnChars;
+    int decimalDigits;
+    StringBuffer columnType;
+    bool keyedField;
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    static HPCCColumnMetaData * createHPCCColumnMetaData(const char * name)
+    {
+        return new HPCCColumnMetaData(name);
+    }
+
+    HPCCColumnMetaData()
+    {
+        columnName.clear();
+        keyedField = false;
+    }
+
+    HPCCColumnMetaData(const char * colname)
+    {
+        columnName.set(colname);
+        keyedField = false;
+    }
+
+    virtual ~HPCCColumnMetaData()
+    {
+#ifdef _DEBUG
+        fprintf(stderr, "leaving columnmetadata.");
+#endif
+    }
+
+    StringBuffer toEclRecString()
+    {
+        StringBuffer result;
+        result.append(this->columnType.str());
+        result.append(" ");
+        result.append(this->columnName.str());
+
+        return result;
+    }
+
+    const char * getColumnType() const
+    {
+        return columnType.str();
+    }
+
+    void setColumnType(const char* columnType)
+    {
+        this->columnType.set(columnType);
+    }
+
+    int getDecimalDigits() const
+    {
+        return decimalDigits;
+    }
+
+    void setDecimalDigits(int decimalDigits = 0)
+    {
+        this->decimalDigits = decimalDigits;
+    }
+
+    int getIndex() const
+    {
+        return index;
+    }
+
+    void setIndex(int index)
+    {
+        this->index = index;
+    }
+
+    const char * getTableName() const
+    {
+        return tableName.str();
+    }
+
+    void setTableName(const char* tableName)
+    {
+        this->tableName.set(tableName);
+    }
+
+    bool isKeyedField() const
+    {
+        return keyedField;
+    }
+
+    void setKeyedField(bool keyedField)
+    {
+        this->keyedField = keyedField;
+    }
+
+    const char * getColumnName() const
+    {
+        return columnName.str();
+    }
+};
+
+#endif /* SQLCOLUMN_HPP_ */

esp/services/ws_sql/SQL2ECL/SQLExpression.cpp

@@ -0,0 +1,977 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#include "SQLExpression.hpp"
+
+/***********************SQLFieldsExpression START**************************************************************************/
+
+SQLFieldsExpression::SQLFieldsExpression(bool allfiles)
+{
+    this->all = allfiles;
+    isExpanded = false;
+}
+
+SQLFieldsExpression::SQLFieldsExpression(const char * table)
+{
+    this->all = false;
+    this->table.set(table);
+    isExpanded = false;
+}
+
+SQLFieldsExpression::~SQLFieldsExpression()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "\ndestroying SQLFieldsExression.\n");
+#endif
+}
+
+ void SQLFieldsExpression::toString(StringBuffer & targetstr, bool fullOutput)
+ {
+     if (table.length() > 0)
+         targetstr.append(table.str()).append(".");
+
+     targetstr.append("*");
+ }
+/***********************SQLFieldsExpression END**************************************************************************/
+
+/***********************SQLFieldValueExpression START**************************************************************************/
+
+ SQLLogicType SQLFieldValueExpression::getLogicType()
+ {
+     const char * type = field.getColumnType();
+
+     if (strnicmp(type,"STRING",6)==0)
+         return String_LogicType;
+     else if (strnicmp(type,"QSTRING",7)==0)
+         return QSstring_LogicType;
+     else if (strnicmp(type,"UNICODE",7)==0)
+         return Unicode_LogicType;
+     else if (strnicmp(type,"VARUNICODE",10)==0)
+         return Unicode_LogicType;
+     else if (strnicmp(type,"VARSTRING",9)==0)
+         return String_LogicType;
+     else if (strnicmp(type,"BOOLEAN",7)==0)
+         return Bool_LogicType;
+     else if (strnicmp(type,"UNSIGNED",8)==0)
+         return Integer_LogicType;
+     else if (strnicmp(type,"REAL",4)==0)
+         return Decimal_LogicType;
+     else if (strnicmp(type,"DECIMAL",7)==0)
+         return Decimal_LogicType;
+     else
+         return Unknown_LogicType;
+ }
+
+ void SQLFieldValueExpression::toECLStringTranslateSource(
+             StringBuffer & eclStr,
+             IProperties * map,
+             bool ignoreMisTranslations,
+             bool forHaving,
+             bool funcParam,
+             bool countFuncParam)
+ {
+
+     StringBuffer translation;
+     map->getProp(getParentTableName(),translation);
+
+     if (translation.length() == 0 && getParentTableName() != NULL)
+         return;
+
+     if ( translation.length() > 0)
+     {
+         if (funcParam && forHaving && translation.length() > 0)
+         {
+             eclStr.append(" ROWS ( ");
+             eclStr.append(translation);
+             eclStr.append(" ) ");
+             if (!countFuncParam)
+             {
+                 eclStr.append(", ");
+                 eclStr.append(getName());
+             }
+         }
+         else
+         {
+             eclStr.append(translation);
+             eclStr.append(".");
+             eclStr.append(getName());
+         }
+     }
+ }
+SQLFieldValueExpression::SQLFieldValueExpression(const char * parentTableName, const char * fieldName)
+{
+    field.setName(fieldName);
+    field.setParenttable(parentTableName);
+}
+
+SQLFieldValueExpression::SQLFieldValueExpression()
+{
+    field.setName("");
+    field.setParenttable("");
+    field.setAlias("");
+}
+
+const char * SQLFieldValueExpression::getParentTableName()
+{
+    return field.getParenttable();
+}
+
+void SQLFieldValueExpression::setParentTableName(const char * parentTableName)
+{
+    field.setParenttable(parentTableName);
+}
+
+const char * SQLFieldValueExpression::getName()
+{
+    return field.getName();
+}
+
+const char * SQLFieldValueExpression::getNameOrAlias()
+{
+    return field.getColumnNameOrAlias();
+}
+void SQLFieldValueExpression::setName(const char * fieldName)
+{
+    field.setName(fieldName);
+}
+
+bool SQLFieldValueExpression::containsKey(const char * colname)
+{
+    return field.isFieldNameOrAalias(colname);
+}
+
+void SQLFieldValueExpression::toString(StringBuffer & targetstr, bool fullOutput)
+{
+    field.toString(targetstr, fullOutput);
+}
+
+const char * SQLFieldValueExpression::getAlias()
+{
+    return field.getAlias();
+}
+
+void SQLFieldValueExpression::setAlias(const char * alias)
+{
+    if (alias)
+        field.setAlias(alias);
+}
+
+bool SQLFieldValueExpression::isAscending()
+{
+    return field.isAscending();
+}
+
+void SQLFieldValueExpression::setAscending(bool ascending)
+{
+    field.setAscending(ascending);
+}
+
+bool SQLFieldValueExpression::isAliasOrName(const char * possiblenameoralias)
+{
+    return field.isFieldNameOrAalias(possiblenameoralias);
+}
+
+void SQLFieldValueExpression::setECLType(const char * type)
+{
+    field.setColumnType(type);
+}
+
+const char * SQLFieldValueExpression::getECLType()
+{
+    return field.getColumnType();
+}
+
+SQLFieldValueExpression::~SQLFieldValueExpression()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "\ndestroying SQLFieldValueExpression.\n");
+#endif
+}
+/***********************SQLFieldValueExpression END**************************************************************************/
+
+/***********************SQLUnaryExpression START**************************************************************************/
+
+SQLUnaryExpression::SQLUnaryExpression()
+{}
+
+SQLUnaryExpression::~SQLUnaryExpression()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "\ndestroying SQLUnaryExpression.\n");
+#endif
+    operand1->Release();
+}
+
+void SQLUnaryExpression::getExpressionFromColumnName(const char * colname, StringBuffer & str)
+{
+        switch (op)
+        {
+            case ISNOTNULL:
+                str.append(" TRUE ");
+                break;
+            case ISNULL:
+                str.append(" FALSE ");
+                break;
+            case NOT_SYM:
+            case NEGATION:
+                str.append(" NOT ");
+                operand1->getExpressionFromColumnName(colname, str);
+                break;
+            default:
+                break;
+        }
+    }
+
+SQLUnaryExpression::SQLUnaryExpression(ISQLExpression* operand1, int opname)
+{
+    this->operand1 =  operand1;
+    this->op = opname;
+}
+
+bool SQLUnaryExpression::containsKey(const char* colname)
+{
+    return operand1->containsKey(colname);
+}
+
+bool SQLUnaryExpression::containsEqualityCondition(IProperties * map, const char * first, const char * second)
+{
+    return operand1->containsEqualityCondition(map, first, second);
+}
+
+void SQLUnaryExpression::toString(StringBuffer & targetstr, bool fullOutput)
+{
+    switch (op)
+    {
+        case ISNOTNULL:
+            targetstr.append(" TRUE ");
+            break;
+        case ISNULL:
+            targetstr.append(" FALSE ");
+            break;
+        case NOT_SYM:
+        case NEGATION:
+            targetstr.append(" NOT ");
+            operand1->toString(targetstr, fullOutput);
+            break;
+        default:
+            operand1->toString(targetstr, fullOutput);
+            break;
+    }
+}
+
+void SQLUnaryExpression::toECLStringTranslateSource(
+                    StringBuffer & eclStr,
+                    IProperties * map,
+                    bool ignoreMisTranslations,
+                    bool forHaving,
+                    bool funcParam,
+                    bool countFuncParam)
+     {
+        switch (op)
+        {
+            case ISNOTNULL:
+                eclStr.append(" TRUE ");
+                break;
+            case ISNULL:
+                eclStr.append(" FALSE ");
+                break;
+            case NOT_SYM:
+            case NEGATION:
+                eclStr.append(" NOT ");
+                operand1->toECLStringTranslateSource(eclStr, map, ignoreMisTranslations, forHaving, funcParam, countFuncParam);
+                break;
+            default:
+                operand1->toECLStringTranslateSource(eclStr, map, ignoreMisTranslations, forHaving, funcParam, countFuncParam);
+                break;
+        }
+     }
+
+/***********************SQLUnaryExpression END**************************************************************************/
+
+/***********************SQLParenthesisExpression START**************************************************************************/
+
+SQLParenthesisExpression::~SQLParenthesisExpression()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "\ndestroying SQLParenthesisExpression.\n");
+#endif
+    innerexpression->Release();
+}
+SQLParenthesisExpression::SQLParenthesisExpression(ISQLExpression* exp)
+{
+    this->innerexpression =  exp;
+}
+
+bool SQLParenthesisExpression::containsKey(const char* colname)
+{
+    return this->containsKey(colname);
+}
+
+void SQLParenthesisExpression::toString(StringBuffer & targetstr, bool fullOutput)
+{
+    targetstr.append("( ");
+    innerexpression->toString(targetstr, fullOutput);
+    targetstr.append(" )");
+}
+ISQLExpression* SQLParenthesisExpression::getInnerExp()
+{
+    return innerexpression;
+}
+/***********************SQLParenthesisExpression END**************************************************************************/
+
+/***********************SQLValueExpression START**************************************************************************/
+
+int SQLValueExpression::setParameterizedNames(int currentindex)
+{
+    if (hasPlaceHolder())
+    {
+        placeHolderName.setf("%sPlaceHolder%d", getPlaceHolderType(), currentindex);
+        return ++currentindex;
+    }
+    else
+        return currentindex;
+}
+
+void SQLValueExpression::toECLStringTranslateSource(
+            StringBuffer & eclStr,
+            IProperties * map,
+            bool ignoreMisTranslations,
+            bool forHaving,
+            bool funcParam,
+            bool countFuncParam)
+{
+     if (hasPlaceHolder())
+        eclStr.append(placeHolderName.str());
+    else
+        eclStr.append(value.str());
+}
+
+SQLValueExpression::SQLValueExpression()
+{
+    type = -1;
+    value = "";
+    field.setName("");
+    field.setParenttable("");
+    field.setAlias("");
+    isAWildCardPattern = false;
+}
+
+SQLValueExpression::~SQLValueExpression()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "\ndestroying SQLValueExpression.\n");
+#endif
+}
+
+SQLValueExpression::SQLValueExpression(int type, const char * value)
+{
+    this->type = type;
+    this->value = value;
+    isAWildCardPattern = false;
+}
+
+void SQLValueExpression::trimTextQuotes()
+{
+    if (this->type == TEXT_STRING)
+    {
+        if (this->value.charAt(0) == '\'' && this->value.charAt(value.length()-1) == '\'')
+        {
+            this->value.remove(this->value.length()-1, 1);
+            this->value.remove(0, 1);
+        }
+    }
+}
+
+void SQLValueExpression::toString(StringBuffer & targetstr, bool fullOutput)
+{
+    if (hasPlaceHolder())
+        targetstr.append(placeHolderName.str());
+    else
+        targetstr.append(value.str());
+}
+
+const char * SQLValueExpression::getName()
+{
+    return field.getName();
+}
+
+const char * SQLValueExpression::getNameOrAlias()
+{
+    return field.getColumnNameOrAlias();
+}
+void SQLValueExpression::setName(const char * fieldName)
+{
+    field.setName(fieldName);
+}
+
+const char * SQLValueExpression::getAlias()
+{
+    return field.getAlias();
+}
+
+void SQLValueExpression::setAlias(const char * alias)
+{
+    if (alias)
+        field.setAlias(alias);
+}
+
+void SQLValueExpression::setECLType(const char * type)
+{
+    field.setColumnType(type);
+}
+
+const char * SQLValueExpression::getECLType()
+{
+    return field.getColumnType();
+}
+
+/***********************SQLValueExpression END**************************************************************************/
+
+/***********************SQLBinaryExpression Start**************************************************************************/
+int SQLBinaryExpression::setParameterizedNames(int currentindex)
+{
+    int op1index = operand1->setParameterizedNames(currentindex);
+    return operand2->setParameterizedNames(op1index);
+}
+
+bool SQLBinaryExpression::containsEqualityCondition(ISQLExpression* operand, IProperties * map, const char * first, const char * second)
+{
+    switch (operand->getExpType())
+    {
+        case Unary_ExpressionType:
+        {
+            SQLUnaryExpression * unary = dynamic_cast<SQLUnaryExpression *>(operand);
+            return unary->containsEqualityCondition(map, first, second);
+        }
+        case Binary_ExpressionType:
+        {
+            SQLBinaryExpression * binary = dynamic_cast<SQLBinaryExpression *>(operand);
+            return binary->isEqualityCondition(map, first, second);
+        }
+        case Parenthesis_ExpressionType:
+        {
+            SQLParenthesisExpression * paren = dynamic_cast<SQLParenthesisExpression *>(operand);
+            return containsEqualityCondition(paren, map, first, second);
+        }
+        default:
+            return false;
+    }
+}
+
+bool SQLBinaryExpression::containsEqualityCondition(IProperties * map, const char * first, const char * second)
+{
+    if (isEqualityCondition(map, first, second))
+    {
+        return true;
+    }
+    else
+    {
+        bool operand1Hasequality = containsEqualityCondition(operand1, map, first, second);
+        bool operand2Hasequality = containsEqualityCondition(operand2, map, first, second);
+
+        if (op == OR_SYM)
+            return (operand1Hasequality && operand2Hasequality);
+        else
+            return (operand1Hasequality || operand2Hasequality);
+    }
+}
+
+bool SQLBinaryExpression::isEqualityCondition(IProperties * map, const char * first, const char * second)
+{
+    StringBuffer operand1Translate;
+    StringBuffer operand2Translate;
+
+    if (operand1->getExpType() == FieldValue_ExpressionType)
+    {
+        SQLFieldValueExpression * op1field = dynamic_cast<SQLFieldValueExpression *>(operand1);
+        map->getProp(op1field->getParentTableName(), operand1Translate);
+    }
+
+    if (operand2->getExpType() == FieldValue_ExpressionType)
+    {
+        SQLFieldValueExpression * op2field = dynamic_cast<SQLFieldValueExpression *>(operand2);
+        map->getProp(op2field->getParentTableName(), operand2Translate);
+    }
+
+    if (operand1Translate.length()<=0 || operand2Translate.length() <= 0)
+        return false;
+
+    return (
+                strcmp(operand1Translate.str(), operand2Translate.str()) != 0 &&
+                (
+                        (strcmp(operand1Translate.str(), first)==0 || strcmp(operand2Translate.str(), first)==0)
+                        &&
+                        (strcmp(operand1Translate.str(), second)==0 || strcmp(operand2Translate.str(), second)==0)
+                )
+            );
+}
+void SQLBinaryExpression::toECLStringTranslateSource(
+            StringBuffer & eclStr,
+            IProperties * map,
+            bool ignoreMisTranslations,
+            bool forHaving,
+            bool funcParam,
+            bool countFuncParam)
+{
+    StringBuffer translation1;
+    StringBuffer translation2;
+
+    operand1->toECLStringTranslateSource(translation1, map, ignoreMisTranslations, forHaving, false, false);
+    operand2->toECLStringTranslateSource(translation2, map, ignoreMisTranslations, forHaving, false, false);
+
+    if (translation1.length()>0 && translation2.length()>0)
+    {
+        if ( op == LIKE_SYM || op == NOT_LIKE)
+        {
+            eclStr.append(getOpStr());
+            eclStr.append("( ");
+            eclStr.append(translation1);
+            eclStr.append(" , ");
+            eclStr.append(translation2);
+            eclStr.append(" , true )");
+        }
+        else
+        {
+            eclStr.append(translation1);
+            eclStr.append(getOpStr());
+            eclStr.append(translation2);
+        }
+
+    }
+    else if (translation1.length()<0 && translation2.length()<0)
+    {
+        return;
+    }
+    else if (ignoreMisTranslations)
+    {
+        /*
+        * If operand1 or operand2 could not be translated using the translation map,
+        * and ignoreMisTranslations = true, we're going to attempt to return an valid
+        * ECL translation of this binary expression. IF the binary expression is of type
+        * OR | AND, we can substitute the mistranslated operand with the appropriate boolean value
+        * to complete the expression with out changing the gist of the expression.
+        *
+        * This is typically done when converting an SQL 'WHERE' clause or 'ON' clause to ECL to
+        * be used in an ECL JOIN function. In any one particular ECL Join funtion only two datasets
+        * are joined, therefore not all portions of the SQL logic clause might be relevant.
+        *
+        */
+        if (op == OR_SYM || op == AND_SYM)
+        {
+            StringBuffer convert( op == OR_SYM ? "FALSE" : "TRUE");
+
+            if (translation1.length()>0)
+            {
+                WARNLOG("Operand 1 of binary expression could not be translated.");
+                eclStr.append(translation1);
+                eclStr.append(getOpStr());
+                eclStr.append(convert);
+            }
+            else
+            {
+                WARNLOG("Operand 2 of binary expression could not be translated.");
+                eclStr.append(convert);
+                eclStr.append(getOpStr());
+                eclStr.append(translation2);
+            }
+        }
+        else
+        {
+            WARNLOG("Binary expression could not be translated.");
+            return;
+        }
+    }
+    else
+    {
+        WARNLOG("Binary expression could not be translated.");
+        return;
+    }
+}
+
+SQLBinaryExpression::SQLBinaryExpression(int op,ISQLExpression* operand1,ISQLExpression* operand2)
+{
+    this->operand1 = operand1;
+    this->op = op;
+    this->operand2 = operand2;
+}
+
+SQLBinaryExpression::~SQLBinaryExpression()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "\ndestroying SQLBinaryExpression.\n");
+#endif
+    if (operand1)
+        operand1->Release();
+    if (operand2)
+        operand2->Release();
+}
+
+bool SQLBinaryExpression::containsKey(const char * colname)
+{
+    return operand1->containsKey(colname) || operand2->containsKey(colname);
+}
+
+void SQLBinaryExpression::toString(StringBuffer & targetstr, bool fullOutput)
+{
+    if ( op == LIKE_SYM || op == NOT_LIKE)
+    {
+        targetstr.append(getOpStr());
+        targetstr.append("( ");
+        operand1->toString(targetstr, fullOutput);
+        targetstr.append(" , ");
+        operand2->toString(targetstr, fullOutput);
+        targetstr.append(" , true )");
+    }
+    else
+    {
+        operand1->toString(targetstr, fullOutput);
+        targetstr.append(getOpStr());
+        operand2->toString(targetstr, fullOutput);
+    }
+}
+
+const char * SQLBinaryExpression::getOpStr()
+{
+    switch (op)
+    {
+        case AND_SYM:
+            return " AND ";
+        case DIVIDE:
+            return " / ";
+        case EQ_SYM:
+            return " = ";
+        case GTH:
+            return " > ";
+        case GET:
+            return " >= ";
+        case LTH:
+            return " < ";
+        case LET:
+            return " <= ";
+        case MINUS:
+            return " - ";
+        case MOD:
+            return " % ";
+        case ASTERISK:
+            return " * ";
+        case NOT_EQ:
+            return " != ";
+        case OR_SYM:
+            return " OR ";
+        case PLUS:
+            return " + ";
+        case IN_SYM:
+            return " IN ";
+        case NOT_IN:
+            return " NOT IN ";
+        case NOT_LIKE:
+            return " NOT STD.Str.WildMatch";
+        case LIKE_SYM:
+            return " STD.Str.WildMatch";
+        default:
+            return " ";
+    }
+}
+int SQLBinaryExpression::getExpressionsCount()
+{
+    return operand1->getExpressionsCount() + operand2->getExpressionsCount();
+}
+/***********************SQLBinaryExpression END**************************************************************************/
+
+/***********************SQLParameterPlaceHolderExpression START**********************************************************/
+const char * SQLParameterPlaceHolderExpression::PARAMPREFIX = "PARAM";
+
+int SQLParameterPlaceHolderExpression::setParameterizedNames(int currentindex)
+{
+    value.set(PARAMPREFIX);
+    value.append(currentindex);
+    return ++currentindex;
+}
+
+SQLParameterPlaceHolderExpression::SQLParameterPlaceHolderExpression()
+{
+}
+
+SQLParameterPlaceHolderExpression::~SQLParameterPlaceHolderExpression()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "\ndestroying SQLParameterPlaceHolderExpression.\n");
+#endif
+}
+
+void SQLParameterPlaceHolderExpression::toString(StringBuffer & targetstr, bool fullOutput)
+{
+    targetstr.append(value.str());
+}
+
+/***********************SQLParameterPlaceHolderExpression END********************************************************/
+
+/***********************SQLFunctionExpression START**********************************************************/
+
+int SQLFunctionExpression::setParameterizedNames(int currentindex)
+{
+    int paramundex = currentindex;
+    ForEachItemIn(paramidx , params)
+    {
+        ISQLExpression & param = params.item(paramidx);
+        paramundex = param.setParameterizedNames(currentindex);
+    }
+
+    return paramundex;
+}
+void SQLFunctionExpression::toECLStringTranslateSource(
+            StringBuffer & eclStr,
+            IProperties * map,
+            bool ignoreMisTranslations,
+            bool forHaving,
+            bool funcParam,
+            bool countFuncParam)
+{
+
+    eclStr.append(function.eclFunctionName);
+    eclStr.append("( ");
+
+    ForEachItemIn(paramidx , params)
+    {
+       ISQLExpression & param = params.item(paramidx);
+       StringBuffer translation;
+       param.toECLStringTranslateSource(translation, map, ignoreMisTranslations, forHaving, true, strncmp(function.name,"COUNT", 5)==0);
+       if (!ignoreMisTranslations && translation.length()<=0)
+           return;
+       if ( paramidx != 0 )
+           eclStr.append(", ");
+       eclStr.append(translation);
+    }
+    eclStr.append(" )");
+}
+SQLFunctionExpression::SQLFunctionExpression(const char* funcname)
+{
+    setFunction(funcname);
+    this->alias = "";
+    distinct = false;
+}
+
+SQLFunctionExpression::SQLFunctionExpression(const char* funcname, const IArrayOf<ISQLExpression> &params)
+{
+    setFunction(funcname);
+    this->params = params;
+    this->alias = "";
+    distinct = false;
+}
+
+const char * SQLFunctionExpression::getAlias()
+{
+    return alias.str();
+}
+
+void SQLFunctionExpression::setAlias(const char * something)
+{
+    this->alias.append(something);
+}
+
+SQLFunctionExpression::~SQLFunctionExpression()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "Leaving SQLFunctionExpression");
+#endif
+    params.kill(false);
+}
+
+void SQLFunctionExpression::getParamsString(StringBuffer & targetstr, bool fullOutput)
+{
+    int paramcount = params.length();
+    for (int i = 0; i < paramcount; i++)
+    {
+        if ( i != 0 )
+            targetstr.append(", ");
+        ISQLExpression & exp = params.item(i);
+        exp.toString(targetstr, fullOutput);
+    }
+}
+
+void SQLFunctionExpression::toString(StringBuffer & targetstr, bool fullOutput)
+{
+    targetstr.append(function.eclFunctionName);
+    targetstr.append(" ( ");
+    getParamsString(targetstr, fullOutput);
+    targetstr.append(" )");
+}
+
+bool SQLFunctionExpression::containsKey(const char * colname)
+{
+    ForEachItemIn(Idx, params)
+    {
+        if (params.item(Idx).containsKey(colname))
+            return true;
+    }
+
+    return false;
+}
+
+int SQLFunctionExpression::getExpressionsCount()
+{
+    int count = 0;
+    ForEachItemIn(paramidx , params)
+    {
+        count += params.item(paramidx).getExpressionsCount();
+    }
+    return count;
+}
+
+const char * SQLFunctionExpression::getNameOrAlias()
+{
+    if (alias.length() > 0)
+           return getAlias();
+       else
+           return getName();
+}
+/***********************SQLFunctionExpression END********************************************************/
+
+
+/************************SQLListExpression Start *************************************************************************/
+
+SQLListExpression::SQLListExpression(){}
+SQLListExpression::~SQLListExpression()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "\ndestroying SQLListExpression.\n");
+#endif
+    entries.kill(false);
+}
+
+int SQLListExpression::getExpressionsCount()
+{
+    int count = 0;
+    ForEachItemIn(paramidx , entries)
+    {
+        count += entries.item(paramidx).getExpressionsCount();
+    }
+    return count;
+}
+
+void SQLListExpression::getExpressionFromColumnName(const char * colname, StringBuffer & str)
+{
+    StringBuffer paramlist;
+
+    StringBuffer paramresult;
+    ForEachItemIn(idx, entries)
+    {
+       entries.item(idx).getExpressionFromColumnName(colname, paramresult.clear());
+    
+       if (paramresult.length() == 0)
+           return;
+    
+       if ( idx > 0 )
+           paramlist.append(", ");
+    
+       paramlist.append(paramresult);
+    }
+    
+    if (paramlist.length()>0)
+        str.appendf(" [ %s ] ", paramlist.str());
+}
+
+void SQLListExpression::getUniqueExpressionColumnNames(StringArray &  uniquenames)
+{
+    ForEachItemIn(paramidx , entries)
+    {
+       ISQLExpression & param = entries.item(paramidx);
+       param.getUniqueExpressionColumnNames(uniquenames);
+    }
+}
+
+void SQLListExpression::eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype)
+{
+    ForEachItemIn(paramidx , entries)
+    {
+        ISQLExpression & param = entries.item(paramidx);
+        param.eclDeclarePlaceHolders(eclstr, op, sibtype);
+    }
+}
+
+SQLLogicType SQLListExpression::getLogicType()
+{
+        return Bool_LogicType;
+}
+
+int SQLListExpression::setParameterizedNames(int currentindex)
+{
+    int paramundex = currentindex;
+    ForEachItemIn(paramidx , entries)
+    {
+        ISQLExpression & param = entries.item(paramidx);
+        paramundex = param.setParameterizedNames(currentindex);
+    }
+
+    return paramundex;
+}
+
+void SQLListExpression::toECLStringTranslateSource(
+                    StringBuffer & eclStr,
+                    IProperties * map,
+                    bool ignoreMisTranslations,
+                    bool forHaving,
+                    bool funcParam,
+                    bool countFuncParam)
+{
+    eclStr.append("[ ");
+    ForEachItemIn(paramidx , entries)
+    {
+       ISQLExpression & param = entries.item(paramidx);
+       StringBuffer translation;
+       param.toECLStringTranslateSource(translation, map, ignoreMisTranslations, forHaving, funcParam, countFuncParam);
+       if (!ignoreMisTranslations && translation.length()<=0)
+           return;
+       if ( paramidx != 0 )
+           eclStr.append(", ");
+       eclStr.append(translation);
+    }
+    eclStr.append(" ]");
+}
+
+
+bool SQLListExpression::containsKey(const char* colname)
+{
+    ForEachItemIn(Idx, entries)
+    {
+        if (entries.item(Idx).containsKey(colname))
+            return true;
+    }
+    return false;
+}
+void SQLListExpression::toString(StringBuffer & targetstr, bool fullOutput)
+{
+    targetstr.append(" [ ");
+    int entrycount = entries.length();
+    ForEachItemIn(Idx, entries)
+    {
+        if ( Idx != 0 )
+            targetstr.append(", ");
+
+        ISQLExpression & exp = entries.item(Idx);
+        exp.toString(targetstr, fullOutput);
+    }
+
+    targetstr.append(" ] ");
+}
+
+void SQLListExpression::appendEntry(ISQLExpression * entry)
+{
+    entries.append(*entry);
+}
+
+/************************SQLListExpression end *************************************************************************/

esp/services/ws_sql/SQL2ECL/SQLExpression.hpp

@@ -0,0 +1,974 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+#ifndef SQLEXPRESSION_HPP_
+#define SQLEXPRESSION_HPP_
+
+#include "ws_sql.hpp"
+#include "SQLColumn.hpp"
+#include "ECLFunction.hpp"
+
+/* undef SOCKET definitions to avoid collision in Antlrdefs.h*/
+#ifdef INVALID_SOCKET
+    //#pragma message( "UNDEFINING INVALID_SOCKET - Will be redefined by ANTLRDEFS.h" )
+    #undef INVALID_SOCKET
+#endif
+#ifdef SOCKET
+    //#pragma message( "UNDEFINING SOCKET - Will be redefined by ANTLRDEFS.h" )
+    #undef SOCKET
+#endif
+#include "HPCCSQLLexer.h"
+/* undef SOCKET definitions to avoid collision in Antlrdefs.h*/
+
+typedef enum _SQLExpressionType
+{
+    Unary_ExpressionType,
+    FieldValue_ExpressionType,
+    Fields_ExpressionType,
+    Parenthesis_ExpressionType,
+    Value_ExpressionType,
+    Binary_ExpressionType,
+    ParameterPlaceHolder_ExpressionType,
+    Function_ExpressionType,
+    List_ExpressionType
+} SQLExpressionType;
+
+typedef enum _SQLLogicType
+{
+    Unknown_LogicType=-1,
+    Bool_LogicType,
+    String_LogicType,
+    QSstring_LogicType,
+    Unicode_LogicType,
+    Numeric_LogicType,
+    Integer_LogicType,
+    Decimal_LogicType
+} SQLLogicType;
+
+interface ISQLExpression : public CInterface, public IInterface
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    /*
+    * Reports whether this expression contains reference to field 'colname'.
+    * @param String name of column
+    *
+    * @return bool Reports whether this expression contains reference to field 'colname'.
+    */
+    virtual bool containsKey(const char * colname) = 0;
+
+    /*
+    * appends string representation of this SQLExpression, fully qualifies field values
+    * depending on input param's value.
+    * @param boolean Fully qualify field values.
+    *
+    * @return appends string Representation of this SQLExpression, fields fully-qualified baed on in param.
+    */
+    virtual void toString(StringBuffer & targetstr, bool fullOutput) = 0;
+
+    /*
+    * Returns number of field value expressions contained within SQL Expression.
+    *
+    * @return int number of field value expressions contained within SQL Expression.
+    */
+    virtual int getExpressionsCount() = 0;
+
+
+    /*
+    * Translates SQLExpression to ECL in flat string form.
+    * @param map IProperties containing source table names as keys and its translation target
+    * @param ignoreMistranslations option to continue translating expression even if
+    * current expression contains table.field where table not found in map
+    * @param forHaving option to translate for ECL HAVING function
+    * @param funcParam option specifying if expression to be translated is a function parameter
+    * @param countFuncParam option specifying if expression to be translated is an ECL Count() parameter
+    *
+    * @return appends string ECL consumable representation of expression.
+    */
+    virtual void toECLStringTranslateSource(StringBuffer & eclStr,
+            IProperties * map,
+            bool ignoreMisTranslations,
+            bool forHaving,
+            bool funcParam,
+            bool countFuncParam)=0;
+
+    /*
+    * Reports if this SQL expression contains an equality condition between table first and second.
+    *
+    * @param map IProperties containing source table names and their possible ECL translation
+    * @param String Name of first table.
+    * @param String Name of second table.
+    * @return bool True if this expression was found to contain equality condition between
+    * table 'first' and table 'second'.
+    */
+    virtual bool containsEqualityCondition(IProperties * map, const char * first, const char * second)
+    {
+        return false;
+    }
+
+    /*
+    * Attempts to label each parameterized expression placeholder, increases the
+    * running index value by the number of param placeholder processed.
+    *
+    * @param int currentindex index used to as postfix for placeholder label
+    *
+    * @return int running index value. To be used for next param placeholder label.
+    */
+    virtual int setParameterizedNames(int currentindex)=0;
+
+    virtual bool needsColumnExpansion() { return false;}
+
+    virtual void appendField(ISQLExpression * field){}
+
+    virtual void setECLType(const char * type) {UNIMPLEMENTED;}
+    virtual const char * getECLType() {UNIMPLEMENTED; return nullptr;}
+    virtual const char * getName() {UNIMPLEMENTED; return nullptr;}
+    virtual void   setName(const char * name) {UNIMPLEMENTED;}
+    virtual const char * getNameOrAlias() {UNIMPLEMENTED; return nullptr;}
+    virtual void   setAlias(const char * alias) {UNIMPLEMENTED;}
+    virtual const char * getAlias() {UNIMPLEMENTED; return nullptr;}
+    virtual SQLExpressionType getExpType()=0;
+
+    virtual SQLLogicType getLogicType(){UNIMPLEMENTED; return Unknown_LogicType;};
+    virtual void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype){UNIMPLEMENTED;};
+    virtual void getUniqueExpressionColumnNames(StringArray & uniquenames)=0;
+    virtual void getExpressionFromColumnName(const char * colname, StringBuffer & str)=0;
+
+    /*
+    * Instructs this expression to be replaced by a typed place holder.
+    *
+    * The purpose of this value placeholder replacement (parameterization)
+    * could be exploited for query normalization, query caching optimization,
+    * query publishing, etc.
+    */
+    virtual bool setValuePlaceHolderType(const char * ecltype){UNIMPLEMENTED; return false;};
+
+    /*
+    * Has this expression been provided a placeholder type?
+    *
+    * select * from t1 where f1 = 5; <- the right side of the binary expression f1 = 5 can be
+    *                                   parameterized as f1 = ${f1.ecltype}
+    */
+    virtual bool hasPlaceHolder(){return false;}
+
+    /*
+    * Get this placeholder inferred type
+    *
+    * select * from t1 where f1 = 5; <- the right side of the binary expression f1 = 5 can be
+    *                                   parameterized as f1 = ${f1.ecltype}
+    *                                   This allows caching of query structure
+    */
+    virtual const char * getPlaceHolderType(){UNIMPLEMENTED; return nullptr;}
+
+    /*
+    * Get this placeholder's generated name
+    */
+    virtual const char * getPlaceHolderName(){UNIMPLEMENTED; return nullptr;}
+};
+
+/*************************************************************************************************/
+class SQLListExpression : implements ISQLExpression
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    SQLListExpression();
+    virtual ~SQLListExpression();
+    void getExpressionFromColumnName(const char * colname, StringBuffer & str);
+    void getUniqueExpressionColumnNames(StringArray &  uniquenames);
+    void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype);
+    SQLLogicType getLogicType();
+    int setParameterizedNames(int currentindex);
+    void toECLStringTranslateSource(StringBuffer & eclStr, IProperties * map, bool ignoreMisTranslations, bool forHaving, bool funcParam, bool countFuncParam);
+    SQLExpressionType getExpType() { return List_ExpressionType;}
+    bool containsKey(const char* colname);
+    void toString(StringBuffer & targetstr, bool fullOutput);
+    int getExpressionsCount();
+    void appendEntry(ISQLExpression * entry);
+
+    bool setValuePlaceHolderType(const char * ecltype){return false;}
+    bool hasPlaceHolder(){ return false;}
+    const char * getPlaceHolderType() {return nullptr;}
+
+
+private:
+    IArrayOf<ISQLExpression> entries;
+};
+
+/*************************************************************************************************/
+class SQLUnaryExpression : implements ISQLExpression
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    void getExpressionFromColumnName(const char * colname, StringBuffer & str);
+    void getUniqueExpressionColumnNames(StringArray &  uniquenames)
+    {
+        operand1->getUniqueExpressionColumnNames(uniquenames);
+    }
+
+    void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype)
+    {
+        operand1->eclDeclarePlaceHolders(eclstr, op, sibtype);
+    }
+
+    SQLLogicType getLogicType(){ return operand1->getLogicType();}
+
+    int setParameterizedNames(int currentindex)
+    {
+        return operand1->setParameterizedNames(currentindex);
+    }
+
+    void toECLStringTranslateSource(StringBuffer & eclStr, IProperties * map, bool ignoreMisTranslations, bool forHaving, bool funcParam, bool countFuncParam);
+    bool containsEqualityCondition(IProperties * map, const char * first, const char * second);
+    SQLExpressionType getExpType() { return Unary_ExpressionType;}
+    SQLUnaryExpression();
+    SQLUnaryExpression(ISQLExpression* operand1, int opname);
+    virtual ~SQLUnaryExpression();
+    bool containsKey(const char* colname);
+    void toString(StringBuffer & targetstr, bool fullOutput);
+
+    int getOp() const
+    {
+        return op;
+    }
+
+    void setOp(int op)
+    {
+        this->op = op;
+    }
+
+    ISQLExpression* getOperand1() const
+    {
+        return operand1;
+    }
+
+    void setOperand1(ISQLExpression* operand1)
+    {
+        this->operand1 = operand1;
+    }
+
+    int getExpressionsCount() { return operand1->getExpressionsCount();}
+
+    bool setValuePlaceHolderType(const char * ecltype){return false;}
+    bool hasPlaceHolder(){ return false;}
+    const char * getPlaceHolderType() {return nullptr;}
+
+private:
+    ISQLExpression * operand1;
+    int op;
+};
+
+/*************************************************************************************************/
+class SQLFieldValueExpression : implements ISQLExpression
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    void getExpressionFromColumnName(const char * colname, StringBuffer & str)
+    {
+        if(stricmp(getName(),colname)==0)
+            toString(str, false);
+    }
+
+    void getUniqueExpressionColumnNames(StringArray & uniquenames)
+    {
+        uniquenames.appendUniq(getName());
+    }
+
+    void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype) {return;}
+    SQLLogicType getLogicType();
+    virtual int setParameterizedNames(int currentindex)
+    {
+        return currentindex;
+    }
+
+    void toECLStringTranslateSource(StringBuffer & eclStr, IProperties * map, bool ignoreMisTranslations, bool forHaving, bool funcParam, bool countFuncParam);
+    SQLExpressionType getExpType() { return FieldValue_ExpressionType;}
+    SQLFieldValueExpression();
+    SQLFieldValueExpression(const char * parentTableName, const char * fieldName);
+    virtual ~SQLFieldValueExpression();
+
+    const char * getParentTableName();
+    void setParentTableName(const char * parentTableName);
+    const char * getName();
+    void setName(const char * name);
+    const char * getAlias();
+    void setAlias(const char * alias);
+    bool isAscending();
+    void setAscending(bool ascending);
+    bool containsKey(const char * colname);
+    void toString(StringBuffer & targetstr, bool fullOutput);
+    int getExpressionsCount() { return 1;}
+    bool isAliasOrName(const char * possiblenameoralias);
+    void setECLType(const char * type);
+    const char * getECLType();
+    const char * getNameOrAlias();
+
+    SQLColumn * queryField()
+    {
+        return &field;
+    }
+
+    bool setValuePlaceHolderType(const char * ecltype){return false;}
+    bool hasPlaceHolder(){ return false;}
+    const char * getPlaceHolderType() {return nullptr;}
+
+private:
+    SQLColumn field;
+};
+
+/*************************************************************************************************/
+class SQLFieldsExpression : implements ISQLExpression
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    void getExpressionFromColumnName(const char * colname, StringBuffer & str){ UNIMPLEMENTED_X("This method should never be accessed for SQLFieldsExpression!");}
+    void getUniqueExpressionColumnNames(StringArray & uniquenames){ UNIMPLEMENTED_X("Method SQLFieldsExpression::getUniqueExpressionColumnNames should never be accessed!");}
+
+    void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype) {return;}
+    virtual SQLLogicType getLogicType(){return Unknown_LogicType;}
+    virtual int setParameterizedNames(int currentindex)
+    {
+        return currentindex;
+    }
+
+    void toECLStringTranslateSource(
+                StringBuffer & eclStr,
+                IProperties * map,
+                bool ignoreMisTranslations,
+                bool forHaving,
+                bool funcParam,
+                bool countFuncParam) { UNIMPLEMENTED_X("This method should never be accessed for SQLFieldsExpression!");}
+
+    SQLExpressionType getExpType() { return Fields_ExpressionType;}
+
+    SQLFieldsExpression(bool allfiles);
+    SQLFieldsExpression(const char * table);
+    virtual ~SQLFieldsExpression();
+    int getExpressionsCount() { return 1;}
+    void toString(StringBuffer & targetstr, bool fullOutput);
+    bool containsKey(const char * colname) {return false;}
+
+    bool isAll() const
+    {
+        return all;
+    }
+
+    void setAll(bool all)
+    {
+        this->all = all;
+    }
+
+    const char* getTable() const
+    {
+        return this->table.str();
+    }
+
+    void setTable(const char* table)
+    {
+        this->table.set(table);
+    }
+
+    bool needsColumnExpansion() { return !isExpanded; }
+
+    bool setValuePlaceHolderType(const char * ecltype){return false;}
+    bool hasPlaceHolder(){ return false;}
+    const char * getPlaceHolderType() {return nullptr;}
+
+private:
+    StringBuffer table;
+    bool all;
+    bool isExpanded;
+};
+
+/*************************************************************************************************/
+class SQLParenthesisExpression : implements ISQLExpression
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    void getExpressionFromColumnName(const char * colname, StringBuffer & str)
+    {
+       StringBuffer result1;
+       innerexpression->getExpressionFromColumnName(colname, result1);
+
+       //if (result1.length > 0 )
+           str.appendf("( %s )", result1.str());
+    }
+
+    void getUniqueExpressionColumnNames(StringArray & uniquenames)
+    {
+        innerexpression->getUniqueExpressionColumnNames(uniquenames);
+    }
+
+    void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype)
+    {
+        innerexpression->eclDeclarePlaceHolders(eclstr, op, sibtype);
+    }
+
+    virtual SQLLogicType getLogicType(){return innerexpression->getLogicType();}
+    virtual int setParameterizedNames(int currentindex)
+    {
+        return innerexpression->setParameterizedNames(currentindex);
+    }
+
+    void toECLStringTranslateSource(
+                        StringBuffer & eclStr,
+                        IProperties * map,
+                        bool ignoreMisTranslations,
+                        bool forHaving,
+                        bool funcParam,
+                        bool countFuncParam)
+        {
+            eclStr.append("( ");
+            innerexpression->toECLStringTranslateSource(eclStr, map, ignoreMisTranslations, forHaving, funcParam, countFuncParam);
+            eclStr.append(" )");
+        }
+
+    ISQLExpression* getInnerExp();
+    SQLExpressionType getExpType() { return Parenthesis_ExpressionType;}
+    SQLParenthesisExpression(ISQLExpression * exp);
+    virtual ~SQLParenthesisExpression();
+
+    bool containsKey(const char * colname);
+    void toString(StringBuffer & targetstr, bool fullOutput);
+    int getExpressionsCount() { return innerexpression->getExpressionsCount();}
+
+    bool setValuePlaceHolderType(const char * ecltype){return false;}
+    bool hasPlaceHolder(){ return false;}
+    const char * getPlaceHolderType() {return nullptr;}
+
+private:
+    ISQLExpression* innerexpression;
+};
+
+/*************************************************************************************************/
+class SQLValueExpression : implements ISQLExpression
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    void getExpressionFromColumnName(const char * colname, StringBuffer & str)
+    {
+        str.appendf(" %s ", value.str());
+    }
+
+    void getUniqueExpressionColumnNames(StringArray & uniquenames) { return; }
+
+    virtual SQLLogicType getLogicType()
+    {
+        const char * type = field.getColumnType();
+
+        if (strnicmp(type,"STRING",6)==0)
+            return String_LogicType;
+        else if (strnicmp(type,"QSTRING",7)==0)
+            return QSstring_LogicType;
+        else if (strnicmp(type,"UNICODE",7)==0)
+            return Unicode_LogicType;
+        else if (strnicmp(type,"VARUNICODE",10)==0)
+            return Unicode_LogicType;
+        else if (strnicmp(type,"VARSTRING",9)==0)
+            return String_LogicType;
+        else if (strnicmp(type,"BOOLEAN",7)==0)
+            return Bool_LogicType;
+        else if (strnicmp(type,"UNSIGNED",8)==0)
+            return Integer_LogicType;
+        else if (strnicmp(type,"REAL",4)==0)
+            return Decimal_LogicType;
+        else if (strnicmp(type,"DECIMAL",7)==0)
+            return Decimal_LogicType;
+        else
+            return Unknown_LogicType;
+    }
+
+    virtual int setParameterizedNames(int currentindex);
+    void toECLStringTranslateSource(
+                StringBuffer & eclStr,
+                IProperties * map,
+                bool ignoreMisTranslations,
+                bool forHaving,
+                bool funcParam,
+                bool countFuncParam);
+
+    SQLExpressionType getExpType() { return Value_ExpressionType;}
+    SQLValueExpression();
+    SQLValueExpression(int type, const char * value);
+    virtual ~SQLValueExpression();
+
+    bool containsKey(const char * colname) {return false;}
+    void toString(StringBuffer & targetstr, bool fullOutput);
+
+    int getType() const
+    {
+        return type;
+    }
+
+    void setType(int type)
+    {
+        this->type = type;
+    }
+
+    const char * getValue() const
+    {
+        return value.str();
+    }
+
+    void setValue(const char * value)
+    {
+        this->value.set(value);
+    }
+
+    int getExpressionsCount()
+    {
+        return 0;
+    }
+
+    virtual void         trimTextQuotes();
+    virtual void         setECLType(const char * type);
+    virtual const char * getECLType();
+    virtual const char * getName();
+    virtual void         setName(const char * name);
+    virtual const char * getNameOrAlias();
+    virtual void         setAlias(const char * alias);
+    virtual const char * getAlias();
+
+    bool setValuePlaceHolderType(const char * ecltype)
+    {
+        if (ecltype && *ecltype)
+        {
+            placeHolderType.set(ecltype);
+            return true;
+        }
+        return false;
+    }
+
+    bool hasPlaceHolder(){ return placeHolderType.length() > 0;}
+    const char * getPlaceHolderType() {return placeHolderType.str();}
+    const char * getPlaceHolderName() {return placeHolderName.str();}
+    void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype)
+    {
+        if(hasPlaceHolder())
+            eclstr.appendf("%s %s := %s : STORED('%s');\n", placeHolderType.str(), placeHolderName.str(), value.str(), placeHolderName.str());
+        if(isAWildCardPattern)
+        {
+            eclstr.appendf("%sTranslated := STD.Str.Translate(  %s , '_%%', '?*');", placeHolderName.str(), placeHolderName.str());
+            placeHolderName.append("Translated");
+        }
+    }
+    void setIsWildCardPattern(bool isthisawildcard) {isAWildCardPattern = isthisawildcard;}
+
+private:
+    int type; //As defined in HPCCSQLParser.h
+    StringBuffer value;
+    SQLColumn field;
+    StringBuffer placeHolderName;
+    StringBuffer placeHolderType;
+    bool isAWildCardPattern;
+};
+
+/*************************************************************************************************/
+class SQLBinaryExpression : implements ISQLExpression
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    void getExpressionFromColumnName(const char * colname, StringBuffer & str)
+    {
+        StringBuffer result1;
+        StringBuffer result2;
+        operand1->getExpressionFromColumnName(colname, result1);
+        if (!result1.length())
+            return; // no need to waste time fetching the right sub-expression
+
+        operand2->getExpressionFromColumnName(colname, result2);
+
+        if (result2.length() > 0)
+            str.appendf(" %s %s %s ", result1.str(), getOpStr(),result2.str());
+
+        /* this was meant to provide sub-expressions based on the colname provided,
+         * however providing the OR'ed sub-expression can be misleading and the logic
+         * arithmetic can be flawed, therefore this else is taken out, but will keep within
+         * comment block for reference:
+         * else if (op == OR_SYM)
+        {
+            if (result1.length() > 0)
+               str.appendf(" %s ", result1.str());
+            else if (result2.length() > 0)
+                str.appendf(" %s ", result2.str());
+        }*/
+    }
+
+    void getUniqueExpressionColumnNames(StringArray & uniquenames)
+    {
+        operand1->getUniqueExpressionColumnNames(uniquenames);
+        operand2->getUniqueExpressionColumnNames(uniquenames);
+    }
+
+    void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype)
+    {
+        operand1->eclDeclarePlaceHolders(eclstr, this->op, operand2->getLogicType());
+        operand2->eclDeclarePlaceHolders(eclstr, this->op, operand1->getLogicType());
+    }
+
+    virtual SQLLogicType getLogicType()
+    {
+        switch (op)
+        {
+            case AND_SYM:
+            case OR_SYM:
+                return Bool_LogicType;
+            case DIVIDE:
+            case GTH:
+            case GET:
+            case LTH:
+            case LET:
+            case MINUS:
+            case MOD:
+            case ASTERISK:
+            case PLUS:
+                return Numeric_LogicType;
+            case EQ_SYM:
+            case NOT_EQ:
+            case IN_SYM:
+            case NOT_IN:
+            {
+                SQLLogicType op1type =operand1->getLogicType();
+                SQLLogicType op2type =operand2->getLogicType();
+                if (op1type != Unknown_LogicType)
+                    return op1type;
+                else if (op2type != Unknown_LogicType)
+                    return op2type;
+                else
+                    return Unknown_LogicType;
+            }
+            default:
+                return Unknown_LogicType;
+        }
+    }
+
+    virtual int setParameterizedNames(int currentindex);
+    bool isEqualityCondition(IProperties * map, const char * first, const char * second);
+    virtual bool containsEqualityCondition(IProperties * map, const char * first, const char * second);
+    static bool containsEqualityCondition(ISQLExpression* operand, IProperties * map, const char * first, const char * second);
+    void toECLStringTranslateSource(
+            StringBuffer & eclStr,
+            IProperties * map,
+            bool ignoreMisTranslations,
+            bool forHaving,
+            bool funcParam,
+            bool countFuncParam);
+
+    SQLExpressionType getExpType() { return Binary_ExpressionType;}
+    SQLBinaryExpression(int op,ISQLExpression* operand1,ISQLExpression* operand2);
+    virtual ~SQLBinaryExpression();
+
+    bool containsKey(const char * colname);
+    void toString(StringBuffer & targetstr, bool fullOutput);
+    int getExpressionsCount();
+
+    int getOp() const
+    {
+        return op;
+    }
+
+    void setOp(int op)
+    {
+        this->op = op;
+    }
+
+    ISQLExpression* getOperand1() const
+    {
+        return operand1;
+    }
+
+    void setOperand1(ISQLExpression* operand1)
+    {
+        this->operand1 = operand1;
+    }
+
+    ISQLExpression* getOperand2() const
+    {
+        return operand2;
+    }
+
+    void setOperand2(ISQLExpression* operand2)
+    {
+        this->operand2 = operand2;
+    }
+
+    bool setValuePlaceHolderType(const char * ecltype){return false;}
+    bool hasPlaceHolder(){ return false;}
+    const char * getPlaceHolderType() {return nullptr;}
+
+private:
+    const char * getOpStr();
+    ISQLExpression* operand1;
+    ISQLExpression* operand2;
+    int op;
+};
+
+
+/*************************************************************************************************/
+class SQLParameterPlaceHolderExpression : implements ISQLExpression
+{
+public:
+    static const char * PARAMPREFIX;
+
+    IMPLEMENT_IINTERFACE;
+
+    void getExpressionFromColumnName(const char * colname, StringBuffer & str) { return; }
+    void getUniqueExpressionColumnNames(StringArray & uniquenames) { return; }
+
+    void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype)
+    {
+        StringBuffer defaulteclvalue;
+        switch (op)
+        {
+            case AND_SYM:
+            case OR_SYM:
+                eclstr.append( "BOOLEAN ");
+                defaulteclvalue.set(" FALSE ");
+                break;
+            case DIVIDE:
+            case MINUS:
+            case MOD:
+            case ASTERISK:
+            case GTH:
+            case GET:
+            case LTH:
+            case LET:
+            case PLUS:
+            case EQ_SYM:
+            case NOT_EQ:
+            case IN_SYM:
+            case NOT_IN:
+            {
+                switch (sibtype)
+                {
+                    case Bool_LogicType:
+                        eclstr.append( "BOOLEAN ");
+                        defaulteclvalue.set(" FALSE ");
+                        break;
+                    case Numeric_LogicType:
+                    case Integer_LogicType:
+                        eclstr.append( "INTEGER ");
+                        defaulteclvalue.set(" 0 ");
+                        break;
+                    case Decimal_LogicType:
+                        eclstr.append( "DECIMAL ");
+                        defaulteclvalue.set(" 0.0 ");
+                        break;
+                    case QSstring_LogicType:
+                        eclstr.append( "QSTRING ");
+                        defaulteclvalue.set(" '' ");
+                        break;
+                    case Unicode_LogicType:
+                        eclstr.append( "UNICODE");
+                        defaulteclvalue.set(" '' ");
+                        break;
+                    case String_LogicType:
+                    default:
+                        eclstr.append( "STRING ");
+                        defaulteclvalue.set(" '' ");
+                        break;
+                }
+                break;
+            }
+            default:
+                eclstr.append( "STRING ");
+                defaulteclvalue.set(" '' ");
+                break;
+        }
+
+        eclstr.append(value.str());
+        eclstr.append(" := ");
+        eclstr.append(defaulteclvalue.str());
+        eclstr.append(" : STORED('");
+        eclstr.append(value.str());
+        eclstr.append("');\n");
+    }
+
+    virtual SQLLogicType getLogicType(){return Unknown_LogicType;}
+    virtual int setParameterizedNames(int currentindex);
+    void toECLStringTranslateSource(
+                    StringBuffer & eclStr,
+                    IProperties * map,
+                    bool ignoreMisTranslations,
+                    bool forHaving,
+                    bool funcParam,
+                    bool countFuncParam)
+    {
+        eclStr.append( value.str() );
+    }
+
+    SQLExpressionType getExpType()
+    {
+        return ParameterPlaceHolder_ExpressionType;
+    }
+
+    SQLParameterPlaceHolderExpression();
+    virtual ~SQLParameterPlaceHolderExpression ();
+
+    bool containsKey(const char * colname) {return false;}
+    void toString(StringBuffer & targetstr, bool fullOutput);
+    int getExpressionsCount() {return 0;}
+    bool setValuePlaceHolderType(const char * ecltype){return false;}
+    bool hasPlaceHolder(){ return false;}
+    const char * getPlaceHolderType() {return nullptr;}
+
+private:
+    int index;
+    StringBuffer value;
+};
+
+/*************************************************************************************************/
+class SQLFunctionExpression : public ISQLExpression
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    void getExpressionFromColumnName(const char * colname, StringBuffer & str)
+    {
+        StringBuffer paramlist;
+
+        StringBuffer paramresult;
+        for (int i = 0; i < params.length(); i++)
+        {
+            params.item(i).getExpressionFromColumnName(colname, paramresult.clear());
+
+            if (paramresult.length() <= 0)
+                return;
+
+            if ( i > 0 )
+                paramlist.append(", ");
+
+            paramlist.append(paramresult);
+        }
+
+        if (paramlist.length()>0)
+            str.appendf(" %s( %s ) ", function.eclFunctionName, paramlist.str());
+    }
+
+    void getUniqueExpressionColumnNames(StringArray & uniquenames)
+    {
+       ForEachItemIn(paramidx , params)
+       {
+           ISQLExpression & param = params.item(paramidx);
+           param.getUniqueExpressionColumnNames(uniquenames);
+       }
+    }
+
+    void eclDeclarePlaceHolders(StringBuffer & eclstr, int op, int sibtype)
+    {
+        ForEachItemIn(paramidx , params)
+        {
+            ISQLExpression & param = params.item(paramidx);
+            param.eclDeclarePlaceHolders(eclstr, op, sibtype);
+        }
+    }
+
+    SQLLogicType getLogicType()
+    {
+        if (strcmp(function.returnType,"NUMERIC")==0)
+        {
+            if (params.length()>0)
+                return params.item(0).getLogicType();
+            else
+                return Numeric_LogicType;
+        }
+        else if (strcmp(function.returnType,"INTEGER")==0)
+            return Integer_LogicType;
+        else
+            return String_LogicType;
+    }
+
+    int setParameterizedNames(int currentindex);
+
+    void toECLStringTranslateSource(
+                StringBuffer & eclStr,
+                IProperties * map,
+                bool ignoreMisTranslations,
+                bool forHaving,
+                bool funcParam,
+                bool countFuncParam);
+
+    SQLExpressionType getExpType() { return Function_ExpressionType;}
+    SQLFunctionExpression(const char* funcname);
+    SQLFunctionExpression(const char* funcname, const IArrayOf<ISQLExpression> &params);
+    virtual ~SQLFunctionExpression();
+    bool containsKey(const char* colname);
+    void toString(StringBuffer & targetstr, bool fullOutput);
+
+    void setName(const char * funcname)
+    {
+        name.set(funcname);
+    }
+
+    void setNameAndDefaultAlias(const char * funcname)
+    {
+        name.set(funcname);
+        alias.set(funcname);
+        alias.append("Out");
+    }
+
+    const char * getName()
+    {
+        return name.str();
+    }
+
+    ECLFunctionDefCfg getFunction() const
+    {
+        return function;
+    }
+
+    void setFunction(const char * funcname)
+    {
+        this->function = ECLFunctions::getEclFuntionDef(funcname);
+    }
+
+    IArrayOf<ISQLExpression> * getParams()
+    {
+        return &params;
+    }
+
+    virtual const char * getAlias();
+    virtual void setAlias(const char * alias);
+    virtual const char * getNameOrAlias();
+    void addParams(ISQLExpression * param)
+    {
+        this->params.append(*param);
+    }
+
+    int getExpressionsCount();
+
+    bool isDistinct() {return distinct;}
+    void setDistinct(bool d) {distinct = d;}
+
+    bool setValuePlaceHolderType(const char * ecltype){return false;}
+    bool hasPlaceHolder(){ return false;}
+    const char * getPlaceHolderType() {return nullptr;}
+
+private:
+    bool distinct;
+    StringBuffer name;
+    StringBuffer alias;
+    ECLFunctionDefCfg function;
+    IArrayOf<ISQLExpression> params;
+    void getParamsString(StringBuffer & targetstr, bool fullOutput);
+};
+
+#endif /* SQLEXPRESSION_HPP_ */

esp/services/ws_sql/SQL2ECL/SQLJoin.cpp

@@ -0,0 +1,94 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#include "SQLJoin.hpp"
+
+SQLJoinType SQLJoin::defaultType = SQLJoinTypeInner;
+
+SQLJoin::SQLJoin()
+{
+    type = defaultType;
+    hasOnclause = false;
+}
+
+SQLJoin::SQLJoin(SQLJoinType jointype)
+{
+    switch (jointype)
+    {
+        case SQLJoinTypeImplicit:
+        case SQLJoinTypeInner:
+        case SQLJoinTypeOuter:
+            type = jointype;
+            break;
+        default:
+            type = defaultType;
+            break;
+    }
+    hasOnclause = false;
+}
+
+SQLJoin::~SQLJoin()
+{
+#ifdef _DEBUG
+    fprintf(stderr, "Leaving SQLJoin");
+#endif
+}
+
+void SQLJoin::getSQLTypeStr(StringBuffer & outstr)
+{
+    switch (type)
+    {
+        case SQLJoinTypeImplicit:
+        case SQLJoinTypeInner:
+            outstr.append(" INNER JOIN ");
+            break;
+        case SQLJoinTypeOuter:
+            outstr.append(" OUTER JOIN ");
+            break;
+        default:
+            outstr.append(" JOIN ");
+            break;
+    }
+}
+
+void SQLJoin::getECLTypeStr(StringBuffer & outstr)
+{
+    switch (type)
+    {
+        case SQLJoinTypeImplicit:
+        case SQLJoinTypeInner:
+            outstr.append(" INNER ");
+            break;
+        case SQLJoinTypeOuter:
+            outstr.append(" FULL OUTER ");
+            break;
+        default:
+            outstr.append(" ");
+            break;
+    }
+}
+
+void SQLJoin::toString(StringBuffer & str)
+{
+    getSQLTypeStr(str);
+
+    if (type != SQLJoinTypeImplicit)
+    {
+        str.append(" ON ");
+        onClause->toString(str, true);
+    }
+}

esp/services/ws_sql/SQL2ECL/SQLJoin.hpp

@@ -0,0 +1,90 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+#ifndef SQLJOIN_HPP_
+#define SQLJOIN_HPP_
+
+#include "ws_sql.hpp"
+#include "SQLExpression.hpp"
+
+typedef enum _SQLJoinType
+{
+    SQLJoinTypeUnknown=-1,
+    SQLJoinTypeInner,
+    SQLJoinTypeOuter,
+    SQLJoinTypeImplicit
+} SQLJoinType;
+
+class SQLJoin : public CInterface, public IInterface
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    static SQLJoin * creatSQLJoin()
+    {
+        return new SQLJoin();
+    }
+
+    static SQLJoin * creatSQLJoin(SQLJoinType jointype)
+    {
+        return new SQLJoin(jointype);
+    }
+
+    SQLJoin();
+    SQLJoin(SQLJoinType jointype);
+    virtual ~SQLJoin();
+
+    ISQLExpression* getOnClause() const
+    {
+        if (!hasOnclause)
+            return NULL;
+        else
+            return onClause.get();
+    }
+
+    void setOnClause(ISQLExpression* onClause)
+    {
+        hasOnclause = true;
+        this->onClause.setown(onClause);
+    }
+
+    SQLJoinType getType() const
+    {
+        return type;
+    }
+
+    void setType(SQLJoinType type)
+    {
+        this->type = type;
+    }
+
+    void toString(StringBuffer & str);
+    void getSQLTypeStr(StringBuffer & outstr);
+    void getECLTypeStr(StringBuffer & outstr);
+
+    bool doesHaveOnclause() const
+    {
+        return hasOnclause;
+    }
+
+private:
+    bool hasOnclause;
+    SQLJoinType type;
+    Owned<ISQLExpression> onClause;
+    static SQLJoinType defaultType;
+};
+
+#endif /* SQLJOIN_HPP_ */

esp/services/ws_sql/SQL2ECL/SQLTable.hpp

@@ -0,0 +1,111 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#ifndef SQLTABLE_HPP_
+#define SQLTABLE_HPP_
+
+#include "ws_sql.hpp"
+#include "SQLJoin.hpp"
+
+class SQLTable : public CInterface, public IInterface
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    static SQLTable * createSQLTable()
+    {
+        return new SQLTable();
+    }
+
+    SQLTable(){}
+
+    virtual ~SQLTable()
+    {
+#ifdef _DEBUG
+        fprintf(stderr, "Leaving SQLTable");
+#endif
+    }
+
+    SQLJoin* getJoin() const
+    {
+        return join.get();
+    }
+
+    void setJoin(SQLJoin* join)
+    {
+        this->join.setown(join);
+    }
+
+    void setNewJoin(SQLJoinType jointype)
+    {
+        this->join.setown(SQLJoin::creatSQLJoin(jointype));
+    }
+
+    bool hasIndexHint()
+    {
+        return indexhint.length() > 0;
+    }
+    const char* getIndexhint() const
+    {
+        return indexhint.str();
+    }
+
+    void setIndexhint(const char* indexhint)
+    {
+        this->indexhint.set(indexhint);
+    }
+
+    const char * getAlias() const
+    {
+        return alias.c_str();
+    }
+
+    void setAlias(const char * alias)
+    {
+        this->alias = alias;
+    }
+
+    const char * getName() const
+    {
+        return name.c_str();
+    }
+
+    void setName(const char * name)
+    {
+        this->name = name;
+    }
+
+    bool hasJoin()
+    {
+        return (join.get() != NULL);
+    }
+
+    const char * translateIfAlias(const char* possibleAlias)
+    {
+        if ((!alias.empty() && alias.compare(possibleAlias) == 0) || (!name.empty() && name.compare(possibleAlias) == 0))
+            return name.c_str();
+        else
+            return "";
+    }
+private:
+    string name;
+    string alias;
+    StringBuffer indexhint;
+    Owned<SQLJoin> join;
+};
+
+#endif /* SQLTABLE_HPP_ */

esp/services/ws_sql/antlr3c.cmake

@@ -0,0 +1,63 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright(C) 2017 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0(the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# generate third party library for inclusion in wssql project
+
+set(ANTLRcCONFIGURE_COMMAND_PARAMS "--silent" "--disable-antlrdebug")
+if(WIN32)
+    set(ANTLR3c_lib "antlr3c.lib")
+else()
+    set(ANTLR3c_lib "libantlr3c.so")
+endif()
+
+if(UNIX)
+    if(${ARCH64BIT} EQUAL 1)
+        set(ANTLRcCONFIGURE_COMMAND_PARAMS ${ANTLRcCONFIGURE_COMMAND_PARAMS} "--enable-64bit")
+        set(osdir "x86_64-linux-gnu")
+    else()
+        set(osdir "i386-linux-gnu")
+    endif()
+elseif(WIN32)
+    set(osdir "lib")
+else()
+    set(osdir "unknown")
+endif()
+
+include(ExternalProject)
+ExternalProject_Add(
+    antlr3c
+    URL "http://www.antlr3.org/download/C/libantlr3c-3.4.tar.gz"
+    DOWNLOAD_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/antlr3c/runtime/C
+    CONFIGURE_COMMAND ${CMAKE_CURRENT_BINARY_DIR}/antlr3c/runtime/C/configure ${ANTLRcCONFIGURE_COMMAND_PARAMS} --prefix=${CMAKE_CURRENT_BINARY_DIR}/antlr3c
+    PREFIX ${CMAKE_CURRENT_BINARY_DIR}/antlr3c
+    BUILD_COMMAND $(MAKE)
+    BUILD_IN_SOURCE 1
+    )
+
+add_library(libantlr3c SHARED IMPORTED GLOBAL)
+set_property(TARGET libantlr3c PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/antlr3c/lib/libantlr3c.so)
+add_dependencies(libantlr3c antlr3c)
+
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/antlr3c/lib/libantlr3c.so
+    DESTINATION ${LIB_DIR}/external
+    COMPONENT Runtime
+    )
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/antlr3c/runtime/C/COPYING
+    DESTINATION ${LIB_DIR}/external
+    COMPONENT Runtime
+    RENAME antlr3c-bsd-license.txt
+    )

esp/services/ws_sql/sourcedoc.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>esp/services/ws_sql</title>
+
+    <para>
+      The esp/services/ws_sql directory contains the sources for the esp/services/ws_sql library.
+    </para>
+</section>

esp/services/ws_sql/ws_sqlPlugin.cpp

@@ -0,0 +1,72 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+#include "ws_sql_esp.ipp"
+
+//ESP Bindings
+#include "http/platform/httpprot.hpp"
+
+//ESP Service
+#include "ws_sqlService.hpp"
+
+#include "espplugin.hpp"
+
+extern "C"
+{
+
+//when we aren't loading dynamically
+// Change the function names when we stick with dynamic loading.
+ESP_FACTORY IEspService * esp_service_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
+{
+    if (strcmp(type, "ws_sql")==0)
+    {
+        CwssqlEx* service = new CwssqlEx;
+        service->init(cfg, process, name);
+
+        return service;
+    }
+    return NULL;
+}
+
+ESP_FACTORY IEspRpcBinding * esp_binding_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
+{
+    if (strcmp(type, "ws_sqlSoapBinding")==0)
+    {
+        CwssqlSoapBinding* binding = new CwssqlSoapBindingEx(cfg, name, process);
+        return binding;
+    }
+    return NULL;
+}
+
+ESP_FACTORY IEspProtocol * esp_protocol_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
+{
+    if (strcmp(type, "http_protocol")==0)
+    {
+        return new CHttpProtocol;
+    }
+    else if(strcmp(type, "secure_http_protocol") == 0)
+    {
+        IPropertyTree *sslSettings;
+        sslSettings = cfg->getPropTree(StringBuffer("Software/EspProcess[@name=\"").append(process).append("\"]").append("/EspProtocol[@name=\"").append(name).append("\"]").str());
+        if(sslSettings != NULL)
+        {
+            return new CSecureHttpProtocol(sslSettings);
+        }
+    }
+    return NULL;
+}
+
+};

esp/services/ws_sql/ws_sqlService.hpp

@@ -0,0 +1,142 @@
+/*##############################################################################
+
+HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+############################################################################## */
+
+#ifndef _ESPWIZ_WS_SQL_HPP__
+#define _ESPWIZ_WS_SQL_HPP__
+
+#include <build-config.h>
+
+#include "ws_sql.hpp"
+#include "ws_sql_esp.ipp"
+
+#include "ws_ecl_wuinfo.hpp"
+#include "ws_workunitsHelpers.hpp"
+#include "ws_dfuService.hpp"
+#include "fileview.hpp"
+#include "environment.hpp"
+#include "TpWrapper.hpp"
+
+#include "HPCCFileCache.hpp"
+#include "HPCCFile.hpp"
+#include "ECLEngine.hpp"
+#include "SQLTable.hpp"
+
+#include "HPCCSQLLexer.h"
+#include "HPCCSQLParser.h"
+
+#include "HPCCSQLTreeWalker.hpp"
+
+#include "dautils.hpp"
+
+#define EMBEDDEDSQLQUERYCOMMENT "\n\n/****************************************************\nOriginal SQL:   \"%s\"\nNormalized SQL: \"%s\"\n****************************************************/\n"
+
+static const char* WSSQLACCESS = "WsSqlAccess";
+static const char* WSSQLRESULT = "WsSQLResult";
+static const char* WSSQLCOUNT  = "WsSQLCount";
+static const char* WSSQLRESULTSCHEMA = "WsSQLResultSchema";
+
+static StringBuffer g_wssqlBuildVersion;
+
+class CwssqlSoapBindingEx : public CwssqlSoapBinding
+{
+public:
+    CwssqlSoapBindingEx(IPropertyTree *cfg, const char *name, const char *process, http_soap_log_level llevel=hsl_none) : CwssqlSoapBinding(cfg, name, process, llevel)
+    {
+    }
+};
+
+class CwssqlEx : public Cwssql
+{
+private:
+    BoolHash validClusters;
+    CriticalSection crit;
+
+    IPropertyTree *cfg;
+    std::map<std::string,std::string> cachedSQLQueries;
+
+    static const unsigned int ExpireSeconds = 60 * 60;
+
+    CriticalSection critCache;
+    bool isQueryCached(const char * sqlQuery);
+    bool getCachedQuery(const char * sqlQuery, StringBuffer & wuid);
+    bool addQueryToCache(const char * sqlQuery, const char * wuid);
+    void removeQueryFromCache(const char * sqlQuery);
+    time_t cacheFlushTime;
+
+    bool isCacheExpired()
+    {
+        time_t timeNow;
+        time(&timeNow);
+        return difftime(timeNow, cacheFlushTime) > ExpireSeconds;
+    }
+
+    void setNewCacheFlushTime()
+    {
+        time(&cacheFlushTime);
+    }
+
+    void setWsSqlBuildVersion(const char* buildVersion)
+    {
+        g_wssqlBuildVersion.clear();
+        if(buildVersion&&*buildVersion)
+            g_wssqlBuildVersion.set(buildVersion);
+
+        g_wssqlBuildVersion.trim();
+    }
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    virtual void init(IPropertyTree *_cfg, const char *_process, const char *_service);
+
+    bool onEcho(IEspContext &context, IEspEchoRequest &req, IEspEchoResponse &resp);
+    bool onPrepareSQL(IEspContext &context, IEspPrepareSQLRequest &req, IEspPrepareSQLResponse &resp);
+    bool onExecuteSQL(IEspContext &context, IEspExecuteSQLRequest &req, IEspExecuteSQLResponse &resp);
+    bool getWUResult(IEspContext &context, const char * wuid, StringBuffer &resp, unsigned start, unsigned count, int sequence, const char * dsname, const char * schemaname);
+    bool onExecutePreparedSQL(IEspContext &context, IEspExecutePreparedSQLRequest &req, IEspExecutePreparedSQLResponse &resp);
+    bool onGetDBSystemInfo(IEspContext &context, IEspGetDBSystemInfoRequest &req, IEspGetDBSystemInfoResponse &resp);
+    bool onGetDBMetaData(IEspContext &context, IEspGetDBMetaDataRequest &req, IEspGetDBMetaDataResponse &resp);
+    bool onGetResults(IEspContext &context, IEspGetResultsRequest &req, IEspGetResultsResponse &resp);
+    bool onGetRelatedIndexes(IEspContext &context, IEspGetRelatedIndexesRequest &req, IEspGetRelatedIndexesResponse &resp);
+    bool onSetRelatedIndexes(IEspContext &context, IEspSetRelatedIndexesRequest &req, IEspSetRelatedIndexesResponse &resp);
+    bool onCreateTableAndLoad(IEspContext &context, IEspCreateTableAndLoadRequest &req, IEspCreateTableAndLoadResponse &resp);
+
+    void refreshValidClusters();
+    bool isValidCluster(const char *cluster);
+    void processMultipleClusterOption(StringArray & clusters, const char  * targetcluster, StringBuffer & hashoptions);
+
+    void fetchRequiredHpccFiles(IArrayOf<SQLTable> * sqltables);
+    static void fetchRequiredHpccFiles(IArrayOf<SQLTable> * sqltables, HpccFiles * hpccfilecache);
+    HPCCSQLTreeWalker * parseSQL(IEspContext &context, StringBuffer & sqltext, bool attemptParameterization = true);
+
+    bool executePublishedQueryByName(IEspContext &context, const char * queryset, const char * queryname, StringBuffer &clonedwuid, const char *paramXml, IArrayOf<IConstNamedValue> *variables, const char * targetcluster, int start, int count);
+    bool executePublishedQueryByWuId(IEspContext &context, const char * targetwuid, StringBuffer &clonedwuid, const char *paramXml, IArrayOf<IConstNamedValue> *variables, const char * targetcluster, int start, int count);
+    bool executePublishedQuery(IEspContext &context, const char * queryset, const char * queryname, StringBuffer &resp, int start, int count, int waittime);
+    bool executePublishedQuery(IEspContext &context, const char * wuid, StringBuffer &resp, int start, int count, int waittime);
+    bool cloneAndExecuteWU(IEspContext &context, const char * originalwuid, StringBuffer &clonedwuid, const char *paramXml, IArrayOf<IConstNamedValue> *variables, IArrayOf<IConstNamedValue> *debugs, const char * targetcluster);
+    bool publishWorkunit(IEspContext &context, const char * queryname, const char * wuid, const char * targetcluster);
+
+    static void createWUXMLParams(StringBuffer & xmlparams, HPCCSQLTreeWalker* parsedSQL, IArrayOf<IConstNamedValue> *variables, IConstWorkUnit * cw);
+    static void createWUXMLParams(StringBuffer & xmlparams, const IArrayOf <ISQLExpression> * parameterlist);
+
+    const char* getWsSqlBuildVersion()
+    {
+        return g_wssqlBuildVersion.str();
+    }
+};
+
+#endif //_ESPWIZ_WS_SQL_HPP__

initfiles/componentfiles/configxml/@temp/CMakeLists.txt

@@ -24,6 +24,7 @@ FOREACH ( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_DynamicESDL.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_wslogging.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/logging_agent.xsl
+    ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_wssql.xsl
 )
     Install ( FILES ${iFILES} DESTINATION componentfiles/configxml/@temp COMPONENT Runtime )
 ENDFOREACH ( iFILES )

initfiles/componentfiles/configxml/@temp/esp_service_wssql.xsl

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xml:space="default" 
+xmlns:set="http://exslt.org/sets" exclude-result-prefixes="set">
+
+<xsl:import href="esp_service_WsSMC.xsl"/>
+
+    <xsl:template match="EspBinding">
+        <xsl:param name="authNode"/>
+
+     <xsl:variable name="serviceNode" select="/Environment/Software/EspService[@name=$espServiceName and @name=current()/@service and Properties/@type='ws_sql']"/>
+        <xsl:apply-templates select="$serviceNode" mode="WsSQL">
+            <xsl:with-param name="bindingNode" select="."/>
+            <xsl:with-param name="authNode" select="$authNode"/>
+        </xsl:apply-templates>
+    </xsl:template>
+
+    <!-- WS-SQL -->
+    <xsl:template match="EspService" mode="WsSQL">
+        <xsl:param name="bindingNode"/>
+        <xsl:param name="authNode"/>
+
+        <xsl:variable name="serviceType" select="'ws_sql'"/>
+        <xsl:variable name="serviceName" select="concat($serviceType, '_', @name, '_', $process)"/>
+        <xsl:variable name="bindName" select="concat($serviceType, '_', $bindingNode/@name, '_', $process)"/>
+        <xsl:variable name="bindType" select="'ws_sqlSoapBinding'"/>
+        <xsl:variable name="servicePlugin">
+            <xsl:call-template name="defineServicePlugin">
+                <xsl:with-param name="plugin" select="'ws_sql'"/>
+            </xsl:call-template>
+        </xsl:variable>
+        <EspService name="{$serviceName}" type="{$serviceType}" plugin="{$servicePlugin}"/>
+        <EspBinding name="{$bindName}" service="{$serviceName}" protocol="{$bindingNode/@protocol}" type="{$bindType}"
+            plugin="{$servicePlugin}" netAddress="0.0.0.0" port="{$bindingNode/@port}" defaultBinding="true">
+            <xsl:call-template name="bindAuthentication">
+                <xsl:with-param name="bindingNode" select="$bindingNode"/>
+                <xsl:with-param name="authMethod" select="$authNode/@method"/>
+                <xsl:with-param name="service" select="'ws_sql'"/>
+            </xsl:call-template>
+        </EspBinding>
+    </xsl:template>
+</xsl:stylesheet>
+

initfiles/componentfiles/configxml/CMakeLists.txt

@@ -55,6 +55,7 @@ FOREACH( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_account.xsd
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_wsecl.xsd
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_wsecl2.xsd
+    ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_wssql.xsd
     ${CMAKE_CURRENT_SOURCE_DIR}/GABConfig.xsd
     ${CMAKE_CURRENT_SOURCE_DIR}/generic.xsd
     ${CMAKE_CURRENT_SOURCE_DIR}/installset.xsd

initfiles/componentfiles/configxml/buildsetCC.xml.in

@@ -17,341 +17,372 @@
 ################################################################################
 -->
 <Environment>
-<Software>
-    <Directories name="${DIR_NAME}">
-      <Category dir="${EXEC_PREFIX}/log/[NAME]/[INST]" name="log"/>
-      <Category dir="${EXEC_PREFIX}/lib/[NAME]/[INST]" name="run"/>
-      <Category dir="${CONFIG_PREFIX}/[NAME]/[INST]" name="conf"/>
-      <Category dir="${EXEC_PREFIX}/lib/[NAME]/[INST]/temp" name="temp"/>
-      <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data/[COMPONENT]" name="data"/>
-      <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data2/[COMPONENT]" name="data2"/>
-      <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data3/[COMPONENT]" name="data3"/>
-      <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-mirror/[COMPONENT]" name="mirror"/>
-      <Category dir="${EXEC_PREFIX}/lib/[NAME]/queries/[INST]" name="query"/>
-      <Category dir="${EXEC_PREFIX}/lock/[NAME]/[INST]" name="lock"/>
-    </Directories>  
-</Software>
-<Programs>
-  <Build name="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}" url="${PREFIX}/${DIR_NAME}">
-   <BuildSet installSet="deploy_map.xml"
-             name="dafilesrv"
-             path="componentfiles/dafilesrv"
-             processName="DafilesrvProcess"
-             schema="dafilesrv.xsd"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="dali"
-             path="componentfiles/dali"
-             processName="DaliServerProcess"
-             schema="dali.xsd"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="backupnode"
-             path="componentfiles/backupnode"
-             processName="BackupNodeProcess"
-             schema="backupnode.xsd"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="daliplugin"
-             path="componentfiles/daliplugin"
-             processName="DaliServerPlugin"
-             schema="daliplugin.xsd"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="dfuplus"
-             path="componentfiles/dfuplus"
-             processName="DfuplusProcess"
-             schema="dfuplus.xsd"
-             overide="no"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="dfuserver"
-             path="componentfiles/dfuserver"
-             processName="DfuServerProcess"
-             schema="dfuserver.xsd"/>
-   <BuildSet deployable="no"
-             installSet="deploy_map.xml"
-             name="DropZone"
-             path="componentfiles/DropZone"
-             processName="DropZone"
-             schema="dropzone.xsd"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="eclagent"
-             path="componentfiles/eclagent"
-             processName="EclAgentProcess"
-             schema="eclagent_config.xsd"/>
-   <BuildSet installSet="deploy_map.xml" 
-             name="eclminus" 
-             path="componentfiles/eclminus"
-             overide="no"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="eclplus"
-             path="componentfiles/eclplus"
-             processName="EclPlusProcess"
-             schema="eclplus.xsd"
-             overide="no"/>
-   <BuildSet installSet="eclccserver_deploy_map.xml"
-             name="eclccserver"
-             path="componentfiles/configxml"
-             processName="EclCCServerProcess"
-             schema="eclccserver.xsd"/>
-   <BuildSet installSet="eclscheduler_deploy_map.xml"
-             name="eclscheduler"
-             path="componentfiles/configxml"
-             processName="EclSchedulerProcess"
-             schema="eclscheduler.xsd"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="esp"
-             path="componentfiles/esp"
-             processName="EspProcess"
-             schema="esp.xsd"/>
-   <BuildSet deployable="no"
-             installSet="deploy_map.xml"
-             name="espsmc"
-             path="componentfiles/espsmc"
-             processName="EspService"
-             schema="espsmcservice.xsd">
-    <Properties defaultPort="8010"
-                defaultResourcesBasedn="ou=SMC,ou=EspServices,ou=ecl"
-                defaultSecurePort="18010"
-                type="WsSMC">
-     <Authenticate access="Read"
-                   description="Root access to SMC service"
-                   path="/"
-                   required="Read"
-                   resource="SmcAccess"/>
-     <AuthenticateFeature description="Access to SMC service"
-                          path="SmcAccess"
-                          resource="SmcAccess"
-                          service="ws_smc"/>
-     <AuthenticateFeature description="Access to thor queues"
-                          path="ThorQueueAccess"
-                          resource="ThorQueueAccess"
-                          service="ws_smc"/>
-      <AuthenticateFeature description="Access to roxie control commands"
-                           path="RoxieControlAccess"
-                           resource="RoxieControlAccess"
-                           service="ws_smc"/>
-     <AuthenticateFeature description="Access to DFU"
-                          path="DfuAccess"
-                          resource="DfuAccess"
-                          service="ws_dfu"/>
-     <AuthenticateFeature description="Access to DFU XRef"
-                          path="DfuXrefAccess"
-                          resource="DfuXrefAccess"
-                          service="ws_dfuxref"/>
-     <AuthenticateFeature description="Access to machine information"
-                          path="MachineInfoAccess"
-                          resource="MachineInfoAccess"
-                          service="ws_machine"/>
-     <AuthenticateFeature description="Access to SNMP metrics information"
-                          path="MetricsAccess"
-                          resource="MetricsAccess"
-                          service="ws_machine"/>
-     <AuthenticateFeature description="Access to DFU workunits"
-                          path="DfuWorkunitsAccess"
-                          resource="DfuWorkunitsAccess"
-                          service="ws_fs"/>
-     <AuthenticateFeature description="Access to DFU exceptions"
-                          path="DfuExceptionsAccess"
-                          resource="DfuExceptions"
-                          service="ws_fs"/>
-     <AuthenticateFeature description="Access to spraying files"
-                          path="FileSprayAccess"
-                          resource="FileSprayAccess"
-                          service="ws_fs"/>
-     <AuthenticateFeature description="Access to despraying of files"
-                          path="FileDesprayAccess"
-                          resource="FileDesprayAccess"
-                          service="ws_fs"/>
-     <AuthenticateFeature description="Access to upload files to dropzone"
-                          path="FileUploadAccess"
-                          resource="FileUploadAccess"
-                          service="ws_fs" />
-     <AuthenticateFeature description="Access to files in dropzone"
-                          path="FileIOAccess"
-                          resource="FileIOAccess"
-                          service="ws_fileio"/>
-     <AuthenticateFeature description="Access to package map"
-                          path="PackageMapAccess"
-                          resource="PackageMapAccess"
-                          service="ws_packageprocess"/>
-     <AuthenticateFeature description="Access to permissions for file scopes"
-                          path="FileScopeAccess"
-                          resource="FileScopeAccess"
-                          service="ws_access"/>
-     <AuthenticateFeature description="Access to WS ECL service"
-                          path="WsEclAccess"
-                          resource="WsEclAccess"
-                          service="ws_ecl"/>
-     <AuthenticateFeature description="Access to cluster topology"
-                          path="ClusterTopologyAccess"
-                          resource="ClusterTopologyAccess"
-                          service="ws_topology"/>
-     <AuthenticateFeature description="Access to own workunits"
-                          path="OwnWorkunitsAccess"
-                          resource="OwnWorkunitsAccess"
-                          service="ws_workunits"/>
-     <AuthenticateFeature description="Access to others&apos; workunits"
-                          path="OthersWorkunitsAccess"
-                          resource="OthersWorkunitsAccess"
-                          service="ws_workunits"/>
-    <AuthenticateFeature description="Access to ESDL configuration service"
-                          path="ESDLConfigAccess"
-                          resource="ESDLConfigAccess"
-                          service="ws_esdlconfig"/>
-     <ProcessFilters>
-      <Platform name="Windows">
-       <ProcessFilter name="any">
-        <Process name="dafilesrv"/>
-       </ProcessFilter>
-       <ProcessFilter multipleInstances="true" name="DfuServerProcess"/>
-       <ProcessFilter multipleInstances="true" name="EclCCServerProcess"/>
-       <ProcessFilter multipleInstances="true" name="EspProcess">
-        <Process name="dafilesrv" remove="true"/>
-       </ProcessFilter>
-      </Platform>
-      <Platform name="Linux">
-       <ProcessFilter name="any">
-        <Process name="dafilesrv"/>
-       </ProcessFilter>
-       <ProcessFilter multipleInstances="true" name="DfuServerProcess"/>
-       <ProcessFilter multipleInstances="true" name="EclCCServerProcess"/>
-       <ProcessFilter multipleInstances="true" name="EspProcess">
-        <Process name="dafilesrv" remove="true"/>
-       </ProcessFilter>
-       <ProcessFilter name="GenesisServerProcess">
-        <Process name="httpd"/>
-        <Process name="atftpd"/>
-        <Process name="dhcpd"/>
-       </ProcessFilter>
-      </Platform>
-     </ProcessFilters>
-    </Properties>
-   </BuildSet>
-   <BuildSet installSet="deploy_map.xml"
-             name="ftslave"
-             path="componentfiles/ftslave"
-             processName="FTSlaveProcess"
-             schema="ftslave_linux.xsd"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="hqltest"
-             path="componentfiles/hqltest"
-             processName="HqlTestProcess"
-             overide="no"/>
-    <BuildSet deployable="no"
-              installSet="deploy_map.xml"
-              name="ldapServer"
-              path="componentfiles/ldapServer"
-              processName="LDAPServerProcess"
-              schema="ldapserver.xsd"/>
-   <BuildSet installSet="roxie_deploy_map.xml"
-             name="roxie"
-             path="componentfiles/configxml"
-             processName="RoxieCluster"
-             schema="roxie.xsd"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="sasha"
-             path="componentfiles/sasha"
-             processName="SashaServerProcess"
-             schema="sasha.xsd"/>
-   <BuildSet deployable="no"
-             installSet="deploy_map.xml"
-             name="SiteCertificate"
-             path="componentfiles/SiteCertificate"
-             processName="SiteCertificate"
-             schema="SiteCertificate.xsd"
-             overide="no"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="soapplus"
-             path="componentfiles/soapplus"
-             processName="SoapPlusProcess"
-             schema="soapplus.xsd"
-             overide="no"/>
-   <BuildSet installSet="deploy_map.xml"
-             name="thor"
-             path="componentfiles/thor"
-             processName="ThorCluster"
-             schema="thor.xsd"/>
-   <BuildSet deployable="no"
-             installSet="deploy_map.xml"
-             name="topology"
-             path="componentfiles/topology"
-             processName="Topology"
-             schema="topology.xsd"/>
-    <BuildSet deployable="no"
-              installSet="deploy_map.xml"
-              name="ws_ecl"
-              path="componentfiles/ws_ecl"
-              processName="EspService"
-              schema="esp_service_wsecl2.xsd">
-      <Properties bindingType="ws_eclSoapBinding"
-                  defaultPort="8002"
-                  defaultResourcesBasedn="ou=WsEcl,ou=EspServices,ou=ecl"
-                  defaultSecurePort="18002"
-                  plugin="ws_ecl"
-                  type="ws_ecl">
-        <Authenticate access="Read"
-                      description="Root access to WS ECL service"
-                      path="/"
-                      required="Read"
-                      resource="WsEclAccess"/>
-        <AuthenticateFeature description="Access to WS ECL service"
-                             path="WsEclAccess"
-                             resource="WsEclAccess"
-                             service="ws_ecl"/>
-      </Properties>
-    </BuildSet>
-   <BuildSet deployable="no"
-             installSet="deploy_map.xml"
-             name="DynamicESDL"
-             path="componentfiles/esdlsvcengine"
-             processName="EspService"
-             schema="esdlsvcengine.xsd">
-     <Properties bindingType="EsdlBinding"
-                 defaultPort="8043"
-                 defaultResourcesBasedn="ou=EspServices,ou=ecl"
-                 defaultSecurePort="18043"
-                 plugin="esdl_svc_engine"
-                 type="DynamicESDL"/>
-    </BuildSet>
-    <BuildSet deployable="no"
-             installSet="deploy_map.xml"
-             name="cassandraloggingagent"
-             path="componentfiles/cassandraloggingagent"
-             processName="CassandraLoggingAgent"
-             schema="cassandraloggingagent.xsd"/>
-    <BuildSet deployable="no"
-             installSet="deploy_map.xml"
-             name="esploggingagent"
-             path="componentfiles/esploggingagent"
-             processName="ESPLoggingAgent"
-             schema="esploggingagent.xsd"/>
-    <BuildSet deployable="no"
-             installSet="deploy_map.xml"
-             name="loggingmanager"
-             path="componentfiles/loggingmanager"
-             processName="LoggingManager"
-             schema="loggingmanager.xsd"/>
-    <BuildSet deployable="no"
-             installSet="deploy_map.xml"
-             name="wslogging"
-             path="componentfiles/wslogging"
-             processName="EspService"
-             schema="wslogging.xsd">
-      <Properties bindingType="loggingservice_binding"
-                defaultPort="8146"
-                defaultResourcesBasedn="ou=EspServices,ou=ecl"
-                defaultSecurePort="18146"
-                plugin="wslogging"
-                type="wslogging">
-        <Authenticate access="Read"
-                   description="Root access to WS Logging service"
-                   path="/"
-                   required="Read"
-                   resource="WsLoggingAccess"/>
-        <AuthenticateFeature description="Access to WS Logging service"
-                   path="WsLoggingAccess"
-                   resource="WsLoggingAccess"
-                   service="wslogging"/>
-    </Properties>
-   </BuildSet>
-  </Build>
-</Programs>
+    <Software>
+        <Directories name="${DIR_NAME}">
+            <Category dir="${EXEC_PREFIX}/log/[NAME]/[INST]" name="log"/>
+            <Category dir="${EXEC_PREFIX}/lib/[NAME]/[INST]" name="run"/>
+            <Category dir="${CONFIG_PREFIX}/[NAME]/[INST]" name="conf"/>
+            <Category dir="${EXEC_PREFIX}/lib/[NAME]/[INST]/temp" name="temp"/>
+            <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data/[COMPONENT]" name="data"/>
+            <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data2/[COMPONENT]" name="data2"/>
+            <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-data3/[COMPONENT]" name="data3"/>
+            <Category dir="${EXEC_PREFIX}/lib/[NAME]/hpcc-mirror/[COMPONENT]" name="mirror"/>
+            <Category dir="${EXEC_PREFIX}/lib/[NAME]/queries/[INST]" name="query"/>
+            <Category dir="${EXEC_PREFIX}/lock/[NAME]/[INST]" name="lock"/>
+        </Directories>  
+    </Software>
+    <Programs>
+        <Build name="${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}" url="${PREFIX}/${DIR_NAME}">
+            <BuildSet installSet="deploy_map.xml"
+                name="dafilesrv"
+                path="componentfiles/dafilesrv"
+                processName="DafilesrvProcess"
+                schema="dafilesrv.xsd"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="dali"
+                path="componentfiles/dali"
+                processName="DaliServerProcess"
+                schema="dali.xsd"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="backupnode"
+                path="componentfiles/backupnode"
+                processName="BackupNodeProcess"
+                schema="backupnode.xsd"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="daliplugin"
+                path="componentfiles/daliplugin"
+                processName="DaliServerPlugin"
+                schema="daliplugin.xsd"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="dfuplus"
+                path="componentfiles/dfuplus"
+                processName="DfuplusProcess"
+                schema="dfuplus.xsd"
+                overide="no"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="dfuserver"
+                path="componentfiles/dfuserver"
+                processName="DfuServerProcess"
+                schema="dfuserver.xsd"/>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="DropZone"
+                path="componentfiles/DropZone"
+                processName="DropZone"
+                schema="dropzone.xsd"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="eclagent"
+                path="componentfiles/eclagent"
+                processName="EclAgentProcess"
+                schema="eclagent_config.xsd"/>
+            <BuildSet installSet="deploy_map.xml" 
+                name="eclminus" 
+                path="componentfiles/eclminus"
+                overide="no"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="eclplus"
+                path="componentfiles/eclplus"
+                processName="EclPlusProcess"
+                schema="eclplus.xsd"
+                overide="no"/>
+            <BuildSet installSet="eclccserver_deploy_map.xml"
+                name="eclccserver"
+                path="componentfiles/configxml"
+                processName="EclCCServerProcess"
+                schema="eclccserver.xsd"/>
+            <BuildSet installSet="eclscheduler_deploy_map.xml"
+                name="eclscheduler"
+                path="componentfiles/configxml"
+                processName="EclSchedulerProcess"
+                schema="eclscheduler.xsd"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="esp"
+                path="componentfiles/esp"
+                processName="EspProcess"
+                schema="esp.xsd"/>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="espsmc"
+                path="componentfiles/espsmc"
+                processName="EspService"
+                schema="espsmcservice.xsd">
+                <Properties defaultPort="8010"
+                    defaultResourcesBasedn="ou=SMC,ou=EspServices,ou=ecl"
+                    defaultSecurePort="18010"
+                    type="WsSMC">
+                    <Authenticate access="Read"
+                        description="Root access to SMC service"
+                        path="/"
+                        required="Read"
+                        resource="SmcAccess"/>
+                    <AuthenticateFeature description="Access to SMC service"
+                        path="SmcAccess"
+                        resource="SmcAccess"
+                        service="ws_smc"/>
+                    <AuthenticateFeature description="Access to thor queues"
+                        path="ThorQueueAccess"
+                        resource="ThorQueueAccess"
+                        service="ws_smc"/>
+                    <AuthenticateFeature description="Access to roxie control commands"
+                        path="RoxieControlAccess"
+                        resource="RoxieControlAccess"
+                        service="ws_smc"/>
+                    <AuthenticateFeature description="Access to DFU"
+                        path="DfuAccess"
+                        resource="DfuAccess"
+                        service="ws_dfu"/>
+                    <AuthenticateFeature description="Access to DFU XRef"
+                        path="DfuXrefAccess"
+                        resource="DfuXrefAccess"
+                        service="ws_dfuxref"/>
+                    <AuthenticateFeature description="Access to machine information"
+                        path="MachineInfoAccess"
+                        resource="MachineInfoAccess"
+                        service="ws_machine"/>
+                    <AuthenticateFeature description="Access to SNMP metrics information"
+                        path="MetricsAccess"
+                        resource="MetricsAccess"
+                        service="ws_machine"/>
+                    <AuthenticateFeature description="Access to DFU workunits"
+                        path="DfuWorkunitsAccess"
+                        resource="DfuWorkunitsAccess"
+                        service="ws_fs"/>
+                    <AuthenticateFeature description="Access to DFU exceptions"
+                        path="DfuExceptionsAccess"
+                        resource="DfuExceptions"
+                        service="ws_fs"/>
+                    <AuthenticateFeature description="Access to spraying files"
+                        path="FileSprayAccess"
+                        resource="FileSprayAccess"
+                        service="ws_fs"/>
+                    <AuthenticateFeature description="Access to despraying of files"
+                        path="FileDesprayAccess"
+                        resource="FileDesprayAccess"
+                        service="ws_fs"/>
+                    <AuthenticateFeature description="Access to upload files to dropzone"
+                        path="FileUploadAccess"
+                        resource="FileUploadAccess"
+                        service="ws_fs" />
+                    <AuthenticateFeature description="Access to files in dropzone"
+                        path="FileIOAccess"
+                        resource="FileIOAccess"
+                        service="ws_fileio"/>
+                    <AuthenticateFeature description="Access to package map"
+                        path="PackageMapAccess"
+                        resource="PackageMapAccess"
+                        service="ws_packageprocess"/>
+                    <AuthenticateFeature description="Access to permissions for file scopes"
+                        path="FileScopeAccess"
+                        resource="FileScopeAccess"
+                        service="ws_access"/>
+                    <AuthenticateFeature description="Access to WS ECL service"
+                        path="WsEclAccess"
+                        resource="WsEclAccess"
+                        service="ws_ecl"/>
+                    <AuthenticateFeature description="Access to cluster topology"
+                        path="ClusterTopologyAccess"
+                        resource="ClusterTopologyAccess"
+                        service="ws_topology"/>
+                    <AuthenticateFeature description="Access to own workunits"
+                        path="OwnWorkunitsAccess"
+                        resource="OwnWorkunitsAccess"
+                        service="ws_workunits"/>
+                    <AuthenticateFeature description="Access to others&apos; workunits"
+                        path="OthersWorkunitsAccess"
+                        resource="OthersWorkunitsAccess"
+                        service="ws_workunits"/>
+                    <AuthenticateFeature description="Access to ESDL configuration service"
+                        path="ESDLConfigAccess"
+                        resource="ESDLConfigAccess"
+                        service="ws_esdlconfig"/>
+                    <ProcessFilters>
+                        <Platform name="Windows">
+                            <ProcessFilter name="any">
+                                <Process name="dafilesrv"/>
+                            </ProcessFilter>
+                            <ProcessFilter multipleInstances="true" name="DfuServerProcess"/>
+                            <ProcessFilter multipleInstances="true" name="EclCCServerProcess"/>
+                            <ProcessFilter multipleInstances="true" name="EspProcess">
+                                <Process name="dafilesrv" remove="true"/>
+                            </ProcessFilter>
+                        </Platform>
+                        <Platform name="Linux">
+                            <ProcessFilter name="any">
+                                <Process name="dafilesrv"/>
+                            </ProcessFilter>
+                            <ProcessFilter multipleInstances="true" name="DfuServerProcess"/>
+                            <ProcessFilter multipleInstances="true" name="EclCCServerProcess"/>
+                            <ProcessFilter multipleInstances="true" name="EspProcess">
+                                <Process name="dafilesrv" remove="true"/>
+                            </ProcessFilter>
+                            <ProcessFilter name="GenesisServerProcess">
+                                <Process name="httpd"/>
+                                <Process name="atftpd"/>
+                                <Process name="dhcpd"/>
+                            </ProcessFilter>
+                        </Platform>
+                    </ProcessFilters>
+                </Properties>
+            </BuildSet>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="ws_sql"
+                path="componentfiles/ws_sql"
+                processName="EspService"
+                schema="esp_service_wssql.xsd">
+                <Properties bindingType="ws_sqlSoapBinding"
+                    defaultPort="8510"
+                    defaultResourcesBasedn="ou=WsSql,ou=EspServices,ou=ecl"
+                    defaultSecurePort="18510"
+                    plugin="ws_sql"
+                    type="ws_sql">
+                    <Authenticate access="Read"
+                        description="Root access to WS SQL service"
+                        path="/"
+                        required="Read"
+                        resource="WsSqlAccess"/>
+                    <AuthenticateFeature description="Access to WS SQL service"
+                        path="WsSQLAccess"
+                        resource="WsSqlAccess"
+                        service="ws_sql"/>
+                    <AuthenticateFeature description="Access to own workunits"
+                        path="OwnWorkunitsAccess"
+                        resource="OwnWorkunitsAccess"
+                        service="ws_sql"/>
+                    <AuthenticateFeature description="Access to others&apos; workunits"
+                        path="OthersWorkunitsAccess"
+                        resource="OthersWorkunitsAccess"
+                        service="ws_sql"/>
+                </Properties>
+            </BuildSet>
+            <BuildSet installSet="deploy_map.xml"
+                name="ftslave"
+                path="componentfiles/ftslave"
+                processName="FTSlaveProcess"
+                schema="ftslave_linux.xsd"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="hqltest"
+                path="componentfiles/hqltest"
+                processName="HqlTestProcess"
+                overide="no"/>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="ldapServer"
+                path="componentfiles/ldapServer"
+                processName="LDAPServerProcess"
+                schema="ldapserver.xsd"/>
+            <BuildSet installSet="roxie_deploy_map.xml"
+                name="roxie"
+                path="componentfiles/configxml"
+                processName="RoxieCluster"
+                schema="roxie.xsd"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="sasha"
+                path="componentfiles/sasha"
+                processName="SashaServerProcess"
+                schema="sasha.xsd"/>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="SiteCertificate"
+                path="componentfiles/SiteCertificate"
+                processName="SiteCertificate"
+                schema="SiteCertificate.xsd"
+                overide="no"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="soapplus"
+                path="componentfiles/soapplus"
+                processName="SoapPlusProcess"
+                schema="soapplus.xsd"
+                overide="no"/>
+            <BuildSet installSet="deploy_map.xml"
+                name="thor"
+                path="componentfiles/thor"
+                processName="ThorCluster"
+                schema="thor.xsd"/>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="topology"
+                path="componentfiles/topology"
+                processName="Topology"
+                schema="topology.xsd"/>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="ws_ecl"
+                path="componentfiles/ws_ecl"
+                processName="EspService"
+                schema="esp_service_wsecl2.xsd">
+                <Properties bindingType="ws_eclSoapBinding"
+                    defaultPort="8002"
+                    defaultResourcesBasedn="ou=WsEcl,ou=EspServices,ou=ecl"
+                    defaultSecurePort="18002"
+                    plugin="ws_ecl"
+                    type="ws_ecl">
+                    <Authenticate access="Read"
+                        description="Root access to WS ECL service"
+                        path="/"
+                        required="Read"
+                        resource="WsEclAccess"/>
+                    <AuthenticateFeature description="Access to WS ECL service"
+                        path="WsEclAccess"
+                        resource="WsEclAccess"
+                        service="ws_ecl"/>
+                </Properties>
+            </BuildSet>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="DynamicESDL"
+                path="componentfiles/esdlsvcengine"
+                processName="EspService"
+                schema="esdlsvcengine.xsd">
+                <Properties bindingType="EsdlBinding"
+                    defaultPort="8043"
+                    defaultResourcesBasedn="ou=EspServices,ou=ecl"
+                    defaultSecurePort="18043"
+                    plugin="esdl_svc_engine"
+                    type="DynamicESDL"/>
+            </BuildSet>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="cassandraloggingagent"
+                path="componentfiles/cassandraloggingagent"
+                processName="CassandraLoggingAgent"
+                schema="cassandraloggingagent.xsd"/>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="esploggingagent"
+                path="componentfiles/esploggingagent"
+                processName="ESPLoggingAgent"
+                schema="esploggingagent.xsd"/>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="loggingmanager"
+                path="componentfiles/loggingmanager"
+                processName="LoggingManager"
+                schema="loggingmanager.xsd"/>
+            <BuildSet deployable="no"
+                installSet="deploy_map.xml"
+                name="wslogging"
+                path="componentfiles/wslogging"
+                processName="EspService"
+                schema="wslogging.xsd">
+                <Properties bindingType="loggingservice_binding"
+                    defaultPort="8146"
+                    defaultResourcesBasedn="ou=EspServices,ou=ecl"
+                    defaultSecurePort="18146"
+                    plugin="wslogging"
+                    type="wslogging">
+                    <Authenticate access="Read"
+                        description="Root access to WS Logging service"
+                        path="/"
+                        required="Read"
+                        resource="WsLoggingAccess"/>
+                    <AuthenticateFeature description="Access to WS Logging service"
+                        path="WsLoggingAccess"
+                        resource="WsLoggingAccess"
+                        service="wslogging"/>
+                </Properties>
+            </BuildSet>
+        </Build>
+    </Programs>
 </Environment>
- 
+

initfiles/componentfiles/configxml/cgencomplist_linux.xml

@@ -113,6 +113,9 @@
 <Component name="ws_ecl" processName='EspService' schema='esp_service_wsecl2.xsd' deployable='no'>
     <File name="@temp/esp_service.xsl" method="esp_service_module"/>
   </Component>  
+  <Component name="ws_sql">
+       <File name="@temp/esp_service_wssql.xsl" method="esp_service_module"/>
+  </Component>
   <Component name="ws_ssn" processName='EspService' schema='ws_ssn.xsd' deployable='no'>
     <File name="@temp/esp_service.xsl" method="esp_service_module" destName="ws_ssn.xml" destPath="@temp"/>
   </Component>

initfiles/componentfiles/configxml/esp_service_wssql.xsd

@@ -0,0 +1,55 @@
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
+    <xs:include schemaLocation="environment.xsd"/>
+    <xs:element name="EspService">
+        <xs:complexType>
+            <xs:attribute name="build" type="buildType" use="required">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>The build name to be deployed</tooltip>
+                        <viewType>hidden</viewType>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="buildSet" type="buildSetType" use="required">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <viewType>hidden</viewType>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="name" type="xs:string" use="optional" default="ws_sql_service">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Name for this ESP service</tooltip>
+                        <required>true</required>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="description" type="xs:string" use="optional">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>SQL interface into HPCC web services</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+        </xs:complexType>
+    </xs:element>
+</xs:schema>