Merge branch 'candidate-6.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

CMakeLists.txt

@@ -155,33 +155,40 @@ if ( PLUGIN )
     HPCC_ADD_SUBDIRECTORY (plugins/redis "REDIS")
     HPCC_ADD_SUBDIRECTORY (plugins/javaembed "JAVAEMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/kafka "KAFKA")
+    HPCC_ADD_SUBDIRECTORY (plugins/sqs "SQS")
     HPCC_ADD_SUBDIRECTORY (plugins/sqlite3 "SQLITE3EMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/mysql "MYSQLEMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/exampleplugin "EXAMPLEPLUGIN")
     HPCC_ADD_SUBDIRECTORY (plugins/couchbase "COUCHBASEEMBED")
-elseif ( NOT MAKE_DOCS_ONLY )
-    HPCC_ADD_SUBDIRECTORY (initfiles)
-    HPCC_ADD_SUBDIRECTORY (tools)
-    HPCC_ADD_SUBDIRECTORY (common)
-    HPCC_ADD_SUBDIRECTORY (dali)
-    HPCC_ADD_SUBDIRECTORY (deploy)
-    HPCC_ADD_SUBDIRECTORY (deployment)
-    HPCC_ADD_SUBDIRECTORY (ecl)
-    HPCC_ADD_SUBDIRECTORY (ecllibrary)
-    HPCC_ADD_SUBDIRECTORY (esp)
-    HPCC_ADD_SUBDIRECTORY (plugins)
-    HPCC_ADD_SUBDIRECTORY (roxie)
-    HPCC_ADD_SUBDIRECTORY (rtl)
-    HPCC_ADD_SUBDIRECTORY (services "PLATFORM")
+elseif ( NOT MAKE_DOCS_ONLY OR MAKE_CONFIGURATOR )
     HPCC_ADD_SUBDIRECTORY (system)
-    HPCC_ADD_SUBDIRECTORY (thorlcr "PLATFORM")
-    HPCC_ADD_SUBDIRECTORY (testing)
+
+    if ( NOT MAKE_DOCS_ONLY )
+        HPCC_ADD_SUBDIRECTORY (initfiles)
+        HPCC_ADD_SUBDIRECTORY (tools)
+        HPCC_ADD_SUBDIRECTORY (common)
+        HPCC_ADD_SUBDIRECTORY (dali)
+        HPCC_ADD_SUBDIRECTORY (deploy)
+        HPCC_ADD_SUBDIRECTORY (deployment)
+        HPCC_ADD_SUBDIRECTORY (ecl)
+        HPCC_ADD_SUBDIRECTORY (ecllibrary)
+        HPCC_ADD_SUBDIRECTORY (esp)
+        HPCC_ADD_SUBDIRECTORY (plugins)
+        HPCC_ADD_SUBDIRECTORY (roxie)
+        HPCC_ADD_SUBDIRECTORY (rtl)
+        HPCC_ADD_SUBDIRECTORY (services "PLATFORM")
+        HPCC_ADD_SUBDIRECTORY (thorlcr "PLATFORM")
+        HPCC_ADD_SUBDIRECTORY (testing)
+
+        if ( NOT WIN32 )
+            HPCC_ADD_SUBDIRECTORY (clienttools "CLIENTTOOLS_ONLY")
+        endif()
+
+    endif ( NOT MAKE_DOCS_ONLY )
+
     if ( MAKE_CONFIGURATOR )
         MESSAGE(STATUS "Configurator app build ON")
-        HPCC_ADD_SUBDIRECTORY (configuration "PLATFORM")
-    endif()
-    if ( NOT WIN32 )
-        HPCC_ADD_SUBDIRECTORY (clienttools "CLIENTTOOLS_ONLY")
+        HPCC_ADD_SUBDIRECTORY (configuration)
     endif()
 endif()
 
@@ -284,14 +291,6 @@ if(TOP_LEVEL_PROJECT)
         message("-- Auto Detecting Packaging type")
         message("-- distro uses ${packageManagement}, revision is ${packageRevisionArch}")
 
-        if("${packageManagement}" STREQUAL "RPM")
-            set(CPACK_RPM_SPEC_MORE_DEFINE
-"%define _use_internal_dependency_generator 0
-%define __getdeps() while read file; do /usr/lib/rpm/rpmdeps -%{1} ${file} | %{__grep} -v libRInside.so | %{__grep} -v libRcpp.so ; done | /bin/sort -u
-%define __find_provides /bin/sh -c '%{__getdeps P}'
-%define __find_requires /bin/sh -c '%{__getdeps R}'")
-        endif()
-
         if("${packageManagement}" STREQUAL "DEB")
             set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}${packageRevisionArch}")
         elseif("${packageManagement}" STREQUAL "RPM")

cmake_modules/FindR.cmake

@@ -44,14 +44,33 @@ IF (NOT R_FOUND)
     SET (RCPP_LIBRARY "")    # Newer versions of Rcpp are header-only, with no associated library.
   ENDIF()
 
+  #Rcpp/config.h
+  #define RCPP_VERSION Rcpp_Version(0,12,3)
+  FILE(STRINGS "${RCPP_INCLUDE_DIR}/Rcpp/config.h" version_string REGEX "#define RCPP_VERSION Rcpp_Version\\(")
+  #major
+  STRING(REGEX REPLACE "#define RCPP_VERSION Rcpp_Version\\(" "" major "${version_string}")
+  STRING(REGEX REPLACE ",[0-9]+,[0-9]+\\)" "" major "${major}")
+  #minor
+  STRING(REGEX REPLACE "#define RCPP_VERSION Rcpp_Version\\([0-9]+," "" minor "${version_string}")
+  STRING(REGEX REPLACE ",[0-9]+\\)" "" minor "${minor}")
+  #patch
+  STRING(REGEX REPLACE "#define RCPP_VERSION Rcpp_Version\\([0-9]+,[0-9]+," "" patch "${version_string}")
+  STRING(REGEX REPLACE "\\)" "" patch "${patch}")
+  SET(RCPP_VERSION_STRING "${major}.${minor}.${patch}")
+
   SET (R_INCLUDE_DIRS ${R_INCLUDE_DIR} ${RINSIDE_INCLUDE_DIR} ${RCPP_INCLUDE_DIR})
   SET (R_LIBRARIES ${R_LIBRARY} ${RINSIDE_LIBRARY} ${RCPP_LIBRARY})
 
-  include(FindPackageHandleStandardArgs)
-  find_package_handle_standard_args(R DEFAULT_MSG
-    R_LIBRARIES
-    R_INCLUDE_DIRS
-  )
+  INCLUDE(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(R
+    REQUIRED_VARS R_LIBRARY
+                  RINSIDE_LIBRARY
+                  R_INCLUDE_DIR
+                  RCPP_INCLUDE_DIR
+                  RINSIDE_INCLUDE_DIR
+                  R_LIBRARIES
+                  R_INCLUDE_DIRS
+    VERSION_VAR RCPP_VERSION_STRING)
 
-  MARK_AS_ADVANCED(R_INCLUDE_DIRS R_LIBRARIES)
+  MARK_AS_ADVANCED(R_INCLUDE_DIRS R_LIBRARIES RINSIDE_LIBRARY)
 ENDIF()

cmake_modules/commonSetup.cmake

@@ -95,6 +95,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(USE_UNSIGNED_CHAR "Build system with default char type is unsigned" OFF)
   option(USE_MYSQL "Enable mysql support" ON)
   option(USE_OPTIONAL "Automatically disable requested features with missing dependencies" ON)
+  option(JLIB_ONLY  "Build JLIB for other projects such as Configurator, Ganglia Monitoring, etc" OFF)
   # Generates code that is more efficient, but will cause problems if target platforms do not support it.
   if (CMAKE_SIZEOF_VOID_P EQUAL 8)
     option(USE_INLINE_TSC "Inline calls to read TSC (time stamp counter)" ON)
@@ -680,7 +681,9 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
 
     if (DOCS_AUTO)
        if ("${CONFIGURATOR_DIRECTORY}" STREQUAL "")
-         set(CONFIGURATOR_DIRECTORY ${HPCC_SOURCE_DIR}/../configurator)
+         set(MAKE_CONFIGURATOR ON)
+         set(JLIB_ONLY ON)
+         set  (CONFIGURATOR_DIRECTORY ${EXECUTABLE_OUTPUT_PATH})
        endif()
     endif()
   ENDIF(MAKE_DOCS)

cmake_modules/docMacros.cmake

@@ -1,102 +1,102 @@
 
 MACRO(RUN_XSLTPROC _xsl _file _out _in_dir _out_dir )
-	PARSE_ARGUMENTS(_XSLT "" "" ${ARGN})
-	SET(TARG "")
-	LIST(LENGTH _XSLT_DEFAULT_ARGS _XSLT_LEN)
-	if( _XSLT_LEN )
-		LIST(GET _XSLT_DEFAULT_ARGS 0 TARG)
-		set(DIRS ${_XSLT_DEFAULT_ARGS})
-		LIST(REMOVE_AT DIRS 0)
-		LIST(LENGTH DIRS length)
-		set(FILES "")
-		if ( length )
-			foreach( D ${DIRS} )
-				file(GLOB_RECURSE _DB_INCLUDES ${D}/*.xml)
-				set(FILES ${FILES} ${_DB_INCLUDES})
-			endforeach()
-		endif()
-	endif()
-	STRING(REGEX REPLACE "([0-9a-z_-]*).xml" "\\1" _file_base "${_file}")
-	SET(_xsl ${_xsl})
-	SET(_file ${_file})
-	SET(_out ${_out})
-	SET(_in_dir ${_in_dir})
-	SET(_out_dir ${_out_dir})
-	IF( TARG )
-		SET(_xslt_target ${ARGN})
-		SET(xinclude "-xinclude")
-	ELSE()
-		SET(_xslt_target)
-		SET(xinclude)
-	ENDIF()
-	CONFIGURE_FILE(${HPCC_SOURCE_DIR}/docs/BuildTools/xsltproc.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/${_out}.cmake @ONLY)
-	ADD_CUSTOM_COMMAND(
-		COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/${_out}.cmake
-		OUTPUT ${_out_dir}/${_out}
-		DEPENDS docbook-expand ${_xsl} ${_in_dir}/${_file} ${_xslt_target} ${FILES} ${DOC_IMAGE_LIST} ${XSLTPROC_DEPENDENCIES}
-		)
-	set_source_files_properties(${_out_dir}/${_out} PROPERTIES GENERATED TRUE)
-	ADD_CUSTOM_TARGET(${_out} DEPENDS ${_out_dir}/${_out} )
+        PARSE_ARGUMENTS(_XSLT "" "" ${ARGN})
+        SET(TARG "")
+        LIST(LENGTH _XSLT_DEFAULT_ARGS _XSLT_LEN)
+        if( _XSLT_LEN )
+                LIST(GET _XSLT_DEFAULT_ARGS 0 TARG)
+                set(DIRS ${_XSLT_DEFAULT_ARGS})
+                LIST(REMOVE_AT DIRS 0)
+                LIST(LENGTH DIRS length)
+                set(FILES "")
+                if ( length )
+                        foreach( D ${DIRS} )
+                                file(GLOB_RECURSE _DB_INCLUDES ${D}/*.xml)
+                                set(FILES ${FILES} ${_DB_INCLUDES})
+                        endforeach()
+                endif()
+        endif()
+        STRING(REGEX REPLACE "([0-9a-z_-]*).xml" "\\1" _file_base "${_file}")
+        SET(_xsl ${_xsl})
+        SET(_file ${_file})
+        SET(_out ${_out})
+        SET(_in_dir ${_in_dir})
+        SET(_out_dir ${_out_dir})
+        IF( TARG )
+                SET(_xslt_target ${ARGN})
+                SET(xinclude "-xinclude")
+        ELSE()
+                SET(_xslt_target)
+                SET(xinclude)
+        ENDIF()
+        CONFIGURE_FILE(${HPCC_SOURCE_DIR}/docs/BuildTools/xsltproc.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/${_out}.cmake @ONLY)
+        ADD_CUSTOM_COMMAND(
+                COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/${_out}.cmake
+                OUTPUT ${_out_dir}/${_out}
+                DEPENDS docbook-expand ${_xsl} ${_in_dir}/${_file} ${_xslt_target} ${FILES} ${DOC_IMAGE_LIST} ${XSLTPROC_DEPENDENCIES}
+                )
+        set_source_files_properties(${_out_dir}/${_out} PROPERTIES GENERATED TRUE)
+        ADD_CUSTOM_TARGET(${_out} DEPENDS ${_out_dir}/${_out} )
         SET(XSLTPROC_DEPENDENCIES)
 ENDMACRO(RUN_XSLTPROC)
 
 MACRO(RUN_FOP _file _out)
-	ADD_CUSTOM_COMMAND(
-		COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs
-		COMMAND ${FOP_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/${_file} -pdf ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs/${_out} 
-		OUTPUT ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs/${_out}
-		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${_file}
-		)
-	set_source_files_properties(${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs/${_out} PROPERTIES GENERATED TRUE)
-	ADD_CUSTOM_TARGET(${_out} DEPENDS ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs/${_out} )
+        ADD_CUSTOM_COMMAND(
+                COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs
+                COMMAND ${FOP_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/${_file} -pdf ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs/${_out}
+                OUTPUT ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs/${_out}
+                DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${_file}
+                )
+        set_source_files_properties(${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs/${_out} PROPERTIES GENERATED TRUE)
+        ADD_CUSTOM_TARGET(${_out} DEPENDS ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs/${_out} )
 ENDMACRO(RUN_FOP)
 
 MACRO(CLEAN_REL_BOOK _file _version_dir _doc_dir _in_dir _out_dir)
-	STRING(REGEX REPLACE "([0-9a-z_-]*).xml" "\\1" _file_base "${_file}")
-	SET(_clean_target "clean_${_file}")
-	SET(VERSION_DIR ${_version_dir})
-	SET(ROOT "book")
-	CONFIGURE_FILE(${HPCC_SOURCE_DIR}/docs/BuildTools/relrem.xsl.in ${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl @ONLY)
-	RUN_XSLTPROC( ${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl ${_file} ${_file} ${_in_dir} ${_out_dir})
-	set_source_files_properties(${_out_dir}/${_file} PROPERTIES GENERATED TRUE)
-	ADD_CUSTOM_TARGET( ${_clean_target} DEPENDS ${_in_dir}/${_file} )
+        STRING(REGEX REPLACE "([0-9a-z_-]*).xml" "\\1" _file_base "${_file}")
+        SET(_clean_target "clean_${_file}")
+        SET(VERSION_DIR ${_version_dir})
+        SET(ROOT "book")
+        CONFIGURE_FILE(${HPCC_SOURCE_DIR}/docs/BuildTools/relrem.xsl.in ${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl @ONLY)
+        RUN_XSLTPROC( ${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl ${_file} ${_file} ${_in_dir} ${_out_dir})
+        set_source_files_properties(${_out_dir}/${_file} PROPERTIES GENERATED TRUE)
+        ADD_CUSTOM_TARGET( ${_clean_target} DEPENDS ${_in_dir}/${_file} )
 ENDMACRO(CLEAN_REL_BOOK)
 
 MACRO(CLEAN_REL_SET _file _version_dir _doc_dir _in_dir _out_dir)
-	STRING(REGEX REPLACE "([0-9a-z_-]*).xml" "\\1" _file_base "${_file}")
-	SET(_clean_target "clean_${_file}")
-	SET(VERSION_DIR ${_version_dir})
-	SET(ROOT "set")
-	CONFIGURE_FILE(${HPCC_SOURCE_DIR}/docs/BuildTools/relrem.xsl.in ${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl @ONLY)
-	RUN_XSLTPROC( ${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl ${_file} ${_file} ${_in_dir} ${_out_dir})
-	set_source_files_properties(${_out_dir}/${_file} PROPERTIES GENERATED TRUE)
-	ADD_CUSTOM_TARGET( ${_clean_target} DEPENDS ${_in_dir}/${_file} )
+        STRING(REGEX REPLACE "([0-9a-z_-]*).xml" "\\1" _file_base "${_file}")
+        SET(_clean_target "clean_${_file}")
+        SET(VERSION_DIR ${_version_dir})
+        SET(ROOT "set")
+        CONFIGURE_FILE(${HPCC_SOURCE_DIR}/docs/BuildTools/relrem.xsl.in ${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl @ONLY)
+        RUN_XSLTPROC( ${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl ${_file} ${_file} ${_in_dir} ${_out_dir})
+        set_source_files_properties(${_out_dir}/${_file} PROPERTIES GENERATED TRUE)
+        ADD_CUSTOM_TARGET( ${_clean_target} DEPENDS ${_in_dir}/${_file} )
 ENDMACRO(CLEAN_REL_SET)
 
 MACRO(DOCBOOK_TO_PDF _xsl _file _name)
 
-	PARSE_ARGUMENTS(_DB "" "" ${ARGN})
-	LIST(LENGTH _DB_DEFAULT_ARGS length)
-	IF(MAKE_DOCS)
-		STRING(REGEX REPLACE "([0-9a-z_-]*).xml" "\\1" _file_base "${_file}")
-		SET(_fo_file ${_file_base}.fo)
-		SET(_pdf_file ${_name}-${version}-${stagever}.pdf)
-		SET( _docs_target "doc_${_pdf_file}")  # File to Name of type.
-		CLEAN_REL_BOOK(${_file} ${VERSION_DIR} ${DOC_IMAGES} ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-		set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl  PROPERTIES GENERATED TRUE)
-		RUN_XSLTPROC(${_xsl} ${_file} ${_fo_file} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR} "clean_${_file}" ${_DB_DEFAULT_ARGS})
+        PARSE_ARGUMENTS(_DB "" "" ${ARGN})
+        LIST(LENGTH _DB_DEFAULT_ARGS length)
+        IF(MAKE_DOCS)
+                STRING(REGEX REPLACE "([0-9a-z_-]*).xml" "\\1" _file_base "${_file}")
+                SET(_fo_file ${_file_base}.fo)
+                SET(_pdf_file ${_name}-${version}-${stagever}.pdf)
+                SET( _docs_target "doc_${_pdf_file}")  # File to Name of type.
+                CLEAN_REL_BOOK(${_file} ${VERSION_DIR} ${DOC_IMAGES} ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+                set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${_file_base}.xsl  PROPERTIES GENERATED TRUE)
+                RUN_XSLTPROC(${_xsl} ${_file} ${_fo_file} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR} "clean_${_file}" ${_DB_DEFAULT_ARGS})
                 IF (DISABLE_PDF)
                    SET(DISABLE_PDF)
                    SET(_target_file ${_fo_file})
                 ELSE()
-		   RUN_FOP(${_fo_file} ${_pdf_file})
-		   set_source_files_properties(${_pdf_file} PROPERTIES GENERATED TRUE)
+                   RUN_FOP(${_fo_file} ${_pdf_file})
+                   set_source_files_properties(${_pdf_file} PROPERTIES GENERATED TRUE)
                    SET(_target_file ${_pdf_file})
                 ENDIF()
-		MESSAGE("-- Adding document: ${_target_file} -  target: ${_docs_target}")
-		ADD_CUSTOM_TARGET(${_docs_target} ALL DEPENDS ${_target_file} )
-		set_property(GLOBAL APPEND PROPERTY DOC_TARGETS "${_docs_target}")
-	ENDIF(MAKE_DOCS)
+                MESSAGE("-- Adding document: ${_target_file} -  target: ${_docs_target}")
+                ADD_CUSTOM_TARGET(${_docs_target} ALL DEPENDS ${_target_file} )
+                set_property(GLOBAL APPEND PROPERTY DOC_TARGETS "${_docs_target}")
+        ENDIF(MAKE_DOCS)
 ENDMACRO(DOCBOOK_TO_PDF targetname_suffix srcfile outfile targetdir deps_list)
 
 MACRO(DOCBOOK_TO_HTML _xsl_file _xml_file _out_dir)
@@ -130,33 +130,37 @@ MACRO(DOCBOOK_TO_HTML _xsl_file _xml_file _out_dir)
 ENDMACRO(DOCBOOK_TO_HTML)
 
 MACRO(FILE_LIST_GENERATOR outxml filename linkname description)
-	set(xmlout "	<file>\n")
-	set(xmlout "${xmlout}		<name>${linkname}</name>\n")
-	set(xmlout "${xmlout}		<description>${description}</description>\n")
-	set(xmlout "${xmlout}		<filename>${filename}</filename>\n")
-	set(xmlout "${xmlout}		<version>${DOC_VERSION}</version>\n")
-	set(xmlout "${xmlout}	</file>\n")
-	message("---- FILE_LIST_GENERATOR: Adding ${filename}")
-	set(${outxml} "${${outxml}}${xmlout}")
+        set(xmlout "        <file>\n")
+        set(xmlout "${xmlout}                <name>${linkname}</name>\n")
+        set(xmlout "${xmlout}                <description>${description}</description>\n")
+        set(xmlout "${xmlout}                <filename>${filename}</filename>\n")
+        set(xmlout "${xmlout}                <version>${DOC_VERSION}</version>\n")
+        set(xmlout "${xmlout}        </file>\n")
+        message("---- FILE_LIST_GENERATOR: Adding ${filename}")
+        set(${outxml} "${${outxml}}${xmlout}")
 ENDMACRO(FILE_LIST_GENERATOR)
 
 MACRO(XSD_TO_XML _xsd_files _in_dir _out_dir)
 
         set(_xml_files)
-	ADD_CUSTOM_COMMAND(
-		COMMAND mkdir -p ${_out_dir}
-		OUTPUT ${_out_dir}
+        ADD_CUSTOM_COMMAND(
+                COMMAND mkdir -p ${_out_dir}
+                OUTPUT ${_out_dir}
                 )
         foreach(_xsd_file ${_xsd_files})
             STRING(REGEX REPLACE "(.*).xsd" "\\1.xml" _xml_file "${_xsd_file}")
-	    ADD_CUSTOM_COMMAND(
-		COMMAND ./configurator -doc -use ${_xsd_file} -b ${_in_dir} -t ${_out_dir}
-		OUTPUT ${_out_dir}/${_xml_file}
+            ADD_CUSTOM_COMMAND(
+                COMMAND ./configurator --doc --use ${_xsd_file} -b ${_in_dir} -t ${_out_dir}
+                OUTPUT ${_out_dir}/${_xml_file}
                 WORKING_DIRECTORY ${CONFIGURATOR_DIRECTORY}
-		DEPENDS ${_out_dir} ${_in_dir}/${_xsd_file} ${_in_dir}/environment.xsd
-		)
+                DEPENDS ${_out_dir} ${_in_dir}/${_xsd_file} ${_in_dir}/environment.xsd
+                )
             list(APPEND _xml_files ${_out_dir}/${_xml_file})
         endforeach()
-	ADD_CUSTOM_TARGET("xsd_to_xml" ALL  DEPENDS ${_out_dir} ${_xml_files} )
+        if (MAKE_CONFIGURATOR)
+            ADD_CUSTOM_TARGET("xsd_to_xml" ALL  DEPENDS ${_out_dir} ${_xml_files} "configurator" )
+        else()
+            ADD_CUSTOM_TARGET("xsd_to_xml" ALL  DEPENDS ${_out_dir} ${_xml_files} )
+        endif()
         set_property(GLOBAL APPEND PROPERTY DOC_TARGETS "xsd_to_xml")
 ENDMACRO(XSD_TO_XML)

common/remote/rmtssh.cpp

@@ -514,7 +514,10 @@ public:
                     res.setLength(res.length()-1);
                 if (reply.item(n)) {
                     errCode = reply.item(n);
-                    multiException->append(*MakeStringExceptionDirect(errCode,res.str()));
+                    if (res.length())
+                        multiException->append(*MakeStringExceptionDirect(errCode,res.str()));
+                    else
+                        multiException->append(*MakeStringExceptionDirect(errCode,cmd.str()));
                 }
             }
             if (errCode)
@@ -524,8 +527,12 @@ public:
             StringBuffer res(replytext.item(0));
             while (res.length()&&(res.charAt(res.length()-1)<=' '))
                 res.setLength(res.length()-1);
-            if (reply.item(0))
-                throw MakeStringExceptionDirect(reply.item(0), res.str());
+            if (reply.item(0)) {
+                if (res.length())
+                    throw MakeStringExceptionDirect(reply.item(0), res.str());
+                else
+                    throw MakeStringExceptionDirect(reply.item(0), cmd.str());
+            }
         }
     }
     void exec(

common/thorhelper/layouttrans.cpp

@@ -501,7 +501,7 @@ IRecordLayoutTranslator::RowTransformContext::~RowTransformContext()
     delete [] ptrs;
 }
 
-CRecordLayoutTranslator::CRecordLayoutTranslator(IDefRecordMeta const * _diskMeta, IDefRecordMeta const * _activityMeta) : diskMeta(const_cast<IDefRecordMeta *>(_diskMeta)), activityMeta(const_cast<IDefRecordMeta *>(_activityMeta)), activityKeySizes(NULL)
+CRecordLayoutTranslator::CRecordLayoutTranslator(IDefRecordMeta const * _diskMeta, IDefRecordMeta const * _activityMeta, IRecordLayoutTranslator::Mode _mode) : diskMeta(const_cast<IDefRecordMeta *>(_diskMeta)), activityMeta(const_cast<IDefRecordMeta *>(_activityMeta)), activityKeySizes(NULL)
 {
     numKeyedDisk = diskMeta->numKeyedFields();
     numKeyedActivity = activityMeta->numKeyedFields();
@@ -516,6 +516,8 @@ CRecordLayoutTranslator::CRecordLayoutTranslator(IDefRecordMeta const * _diskMet
         topMappingLevel.calculateMappings(diskMeta->queryRecord(), numKeyedDisk, activityMeta->queryRecord(), numKeyedActivity);
         calculateActivityKeySizes();
         calculateKeysTransformed();
+        if (keysTransformed && _mode != TranslateAll)
+            throw makeFailure(IRecordLayoutTranslator::Failure::KeyedDisallowed)->append("Translation of key fields would be required");
         transformer.build(numTransformers, mappings);
     }
     catch(Failure * f)
@@ -543,7 +545,7 @@ void CRecordLayoutTranslator::calculateActivityKeySizes()
 
 void CRecordLayoutTranslator::calculateKeysTransformed()
 {
-    keysTransformed = true;;
+    keysTransformed = true;
     if(numKeyedActivity != numKeyedDisk)
         return;
     for(unsigned diskFieldNum=0; diskFieldNum<numKeyedDisk; ++diskFieldNum)
@@ -651,9 +653,9 @@ void ExpandedSegmentMonitorList::setMergeBarrier(unsigned offset)
     // MORE - It's possible that I need to do something here??
 }
 
-IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta)
+IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta, IRecordLayoutTranslator::Mode _mode)
 {
-    Owned<IRecordLayoutTranslator> layoutTrans = new CRecordLayoutTranslator(diskMeta, activityMeta);
+    Owned<IRecordLayoutTranslator> layoutTrans = new CRecordLayoutTranslator(diskMeta, activityMeta, _mode);
     if(!layoutTrans->querySuccess())
     {
         StringBuffer cause;
@@ -663,7 +665,7 @@ IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * di
     return layoutTrans.getClear();
 };
 
-extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(size32_t diskMetaSize, const void *diskMetaData, size32_t activityMetaSize, const void *activityMetaData)
+extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(size32_t diskMetaSize, const void *diskMetaData, size32_t activityMetaSize, const void *activityMetaData, IRecordLayoutTranslator::Mode _mode)
 {
     MemoryBuffer activityMetaSerialized;
     activityMetaSerialized.setBuffer(activityMetaSize, (void *) activityMetaData, false);
@@ -673,7 +675,7 @@ extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(siz
     diskMetaSerialized.setBuffer(diskMetaSize, (void *) diskMetaData, false);
     Owned<IDefRecordMeta> diskMeta = deserializeRecordMeta(diskMetaSerialized, true);
 
-    return createRecordLayoutTranslator(diskMeta, activityMeta);
+    return createRecordLayoutTranslator(diskMeta, activityMeta, _mode);
 }
 
 #ifdef DEBUG_HELPERS_REQUIRED
@@ -720,21 +722,21 @@ StringBuffer & CRecordLayoutTranslator::getMappingsAsString(StringBuffer & out)
 
 #endif
 
-CacheKey::CacheKey(size32_t _s1, void const * _d1, size32_t _s2, void const * _d2)
-    : s1(_s1), d1(static_cast<byte const *>(_d1)), s2(_s2), d2(static_cast<byte const *>(_d2))
+CacheKey::CacheKey(IRecordLayoutTranslator::Mode _mode, size32_t _s1, void const * _d1, size32_t _s2, void const * _d2)
+    : mode(_mode), s1(_s1), d1(static_cast<byte const *>(_d1)), s2(_s2), d2(static_cast<byte const *>(_d2))
 {
-    hashval = hashc(d1, s1, 0);
+    hashval = hashc(d1, s1, (unsigned) mode);
     hashval = hashc(d2, s2, hashval);
 }
 
-CacheValue::CacheValue(size32_t s1, void const * d1, size32_t s2, void const * d2, IRecordLayoutTranslator * _trans)
-    : b1(s1, d1), b2(s2, d2), key((size32_t)b1.length(), b1.get(), (size32_t)b2.length(), b2.get()), trans(_trans)
+CacheValue::CacheValue(IRecordLayoutTranslator::Mode _mode, size32_t s1, void const * d1, size32_t s2, void const * d2, IRecordLayoutTranslator * _trans)
+    : b1(s1, d1), b2(s2, d2), key(_mode, (size32_t)b1.length(), b1.get(), (size32_t)b2.length(), b2.get()), trans(_trans)
 {
 }
 
-IRecordLayoutTranslator * CRecordLayoutTranslatorCache::get(size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta)
+IRecordLayoutTranslator * CRecordLayoutTranslatorCache::get(IRecordLayoutTranslator::Mode mode, size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta)
 {
-    CacheKey key(diskMetaSize, diskMetaData, activityMetaSize, activityMetaData);
+    CacheKey key(mode, diskMetaSize, diskMetaData, activityMetaSize, activityMetaData);
     CacheValue * value = find(&key);
     if(!value)
     {
@@ -751,9 +753,9 @@ IRecordLayoutTranslator * CRecordLayoutTranslatorCache::get(size32_t diskMetaSiz
         diskMetaSerialized.setBuffer(diskMetaSize, (void *) diskMetaData, false);
         Owned<IDefRecordMeta> diskMeta = deserializeRecordMeta(diskMetaSerialized, true);
 
-        Owned<IRecordLayoutTranslator> trans = createRecordLayoutTranslator(diskMeta, activityMeta);
+        Owned<IRecordLayoutTranslator> trans = createRecordLayoutTranslator(diskMeta, activityMeta, mode);
 
-        value = new CacheValue(diskMetaSize, diskMetaData, activityMetaSize, activityMetaData, trans.getLink());
+        value = new CacheValue(mode, diskMetaSize, diskMetaData, activityMetaSize, activityMetaData, trans.getLink());
         addNew(value);
     }
     return value->getTranslator();
@@ -868,11 +870,11 @@ public:
         Owned<IRecordLayoutTranslatorCache> cache = createRecordLayoutTranslatorCache();
         CPPUNIT_ASSERT(cache.get() != 0);
         CPPUNIT_ASSERT(cache->count() == 0);
-        Owned<IRecordLayoutTranslator> t1 = cache->get(buff[0].length(), buff[0].bufferBase(), buff[1].length(), buff[1].bufferBase(), NULL);
+        Owned<IRecordLayoutTranslator> t1 = cache->get(IRecordLayoutTranslator::TranslateAll, buff[0].length(), buff[0].bufferBase(), buff[1].length(), buff[1].bufferBase(), NULL);
         CPPUNIT_ASSERT(cache->count() == 1);
-        Owned<IRecordLayoutTranslator> t2 = cache->get(buff[0].length(), buff[0].bufferBase(), buff[1].length(), buff[1].bufferBase(), NULL);
+        Owned<IRecordLayoutTranslator> t2 = cache->get(IRecordLayoutTranslator::TranslateAll, buff[0].length(), buff[0].bufferBase(), buff[1].length(), buff[1].bufferBase(), NULL);
         CPPUNIT_ASSERT(cache->count() == 1);
-        Owned<IRecordLayoutTranslator> t3 = cache->get(buff[0].length(), buff[0].bufferBase(), buff[2].length(), buff[2].bufferBase(), NULL);
+        Owned<IRecordLayoutTranslator> t3 = cache->get(IRecordLayoutTranslator::TranslateAll, buff[0].length(), buff[0].bufferBase(), buff[2].length(), buff[2].bufferBase(), NULL);
         CPPUNIT_ASSERT(cache->count() == 2);
         CPPUNIT_ASSERT(t1.get() == t2.get());
         CPPUNIT_ASSERT(t1.get() != t3.get());
@@ -1114,14 +1116,14 @@ private:
 
     void doTranslate(unsigned disk, unsigned activity)
     {
-        Owned<IRecordLayoutTranslator> trans = new CRecordLayoutTranslator(&meta.item(disk), &meta.item(activity));
+        Owned<IRecordLayoutTranslator> trans = new CRecordLayoutTranslator(&meta.item(disk), &meta.item(activity), IRecordLayoutTranslator::TranslateAll);
         CPPUNIT_ASSERT(trans.get() != NULL);
         CPPUNIT_ASSERT(trans->querySuccess());
     }
     
     void doTranslateFail(unsigned disk, unsigned activity, unsigned code)
     {
-        Owned<IRecordLayoutTranslator> trans = new CRecordLayoutTranslator(&meta.item(disk), &meta.item(activity));
+        Owned<IRecordLayoutTranslator> trans = new CRecordLayoutTranslator(&meta.item(disk), &meta.item(activity), IRecordLayoutTranslator::TranslateAll);
         CPPUNIT_ASSERT(trans.get() != 0);
         CPPUNIT_ASSERT(!trans->querySuccess());
         CPPUNIT_ASSERT(trans->queryFailure().queryCode() == code);

common/thorhelper/layouttrans.hpp

@@ -20,7 +20,6 @@
 
 #include "deffield.hpp"
 #include "rtlkey.hpp"
-#include "jhtree.hpp"
 #include "thorhelper.hpp"
 
 #ifdef _DEBUG
@@ -33,7 +32,7 @@ public:
     class THORHELPER_API Failure : public CInterface
     {
     public:
-        typedef enum { BadStructure = 1, MissingDiskField, UnkeyedDiskField, UntranslatableField, UnsupportedFilter } Code;
+        typedef enum { BadStructure = 1, MissingDiskField, UnkeyedDiskField, UntranslatableField, UnsupportedFilter, KeyedDisallowed } Code;
         
         virtual Code queryCode() const = 0;
         virtual StringBuffer & getDetail(StringBuffer & out) const = 0;
@@ -66,6 +65,8 @@ public:
         offset_t fposIn;
     };
 
+    typedef enum { NoTranslation = 0, TranslateAll = 1, TranslatePayload = 2 } Mode;
+
     virtual bool querySuccess() const = 0;
     virtual Failure const & queryFailure() const = 0;
     virtual void checkSizes(char const * filename, size32_t activitySize, size32_t diskSize) const = 0;
@@ -79,13 +80,13 @@ public:
 #endif
 };
 
-extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta);
-extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(size32_t diskMetaSize, const void *diskMetaData, size32_t activityMetaSize, const void *activityMetaData);
+extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta, IRecordLayoutTranslator::Mode _mode);
+extern THORHELPER_API IRecordLayoutTranslator * createRecordLayoutTranslator(size32_t diskMetaSize, const void *diskMetaData, size32_t activityMetaSize, const void *activityMetaData, IRecordLayoutTranslator::Mode _mode);
 
 class THORHELPER_API IRecordLayoutTranslatorCache : public IInterface
 {
 public:
-    virtual IRecordLayoutTranslator * get(size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta = NULL) = 0; //if activityMeta is NULL, it is calculated by deserializing from buffer --- but option to supply so caller can deserialize once and use many
+    virtual IRecordLayoutTranslator * get(IRecordLayoutTranslator::Mode mode, size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta = NULL) = 0; //if activityMeta is NULL, it is calculated by deserializing from buffer --- but option to supply so caller can deserialize once and use many
     virtual unsigned count() const = 0;
 };
 

common/thorhelper/layouttrans.ipp

@@ -191,7 +191,7 @@ private:
 class CRecordLayoutTranslator : public IRecordLayoutTranslator, public CInterface
 {
 public:
-    CRecordLayoutTranslator(IDefRecordMeta const * _diskMeta, IDefRecordMeta const * _activityMeta);
+    CRecordLayoutTranslator(IDefRecordMeta const * _diskMeta, IDefRecordMeta const * _activityMeta, IRecordLayoutTranslator::Mode _mode);
     ~CRecordLayoutTranslator() { delete [] activityKeySizes; }
     IMPLEMENT_IINTERFACE;
     virtual bool querySuccess() const { return !failure; }
@@ -229,10 +229,11 @@ private:
 class CacheKey
 {
 public:
-    CacheKey(size32_t _s1, void const * _d1, size32_t _s2, void const * _d2);
+    CacheKey(IRecordLayoutTranslator::Mode mode, size32_t _s1, void const * _d1, size32_t _s2, void const * _d2);
     unsigned getHash() const { return hashval; }
-    bool operator==(CacheKey const & other) const { return((s1 == other.s1) && (s2 == other.s2) && (memcmp(d1, other.d1, s1) == 0) && (memcmp(d2, other.d2, s2) == 0)); }
+    bool operator==(CacheKey const & other) const { return((mode == other.mode) && (s1 == other.s1) && (s2 == other.s2) && (memcmp(d1, other.d1, s1) == 0) && (memcmp(d2, other.d2, s2) == 0)); }
 private:
+    IRecordLayoutTranslator::Mode mode;
     size32_t s1;
     byte const * d1;
     size32_t s2;
@@ -243,7 +244,7 @@ private:
 class CacheValue
 {
 public:
-    CacheValue(size32_t s1, void const * d1, size32_t s2, void const * d2, IRecordLayoutTranslator * _trans);
+    CacheValue(IRecordLayoutTranslator::Mode _mode, size32_t s1, void const * d1, size32_t s2, void const * d2, IRecordLayoutTranslator * _trans);
     CacheKey const & queryKey() const { return key; }
     IRecordLayoutTranslator * getTranslator() const { return trans.getLink(); }
 private:
@@ -260,7 +261,7 @@ class CRecordLayoutTranslatorCache : public CacheTable, public IRecordLayoutTran
 public:
     IMPLEMENT_IINTERFACE;
     virtual ~CRecordLayoutTranslatorCache() { _releaseAll(); }
-    virtual IRecordLayoutTranslator * get(size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta = NULL);
+    virtual IRecordLayoutTranslator * get(IRecordLayoutTranslator::Mode mode, size32_t diskMetaSize, void const  * diskMetaData, size32_t activityMetaSize, void const * activityMetaData, IDefRecordMeta const * activityMeta = NULL);
     virtual unsigned count() const { return CacheTable::count(); }
 
 private:

common/thorhelper/thorsoapcall.cpp

@@ -290,7 +290,9 @@ private:
             catch (IException *e)
             {
                 StringBuffer s;
-                throw MakeStringException(ROXIE_ABORT_EVENT, "%s", e->errorMessage(s).str());
+                e->errorMessage(s);
+                e->Release();
+                throw MakeStringException(ROXIE_ABORT_EVENT, "%s", s.str());
             }
         }
     }

common/workunit/package.h

@@ -21,6 +21,7 @@
 #include "errorlist.h"
 #include "dadfs.hpp"
 #include "workunit.hpp"
+#include "layouttrans.hpp"
 
 // error codes
 #define PACKAGE_TARGET_NOT_FOUND      PACKAGE_ERROR_START
@@ -37,7 +38,7 @@ interface IHpccPackage : extends IInterface
     virtual const char *locateSuperFile(const char *superFileName) const = 0;
 
     virtual const char *queryEnv(const char *varname) const = 0;
-    virtual bool getEnableFieldTranslation() const = 0;
+    virtual IRecordLayoutTranslator::Mode getEnableFieldTranslation() const = 0;
     virtual bool isCompulsory() const = 0;
     virtual bool isPreload() const = 0;
     virtual const IPropertyTree *queryTree() const = 0;

common/workunit/pkgimpl.hpp

@@ -118,13 +118,20 @@ protected:
         return (node) ? node->getPropBool("@resolveLocally", false) : true;  // default is false for explicit package files, but true for the default empty package
     }
 
-    virtual bool getSysFieldTranslationEnabled() const {return false;}
-    virtual bool getEnableFieldTranslation() const
+    virtual IRecordLayoutTranslator::Mode getSysFieldTranslationEnabled() const { return IRecordLayoutTranslator::NoTranslation; }
+    virtual IRecordLayoutTranslator::Mode getEnableFieldTranslation() const
     {
         const char *val = queryEnv("control:enableFieldTranslation");
         if (!val) val = queryEnv("enableFieldTranslation"); // Backward compatibility
         if (val)
-            return strToBool(val);
+        {
+            if (strieq(val, "payload"))
+                return IRecordLayoutTranslator::TranslatePayload;
+            else if (strToBool(val))
+                return IRecordLayoutTranslator::TranslateAll;
+            else
+                return IRecordLayoutTranslator::NoTranslation;
+        }
         else
             return getSysFieldTranslationEnabled();
     }

common/workunit/workunit.cpp

@@ -2871,6 +2871,19 @@ public:
         return root->numChildren();
     }
 
+    /**
+     * Add a filter to an xpath query, with the appropriate filter flags
+     */
+    static void appendFilterToQueryString(StringBuffer& query, int flags, const char* name, const char* value)
+    {
+        query.append('[').append(name).append('=');
+        if (flags & WUSFnocase)
+            query.append('?');
+        if (flags & WUSFwild)
+            query.append('~');
+        query.append('"').append(value).append("\"]");
+    };
+
     IConstWorkUnitIterator* getWorkUnitsSorted( WUSortField sortorder, // field to sort by (and flags for desc sort etc)
                                                 WUSortField *filters,   // NULL or list of fields to filter on (terminated by WUSFterm)
                                                 const void *filterbuf,  // (appended) string values for filters
@@ -2881,6 +2894,69 @@ public:
                                                 ISecManager *secmgr,
                                                 ISecUser *secuser)
     {
+        class CQueryOrFilter : public CInterface
+        {
+            unsigned flags;
+            StringAttr name;
+            StringArray values;
+        public:
+            IMPLEMENT_IINTERFACE;
+            CQueryOrFilter(unsigned _flags, const char *_name, const char *_value)
+            : flags(_flags), name(_name)
+            {
+                values.appendListUniq(_value, "|");
+            };
+
+            const char* queryName() { return name.get(); };
+            unsigned querySearchFlags() { return flags; };
+            const StringArray& queryValues() const { return values; };
+        };
+        class CMultiPTreeIterator : public CInterfaceOf<IPropertyTreeIterator>
+        {
+        public:
+            virtual bool first() override
+            {
+                curSource = 0;
+                while (sources.isItem(curSource))
+                {
+                    if (sources.item(curSource).first())
+                        return true;
+                    curSource++;
+                }
+                return false;
+            }
+            virtual bool next() override
+            {
+                if (sources.isItem(curSource))
+                {
+                    if (sources.item(curSource).next())
+                        return true;
+                    curSource++;
+                    while (sources.isItem(curSource))
+                    {
+                        if (sources.item(curSource).first())
+                            return true;
+                        curSource++;
+                    }
+                }
+                return false;
+            }
+            virtual bool isValid() override
+            {
+                return sources.isItem(curSource);
+            }
+            virtual IPropertyTree & query() override
+            {
+                return sources.item(curSource).query();
+            }
+            void addSource(IPropertyTreeIterator &source)
+            {
+                sources.append(source);
+            }
+        private:
+            IArrayOf<IPropertyTreeIterator> sources;
+            unsigned curSource = 0;
+        };
         class CWorkUnitsPager : public CSimpleInterface, implements IElementsPager
         {
             StringAttr xPath;
@@ -2888,12 +2964,13 @@ public:
             StringAttr nameFilterLo;
             StringAttr nameFilterHi;
             StringArray unknownAttributes;
+            Owned<CQueryOrFilter> orFilter;
 
         public:
             IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-            CWorkUnitsPager(const char* _xPath, const char *_sortOrder, const char* _nameFilterLo, const char* _nameFilterHi, StringArray& _unknownAttributes)
-                : xPath(_xPath), sortOrder(_sortOrder), nameFilterLo(_nameFilterLo), nameFilterHi(_nameFilterHi)
+            CWorkUnitsPager(const char* _xPath, CQueryOrFilter* _orFilter, const char *_sortOrder, const char* _nameFilterLo, const char* _nameFilterHi, StringArray& _unknownAttributes)
+                : xPath(_xPath), orFilter(_orFilter), sortOrder(_sortOrder), nameFilterLo(_nameFilterLo), nameFilterHi(_nameFilterHi)
             {
                 ForEachItemIn(x, _unknownAttributes)
                     unknownAttributes.append(_unknownAttributes.item(x));
@@ -2903,7 +2980,36 @@ public:
                 Owned<IRemoteConnection> conn = querySDS().connect("WorkUnits", myProcessSession(), 0, SDS_LOCK_TIMEOUT);
                 if (!conn)
                     return NULL;
-                Owned<IPropertyTreeIterator> iter = conn->getElements(xPath);
+                Owned<IPropertyTreeIterator> iter;
+                if (!orFilter)
+                {
+                    iter.setown(conn->getElements(xPath.get()));
+                }
+                else
+                {
+                    Owned <CMultiPTreeIterator> multi = new CMultiPTreeIterator;
+                    bool added = false;
+                    const char* fieldName = orFilter->queryName();
+                    unsigned flags = orFilter->querySearchFlags();
+                    const StringArray& values = orFilter->queryValues();
+                    ForEachItemIn(i, values)
+                    {
+                        StringBuffer path = xPath.get();
+                        const char* value = values.item(i);
+                        if (!isEmptyString(value))
+                        {
+                            appendFilterToQueryString(path, flags, fieldName, value);
+                            IPropertyTreeIterator *itr = conn->getElements(path.str());
+                            if (itr)
+                            {
+                                multi->addSource(*itr);
+                                added = true;
+                            }
+                        }
+                    }
+                    if (added)
+                        iter.setown(multi.getClear());
+                }
                 if (!iter)
                     return NULL;
                 sortElements(iter, sortOrder.get(), nameFilterLo.get(), nameFilterHi.get(), unknownAttributes, elements);
@@ -2946,6 +3052,7 @@ public:
                 return ret;
             }
         };
+        Owned<CQueryOrFilter> orFilter;
         Owned<ISortedElementsTreeFilter> sc = new CScopeChecker(secmgr,secuser);
         StringBuffer query;
         StringBuffer so;
@@ -2984,12 +3091,19 @@ public:
                 }
                 else
                 {
-                    query.append('[').append(getEnumText(subfmt,workunitSortFields)).append('=');
-                    if (fmt&WUSFnocase)
-                        query.append('?');
-                    if (fmt&WUSFwild)
-                        query.append('~');
-                    query.append('"').append(fv).append("\"]");
+                    const char* fieldName = getEnumText(subfmt,workunitSortFields);
+                    if (!strchr(fv, '|'))
+                        appendFilterToQueryString(query, fmt, fieldName, fv);
+                    else if (orFilter)
+                        throw MakeStringException(WUERR_InvalidUserInput, "Multiple OR filters not allowed");
+                    else
+                    {
+                        if (!strieq(fieldName, getEnumText(WUSFstate,workunitSortFields)) &&
+                            !strieq(fieldName, getEnumText(WUSFuser,workunitSortFields)) &&
+                            !strieq(fieldName, getEnumText(WUSFcluster,workunitSortFields)))
+                            throw MakeStringException(WUERR_InvalidUserInput, "OR filters not allowed for %s", fieldName);
+                        orFilter.setown(new CQueryOrFilter(fmt, fieldName, fv));
+                    }
                 }
                 fv = fv + strlen(fv)+1;
             }
@@ -3010,7 +3124,7 @@ public:
             so.append(getEnumText(sortorder&0xff,workunitSortFields));
         }
         IArrayOf<IPropertyTree> results;
-        Owned<IElementsPager> elementsPager = new CWorkUnitsPager(query.str(), so.length()?so.str():NULL, namefilterlo.get(), namefilterhi.get(), unknownAttributes);
+        Owned<IElementsPager> elementsPager = new CWorkUnitsPager(query.str(), orFilter.getClear(), so.length()?so.str():NULL, namefilterlo.get(), namefilterhi.get(), unknownAttributes);
         Owned<IRemoteConnection> conn=getElementsPaged(elementsPager,startoffset,maxnum,secmgr?sc:NULL,"",cachehint,results,total,NULL);
         return new CConstWUArrayIterator(results);
     }

common/workunit/wuerror.hpp

@@ -52,4 +52,5 @@
 #define WUERR_WorkunitPluginError               5027
 #define WUERR_WorkunitVersionMismatch           5028
 #define WUERR_InvalidFieldUsage                 5029
+#define WUERR_InvalidUserInput                  5030
 #endif

configuration/configurator/BuildSet.cpp

@@ -16,6 +16,7 @@
 ############################################################################## */
 
 #include "jptree.hpp"
+#include "jfile.hpp"
 #include "XMLTags.h"
 #include "BuildSet.hpp"
 #include "SchemaCommon.hpp"
@@ -39,6 +40,9 @@ CBuildSetManager* CBuildSetManager::getInstance(const char* pBuildSetFile, const
         pBuildSetDirectory = s_pBuildSetManager->m_buildSetDir;
 
         s_pBuildSetManager->m_buildSetPath.setf("%s%s%s", pBuildSetDirectory, pBuildSetDirectory[strlen(pBuildSetDirectory)-1] == '/' ? "" : "/", pBuildSetFile);
+
+        if (!checkFileExists(s_pBuildSetManager->m_buildSetPath.str()))
+            return s_pBuildSetManager;
         if (s_pBuildSetManager->populateBuildSet() == false)
         {
             delete s_pBuildSetManager;

configuration/configurator/schemas/SchemaAttributes.cpp

@@ -195,6 +195,7 @@ void CAttribute::populateEnvXPath(StringBuffer strXPath, unsigned int index)
 {
     assert(this->getName() != nullptr);
 
+    //These two lines cause installset.xsd segmentation fault in debug. Will come back to this later
     const char *pChar = strrchr(strXPath.str(),'[');
     assert(pChar != nullptr && strlen(pChar) >= 3);
 

configuration/configurator/schemas/SchemaElement.cpp

@@ -269,7 +269,7 @@ void CElement::getDocumentation(::StringBuffer &strDoc) const
 
     assert(strlen(this->getName()) > 0);
 
-    if (pGrandParentNode->getNodeType() == XSD_SCHEMA)
+    if (strDoc.length() == 0)
     {
         ::StringBuffer strName(this->getName());
 

configuration/configurator/schemas/SchemaSimpleType.cpp

@@ -28,12 +28,12 @@ using namespace CONFIGURATOR;
 #define StringBuffer ::StringBuffer
 #define IPropertyTree ::IPropertyTree
 
-CXSDNodeBase* CSimpleType::getNodeByTypeAndNameAscending(NODE_TYPES eNodeType, const char *pName)
+const CXSDNodeBase* CSimpleType::getNodeByTypeAndNameAscending(NODE_TYPES eNodeType, const char *pName) const
 {
     return (this->checkSelf(eNodeType, pName, this->getName()) ? this : nullptr);
 }
 
-CXSDNodeBase* CSimpleType::getNodeByTypeAndNameDescending(NODE_TYPES eNodeType, const char *pName)
+const CXSDNodeBase* CSimpleType::getNodeByTypeAndNameDescending(NODE_TYPES eNodeType, const char *pName) const
 {
     return (this->checkSelf(eNodeType, pName, this->getName()) ? this : nullptr);
 }

configuration/configurator/schemas/SchemaSimpleType.hpp

@@ -36,8 +36,8 @@ public:
     {
     }
 
-    virtual CXSDNodeBase* getNodeByTypeAndNameAscending(NODE_TYPES eNodeType, const char *pName);
-    virtual CXSDNodeBase* getNodeByTypeAndNameDescending(NODE_TYPES eNodeType, const char *pName);
+    virtual const CXSDNodeBase* getNodeByTypeAndNameAscending(NODE_TYPES eNodeType, const char *pName) const override;
+    virtual const CXSDNodeBase* getNodeByTypeAndNameDescending(NODE_TYPES eNodeType, const char *pName) const override;
     virtual void dump(::std::ostream& cout, unsigned int offset = 0) const;
     virtual void getDocumentation(StringBuffer &strDoc) const;
     virtual void populateEnvXPath(StringBuffer strXPath, unsigned int index = 1);

docs/ECLStandardLibraryReference/SLR-Mods/MonitorFile.xml

@@ -2,7 +2,7 @@
 <!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <sect1 id="MonitorFile">
-  <title><emphasis role="bold">MonitorFile</emphasis></title>
+  <title>MonitorFile</title>
 
   <para><emphasis role="bold">STD.File.MonitorFile<indexterm>
       <primary>STD.File.MonitorFile</primary>

docs/ECLStandardLibraryReference/SLR-Mods/MonitorLogicalFileName.xml

@@ -2,7 +2,7 @@
 <!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <sect1 id="MonitorLogicalFileName">
-  <title><emphasis role="bold">MonitorLogicalFileName</emphasis></title>
+  <title>MonitorLogicalFileName</title>
 
   <para><emphasis role="bold">STD.File.MonitorLogicalFileName<indexterm>
       <primary>STD.File.MonitorLogicalFileName</primary>

docs/HPCCClientTools/CT_Mods/CT_ESDL_CLI.xml

@@ -378,9 +378,9 @@
       <sect2 id="CT_ESDL_CLI_esdl_publish" role="brk">
         <title>esdl publish</title>
 
-        <para><emphasis role="bold">esdl publish &lt;servicename&gt;
-        &lt;filename.ecm&gt; )[<emphasis>options</emphasis>]
-        </emphasis></para>
+        <para><emphasis role="bold">esdl publish
+        &lt;filename.(ecm|esdl|xml)&gt; &lt;servicename&gt;
+        [<emphasis>options</emphasis>] </emphasis></para>
 
         <para><informaltable colsep="1" frame="all" rowsep="1">
             <tgroup cols="2">
@@ -390,16 +390,17 @@
 
               <tbody>
                 <row>
-                  <entry>servicename</entry>
+                  <entry>filename</entry>
 
-                  <entry>The name of the service to publish</entry>
+                  <entry>The ESDL (*.ecm, *.esdl, or *.xml) file containing
+                  the service definitions.</entry>
                 </row>
 
                 <row>
-                  <entry>filename.ecm</entry>
+                  <entry>servicename</entry>
 
-                  <entry>The ESDL (*.ecm) file containing the service
-                  definitions.</entry>
+                  <entry>The name of the service to publish. Optional if the
+                  ESDL definition contains only one service. </entry>
                 </row>
 
                 <row>
@@ -461,7 +462,7 @@
 
         <para>Examples:</para>
 
-        <programlisting>esdl publish mathservice mathservice.ecm -s nnn.nnn.nnn.nnn --port 8010
+        <programlisting>esdl publish mathservice.ecm mathservice -s nnn.nnn.nnn.nnn --port 8010
 </programlisting>
       </sect2>
 
@@ -660,7 +661,8 @@
                 <entry>ESDLServiceName</entry>
 
                 <entry>The Name of the ESDL Service (as defined in the ESDL
-                Definition)</entry>
+                Definition). Optional if the ESDL definition contains only one
+                service. </entry>
               </row>
 
               <row>

docs/HPCCClientTools/CT_Mods/CT_Overview_withoutIDE.xml

@@ -228,9 +228,14 @@
       <para><emphasis role="bold">Windows</emphasis></para>
 
       <para>Client tools for Windows installs in a directory such as:
-      C:\Program Files (x86)\HPCCSystems\4.2.0\clientools where the number
-      (4.2.0 for example) corresponds to the version of the client
-      tools.</para>
+      C:\Program Files (x86)\HPCCSystems\6.2.0\clienttools\bin where the
+      number (6.2.0 for example) corresponds to the version of the client
+      tools. </para>
+
+      <para>If you want access to one version of the command line client tools
+      from any folder, you can add the \bin folder to your Path in Windows
+      (for example, <emphasis role="bold">C:\Program Files
+      (x86)\HPCCSystems\6.2.0\clienttools\bin</emphasis> ) </para>
 
       <para>The Windows installer will prompt you to delete the previous
       version during installation. If you want to keep both, decline the offer

docs/UsingConfigManager/UsingConfigManager.xml

@@ -163,14 +163,14 @@
           <listitem>
             <?dbfo keep-together="always"?>
 
-            <para>Enter the IP addresses or hostname(s). </para>
+            <para>Enter the IP addresses or hostname(s).</para>
 
             <para>IP Addresses can be specified individually using semi-colon
             delimiters. You can also specify a range of IPs using a hyphen
             (for example, nnn.nnn.nnn.x-y). In the image below, we specified
             the IP addresses 10.239.219.1 through 10.239.219.100 using the
             range syntax, and also a single IP 10.239.219.111. Alternatively,
-            you can enter the hostnames. </para>
+            you can enter the hostnames.</para>
 
             <para><graphic fileref="images/GS_ConfigMgrWiz002.jpg"
             vendor="configmgrSS" /></para>
@@ -1293,8 +1293,8 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
 
           <orderedlist>
             <listitem>
-              <para>Select ESP - MyEsp in the Navigator panel on the left
-              side.</para>
+              <para>Select <emphasis role="bold">ESP - myesp</emphasis> in the
+              Navigator panel on the left side.</para>
             </listitem>
 
             <listitem>
@@ -1920,7 +1920,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           <listitem>
             <?dbfo keep-together="always"?>
 
-            <para>right-click on the Master and select Add Slaves.</para>
+            <para>Right-click on the Master and select Add Slaves.</para>
 
             <para>
               <graphic fileref="images/CM-111.jpg" vendor="configmgrSS" />

ecl/eclagent/eclagent.cpp

@@ -40,6 +40,7 @@
 #include "workunit.hpp"
 #include "eventqueue.hpp"
 #include "schedulectrl.hpp"
+#include "jhtree.hpp"
 
 #include "mpbase.hpp"
 #include "daclient.hpp"

ecl/eclagent/eclgraph.cpp

@@ -1597,6 +1597,7 @@ void EclAgent::executeThorGraph(const char * graphName)
             thorMaster.getUrlStr(s);
             s.append("; (").append(e->errorCode()).append(", ");
             e->errorMessage(s).append(")");
+            e->Release();
             throw MakeStringExceptionDirect(-1, s.str());
         }
         ThorReplyCodes replyCode;

ecl/eclccserver/eclccserver.cpp

@@ -322,7 +322,9 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
         query->getQueryText(eclQuery);
         query->getQueryMainDefinition(mainDefinition);
 
-        StringBuffer eclccProgName("eclcc");
+        StringBuffer eclccProgName;
+        splitDirTail(queryCurrentProcessPath(), eclccProgName);
+        eclccProgName.append("eclcc");
         StringBuffer eclccCmd(" -shared");
         if (eclQuery.length())
             eclccCmd.append(" -");

ecl/hql/hqlthql.cpp

@@ -448,7 +448,7 @@ StringBuffer &HqltHql::callEclFunction(StringBuffer &s, IHqlExpression * expr, b
 {
     assertex(expr->isNamedSymbol());
     IHqlExpression * funcdef = expr->queryFunctionDefinition();
-    assertex(funcdef->getOperator() == no_funcdef);
+    assertex(funcdef->getOperator() == no_funcdef || funcdef->getOperator() == no_internalselect);
     IHqlExpression * formals = queryFunctionParameters(funcdef);
 
     s.append('(');

ecl/hthor/hthorkey.cpp

@@ -90,7 +90,7 @@ bool rltEnabled(IConstWorkUnit const * wu)
         return wu->getDebugValueBool("hthorLayoutTranslationEnabled", false);
 }
 
-IRecordLayoutTranslator * getRecordLayoutTranslator(IDefRecordMeta const * activityMeta, size32_t activityMetaSize, void const * activityMetaBuff, IDistributedFile * df, IRecordLayoutTranslatorCache * cache)
+IRecordLayoutTranslator * getRecordLayoutTranslator(IDefRecordMeta const * activityMeta, size32_t activityMetaSize, void const * activityMetaBuff, IDistributedFile * df, IRecordLayoutTranslatorCache * cache, IRecordLayoutTranslator::Mode mode)
 {
     IPropertyTree const & props = df->queryAttributes();
     MemoryBuffer diskMetaBuff;
@@ -107,9 +107,9 @@ IRecordLayoutTranslator * getRecordLayoutTranslator(IDefRecordMeta const * activ
     try
     {
         if(cache)
-            return cache->get(diskMetaBuff.length(), diskMetaBuff.bufferBase(), activityMetaSize, activityMetaBuff, activityMeta);
+            return cache->get(mode, diskMetaBuff.length(), diskMetaBuff.bufferBase(), activityMetaSize, activityMetaBuff, activityMeta);
         else
-            return createRecordLayoutTranslator(diskMetaBuff.length(), diskMetaBuff.bufferBase(), activityMetaSize, activityMetaBuff);
+            return createRecordLayoutTranslator(diskMetaBuff.length(), diskMetaBuff.bufferBase(), activityMetaSize, activityMetaBuff, mode);
     }
     catch (IException *E)
     {
@@ -742,7 +742,7 @@ IRecordLayoutTranslator * CHThorIndexReadActivityBase::getLayoutTranslator(IDist
         activityRecordMeta.setown(deserializeRecordMeta(buff, true));
     }
 
-    return getRecordLayoutTranslator(activityRecordMeta, activityRecordMetaSize, activityRecordMetaBuff, f, agent.queryRecordLayoutTranslatorCache());
+    return getRecordLayoutTranslator(activityRecordMeta, activityRecordMetaSize, activityRecordMetaBuff, f, agent.queryRecordLayoutTranslatorCache(), IRecordLayoutTranslator::TranslateAll);
 }
 
 void CHThorIndexReadActivityBase::verifyIndex(IKeyIndex * idx)
@@ -4062,7 +4062,7 @@ protected:
             activityRecordMeta.setown(deserializeRecordMeta(buff, true));
         }
 
-        return getRecordLayoutTranslator(activityRecordMeta, activityRecordMetaSize, activityRecordMetaBuff, f, agent.queryRecordLayoutTranslatorCache());
+        return getRecordLayoutTranslator(activityRecordMeta, activityRecordMetaSize, activityRecordMetaBuff, f, agent.queryRecordLayoutTranslatorCache(), IRecordLayoutTranslator::TranslateAll);
     }
 
     virtual void verifyIndex(IDistributedFile * f, IKeyIndex * idx, IRecordLayoutTranslator * trans)

ecl/regress/issue17182.ecl

@@ -0,0 +1,114 @@
+IMPORT Std;
+
+ParentMod := MODULE, VIRTUAL
+
+    SHARED DEFAULT_GENERATION_CNT := 3; // Update Init() docs if changed
+    SHARED MIN_GENERATION_CNT := 2;
+    SHARED SUPERFILE_SUFFIX := 'gen_';
+    SHARED SUBFILE_SUFFIX := 'file_';
+
+    SHARED _BuildSuperfilePathPrefix(STRING parent) := Std.Str.ToLowerCase(parent) + '::' + SUPERFILE_SUFFIX;
+
+    SHARED _BuildSuperfilePath(STRING parent, UNSIGNED1 generationNum) := _BuildSuperfilePathPrefix(parent) + generationNum;
+
+    SHARED _CreateSuperfilePathDS(STRING parent, UNSIGNED1 numGenerations) := DATASET
+        (
+            numGenerations,
+            TRANSFORM
+                (
+                    {
+                        UNSIGNED1   n, // Generation number
+                        STRING      f  // Superfile path
+                    },
+                    SELF.n := COUNTER,
+                    SELF.f := _BuildSuperfilePath(parent, COUNTER)
+                )
+        );
+
+    EXPORT _NumGenerationsAvailable(STRING dataStorePath) := FUNCTION
+        generationPattern := _BuildSuperfilePathPrefix(REGEXREPLACE('^~', dataStorePath, '')) + '*';
+        foundGenerationPaths := NOTHOR(Std.File.LogicalFileList(generationPattern, FALSE, TRUE));
+        expectedPaths := _CreateSuperfilePathDS(REGEXREPLACE('^~', dataStorePath, ''), COUNT(foundGenerationPaths));
+        joinedPaths := JOIN(foundGenerationPaths, expectedPaths, LEFT.name = RIGHT.f);
+        numJoinedPaths := COUNT(joinedPaths) : INDEPENDENT;
+        numExpectedPaths := COUNT(expectedPaths) : INDEPENDENT;
+        isSame := numJoinedPaths = numExpectedPaths;
+        isNumGenerationsValid := numJoinedPaths >= MIN_GENERATION_CNT;
+
+        RETURN WHEN(numJoinedPaths, ASSERT(isSame AND isNumGenerationsValid, 'Invalid structure: Unexpected superfile structure found for ' + dataStorePath, FAIL));
+    END;
+
+    //-----------------------------
+
+    EXPORT Init(STRING dataStorePath, UNSIGNED1 numGenerations = DEFAULT_GENERATION_CNT) := FUNCTION
+        clampedGenerations := MAX(MIN_GENERATION_CNT, numGenerations);
+        generationPaths := _CreateSuperfilePathDS(dataStorePath, clampedGenerations);
+        createParentAction := Std.File.CreateSuperFile(dataStorePath);
+        createGenerationsAction := NOTHOR(APPLY(generationPaths, Std.File.CreateSuperFile(f)));
+        appendGenerationsAction := NOTHOR(APPLY(generationPaths, Std.File.AddSuperFile(dataStorePath, f)));
+
+        RETURN ORDERED
+            (
+                createParentAction;
+                createGenerationsAction;
+                Std.File.StartSuperFileTransaction();
+                appendGenerationsAction;
+                Std.File.FinishSuperFileTransaction();
+            );
+    END;
+
+    EXPORT NumGenerationsAvailable(STRING dataStorePath) := FUNCTION
+        numGens := _NumGenerationsAvailable(dataStorePath) : INDEPENDENT;
+
+        RETURN numGens;
+    END;
+
+    EXPORT NumGenerationsInUse(STRING dataStorePath) := FUNCTION
+        numPartitions := NumGenerationsAvailable(dataStorePath);
+        generationPaths := _CreateSuperfilePathDS(dataStorePath, numPartitions);
+        generationsUsed := NOTHOR
+            (
+                PROJECT
+                    (
+                        generationPaths,
+                        TRANSFORM
+                            (
+                                {
+                                    RECORDOF(LEFT),
+                                    BOOLEAN     hasFiles
+                                },
+                                SELF.hasFiles := Std.File.GetSuperFileSubCount(LEFT.f) > 0,
+                                SELF := LEFT
+                            )
+                    )
+            );
+
+        RETURN MAX(generationsUsed(hasFiles), n);
+    END;
+
+END; // ParentMod
+
+//----------------------------
+
+ChildMod := MODULE(ParentMod)
+
+    EXPORT Tests := MODULE
+
+        SHARED dataStoreName := '~err::test::' + Std.System.Job.WUID();
+        SHARED numGens := 3;
+
+        EXPORT DoTest := SEQUENTIAL
+            (
+                Init(dataStoreName, numGens);
+                //ASSERT(ParentMod.NumGenerationsInUse(dataStoreName) = 0);
+                ASSERT(NumGenerationsInUse(dataStoreName) = 0);
+                TRUE;
+            );
+
+    END;
+
+END; // ChildMod
+
+//----------------------------
+
+ChildMod.Tests.DoTest;

esp/services/ws_dfu/CMakeLists.txt

@@ -49,6 +49,7 @@ include_directories (
          ./../../../rtl/eclrtl 
          ./../../../rtl/nbcd
          ./../../../common/environment 
+         ./../../../common/thorhelper 
          ./../../services 
          ./../../../dali/ft 
          ./../common 

esp/services/ws_packageprocess/CMakeLists.txt

@@ -38,6 +38,9 @@ include_directories (
          ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp
          ${HPCC_SOURCE_DIR}/dali/base
          ${HPCC_SOURCE_DIR}/common/environment
+         ${HPCC_SOURCE_DIR}/common/deftype
+         ${HPCC_SOURCE_DIR}/common/thorhelper
+         ${HPCC_SOURCE_DIR}/rtl/eclrtl
          ${HPCC_SOURCE_DIR}/dali/dfu
          ${HPCC_SOURCE_DIR}/common/remote
          ${HPCC_SOURCE_DIR}/common/workunit

esp/services/ws_workunits/CMakeLists.txt

@@ -54,6 +54,7 @@ include_directories (
          ./../../../common/remote
          ./../../../system/jlib
          ./../../../common/environment
+         ./../../../rtl/eclrtl
          ./../../../roxie/roxie
          ./../../services
          ./../common

esp/src/eclwatch/ECLPlaygroundWidget.js

@@ -285,7 +285,7 @@ define([
             var fetchedCount = 0;
             for (var i = 0; i < graphs.length; ++i) {
                 var context = this;
-                this.wu.fetchGraphXgmml(i, function (xgmml) {
+                this.wu.fetchGraphXgmml(i, null, function (xgmml) {
                     ++fetchedCount;
                     context.graphControl.loadXGMML(xgmml, fetchedCount > 1);
                     if (fetchedCount === graphs.length) {

esp/src/eclwatch/ESPWorkunit.js

@@ -828,10 +828,10 @@ define([
                 onGetGraphs: onFetchGraphs
             });
         },
-        fetchGraphXgmmlByName: function (name, onFetchGraphXgmml, force) {
+        fetchGraphXgmmlByName: function (name, subGraphId, onFetchGraphXgmml, force) {
             var idx = this.getGraphIndex(name);
             if (idx >= 0) {
-                this.fetchGraphXgmml(idx, onFetchGraphXgmml, force);
+                this.fetchGraphXgmml(idx, subGraphId, onFetchGraphXgmml, force);
             } else {
                 topic.publish("hpcc/brToaster", {
                     Severity: "Error",
@@ -843,10 +843,13 @@ define([
                 onFetchGraphXgmml("", "");
             }
         },
-        fetchGraphXgmml: function (idx, onFetchGraphXgmml, force) {
-            if (!force && this.graphs && this.graphs[idx] && this.graphs[idx].xgmml) {
+        fetchGraphXgmml: function (idx, subGraphId, onFetchGraphXgmml, force) {
+            if (!force && !subGraphId && this.graphs && this.graphs[idx] && this.graphs[idx].xgmml) {
                 onFetchGraphXgmml(this.graphs[idx].xgmml, this.graphs[idx].svg);
                 return;
+            } else if (!force && subGraphId && this.subgraphs && this.subgraphs[idx + "." + subGraphId] && this.subgraphs[idx + "." + subGraphId].xgmml) {
+                onFetchGraphXgmml(this.subgraphs[idx + "." + subGraphId].xgmml, this.subgraphs[idx + "." + subGraphId].svg);
+                return;
             }
 
             this._assertHasWuid();
@@ -854,12 +857,24 @@ define([
             WsWorkunits.WUGetGraph({
                 request: {
                     Wuid: this.Wuid,
-                    GraphName: this.graphs[idx].Name
+                    GraphName: this.graphs[idx].Name,
+                    SubGraphId: subGraphId
                 }
             }).then(function (response) {
                 if (lang.exists("WUGetGraphResponse.Graphs.ECLGraphEx", response) && response.WUGetGraphResponse.Graphs.ECLGraphEx.length) {
-                    context.graphs[idx].xgmml = response.WUGetGraphResponse.Graphs.ECLGraphEx[0].Graph;
-                    onFetchGraphXgmml(context.graphs[idx].xgmml, context.graphs[idx].svg);
+                    if (subGraphId) {
+                        if (!context.subgraphs) {
+                            context.subgraphs = {};
+                        }
+                        if (!context.subgraphs[idx + "." + subGraphId]) {
+                            context.subgraphs[idx + "." + subGraphId] = {};
+                        }
+                        context.subgraphs[idx + "." + subGraphId].xgmml = "<graph>" + response.WUGetGraphResponse.Graphs.ECLGraphEx[0].Graph + "</graph>";
+                        onFetchGraphXgmml(context.subgraphs[idx + "." + subGraphId].xgmml, context.subgraphs[idx + "." + subGraphId].svg);
+                    } else {
+                        context.graphs[idx].xgmml = response.WUGetGraphResponse.Graphs.ECLGraphEx[0].Graph;
+                        onFetchGraphXgmml(context.graphs[idx].xgmml, context.graphs[idx].svg);
+                    }
                 } else {
                     topic.publish("hpcc/brToaster", {
                         Severity: "Error",

esp/src/eclwatch/GraphPageWidget.js

@@ -506,7 +506,7 @@ define([
             this.main.setMessage(this.i18n.FetchingData);
             this.local.setMessage(this.i18n.FetchingData);
             var context = this;
-            wu.fetchGraphXgmmlByName(graphName, function (xgmml, svg) {
+            wu.fetchGraphXgmmlByName(graphName, null, function (xgmml, svg) {
                 context.overview.setMessage("");
                 context.main.setMessage("");
                 context.local.setMessage("");
@@ -518,7 +518,7 @@ define([
 
         refreshGraphFromWU: function (wu, graphName) {
             var context = this;
-            wu.fetchGraphXgmmlByName(graphName, function (xgmml) {
+            wu.fetchGraphXgmmlByName(graphName, null, function (xgmml) {
                 context.mergeGraphFromXGMML(xgmml);
             }, true);
         },
@@ -724,7 +724,7 @@ define([
 
         displayGraphs: function (graphs) {
             for (var i = 0; i < graphs.length; ++i) {
-                this.wu.fetchGraphXgmml(i, function (xgmml) {
+                this.wu.fetchGraphXgmml(i, null, function (xgmml) {
                     this.main.loadXGMML(xgmml, true);
                 });
             }

esp/src/eclwatch/GraphTreeWidget.js

@@ -427,6 +427,7 @@ define([
             }
 
             this.graphName = params.GraphName;
+            this.subGraphId = params.SubGraphId;
             this.widget.TimingsTreeMap.init(lang.mixin({
                 query: {
                     graphsOnly: true,
@@ -458,9 +459,9 @@ define([
                         onGetGraphs: function (graphs) {
                             if (firstLoad === true) {
                                 firstLoad = false;
-                                context.loadGraphFromWu(context.wu, context.graphName);
+                                context.loadGraphFromWu(context.wu, context.graphName, context.subGraphId);
                             } else {
-                                context.refreshGraphFromWU(context.wu, context.graphName);
+                                context.refreshGraphFromWU(context.wu, context.graphName, context.subGraphId);
                             }
                         },
                         onGetTimers: function (timers) {
@@ -478,7 +479,7 @@ define([
 
         refreshData: function () {
             if (this.isWorkunit()) {
-                this.loadGraphFromWu(this.wu, this.graphName, true);
+                this.loadGraphFromWu(this.wu, this.graphName, this.subGraphId, true);
             } else if (this.isQuery()) {
                 this.loadGraphFromQuery(this.targetQuery, this.queryId, this.graphName);
             }
@@ -523,11 +524,11 @@ define([
             this.loadEdges();
         },
 
-        loadGraphFromWu: function (wu, graphName, refresh) {
+        loadGraphFromWu: function (wu, graphName, subGraphId, refresh) {
             var deferred = new Deferred();
             this.main.setMessage(this.i18n.FetchingData);
             var context = this;
-            wu.fetchGraphXgmmlByName(graphName, function (xgmml, svg) {
+            wu.fetchGraphXgmmlByName(graphName, subGraphId, function (xgmml, svg) {
                 context.main.setMessage("");
                 context.loadGraphFromXGMML(xgmml, svg);
                 deferred.resolve();
@@ -535,9 +536,9 @@ define([
             return deferred.promise;
         },
 
-        refreshGraphFromWU: function (wu, graphName) {
+        refreshGraphFromWU: function (wu, graphName, subGraphId) {
             var context = this;
-            wu.fetchGraphXgmmlByName(graphName, function (xgmml) {
+            wu.fetchGraphXgmmlByName(graphName, subGraphId, function (xgmml) {
                 context.mergeGraphFromXGMML(xgmml);
             }, true);
         },
@@ -798,7 +799,7 @@ define([
 
         displayGraphs: function (graphs) {
             for (var i = 0; i < graphs.length; ++i) {
-                this.wu.fetchGraphXgmml(i, function (xgmml) {
+                this.wu.fetchGraphXgmml(i, null, function (xgmml) {
                     this.main.loadXGMML(xgmml, true);
                 });
             }

esp/src/eclwatch/templates/GraphTreeWidget.html

@@ -39,10 +39,6 @@
                 </select>
             </div>
             <div id="${id}OverviewTabContainer" data-dojo-props="region: 'center', tabPosition: 'bottom'" data-dojo-type="dijit.layout.StackContainer">
-                <div id="${id}TreeGridCP" tooltip="${i18n.Tree}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconFolderTree', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
-                    <div id="${id}TreeGrid">
-                    </div>
-                </div>
                 <div id="${id}SubgraphsGridCP" tooltip="${i18n.Subgraphs}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconFolderList', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
                     <div id="${id}SubgraphsGrid">
                     </div>
@@ -55,6 +51,10 @@
                     <div id="${id}EdgesGrid">
                     </div>
                 </div>
+                <div id="${id}TreeGridCP" tooltip="${i18n.Tree}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconFolderTree', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
+                    <div id="${id}TreeGrid">
+                    </div>
+                </div>
                 <div id="${id}TimingsTreeMap" tooltip="${i18n.TimingsMap}" data-dojo-props="iconClass:'iconTreeMap', showTitle: false" data-dojo-type="TimingTreeMapWidget">
                 </div>
                 <div id="${id}ActivitiesTreeMap" tooltip="${i18n.ActivityMap}" data-dojo-props="iconClass:'iconTreeMap', showTitle: false" data-dojo-type="TimingTreeMapWidget">

initfiles/bash/etc/init.d/dafilesrv.in

@@ -34,9 +34,11 @@
 #                         chkconfig Parameters
 ##-----------------------------------------------------------------------------
 
+### BEGIN INIT INFO
 # chkconfig: 235 30 80
 # Description: Starts the HPCC processes
 # Provides: dafilesrv
+### END INIT INFO
 
 ##-----------------------------------------------------------------------------
 #                         General Purpose Functions

initfiles/bash/etc/init.d/hpcc-init.in

@@ -33,10 +33,12 @@
 #                         chkconfig Parameters
 ##-----------------------------------------------------------------------------
 
+### BEGIN INIT INFO
 # chkconfig: 235 40 90
 # Description: Starts the HPCC processes
 # Provides: hpcc-init
 # Required-Start: dafilesrv
+### END INIT INFO
 
 ##-----------------------------------------------------------------------------
 #                         General Purpose Functions                           
@@ -139,9 +141,6 @@ fi
 
 check_getopt
 
-#changing ownership of the home directory for user.
-chown -cR $user:$group ${home}/${user} 1>/dev/null 2>/dev/null
-
 basepath=`pwd`
 
 COMPS=$(${configgen_path}/configgen -env ${envfile} -list)
@@ -278,7 +277,8 @@ log "Debug log written to $LOG_DIR/hpcc-init.debug"
 [ -e $LOG_DIR/hpcc-init.debug ] && rm -rf ${LOG_DIR}/hpcc-init.debug
 touch ${LOG_DIR}/hpcc-init.debug
 chown ${user}:${group} ${LOG_DIR}/hpcc-init.debug
-exec 2>$LOG_DIR/hpcc-init.debug
+PS4='+\011$(date "+%T.%N")\011'
+exec 3>&2 2>$LOG_DIR/hpcc-init.debug
 set -x
 
 if [ -z ${component} ]; then

initfiles/bin/init_thor.in

@@ -89,8 +89,6 @@ kill_slaves()
             else
                 log "Frunssh successful"
             fi
-        else
-            log "Could not locate $instancedir/uslaves file"
         fi
     fi
 

initfiles/bin/init_thorslave.in

@@ -45,7 +45,7 @@ stop_slaves()
     local isAlive=0
 
     log "Attempting to kill $slavename with SIGTERM"
-    killall -SIGTERM $slavename > /dev/null
+    killall -SIGTERM $slavename > /dev/null 2>&1
     while [[ $isAlive -eq 0 && $timer -gt 0 ]];do
         killall -0 $slavename > /dev/null 2>&1
         isAlive=$?

initfiles/componentfiles/configxml/esp.xsl

@@ -255,7 +255,7 @@
        <xsl:for-each select="preceding-sibling::EspBinding[@port=$port]">
            <xsl:variable name="type2" select="/Environment/Software/EspService[@name=current()/@service]/Properties/@type"/>
             <xsl:choose>
-                <xsl:when test="$type2=$type">
+                <xsl:when test="($type2=$type) and ($type!='DynamicESDL')">
                     <xsl:message terminate="yes">
                         <xsl:text>Port conflict in ESP binding '</xsl:text>
                         <xsl:value-of select="$name"/>

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -578,12 +578,19 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="fieldTranslationEnabled" type="xs:boolean" use="optional" default="false">
+    <xs:attribute name="fieldTranslationEnabled" use="optional" default="false">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>Enables translation (where possible) of mismatched index formats on-the-fly</tooltip>
+          <tooltip>Enables translation (where possible) of mismatched index layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only</tooltip>
         </xs:appinfo>
       </xs:annotation>
+      <xs:simpleType>
+        <xs:restriction base="xs:string">
+          <xs:enumeration value="false"/>
+          <xs:enumeration value="true"/>
+          <xs:enumeration value="payload"/>
+        </xs:restriction>
+      </xs:simpleType>
     </xs:attribute>
     <xs:attribute name="highTimeout" type="xs:nonNegativeInteger" use="optional" default="2000">
       <xs:annotation>

initfiles/etc/DIR_NAME/CMakeLists.txt

@@ -14,37 +14,87 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
+
 configure_file("version.in" "version")
 configure_file("environment.conf.in" "environment.conf" IMMEDIATE)
 configure_file("environment.xml.in" "environment.xml" IMMEDIATE)
-FOREACH( iFILES
+foreach(iFILES
     ${CMAKE_CURRENT_BINARY_DIR}/environment.conf
-    ${CMAKE_CURRENT_BINARY_DIR}/version
-)
-    Install ( FILES ${iFILES}  DESTINATION etc/${DIR_NAME} COMPONENT Runtime )
-ENDFOREACH ( iFILES)
+    ${CMAKE_CURRENT_BINARY_DIR}/version)
+    install(FILES ${iFILES}  DESTINATION etc/${DIR_NAME} COMPONENT Runtime)
+endforeach(iFILES)
 
-Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/environment.xml DESTINATION etc/${DIR_NAME}/rpmnew COMPONENT Runtime )
-Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/environment.conf DESTINATION etc/${DIR_NAME}/rpmnew COMPONENT Runtime )
-Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/genenvrules.conf DESTINATION etc/${DIR_NAME}/rpmnew COMPONENT Runtime )
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/environment.xml DESTINATION etc/${DIR_NAME}/rpmnew COMPONENT Runtime)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/environment.conf DESTINATION etc/${DIR_NAME}/rpmnew COMPONENT Runtime)
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/genenvrules.conf DESTINATION etc/${DIR_NAME}/rpmnew COMPONENT Runtime)
 
-ADD_SUBDIRECTORY(configmgr)
+add_subdirectory(configmgr)
 
+# Developer build
+# target: configure
 if(NOT "${DESTDIR}" STREQUAL "")
     message(STATUS "DESTDIR is set to ${DESTDIR}. Enabling configure target.")
-    add_custom_command(OUTPUT 
-        ${HOME_DIR}/${RUNTIME_USER}/.ssh/authorized_keys
-        ${HOME_DIR}/${RUNTIME_USER}/.ssh/id_rsa.pub
-        ${HOME_DIR}/${RUNTIME_USER}/.ssh/id_rsa
+
+    set(dirlist ${RUNTIME_PATH} ${LOG_PATH} ${LOCK_PATH} ${PID_PATH} ${INIT_PATH} ${CONFIG_DIR})
+    foreach(dir ${dirlist})
+        add_custom_command(OUTPUT ${dir}
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${dir}
+            COMMENT "creating ${dir}")
+    endforeach()
+    add_custom_target(configure_directories
+        DEPENDS ${dirlist})
+
+    add_custom_command(
+        OUTPUT  ${HOME_DIR}/${RUNTIME_USER}/.ssh/authorized_keys
+                ${HOME_DIR}/${RUNTIME_USER}/.ssh/id_rsa.pub
+                ${HOME_DIR}/${RUNTIME_USER}/.ssh/id_rsa
         COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/configure_keys.sh
         COMMENT "Attempting to generate ssh keys if not present"
-        WORKING_DIRECTORY ${HOME_DIR}/${RUNTIME_USER}/.ssh VERBATIM)
-    add_custom_target(configure
-        COMMAND ${CMAKE_COMMAND} -E make_directory ${CONFIG_DIR}
+        WORKING_DIRECTORY ${HOME_DIR}/${RUNTIME_USER}/.ssh
+        VERBATIM)
+    add_custom_target(configure_ssh_keys
+        DEPENDS ${HOME_DIR}/${RUNTIME_USER}/.ssh/authorized_keys
+                ${HOME_DIR}/${RUNTIME_USER}/.ssh/id_rsa.pub
+                ${HOME_DIR}/${RUNTIME_USER}/.ssh/id_rsa)
+    
+    add_custom_command(OUTPUT ${CONFIG_DIR}/${ENV_XML_FILE}
         COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${ENV_XML_FILE} ${CONFIG_DIR}
+        DEPENDS configure_directories ${CMAKE_CURRENT_BINARY_DIR}/environment.xml)
+    add_custom_target(configure_environment_xml
+        DEPENDS ${CONFIG_DIR}/${ENV_XML_FILE})
+
+    add_custom_command(OUTPUT ${CONFIG_DIR}/${ENV_CONF_FILE}
         COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${ENV_CONF_FILE} ${CONFIG_DIR}
+        DEPENDS configure_directories ${CMAKE_CURRENT_BINARY_DIR}/environment.conf)
+    add_custom_target(configure_environment_conf
+        DEPENDS ${CONFIG_DIR}/${ENV_CONF_FILE})
+    
+    add_custom_command(OUTPUT ${CONFIG_DIR}/genenvrules.conf
         COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/genenvrules.conf ${CONFIG_DIR}
-        DEPENDS ${HOME_DIR}/${RUNTIME_USER}/.ssh/authorized_keys
-        ${HOME_DIR}/${RUNTIME_USER}/.ssh/id_rsa.pub
-        ${HOME_DIR}/${RUNTIME_USER}/.ssh/id_rsa)
+        DEPENDS configure_directories ${CMAKE_CURRENT_SOURCE_DIR}/genenvrules.conf)
+    add_custom_target(configure_genenvrules_conf
+        DEPENDS ${CONFIG_DIR}/genenvrules.conf)
+   
+    add_custom_command(OUTPUT ${INIT_PATH}/dafilesrv
+        COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/initfiles/bash/etc/init.d/dafilesrv ${INIT_PATH}/dafilesrv
+        DEPENDS configure_directories
+                ProcessFiles-initfiles-bash-etc-init.d
+                ${CMAKE_BINARY_DIR}/initfiles/bash/etc/init.d/dafilesrv)
+    add_custom_command(OUTPUT ${INIT_PATH}/hpcc-init
+        COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/initfiles/bash/etc/init.d/hpcc-init ${INIT_PATH}/hpcc-init
+        DEPENDS configure_directories
+                ProcessFiles-initfiles-bash-etc-init.d
+                ${CMAKE_BINARY_DIR}/initfiles/bash/etc/init.d/hpcc-init)
+    add_custom_target(configure_service_files
+        DEPENDS ${INIT_PATH}/hpcc-init
+                ${INIT_PATH}/dafilesrv)
+
+    add_custom_target(configure ALL
+        DEPENDS configure_ssh_keys
+                configure_directories
+                configure_environment_xml
+                configure_environment_conf
+                configure_genenvrules_conf
+                configure_service_files
+        COMMENT "Configuring ${DESTDIR} for deployment")
 endif()

initfiles/sbin/configmgr.in

@@ -27,7 +27,8 @@ source ${INSTALL_DIR}/etc/init.d/hpcc_common
 
 createConf ()
 {
-    awk -f ${reg_path}/regex.awk -v NEW_ENVFILE=$1 -v NEW_PORT=$2 -v NEW_CONFFILE=$3< ${path}${configs}/configmgr/esp.xml >${runtime}/${compName}/esp.xml
+    dir_name=$(basename ${INSTALL_DIR})
+    awk -f ${reg_path}/regex.awk -v NEW_ENVFILE=$1 -v NEW_PORT=$2 -v NEW_CONFFILE=$3< ${path}/etc/${dir_name}/configmgr/esp.xml >${runtime}/${compName}/esp.xml
 }
 
 

plugins/CMakeLists.txt

@@ -38,4 +38,4 @@ add_subdirectory (redis)
 add_subdirectory (kafka)
 add_subdirectory (exampleplugin)
 add_subdirectory (couchbase)
-
+add_subdirectory(sqs)

plugins/Rembed/CMakeLists.txt

@@ -25,7 +25,7 @@
 project(Rembed)
 
 if(REMBED)
-    ADD_PLUGIN(Rembed PACKAGES R)
+    ADD_PLUGIN(Rembed PACKAGES R MINVERSION 0.12.1)
     if(MAKE_REMBED)
         set(
             SRCS
@@ -49,15 +49,22 @@ if(REMBED)
         endif()
 
         HPCC_ADD_LIBRARY(Rembed SHARED ${SRCS})
-        install(
-            TARGETS Rembed
-            DESTINATION plugins)
-
+        SET_DEPENDENCIES(CPACK_DEBIAN_PACKAGE_DEPENDS R-base-core r-cran-rcpp r-cran-inline)
+        SET_DEPENDENCIES(CPACK_RPM_PACKAGE_REQUIRES R-Rcpp)
         target_link_libraries(
             Rembed
             ${R_LIBRARIES}
             eclrtl
             jlib)
+
+        install(
+            TARGETS Rembed
+            DESTINATION plugins)
+        install(
+            FILES ${RINSIDE_LIBRARY}
+            DESTINATION ${LIB_DIR}
+            PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
+            COMPONENT Runtime)
     endif()
 endif()
 

plugins/Rembed/Rembed.cpp

@@ -423,7 +423,6 @@ public:
             theBoolSet = Rcpp::LogicalVector(elements);
             l[r] = theBoolSet;
             break;
-        case type_unsigned:
         case type_int:
             theIntSet = Rcpp::IntegerVector(elements);
             l[r] = theIntSet;

plugins/cassandra/cassandrawu.cpp

@@ -1526,7 +1526,7 @@ public:
     {
         return field;
     }
-private:
+protected:
     const char *xpath;
     StringAttr pattern;
     StringAttr value;
@@ -1534,6 +1534,35 @@ private:
     bool wild;
 };
 
+class MultiValuePostFilter : public PostFilter
+{
+public:
+    MultiValuePostFilter(WUSortField _field, const char *_value)
+      : PostFilter(_field, _value, false)
+    {
+        setValue(_value);
+    }
+    virtual bool matches(IPTree &p) const
+    {
+        const char *val = p.queryProp(xpath);
+        if (val)
+        {
+            ForEachItemIn(idx, values)
+            {
+                if (strieq(val, values.item(idx)))
+                    return true;
+            }
+        }
+        return false;
+    }
+    void setValue(const char *_value)
+    {
+        values.appendList(_value, "|");
+    }
+private:
+    StringArray values;
+};
+
 class AppValuePostFilter : public CInterfaceOf<IPostFilter>
 {
 public:
@@ -3417,6 +3446,8 @@ public:
                         if (s.length())
                             remoteWildFilters.append(*new PostFilter(field, s, true));  // Trailing-only wildcards can be done remotely
                     }
+                    else if (strchr(fv, '|'))
+                        goodFilters.append(*new MultiValuePostFilter(field, fv));
                     else
                         goodFilters.append(*new PostFilter(field, fv, false));
                     break;
@@ -3424,7 +3455,10 @@ public:
                 case WUSFpriority:
                 case WUSFprotected:
                     // These can't be wild, but are not very good filters
-                    poorFilters.append(*new PostFilter(field, fv, false));
+                    if (strchr(fv, '|'))
+                        poorFilters.append(*new MultiValuePostFilter(field, fv));
+                    else
+                        poorFilters.append(*new PostFilter(field, fv, false));
                     break;
                 case WUSFwuid: // Acts as wuidLo when specified as a filter
                 case WUSFwuidhigh:
@@ -3515,14 +3549,40 @@ public:
                 merger->addPostFilters(poorFilters, 0);
                 merger->addPostFilters(remoteWildFilters, 0);
                 const IPostFilter &best = goodFilters.item(0);
-                merger->addResult(*new CassandraResult(fetchDataForKeyWithFilter(best.queryXPath(), best.queryValue(), wuidFilters, sortorder, merger->hasPostFilters() ? 0 : pageSize+startOffset)));
+                const char *queryValue = best.queryValue();
+                if (strchr(queryValue, '|'))
+                {
+                    StringArray values;
+                    values.appendListUniq(queryValue, "|");
+                    ForEachItemIn(vidx, values)
+                    {
+                        const char *thisValue = values.item(vidx);
+                        if (!isEmptyString(thisValue))
+                            merger->addResult(*new CassandraResult(fetchDataForKeyWithFilter(best.queryXPath(), thisValue, wuidFilters, sortorder, merger->hasPostFilters() ? 0 : pageSize+startOffset)));
+                    }
+                }
+                else
+                    merger->addResult(*new CassandraResult(fetchDataForKeyWithFilter(best.queryXPath(), best.queryValue(), wuidFilters, sortorder, merger->hasPostFilters() ? 0 : pageSize+startOffset)));
             }
             else if (poorFilters.length())
             {
                 merger->addPostFilters(poorFilters, 1);
                 merger->addPostFilters(remoteWildFilters, 0);
                 const IPostFilter &best= poorFilters.item(0);
-                merger->addResult(*new CassandraResult(fetchDataForKeyWithFilter(best.queryXPath(), best.queryValue(), wuidFilters, sortorder, merger->hasPostFilters() ? 0 : pageSize+startOffset)));
+                const char *queryValue =best.queryValue();
+                if (strchr(queryValue, '|'))
+                {
+                    StringArray values;
+                    values.appendListUniq(queryValue, "|");
+                    ForEachItemIn(vidx, values)
+                    {
+                        const char *thisValue = values.item(vidx);
+                        if (!isEmptyString(thisValue))
+                            merger->addResult(*new CassandraResult(fetchDataForKeyWithFilter(best.queryXPath(), thisValue, wuidFilters, sortorder, merger->hasPostFilters() ? 0 : pageSize+startOffset)));
+                    }
+                }
+                else
+                    merger->addResult(*new CassandraResult(fetchDataForKeyWithFilter(best.queryXPath(), best.queryValue(), wuidFilters, sortorder, merger->hasPostFilters() ? 0 : pageSize+startOffset)));
             }
             else if (remoteWildFilters.length())
             {

plugins/sqs/CMakeLists.txt

@@ -0,0 +1,72 @@
+# minimal CMakeLists.txt for the AWS SDK for C++
+cmake_minimum_required(VERSION 2.8)
+
+# "my-example" is just an example value.
+project(sqs)
+
+
+if(SQS)
+   ADD_PLUGIN(sqs)
+   if(MAKE_SQS)
+    # Locate the AWS SDK for C++ package.
+    # Requires that you build with:
+    # -Daws-sdk-cpp_DIR=/path/to/sdk_build
+    # or export/set:
+    # CMAKE_PREFIX_PATH=/path/to/sdk_build
+    find_package(aws-sdk-cpp)
+    if(!aws-sdk-cpp_FOUND) 
+         message(FATAL_ERROR
+          "AWS SDK not found and required that you build with: 
+		-Daws-sdk-cpp_DIR=/path/to/sdk_build 
+           or export/set:
+		CMAKE_PREFIX_PATH=/path/to/sdk_buid")
+    endif()	
+
+    # Link to the SDK shared libraries.
+    add_definitions(-DUSE_IMPORT_EXPORT)
+#    add_subdirectory(doc)
+    include_directories (BEFORE ${INCLUDE_DIR})
+    set(
+            SRCS
+            sqs.h
+            sqs.cpp)
+
+   include_directories(
+            ./../../system/include
+            ./../../rtl/eclrtl
+            ./../../rtl/include
+            ./../../common/deftype
+            ./../../system/jlib
+            ${PROJECT_BINARY_DIR}/include
+            ${CMAKE_BINARY_DIR} )
+
+       add_definitions(-D_USRDLL -DECL_SQS_EXPORTS)
+        HPCC_ADD_LIBRARY(sqs SHARED ${SRCS})
+
+        if(${CMAKE_VERSION} VERSION_LESS "2.8.9")
+            message(WARNING "Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+        elseif(NOT APPLE)
+            set_target_properties(sqs PROPERTIES NO_SONAME 1)
+        endif()
+
+        install(
+            TARGETS sqs
+            DESTINATION plugins)
+
+
+        target_link_libraries(
+            sqs 
+            aws-cpp-sdk-sqs
+            eclrtl
+            jlib
+            ${ZLIB_LIBRARIES})
+
+   endif()
+endif()
+
+if(PLATFORM OR CLIENTTOOLS_ONLY)
+    install(
+        FILES ${CMAKE_CURRENT_SOURCE_DIR}/sqs.ecllib
+        DESTINATION plugins
+        COMPONENT Runtime)
+endif()

plugins/sqs/README.md

@@ -0,0 +1,14 @@
+#[Description] (SQS plugin allows to push message using ECL code)
+   * First you have to put your aws accessKey in the credential file into .aws folder. This folder is located in HPCC's home dir: "~/hpcc/.aws/" if it doesn't exist, you should create it.
+
+#[Example]
+ * import sqs;
+	publisher:=sqs.SQSPublisher('QueueName');
+	publisher.PublishMessage('Put your message here');
+
+
+
+
+
+
+

plugins/sqs/cmake_install.cmake

@@ -0,0 +1,44 @@
+# Install script for directory: /hpcc/HPCC-Platform/plugins/sqs
+
+# Set the install prefix
+if(NOT DEFINED CMAKE_INSTALL_PREFIX)
+  set(CMAKE_INSTALL_PREFIX "/usr/local")
+endif()
+string(REGEX REPLACE "/$" "" CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}")
+
+# Set the install configuration name.
+if(NOT DEFINED CMAKE_INSTALL_CONFIG_NAME)
+  if(BUILD_TYPE)
+    string(REGEX REPLACE "^[^A-Za-z0-9_]+" ""
+           CMAKE_INSTALL_CONFIG_NAME "${BUILD_TYPE}")
+  else()
+    set(CMAKE_INSTALL_CONFIG_NAME "")
+  endif()
+  message(STATUS "Install configuration: \"${CMAKE_INSTALL_CONFIG_NAME}\"")
+endif()
+
+# Set the component getting installed.
+if(NOT CMAKE_INSTALL_COMPONENT)
+  if(COMPONENT)
+    message(STATUS "Install component: \"${COMPONENT}\"")
+    set(CMAKE_INSTALL_COMPONENT "${COMPONENT}")
+  else()
+    set(CMAKE_INSTALL_COMPONENT)
+  endif()
+endif()
+
+# Install shared libraries without execute permission?
+if(NOT DEFINED CMAKE_INSTALL_SO_NO_EXE)
+  set(CMAKE_INSTALL_SO_NO_EXE "1")
+endif()
+
+if(CMAKE_INSTALL_COMPONENT)
+  set(CMAKE_INSTALL_MANIFEST "install_manifest_${CMAKE_INSTALL_COMPONENT}.txt")
+else()
+  set(CMAKE_INSTALL_MANIFEST "install_manifest.txt")
+endif()
+
+string(REPLACE ";" "\n" CMAKE_INSTALL_MANIFEST_CONTENT
+       "${CMAKE_INSTALL_MANIFEST_FILES}")
+file(WRITE "/hpcc/HPCC-Platform/plugins/sqs/${CMAKE_INSTALL_MANIFEST}"
+     "${CMAKE_INSTALL_MANIFEST_CONTENT}")

plugins/sqs/sqs.cpp

@@ -0,0 +1,451 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */ 
+
+
+#include <aws/core/Aws.h>
+#include <aws/sqs/SQSClient.h>
+#include <aws/sqs/model/CreateQueueRequest.h>
+#include <aws/sqs/model/AddPermissionRequest.h>
+#include <aws/sqs/model/DeleteQueueRequest.h>
+#include <aws/sqs/model/ListQueuesRequest.h>
+#include <aws/sqs/model/ListQueuesResult.h>
+#include <aws/sqs/model/SendMessageRequest.h>
+#include <aws/sqs/model/SendMessageResult.h>
+#include <aws/sqs/model/ReceiveMessageRequest.h>
+#include <aws/sqs/model/ReceiveMessageResult.h>
+#include <aws/sqs/model/GetQueueUrlRequest.h>
+#include <aws/sqs/model/GetQueueUrlResult.h>
+#include "platform.h"
+#include "jexcept.hpp"
+#include "jthread.hpp"
+#include "deftype.hpp"
+#include "eclhelper.hpp"
+#include "eclrtl.hpp"
+#include "eclrtl_imp.hpp"
+#include <iostream>
+#include <string>
+#include <algorithm>
+#include <cstring>
+#include <cctype>
+#include "sqs.h"
+
+using namespace std;
+
+static const char * compatibleVersions[] = {
+  "AWS Amazon SQS based on SDK AWS",
+  NULL };
+
+static const char *version = "SQS Version depends on AWS";
+
+extern "C" DECL_EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+  if (pb->size == sizeof(ECLPluginDefinitionBlock))
+    {
+      ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
+      pbx->compatibleVersions = compatibleVersions;
+      pb->magicVersion = PLUGIN_VERSION;
+      pb->version = version;
+      pb->moduleName = "SQS";
+      pb->ECL = NULL;
+      pb->flags = PLUGIN_MULTIPLE_VERSIONS;
+      pb->description = "SQS based on AWS SDK";
+      return true;
+    }
+  else
+    {
+      return false;
+    }	
+}
+
+
+/**
+ * This method creates an instance of SQSHPCC
+ * @param _queueName this name gave to queue
+ *
+ */
+
+SQSHPCCPlugin::SQSHPCC::SQSHPCC(const string& _queuename) 
+{
+
+  if(_queuename.empty())
+    {
+      throw runtime_error("QueueName is required");
+    }
+  this->queueName=_queuename;
+  cout << "The queue is " << this->queueName << endl;
+  
+}
+
+
+/**
+ * Destructor  
+ * Release the SQS connection
+ */
+SQSHPCCPlugin::SQSHPCC::~SQSHPCC()
+{
+
+  // Aws::ShutdownAPI(this->options);
+  this->queueName="";
+  delete(this->sqsClient);
+}
+
+/**
+ *  
+ *
+ *
+ **/
+
+SQSHPCCPlugin::Response  SQSHPCCPlugin::SQSHPCC::sendMessage(const char* message) 
+{
+
+  SQSHPCCPlugin::Response ref = {};
+
+  cout << "SendMessage is " << this->queueUrl << endl; 
+
+  try {
+    Aws::SQS::Model::SendMessageRequest sendMessageRequest;
+    sendMessageRequest.SetQueueUrl(this->queueUrl);
+    sendMessageRequest.SetMessageBody(message);
+ 
+    Aws::SQS::Model::SendMessageOutcome sendMessageOutcome = this->sqsClient->SendMessage(sendMessageRequest);
+     
+    if(!sendMessageOutcome.IsSuccess() || sendMessageOutcome.GetResult().GetMessageId().length() == 0 )
+      {
+	cout << "Error occurred during the sending " << endl;
+	ref.code=-1;
+	ref.body="Error occurred during message sending";
+      }
+    else 
+      {
+	ref.code=3;
+	ref.body=convertAwsStringToCharPtr(sendMessageOutcome.GetResult().GetMessageId());
+      }	
+     
+  } catch (const char* message) {
+    cout<< "Error occurred during sending message [ " << message << " ]" << endl;
+  }
+
+  return ref;
+}
+
+
+/**
+ *
+ *  This function allows to disconnect the link with AWS SQS
+ *
+ **/
+
+bool SQSHPCCPlugin::SQSHPCC::disconnect() 
+{
+  try 
+    {
+      Aws::ShutdownAPI(this->options);
+      return true;
+    }
+  catch(...)
+    {
+      return false;
+    }	
+}
+
+
+/**
+ *
+ *
+ */
+bool SQSHPCCPlugin::SQSHPCC::QueueExists() 
+{
+  Aws::SQS::Model::GetQueueUrlRequest gqu_req;
+  gqu_req.SetQueueName(this->queueName.c_str());
+  bool exists=false;
+  try
+    {
+      Aws::SQS::Model::GetQueueUrlOutcome gqu_out = this->sqsClient->GetQueueUrl(gqu_req);
+      exists =gqu_out.IsSuccess(); 
+    }
+  catch(const char* message) 
+    {
+      cout << "Error: " << message << endl;
+    }
+   
+  return exists;
+}
+
+SQSHPCCPlugin::Response SQSHPCCPlugin::SQSHPCC::createQueue()
+{
+
+  SQSHPCCPlugin::Response ref = {};
+
+  Aws::SQS::Model::CreateQueueRequest cq_req;
+  cq_req.SetQueueName(this->queueName.c_str());
+
+  Aws::SQS::Model::CreateQueueOutcome  cq_out = this->sqsClient->CreateQueue(cq_req);
+  if (cq_out.IsSuccess()) 
+    {
+      cout << "Successfully created queue " << this->queueName << std::endl;
+    }
+  else 
+    {
+      cout << "Error creating queue " << this->queueName << ": " <<
+	cq_out.GetError().GetMessage() << std::endl;
+    }
+
+  return ref;
+}
+
+
+SQSHPCCPlugin::Response SQSHPCCPlugin::SQSHPCC::deleteQueue()
+{
+
+  SQSHPCCPlugin::Response ref = {};
+
+  Aws::SQS::Model::DeleteQueueRequest cq_req;
+  cq_req.SetQueueUrl(this->queueUrl);
+
+  auto  cq_out = this->sqsClient->DeleteQueue(cq_req);
+  if (cq_out.IsSuccess()) 
+    {
+      cout << "Successfully deleted queue " << this->queueName << std::endl;
+    } 
+  else 
+    {
+      cout << "Error deleting queue " << this->queueName << ": " <<
+	cq_out.GetError().GetMessage() << std::endl;
+    }
+
+  return ref;
+}
+
+
+SQSHPCCPlugin::Response SQSHPCCPlugin::SQSHPCC::receiveMessage() 
+{
+
+  SQSHPCCPlugin::Response ref = {};
+  try
+    {
+      Aws::SQS::Model::ReceiveMessageRequest receiveMessageRequest;
+      receiveMessageRequest.SetQueueUrl(this->queueUrl);
+      receiveMessageRequest.SetMaxNumberOfMessages(1);
+      receiveMessageRequest.AddMessageAttributeNames("All");
+
+      Aws::SQS::Model::ReceiveMessageOutcome receiveMessageOutcome = this->sqsClient->ReceiveMessage(receiveMessageRequest);
+      if(!receiveMessageOutcome.IsSuccess() || receiveMessageOutcome.GetResult().GetMessages().size() == 0)
+	{
+	  std::cout << "Error on receive: " << receiveMessageOutcome.GetError().GetMessage() << std::endl;
+	  ref.code=2;
+	  return ref;
+	}
+ 
+      Aws::SQS::Model::Message msg = receiveMessageOutcome.GetResult().GetMessages()[0];
+      cout << msg.GetBody() << endl;
+    } 
+  catch(const char* message) 
+    {
+      cout << "Error: " << message << endl; 
+    }
+   
+  return ref;
+}
+
+
+void SQSHPCCPlugin::SQSHPCC::setQueueUrlFromQueueName() 
+{
+
+  Aws::SQS::Model::GetQueueUrlRequest gqu_req;
+  gqu_req.SetQueueName(this->queueName.c_str());
+
+  try 
+    {
+      Aws::SQS::Model::GetQueueUrlOutcome gqu_out = this->sqsClient->GetQueueUrl(gqu_req);
+      if(gqu_out.IsSuccess()) 
+	{
+	  std::cout << "Queue " << this->queueName.c_str() << " has url " << std::endl;
+	  this->queueUrl=gqu_out.GetResult().GetQueueUrl();
+        } 
+      else
+	{
+	  std::cout << "Error getting url for queue " << this->queueName.c_str() << ": " << std::endl;
+	  throw runtime_error(gqu_out.GetError().GetMessage().c_str()) ;
+        }
+
+    } catch(const char* message) {
+    cout << "Error: " << message << endl;
+  }
+
+}
+
+void SQSHPCCPlugin::SQSHPCC::setSQSConfiguration(const string& protocol, const string& region)
+{
+
+  Aws::InitAPI(this->options);
+  Aws::Client::ClientConfiguration config;
+  
+  if(!protocol.empty())
+    {
+      config.scheme = Aws::Http::Scheme::HTTPS;
+    }
+
+  if(region.empty()) throw string("Region mustn't be empty");
+  
+  if(RegionExists(region))
+    {
+      config.region = getRegion(region);
+    }
+
+  this->sqsClient= new Aws::SQS::SQSClient(config);
+
+  cout << "Queue URL is "  << this->queueUrl << endl;
+}
+
+
+bool SQSHPCCPlugin::SQSHPCC::RegionExists(const string& region)
+{
+  
+  if(region.empty())
+    {
+      return false;
+    }
+
+  char *reg = convertStringToChar(region); 
+  
+  return (
+          strieq(reg,"US_EAST_1") ||
+    	  strieq(reg,"US_WEST_1") ||
+    	  strieq(reg,"EU_WEST_1") ||
+    	  strieq(reg,"EU_CENTRAL_1") ||
+    	  strieq(reg,"AP_SOUTHEAST_1") ||
+     	  strieq(reg,"AP_SOUTHEAST_2")
+	  ); 
+}
+
+
+const char *const SQSHPCCPlugin::SQSHPCC::getRegion(const string& region) 
+{
+
+  char *reg = convertStringToChar(region); 
+    
+  if(strieq(reg,"US_EAST_1"))  return Aws::Region::US_EAST_1;
+  if(strieq(reg,"US_WEST_1")) return Aws::Region::US_WEST_1;
+  if(strieq(reg,"EU_WEST_1")) return Aws::Region::EU_WEST_1;
+  if(strieq(reg,"EU_CENTRAL_1")) return Aws::Region::EU_CENTRAL_1;
+  if(strieq(reg,"AP_SOUTHEAST_1")) return Aws::Region::AP_SOUTHEAST_1;
+  if(strieq(reg,"AP_SOUTHEAST_2"))  return Aws::Region::AP_SOUTHEAST_2;
+
+  throw string("Your region must be among these regions [ US_EAST_1, US_WEST_1, EU_WEST_1, EU_CENTRAL_1, AP_SOUTHEAST_1, AP_SOUTHEAST_2 ], please check your region");
+}
+
+
+void SQSHPCCPlugin::SQSHPCC::upstr(char *s)
+{
+  char  *p;
+
+  for (p = s; *p != '\0'; p++) 
+    *p = (char) toupper(*p);
+}
+
+
+
+char*  SQSHPCCPlugin::SQSHPCC::convertStringToChar(const string& str) 
+{
+  char *reg = new char[str.length() +1];
+  strcpy(reg,str.c_str());
+  return reg;
+}
+
+string SQSHPCCPlugin::SQSHPCC::convertAwsStringToCharPtr(Aws::String str)
+{
+  string res(str.c_str());
+  return res;
+}
+
+/**
+ *  These function expose the contract for ECL  
+ *
+ */
+
+namespace SQSHPCCPlugin
+{
+
+  ECL_SQS_API bool ECL_SQS_CALL publishMessage(ICodeContext * ctx,const char* region, const char* queueName, const char* message)
+  {
+
+    if(strlen(queueName) == 0) 
+      {
+	cout << "QueueName is Empty" << endl;
+	throw runtime_error("The queueName mustn't be empty!!!");
+      }
+    try 
+      { 
+	SQSHPCCPlugin::SQSHPCC hpcc(queueName);
+	hpcc.setSQSConfiguration("HTTPS",region);
+	hpcc.setQueueUrlFromQueueName();
+	SQSHPCCPlugin::Response response = hpcc.sendMessage(message);
+	return true;
+      }	
+    catch(...)
+      {
+	throw;
+      }
+	  
+    return false;
+  }
+
+  ECL_SQS_API bool ECL_SQS_CALL QueueExists(ICodeContext* ctx,const char* region, const char* queueName)
+  {
+
+    SQSHPCCPlugin::SQSHPCC hpcc(queueName);
+    hpcc.setSQSConfiguration("HTPPS",region);
+    hpcc.setQueueUrlFromQueueName();
+    bool exists = hpcc.QueueExists();
+    return exists;
+  }
+
+  ECL_SQS_API bool createQueue(ICodeContext* ctx,const char* region, const char* queueName)
+  {
+    SQSHPCCPlugin::SQSHPCC hpcc(queueName);
+    hpcc.setSQSConfiguration("HTTPS",region);
+    try
+      {
+	SQSHPCCPlugin::Response response = hpcc.createQueue();
+	return true;
+      }
+    catch(...)
+      {
+	return false;
+      }	
+	    
+  }
+
+  ECL_SQS_API bool ECL_SQS_CALL deleteQueue(ICodeContext * ctx,const char* region, const char* queueName)
+  {
+
+    SQSHPCCPlugin::SQSHPCC hpcc(queueName);
+    hpcc.setSQSConfiguration("HTTPS",region);
+    hpcc.setQueueUrlFromQueueName();
+    try
+      {
+	hpcc.deleteQueue();
+	return true;
+      }
+    catch(...)
+      {
+	return false;
+      }
+  }
+}
+
+

plugins/sqs/sqs.ecllib

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the License);
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an AS IS BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+// Service definition
+SHARED sqs := SERVICE : plugin('sqs'), namespace('SQSHPCCPlugin')
+
+    BOOLEAN PublishMessage(CONST VARSTRING region, CONST VARSTRING queueName, CONST VARSTRING message) : cpp,action,context,entrypoint='publishMessage';
+    BOOLEAN IsQueueExist(CONST VARSTRING region,CONST VARSTRING queueName):cpp,action,context,entrypoint='isQueueExist';
+    BOOLEAN CreateQueue(CONST VARSTRING region,CONST VARSTRING queueName):cpp,action,context,entrypoint='createQueue';
+END;
+
+/**
+ * Module wrapping message publishing functions.
+ *
+ * @param   queue           The name of the queue this module will be publishing to;
+ *                          cannot be an empty string; REQUIRED
+ * @param   region          The AWS region is the zone where is the queue SQS is located. By default is "eu_west_1"       
+ */
+EXPORT SQSPublisher(VARSTRING queue, VARSTRING region = 'eu_west_1') := MODULE
+
+
+  EXPORT BOOLEAN PublishMessage(CONST VARSTRING message) := sqs.PublishMessage(region,queue,message);
+  EXPORT BOOLEAN IsQueueExist():= sqs.isQueueExist(region,queue);
+  EXPORT BOOLEAN CreateQueue():= sqs.createQueue(region,queue);
+END;

plugins/sqs/sqs.h

@@ -0,0 +1,138 @@
+
+/*##############################################################################
+
+  HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  ############################################################################## */
+
+#ifndef SQSHPCC_H
+#define SQSHPCC_H
+
+
+
+#ifdef _WIN32
+#define ECL_SQS_CALL _cdecl
+#else
+#define ECL_SQS_CALL
+#endif
+
+#ifdef ECL_SQS_EXPORTS
+#define ECL_SQS_API DECL_EXPORT
+#else
+#define ECL_SQS_API DECL_IMPORT
+#endif
+
+#include "platform.h"
+#include "jthread.hpp"
+#include "hqlplugins.hpp"
+#include "eclrtl_imp.hpp"
+#include "eclhelper.hpp"
+#include <aws/core/Aws.h>
+#include <aws/sqs/SQSClient.h>
+#include <iostream>
+#include <stdio.h>
+#include <string>
+#include <unistd.h>
+#include <libgen.h>
+
+
+#ifdef ECL_SQS_EXPORTS
+
+extern  "C" 
+{
+  ECL_SQS_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
+  ECL_SQS_API void setPluginContext(IPluginContext * _ctx);
+}
+#endif
+
+extern "C++" 
+{
+  namespace SQSHPCCPlugin
+  {
+    using namespace std;
+    typedef struct {
+      int code;
+      std::string body;
+      bool success;
+    } Response;
+
+
+
+  
+    class SQSHPCC
+    {
+
+    public:
+      explicit SQSHPCC(const std::string& _queueName);
+      ~SQSHPCC();
+ 
+      Response sendMessage(const char* message);
+      Response createQueue();
+      Response deleteQueue();
+      Response deleteMessage(const std::string& message);
+      Response receiveMessage();
+
+      void setSQSConfiguration(const std::string& protocol,
+			       const std::string& region); 
+      void setAwsCredentials(const char* accessKeyId, 
+			     const char* secretKey); 
+      bool disconnect();
+      bool QueueExists();
+
+      void setQueueUrlFromQueueName();
+
+    protected:
+
+    private:
+      std::string queueName;
+      Aws::String queueUrl;
+      Aws::SQS::SQSClient* sqsClient;
+      Aws::SDKOptions options;
+      Aws::Auth::AWSCredentials* credentials;
+      bool RegionExists(const std::string& region);
+      const char *const getRegion(const std::string& region);
+      void upstr(char* s);
+      std::string convertAwsStringToCharPtr(Aws::String str);
+      char*  convertStringToChar(const string& str);
+    };
+
+
+    //----------------------------------------------------------------------
+
+    /**
+     * Queues the message for publishing to aws queue
+     * 
+     * @param   Reqion          
+     * @param   QueueName
+     * @param   message            The message to send
+     *
+     * @return  true if the message was cached successfully
+     */
+    ECL_SQS_API bool ECL_SQS_CALL publishMessage(ICodeContext * ctx,const char* region, const char* queueName, const char* message); 
+
+    //---------------------------------------------------------------------
+
+    /** 
+     *
+     *
+     */
+    ECL_SQS_API bool ECL_SQS_CALL createQueue(ICodeContext * ctx,const char* region, const char* queueName);
+
+    ECL_SQS_API bool ECL_SQS_CALL QueueExists(ICodeContext* ctx,const char* region, const char* queueName);
+     
+    ECL_SQS_API bool ECL_SQS_CALL deleteQueue(ICodeContext * ctx,const char* region, const char* queueName);      
+	
+  }
+}
+#endif

plugins/sqs/sqs.pc

@@ -0,0 +1,11 @@
+prefix=/usr
+exec_prefix=${prefix}
+libdir=${exec_prefix}/lib
+includedir=${prefix}/include
+
+Name: SQSHPCC
+Description: This is a plugin for HPCC allowing to dialog with Amazon's SQS 
+Version: 0.0.1
+Requires:
+Libs: -L${libdir} -lssl -lcrypto -lz
+Cflags: -I${includedir}

roxie/ccd/ccd.hpp

@@ -31,6 +31,7 @@
 #include "roxiedebug.ipp"
 #include "eclrtl.hpp"
 #include "workunit.hpp"
+#include "layouttrans.hpp"
 
 #ifdef CCD_EXPORTS
 #define CCD_API DECL_EXPORT
@@ -417,7 +418,7 @@ struct PartNoType
 extern unsigned statsExpiryTime;
 extern time_t startupTime;
 extern unsigned miscDebugTraceLevel;
-extern bool fieldTranslationEnabled;
+extern IRecordLayoutTranslator::Mode fieldTranslationEnabled;
 
 extern unsigned defaultParallelJoinPreload;
 extern unsigned defaultConcatPreload;

roxie/ccd/ccdactivities.cpp

@@ -156,7 +156,7 @@ public:
         return ret.appendf("%p", this);
     }
 
-    bool getEnableFieldTranslation() const
+    IRecordLayoutTranslator::Mode getEnableFieldTranslation() const
     {
         return queryFactory.queryOptions().enableFieldTranslation;
     }
@@ -308,7 +308,7 @@ protected:
     bool resent;
     bool isOpt;
     bool variableFileName;
-    bool allowFieldTranslation;
+    IRecordLayoutTranslator::Mode allowFieldTranslation;
     Owned<const IResolvedFile> varFileInfo;
 
     virtual void setPartNo(bool filechanged) = 0;

roxie/ccd/ccdcontext.cpp

@@ -2321,6 +2321,7 @@ protected:
                 thorMaster.getUrlStr(s);
                 s.append("; (").append(e->errorCode()).append(", ");
                 e->errorMessage(s).append(")");
+                e->Release();
                 throw MakeStringExceptionDirect(-1, s.str());
             }
             ThorReplyCodes replyCode;

roxie/ccd/ccdfile.cpp

@@ -1962,7 +1962,7 @@ public:
         }
         return f.getClear();
     }
-    virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, bool allowFieldTranslation) const
+    virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, IRecordLayoutTranslator::Mode allowFieldTranslation) const override
     {
         unsigned maxParts = 0;
         ForEachItemIn(subFile, subFiles)
@@ -1981,8 +1981,8 @@ public:
             if (translators)
             {
                 if (fdesc && thisDiskMeta && activityMeta && !thisDiskMeta->equals(activityMeta))
-                    if (allowFieldTranslation)
-                        translators->append(createRecordLayoutTranslator(lfn, thisDiskMeta, activityMeta));
+                    if (allowFieldTranslation != IRecordLayoutTranslator::NoTranslation)
+                        translators->append(createRecordLayoutTranslator(lfn, thisDiskMeta, activityMeta, allowFieldTranslation));
                     else
                     {
                         DBGLOG("Key layout mismatch: %s", lfn.get());

roxie/ccd/ccdfile.hpp

@@ -21,6 +21,7 @@
 #include "eclhelper.hpp"
 #include "ccddali.hpp"
 #include "dautils.hpp"
+#include "layouttrans.hpp"
 
 enum RoxieFileStatus { FileSizeMismatch, FileDateMismatch, FileCRCMismatch, FileIsValid, FileNotFound };
 enum RoxieFileType { ROXIE_KEY, ROXIE_FILE, ROXIE_PATCH, ROXIE_BASEINDEX };
@@ -91,7 +92,7 @@ interface IResolvedFile : extends ISimpleSuperFileEnquiry
     virtual void serializePartial(MemoryBuffer &mb, unsigned channel, bool localInfoOnly) const = 0;
 
     virtual IFileIOArray *getIFileIOArray(bool isOpt, unsigned channel) const = 0;
-    virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, bool allowFieldTranslation) const = 0;
+    virtual IKeyArray *getKeyArray(IDefRecordMeta *activityMeta, TranslatorArray *translators, bool isOpt, unsigned channel, IRecordLayoutTranslator::Mode allowFieldTranslation) const = 0;
     virtual IFilePartMap *getFileMap() const = 0;
     virtual unsigned getNumParts() const = 0;
     virtual IInMemoryIndexManager *getIndexManager(bool isOpt, unsigned channel, IFileIOArray *files, IRecordSize *recs, bool preload, int numKeys) const = 0;

roxie/ccd/ccdmain.cpp

@@ -78,7 +78,7 @@ unsigned defaultTraceLimit = 10;
 unsigned watchActivityId = 0;
 unsigned testSlaveFailure = 0;
 unsigned restarts = 0;
-bool fieldTranslationEnabled = false;
+IRecordLayoutTranslator::Mode fieldTranslationEnabled = IRecordLayoutTranslator::NoTranslation;
 bool mergeSlaveStatistics = true;
 PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
 bool runOnce = false;
@@ -798,7 +798,15 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         coresPerQuery = topology->getPropInt("@coresPerQuery", 0);
 
         diskReadBufferSize = topology->getPropInt("@diskReadBufferSize", 0x10000);
-        fieldTranslationEnabled = topology->getPropBool("@fieldTranslationEnabled", false);
+        fieldTranslationEnabled = IRecordLayoutTranslator::NoTranslation;
+        const char *val = topology->queryProp("@fieldTranslationEnabled");
+        if (val)
+        {
+            if (strieq(val, "payload"))
+                fieldTranslationEnabled = IRecordLayoutTranslator::TranslatePayload;
+            else if (strToBool(val))
+                fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAll;
+        }
 
         pretendAllOpt = topology->getPropBool("@ignoreMissingFiles", false);
         memoryStatsInterval = topology->getPropInt("@memoryStatsInterval", 60);

roxie/ccd/ccdquery.cpp

@@ -397,6 +397,21 @@ void QueryOptions::updateFromWorkUnit(bool &value, IConstWorkUnit &wu, const cha
     value = wu.getDebugValueBool(name, value);
 }
 
+void QueryOptions::updateFromWorkUnit(IRecordLayoutTranslator::Mode &value, IConstWorkUnit &wu, const char *name)
+{
+    SCMStringBuffer val;
+    wu.getDebugValue(name, val);
+    if (val.length())
+    {
+        if (strieq(val.str(), "payload"))
+            value = IRecordLayoutTranslator::TranslatePayload;
+        else if (strToBool(val.str()))
+            value = IRecordLayoutTranslator::TranslateAll;
+        else
+            value = IRecordLayoutTranslator::NoTranslation;
+    }
+}
+
 void QueryOptions::setFromContext(const IPropertyTree *ctx)
 {
     if (ctx)
@@ -1971,11 +1986,11 @@ extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned
     return createSlaveQueryFactory(wu->queryWuid(), dll.getClear(), queryRootRoxiePackage(), channelNo, NULL, true, false);  // MORE - if use a constant for id might cache better?
 }
 
-IRecordLayoutTranslator * createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta)
+IRecordLayoutTranslator * createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta, IRecordLayoutTranslator::Mode mode)
 {
     try
     {
-        return ::createRecordLayoutTranslator(diskMeta, activityMeta);
+        return ::createRecordLayoutTranslator(diskMeta, activityMeta, mode);
     }
     catch (IException *E)
     {

roxie/ccd/ccdquery.hpp

@@ -116,7 +116,7 @@ public:
 
     bool checkingHeap;
     bool disableLocalOptimizations;
-    bool enableFieldTranslation;
+    IRecordLayoutTranslator::Mode enableFieldTranslation;
     bool skipFileFormatCrcCheck;
     bool stripWhitespaceFromStoredDataset;
     bool timeActivities;
@@ -130,6 +130,7 @@ private:
     static void updateFromWorkUnit(int &value, IConstWorkUnit &wu, const char *name);
     static void updateFromWorkUnit(unsigned &value, IConstWorkUnit &wu, const char *name);
     static void updateFromWorkUnit(bool &value, IConstWorkUnit &wu, const char *name);
+    static void updateFromWorkUnit(IRecordLayoutTranslator::Mode &value, IConstWorkUnit &wu, const char *name);
     static void updateFromContextM(memsize_t &val, const IPropertyTree *ctx, const char *name, const char *name2 = NULL); // Needs different name to ensure works in 32-bit where memsize_t and unsigned are same type
     static void updateFromContext(int &val, const IPropertyTree *ctx, const char *name, const char *name2 = NULL);
     static void updateFromContext(unsigned &val, const IPropertyTree *ctx, const char *name, const char *name2 = NULL);
@@ -301,7 +302,7 @@ extern const IQueryDll *createQueryDll(const char *dllName);
 extern const IQueryDll *createExeQueryDll(const char *exeName);
 extern const IQueryDll *createWuQueryDll(IConstWorkUnit *wu);
 
-extern IRecordLayoutTranslator *createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta);
+extern IRecordLayoutTranslator *createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta, IRecordLayoutTranslator::Mode _mode);
 extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo, bool isDynamic, bool forceRetry);
 extern IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned _channelNo, const IPropertyTree *stateInfo, bool isDynamic, bool forceRetry);
 extern IQueryFactory *getQueryFactory(hash64_t hashvalue, unsigned channel);

roxie/ccd/ccdserver.cpp

@@ -23716,7 +23716,7 @@ public:
     bool maySkip;
     bool sorted;
     bool variableFileName;
-    bool enableFieldTranslation;
+    IRecordLayoutTranslator::Mode enableFieldTranslation;
     unsigned maxSeekLookahead;
     Owned<const IResolvedFile> indexfile;
 
@@ -24843,7 +24843,7 @@ public:
                     bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isIndexOpt']/@value");
                     indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true));
                     if (indexfile)
-                        keySet.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, isLocal ? queryFactory.queryChannel() : 0, false));
+                        keySet.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, isLocal ? queryFactory.queryChannel() : 0, IRecordLayoutTranslator::NoTranslation));
                 }
                 if (fileName && !allFilesDynamic && !queryFactory.isDynamic())
                 {
@@ -26446,7 +26446,7 @@ class CRoxieServerKeyedJoinActivityFactory : public CRoxieServerMultiInputFactor
     unsigned joinFlags;
     bool isHalfKeyed;
     bool isLocal;
-    bool enableFieldTranslation;
+    IRecordLayoutTranslator::Mode enableFieldTranslation;
     bool variableFetchFileName;
     bool variableIndexFileName;
     bool isSimple;

roxie/ccd/ccdstate.cpp

@@ -431,7 +431,7 @@ protected:
     virtual aindex_t getBaseCount() const = 0;
     virtual const CRoxiePackageNode *getBaseNode(aindex_t pos) const = 0;
 
-    virtual bool getSysFieldTranslationEnabled() const {return fieldTranslationEnabled;} //roxie configured value
+    virtual IRecordLayoutTranslator::Mode getSysFieldTranslationEnabled() const { return fieldTranslationEnabled; } //roxie configured value
 
     // Use local package file only to resolve subfile into physical file info
     IResolvedFile *resolveLFNusingPackage(const char *fileName) const
@@ -598,7 +598,7 @@ protected:
     void doPreload(unsigned channel, const IResolvedFile *resolved)
     {
         if (resolved->isKey())
-            keyArrays.append(*resolved->getKeyArray(NULL, NULL, false, channel, false));
+            keyArrays.append(*resolved->getKeyArray(NULL, NULL, false, channel, IRecordLayoutTranslator::NoTranslation));
         else
             fileArrays.append(*resolved->getIFileIOArray(false, channel));
     }
@@ -739,7 +739,7 @@ public:
     {
         return CPackageNode::queryEnv(varname);
     }
-    virtual bool getEnableFieldTranslation() const
+    virtual IRecordLayoutTranslator::Mode getEnableFieldTranslation() const override
     {
         return CPackageNode::getEnableFieldTranslation();
     }
@@ -2188,7 +2188,16 @@ private:
         case 'F':
             if (stricmp(queryName, "control:fieldTranslationEnabled")==0)
             {
-                fieldTranslationEnabled = control->getPropBool("@val", true);
+                const char *val = control->queryProp("@val");
+                if (val)
+                {
+                    if (strieq(val, "payload"))
+                        fieldTranslationEnabled = IRecordLayoutTranslator::TranslatePayload;
+                    else if (!val || strToBool(val))
+                        fieldTranslationEnabled = IRecordLayoutTranslator::TranslateAll;
+                    else
+                        fieldTranslationEnabled = IRecordLayoutTranslator::NoTranslation;
+                }
                 topology->setPropInt("@fieldTranslationEnabled", fieldTranslationEnabled);
             }
             else if (stricmp(queryName, "control:flushJHtreeCacheOnOOM")==0)

roxie/roxie/CMakeLists.txt

@@ -43,6 +43,7 @@ include_directories (
          ./../../roxie/roxie 
          ./../../roxie/roxiemem 
          ./../../common/thorhelper
+         ./../../common/deftype
          ./../../common/workunit
     )
 

services/runagent/frunssh.cpp

@@ -76,7 +76,8 @@ int main( int argc, char *argv[] )
                 buf.setLength(buf.length()-1);
                 buf.clip();
             }
-            PROGLOG("%d: ssh(%d): %s",i+1,unsArray.item(i),buf.str());
+            if (buf.length())
+                PROGLOG("%d: ssh(%d): %s",i+1,unsArray.item(i),buf.str());
         }
     }
     catch(IException *e)

system/CMakeLists.txt

@@ -13,11 +13,15 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-HPCC_ADD_SUBDIRECTORY (hrpc)
+
 HPCC_ADD_SUBDIRECTORY (include)
 HPCC_ADD_SUBDIRECTORY (jhtree)
 HPCC_ADD_SUBDIRECTORY (jlib)
-HPCC_ADD_SUBDIRECTORY (mp)
-HPCC_ADD_SUBDIRECTORY (security)
-HPCC_ADD_SUBDIRECTORY (xmllib)
-HPCC_ADD_SUBDIRECTORY (xmllibtest "PLATFORM")
+
+if (NOT JLIB_ONLY)
+   HPCC_ADD_SUBDIRECTORY (hrpc)
+   HPCC_ADD_SUBDIRECTORY (mp)
+   HPCC_ADD_SUBDIRECTORY (security)
+   HPCC_ADD_SUBDIRECTORY (xmllib)
+   HPCC_ADD_SUBDIRECTORY (xmllibtest "PLATFORM")
+endif( )

system/jlib/jargv.cpp

@@ -56,7 +56,7 @@ bool processArgvFilenamesFromFile(IFileArray & filenames, const char * filename)
     {
         if (fgets(buffer, sizeof(buffer), in))
         {
-            unsigned len = strlen(buffer);
+            size_t len = strlen(buffer);
             while (len && !isprint(buffer[len-1]))
                 len--;
             buffer[len] = 0;

system/jlib/jdebug.cpp

@@ -287,6 +287,7 @@ cycle_t nanosec_to_cycle(__int64 ns)
     return (__int64)((double)ns / cycleToNanoScale);
 }
 
+#if !(defined(INLINE_GET_CYCLES_NOW) && defined(HAS_GOOD_CYCLE_COUNTER))
 cycle_t jlib_decl get_cycles_now()
 {
     if (useRDTSC)
@@ -295,6 +296,7 @@ cycle_t jlib_decl get_cycles_now()
     QueryPerformanceCounter(&temp);
     return temp.QuadPart;
 }
+#endif
 
 double getCycleToNanoScale()
 {
@@ -2785,7 +2787,7 @@ void getHardwareInfo(HardwareInfo &hdwInfo, const char *primDiskPath, const char
 
     MEMORYSTATUS memstatus;
     GlobalMemoryStatus(&memstatus);
-    hdwInfo.totalMemory = memstatus.dwTotalPhys / (1024*1024); // in MB
+    hdwInfo.totalMemory = (unsigned)(memstatus.dwTotalPhys / (1024*1024)); // in MB
 
     ULARGE_INTEGER diskAvailStruct;
     ULARGE_INTEGER diskTotalStruct;
@@ -2903,7 +2905,7 @@ public:
             }
         }
         else if (file&&fgets (ln, sizeof(ln), file)) {
-            unsigned i = strlen(ln);
+            size_t i = strlen(ln);
             while (i&&((ln[i-1]==10)||(ln[i-1]==13)))
                 i--;
             ln[i] = 0;
@@ -3221,7 +3223,7 @@ static int MemoryLeakReportHook(int nRptType,char *szMsg,int  *retVal)
             // this works  better in VS 2008 libraries (which fault in fopen)
             int handle = _open(_logFile, O_RDWR | O_CREAT, _S_IREAD | _S_IWRITE);
             _lseek(handle,0,SEEK_END);
-            _write(handle,szMsg,strlen(szMsg));
+            _write(handle,szMsg,(unsigned)strlen(szMsg));
             _close(handle);
 #else
             FILE *handle = fopen(_logFile, "a");

system/jlib/jexcept.cpp

@@ -635,7 +635,7 @@ static BOOL GetLogicalAddress( PVOID addr, PTSTR szModule, DWORD len, DWORD& sec
     szModule[0] = 0;
     section = 0;
     offset = 0;
-    if ((unsigned)addr<0x10000)
+    if ((unsigned)(memsize_t)addr<0x10000)
         return FALSE;
     
     
@@ -644,7 +644,7 @@ static BOOL GetLogicalAddress( PVOID addr, PTSTR szModule, DWORD len, DWORD& sec
     if ( !VirtualQuery( addr, &mbi, sizeof(mbi) ) )
         return FALSE;
     
-    DWORD hMod = (DWORD)mbi.AllocationBase;
+    memsize_t hMod = (memsize_t)mbi.AllocationBase;
     
     if ( !GetModuleFileName( (HMODULE)hMod, szModule, len ) )
         return FALSE;
@@ -655,7 +655,7 @@ static BOOL GetLogicalAddress( PVOID addr, PTSTR szModule, DWORD len, DWORD& sec
     
     PIMAGE_SECTION_HEADER pSection = IMAGE_FIRST_SECTION( pNtHdr );
     
-    DWORD rva = (DWORD)addr - hMod; 
+    memsize_t rva = (memsize_t)addr - hMod;
     
     for (unsigned i = 0; i < pNtHdr->FileHeader.NumberOfSections; i++, pSection++ )
     {

system/jlib/jlog.cpp

@@ -2342,8 +2342,8 @@ bool CSysLogEventLogger::win32Report(unsigned eventtype, unsigned category, unsi
             DWORD type = REG_EXPAND_SZ;
             if ((RegQueryValueEx(hk,"EventMessageFile",NULL, &type, NULL, &sizedata)!=0)||!sizedata) {
                 StringAttr str("%SystemRoot%\\System32\\JELOG.dll"); 
-                RegSetValueEx(hk,"EventMessageFile", 0, REG_EXPAND_SZ, (LPBYTE) str.get(), str.length() + 1);
-                RegSetValueEx(hk,"CategoryMessageFile", 0, REG_EXPAND_SZ, (LPBYTE) str.get(), str.length() + 1);
+                RegSetValueEx(hk,"EventMessageFile", 0, REG_EXPAND_SZ, (LPBYTE) str.get(), (DWORD)str.length() + 1);
+                RegSetValueEx(hk,"CategoryMessageFile", 0, REG_EXPAND_SZ, (LPBYTE) str.get(), (DWORD)str.length() + 1);
                 DWORD dwData = EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | EVENTLOG_INFORMATION_TYPE | EVENTLOG_AUDIT_SUCCESS | EVENTLOG_AUDIT_FAILURE; 
                 RegSetValueEx(hk, "TypesSupported", 0, REG_DWORD,  (LPBYTE) &dwData,  sizeof(DWORD));
                 dwData = 16;

system/jlib/jstring.hpp

@@ -300,7 +300,7 @@ public:
     virtual void set(const char *val) { attr.set(val); };
     virtual void clear() { attr.clear(); };
     virtual void setLen(const char *val, unsigned length) { attr.set(val, length); };
-    virtual unsigned length() const { return attr.length(); };
+    virtual unsigned length() const { return (unsigned)attr.length(); };
 
 private:
     StringAttr & attr;

system/security/LdapSecurity/ldapconnection.cpp

@@ -2687,8 +2687,8 @@ public:
             };
 
 
-            StringBuffer emplID(user.getEmployeeID());
-            char *employeeID_values[] = {(char*)emplID.str(), NULL };
+            const char * emplID = user.getEmployeeID();
+            char *employeeID_values[] = {(emplID && *emplID) ? (char*)emplID : nullptr, nullptr };
             LDAPMod employeeID_attr =
             {
                 LDAP_MOD_REPLACE,
@@ -2711,8 +2711,7 @@ public:
                 attrs[ind++] = &cn_attr;
             }
 
-            if (!emplID.isEmpty())
-                attrs[ind++] = &employeeID_attr;
+            attrs[ind++] = &employeeID_attr;
             
             attrs[ind] = NULL;
             
@@ -5909,7 +5908,7 @@ private:
         {
             attrs[ind++] = &username_attr;
             attrs[ind++] = &dispname_attr;
-            if (user.getEmployeeID())
+            if (employeeID && *employeeID)
                 attrs[ind++] = &employeeID_attr;
         }
         else

testing/regress/ecl/key/setuppersist.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><country>Spain</country><population>40397842</population></Row>
+ <Row><country>Sweden</country><population>9016596</population></Row>
+ <Row><country>Switzerland</country><population>7523934</population></Row>
+ <Row><country>UK</country><population>60609153</population></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><country>Spain</country><population>40397842</population></Row>
+ <Row><country>Sweden</country><population>9016596</population></Row>
+ <Row><country>Switzerland</country><population>7523934</population></Row>
+ <Row><country>United Kingdom</country><population>60609153</population></Row>
+</Dataset>

testing/regress/ecl/persist_refresh.ecl

@@ -37,8 +37,14 @@ ds2 := DATASET([{'Spain', 40397842},
 
 persistRefresh := #IFDEFINED(root.persistRefresh, true);
 
-ds := if(persistRefresh=true, ds2, ds1);
-
-CountriesDS := ds:PERSIST('~REGRESS::PersistRefresh', SINGLE, REFRESH(persistRefresh));
+#if (persistRefresh)
+    persistFileName := '~REGRESS::PersistRefresh';
+    ds := ds2;
+#else
+    persistFileName := '~REGRESS::PersistNoRefresh';
+    ds := ds1;
+#end
+
+CountriesDS := ds:PERSIST(persistFileName, SINGLE, REFRESH(persistRefresh));
 
 OUTPUT(CountriesDS);

testing/regress/ecl/setup/setuppersist.ecl

@@ -0,0 +1,39 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the 'License');
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an 'AS IS' BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+countryRecord := RECORD
+    string country;
+    integer4 population;
+END;
+
+ds1 := DATASET([{'Spain', 40397842},
+                {'Sweden', 9016596},
+                {'Switzerland', 7523934},
+                {'UK',60609153}], countryRecord);
+
+ds2 := DATASET([{'Spain', 40397842},
+                {'Sweden', 9016596},
+                {'Switzerland', 7523934},
+                {'United Kingdom', 60609153}], countryRecord);
+
+pds1 := ds1:PERSIST('~REGRESS::PersistRefresh', SINGLE, REFRESH(true));
+output(pds1);
+
+pds2 := ds2:PERSIST('~REGRESS::PersistNoRefresh', SINGLE, REFRESH(true));
+output(pds2);
+
+

testing/regress/hpcc/common/logger.py

@@ -128,6 +128,9 @@ class Logger(object):
                 if record.levelname == 'DEBUG':
                     msg +=" [asctime:"+record.asctime+", process:"+str(record.process)+", processName:"+record.processName+", thread:"+str(record.thread)+", threadName:"+record.threadName+"]"
                     msg = "{0:3d}".format(taskId) + ". Debug-[debug-"+record.asctime+"]: "+msg
+                if record.levelname == 'ERROR':
+                    msg +=" [asctime:"+record.asctime+", process:"+str(record.process)+", processName:"+record.processName+", thread:"+str(record.thread)+", threadName:"+record.threadName+"]"
+                    msg = "{0:3d}".format(taskId) + ". Error-[error-"+record.asctime+"]: "+msg
                 if record.levelname == 'CRITICAL':
                     msg += " [level: "+record.levelname+" ]"
                     msg = "{0:3d}".format(taskId) +". " + msg

testing/regress/hpcc/regression/regress.py

@@ -535,7 +535,7 @@ class Regression:
                                           password=self.config.password,
                                           retryCount=self.config.maxAttemptCount)
                 except Error as e:
-                    logging.debug("Exception raised:'%s'"  % ( str(e)),  extra={'taskId':cnt})
+                    logging.debug("Exception raised:'%s' (line: %s )"  % ( str(e), str(inspect.stack()[0][2]) ),  extra={'taskId':cnt})
                     res = False
                     wuid = 'Not found'
                     query.setWuid(wuid)
@@ -543,7 +543,7 @@ class Regression:
                     report[0].addResult(query)
                     pass
                 except:
-                    logging.error("Unexpected error:'%s'" %( sys.exc_info()[0]) ,  extra={'taskId':cnt})
+                    logging.error("Unexpected error:'%s' (line: %s )" %( sys.exc_info()[0], str(inspect.stack()[0][2]) ) ,  extra={'taskId':cnt})
 
                 wuid = query.getWuid()
                 logging.debug("CMD result: '%s', wuid:'%s'"  % ( res,  wuid),  extra={'taskId':cnt})

testing/regress/hpcc/util/ecl/file.py

@@ -223,7 +223,7 @@ class ECLFile:
         return path
 
     def getResults(self):
-        return os.path.join(self.dir_r, self.xml_r)
+        return os.path.join(self.dir_r, self.getJobname() + '.xml')
 
     def getArchive(self):
         logging.debug("%3d. getArchive (isVersions:'%s')", self.taskId, self.isVersions )

tools/wutool/wutool.cpp

@@ -487,7 +487,7 @@ class WuTool : public CppUnit::TestFixture
         CPPUNIT_TEST(testQuery);
         CPPUNIT_TEST(testGraph);
         CPPUNIT_TEST(testGraphProgress);
-        CPPUNIT_TEST(testGlobal); 
+        CPPUNIT_TEST(testGlobal);
     CPPUNIT_TEST_SUITE_END();
 protected:
     static StringArray wuids;
@@ -1329,16 +1329,76 @@ protected:
         WUSortField sortByOwner[] = { WUSFuser, WUSFstate, WUSFterm };
         start = msTick();
         wus.setown(factory->getWorkUnitsSorted((WUSortField) (WUSFwuid | WUSFreverse), sortByOwner, "WuTestUser00\0completed", 0, 10000, NULL, NULL));
-        numIterated = 0;
+        unsigned numCompleted = 0;
         ForEach(*wus)
         {
             IConstWorkUnitInfo &wu = wus->query();
             ASSERT(streq(wu.queryUser(), "WuTestUser00"));
             ASSERT(wu.getState()==WUStateCompleted);
-            numIterated++;
+            numCompleted++;
+        }
+        DBGLOG("%d owned,completed workunits listed the hard way in %d ms", numCompleted, msTick()-start);
+        ASSERT(numCompleted > 0);  // Not sure what the exact answer should be! Around 5/6 of 1/50th ?
+
+        start = msTick();
+        wus.setown(factory->getWorkUnitsSorted((WUSortField) (WUSFwuid | WUSFreverse), sortByOwner, "WuTestUser00\0scheduled", 0, 10000, NULL, NULL));
+        unsigned numScheduled = 0;
+        ForEach(*wus)
+        {
+            IConstWorkUnitInfo &wu = wus->query();
+            ASSERT(streq(wu.queryUser(), "WuTestUser00"));
+            ASSERT(wu.getState()==WUStateScheduled);
+            numScheduled++;
+        }
+        DBGLOG("%d owned,scheduled workunits listed the hard way in %d ms", numScheduled, msTick()-start);
+        ASSERT(numScheduled > 0);
+        ASSERT(numScheduled + numCompleted == (testSize+49)/50);
+
+        // Use multiple states
+        DBGLOG("Testing multiple states");
+        start = msTick();
+        wus.setown(factory->getWorkUnitsSorted((WUSortField) (WUSFwuid | WUSFreverse), sortByOwner, "WuTestUser00\0completed|scheduled", 0, 10000, NULL, NULL));
+        unsigned numCompleted2 = 0;
+        unsigned numScheduled2 = 0;
+        ForEach(*wus)
+        {
+            IConstWorkUnitInfo &wu = wus->query();
+            ASSERT(streq(wu.queryUser(), "WuTestUser00"));
+            ASSERT((wu.getState()==WUStateCompleted) || (wu.getState()==WUStateScheduled));
+            if ((wu.getState()==WUStateCompleted))
+                numCompleted2++;
+            else
+                numScheduled2++;
+        }
+        DBGLOG("%d owned,completed and %d owned,scheduled workunits listed via multiple in %d ms", numCompleted2, numScheduled2, msTick()-start);
+        ASSERT(numCompleted == numCompleted2);
+        ASSERT(numScheduled == numScheduled2);
+
+        // Use multiple states only
+        DBGLOG("Testing multiple states only");
+        start = msTick();
+        WUSortField filterByState[] = { WUSFstate, WUSFterm };
+        wus.setown(factory->getWorkUnitsSorted((WUSortField) (WUSFwuid | WUSFreverse), filterByState, "completed|scheduled", 0, 10000, NULL, NULL));
+        unsigned numCompleted3 = 0;
+        unsigned numScheduled3 = 0;
+        ForEach(*wus)
+        {
+            IConstWorkUnitInfo &wu = wus->query();
+            ASSERT((wu.getState()==WUStateCompleted) || (wu.getState()==WUStateScheduled));
+            if (strlen(wu.queryUser()) > 10 && memcmp(wu.queryUser(), "WuTestUser", 10)==0)  // Ignore any wu's not part of the test set
+            {
+                if ((wu.getState()==WUStateCompleted))
+                    numCompleted3++;
+                else
+                    numScheduled3++;
+            }
         }
-        DBGLOG("%d owned workunits listed the hard way in %d ms", numIterated, msTick()-start);
-        ASSERT(numIterated <= (testSize+49)/50);  // Not sure what the exact answer should be!
+        DBGLOG("%d completed and %d scheduled workunits listed via multiple in %d ms", numCompleted3, numScheduled3, msTick()-start);
+        ASSERT(numCompleted3 >= numCompleted2);
+        ASSERT(numScheduled3 >= numScheduled2);
+        ASSERT(numCompleted3 + numScheduled3 == testSize);
+
+
 
         // Get Scheduled Workunits
         start = msTick();