Merge branch 'candidate-6.0.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.gitmodules

@@ -31,3 +31,6 @@
 [submodule "esp/src/crossfilter"]
 	path = esp/src/crossfilter
 	url = https://github.com/hpcc-systems/crossfilter.git
+[submodule "plugins/kafka/librdkafka"]
+	path = plugins/kafka/librdkafka
+	url = https://github.com/hpcc-systems/librdkafka.git

CMakeLists.txt

@@ -47,11 +47,16 @@
 #
 #########################################################
 
-project (hpccsystems-platform)
-cmake_minimum_required (VERSION 2.8.11)
+project(hpccsystems-platform)
+cmake_minimum_required(VERSION 2.8.11)
+
+set(TOP_LEVEL_PROJECT ON)
+if(NOT CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
+    set(TOP_LEVEL_PROJECT OFF)
+endif(NOT CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
 
 include(CTest)
-ENABLE_TESTING()
+enable_testing()
 
 set(HPCC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(CMAKE_MODULE_PATH "${HPCC_SOURCE_DIR}/cmake_modules/")
@@ -61,11 +66,9 @@ include(${HPCC_SOURCE_DIR}/version.cmake)
 ###
 ## Build Level
 ###
-if( NOT BUILD_LEVEL )
-    set ( BUILD_LEVEL "COMMUNITY" )
-endif()
-###
-
+if(NOT BUILD_LEVEL)
+    set(BUILD_LEVEL "COMMUNITY")
+endif(NOT BUILD_LEVEL)
 
 ###
 ## Config Block
@@ -91,121 +94,123 @@ option(ENV_XML_FILE "Set the environment xml file name.")
 option(ENV_CONF_FILE "Set the environment conf file name.")
 option(LICENSE_FILE "Set the license file to use.")
 
-if( NOT LICENSE_FILE )
+if(NOT LICENSE_FILE)
     set(LICENSE_FILE "LICENSE.txt")
 endif()
 
 include(${HPCC_SOURCE_DIR}/cmake_modules/optionDefaults.cmake)
-###
-
 include(${HPCC_SOURCE_DIR}/cmake_modules/commonSetup.cmake)
 
-if ( NOT MAKE_DOCS_ONLY )
-    HPCC_ADD_SUBDIRECTORY (initfiles)
-    HPCC_ADD_SUBDIRECTORY (tools)
-    HPCC_ADD_SUBDIRECTORY (common)
-    HPCC_ADD_SUBDIRECTORY (dali)
-    HPCC_ADD_SUBDIRECTORY (deploy)
-    HPCC_ADD_SUBDIRECTORY (deployment)
-    HPCC_ADD_SUBDIRECTORY (ecl)
-    HPCC_ADD_SUBDIRECTORY (ecllibrary)
-    HPCC_ADD_SUBDIRECTORY (esp)
-    HPCC_ADD_SUBDIRECTORY (plugins)
-    HPCC_ADD_SUBDIRECTORY (roxie)
-    HPCC_ADD_SUBDIRECTORY (rtl)
-    HPCC_ADD_SUBDIRECTORY (services "PLATFORM")
-    HPCC_ADD_SUBDIRECTORY (system)
-    HPCC_ADD_SUBDIRECTORY (thorlcr "PLATFORM")
-    HPCC_ADD_SUBDIRECTORY (testing)
-   if ( NOT WIN32 )
-       HPCC_ADD_SUBDIRECTORY (clienttools "CLIENTTOOLS_ONLY")
-   endif()
-
+if(NOT MAKE_DOCS_ONLY)
+    HPCC_ADD_SUBDIRECTORY(initfiles)
+    HPCC_ADD_SUBDIRECTORY(tools)
+    HPCC_ADD_SUBDIRECTORY(common)
+    HPCC_ADD_SUBDIRECTORY(dali)
+    HPCC_ADD_SUBDIRECTORY(deploy)
+    HPCC_ADD_SUBDIRECTORY(deployment)
+    HPCC_ADD_SUBDIRECTORY(ecl)
+    HPCC_ADD_SUBDIRECTORY(ecllibrary)
+    HPCC_ADD_SUBDIRECTORY(esp)
+    HPCC_ADD_SUBDIRECTORY(plugins)
+    HPCC_ADD_SUBDIRECTORY(roxie)
+    HPCC_ADD_SUBDIRECTORY(rtl)
+    HPCC_ADD_SUBDIRECTORY(services "PLATFORM")
+    HPCC_ADD_SUBDIRECTORY(system)
+    HPCC_ADD_SUBDIRECTORY(thorlcr "PLATFORM")
+    HPCC_ADD_SUBDIRECTORY(testing)
+    if(NOT WIN32)
+        HPCC_ADD_SUBDIRECTORY(clienttools "CLIENTTOOLS_ONLY")
+    endif()
 endif()
-HPCC_ADD_SUBDIRECTORY (docs "PLATFORM")
-if (APPLE OR WIN32)
-  HPCC_ADD_SUBDIRECTORY (lib2)
-endif (APPLE OR WIN32)
+
+HPCC_ADD_SUBDIRECTORY(docs "PLATFORM")
+if(APPLE OR WIN32)
+    HPCC_ADD_SUBDIRECTORY(lib2)
+endif(APPLE OR WIN32)
 
 ###
 ## CPack install and packaging setup.
 ###
-INCLUDE(InstallRequiredSystemLibraries)
-if ( PLATFORM )
-    set(CPACK_PACKAGE_NAME "hpccsystems-platform")
-    set(PACKAGE_FILE_NAME_PREFIX "hpccsystems-platform-${projname}")
-else()
-    set(CPACK_PACKAGE_NAME "hpccsystems-clienttools-${majorver}.${minorver}")
-    set(PACKAGE_FILE_NAME_PREFIX  "hpccsystems-clienttools-${projname}")
-endif()
+include(InstallRequiredSystemLibraries)
 
-set (VER_SEPARATOR "-")
-if ("${stagever}" MATCHES "^rc[0-9]+$")
-    set (VER_SEPARATOR "~")
+set(VER_SEPARATOR "-")
+if("${stagever}" MATCHES "^rc[0-9]+$")
+    set(VER_SEPARATOR "~")
 endif()
 
-SET(CPACK_PACKAGE_VERSION_MAJOR ${majorver})
-SET(CPACK_PACKAGE_VERSION_MINOR ${minorver})
-SET(CPACK_PACKAGE_VERSION_PATCH ${point}${VER_SEPARATOR}${stagever})
-set ( CPACK_PACKAGE_CONTACT "HPCCSystems <ossdevelopment@lexisnexis.com>" )
-set( CPACK_SOURCE_GENERATOR TGZ )
-
-set ( CPACK_RPM_PACKAGE_VERSION "${version}" )
-SET(CPACK_RPM_PACKAGE_RELEASE "${stagever}")
-SET(CPACK_RPM_PACKAGE_VENDOR "HPCC Systems®")
-SET(CPACK_PACKAGE_VENDOR "HPCC Systems®")
-
-if ( ${ARCH64BIT} EQUAL 1 )
-    set ( CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
-else( ${ARCH64BIT} EQUAL 1 )
-    set ( CPACK_RPM_PACKAGE_ARCHITECTURE "i386")
-endif ( ${ARCH64BIT} EQUAL 1 )
-set(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CPACK_RPM_PACKAGE_ARCHITECTURE}")
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
-    set(CPACK_STRIP_FILES TRUE)
-endif()
+if(TOP_LEVEL_PROJECT)
+    if(PLATFORM)
+        set(CPACK_PACKAGE_NAME "hpccsystems-platform")
+        set(PACKAGE_FILE_NAME_PREFIX "hpccsystems-platform-${projname}")
+    else()
+        set(CPACK_PACKAGE_NAME "hpccsystems-clienttools-${majorver}.${minorver}")
+        set(PACKAGE_FILE_NAME_PREFIX  "hpccsystems-clienttools-${projname}")
+    endif()
 
+    set(CPACK_PACKAGE_VERSION_MAJOR ${majorver})
+    set(CPACK_PACKAGE_VERSION_MINOR ${minorver})
+    set(CPACK_PACKAGE_VERSION_PATCH ${point}${VER_SEPARATOR}${stagever})
+    set(CPACK_PACKAGE_CONTACT "HPCCSystems <ossdevelopment@lexisnexis.com>")
+    set(CPACK_SOURCE_GENERATOR TGZ)
+
+    set(CPACK_RPM_PACKAGE_VERSION "${version}")
+    set(CPACK_RPM_PACKAGE_RELEASE "${stagever}")
+    set(CPACK_RPM_PACKAGE_VENDOR "HPCC Systems®")
+    set(CPACK_PACKAGE_VENDOR "HPCC Systems®")
+
+    if(${ARCH64BIT} EQUAL 1)
+        set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
+    else(${ARCH64BIT} EQUAL 1)
+        set(CPACK_RPM_PACKAGE_ARCHITECTURE "i386")
+    endif(${ARCH64BIT} EQUAL 1)
+
+    set(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CPACK_RPM_PACKAGE_ARCHITECTURE}")
+    if("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
+        set(CPACK_STRIP_FILES TRUE)
+    endif()
 
-if ( APPLE )
-elseif ( UNIX )
-    EXECUTE_PROCESS (
-                COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/distrocheck.sh
-                    OUTPUT_VARIABLE packageManagement
-                        ERROR_VARIABLE  packageManagement
-                )
-    EXECUTE_PROCESS (
-                COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/getpackagerevisionarch.sh
-                    OUTPUT_VARIABLE packageRevisionArch
-                        ERROR_VARIABLE  packageRevisionArch
-                )
-    EXECUTE_PROCESS (
-                COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/getpackagerevisionarch.sh --noarch
-                    OUTPUT_VARIABLE packageRevision
-                        ERROR_VARIABLE  packageRevision
-                )
-
-    message ( "-- Auto Detecting Packaging type")
-    message ( "-- distro uses ${packageManagement}, revision is ${packageRevisionArch}" )
-    if ( "${packageManagement}" STREQUAL "RPM" AND WITH_PLUGINS )
-        set ( CPACK_RPM_SPEC_MORE_DEFINE
+    if(UNIX AND NOT APPLE)
+        execute_process(
+            COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/distrocheck.sh
+            OUTPUT_VARIABLE packageManagement
+            ERROR_VARIABLE  packageManagement
+            )
+        execute_process(
+            COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/getpackagerevisionarch.sh
+            OUTPUT_VARIABLE packageRevisionArch
+            ERROR_VARIABLE  packageRevisionArch
+            )
+        execute_process(
+            COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/getpackagerevisionarch.sh --noarch
+            OUTPUT_VARIABLE packageRevision
+            ERROR_VARIABLE  packageRevision
+            )
+
+        message("-- Auto Detecting Packaging type")
+        message("-- distro uses ${packageManagement}, revision is ${packageRevisionArch}")
+
+        if("${packageManagement}" STREQUAL "RPM" AND WITH_PLUGINS)
+            set(CPACK_RPM_SPEC_MORE_DEFINE
 "%define _use_internal_dependency_generator 0
 %define __getdeps() while read file; do /usr/lib/rpm/rpmdeps -%{1} ${file}; done | /bin/sort -u
 %define __find_provides /bin/sh -c '%{__getdeps P}'
-%define __find_requires /bin/sh -c '%{__grep} -v libRembed.so | %{__getdeps R}'" )
-        set ( PACKAGE_FILE_NAME_PREFIX "${PACKAGE_FILE_NAME_PREFIX}-with-plugins")
-    endif()
-    if ( "${packageManagement}" STREQUAL "DEB" )
-        set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}${packageRevisionArch}")
-    elseif ( "${packageManagement}" STREQUAL "RPM" )
-        set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}.${packageRevisionArch}")
-    else()
-        set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}_${stagever}${CPACK_SYSTEM_NAME}")
+%define __find_requires /bin/sh -c '%{__grep} -v libRembed.so | %{__getdeps R}'")
+            set(PACKAGE_FILE_NAME_PREFIX "${PACKAGE_FILE_NAME_PREFIX}-with-plugins")
+        endif()
+
+        if("${packageManagement}" STREQUAL "DEB")
+            set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}${packageRevisionArch}")
+        elseif("${packageManagement}" STREQUAL "RPM")
+            set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}.${packageRevisionArch}")
+        else()
+            set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}_${stagever}${CPACK_SYSTEM_NAME}")
+        endif()
     endif ()
-endif ()
-MESSAGE ( "-- Current release version is ${CPACK_PACKAGE_FILE_NAME}" )
-set( CPACK_SOURCE_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}" )
-set( CPACK_SOURCE_GENERATOR TGZ )
-set( CPACK_SOURCE_IGNORE_FILES
+
+    message("-- Current release version is ${CPACK_PACKAGE_FILE_NAME}")
+    set(CPACK_SOURCE_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${stagever}")
+    set(CPACK_SOURCE_GENERATOR TGZ)
+    set(CPACK_SOURCE_IGNORE_FILES
         "~$"
         "\\\\.cvsignore$"
         "^${PROJECT_SOURCE_DIR}.*/CVS/"
@@ -229,137 +234,141 @@ set( CPACK_SOURCE_IGNORE_FILES
         "^${PROJECT_SOURCE_DIR}/ecl/regress/"
     "^${PROJECT_SOURCE_DIR}/testing/"
         )
+endif(TOP_LEVEL_PROJECT)
 
 ###
 ## Run file configuration to set build tag along with install lines for generated
 ## config files.
 ###
-set( BUILD_TAG "${projname}_${version}-${stagever}")
-if (USE_GIT_DESCRIBE OR CHECK_GIT_TAG)
-    FETCH_GIT_TAG (${CMAKE_SOURCE_DIR} ${projname}_${version} GIT_BUILD_TAG)
-    message ("-- Git tag is '${GIT_BUILD_TAG}'")
-    if (NOT "${GIT_BUILD_TAG}" STREQUAL "${BUILD_TAG}")
-        if (CHECK_GIT_TAG)
-            message(FATAL_ERROR "Git tag '${GIT_BUILD_TAG}' does not match source version '${BUILD_TAG}'" )
+set(BUILD_TAG "${projname}_${version}-${stagever}")
+if(USE_GIT_DESCRIBE OR CHECK_GIT_TAG)
+    FETCH_GIT_TAG(${CMAKE_SOURCE_DIR} ${projname}_${version} GIT_BUILD_TAG)
+    message("-- Git tag is '${GIT_BUILD_TAG}'")
+    if(NOT "${GIT_BUILD_TAG}" STREQUAL "${BUILD_TAG}")
+        if(CHECK_GIT_TAG)
+            message(FATAL_ERROR "Git tag '${GIT_BUILD_TAG}' does not match source version '${BUILD_TAG}'")
         else()
             if(NOT "${GIT_BUILD_TAG}" STREQUAL "") # probably means being built from a tarball...
-                set( BUILD_TAG "${BUILD_TAG}[${GIT_BUILD_TAG}]")
+                set(BUILD_TAG "${BUILD_TAG}[${GIT_BUILD_TAG}]")
             endif()
         endif()
     endif()
 endif()
-message ("-- Build tag is '${BUILD_TAG}'")
-if (NOT "${BASE_BUILD_TAG}" STREQUAL "")
+message("-- Build tag is '${BUILD_TAG}'")
+if(NOT "${BASE_BUILD_TAG}" STREQUAL "")
     set(BASE_BUILD_TAG "${BUILD_TAG}")
 endif()
-message ("-- Base build tag is '${BASE_BUILD_TAG}'")
-configure_file(${HPCC_SOURCE_DIR}/build-config.h.cmake "build-config.h" )
+message("-- Base build tag is '${BASE_BUILD_TAG}'")
+configure_file(${HPCC_SOURCE_DIR}/build-config.h.cmake "build-config.h")
 
 #set( CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON )
 #set( CPACK_DEB_PACKAGE_COMPONENT ON )
 
-if ( UNIX )
-    if ( "${packageManagement}" STREQUAL "DEB" )
-        set ( CPACK_GENERATOR "${packageManagement}" )
-        message("-- Will build DEB package")
-        ###
-        ## CPack instruction required for Debian
-        ###
-        message ("-- Packing BASH installation files")
-        if ( CLIENTTOOLS_ONLY )
-            set ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
+if(TOP_LEVEL_PROJECT)
+    if(UNIX)
+        if("${packageManagement}" STREQUAL "DEB")
+            set(CPACK_GENERATOR "${packageManagement}")
+            message("-- Will build DEB package")
+            ###
+            ## CPack instruction required for Debian
+            ###
+            message("-- Packing BASH installation files")
+            if(CLIENTTOOLS_ONLY)
+                set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
 "${CMAKE_CURRENT_BINARY_DIR}/clienttools/install/postinst;${CMAKE_CURRENT_BINARY_DIR}/clienttools/install/prerm; ${CMAKE_CURRENT_BINARY_DIR}/clienttools/install/postrm")
-        else ( CLIENTTOOLS_ONLY )
-            set ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postinst;${CMAKE_CURRENT_BINARY_DIR}/initfiles/sbin/prerm;${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postrm" )
-        endif ( CLIENTTOOLS_ONLY )
-
-        # Standard sections values:
-        # https://www.debian.org/doc/debian-policy/ch-archive.html/#s-subsections
-        SET(CPACK_DEBIAN_PACKAGE_SECTION "devel")
-
-    elseif ( "${packageManagement}" STREQUAL "RPM" )
-        set ( CPACK_GENERATOR "${packageManagement}" )
-        ###
-        ## CPack instruction required for RPM
-        ###
-        message("-- Will build RPM package")
-        message ("-- Packing BASH installation files")
-        if ( CLIENTTOOLS_ONLY )
-            set ( CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/clienttools/install/postinst" )
-            set ( CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/clienttools/install/prerm" )
-            set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/clienttools/install/postrm" )
-            set ( CPACK_RPM_PACKAGE_GROUP "development/libraries")
-            set ( CPACK_RPM_PACKAGE_SUMMARY "HPCC Systems® Client Tools." )
-        else ( CLIENTTOOLS_ONLY )
-            set ( CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postinst" )
-            set ( CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/sbin/prerm" )
-            set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postrm" )
-            # Standard group names: http://fedoraroject.org/wiki/RPMGroups
-            set ( CPACK_RPM_PACKAGE_GROUP "development/system")
-            set ( CPACK_RPM_PACKAGE_SUMMARY "${PACKAGE_FILE_NAME_PREFIX}")
-        endif ( CLIENTTOOLS_ONLY )
+            else(CLIENTTOOLS_ONLY)
+                set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postinst;${CMAKE_CURRENT_BINARY_DIR}/initfiles/sbin/prerm;${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postrm")
+            endif(CLIENTTOOLS_ONLY)
+
+            # Standard sections values:
+            # https://www.debian.org/doc/debian-policy/ch-archive.html/#s-subsections
+            set(CPACK_DEBIAN_PACKAGE_SECTION "devel")
+
+        elseif("${packageManagement}" STREQUAL "RPM")
+            set(CPACK_GENERATOR "${packageManagement}")
+            ###
+            ## CPack instruction required for RPM
+            ###
+            message("-- Will build RPM package")
+            message("-- Packing BASH installation files")
+            if(CLIENTTOOLS_ONLY)
+                set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/clienttools/install/postinst")
+                set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/clienttools/install/prerm")
+                set(CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/clienttools/install/postrm")
+                set(CPACK_RPM_PACKAGE_GROUP "development/libraries")
+                set(CPACK_RPM_PACKAGE_SUMMARY "HPCC Systems® Client Tools.")
+            else(CLIENTTOOLS_ONLY)
+                set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postinst")
+                set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/sbin/prerm")
+                set(CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postrm")
+                # Standard group names: http://fedoraroject.org/wiki/RPMGroups
+                set(CPACK_RPM_PACKAGE_GROUP "development/system")
+                set(CPACK_RPM_PACKAGE_SUMMARY "${PACKAGE_FILE_NAME_PREFIX}")
+            endif(CLIENTTOOLS_ONLY)
+        else()
+            message("WARNING: Unsupported package ${packageManagement}.")
+        endif ()
+    endif(UNIX)
+
+    if(EXISTS ${HPCC_SOURCE_DIR}/cmake_modules/dependencies/${packageRevision}.cmake)
+        include(${HPCC_SOURCE_DIR}/cmake_modules/dependencies/${packageRevision}.cmake)
     else()
-        message("WARNING: Unsupported package ${packageManagement}.")
-    endif ()
+        message("-- WARNING: DEPENDENCY FILE FOR ${packageRevision} NOT FOUND, Using deps template.")
+        include(${HPCC_SOURCE_DIR}/cmake_modules/dependencies/template.cmake)
+    endif()
 
-endif ( UNIX )
-if ( EXISTS ${HPCC_SOURCE_DIR}/cmake_modules/dependencies/${packageRevision}.cmake )
-    include( ${HPCC_SOURCE_DIR}/cmake_modules/dependencies/${packageRevision}.cmake )
-else()
-    message("-- WARNING: DEPENDENCY FILE FOR ${packageRevision} NOT FOUND, Using deps template.")
-    include( ${HPCC_SOURCE_DIR}/cmake_modules/dependencies/template.cmake )
-endif()
+    if(UNIX)
+        set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}")
+    endif(UNIX)
 
-if ( UNIX )
-    set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" )
-endif ( UNIX )
-if ( PLATFORM )
-    set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "${PACKAGE_FILE_NAME_PREFIX}")
-else ( PLATFORM )
-    if ( APPLE OR WIN32 )
-        set ( CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${version}-${stagever}${CPACK_SYSTEM_NAME}" )
-    endif()
-    set ( CPACK_MONOLITHIC_INSTALL TRUE )
-    file(WRITE "${PROJECT_BINARY_DIR}/welcome.txt"
-        "HPCC Systems® - Client Tools\r"
-        "===========================\r\r"
-        "This installer will install the HPCC Systems® Client Tools.")
-    set ( CPACK_RESOURCE_FILE_README "${PROJECT_BINARY_DIR}/welcome.txt" )
-    set ( CPACK_RESOURCE_FILE_LICENSE "${HPCC_SOURCE_DIR}/${LICENSE_FILE}" )
-    set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HPCC Systems® Client Tools." )
-    if (WIN32)
-      if ( "${SIGN_DIRECTORY}" STREQUAL "" )
-        set ( SIGN_DIRECTORY "${HPCC_SOURCE_DIR}/../sign" )
-      endif ()
-
-      set ( CPACK_NSIS_DISPLAY_NAME "Client Tools" )
-      set ( CPACK_PACKAGE_INSTALL_DIRECTORY "${DIR_NAME}\\\\${version}\\\\clienttools")
-      set ( CPACK_PACKAGE_INSTALL_REGISTRY_KEY "clienttools_${version}")
-      set ( CPACK_NSIS_ENABLE_UNINSTALL_BEFORE_INSTALL "ON" )
-      set ( CPACK_NSIS_URL_INFO_ABOUT "http:\\\\\\\\hpccsystems.com" )
-      set ( CPACK_NSIS_CONTACT ${CPACK_PACKAGE_CONTACT} )
-      set ( CPACK_NSIS_DEFINES "
-        !define MUI_STARTMENUPAGE_DEFAULTFOLDER \\\"${CPACK_PACKAGE_VENDOR}\\\\${version}\\\\${CPACK_NSIS_DISPLAY_NAME}\\\"
-        !define MUI_FINISHPAGE_NOAUTOCLOSE
-      ")
-
-      file(STRINGS "${SIGN_DIRECTORY}/passphrase.txt" PFX_PASSWORD LIMIT_COUNT 1)
-
-      add_custom_target(SIGN
-          COMMAND signtool sign /f "${SIGN_DIRECTORY}/hpcc_code_signing.pfx"
+    if(PLATFORM)
+        set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "${PACKAGE_FILE_NAME_PREFIX}")
+    else(PLATFORM)
+        if(APPLE OR WIN32)
+            set(CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${version}-${stagever}${CPACK_SYSTEM_NAME}")
+        endif()
+        set(CPACK_MONOLITHIC_INSTALL TRUE)
+        file(WRITE "${PROJECT_BINARY_DIR}/welcome.txt"
+            "HPCC Systems® - Client Tools\r"
+            "===========================\r\r"
+            "This installer will install the HPCC Systems® Client Tools.")
+        set(CPACK_RESOURCE_FILE_README "${PROJECT_BINARY_DIR}/welcome.txt")
+        set(CPACK_RESOURCE_FILE_LICENSE "${HPCC_SOURCE_DIR}/${LICENSE_FILE}")
+        set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "HPCC Systems® Client Tools.")
+        if(WIN32)
+            if("${SIGN_DIRECTORY}" STREQUAL "")
+                set(SIGN_DIRECTORY "${HPCC_SOURCE_DIR}/../sign")
+            endif()
+
+            set(CPACK_NSIS_DISPLAY_NAME "Client Tools")
+            set(CPACK_PACKAGE_INSTALL_DIRECTORY "${DIR_NAME}\\\\${version}\\\\clienttools")
+            set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY "clienttools_${version}")
+            set(CPACK_NSIS_ENABLE_UNINSTALL_BEFORE_INSTALL "ON")
+            set(CPACK_NSIS_URL_INFO_ABOUT "http:\\\\\\\\hpccsystems.com")
+            set(CPACK_NSIS_CONTACT ${CPACK_PACKAGE_CONTACT})
+            set(CPACK_NSIS_DEFINES "
+                !define MUI_STARTMENUPAGE_DEFAULTFOLDER \\\"${CPACK_PACKAGE_VENDOR}\\\\${version}\\\\${CPACK_NSIS_DISPLAY_NAME}\\\"
+                !define MUI_FINISHPAGE_NOAUTOCLOSE
+            ")
+
+            file(STRINGS "${SIGN_DIRECTORY}/passphrase.txt" PFX_PASSWORD LIMIT_COUNT 1)
+
+            add_custom_target(SIGN
+                COMMAND signtool sign /f "${SIGN_DIRECTORY}/hpcc_code_signing.pfx"
 /p "${PFX_PASSWORD}" /t "http://timestamp.verisign.com/scripts/timstamp.dll"
 "${CMAKE_BINARY_DIR}/${PACKAGE_FILE_NAME_PREFIX}*.exe"
-          COMMENT "Digital Signature"
-      )
-      add_dependencies(SIGN PACKAGE)
-      set_property(TARGET SIGN PROPERTY FOLDER "CMakePredefinedTargets")
-    endif()
-endif( PLATFORM )
+                COMMENT "Digital Signature"
+            )
+            add_dependencies(SIGN PACKAGE)
+            set_property(TARGET SIGN PROPERTY FOLDER "CMakePredefinedTargets")
+        endif()
+    endif(PLATFORM)
+endif(TOP_LEVEL_PROJECT)
 
 ###
 ## Below are the non-compile based install scripts required for
 ## the hpcc platform.
 ###
 
-Install ( FILES ${HPCC_SOURCE_DIR}/${LICENSE_FILE} DESTINATION "." COMPONENT Runtime )
-include (CPack)
+install(FILES ${HPCC_SOURCE_DIR}/${LICENSE_FILE} DESTINATION "." COMPONENT Runtime)
+include(CPack)

cmake_modules/commonSetup.cmake

@@ -93,6 +93,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(USE_RINSIDE "Enable R support" ON)
   option(USE_MEMCACHED "Enable Memcached support" ON)
   option(USE_REDIS "Enable Redis support" ON)
+  option(USE_KAFKA "Enable Kafka support" ON)
 
   if (APPLE OR WIN32)
       option(USE_TBB "Enable Threading Building Block support" OFF)
@@ -112,6 +113,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       set( USE_CASSANDRA OFF )
       set( USE_MEMCACHED OFF )
       set( USE_REDIS OFF )
+      set( USE_KAFKA OFF )
   endif()
 
   if ( USE_XALAN AND USE_LIBXSLT )
@@ -513,8 +515,8 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       ENDIF()
 
       IF (CMAKE_COMPILER_IS_GNUCXX)
-        IF ("${CMAKE_CXX_COMPILER_VERSION}" VERSION_LESS "4.1.1")
-          MESSAGE(FATAL_ERROR "You need Gnu c++ version 4.1.1 or later to build this project (version ${CMAKE_CXX_COMPILER_VERSION} detected)")
+        IF ("${CMAKE_CXX_COMPILER_VERSION}" VERSION_LESS "4.7.3")
+          MESSAGE(FATAL_ERROR "You need Gnu c++ version 4.7.3 or later to build this project (version ${CMAKE_CXX_COMPILER_VERSION} detected)")
         ENDIF()
       ENDIF()
     ENDIF()

common/thorhelper/roxiehelper.cpp

@@ -1307,6 +1307,10 @@ public:
             return 20;
         return 10;
     }
+    virtual unsigned getActivityId() const
+    {
+        return activityId;
+    }
     virtual bool freeBufferedRows(bool critical)
     {
         roxiemem::RoxieOutputRowArrayLock block(rowsToSort);

common/thorhelper/roxierow.cpp

@@ -177,7 +177,7 @@ public:
     {
         return meta.queryOriginal();
     }
-    virtual unsigned queryActivityId()
+    virtual unsigned queryActivityId() const
     {
         return activityId;
     }

common/thorhelper/thorcommon.cpp

@@ -1105,7 +1105,7 @@ IRowInterfaces *createRowInterfaces(IOutputMetaData *meta, unsigned actid, ICode
         {
             return meta;
         }
-        unsigned queryActivityId()
+        unsigned queryActivityId() const
         {
             return actid;
         }

common/thorhelper/thorcommon.hpp

@@ -71,7 +71,7 @@ interface IRowInterfaces: extends IInterface
     virtual IOutputRowSerializer * queryRowSerializer()=0; 
     virtual IOutputRowDeserializer * queryRowDeserializer()=0; 
     virtual IOutputMetaData *queryRowMetaData()=0;
-    virtual unsigned queryActivityId()=0;
+    virtual unsigned queryActivityId() const=0;
     virtual ICodeContext *queryCodeContext()=0;
 };
 

common/thorhelper/thorrparse.cpp

@@ -3391,43 +3391,46 @@ bool RegexParser::performMatch(IMatchedAction & action, const void * row, unsign
         const byte * end = endData - algo->minPatternLength;
 
         RegexState state(cache, algo->kind, helper, this, algo->inputFormat, len, start);
-        state.row = row;
-        state.processor = &action;
-        state.best = NULL;
-        for (const byte * curScan = start; curScan <= end;)
+        if (len >= algo->minPatternLength)
         {
-            state.cur = curScan;
-            state.top.start = curScan;
-            state.nextScanPosition = NULL;
-            state.score = 0;
-            if (!algo->singleChoicePerLine)
-                state.best = NULL;
-            if ((size32_t)(endData - curScan) > maxSize)
+            state.row = row;
+            state.processor = &action;
+            state.best = NULL;
+            for (const byte * curScan = start; curScan <= end;)
             {
-                state.end = curScan + (maxSize + charWidth);
-                state.lengthIsLimited = true;
-            }
-            else
-            {
-                state.end = endData;
-                state.lengthIsLimited = false;
-            }
-            algo->match(state);
-            if (state.numMatched >= algo->keepLimit)
-                break;
-            if (state.numMatched > algo->atMostLimit)
-            {
-                results.reset();
-                return false;
+                state.cur = curScan;
+                state.top.start = curScan;
+                state.nextScanPosition = NULL;
+                state.score = 0;
+                if (!algo->singleChoicePerLine)
+                    state.best = NULL;
+                if ((size32_t)(endData - curScan) > maxSize)
+                {
+                    state.end = curScan + (maxSize + charWidth);
+                    state.lengthIsLimited = true;
+                }
+                else
+                {
+                    state.end = endData;
+                    state.lengthIsLimited = false;
+                }
+                algo->match(state);
+                if (state.numMatched >= algo->keepLimit)
+                    break;
+                if (state.numMatched > algo->atMostLimit)
+                {
+                    results.reset();
+                    return false;
+                }
+                if (algo->scanAction == INlpParseAlgorithm::NlpScanWhole)
+                    break;
+                if (state.numMatched && (algo->scanAction == INlpParseAlgorithm::NlpScanNone))
+                    break;
+                if (state.nextScanPosition && (algo->scanAction == INlpParseAlgorithm::NlpScanNext) && (curScan != state.nextScanPosition))
+                    curScan = state.nextScanPosition;
+                else
+                    curScan += charWidth;
             }
-            if (algo->scanAction == INlpParseAlgorithm::NlpScanWhole)
-                break;
-            if (state.numMatched && (algo->scanAction == INlpParseAlgorithm::NlpScanNone))
-                break;
-            if (state.nextScanPosition && (algo->scanAction == INlpParseAlgorithm::NlpScanNext) && (curScan != state.nextScanPosition))
-                curScan = state.nextScanPosition;
-            else
-                curScan += charWidth;
         }
 
         if (state.numMatched == 0)

common/workunit/workunit.cpp

@@ -2876,9 +2876,12 @@ public:
         StringAttr namefilterlo;
         StringAttr namefilterhi;
         StringArray unknownAttributes;
-        if (filters) {
-            const char *fv = (const char *)filterbuf;
-            for (unsigned i=0;filters[i]!=WUSFterm;i++) {
+        if (filters)
+        {
+            const char *fv = (const char *) filterbuf;
+            for (unsigned i=0;filters[i]!=WUSFterm;i++)
+            {
+                assertex(fv);
                 int fmt = filters[i];
                 int subfmt = (fmt&0xff);
                 if (subfmt==WUSFwuid)
@@ -2889,17 +2892,21 @@ public:
                     namefilter.set(fv);
                 else if (subfmt==WUSFappvalue)
                 {
-                    query.append("[Application/").append(fv).append("=?~\"");
+                    const char *app = fv;
                     fv = fv + strlen(fv)+1;
-                    query.append(fv).append("\"]");
+                    query.append("[Application/").append(app);
+                    if (*fv)
+                        query.append("=?~\"").append(fv).append('\"');
+                    query.append("]");
                 }
-                else if (!fv || !*fv)
+                else if (!*fv)
                 {
                     unknownAttributes.append(getEnumText(subfmt,workunitSortFields));
                     if (subfmt==WUSFtotalthortime)
                         sortorder = (WUSortField) (sortorder | WUSFnumeric);
                 }
-                else {
+                else
+                {
                     query.append('[').append(getEnumText(subfmt,workunitSortFields)).append('=');
                     if (fmt&WUSFnocase)
                         query.append('?');

dali/base/dasds.cpp

@@ -3284,15 +3284,6 @@ class CLock : public CInterface, implements IInterface
 
     LockStatus doLock(unsigned mode, unsigned timeout, ConnectionId id, SessionId sessionId, IUnlockCallback &callback, bool change=false)
     {
-        class CLockCallbackUnblock
-        {
-        public:
-            CLockCallbackUnblock(IUnlockCallback &_callback) : callback(_callback) { callback.unblock(); }
-            ~CLockCallbackUnblock() { callback.block(); }
-        private:
-            IUnlockCallback &callback;
-        };
-
         if (INFINITE == timeout)
         {
             loop
@@ -3308,8 +3299,9 @@ class CLock : public CInterface, implements IInterface
                     waiting++;
                     {
                         CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
-                        CLockCallbackUnblock cb(callback);
+                        callback.unblock();
                         timedout = !sem.wait(LOCKSESSCHECK);
+                        callback.block();
                     }
                     if (timedout)
                     {
@@ -3322,8 +3314,9 @@ class CLock : public CInterface, implements IInterface
                         }
                         {
                             CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
-                            CLockCallbackUnblock cb(callback);
+                            callback.unblock();
                             validateConnectionSessions();
+                            callback.block();
                         }
                     }
                 }
@@ -3345,10 +3338,11 @@ class CLock : public CInterface, implements IInterface
                     waiting++;
                     {
                         CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
-                        CLockCallbackUnblock cb(callback);
+                        callback.unblock();
                         unsigned remaining;
                         if (tm.timedout(&remaining) || !sem.wait(remaining>LOCKSESSCHECK?LOCKSESSCHECK:remaining))
                             timedout = true;
+                        callback.block();
                     }
                     if (timedout) {
                         if (!sem.wait(0))
@@ -3357,8 +3351,9 @@ class CLock : public CInterface, implements IInterface
                         bool disconnects;
                         {
                             CHECKEDCRITICALUNBLOCK(crit, fakeCritTimeout);
-                            CLockCallbackUnblock cb(callback);
+                            callback.unblock();
                             disconnects = validateConnectionSessions();
+                            callback.block();
                         }
                         if (tm.timedout())
                         {

docs/ECLWatch/TheECLWatchMan.xml

@@ -182,9 +182,9 @@
       <sect2 id="ECLWatch_GlobalSearch">
         <title>Global Search</title>
 
-        <para>On the navigation bar at the top of the ECL Watch page, about
-        half way across the page is the search box. <figure>
-            <title>Search box</title>
+        <para>The global search box can be found on the navigation bar at the
+        top of the ECL Watch page.<figure>
+            <title>Global Search box</title>
 
             <mediaobject>
               <imageobject>
@@ -193,8 +193,74 @@
             </mediaobject>
           </figure></para>
 
-        <para>You can search for workunits, users, files, or even ECL, using
-        the search box. The search box supports wild cards.</para>
+        <para>You can search ECL Workunits, DFU Workuntis, Logical Files, and
+        Queries using the global search box. The global search box also
+        supports wild cards. To limit or filter your search results you can
+        use keywords as displayed in the empty search box.</para>
+
+        <para><variablelist>
+            <varlistentry>
+              <term>file:</term>
+
+              <listitem>
+                <para>Preface the search string with
+                <emphasis>file:</emphasis> to search Logical Files.</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>wuid:</term>
+
+              <listitem>
+                <para>Preface the search string with
+                <emphasis>wuid:</emphasis> to search only Workunit ids.</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>ecl:</term>
+
+              <listitem>
+                <para>Preface the search string with <emphasis>ecl:</emphasis>
+                to search only the ECL workunits.</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>dfu:</term>
+
+              <listitem>
+                <para>Preface the search string with <emphasis>dfu:</emphasis>
+                to search only DFU workunits.</para>
+              </listitem>
+            </varlistentry>
+
+            <varlistentry>
+              <term>query:</term>
+
+              <listitem>
+                <para>Preface the search string with
+                <emphasis>query:</emphasis> to search only published
+                queries.</para>
+              </listitem>
+            </varlistentry>
+          </variablelist>Examples of using the global search:</para>
+
+        <para>Enter <emphasis>W201510*</emphasis> into the search box, and it
+        will return all of the workunits from October 2015.</para>
+
+        <para>Enter <emphasis>file:keys</emphasis> into the search box, and it
+        will return all of the logical files that contain "keys". <figure>
+            <title>Global Search Example</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="images/ECLWA015.jpg" />
+              </imageobject>
+            </mediaobject>
+          </figure></para>
+
+        <!-- Enter <i>ecl:output</i> into the global search box and it will return all of the workunits that contain "output" in the ECL page of ECL Watch-->
       </sect2>
 
       <sect2 id="ECLWatch_AutoRefresh">

ecl/hql/CMakeLists.txt

@@ -113,6 +113,8 @@ if (WIN32)
     )
 else()
     add_custom_command ( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h
+        #delete output files before building to force rebuilds with errors to fail rather than continue unnoticed, using the existing output files.
+        COMMAND rm -f ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp
         #pipe result through grep to remove warnings that are hard to suppress, and pipe through cat to prevent error code from no matches aborting the compile
         COMMAND ${bisoncmdprefix} ${bisoncmd} --report=state --defines=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.h --output=${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hqlgram.y 2>&1 | grep -v "unused value" | cat
         DEPENDS hqlgram.y

ecl/hql/hql.hpp

@@ -117,7 +117,7 @@ typedef IAtom ISourcePath;
 struct HQL_API ECLlocation
 {
 public:
-    inline ECLlocation() {}
+    inline ECLlocation() : lineno(0), column(0), position(0), sourcePath(NULL) {}
     ECLlocation(const IHqlExpression * _expr) { if (!extractLocationAttr(_expr)) clear(); }
     ECLlocation(int _line, int _column, int _position, ISourcePath * _sourcePath) { set(_line, _column, _position, _sourcePath); }
 

ecl/hql/hqlgram.hpp

@@ -456,6 +456,7 @@ public:
     void checkIntegerOrString(attribute & e1);
     void checkNumeric(attribute &e1);
     ITypeInfo *checkNumericGetType(attribute &e1);
+    void checkInlineDatasetOptions(const attribute & attr);
     void checkLibraryParametersMatch(const attribute & errpos, bool isParametered, const HqlExprArray & activeParameters, IHqlExpression * definition);
     void checkReal(attribute &e1);
     ITypeInfo *checkStringIndex(attribute & strAttr, attribute & idxAttr);

ecl/hql/hqlgram.y

@@ -3101,7 +3101,7 @@ datasetFlag
                             $$.setExpr(createExprAttribute(distributedAtom));
                             $$.setPosition($1);
                         }
-    | localAttribute
+    | commonAttribute
     ;
 
 optIndexFlags
@@ -5635,7 +5635,7 @@ primexpr1
                         {   $$.inherit($2); }
     | COUNT '(' startTopFilter aggregateFlags ')' endTopFilter
                         {
-                            $$.setExpr(createValue(no_count, LINK(parser->defaultIntegralType), $3.getExpr(), $4.getExpr()));
+                            $$.setExpr(createValueF(no_count, LINK(parser->defaultIntegralType), $3.getExpr(), $4.getExpr(), NULL));
                         }
     | COUNT '(' GROUP optExtraFilter ')'
                         {
@@ -5667,7 +5667,7 @@ primexpr1
                         }
     | EXISTS '(' dataSet aggregateFlags ')'
                         {
-                            $$.setExpr(createBoolExpr(no_exists, $3.getExpr(), $4.getExpr()));
+                            $$.setExpr(createValueF(no_exists, makeBoolType(), $3.getExpr(), $4.getExpr(), NULL));
                             $$.setPosition($1);
                         }
     | EXISTS '(' dictionary ')'
@@ -6034,7 +6034,7 @@ primexpr1
                         {
                             parser->normalizeExpression($5);
                             IHqlExpression *e5 = $5.getExpr();
-                            $$.setExpr(createValue(no_max, e5->getType(), $3.getExpr(), e5, $6.getExpr()));
+                            $$.setExpr(createValueF(no_max, e5->getType(), $3.getExpr(), e5, $6.getExpr(), NULL));
                         }
     | MAX '(' GROUP ',' expression ')'
                         {
@@ -6046,7 +6046,7 @@ primexpr1
                         {
                             parser->normalizeExpression($5);
                             IHqlExpression *e5 = $5.getExpr();
-                            $$.setExpr(createValue(no_min, e5->getType(), $3.getExpr(), e5, $6.getExpr()));
+                            $$.setExpr(createValueF(no_min, e5->getType(), $3.getExpr(), e5, $6.getExpr(), NULL));
                         }
     | MIN '(' GROUP ',' expression ')'
                         {
@@ -6067,7 +6067,7 @@ primexpr1
                             Owned<ITypeInfo> temp = parser->checkPromoteNumeric($5, true);
                             OwnedHqlExpr value = $5.getExpr();
                             Owned<ITypeInfo> type = getSumAggType(value);
-                            $$.setExpr(createValue(no_sum, LINK(type), $3.getExpr(), ensureExprType(value, type), $6.getExpr()));
+                            $$.setExpr(createValueF(no_sum, LINK(type), $3.getExpr(), ensureExprType(value, type), $6.getExpr(), NULL));
                         }
     | SUM '(' GROUP ',' expression optExtraFilter ')'
                         {
@@ -6075,12 +6075,12 @@ primexpr1
                             Owned<ITypeInfo> temp = parser->checkPromoteNumeric($5, true);
                             OwnedHqlExpr value = $5.getExpr();
                             Owned<ITypeInfo> type = getSumAggType(value);
-                            $$.setExpr(createValue(no_sumgroup, LINK(type), ensureExprType(value, type), $6.getExpr()));
+                            $$.setExpr(createValueF(no_sumgroup, LINK(type), ensureExprType(value, type), $6.getExpr(), NULL));
                         }
     | AVE '(' startTopFilter ',' expression aggregateFlags ')' endTopFilter
                         {
                             parser->normalizeExpression($5, type_numeric, false);
-                            $$.setExpr(createValue(no_ave, makeRealType(8), $3.getExpr(), $5.getExpr(), $6.getExpr()));
+                            $$.setExpr(createValueF(no_ave, makeRealType(8), $3.getExpr(), $5.getExpr(), $6.getExpr(), NULL));
                         }
     | AVE '(' GROUP ',' expression optExtraFilter')'
                         {
@@ -6090,7 +6090,7 @@ primexpr1
     | VARIANCE '(' startTopFilter ',' expression aggregateFlags ')' endTopFilter
                         {
                             parser->normalizeExpression($5, type_numeric, false);
-                            $$.setExpr(createValue(no_variance, makeRealType(8), $3.getExpr(), $5.getExpr(), $6.getExpr()));
+                            $$.setExpr(createValueF(no_variance, makeRealType(8), $3.getExpr(), $5.getExpr(), $6.getExpr(), NULL));
                         }
     | VARIANCE '(' GROUP ',' expression optExtraFilter')'
                         {
@@ -6101,7 +6101,7 @@ primexpr1
                         {
                             parser->normalizeExpression($5, type_numeric, false);
                             parser->normalizeExpression($7, type_numeric, false);
-                            $$.setExpr(createValue(no_covariance, makeRealType(8), $3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr()));
+                            $$.setExpr(createValueF(no_covariance, makeRealType(8), $3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr(), NULL));
                         }
     | COVARIANCE '(' GROUP ',' expression ',' expression optExtraFilter')'
                         {
@@ -6113,7 +6113,7 @@ primexpr1
                         {
                             parser->normalizeExpression($5, type_numeric, false);
                             parser->normalizeExpression($7, type_numeric, false);
-                            $$.setExpr(createValue(no_correlation, makeRealType(8), $3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr()));
+                            $$.setExpr(createValueF(no_correlation, makeRealType(8), $3.getExpr(), $5.getExpr(), $7.getExpr(), $8.getExpr(), NULL));
                         }
     | CORRELATION '(' GROUP ',' expression ',' expression optExtraFilter')'
                         {
@@ -6801,13 +6801,18 @@ xmlEncodeFlags
 
 aggregateFlags
     :                   { $$.setNullExpr(); }
-    | ',' KEYED         { $$.setExpr(createAttribute(keyedAtom)); $$.setPosition($2); }
-    | ',' prefetchAttribute
+    | aggregateFlags ',' aggregateFlag
                         {
-                            $$.setExpr($2.getExpr(), $2);
+                            $$.setExpr(createComma($1.getExpr(), $3.getExpr()), $1);
                         }
     ;
 
+aggregateFlag
+    : KEYED             { $$.setExpr(createAttribute(keyedAtom), $1); }
+    | prefetchAttribute
+    | hintAttribute
+    ;
+
 transfer
     : TYPE_LPAREN typeDef TYPE_RPAREN 
                         { $$ = $2; }
@@ -8787,14 +8792,8 @@ simpleDataSet
                             IHqlExpression * counter = $7.getExpr();
                             if (counter)
                                 counter = createAttribute(_countProject_Atom, counter);
-                            OwnedHqlExpr options = $8.getExpr();
-                            if (options)
-                            {
-                                if (options->numChildren() > 0)
-                                    parser->reportError(ERR_DSPARAM_INVALIDOPTCOMB, $8, "The DATASET options DISTRIBUTED, LOCAL, and NOLOCAL are not permutable.");
-                            }
-                            $$.setExpr(createDataset(no_dataset_from_transform, $3.getExpr(), createComma($6.getExpr(), counter, options.getClear())));
-                            $$.setPosition($1);
+                            parser->checkInlineDatasetOptions($8);
+                            $$.setExpr(createDataset(no_dataset_from_transform, $3.getExpr(), createComma($6.getExpr(), counter, $8.getExpr())), $1);
                         }
     | ENTH '(' dataSet ',' expression optCommonAttrs ')'
                         {

ecl/hql/hqlgram2.cpp

@@ -2153,6 +2153,20 @@ void HqlGram::checkConstant(attribute & attr)
 }
 
 
+void HqlGram::checkInlineDatasetOptions(const attribute & attr)
+{
+    IHqlExpression * options = attr.queryExpr();
+    unsigned optionCount = 0;
+    if (queryAttributeInList(distributedAtom, options))
+        optionCount++;
+    if (queryAttributeInList(localAtom, options))
+        optionCount++;
+    if (queryAttributeInList(noLocalAtom, options))
+        optionCount++;
+    if (optionCount > 1)
+        reportError(ERR_DSPARAM_INVALIDOPTCOMB, attr, "The DATASET options DISTRIBUTED, LOCAL, and NOLOCAL cannot be combined.");
+}
+
 IHqlExpression * HqlGram::checkConcreteModule(const attribute & errpos, IHqlExpression * expr)
 {
     return checkCreateConcreteModule(this, expr, errpos.pos);

ecl/hql/hqlutil.cpp

@@ -4594,11 +4594,16 @@ extern HQL_API IHqlExpression * convertScalarAggregateToDataset(IHqlExpression *
         field.setown(createField(valueId, expr->getType(), NULL));
 
     IHqlExpression * aggregateRecord = createRecord(field);
-    IHqlExpression * keyedAttr = expr->queryAttribute(keyedAtom);
-    IHqlExpression * prefetchAttr = expr->queryAttribute(prefetchAtom);
 
     HqlExprArray valueArgs;
-    unwindChildren(valueArgs, expr, 1);
+    ForEachChildFrom(i1, expr, 1)
+    {
+        IHqlExpression * cur = expr->queryChild(i1);
+        //keyed is currently required on the aggregate operator
+        if (!cur->isAttribute() || (cur->queryName() == keyedAtom))
+            valueArgs.append(*LINK(cur));
+    }
+
     IHqlExpression * newValue = createValue(newop, expr->getType(), valueArgs);
     IHqlExpression * assign = createAssign(createSelectExpr(getSelf(aggregateRecord), LINK(field)), newValue);
     IHqlExpression * transform = createValue(no_newtransform, makeTransformType(aggregateRecord->getType()), assign);
@@ -4608,7 +4613,17 @@ extern HQL_API IHqlExpression * convertScalarAggregateToDataset(IHqlExpression *
     if (dataset->queryType()->getTypeCode() == type_groupedtable)
         dataset = createDataset(no_group, dataset, NULL);
 
-    IHqlExpression * project = createDataset(no_newaggregate, dataset, createComma(aggregateRecord, transform, LINK(keyedAttr), LINK(prefetchAttr)));
+    HqlExprArray args;
+    args.append(*dataset);
+    args.append(*aggregateRecord);
+    args.append(*transform);
+    ForEachChild(i2, expr)
+    {
+        IHqlExpression * cur = expr->queryChild(i2);
+        if (cur->isAttribute())
+            args.append(*LINK(cur));
+    }
+    IHqlExpression * project = createDataset(no_newaggregate, args);
     return createRow(no_selectnth, project, createConstantOne());
 }
 

ecl/hqlcpp/hqlcpp.cpp

@@ -1752,7 +1752,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.alwaysUseGraphResults,"alwaysUseGraphResults",false),
         DebugOption(options.noConditionalLinks,"noConditionalLinks",false),
         DebugOption(options.reportAssertFilenameTail,"reportAssertFilenameTail",false),        
-        DebugOption(options.newBalancedSpotter,"newBalancedSpotter",false),
+        DebugOption(options.newBalancedSpotter,"newBalancedSpotter",true),
         DebugOption(options.keyedJoinPreservesOrder,"keyedJoinPreservesOrder",true),
         DebugOption(options.expandSelectCreateRow,"expandSelectCreateRow",false),
         DebugOption(options.obfuscateOutput,"obfuscateOutput",false),

ecl/hqlcpp/hqlcppds.cpp

@@ -2921,7 +2921,7 @@ public:
     virtual void * finalizeRow(size32_t newSize, void * row, size32_t oldSize) { throwUnexpected(); }
 
     virtual IOutputMetaData * queryOutputMeta() { return NULL; }
-    virtual unsigned queryActivityId() { return 0; }
+    virtual unsigned queryActivityId() const { return 0; }
     virtual StringBuffer &getId(StringBuffer & out) { return out; }
     virtual IOutputRowSerializer *createDiskSerializer(ICodeContext *ctx = NULL) { throwUnexpected(); }
     virtual IOutputRowDeserializer *createDiskDeserializer(ICodeContext *ctx) { throwUnexpected(); }

ecl/hqlcpp/hqlhtcpp.cpp

@@ -452,8 +452,12 @@ public:
                 return true;
             switch (search->getOperator())
             {
-            case no_selectnth:
             case no_newaggregate:
+                //Hash aggregate is NOT a trivial operation.
+                if (queryRealChild(search, 3))
+                    return false;
+                break;
+            case no_selectnth:
             case no_filter:
                 break;
             case no_select:

ecl/hqlcpp/hqlresource.cpp

@@ -2791,6 +2791,9 @@ ResourcerInfo::ResourcerInfo(IHqlExpression * _original, CResourceOptions * _opt
     visited = false;
     lastPass = 0;
     balancedExternalUses = 0;
+    balancedInternalUses = 0;
+    balancedVisiting = false;
+    removedParallelPullers = false;
 #ifdef TRACE_BALANCED
     balanceId = 0;
 #endif
@@ -2948,6 +2951,8 @@ void ResourcerInfo::resetBalanced()
     curBalanceLink = 0;
     balancedVisiting = false;
     balancedExternalUses = 0;
+    balancedInternalUses = 0;
+    removedParallelPullers = false;
 }
 
 bool ResourcerInfo::spillSharesSplitter()
@@ -5027,6 +5032,7 @@ void CSplitterInfo::addLink(IHqlExpression * source, IHqlExpression * sink, bool
 
         sourceInfo->balancedLinks.append(*link);
         sinkInfo->balancedLinks.append(*LINK(link));
+        sourceInfo->balancedInternalUses++;
 
 #ifdef TRACE_BALANCED
         printf("\tn%u -> n%u;\n", sourceInfo->balanceId, sinkInfo->balanceId);
@@ -5039,7 +5045,18 @@ void CSplitterInfo::addLink(IHqlExpression * source, IHqlExpression * sink, bool
     }
 }
 
-bool CSplitterInfo::allInputsPulledIndependently(IHqlExpression * expr) const
+void CSplitterLink::mergeSinkLink(CSplitterLink & sinkLink)
+{
+    IHqlExpression * newSink = sinkLink.querySink();
+    assertex(newSink);
+    assertex(sinkLink.hasSource(querySink()));
+    sink.set(newSink);
+    ResourcerInfo * sinkInfo = queryResourceInfo(newSink);
+    unsigned sinkPos = sinkInfo->balancedLinks.find(sinkLink);
+    sinkInfo->balancedLinks.replace(OLINK(*this), sinkPos);
+}
+
+bool CSplitterInfo::allInputsPulledIndependently(IHqlExpression * expr)
 {
     switch (expr->getOperator())
     {
@@ -5077,13 +5094,7 @@ bool CSplitterInfo::isBalancedSplitter(IHqlExpression * expr) const
     ResourcerInfo * info = queryResourceInfo(expr);
     if (!info->balanced)
         return false;
-    unsigned numOutputs = info->balancedExternalUses;
-    ForEachItemIn(i, info->balancedLinks)
-    {
-        CSplitterLink & cur = info->balancedLinks.item(i);
-        if (cur.hasSource(expr))
-            numOutputs++;
-    }
+    unsigned numOutputs = info->balancedExternalUses + info->balancedInternalUses;
     return (numOutputs > 1);
 }
 
@@ -5126,10 +5137,7 @@ void CSplitterInfo::gatherPotentialSplitters(IHqlExpression * expr, IHqlExpressi
         if (alreadyVisited)
             return;
 
-        if (allInputsPulledIndependently(expr))
-            sink = NULL;
-        else
-            sink = expr;
+        sink = expr;
     }
 
     if (info->containsActivity)
@@ -5275,10 +5283,121 @@ IHqlExpression * EclResourcer::walkPotentialSplitterLinks(CSplitterInfo & connec
     return NULL;
 }
 
+bool EclResourcer::removePassThrough(CSplitterInfo & connections, ResourcerInfo & info)
+{
+    if (info.balancedLinks.ordinality() != 2)
+        return false;
+
+    CSplitterLink & link0 = info.balancedLinks.item(0);
+    CSplitterLink & link1 = info.balancedLinks.item(1);
+
+    CSplitterLink * sourceLink;
+    CSplitterLink * sinkLink;
+    if (link0.hasSource(info.original) && link1.hasSink(info.original))
+    {
+        sourceLink = &link1;
+        sinkLink = &link0;
+    }
+    else if (link0.hasSink(info.original) && link1.hasSource(info.original))
+    {
+        sourceLink = &link0;
+        sinkLink = &link1;
+    }
+    else
+        return false;
+
+    if (!sinkLink->querySink())
+        return false;
+
+#ifdef TRACE_BALANCED
+    printf("//remove node %u since now pass-through\n", info.balanceId);
+#endif
+
+    sourceLink->mergeSinkLink(*sinkLink);
+    return true;
+}
+
+void EclResourcer::removeDuplicateIndependentLinks(CSplitterInfo & connections, ResourcerInfo & info)
+{
+    IHqlExpression * expr = info.original;
+    loop
+    {
+        bool again = false;
+        for (unsigned i=0; i < info.balancedLinks.ordinality(); i++)
+        {
+            CSplitterLink & cur = info.balancedLinks.item(i);
+            if (cur.hasSource(expr))
+            {
+                IHqlExpression * sink = cur.queryOther(expr);
+                assertex(sink);
+                ResourcerInfo & sinkInfo = *queryResourceInfo(sink);
+                if (CSplitterInfo::allInputsPulledIndependently(sink))
+                {
+                    unsigned numRemoved = 0;
+                    for (unsigned j=info.balancedLinks.ordinality()-1; j > i; j--)
+                    {
+                        CSplitterLink & next = info.balancedLinks.item(j);
+                        if (next.hasSource(expr) && next.hasSink(sink))
+                        {
+                            info.balancedLinks.remove(j);
+                            sinkInfo.balancedLinks.zap(next);
+                            numRemoved++;
+                        }
+                    }
+
+#ifdef TRACE_BALANCED
+                    if (numRemoved)
+                        printf("//removed %u duplicate links from %u to %u\n", numRemoved, info.balanceId, sinkInfo.balanceId);
+#endif
+
+                }
+
+                //Removing duplicate links has turned the source item into a pass-through.
+                //Replace references to the sink activity with references to its sink
+                //to possibly allow more to be removed.
+                if (removePassThrough(connections, sinkInfo))
+                {
+#ifdef TRACE_BALANCED
+                    printf("//remove %u now pass-through\n", sinkInfo.balanceId);
+#endif
+                    again = true;
+                }
+            }
+        }
+        if (!again)
+            break;
+    }
+}
+
+
+void EclResourcer::optimizeIndependentLinks(CSplitterInfo & connections, ResourcerInfo & info)
+{
+    if (info.removedParallelPullers)
+        return;
+    info.removedParallelPullers = true;
+
+    removeDuplicateIndependentLinks(connections, info);
+
+    //Recurse over inputs to this activity (each call may remove links)
+    for (unsigned i=0; i < info.balancedLinks.ordinality(); i++)
+    {
+        CSplitterLink & cur = info.balancedLinks.item(i);
+        if (cur.hasSink(info.original))
+            optimizeIndependentLinks(connections, *queryResourceInfo(cur.querySource()));
+    }
+}
+
+
 void EclResourcer::optimizeConditionalLinks(CSplitterInfo & connections)
 {
     //MORE: IF() can be special cased.  If it creates two identical links then one of them can be removed
     //Implement by post processing the links and removing duplicates
+    ForEachItemIn(i, connections.sinks)
+    {
+        IHqlExpression & cur = connections.sinks.item(i);
+        ResourcerInfo * info = queryResourceInfo(&cur);
+        optimizeIndependentLinks(connections, *info);
+    }
 }
 
 void EclResourcer::walkPotentialSplitters(CSplitterInfo & connections)

ecl/hqlcpp/hqlresource.ipp

@@ -276,6 +276,10 @@ public:
 
     bool hasSink(IHqlExpression * expr) const { return sink == expr->queryBody(); }
     bool hasSource(IHqlExpression * expr) const { return source == expr->queryBody(); }
+    IHqlExpression * querySource() const { return source; }
+    IHqlExpression * querySink() const { return sink; }
+
+    void mergeSinkLink(CSplitterLink & sinkLink);
 
 private:
     LinkedHqlExpr source;
@@ -290,7 +294,7 @@ public:
     ~CSplitterInfo();
 
     void addLink(IHqlExpression * source, IHqlExpression * sink, bool isExternal);
-    bool allInputsPulledIndependently(IHqlExpression * expr) const;
+    static bool allInputsPulledIndependently(IHqlExpression * expr);
     void gatherPotentialSplitters(IHqlExpression * expr, IHqlExpression * sink, ResourceGraphInfo * graph, bool isDependency);
     bool isSplitOrBranch(IHqlExpression * expr) const;
     bool isBalancedSplitter(IHqlExpression * expr) const;
@@ -430,6 +434,7 @@ public:
     unsigned curBalanceLink;
     unsigned lastPass;
     unsigned balancedExternalUses;
+    unsigned balancedInternalUses;
 #ifdef TRACE_BALANCED
     unsigned balanceId;
 #endif
@@ -447,6 +452,7 @@ public:
     bool projectResult:1;
     bool visited:1;
     bool balancedVisiting:1;
+    bool removedParallelPullers:1;
 };
 
 class EclResourceDependencyGatherer;
@@ -520,6 +526,9 @@ protected:
     void spotSharedInputs(IHqlExpression * expr, ResourceGraphInfo * graph);
     void spotSharedInputs();
 
+    bool removePassThrough(CSplitterInfo & connections, ResourcerInfo & info);
+    void removeDuplicateIndependentLinks(CSplitterInfo & connections, ResourcerInfo & info);
+    void optimizeIndependentLinks(CSplitterInfo & connections, ResourcerInfo & info);
     void optimizeConditionalLinks(CSplitterInfo & connections);
     IHqlExpression * walkPotentialSplitterLinks(CSplitterInfo & connections, IHqlExpression * expr, const CSplitterLink * link);
     IHqlExpression * walkPotentialSplitters(CSplitterInfo & connections, IHqlExpression * expr, const CSplitterLink & link);

ecl/hqlcpp/hqlttcpp.cpp

@@ -4681,7 +4681,9 @@ IHqlExpression * OptimizeActivityTransformer::optimizeCompare(IHqlExpression * l
         if (lhs->getOperator() == no_count)
         {
             IHqlExpression * ds = lhs->queryChild(0);
-            OwnedHqlExpr ret = createValue(no_exists, makeBoolType(), LINK(ds));
+            HqlExprArray args;
+            unwindChildren(args, lhs);
+            OwnedHqlExpr ret = createValue(no_exists, makeBoolType(), args);
             if (existOp == no_not)
                 return createValue(no_not, makeBoolType(), ret.getClear());
             return ret.getClear();

ecl/hthor/hthor.cpp

@@ -3894,6 +3894,12 @@ unsigned CHThorGroupSortActivity::getSpillCost() const
     return 10;
 }
 
+
+unsigned CHThorGroupSortActivity::getActivityId() const
+{
+    return activityId;
+}
+
 bool CHThorGroupSortActivity::freeBufferedRows(bool critical)
 {
     roxiemem::RoxieOutputRowArrayLock block(sorter->getRowArray());

ecl/hthor/hthor.ipp

@@ -1067,6 +1067,7 @@ public:
 
     //interface roxiemem::IBufferedRowCallback
     virtual unsigned getSpillCost() const;
+    virtual unsigned getActivityId() const;
     virtual bool freeBufferedRows(bool critical);
 
 private:

ecl/regress/count7.ecl

@@ -39,5 +39,5 @@ output(count(namesTable5) = 5);
 namesTable6 := dataset('x6',namesRecord,FLAT);
 output(count(namesTable6) != 5);
 namesTable7 := dataset('x7',namesRecord,FLAT);
-output(count(namesTable7) != 5);
-output(count(namesTable7) > 0);
+output(count(namesTable7, HINT(goQuitefast(true))) != 5);
+output(count(namesTable7, HINT(goReallyReallyFast(true))) > 0);

ecl/regress/dataset24.ecl

@@ -27,6 +27,6 @@ namesTable2 := dataset([
         {'Hawthorn','Gavin',31+zero},
         {'Hawthorn','Peter',30+zero},
         {'Smithe','Simon',10+zero},
-        {'X','Z',zero}], namesRecord);
+        {'X','Z',zero}], namesRecord, HINT(ThisIsAHint(true)));
 
 output(namesTable2,,'out.d00',overwrite);

ecl/regress/dataset_transform.ecl

@@ -31,7 +31,7 @@ end;
 
 // zero
 output(true);
-ds := DATASET(0, t1(COUNTER));
+ds := DATASET(0, t1(COUNTER), HINT(fasterThanAFastThing(true)));
 output(ds);
 
 // plain

ecl/regress/readahead2.ecl

@@ -506,7 +506,7 @@ outrec it(in L) := TRANSFORM
   self := L;
 END;
 
-OUTPUT(SUM(in, TS_WordIndex(KEYED(word = in.word))[1].doc, prefetch(5, parallel))) : independent;
+OUTPUT(SUM(in, TS_WordIndex(KEYED(word = in.word))[1].doc, prefetch(5, parallel), hint(thisIsAHint('Help!')))) : independent;
 OUTPUT(TABLE(in, { sum(group, TS_WordIndex(KEYED(word = in.word))[1].doc), count(group) }, prefetch(5, parallel))) : independent;
 OUTPUT(TABLE(in, { sum(group, TS_WordIndex(KEYED(word = in.word))[1].doc), count(group) }, word[1], prefetch(5, parallel))) : independent;
 

ecl/wutest/wutest.cpp

@@ -1721,7 +1721,7 @@ protected:
 
         size32_t lenResult;
         void * result;
-        wsWorkunitList(&ctx, lenResult, result, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, true, false);
+        wsWorkunitList(&ctx, lenResult, result, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, true, false, NULL);
         /* export WsWorkunitRecord := record "
                                     " string24 wuid;"                         0
                                     " string owner{maxlength(64)};"           24
@@ -1759,7 +1759,7 @@ protected:
 
         // Now filter owner via generic mechanism
         unsigned start = msTick();
-        wsWorkunitList(&ctx, lenResult, result, NULL, NULL, "WuTestUser00", NULL, NULL, "completed", NULL, NULL, NULL, NULL, NULL, true, false);
+        wsWorkunitList(&ctx, lenResult, result, NULL, NULL, "WuTestUser00", NULL, NULL, "completed", NULL, NULL, NULL, NULL, NULL, true, false, NULL);
         ASSERT(lenResult % sizeof(resultStruct) == 0);
         unsigned numResults = lenResult/sizeof(resultStruct);
         resultStruct *it = (resultStruct *) result;

ecllibrary/std/system/Workunit.ecl

@@ -63,10 +63,11 @@ EXPORT BOOLEAN WorkunitExists(varstring wuid, boolean online=true, boolean archi
  * @param eclcontains   the text to search for in the workunit�s ECL code. This may contain wildcard ( * ? ) characters.
  * @param online        the flag specifying whether the search is performed online.
  * @param archived      the flag specifying whether the search is performed in the archives.
+ * @param appvalues     application values to search for. Use a string of the form appname/key=value or appname/*=value.
  */
 
 EXPORT dataset(WorkunitRecord) WorkunitList(
-                                         varstring lowwuid, 
+                                         varstring lowwuid='',
                                          varstring highwuid='', 
                                          varstring username='', 
                                          varstring cluster='', 
@@ -78,13 +79,14 @@ EXPORT dataset(WorkunitRecord) WorkunitList(
                                          varstring roxiecluster='',
                                          varstring eclcontains='',
                                          boolean online=true,
-                                         boolean archived=false
+                                         boolean archived=false,
+                                         varstring appvalues=''
                                         ) :=
     lib_workunitservices.WorkUnitServices.WorkunitList(
                                         lowwuid, highwuid, 
                                         username, cluster, jobname, state, priority,
                                         fileread, filewritten, roxiecluster, eclcontains,
-                                        online, archived);
+                                        online, archived, appvalues);
 
 /*
  * Returns a valid Workunit identifier for the specified date and time.  This is useful for creating ranges of wuids 
@@ -163,5 +165,17 @@ EXPORT dataset(TimingRecord) WorkunitTimings(varstring wuid) :=
 EXPORT dataset(StatisticRecord) WorkunitStatistics(varstring wuid, boolean includeActivities = false, varstring _filter = '') :=
   lib_workunitservices.WorkUnitServices.WorkunitStatistics(wuid, includeActivities, _filter);
 
+/*
+ * Sets an application value in current workunit. Returns true if the value was set successfully.
+ *
+ * @param app           the app name to set.
+ * @param key           the name of the value to set.
+ * @param value         the value to set.
+ * @param overwrite     whether an existing value should be overwritten (default=true).
+*/
 
-END;
+EXPORT boolean SetWorkunitAppValue(varstring app, varstring key, varstring value, boolean overwrite=true) :=
+  lib_workunitservices.WorkUnitServices.SetWorkunitAppValue(app, key, value, overwrite);
+
+
+END;

esp/services/WsDeploy/WsDeployEngine.hpp

@@ -46,7 +46,7 @@ public:
 
     virtual void printStatus(IDeployTask* task);
     virtual void printStatus(StatusType type, const char* processType, const char* process, 
-                             const char* instance, const char* format=NULL, ...) __attribute((format(printf,6,7)));
+                             const char* instance, const char* format=NULL, ...) __attribute__((format(printf,6,7)));
    virtual bool onDisconnect(const char* target);
     bool getAbortStatus() const
     {

esp/src/eclwatch/ActivityWidget.js

@@ -26,6 +26,7 @@ define([
     "dijit/form/ToggleButton",
     "dijit/ToolbarSeparator",
     "dijit/layout/ContentPane",
+    "dijit/Tooltip",
 
     "dgrid/selector",
     "dgrid/tree",
@@ -37,7 +38,7 @@ define([
     "hpcc/ESPUtil"
 
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, on,
-                registry, Button, ToggleButton, ToolbarSeparator, ContentPane,
+                registry, Button, ToggleButton, ToolbarSeparator, ContentPane, Tooltip,
                 selector, tree,
                 GridDetailsWidget, ESPRequest, ESPActivity, DelayLoadWidget, ESPUtil) {
     return declare("ActivityWidget", [GridDetailsWidget], {
@@ -49,6 +50,7 @@ define([
 
         _onAutoRefresh: function (event) {
             this.activity.disableMonitor(!this.autoRefreshButton.get("checked"));
+            this.createStackControllerTooltip(this.id + "AutoRefresh", this.i18n.AutoRefresh + ": " + this.autoRefreshButton.get("checked"));
         },
 
         _onPause: function (event, params) {
@@ -206,13 +208,14 @@ define([
                     context.refreshGrid();
                 }
             });
+            this.createStackControllerTooltip(this.id + "AutoRefresh", this.i18n.AutoRefresh + ": " + this.autoRefreshButton.get("checked"));
         },
 
         createGrid: function (domID) {
             var context = this;
 
             this.openButton = registry.byId(this.id + "Open");
-            this.refreshButton = registry.byId(this.id + "Refresh");            
+            this.refreshButton = registry.byId(this.id + "Refresh");
             this.autoRefreshButton = new ToggleButton({
                 id: this.id + "AutoRefresh",
                 iconClass:'iconAutoRefresh',
@@ -583,6 +586,15 @@ define([
             this.wuMoveUpButton.set("disabled", !wuCanUp);
             this.wuMoveDownButton.set("disabled", !wuCanDown);
             this.wuMoveBottomButton.set("disabled", !wuCanDown);
+        },
+
+        createStackControllerTooltip: function (widgetID, text) {
+            return new Tooltip({
+                connectId: [widgetID],
+                label: text,
+                showDelay: 1,
+                position: ["below"]
+            });
         }
     });
 });

esp/src/eclwatch/ESPGraph.js

@@ -298,7 +298,7 @@ define([
 
     var Subgraph = declare([GraphItem], {
         constructor: function (graph, id) {
-            this._globalType = "Cluster";
+            this._globalType = id === "0" ? "Graph" : "Cluster";
             this.__hpcc_subgraphs = [];
             this.__hpcc_vertices = [];
             this.__hpcc_edges = [];
@@ -498,6 +498,15 @@ define([
                                 break;
                             case "edge":
                                 var edge = this.walkDocument(childNode, childNode.getAttribute("id"));
+                                if (edge.count) {
+                                    edge._eclwatchCount = edge.count.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
+                                }
+                                if (edge.inputProgress) {
+                                    edge._eclwatchInputProgress = "[" + edge.inputProgress.replace(/\B(?=(\d{3})+(?!\d))/g, ",") + "]";
+                                }
+                                if (edge.maxskew && edge.minskew) {
+                                    edge._eclwatchSkew = "+" + edge.maxskew + "%%, -" + edge.minskew + "%%";
+                                }
                                 retVal.addEdge(edge);
                                 break;
                             default:

esp/src/eclwatch/GraphTreeWidget.js

@@ -82,6 +82,14 @@ define([
         found: [],
         foundIndex: 0,
 
+        constructor: function (args) {
+            if (args.forceNative) {
+                this.graphType = "GraphWidget";
+            } else {
+                this.graphType = "JSGraphWidget";
+            }
+        },
+
         buildRendering: function (args) {
             this.inherited(arguments);
         },
@@ -233,7 +241,7 @@ define([
             pMenu.addChild(new MenuSeparator());
             pMenu.addChild(new CheckedMenuItem({
                 label: this.i18n.Activities,
-                checked: true,
+                checked: false,
                 onClick: function (evt) {
                     if (this.checked) {
                         context.treeGrid.set("query", {
@@ -289,8 +297,7 @@ define([
             if (this.findText != this.widget.FindField.value) {
                 this.findText = this.widget.FindField.value;
                 this.found = this.global.findAsGlobalID(this.findText);
-                this.global.setSelectedAsGlobalID(this.found);
-                this.syncSelectionFrom(this.global);
+                this.syncSelectionFrom(this.found);
                 this.foundIndex = -1;
             }
             this.foundIndex += prev ? -1 : +1;
@@ -446,7 +453,7 @@ define([
         },
 
         loadGraphFromXGMML: function (xgmml) {
-            if (this.global.loadXGMML(xgmml, false, this.graphTimers)) {
+            if (this.global.loadXGMML(xgmml, false, this.graphTimers, true)) {
                 this.global.setMessage("...");  //  Just in case it decides to render  ---
                 var initialSelection = [];
                 var mainRoot = [0];
@@ -465,7 +472,7 @@ define([
         },
 
         mergeGraphFromXGMML: function (xgmml) {
-            if (this.global.loadXGMML(xgmml, true, this.graphTimers)) {
+            if (this.global.loadXGMML(xgmml, true, this.graphTimers, true)) {
                 this.global.setMessage("...");  //  Just in case it decides to render  ---
                 this.refreshMainXGMML();
                 this.loadSubgraphs();
@@ -580,7 +587,10 @@ define([
             } else if (this.isQuery()) {
                 this.treeStore.appendColumns(columns, ["localTime", "totalTime", "label", "ecl"], ["DescendantCount", "definition", "SubgraphCount", "ActivityCount", "ChildCount", "Depth"]);
             }
-            this.treeGrid.set("query", {id:"0"});
+            this.treeGrid.set("query", {
+                id: "0",
+                __hpcc_notActivity: true
+            });
             this.treeGrid.set("columns", columns);
             this.treeGrid.refresh();
         },
@@ -657,6 +667,8 @@ define([
                         selectedGlobalIDs.push(items[i]._globalID);
                     }
                 }
+            } else if (sourceControl === this.found ) {
+                selectedGlobalIDs = this.found;
             } else {
                 selectedGlobalIDs = sourceControl.getSelectionAsGlobalID();
             }

esp/src/eclwatch/GraphWidget.js

@@ -41,10 +41,18 @@ define([
 
     "dijit/Toolbar", 
     "dijit/ToolbarSeparator", 
+
+    "dijit/TooltipDialog",
+    "dijit/form/Form",
+    "dijit/form/CheckBox",
+    "dijit/form/TextBox",
+    "dijit/form/DropDownButton",
     "dijit/form/Button",
     "dijit/form/ComboBox",
-    "dijit/form/NumberSpinner"
-    
+    "dijit/form/NumberSpinner",
+    "dijit/Fieldset",
+
+    "hpcc/TableContainer"
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, Deferred, has, dom, domConstruct, domClass, domStyle, Memory, Observable, QueryResults, Evented,
             registry, BorderContainer, ContentPane,
             _Widget, ESPUtil,
@@ -413,6 +421,7 @@ define([
             _onClickRefresh: function () {
                 var graphView = this.getCurrentGraphView();
                 graphView.refreshLayout(this);
+                this.refreshRootState(graphView.rootGlobalIDs);
             },
 
             _onClickPrevious: function () {
@@ -470,6 +479,12 @@ define([
                 }
             },
 
+            _onOptionsApply: function () {
+            },
+
+            _onOptionsReset: function () {
+            },
+
             onSelectionChanged: function (items) {
             },
 
@@ -489,10 +504,13 @@ define([
                 this.next = registry.byId(this.id + "Next");
                 this.previous = registry.byId(this.id + "Previous");
                 this.zoomDropCombo = registry.byId(this.id + "ZoomDropCombo");
+                this.depthLabel = registry.byId(this.id + "DepthLabel");
                 this.depth = registry.byId(this.id + "Depth");
                 this.distance = registry.byId(this.id + "Distance");
                 this.syncSelectionSplitter = registry.byId(this.id + "SyncSelectionSplitter");
                 this.syncSelection = registry.byId(this.id + "SyncSelection");
+                this.optionsDropDown = registry.byId(this.id + "OptionsDropDown");
+                this.optionsForm = registry.byId(this.id + "OptionsForm");
             },
 
             startup: function (args) {
@@ -513,6 +531,10 @@ define([
             },
 
             //  Plugin wrapper  ---
+            hasOptions: function () {
+                return false;
+            },
+
             createTreeStore: function () {
                 var store = new GraphTreeStore();
                 return Observable(store);
@@ -578,9 +600,10 @@ define([
                 }
             },
 
-            loadXGMML: function (xgmml, merge, timers) {
+            loadXGMML: function (xgmml, merge, timers, skipRender) {
                 if (this.hasPlugin() && this.xgmml !== xgmml) {
                     this.xgmml = xgmml;
+                    this._plugin._skipRender = skipRender;
                     if (merge) {
                         this._plugin.mergeXGMML(xgmml);
                     } else {
@@ -889,9 +912,9 @@ define([
             },
 
             _onLayoutFinished: function() {
+                this.setMessage('');
                 this.centerOnItem(0, true);
                 this.dot = this._plugin.getDOT();
-                this.setMessage('');
                 if (this.onLayoutFinished) {
                     this.onLayoutFinished();
                 }
@@ -1172,8 +1195,11 @@ define([
                     depthDisabled = !selectedGlobalIDs.length || !(typeSummary.Graph || typeSummary.Cluster);
                     distanceDisabled = !(typeSummary.Vertex || typeSummary.Edge);
                 }
+                depthDisabled = depthDisabled || (this.hasOptions() && !this.option("subgraph"))
+
                 this.setDisabled(this.id + "Depth", depthDisabled);
                 this.setDisabled(this.id + "Distance", distanceDisabled);
+                this.setDisabled(this.id + "OptionsDropDown", !this.hasOptions());
             }
         });
     });

esp/src/eclwatch/GraphsWidget.js

@@ -114,23 +114,24 @@ define([
 
         createGrid: function (domID) {
             var context = this;
-            this.openSafeMode = new Button({
-                label: this.i18n.OpenSafeMode,
+            this.openLegacyMode = new Button({
+                label: this.i18n.OpenLegacyMode,
                 onClick: function (event) {
                     context._onOpen(event, {
-                        safeMode: true
+                        legacyMode: true
                     });
                 }
             }).placeAt(this.widget.Open.domNode, "after");
-            this.openTreeMode = new Button({
-                label: this.i18n.OpenTreeMode,
-                onClick: function (event) {
-                    context._onOpen(event, {
-                        treeMode: true
-                    });
-                }
-            }).placeAt(this.widget.Open.domNode, "after");
-
+            if (dojoConfig.isPluginInstalled()) {
+                this.openNativeMode = new Button({
+                    label: this.i18n.OpenNativeMode,
+                    onClick: function (event) {
+                        context._onOpen(event, {
+                            nativeMode: true
+                        });
+                    }
+                }).placeAt(this.widget.Open.domNode, "after");
+            }
             var retVal = new declare([ESPUtil.Grid(false, true)])({
                 store: this.store,
                 columns: {
@@ -194,10 +195,10 @@ define([
 
         getDetailID: function (row, params) {
             var retVal = "Detail" + row[this.idProperty];
-            if (params && params.treeMode) {
-                retVal += "Tree";
-            } else if (params && params.safeMode) {
-                retVal += "Safe";
+            if (params && params.nativeMode) {
+                retVal += "Native";
+            } else if (params && params.legacyMode) {
+                retVal += "Legacy";
             }
             return retVal;
         },
@@ -228,16 +229,27 @@ define([
                 }
             }
             var title = row.Name;
-            if (params && params.treeMode) {
-                title += " (T)";
-            } else if (params && params.safeMode) {
-                title += " (S)";
+            var delayWidget = "GraphTreeWidget";
+            var delayProps = {
+                forceJS: true
+            };
+            if (params && params.nativeMode) {
+                title += " (N)";
+                delayWidget = "GraphTreeWidget";
+                delayProps = {
+                    forceNative: true
+                };
+            } else if (params && params.legacyMode) {
+                delayWidget = "GraphPageWidget";
+                title += " (L)";
+                delayProps = {};
             }
             return new DelayLoadWidget({
                 id: id,
                 title: title,
                 closable: true,
-                delayWidget: (params && params.treeMode) ? "GraphTreeWidget" : "GraphPageWidget",
+                delayWidget: delayWidget,
+                delayProps: delayProps,
                 hpcc: {
                     type: "graph",
                     params: localParams
@@ -298,8 +310,10 @@ define([
         refreshActionState: function (selection) {
             this.inherited(arguments);
 
-            this.openTreeMode.set("disabled", !selection.length);
-            this.openSafeMode.set("disabled", !selection.length);
+            if (this.openNativeMode) {
+                this.openNativeMode.set("disabled", !selection.length);
+            }
+            this.openLegacyMode.set("disabled", !selection.length);
         },
 
         syncSelectionFrom: function (sourceControl) {

esp/src/eclwatch/JSGraphWidget.js

@@ -25,9 +25,90 @@ define([
     "hpcc/ESPGraph"
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, Evented,
             GraphWidget, ESPGraph) {
+
+    var persist = {
+        remove: function (key) {
+            if (typeof (Storage) !== "undefined") {
+                localStorage.removeItem("JSGraphWidget_" + key);
+            }
+        },
+        set: function (key, val) {
+            if (typeof (Storage) !== "undefined") {
+                localStorage.setItem("JSGraphWidget_" + key, val);
+            }
+        },
+        setObj: function (key, val) {
+            this.set(key, JSON.stringify(val));
+        },
+        get: function (key, defValue) {
+            if (typeof (Storage) !== "undefined") {
+                var retVal = localStorage.getItem("JSGraphWidget_" + key);
+                return retVal === null ? defValue : retVal;
+            }
+            return "";
+        },
+        getObj: function (key, defVal) {
+            try {
+                return JSON.parse(this.get(key, defVal));
+            } catch (e) {
+                return {};
+            }
+        },
+        exists: function (key) {
+            if (typeof (Storage) !== "undefined") {
+                var retVal = localStorage.getItem("JSGraphWidget_" + key);
+                return retVal === null;
+            }
+            return false;
+        }
+    };
+
+    var faCharFactory = function (kind) {
+        switch (kind) {
+            case "2": return "\uf0c7";      //  Disk Write
+            case "3": return "\uf15d";      //  sort
+            case "5": return "\uf0b0";      //  Filter
+            case "6": return "\uf1e0";      //  Split
+            case "12": return "\uf039";     //  First N
+            case "15": return "\uf126";     //  Lightweight Join
+            case "17": return "\uf126";     //  Lookup Join
+            case "22": return "\uf1e6";     //  Pipe Output
+            case "23": return "\uf078";     //  Funnel
+            case "25": return "\uf0ce";     //  Inline Dataset
+            case "26": return "\uf074";     //  distribute
+            case "29": return "\uf005";     //  Store Internal Result
+            case "36": return "\uf128";     //  If
+            case "44": return "\uf0c7";     //  write csv
+            case "47": return "\uf0c7";     //  write 
+            case "54": return "\uf013";     //  Workunit Read
+            case "56": return "\uf0c7";     //  Spill
+            case "59": return "\uf126";     //  Merge
+            case "61": return "\uf0c7";     //  write xml
+            case "82": return "\uf1c0";     //  Projected Disk Read Spill 
+            case "88": return "\uf1c0";     //  Projected Disk Read Spill 
+            case "92": return "\uf129";     //  Limted Index Read
+            case "93": return "\uf129";     //  Limted Index Read
+            case "99": return "\uf1c0";     //  CSV Read
+            case "105": return "\uf1c0";    //  CSV Read
+
+            case "7": return "\uf090";      //  Project
+            case "9": return "\uf0e2";      //  Local Iterate
+            case "16": return "\uf005";     //  Output Internal
+            case "19": return "\uf074";     //  Hash Distribute
+            case "21": return "\uf275";     //  Normalize
+            case "35": return "\uf0c7";     //  CSV Write
+            case "37": return "\uf0c7";     //  Index Write
+            case "71": return "\uf1c0";     //  Disk Read Spill
+            case "133": return "\uf0ce";    //  Inline Dataset
+            case "148": return "\uf0ce";    //  Inline Dataset
+            case "168": return "\uf275";    //  Local Denormalize
+        }
+        return "\uf063";
+    };
+
     var loadJSPlugin = function (callback) {
-        require(["src/hpcc-viz", "src/hpcc-viz-common", "src/hpcc-viz-graph"], function () {
-            require(["src/common/Shape", "src/common/TextBox", "src/graph/Graph", "src/graph/Vertex", "src/graph/Edge"], function (Shape, TextBox, Graph, Vertex, Edge) {
+        require(["src/hpcc-viz", "src/hpcc-viz-common", "src/hpcc-viz-graph", "src/hpcc-viz-layout"], function () {
+            require(["src/common/Shape", "src/common/Icon", "src/common/TextBox", "src/graph/Graph", "src/graph/Vertex", "src/graph/Edge", "src/layout/Layered"], function (Shape, Icon, TextBox, Graph, Vertex, Edge, Layered) {
                 callback(declare([Evented], {
                     KeyState_None: 0,
                     KeyState_Shift: 1,
@@ -37,20 +118,53 @@ define([
                     constructor: function (domNode) {
                         this.graphData = new ESPGraph();
                         this.graphWidget = new Graph()
-                            .target(domNode.id)
                             .allowDragging(false)
-                            .render()
                         ;
                         var context = this;
                         this.graphWidget.vertex_click = function (item, event) {
                             context.emit("SelectionChanged", [item]);
                         }
+                        this.graphWidget.edge_click = function (item, event) {
+                            context.emit("SelectionChanged", [item]);
+                        }
                         this.graphWidget.vertex_dblclick = function (item, event) {
                             context.emit("MouseDoubleClick", item, (event.shiftKey ? context.KeyState_Shift : 0) + (event.ctrlKey ? context.KeyState_Control : 0) + (event.altKey ? context.KeyState_Menu : 0));
                         }
+                        this.messageWidget = new TextBox()
+                            .shape_colorFill("#006CCC")
+                            .shape_colorStroke("#003666")
+                            .text_colorFill("#FFFFFF")
+                        ;
+                        this.layout = new Layered()
+                            .target(domNode.id)
+                            .widgets([ this.messageWidget, this.graphWidget])
+                            .render()
+                        ;
+                        this._options = {};
+                    },
+
+                    option: function (key, _) {
+                        if (arguments.length < 1) throw Error("Invalid Call:  option");
+                        if (arguments.length === 1) return this._options[key];
+                        this._options[key] = _ instanceof Array ? _.length > 0 : _;
+                        return this;
+                    },
+
+                    optionsReset: function (options) {
+                        options = options || this._optionsDefault;
+                        for (var key in options) {
+                            this.option(key, options[key]);
+                        }
                     },
 
                     setMessage: function (msg) {
+                        if (msg !== this._prevMsg) {
+                            this.messageWidget.text(msg).render();
+                            if ((msg && this.graphWidget.visible()) || (!msg && !this.graphWidget.visible())) {
+                                this.graphWidget.visible(msg ? false : true).render();
+                            }
+                            this._prevMsg = msg;
+                        }
                     },
 
                     setScale: function (scale) {
@@ -59,10 +173,7 @@ define([
                     },
 
                     centerOnItem: function (item, scaleToFit, widthOnly) {
-                        if (item === 0) {
-                            item = this.graphData.subgraphs[0];
-                        }
-                        var bounds = this.graphWidget.getBounds([item.__widget]);
+                        var bounds = item === 0 ? this.graphWidget.getVertexBounds() : this.graphWidget.getBounds([item.__widget]);
                         if (scaleToFit) {
                             if (widthOnly) {
                                 bounds[0][1] = 0;
@@ -125,8 +236,27 @@ define([
                     },
 
                     find: function (findText) {
+                        var findProp = "";
+                        var findTerm = findText;
+                        var findTextParts = findText.split(":");
+                        if (findTextParts.length > 1) {
+                            findProp = findTextParts[0];
+                            findTextParts.splice(0, 1);
+                            findTerm = findTextParts.join(":");
+                        }
                         return this.graphData.vertices.filter(function (item) {
-                            return (item.label.toLowerCase().indexOf(findText.toLowerCase()) >= 0);
+                            if (findProp) {
+                                if (item.hasOwnProperty(findProp)) {
+                                    return (item[findProp].toString().toLowerCase().indexOf(findTerm.toLowerCase()) >= 0);
+                                }
+                            } else {
+                                for (var key in item) {
+                                    if (item.hasOwnProperty(key) && item[key].toString().toLowerCase().indexOf(findTerm.toLowerCase()) >= 0) {
+                                        return true;
+                                    }
+                                }
+                            }
+                            return false;
                         });
                     },
 
@@ -183,52 +313,109 @@ define([
                         this.graphWidget.clear();
                     },
 
-                    mergeXGMML: function (xgmml, timers) {
-                        return this.loadXGMML(xgmml, true, timers)
+                    mergeXGMML: function (xgmml) {
+                        this._loadXGMML(xgmml, true);
                     },
 
-                    loadXGMML: function (xgmml, merge, timers) {
-                        var retVal = this.inherited(arguments);
+                    loadXGMML: function (xgmml) {
+                        this._loadXGMML(xgmml, false);
+                    },
+
+                    _loadXGMML: function (xgmml, merge) {
                         if (merge) {
                             this.graphData.merge(xgmml);
                         } else {
                             this.graphData.load(xgmml);
                         }
+                        if (!this._skipRender) {
+                            this.rebuild(merge);
+                        }
+                    },
+
+                    format: function (labelTpl, obj) {
+                        var retVal = "";
+                        var lpos = labelTpl.indexOf("%");
+                        var rpos = -1;
+                        while (lpos >= 0) {
+                            retVal += labelTpl.substring(rpos + 1, lpos);
+                            rpos = labelTpl.indexOf("%", lpos + 1);
+                            if (rpos < 0) {
+                                console.log("Invalid Label Template");
+                                break;
+                            }
+                            var key = labelTpl.substring(lpos + 1, rpos);
+                            retVal += !key ? "%" : (obj[labelTpl.substring(lpos + 1, rpos)] || "");
+                            lpos = labelTpl.indexOf("%", rpos + 1);
+                        }
+                        retVal += labelTpl.substring(rpos + 1, labelTpl.length);
+                        return retVal.split("\\n").join("\n");
+                    },
+
+                    rebuild: function (merge) {
+                        merge = merge || false;
                         var vertices = [];
                         var edges = [];
                         var hierarchy = [];
 
-                        arrayUtil.forEach(this.graphData.subgraphs, function (subgraph, idx) {
-                            if (!subgraph.__widget) {
-                                subgraph.__widget = new Shape()
-                                    .shape("rect")
-                                    .width(0)
-                                    .height(0)
-                                ;
-                                subgraph.__widget.__hpcc_globalID = subgraph.__hpcc_id;
-                            }
-                            vertices.push(subgraph.__widget);
-                        }, this);
+                        if (this.option("subgraph")) {
+                            arrayUtil.forEach(this.graphData.subgraphs, function (subgraph, idx) {
+                                if (!merge || !subgraph.__widget) {
+                                    subgraph.__widget = new Shape()
+                                        .shape("rect")
+                                        .width(0)
+                                        .height(0)
+                                    ;
+                                    subgraph.__widget.__hpcc_globalID = subgraph.__hpcc_id;
+                                }
+                                vertices.push(subgraph.__widget);
+                            }, this);
+                        }
+                        var labelTpl = this.option("vlabel");
+                        var tooltipTpl = this.option("vtooltip");
                         arrayUtil.forEach(this.graphData.vertices, function (item, idx) {
-                            if (!item.__widget) {
+                            if (!merge || !item.__widget) {
+                                var label = this.format(labelTpl, item);
+                                var tooltip = this.format(tooltipTpl, item);
                                 switch (item._kind) {
                                     case "point":
                                         item.__widget = new Shape()
-                                            .radius(3)
+                                            .radius(7)
+                                            .tooltip(label)
                                         ;
                                         break;
                                     default:
-                                        item.__widget = new TextBox()
-                                            .text(item.label)
-                                        ;
+                                        if (this.option("vicon") && this.option("vlabel")) {
+                                            item.__widget = new Vertex()
+                                                .faChar(faCharFactory(item._kind))
+                                                .text(label)
+                                                .tooltip(tooltip)
+                                            ;
+                                        } else if (this.option("vicon")) {
+                                            item.__widget = new Icon()
+                                                .faChar(faCharFactory(item._kind))
+                                                .tooltip(tooltip)
+                                            ;
+                                        } else if (this.option("vlabel")) {
+                                            item.__widget = new TextBox()
+                                                .text(label)
+                                                .tooltip(tooltip)
+                                            ;
+                                        } else {
+                                            item.__widget = new Shape()
+                                                .radius(7)
+                                                .tooltip(tooltip)
+                                            ;
+                                        }
                                         break;
                                 }
                                 item.__widget.__hpcc_globalID = item.__hpcc_id;
                             }
                             vertices.push(item.__widget);
                         }, this);
+                        labelTpl = this.option("elabel");
+                        tooltipTpl = this.option("etooltip");
                         arrayUtil.forEach(this.graphData.edges, function (item, idx) {
-                            if (!item.__widget) {
+                            if (!merge || !item.__widget) {
                                 var strokeDasharray = null;
                                 var weight = 100;
                                 if (item._dependsOn) {
@@ -241,25 +428,8 @@ define([
                                     strokeDasharray = "5,5,10,5";
                                 }
 
-                                var label = item.label ? item.label : "";
-                                if (item.count) {
-                                    if (label) {
-                                        label += "\n";
-                                    }
-                                    label += item.count.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
-                                }
-                                if (item.inputProgress) {
-                                    if (label) {
-                                        label += "\n";
-                                    }
-                                    label += "[" + item.inputProgress.replace(/\B(?=(\d{3})+(?!\d))/g, ",") + "]";
-                                }
-                                if (item.maxskew && item.minskew) {
-                                    if (label) {
-                                        label += "\n";
-                                    }
-                                    label += "+" + item.maxskew + "%, -" + item.minskew + "%";
-                                }
+                                var label = this.format(labelTpl, item);
+                                var tooltip = this.format(tooltipTpl, item);
                                 item.__widget = new Edge()
                                     .sourceVertex(item.getSource().__widget)
                                     .targetVertex(item.getTarget().__widget)
@@ -267,27 +437,29 @@ define([
                                     .weight(weight)
                                     .strokeDasharray(strokeDasharray)
                                     .text(label)
+                                    .tooltip(tooltip)
                                 ;
                                 item.__widget.__hpcc_globalID = item.__hpcc_id;
                             }
                             edges.push(item.__widget);
                         }, this);
-                        arrayUtil.forEach(this.graphData.subgraphs, function (subgraph, idx) {
-                            arrayUtil.forEach(subgraph.__hpcc_subgraphs, function (item, idx) {
-                                if (!subgraph.__widget || !item.__widget) {
-                                    var d = 0;
-                                }
-                                hierarchy.push({ parent: subgraph.__widget, child: item.__widget });
-                            }, this);
-                            arrayUtil.forEach(subgraph.__hpcc_vertices, function (item, idx) {
-                                if (!subgraph.__widget || !item.__widget) {
-                                    var d = 0;
-                                }
-                                hierarchy.push({ parent: subgraph.__widget, child: item.__widget });
+                        if (this.option("subgraph")) {
+                            arrayUtil.forEach(this.graphData.subgraphs, function (subgraph, idx) {
+                                arrayUtil.forEach(subgraph.__hpcc_subgraphs, function (item, idx) {
+                                    if (!subgraph.__widget || !item.__widget) {
+                                        var d = 0;
+                                    }
+                                    hierarchy.push({ parent: subgraph.__widget, child: item.__widget });
+                                }, this);
+                                arrayUtil.forEach(subgraph.__hpcc_vertices, function (item, idx) {
+                                    if (!subgraph.__widget || !item.__widget) {
+                                        var d = 0;
+                                    }
+                                    hierarchy.push({ parent: subgraph.__widget, child: item.__widget });
+                                }, this);
                             }, this);
-                        }, this);
+                        }
                         this.graphWidget.data({ vertices: vertices, edges: edges, hierarchy: hierarchy, merge: merge });
-                        return retVal;
                     }
                 }));
             });
@@ -300,10 +472,32 @@ define([
             this.graphData = new ESPGraph();
         },
 
+        hasOptions: function(key, val) {
+            return this.hasPlugin();
+        },
+
+        _onOptionsApply: function () {
+            var optionsValues = this.optionsForm.getValues();
+            persist.setObj("options", optionsValues);
+            this.optionsDropDown.closeDropDown();
+            this._plugin.optionsReset(optionsValues);
+            this._plugin.rebuild();
+            this._onClickRefresh();
+        },
+
+        _onOptionsReset: function () {
+            this.optionsForm.setValues(this._plugin._optionsDefault);
+            this._plugin.optionsReset(this._plugin._optionsDefault);
+        },
+
+        option: function(key, val) {
+            return this._plugin.option.apply(this._plugin, arguments);
+        },
+
         resize: function (size) {
             this.inherited(arguments);
             if (this.hasPlugin()) {
-                this._plugin.graphWidget
+                this._plugin.layout
                     .resize()
                     .render()
                 ;
@@ -315,6 +509,10 @@ define([
                 var context = this;
                 loadJSPlugin(function (JSPlugin) {
                     context._plugin = new JSPlugin(context.graphContentPane.domNode);
+                    context._plugin._optionsDefault = context.optionsForm.getValues();
+                    var optionsValues = lang.mixin({}, context._plugin._optionsDefault, persist.getObj("options"));
+                    context._plugin.optionsReset(optionsValues);
+                    context.optionsForm.setValues(optionsValues);
                     context.version = {
                         major: 6,
                         minor: 0

esp/src/eclwatch/css/hpcc.css

@@ -1077,25 +1077,14 @@ margin-left:-20px;
     fill: white;
 }
 
-.claro .graph_Graph .edge .common_Shape {
-    fill: white;
-}
-
-.claro .graph_Graph .graphVertex > .common_Shape .common_Shape {
+.claro .graph_Graph .graphVertex > .common_Shape rect.common_Shape {
     stroke: black;
     fill: none;
 }
 
-.claro .graph_Graph .graphVertex > .common_Shape.selected .common_Shape {
-    stroke: #1f77b4;
-}
-
-.claro .graph_Graph .graphVertex .common_TextBox .common_Shape {
-    stroke: black;
-    fill: white;
+.claro .graph_Graph .graphVertex > .common_Shape .common_Shape {
 }
 
-.claro .graph_Graph .graphVertex .common_TextBox.selected .common_Shape {
-    fill: #dcf1ff;
-    stroke: #1f77b4;
+.claro .graph_Graph .graphVertex > .common_Shape.selected .common_Shape {
+    stroke: red
 }

esp/src/eclwatch/dojoConfig.js

@@ -58,6 +58,18 @@ var dojoConfig = (function () {
             //  Visualization Paths  ---
             "crossfilter": urlInfo.basePath + "/crossfilter/crossfilter.min",
             "font-awesome.css": urlInfo.basePath + "/Visualization/dist-amd/font-awesome/css/font-awesome.min.css"
+
+/*  HPCC Visualization Debug  
+            ,
+            "css": urlInfo.basePath + "/Visualization/node_modules/require-css/css",
+            "d3": urlInfo.basePath + "/Visualization/bower_components/d3/d3",
+            "c3": urlInfo.basePath + "/Visualization/bower_components/c3/c3",
+            "dagre": urlInfo.basePath + "/Visualization/bower_components/dagre/index",
+            "topojson": urlInfo.basePath + "/Visualization/bower_components/topojson/topojson",
+            "colorbrewer": urlInfo.basePath + "/Visualization/bower_components/colorbrewer/colorbrewer",
+            "d3-cloud": urlInfo.basePath + "/Visualization/bower_components/d3-cloud/build/d3.layout.cloud",
+            "font-awesome": urlInfo.basePath + "/Visualization/bower_components/font-awesome/css/font-awesome"
+*/
         },
         packages: [{
             name: "hpcc",
@@ -78,6 +90,11 @@ var dojoConfig = (function () {
             name: "d3",
             location: urlInfo.basePath + "/Visualization/dist-amd",
             main: "hpcc-viz-common"
+/*  HPCC Visualization Debug  
+        }, {
+            name: "src",
+            location: urlInfo.basePath + "/Visualization/src"
+*/
         }, {
             name: "this",
             location: urlInfo.thisPath

esp/src/eclwatch/nls/hpcc.js

@@ -83,6 +83,7 @@ define({root:
     Deactivate: "Deactivate",
     Debug: "Debug",
     DEF: "DEF",
+    Defaults: "Defaults",
     Delete: "Delete",
     Deleted: "Deleted",
     DeleteSelectedFiles: "Delete Selected Files?",
@@ -198,6 +199,7 @@ define({root:
     High: "High",
     History: "History",
     HPCCSystems: "HPCC Systems®",
+    Icon: "Icon",
     ID: "ID",
     Inactive: "Inactive",
     Index: "Index",
@@ -306,8 +308,8 @@ define({root:
     OpenInNewPage: "Open in New Page",
     OpenInNewPageNoFrame: "Open in New Page (No Frame)",
     OpenLegacyECLWatch: "Open Legacy ECL Watch",
-    OpenSafeMode: "Open (safe mode)",
-    OpenTreeMode: "Open (tree mode)",
+    OpenLegacyMode: "Open (legacy)",
+    OpenNativeMode: "Open (native)",
     OpenSource: "Open Source",
     Operations: "Operations",
     Options: "Options",
@@ -543,6 +545,7 @@ define({root:
     To: "To",
     ToDate: "To Date",
     Toenablegraphviews: "To enable graph views, please install the Graph View Control plugin",
+    Tooltip: "Tooltip",
     Top: "Top",
     Topology: "Topology",
     ToSizes: "To Sizes",

esp/src/eclwatch/templates/GraphTreeWidget.html

@@ -1,29 +1,31 @@
 <div class="${baseClass}">
     <div id="${id}BorderContainer" class="${baseClass}BorderContainer" style="width: 100%; height: 100%" data-dojo-type="dijit.layout.BorderContainer">
-        <div id="${id}Toolbar" class="topPanel" data-dojo-props="region: 'top'" data-dojo-type="dijit.Toolbar">
-            <div id="${id}Refresh" data-dojo-attach-event="onClick:_onRefresh" data-dojo-props="iconClass:'iconRefresh'" data-dojo-type="dijit.form.Button">${i18n.Refresh}</div>
-            <span data-dojo-type="dijit.ToolbarSeparator"></span>
-            <div id="${id}FindField" style="width: 120px" data-dojo-props="placeHolder:'${i18n.Find}'" data-dojo-type="dijit.form.TextBox">${i18n.Find}</div>
-            <div id="${id}Find" data-dojo-attach-event="onClick:_onFind" data-dojo-props="iconClass:'iconFind', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Find}</div>
-            <div id="${id}FindPrevious" data-dojo-attach-event="onClick:_onFindPrevious" data-dojo-props="iconClass:'iconLeft', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.FindPrevious}</div>
-            <div id="${id}FindNext" data-dojo-attach-event="onClick:_onFindNext" data-dojo-props="iconClass:'iconRight', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.FindNext}</div>
-            <span data-dojo-type="dijit.ToolbarSeparator"></span>
-            <div id="${id}AdvancedMenu" data-dojo-type="dijit.form.DropDownButton">
-                <span>${i18n.Advanced}</span>
-                <div data-dojo-type="dijit.Menu" >
-                    <div id="${id}GetSVG" data-dojo-attach-event="onClick:_onGetSVG" data-dojo-type="dijit.MenuItem">${i18n.ShowSVG}</div>
-                    <div id="${id}RenderSVG" data-dojo-attach-event="onClick:_onRenderSVG" data-dojo-type="dijit.MenuItem">${i18n.RenderSVG}</div>
-                    <div id="${id}GetXGMML" data-dojo-attach-event="onClick:_onGetXGMML" data-dojo-type="dijit.MenuItem">${i18n.EditXGMML}</div>
-                    <div id="${id}EditDOT" data-dojo-attach-event="onClick:_onEditDOT" data-dojo-type="dijit.MenuItem">${i18n.EditDOT}</div>
-                    <span data-dojo-type="dijit.MenuSeparator"></span>
-                    <div id="${id}GetGraphAttributes" data-dojo-attach-event="onClick:_onGetGraphAttributes" data-dojo-type="dijit.MenuItem">${i18n.EditGraphAttributes}</div>
-                    <span data-dojo-type="dijit.MenuSeparator"></span>
-                    <div id="${id}About" data-dojo-attach-event="onClick:_onAbout" data-dojo-type="dijit.MenuItem">${i18n.AboutGraphControl}</div>
+        <div id="${id}ToolbarContentPane" class="${baseClass}ToolbarContentPane" style="padding: 0px; overflow: hidden" data-dojo-props="region: 'top'" data-dojo-type="dijit.layout.ContentPane">
+            <div id="${id}Toolbar" class="topPanel dijit dijitToolbar" role="toolbar">
+                <div id="${id}Refresh" data-dojo-attach-event="onClick:_onRefresh" data-dojo-props="iconClass:'iconRefresh'" data-dojo-type="dijit.form.Button">${i18n.Refresh}</div>
+                <span data-dojo-type="dijit.ToolbarSeparator"></span>
+                <div id="${id}FindField" style="width: 120px" data-dojo-props="placeHolder:'${i18n.Find}'" data-dojo-type="dijit.form.TextBox">${i18n.Find}</div>
+                <div id="${id}Find" data-dojo-attach-event="onClick:_onFind" data-dojo-props="iconClass:'iconFind', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Find}</div>
+                <div id="${id}FindPrevious" data-dojo-attach-event="onClick:_onFindPrevious" data-dojo-props="iconClass:'iconLeft', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.FindPrevious}</div>
+                <div id="${id}FindNext" data-dojo-attach-event="onClick:_onFindNext" data-dojo-props="iconClass:'iconRight', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.FindNext}</div>
+                <span data-dojo-type="dijit.ToolbarSeparator"></span>
+                <div id="${id}AdvancedMenu" data-dojo-type="dijit.form.DropDownButton">
+                    <span>${i18n.Advanced}</span>
+                    <div data-dojo-type="dijit.Menu">
+                        <div id="${id}GetSVG" data-dojo-attach-event="onClick:_onGetSVG" data-dojo-type="dijit.MenuItem">${i18n.ShowSVG}</div>
+                        <div id="${id}RenderSVG" data-dojo-attach-event="onClick:_onRenderSVG" data-dojo-type="dijit.MenuItem">${i18n.RenderSVG}</div>
+                        <div id="${id}GetXGMML" data-dojo-attach-event="onClick:_onGetXGMML" data-dojo-type="dijit.MenuItem">${i18n.EditXGMML}</div>
+                        <div id="${id}EditDOT" data-dojo-attach-event="onClick:_onEditDOT" data-dojo-type="dijit.MenuItem">${i18n.EditDOT}</div>
+                        <span data-dojo-type="dijit.MenuSeparator"></span>
+                        <div id="${id}GetGraphAttributes" data-dojo-attach-event="onClick:_onGetGraphAttributes" data-dojo-type="dijit.MenuItem">${i18n.EditGraphAttributes}</div>
+                        <span data-dojo-type="dijit.MenuSeparator"></span>
+                        <div id="${id}About" data-dojo-attach-event="onClick:_onAbout" data-dojo-type="dijit.MenuItem">${i18n.AboutGraphControl}</div>
+                    </div>
                 </div>
+                <span data-dojo-type="dijit.ToolbarSeparator"></span>
+                <label id="${id}Warning"></label>
+                <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
             </div>
-            <span data-dojo-type="dijit.ToolbarSeparator"></span>
-            <label id="${id}Warning"></label>
-            <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
         </div>
         <div id="${id}MainGraphWidget" data-dojo-props="region: 'center'" data-dojo-type="${graphType}">
         </div>
@@ -64,11 +66,10 @@
     </div>
     <div id="${id}XGMMLDialog" title="${i18n.XGMML}" data-dojo-type="dijit.Dialog">
         <div class="dijitDialogPaneContentArea">
-            <textarea id="${id}XGMMLTextArea" rows="25" cols="80" data-dojo-type="dijit.form.SimpleTextarea">
-            </textarea>
+            <textarea id="${id}XGMMLTextArea" rows="25" cols="80" data-dojo-type="dijit.form.SimpleTextarea"></textarea>
         </div>
         <div class="dijitDialogPaneActionBar">
-            <button id="${id}XGMMLDialogApply" type="submit" data-dojo-type="dijit.form.Button" >${i18n.Apply}</button>
+            <button id="${id}XGMMLDialogApply" type="submit" data-dojo-type="dijit.form.Button">${i18n.Apply}</button>
             <button id="${id}XGMMLDialogCancel" type="button" data-dojo-type="dijit.form.Button">${i18n.Cancel}</button>
         </div>
     </div>

esp/src/eclwatch/templates/GraphWidget.html

@@ -20,15 +20,45 @@
                 </select>
                 <span data-dojo-type="dijit.ToolbarSeparator"></span>
                 <span data-dojo-attach-point="containerNode"></span>
-                <div title="${i18n.DepthTooltip}" data-dojo-props="iconClass:'iconDepth', showLabel:false, disabled: true" data-dojo-type="dijit.form.Button"></div>
+                <div id="${id}DepthLabel" title="${i18n.DepthTooltip}" data-dojo-props="iconClass:'iconDepth', showLabel:false, disabled: true" data-dojo-type="dijit.form.Button"></div>
                 <input id="${id}Depth" style="width: 60px" value="2" title="${i18n.DepthTooltip}" data-dojo-attach-event="onChange:_onDepthChange" data-dojo-props="intermediateChanges:true, constraints:{min:0,max:1000}" data-dojo-type="dijit.form.NumberSpinner" />
                 <div id="${id}DistanceLabel" title="${i18n.DistanceTooltip}" data-dojo-props="iconClass:'iconDistance', showLabel:false, disabled: true" data-dojo-type="dijit.form.Button"></div>
                 <input id="${id}Distance" style="width: 60px" value="2" title="${i18n.DistanceTooltip}" data-dojo-attach-event="onChange:_onDistanceChange" data-dojo-props="intermediateChanges:true, constraints:{min:0,max:1000}" data-dojo-type="dijit.form.NumberSpinner" />
                 <span id="${id}SyncSelectionSplitter" data-dojo-type="dijit.ToolbarSeparator"></span>
                 <div id="${id}SyncSelection" data-dojo-attach-event="onClick:_onSyncSelection" data-dojo-props="iconClass:'iconSync', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.ResetViewToSelection}</div>
+                <span data-dojo-type="dijit.ToolbarSeparator"></span>
+                <div id="${id}OptionsDropDown" data-dojo-type="dijit.form.DropDownButton">
+                    <span>${i18n.Options}</span>
+                    <div data-dojo-type="dijit.TooltipDialog">
+                        <div id="${id}OptionsForm" style="width: 530px;" onsubmit="return false;" data-dojo-type="dijit.form.Form">
+                            <div data-dojo-type="hpcc.TableContainer">
+                                <input id="${id}showSubgraphs" title="${i18n.Subgraphs}:" name="subgraph" data-dojo-type="dijit.form.CheckBox" />
+                            </div>
+                            <div data-dojo-type="dijit.Fieldset">
+                                <legend>${i18n.Activities}</legend>
+                                <div data-dojo-type="hpcc.TableContainer">
+                                    <input title="${i18n.Icon}:" name="vicon" checked data-dojo-type="dijit.form.CheckBox" />
+                                    <input title="${i18n.Label}:" name="vlabel" value="%label%" style="width: 95%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.TextBox" />
+                                    <input title="${i18n.Tooltip}:" name="vtooltip" value="%ecl%" style="width: 95%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.TextBox" />
+                                </div>
+                            </div>
+                            <div data-dojo-type="dijit.Fieldset">
+                                <legend>${i18n.Edges}</legend>
+                                <div data-dojo-type="hpcc.TableContainer">
+                                    <input title="${i18n.Label}:" name="elabel" value="%label%\n%_eclwatchCount%\n%_eclwatchInputProgress%\n%_eclwatchSkew%" style="width: 95%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.TextBox" />
+                                    <input title="${i18n.Tooltip}:" name="etooltip" value="" style="width: 95%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.TextBox" />
+                                </div>
+                            </div>
+                            <div class="dijitDialogPaneActionBar">
+                                <button type="submit" data-dojo-attach-event="onClick:_onOptionsApply" data-dojo-type="dijit.form.Button">${i18n.Apply}</button>
+                                <button data-dojo-attach-event="onClick:_onOptionsReset" data-dojo-type="dijit.form.Button">${i18n.Defaults}</button>
+                            </div>
+                        </div>
+                    </div>
+                </div>
             </div>
         </div>
-        <div id="${id}GraphContentPane" class="${baseClass}GraphContentPane" style="padding: 0px; overflow: hidden; background-color: #707070" data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
+        <div id="${id}GraphContentPane" class="${baseClass}GraphContentPane" style="padding: 0px; overflow: hidden" data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
         </div>
     </div>
 </div>

plugins/CMakeLists.txt

@@ -33,3 +33,4 @@ add_subdirectory (Rembed)
 add_subdirectory (cassandra)
 add_subdirectory (memcached)
 add_subdirectory (redis)
+add_subdirectory (kafka)

plugins/kafka/CMakeLists.txt

@@ -0,0 +1,127 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# Component: kafka
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for kafka
+#####################################################
+
+project( kafka )
+
+if (USE_KAFKA)
+
+    ADD_PLUGIN(kafka PACKAGES OPTION MAKE_KAFKA)
+
+    if ( MAKE_KAFKA )
+
+        # librdkafka packages do not include everything we need to properly
+        # build against it; until/if they are ever fixed we will need to build
+        # our own version of librdkafka from source and include it ourselves
+
+        if( NOT EXISTS "${PROJECT_SOURCE_DIR}/librdkafka/configure" )
+            message( FATAL_ERROR
+"   The librdkafka submodule is not available.
+   This normally indicates that the git submodule has not been fetched.
+   Please run git submodule update --init --recursive")
+        endif()
+
+        if (APPLE)
+            set ( LIBRDKAFKA_LIB ${PROJECT_BINARY_DIR}/lib/librdkafka.dylib )
+            set ( LIBRDKAFKA_LIB_REAL ${PROJECT_BINARY_DIR}/lib/librdkafka.1.dylib )
+            set ( LIBRDKAFKACPP_LIB ${PROJECT_BINARY_DIR}/lib/librdkafka++.dylib )
+            set ( LIBRDKAFKACPP_LIB_REAL ${PROJECT_BINARY_DIR}/lib/librdkafka++.1.dylib )
+        else()
+            set ( LIBRDKAFKA_LIB ${PROJECT_BINARY_DIR}/lib/librdkafka.so )
+            set ( LIBRDKAFKA_LIB_REAL ${PROJECT_BINARY_DIR}/lib/librdkafka.so.1 )
+            set ( LIBRDKAFKACPP_LIB ${PROJECT_BINARY_DIR}/lib/librdkafka++.so )
+            set ( LIBRDKAFKACPP_LIB_REAL ${PROJECT_BINARY_DIR}/lib/librdkafka++.so.1 )
+        endif()
+
+        # librdkafka does not support out-of-source builds, so let's copy all
+        # of its source code to our binary directory and build there; further,
+        # we need to pull some working directory shenanigans for each command
+        # in order to make the built scripts function correctly
+
+        add_custom_command ( OUTPUT ${LIBRDKAFKA_LIB}
+                COMMAND cp -r ${PROJECT_SOURCE_DIR}/librdkafka ${PROJECT_BINARY_DIR}/src
+                COMMAND cd ${PROJECT_BINARY_DIR}/src && ./configure --prefix=${PROJECT_BINARY_DIR}
+                COMMAND cd ${PROJECT_BINARY_DIR}/src && make && make install
+                COMMENT Copying and building librdkafka
+            )
+
+        add_custom_target ( librdkafka-build ALL DEPENDS ${LIBRDKAFKA_LIB} )
+
+        # Add both libraries from librdkafka
+
+        add_library ( librdkafka SHARED IMPORTED )
+        set_property ( TARGET librdkafka PROPERTY IMPORTED_LOCATION ${LIBRDKAFKA_LIB} )
+        add_dependencies ( librdkafka librdkafka-build )
+
+        add_library ( librdkafkacpp STATIC IMPORTED )
+        set_property ( TARGET librdkafkacpp PROPERTY IMPORTED_LOCATION ${LIBRDKAFKACPP_LIB} )
+        add_dependencies ( librdkafkacpp librdkafka-build )
+
+        set (   SRCS
+                kafka.hpp
+                kafka.cpp
+            )
+
+        include_directories (
+                ./../../system/include
+                ./../../rtl/eclrtl
+                ./../../rtl/include
+                ./../../common/deftype
+                ./../../system/jlib
+                ${PROJECT_BINARY_DIR}/include
+                ${CMAKE_BINARY_DIR}
+            )
+
+        ADD_DEFINITIONS( -D_USRDLL -DECL_KAFKA_EXPORTS )
+        HPCC_ADD_LIBRARY( kafka SHARED ${SRCS} )
+
+        if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
+            message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+        elseif(NOT APPLE)
+            set_target_properties( kafka PROPERTIES NO_SONAME 1 )
+        endif()
+
+        install ( TARGETS kafka
+                DESTINATION plugins
+            )
+
+        # Install our built librdkafka libraries into the RPM
+        install ( FILES ${LIBRDKAFKA_LIB} ${LIBRDKAFKA_LIB_REAL} ${LIBRDKAFKACPP_LIB} ${LIBRDKAFKACPP_LIB_REAL}
+                DESTINATION ${LIB_DIR}
+                COMPONENT Runtime
+            )
+
+        target_link_libraries ( kafka
+                librdkafka
+                librdkafkacpp
+                eclrtl
+                jlib
+                ${ZLIB_LIBRARIES}
+            )
+
+    endif()
+
+endif()
+
+#Even if not making the kafka plugin, we want to install the header
+install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/kafka.ecllib DESTINATION plugins COMPONENT Runtime)

plugins/kafka/README.md

@@ -0,0 +1,359 @@
+#ECL Apache Kafka Plugin
+
+This is the ECL plugin to access [Apache Kafka](https://kafka.apache.org), a
+publish-subscribe messaging system.  ECL string data can be both published to
+and consumed from Apache Kafka brokers.
+
+Client access is via a third-party C++ plugin,
+[librdkafka](https://github.com/edenhill/librdkafka).
+
+##Installation and Dependencies
+
+[librdkafka](https://github.com/edenhill/librdkafka) is included as a git
+submodule in HPCC-Platform.  It will be built and integrated automatically when
+you build the HPCC-Platform project.
+
+The recommended method for obtaining Apache Kafka is via
+[download](https://kafka.apache.org/downloads.html).
+
+Note that Apache Kafka has its own set of dependencies, most notably
+[zookeeper](https://zookeeper.apache.org).  The Kafka download file does contain
+a Zookeeper installation, so for testing purposes you need to download only
+Apache Kafka and follow the excellent
+[instructions](https://kafka.apache.org/documentation.html#quickstart).  Those
+instructions will tell you how to start Zookeeper and Apache Kafka, then test
+your installation by creating a topic and interacting with it.
+
+*Note:* Apache Kafka version 0.8.2 or later is recommended.
+
+##Plugin Configuration
+
+The Apache Kafka plugin uses sensible default configuration values but these can
+be modified via configuration files.
+
+There are two types of configurations:  Global and per-topic.  Some
+configuration parameters are applicable only to publishers (producers, in Apache
+Kafka's terminology), others only to consumers, and some to both.  Details on
+the supported configuration parameters can be found on the [librdkafka
+configuration
+page](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
+
+A configuration file is a simple text document with a series of key/value
+parameters, formatted like:
+
+    key=value
+    key=value
+    ...
+    key=value
+
+A '#' character at the beginning of a line denotes a comment.  Note that this is
+the only kind of comment supported in configuration files.
+
+Whenever a new connection is created (either publisher or consumer) the plugin
+will scan for configuration files.  All configuration files will reside in the
+HPCC configuration directory, which is `/etc/HPCCSystems`.  The global
+configuration file should be named `kafka_global.conf`.  Per-topic configuration
+files are also supported, and they can be different for a publisher or a
+consumer.  For a publisher, the naming convention is
+`kafka_publisher_topic_<TopicName>.conf` and for a consumer it is
+`kafka_consumer_topic_<TopicName>.conf`.  In both cases, `<TopicName>` is the
+name of the topic you are publishing to or consuming from.
+
+Configuration parameters loaded from a file override those set by the plugin
+with one exception:  the `metadata.broker.list` setting, if found in a
+configuration file, is ignored.  Apache Kafka brokers are always set in ECL.
+
+The following configuration parameters are set by the plugin for publishers,
+overriding their normal default values:
+
+    queue.buffering.max.messages=1000000
+    compression.codec=snappy
+    message.send.max.retries=3
+    retry.backoff.ms=500
+
+The following configuration parameters are set by the plugin for consumers,
+overriding their normal default values:
+
+    compression.codec=snappy
+    queued.max.messages.kbytes=10000000
+    fetch.message.max.bytes=10000000
+    auto.offset.reset=smallest
+
+##Publishing messages with the plugin
+
+Publishing string messages begins with instantiating an ECL module that defines
+the Apache Kafka cluster and the topic into which the messages will be posted. 
+The definition of the module is:
+
+    Publisher(VARSTRING topic, VARSTRING brokers = 'localhost') := MODULE
+        ...
+    END
+
+The module requires you to designate a topic by name and, optionally, at least
+one Apache Kafka broker.  The format of the broker is `BrokerName[:port]` where
+`BrokerName` is either an IP address or a DNS name of a broker.  You can
+optionally include a port number if the default Apache Kafka broker port is not
+used.  Multiple brokers can be listed, separated by commas.  Only one broker in
+an Apache Kafka cluster is required; the rest can be discovered once a
+connection is made.
+
+Example instantiating a publishing module:
+
+    p := kafka.Publisher('MyTopic', '10.211.55.13');
+
+The module contains an exported function for publishing a message, defined as:
+
+    BOOLEAN PublishMessage(CONST VARSTRING message, CONST VARSTRING key = '');
+
+The module function requires a string message and allows you to specify a 'key'
+that affects how Apache Kafka stores the message.  Key values act a lot like the
+expression argument in ECL's DISTRIBUTE() function:  Messages with the same key
+value wind up on the same Apache Kafka partition within the topic.  This can
+affect how consumers retrieve the published messages.  More details regarding
+partitions and how keys are used can be found Apache Kafka's
+[introduction](https://kafka.apache.org/documentation.html#introduction).  If a
+key value is not supplied than the messages are distributed among the available
+partitions for that topic.
+
+Examples:
+
+    p.PublishMessage('This is a test message');
+    p.PublishMessage('A keyed message', 'MyKey');
+    p.PublishMessage('Another keyed message', 'MyKey');
+
+Note that keys are not retrieved by the ECL Apache Kafka consumer.  They are
+used only to determine how the messages are stored.
+
+You can find out how many partitions are available in a publisher's topic by
+calling the following module function:
+
+    partitionCount := p.GetTopicPartitionCount();
+
+`GetTopicPartitionCount()` returns zero if the topic has not been created or
+there are has been an error.
+
+##Consuming messages with the plugin
+
+As with publishing, consuming string messages begins with instantiating an ECL
+module that defines the Apache Kafka cluster and the topic from which the
+messages will be read.  The definition of the module is:
+
+    Consumer(VARSTRING topic,
+             VARSTRING brokers = 'localhost',
+             VARSTRING consumerGroup = 'hpcc') := MODULE
+        ...
+    END
+
+The module requires you to designate a topic by name.  Optionally, you may also
+cite at least one Apache Kafka broker and a consumer group.  The format and
+requirements for a broker are the same as for instantiating a Producer module. 
+Consumer groups in Apache Kafka allow multiple consumer instances, like Thor
+nodes, to form a "logical consumer" and be able to retrieve messages in parallel
+and without duplication.  See the "Consumers" subtopic in Apache Kafka's
+[introduction](https://kafka.apache.org/documentation.html#introduction) for
+more details.
+
+Example:
+
+    c := kafka.Consumer('MyTopic', '10.211.55.13');
+
+The module contains an exported function for consuming messages, defined as:
+
+    DATASET(KafkaMessage) GetMessages(INTEGER4 maxRecords);
+
+This function returns a new dataset containing messages consumed by the topic
+defined in the module.  The layout for that dataset is:
+
+    KafkaMessage := RECORD
+        UNSIGNED4   partition;
+        INTEGER8    offset;
+        STRING      message;
+    END;
+
+Example retrieving up to 10,000 messages:
+
+    myMessages := c.GetMessages(10000);
+
+After you consume some messages it may be beneficial to track the last-read
+offset from each Apache Kafka topic partition.  The following module function
+does that:
+
+    DATASET(KafkaMessageOffset) LastMessageOffsets(DATASET(KafkaMessage) messages);
+
+Basically, you pass in the just-consumed message dataset to the function and get
+back a small dataset containing just the partition numbers and the last-read
+message's offset.  The layout of the returned dataset is:
+
+    KafkaMessageOffset := RECORD
+        UNSIGNED4   partitionNum;
+        INTEGER8    offset;
+    END;
+
+Example call:
+
+    myOffsets := c.LastMessageOffsets(myMessages);
+
+If you later find out that you need to "rewind" your consumption -- read old
+messages, in other words -- you can use the data within a KafkaMessageOffset
+dataset to reset your consumers, making the next `GetMessages()` call pick up
+from that point.  Use the following module function to reset the offsets:
+
+    UNSIGNED4 SetMessageOffsets(DATASET(KafkaMessageOffset) offsets);
+
+The function returns the number of partitions reset (which should equal the
+number of records you're handing the function).
+
+Example call:
+
+    numPartitionsReset := c.SetMessageOffsets(myOffsets);
+
+You can easily reset all topic partitions to their earliest point with the
+following module function:
+
+    UNSIGNED4 ResetMessageOffsets();
+
+This function returns the number of partitions reset.
+
+Example call:
+
+    numPartitionsReset := c.ResetMessageOffsets();
+
+You can find out how many partitions are available in a consumers's topic by
+calling the following module function:
+
+    partitionCount := c.GetTopicPartitionCount();
+
+`GetTopicPartitionCount()` returns zero if the topic has not been created or
+there are has been an error.
+
+##Complete ECL Examples
+
+The following code will publish 100K messages to a topic named 'MyTestTopic' on
+an Apache Kafka broker located at address 10.211.55.13.  If you are running a
+single-node HPCC cluster and have installed Kafka on the same node, you can use
+'localhost' instead (or omit the parameter, as it defaults to 'localhost').
+
+###Publishing
+
+    IMPORT kafka;
+
+    MyDataLayout := RECORD
+        STRING  message;
+    END;
+
+    ds := DATASET
+        (
+            100000,
+            TRANSFORM
+                (
+                    MyDataLayout,
+                    SELF.message := 'Test message ' + (STRING)COUNTER
+                ),
+            DISTRIBUTED
+        );
+
+    p := kafka.Publisher('MyTestTopic', brokers := '10.211.55.13');
+
+    APPLY(ds, ORDERED(p.PublishMessage(message)));
+
+###Consuming
+
+This code will read the messages written by the publishing example, above.  It
+will also show the number of partitions in the topic and the offsets of the
+last-read messages.
+
+    IMPORT kafka;
+
+    c := kafka.Consumer('MyTestTopic', brokers := '10.211.55.13');
+
+    ds := c.GetMessages(200000);
+    offsets := c.LastMessageOffsets(ds);
+    partitionCount := c.GetTopicPartitionCount();
+
+    OUTPUT(ds, NAMED('MessageSample'));
+    OUTPUT(COUNT(ds), NAMED('MessageCount'));
+    OUTPUT(offsets, NAMED('LastMessageOffsets'));
+    OUTPUT(partitionCount, NAMED('PartitionCount'));
+
+###Resetting Offsets
+
+Resetting offsets is useful when you have a topic already published with
+messages and you need to reread its messages from the very beginning.
+
+    IMPORT kafka;
+
+    c := kafka.Consumer('MyTestTopic', brokers := '10.211.55.13');
+
+    c.ResetMessageOffsets();
+
+##Behaviour and Implementation Details
+
+###Partitioning within Apache Kafka Topics
+
+Topic partitioning is covered in Apache Kafka's
+[introduction](https://kafka.apache.org/documentation.html#introduction).  There
+is a performance relationship between the number of partitions in a topic and
+the size of the HPCC cluster when consuming messages.  Ideally, the number of
+partitions will exactly equal the number of HPCC nodes consuming messages.  For
+Thor, this means the total number of slaves rather than the number of nodes, as
+that can be different in a multi-slave setup.  For Roxie, the number is always
+one.  If there are fewer partitions than nodes (slaves) then not all of your
+cluster will be utilized when consuming messages; if there are more partitions
+than nodes (slaves) then some nodes will be performing extra work, consuming
+from multiple partitions.  In either mismatch case, you may want to consider
+using the ECL DISTRIBUTE() function to redistribute your data before processing.
+
+When messages are published without a 'key' argument to a topic that has more
+than one partition, Apache Kafka will distribute those messages among the
+partitions.  The distribution is not perfect.  For example, if you publish 20
+messages to a topic with two partitions, one partition may wind up with 7
+messages and the other with 13 (or some other mix of message counts that total
+20).  When testing your code, be aware of this behavior and always request more
+messages than you publish.  In the examples above, 100K messages were published
+but up to 200K messages were requested.  This ensures that you receive all of
+the messages you publish.  This is typically not an issue in a production
+environment, as your requested consumption message count is more a function of
+how much data you're willing to process in one step than with how many messages
+are actually stored in the topic.
+
+Be aware that, by default, Apache Kafka will automatically create a topic that
+has never been seen before if someone publishes to it, and that topic will have
+only one partition.  Both actions -- whether a topic is automatically created
+and how many partitions it will have -- are configurable within Apache Kafka.
+
+###Publisher Connections
+
+This plugin caches the internal publisher objects and their connections. 
+Publishing from ECL, technically, only writes the messages to a local cache. 
+Those messages are batched and set to Apache Kafka for higher performance in a
+background thread.  Because this batching can extend far beyond the time ECL
+spends sending the data to the local cache, the objects (and their connections)
+need to hang around for some additional time.  The upside is that the cached
+objects and connections will be reused for subsequent publish operations,
+speeding up the entire process.
+
+###Consumer Connections
+
+Unlike publisher objects, one consumer object is created per thread for each
+connection.  A connection is to a specific broker, topic, consumer group, and
+partition number combination.  The consumer objects and connections live only as
+long as needed.
+
+###Saved Topic Offsets
+
+By default, consumers save to a file the offset of the last-read message from a
+given topic, consumer group, and partition combination.  The offset is saved so
+that the next time the consumer is fired up for that particular connection
+combination, the consumption process can pick up where it left off.  The file is
+saved to the HPCC engine's data directory which is typically
+`/var/lib/HPCCSystems/mythor/`, `/var/lib/HPCCSystems/myroxie/` or
+`/var/lib/HPCCSystems/myeclagent/` depending on the engine you're using (the
+exact path may be different if you have named an engine differently in your HPCC
+configuration).  The format of the saved offset filename is
+`<TopicName>-<PartitionNum>-<ConsumerGroup>.offset`.
+
+Note that saving partition offsets is engine-specific.  One practical
+consideration of this is that you cannot have one engine (e.g. Thor) consume
+from a given topic and then have another engine (e.g. Roxie) consume the next
+set of messages from that topic.  Both engines can consume messages without a
+problem, but they will not track each other's last-read positions.

plugins/kafka/kafka.ecllib

@@ -0,0 +1,255 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the License);
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an AS IS BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+// Record structure containing message offset positioning
+EXPORT KafkaMessageOffset := RECORD
+    UNSIGNED4   partitionNum;
+    INTEGER8    offset;
+END;
+
+// Record structure that will be used to return Kafka messages to ECL
+EXPORT KafkaMessage := RECORD
+    KafkaMessageOffset;
+    STRING      message;
+END;
+
+// Service definition
+SHARED kafka := SERVICE : plugin('kafka'), namespace('KafkaPlugin')
+
+    BOOLEAN PublishMessage(CONST VARSTRING brokers, CONST VARSTRING topic, CONST VARSTRING message, CONST VARSTRING key) : cpp,action,context,entrypoint='publishMessage';
+    INTEGER4 getTopicPartitionCount(CONST VARSTRING brokers, CONST VARSTRING topic) : cpp,action,context,entrypoint='getTopicPartitionCount';
+    STREAMED DATASET(KafkaMessage) GetMessageDataset(CONST VARSTRING brokers, CONST VARSTRING topic, CONST VARSTRING consumerGroup, INTEGER4 partitionNum, INTEGER8 maxRecords) : cpp,action,context,entrypoint='getMessageDataset';
+    INTEGER8 SetMessageOffset(CONST VARSTRING brokers, CONST VARSTRING topic, CONST VARSTRING consumerGroup, INTEGER4 partitionNum, INTEGER8 newOffset) : cpp,action,context,entrypoint='setMessageOffset';
+
+END;
+
+/**
+ * Module wrapping message publishing functions.
+ *
+ * @param   topic           The name of the topic this module will be publishing to;
+ *                          cannot be an empty string; REQUIRED
+ * @param   brokers         One or more Kafka broker; each broker should be in the
+ *                          form 'Name[:port]' where 'Name' may be either a DNS name
+ *                          or an IP address; multiple brokers should be delimited
+ *                          with a comma; brokers can also be set in the
+ *                          kafka_global.conf configuration file, in which case
+ *                          you should pass an empty string; OPTIONAL,
+ *                          defaulting to 'localhost'
+ */
+EXPORT Publisher(VARSTRING topic, VARSTRING brokers = 'localhost') := MODULE
+
+    /**
+     * Get the number of partitions currently set up for this topic
+     *
+     * @return  The number of partitions or zero if either the topic does not
+     *          exist or there was an error
+     */
+    EXPORT INTEGER4 GetTopicPartitionCount() := kafka.getTopicPartitionCount(brokers, topic);
+
+    /**
+     * Queue one message for publishing to the current Kafka topic
+     *
+     * @param   message     The message to publish; must not be an empty string;
+     *                      REQUIRED
+     * @param   key         A key to attach to the message, used by Kafka to
+     *                      route the message to a particular partition (keys
+     *                      with the same value wind up on the same partition);
+     *                      an empty string indicates no key value; OPTIONAL,
+     *                      defaults to an empty string
+     *
+     * @return  TRUE
+     */
+    EXPORT BOOLEAN PublishMessage(CONST VARSTRING message, CONST VARSTRING key = '') := kafka.PublishMessage(brokers, topic, message, key);
+
+END;
+
+/**
+ * Module wrapping message consuming functions.
+ *
+ * @param   topic           The name of the topic this module will be publishing to;
+ *                          cannot be an empty string; REQUIRED
+ * @param   brokers         One or more Kafka broker; each broker should be in the
+ *                          form 'Name[:port]' where 'Name' may be either a DNS name
+ *                          or an IP address; multiple brokers should be delimited
+ *                          with a comma; brokers can also be set in the
+ *                          kafka_global.conf configuration file, in which case
+ *                          you should pass an empty string; OPTIONAL,
+ *                          defaulting to 'localhost'
+ * @param   consumerGroup   The name of the Kafka consumer group to use for any
+ *                          message consumption;
+ *                          (see https://kafka.apache.org/documentation.html#introduction);
+ *                          OPTIONAL, defaults to 'hpcc'
+ */
+EXPORT Consumer(VARSTRING topic, VARSTRING brokers = 'localhost', VARSTRING consumerGroup = 'hpcc') := MODULE
+
+    /**
+     * Get the number of partitions currently set up for this topic
+     *
+     * @return  The number of partitions or zero if either the topic does not
+     *          exist or there was an error
+     */
+    EXPORT INTEGER4 GetTopicPartitionCount() := kafka.getTopicPartitionCount(brokers, topic);
+
+    /**
+     * Consume previously-published messages from the current topic.
+     *
+     * @param   maxRecords  The maximum number of records to retrieve; pass
+     *                      zero to return as many messages as there are
+     *                      queued (dangerous); REQUIRED
+     *
+     * @return  A new dataset containing the retrieved messages
+     */
+    EXPORT DATASET(KafkaMessage) GetMessages(INTEGER8 maxRecords) := FUNCTION
+
+        // Record structure to hold messages from multiple partitions
+        MultiNodeMessageRec := RECORD
+            DATASET(KafkaMessage)   messages;
+        END;
+
+        numberOfPartitions := GetTopicPartitionCount() : INDEPENDENT;
+        maxRecordsPerNode := MAX(maxRecords DIV numberOfPartitions, 1);
+
+        // Container holding messages from all partitions; in a multi-node setup
+        // the work will be distributed among the nodes (at least up to the
+        // number of partitions); note that 'COUNTER - 1' is actually the
+        // Kafka partition number that will be read
+        messageContainer := DATASET
+            (
+                numberOfPartitions,
+                TRANSFORM
+                    (
+                        MultiNodeMessageRec,
+                        SELF.messages := kafka.GetMessageDataset(brokers, topic, consumerGroup, COUNTER - 1, maxRecordsPerNode)
+                    ),
+                DISTRIBUTED
+            );
+
+        // Map messages from multiple partitions back to final record structure
+        resultDS := NORMALIZE
+            (
+                messageContainer,
+                LEFT.messages,
+                TRANSFORM
+                    (
+                        KafkaMessage,
+                        SELF := RIGHT
+                    ),
+                LOCAL
+            );
+
+        RETURN resultDS;
+
+    END;
+
+    /**
+     * Given a set of messages, presumably just consumed from an Apache Kafka
+     * cluster, summarize the last-read message offsets on a per-partition basis.
+     * This is useful for logging/saving the last messages read during a
+     * particular run, which can then be used to restore system state if you
+     * have to re-consume older messages (see SetMessageOffsets() function).
+     *
+     * @param   messages    A dataset of consumed messages; REQUIRED
+     *
+     * @return  A new dataset containing a summary of partitions and their
+     *          associated last-read message offsets.
+     */
+    EXPORT DATASET(KafkaMessageOffset) LastMessageOffsets(DATASET(KafkaMessage) messages) := FUNCTION
+        t := TABLE
+            (
+                messages,
+                {
+                    partitionNum,
+                    INTEGER8    offset := MAX(GROUP, offset)
+                },
+                partitionNum,
+                MERGE
+            );
+
+        f := PROJECT(t, TRANSFORM(KafkaMessageOffset, SELF := LEFT));
+
+        RETURN f;
+    END;
+
+    /**
+     * Resets the last-read partition offsets to the values in the given dataset.
+     * This is useful for "rewinding" message reading to an earlier point.  The
+     * next call to GetMessages() will start consuming at the points described
+     * in the dataset.
+     *
+     * @param   offsets     A dataset of of partitions and the offsets to which
+     *                      you want to set each, like the result from a call
+     *                      to LastMessageOffsets(); REQUIRED
+     *
+     * @return  The number of partitions set
+     */
+    EXPORT UNSIGNED4 SetMessageOffsets(DATASET(KafkaMessageOffset) offsets) := FUNCTION
+
+        // Distribute the offset data so that each partition lines up on the right node
+        distOffsets := DISTRIBUTE(offsets, partitionNum);
+
+        // Temporary result layout that will capture a COUNTER value generated
+        // by PROJECT, which in turn ensures that the LOCAL flag is actually used
+        // and our data distribution is honored (the distribution is required in
+        // order to ensure that kafka.SetMessageOffset() is called on the correct
+        // Thor nodes)
+        ResultLayout := RECORD
+            KafkaMessageOffset;
+            UNSIGNED4   c;
+        END;
+
+        // Set the offset for each partition on each node
+        result := PROJECT
+            (
+                distOffsets,
+                TRANSFORM
+                    (
+                        ResultLayout,
+                        SELF.offset := kafka.SetMessageOffset(brokers, topic, consumerGroup, LEFT.partitionNum, LEFT.offset),
+                        SELF.c := COUNTER,
+                        SELF := LEFT
+                    ),
+                LOCAL
+            );
+
+        RETURN COUNT(result(offset >= -1));
+    END;
+
+    /**
+     * Convenience function.  Resets all topic partitions to their earliest
+     * point.
+     *
+     * @return  The number of partitions reset
+     */
+    EXPORT UNSIGNED4 ResetMessageOffsets() := FUNCTION
+
+        numberOfPartitions := GetTopicPartitionCount() : INDEPENDENT;
+
+        offsets := DATASET
+            (
+                numberOfPartitions,
+                TRANSFORM
+                    (
+                        KafkaMessageOffset,
+                        SELF.partitionNum := COUNTER - 1,
+                        SELF.offset := -1
+                    )
+            );
+
+        RETURN SetMessageOffsets(offsets);
+    END;
+
+END;

plugins/kafka/kafka.hpp

@@ -0,0 +1,441 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef ECL_KAFKA_INCL
+#define ECL_KAFKA_INCL
+
+#ifdef _WIN32
+#define ECL_KAFKA_CALL _cdecl
+#ifdef ECL_KAFKA_EXPORTS
+#define ECL_KAFKA_API __declspec(dllexport)
+#else
+#define ECL_KAFKA_API __declspec(dllimport)
+#endif
+#else
+#define ECL_KAFKA_CALL
+#define ECL_KAFKA_API
+#endif
+
+#include "platform.h"
+#include "jthread.hpp"
+#include "hqlplugins.hpp"
+#include "eclrtl_imp.hpp"
+#include "eclhelper.hpp"
+
+#include <atomic>
+#include <string>
+#include <time.h>
+
+#include "librdkafka/rdkafkacpp.h"
+
+#ifdef ECL_KAFKA_EXPORTS
+extern "C"
+{
+    ECL_KAFKA_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
+}
+#endif
+
+extern "C++"
+{
+    namespace KafkaPlugin
+    {
+        class KafkaObj;
+        class Poller;
+        class Publisher;
+        class Consumer;
+        class KafkaStreamedDataset;
+
+        /** @class KafkaObj
+         *
+         *  Parent class for both Publisher and Consumer classes.  Provides
+         *  easy way for a Poller object to access either for callbacks, etc.
+         */
+        class KafkaObj
+        {
+            public:
+
+                /**
+                 * Returns a pointer to the librdkafka object that can be either
+                 * a producer or consumer.
+                 */
+                virtual RdKafka::Handle* handle() = 0;
+        };
+
+        //----------------------------------------------------------------------
+
+        /** @class Poller
+         *
+         *  Background execution of librdkafka's poll() function, which is
+         *  required in order to batch I/O.  One Poller will be created
+         *  for each Publisher and Consumer object actively used
+         */
+        class Poller : public Thread
+        {
+            public:
+
+                /**
+                 * Constructor
+                 *
+                 * @param   _parentPtr      Pointer to Publisher or Consumer object
+                 *                          that created this object
+                 * @param   _pollTimeout    The number of milliseconds to wait
+                 *                          for events within librdkafka
+                 */
+                Poller(KafkaObj* _parentPtr, __int32 _pollTimeout);
+
+                /**
+                 * Starts execution of the thread main event loop
+                 */
+                virtual void start();
+
+                /**
+                 * Stops execution of the thread main event loop.  Note that we
+                 * wait until the main event loop has actually stopped before
+                 * returning.
+                 */
+                void stop();
+
+                /**
+                 * Entry point to the thread main event loop.  Exiting this
+                 * method means that the thread should stop.
+                 */
+                virtual int run();
+
+            private:
+
+                std::atomic_bool    shouldRun;      //!< If true, we should execute our thread's main event loop
+                KafkaObj*           parentPtr;      //!< Pointer to object that started this threaded execution
+                __int32             pollTimeout;    //!< The amount of time (in ms) we give to librdkafka's poll() function
+        };
+
+        //----------------------------------------------------------------------
+
+        class Publisher : public KafkaObj, public RdKafka::EventCb, public RdKafka::DeliveryReportCb
+        {
+            public:
+
+                /**
+                 * Constructor
+                 *
+                 * @param   _brokers        One or more Kafka brokers, in the
+                 *                          format 'name[:port]' where 'name'
+                 *                          is either a host name or IP address;
+                 *                          multiple brokers can be delimited
+                 *                          with commas
+                 * @param   _topic          The name of the topic we will be
+                 *                          publishing to
+                 * @param   _pollTimeout    The number of milliseconds to wait
+                 *                          for events within librdkafka
+                 * @param   _traceLevel     Current logging level
+                 */
+                Publisher(const std::string& _brokers, const std::string& _topic, __int32 _pollTimeout, int _traceLevel);
+
+                virtual ~Publisher();
+
+                /**
+                 * @return  A pointer to the librdkafka producer object.
+                 */
+                virtual RdKafka::Handle* handle();
+
+                /**
+                 * @return  Updates the touch time and returns it.
+                 */
+                time_t updateTimeTouched();
+
+                /**
+                 * @return  The time at which this object was created
+                 */
+                time_t getTimeTouched() const;
+
+                /**
+                 * If needed, establish connection to Kafka cluster using the
+                 * parameters stored within this object.
+                 */
+                void ensureSetup();
+
+                /**
+                 * Stops the attached poller's main event loop.  This should be
+                 * called before deletion.
+                 */
+                void shutdownPoller();
+
+                /**
+                 * @return  Returns the number of messages currently waiting
+                 *          in the local outbound queue, ready for transmission
+                 *          to the Kafka cluster
+                 */
+                __int32 messagesWaitingInQueue();
+
+                /**
+                 * Send one message
+                 *
+                 * @param   message     The message to send
+                 * @param   key         The key to attach to the message
+                 */
+                void sendMessage(const std::string& message, const std::string& key);
+
+                /**
+                 * Callback function.  librdkafka will call here, outside of a
+                 * poll(), when it has interesting things to tell us
+                 *
+                 * @param   event       Reference to an Event object provided
+                 *                      by librdkafka
+                 */
+                virtual void event_cb(RdKafka::Event& event);
+
+                /**
+                 * Callback function.  librdkafka will call here to notify
+                 * us of problems with delivering messages to the server
+                 *
+                 * @param   message     Reference to an Message object provided
+                 *                      by librdkafka
+                 */
+                virtual void dr_cb (RdKafka::Message& message);
+
+            private:
+
+                std::string                     brokers;        //!< One or more Kafka bootstrap brokers; comma-delimited; NameOrIP[:port]
+                std::string                     topic;          //!< The name of the topic to publish to
+                RdKafka::Producer*              producerPtr;    //!< Pointer to librdkafka producer object
+                std::atomic<RdKafka::Topic*>    topicPtr;       //!< Pointer to librdkafka topic object
+                CriticalSection                 lock;           //!< Mutex to ensure that only one thread creates the librdkafka object pointers
+                Poller*                         pollerPtr;      //!< Pointer to the threaded Poller object that gives time to librdkafka
+                __int32                         pollTimeout;    //!< The amount of time (in ms) we give to librdkafka's poll() function
+                time_t                          timeCreated;    //!< The time at which this object was created
+                int                             traceLevel;     //!< The current logging level
+        };
+
+        //----------------------------------------------------------------------
+
+        class Consumer : public KafkaObj, public RdKafka::EventCb
+        {
+            public:
+
+                /**
+                 * Constructor
+                 *
+                 * @param   _brokers        One or more Kafka brokers, in the
+                 *                          format 'name[:port]' where 'name'
+                 *                          is either a host name or IP address;
+                 *                          multiple brokers can be delimited
+                 *                          with commas
+                 * @param   _topic          The name of the topic we will be
+                 *                          consuming from
+                 * @param   _partitionNum   The topic partition number we will be
+                 *                          consuming from
+                 * @param   _traceLevel     Current logging level
+                 */
+                Consumer(const std::string& _brokers, const std::string& _topic, const std::string& _consumerGroup, __int32 _partitionNum, int _traceLevel);
+
+                virtual ~Consumer();
+
+                /**
+                 * @return  A pointer to the librdkafka consumer object.
+                 */
+                virtual RdKafka::Handle* handle();
+
+                /**
+                 * If needed, establish connection to Kafka cluster using the
+                 * parameters stored within this object.
+                 */
+                void ensureSetup();
+
+                /**
+                 * @return  Returns one new message from the inbound Kafka
+                 *          topic.  A NON-NULL RESULT MUST EVENTUALLY BE
+                 *          DISPOSED OF WITH A CALL TO delete().
+                 */
+                RdKafka::Message* getOneMessage();
+
+                /**
+                 * Retrieves many messages from the inbound Kafka topic and
+                 * returns them as a streamed dataset.  Note that this is a
+                 * per-brokers/per-topic/per-partition retrieval.
+                 *
+                 * @param   allocator       The allocator to use with RowBuilder
+                 * @param   maxRecords      The maximum number of records
+                 *                          to retrieved
+                 *
+                 * @return  An IRowStream streamed dataset object pointer
+                 */
+                KafkaStreamedDataset* getMessageDataset(IEngineRowAllocator* allocator, __int64 maxRecords = 1);
+
+                /**
+                 * @return  StringBuffer object containing the path to this
+                 *          consumer's offset file
+                 */
+                StringBuffer offsetFilePath() const;
+
+                /**
+                 * Commits the given offset to storage so we can pick up
+                 * where we left off in a subsequent read.
+                 *
+                 * @param   offset          The offset to store
+                 */
+                void commitOffset(__int64 offset) const;
+
+                /**
+                 * If the offset file does not exist, create one with a
+                 * default offset
+                 */
+                void initFileOffsetIfNotExist() const;
+
+                /**
+                 * Callback function.  librdkafka will call here, outside of a
+                 * poll(), when it has interesting things to tell us
+                 *
+                 * @param   event       Reference to an Event object provided
+                 *                      by librdkafka
+                 */
+                virtual void event_cb(RdKafka::Event& event);
+
+            private:
+
+                std::string                     brokers;        //!< One or more Kafka bootstrap brokers; comma-delimited; NameOrIP[:port]
+                std::string                     topic;          //!< The name of the topic to consume from
+                std::string                     consumerGroup;  //!< The name of the consumer group for this consumer object
+                RdKafka::Consumer*              consumerPtr;    //!< Pointer to librdkafka consumer object
+                std::atomic<RdKafka::Topic*>    topicPtr;       //!< Pointer to librdkafka topic object
+                CriticalSection                 lock;           //!< Mutex to ensure that only one thread creates the librdkafka object pointers or starts/stops the queue
+                __int32                         partitionNum;   //!< The partition within the topic from which we will be pulling messages
+                bool                            queueStarted;   //!< If true, we have started the process of reading from the queue
+                int                             traceLevel;     //!< The current logging level
+        };
+
+        //----------------------------------------------------------------------
+
+        class KafkaStreamedDataset : public RtlCInterface, implements IRowStream
+        {
+            public:
+
+                /**
+                 * Constructor
+                 *
+                 * @param   _consumerPtr        Pointer to the Consumer object
+                 *                              from which we will be retrieving
+                 *                              records
+                 * @param   _resultAllocator    The memory allocator used to build
+                 *                              the result rows; this is provided
+                 *                              by the platform during the
+                 *                              plugin call
+                 * @param   _traceLevel         The current logging level
+                 * @param   _maxRecords         The maximum number of records
+                 *                              to return; use 0 to return all
+                 *                              available records
+                 */
+                KafkaStreamedDataset(Consumer* _consumerPtr, IEngineRowAllocator* _resultAllocator, int _traceLevel, __int64 _maxRecords = -1);
+
+                virtual ~KafkaStreamedDataset();
+
+                RTLIMPLEMENT_IINTERFACE
+
+                virtual const void* nextRow();
+
+                virtual void stop();
+
+            private:
+
+                Consumer*                       consumerPtr;        //!< Pointer to the Consumer object that we will read from
+                Linked<IEngineRowAllocator>     resultAllocator;    //!< Pointer to allocator used when building result rows
+                int                             traceLevel;         //!< The current logging level
+                bool                            shouldRead;         //!< If true, we should continue trying to read more messages
+                __int64                         maxRecords;         //!< The maximum number of messages to read
+                __int64                         consumedRecCount;   //!< The number of messages actually read
+                __int64                         lastMsgOffset;      //!< The offset of the last message read from the consumer
+        };
+
+        //----------------------------------------------------------------------
+
+        /**
+         * Queues the message for publishing to a topic on a Kafka cluster.
+         *
+         * @param   brokers             One or more Kafka brokers, in the
+         *                              format 'name[:port]' where 'name'
+         *                              is either a host name or IP address;
+         *                              multiple brokers can be delimited
+         *                              with commas
+         * @param   topic               The name of the topic
+         * @param   message             The message to send
+         * @param   key                 The key to use for the message
+         *
+         * @return  true if the message was cached successfully
+         */
+        ECL_KAFKA_API bool ECL_KAFKA_CALL publishMessage(const char* brokers, const char* topic, const char* message, const char* key);
+
+        /**
+         * Get the number of partitions currently set up for a topic on a cluster.
+         *
+         * @param   brokers             One or more Kafka brokers, in the
+         *                              format 'name[:port]' where 'name'
+         *                              is either a host name or IP address;
+         *                              multiple brokers can be delimited
+         *                              with commas
+         * @param   topic               The name of the topic
+         *
+         * @return  The number of partitions or zero if either the topic does not
+         *          exist or there was an error
+         */
+        ECL_KAFKA_API __int32 ECL_KAFKA_CALL getTopicPartitionCount(const char* brokers, const char* topic);
+
+        /**
+         * Retrieves a set of messages on a topic from a Kafka cluster.
+         *
+         * @param   ctx                 Platform-provided context point
+         * @param   allocator           Platform-provided memory allocator used
+         *                              to help build data rows for returning
+         * @param   brokers             One or more Kafka brokers, in the
+         *                              format 'name[:port]' where 'name'
+         *                              is either a host name or IP address;
+         *                              multiple brokers can be delimited
+         *                              with commas
+         * @param   topic               The name of the topic
+         * @param   consumerGroup       The name of the consumer group to use; see
+         *                              https://kafka.apache.org/documentation.html#introduction
+         * @param   partitionNum        The topic partition from which to pull
+         *                              messages; this is a zero-based index
+         * @param   maxRecords          The maximum number of records return;
+         *                              pass zero to return as many messages
+         *                              as possible (dangerous)
+         *
+         * @return  An IRowStream pointer representing the fetched messages
+         *          or NULL if no messages could be retrieved
+         */
+        ECL_KAFKA_API IRowStream* ECL_KAFKA_CALL getMessageDataset(ICodeContext* ctx, IEngineRowAllocator* allocator, const char* brokers, const char* topic, const char* consumerGroup, __int32 partitionNum, __int64 maxRecords);
+
+        /**
+         * Resets the saved offsets for a partition.
+         *
+         * @param   ctx                 Platform-provided context point
+         * @param   brokers             One or more Kafka brokers, in the
+         *                              format 'name[:port]' where 'name'
+         *                              is either a host name or IP address;
+         *                              multiple brokers can be delimited
+         *                              with commas
+         * @param   topic               The name of the topic
+         * @param   consumerGroup       The name of the consumer group to use; see
+         *                              https://kafka.apache.org/documentation.html#introduction
+         * @param   partitionNum        The topic partition from which to pull
+         *                              messages; this is a zero-based index
+         * @param   newOffset           The new offset to save
+         *
+         * @return  The offset that was saved
+         */
+        ECL_KAFKA_API __int64 ECL_KAFKA_CALL setMessageOffset(ICodeContext* ctx, const char* brokers, const char* topic, const char* consumerGroup, __int32 partitionNum, __int64 newOffset);
+    }
+}
+
+#endif

plugins/kafka/librdkafka

@@ -0,0 +1 @@
+Subproject commit 3e1babf4f26a7d12bbd272c1cdf4aa6a44000d4a

plugins/workunitservices/workunitservices.cpp

@@ -54,11 +54,9 @@ Persists changed?
 #include "workunitservices.ipp"
 #include "environment.hpp"
 
-#define WORKUNITSERVICES_VERSION "WORKUNITSERVICES 1.0.1"
+#define WORKUNITSERVICES_VERSION "WORKUNITSERVICES 1.0.2"
 
 static const char * compatibleVersions[] = {
-    "WORKUNITSERVICES 1.0 ",  // a version was released with a space here in signature... 
-    "WORKUNITSERVICES 1.0.1", 
     NULL };
 
 static const char * EclDefinition =
@@ -122,10 +120,10 @@ static const char * EclDefinition =
                             " string description;"
                             " string unit;"
                         " end;\n"
-"export WorkunitServices := SERVICE\n"
-"   boolean WorkunitExists(const varstring wuid, boolean online=true, boolean archived=false) : c,context,entrypoint='wsWorkunitExists'; \n"
+"export WorkunitServices := SERVICE : cpp\n"
+"   boolean WorkunitExists(const varstring wuid, boolean online=true, boolean archived=false) : context,entrypoint='wsWorkunitExists'; \n"
 "   dataset(WsWorkunitRecord) WorkunitList("
-                                        " const varstring lowwuid," 
+                                        " const varstring lowwuid='',"
                                         " const varstring highwuid=''," 
                                         " const varstring username=''," 
                                         " const varstring cluster=''," 
@@ -137,17 +135,18 @@ static const char * EclDefinition =
                                         " const varstring roxiecluster='',"
                                         " const varstring eclcontains='',"
                                         " boolean online=true,"
-                                        " boolean archived=false"
-                                        ") : c,context,entrypoint='wsWorkunitList'; \n"
-"  varstring WUIDonDate(unsigned4 year,unsigned4 month,unsigned4 day,unsigned4 hour, unsigned4 minute) : c,entrypoint='wsWUIDonDate'; \n"
-"  varstring WUIDdaysAgo(unsigned4  daysago) : c,entrypoint='wsWUIDdaysAgo'; \n"
-"  dataset(WsTimeStamp) WorkunitTimeStamps(const varstring wuid) : c,context,entrypoint='wsWorkunitTimeStamps'; \n"
-"  dataset(WsMessage) WorkunitMessages(const varstring wuid) : c,context,entrypoint='wsWorkunitMessages'; \n"
-"  dataset(WsFileRead) WorkunitFilesRead(const varstring wuid) : c,context,entrypoint='wsWorkunitFilesRead'; \n"
-"  dataset(WsFileWritten) WorkunitFilesWritten(const varstring wuid) : c,context,entrypoint='wsWorkunitFilesWritten'; \n"
-"  dataset(WsTiming) WorkunitTimings(const varstring wuid) : c,context,entrypoint='wsWorkunitTimings'; \n"
-"  streamed dataset(WsStatistic) WorkunitStatistics(const varstring wuid, boolean includeActivities = false, const varstring _filter = '') : c,context,entrypoint='wsWorkunitStatistics'; \n"
-    
+                                        " boolean archived=false,"
+                                        " const varstring appvalues=''"
+                                        ") : context,entrypoint='wsWorkunitList'; \n"
+"  varstring WUIDonDate(unsigned4 year,unsigned4 month,unsigned4 day,unsigned4 hour, unsigned4 minute) : entrypoint='wsWUIDonDate'; \n"
+"  varstring WUIDdaysAgo(unsigned4  daysago) : entrypoint='wsWUIDdaysAgo'; \n"
+"  dataset(WsTimeStamp) WorkunitTimeStamps(const varstring wuid) : context,entrypoint='wsWorkunitTimeStamps'; \n"
+"  dataset(WsMessage) WorkunitMessages(const varstring wuid) : context,entrypoint='wsWorkunitMessages'; \n"
+"  dataset(WsFileRead) WorkunitFilesRead(const varstring wuid) : context,entrypoint='wsWorkunitFilesRead'; \n"
+"  dataset(WsFileWritten) WorkunitFilesWritten(const varstring wuid) : context,entrypoint='wsWorkunitFilesWritten'; \n"
+"  dataset(WsTiming) WorkunitTimings(const varstring wuid) : context,entrypoint='wsWorkunitTimings'; \n"
+"  streamed dataset(WsStatistic) WorkunitStatistics(const varstring wuid, boolean includeActivities = false, const varstring _filter = '') : context,entrypoint='wsWorkunitStatistics'; \n"
+"  boolean setWorkunitAppValue(const varstring app, const varstring key, const varstring value, boolean overwrite=true) : context,entrypoint='wsSetWorkunitAppValue'; \n"
 "END;";
 
 #define WAIT_SECONDS 30
@@ -337,11 +336,12 @@ static bool serializeWUInfo(IConstWorkUnitInfo &info,MemoryBuffer &mb)
     return true;
 }
 
-
 }//namespace
 
 using namespace nsWorkunitservices;
 
+static const unsigned MAX_FILTERS=20;
+
 WORKUNITSERVICES_API void wsWorkunitList(
     ICodeContext *ctx,
     size32_t & __lenResult,
@@ -358,7 +358,8 @@ WORKUNITSERVICES_API void wsWorkunitList(
     const char *roxiecluster,  // Not in use - retained for compatibility only
     const char *eclcontains,
     bool online,
-    bool archived 
+    bool archived,
+    const char *appvalues
 )
 {
     MemoryBuffer mb;
@@ -409,7 +410,7 @@ WORKUNITSERVICES_API void wsWorkunitList(
     }
     if (online)
     {
-        WUSortField filters[20];  // NOTE - increase if you add a LOT more parameters!
+        WUSortField filters[MAX_FILTERS+1];  // NOTE - increase if you add a LOT more parameters! The +1 is to allow space for the terminator
         unsigned filterCount = 0;
         MemoryBuffer filterbuf;
 
@@ -437,6 +438,37 @@ WORKUNITSERVICES_API void wsWorkunitList(
         addWUQueryFilter(filters, filterCount, filterbuf, eclcontains, (WUSortField) (WUSFecl | WUSFwild));
         addWUQueryFilter(filters, filterCount, filterbuf, lowwuid, WUSFwuid);
         addWUQueryFilter(filters, filterCount, filterbuf, highwuid, WUSFwuidhigh);
+        if (appvalues && *appvalues)
+        {
+            StringArray appFilters;
+            appFilters.appendList(appvalues, "|");   // Multiple filters separated by |
+            ForEachItemIn(idx, appFilters)
+            {
+                StringArray appFilter; // individual filter of form appname/key=value or appname/*=value
+                appFilter.appendList(appFilters.item(idx), "=");
+                const char *appvalue;
+                switch (appFilter.length())
+                {
+                case 1:
+                    appvalue = NULL;
+                    break;
+                case 2:
+                    appvalue = appFilter.item(1);
+                    break;
+                default:
+                    throw MakeStringException(-1,"WORKUNITSERVICES: Invalid application value filter %s (expected format is 'appname/keyname=value')", appFilters.item(idx));
+                }
+                const char *appkey = appFilter.item(0);
+                if (!strchr(appkey, '/'))
+                    throw MakeStringException(-1,"WORKUNITSERVICES: Invalid application value filter %s (expected format is 'appname/keyname=value')", appFilters.item(idx));
+                if (filterCount>=MAX_FILTERS)
+                    throw MakeStringException(-1,"WORKUNITSERVICES: Too many filters");
+                filterbuf.append(appkey);
+                filterbuf.append(appvalue);
+                filters[filterCount++] = WUSFappvalue;
+            }
+        }
+
         filters[filterCount] = WUSFterm;
         Owned<IWorkUnitFactory> wuFactory = getWorkunitFactory(ctx);
         Owned<IConstWorkUnitIterator> it = wuFactory->getWorkUnitsSorted((WUSortField) (WUSFwuid | WUSFreverse), filters, filterbuf.bufferBase(), 0, INT_MAX, NULL, NULL); // MORE - need security flags here!
@@ -698,6 +730,17 @@ WORKUNITSERVICES_API IRowStream * wsWorkunitStatistics( ICodeContext *ctx, IEngi
     return new StreamedStatistics(wu, allocator, stats);
 }
 
+WORKUNITSERVICES_API bool wsSetWorkunitAppValue( ICodeContext *ctx, const char *appname, const char *key, const char *value, bool overwrite)
+{
+    if (appname && *appname && key && *key && value && *value)
+    {
+        WorkunitUpdate w(ctx->updateWorkUnit());
+        w->setApplicationValue(appname, key, value, overwrite);
+        return true;
+    }
+    return false;
+}
+
 
 WORKUNITSERVICES_API void setPluginContext(IPluginContext * _ctx) { parentCtx = _ctx; }
 

plugins/workunitservices/workunitservices.hpp

@@ -34,15 +34,17 @@
 #include "workunit.hpp"
 #include "eclhelper.hpp"
 
-extern "C" {
-WORKUNITSERVICES_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
-WORKUNITSERVICES_API void setPluginContext(IPluginContext * _ctx);
+extern "C"
+{
+  WORKUNITSERVICES_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
+  WORKUNITSERVICES_API void setPluginContext(IPluginContext * _ctx);
+}
+
 WORKUNITSERVICES_API char * WORKUNITSERVICES_CALL wsGetBuildInfo(void);
 
 WORKUNITSERVICES_API bool WORKUNITSERVICES_CALL wsWorkunitExists(ICodeContext *ctx, const char *wuid, bool online, bool archived);
 
-WORKUNITSERVICES_API void WORKUNITSERVICES_CALL wsWorkunitList(
-                                                                ICodeContext *ctx,
+WORKUNITSERVICES_API void WORKUNITSERVICES_CALL wsWorkunitList( ICodeContext *ctx,
                                                                 size32_t & __lenResult,
                                                                 void * & __result, 
                                                                 const char *lowwuid,
@@ -57,7 +59,8 @@ WORKUNITSERVICES_API void WORKUNITSERVICES_CALL wsWorkunitList(
                                                                 const char *roxiecluster,
                                                                 const char *eclcontains,
                                                                 bool online,
-                                                                bool archived );
+                                                                bool archived,
+                                                                const char *appvalues);
 
 
 WORKUNITSERVICES_API char * wsWUIDonDate(unsigned year,unsigned month,unsigned day,unsigned hour,unsigned minute);
@@ -69,8 +72,7 @@ WORKUNITSERVICES_API void WORKUNITSERVICES_CALL wsWorkunitFilesWritten( ICodeCon
 WORKUNITSERVICES_API void WORKUNITSERVICES_CALL wsWorkunitTimings( ICodeContext *ctx, size32_t & __lenResult, void * & __result, const char *wuid );
 WORKUNITSERVICES_API IRowStream * WORKUNITSERVICES_CALL wsWorkunitStatistics( ICodeContext *ctx, IEngineRowAllocator * allocator, const char *wuid, bool includeActivities, const char * filterText);
 
-
-}
+WORKUNITSERVICES_API bool WORKUNITSERVICES_CALL wsWorkunitTimings( ICodeContext *ctx, const char *wuid, const char * appname, const char *key, const char *value, bool overwrrite);
 
 #endif
 

roxie/ccd/ccdserver.cpp

@@ -971,7 +971,7 @@ public:
             DBGLOG("[%s] %s", prefix, text);
     }
 
-    virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const __attribute((format(printf,6,0)))
+    virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const __attribute__((format(printf,6,0)))
     {
         if (ctx)
             ctx->CTXLOGaeva(E, file, line, prefix, format, args);

roxie/roxiemem/roxiemem.hpp

@@ -93,6 +93,7 @@ const static unsigned SpillAllCost = (unsigned)-1;
 interface IBufferedRowCallback
 {
     virtual unsigned getSpillCost() const = 0; // lower values get freed up first.
+    virtual unsigned getActivityId() const = 0;
     virtual bool freeBufferedRows(bool critical) = 0; // return true if and only if managed to free something.
 };
 
@@ -460,7 +461,6 @@ interface IRowManager : extends IInterface
     virtual void *finalizeRow(void *final, memsize_t originalSize, memsize_t finalSize, unsigned activityId) = 0;
     virtual unsigned allocated() = 0;
     virtual unsigned numPagesAfterCleanup(bool forceFreeAll) = 0; // calls releaseEmptyPages() then returns
-    virtual bool releaseEmptyPages(bool forceFreeAll) = 0; // ensures any empty pages are freed back to the heap
     virtual unsigned getMemoryUsage() = 0;
     virtual bool attachDataBuff(DataBuffer *dataBuff) = 0 ;
     virtual void noteDataBuffReleased(DataBuffer *dataBuff) = 0 ;
@@ -490,6 +490,7 @@ interface IRowManager : extends IInterface
     virtual void setMinimizeFootprint(bool value, bool critical) = 0;
     //If set, and changes to the callback list always triggers the callbacks to be called.
     virtual void setReleaseWhenModifyCallback(bool value, bool critical) = 0;
+    virtual IRowManager * querySlaveRowManager(unsigned slave) = 0;  // 0..numSlaves-1
 };
 
 extern roxiemem_decl void setDataAlignmentSize(unsigned size);
@@ -512,6 +513,7 @@ interface IActivityMemoryUsageMap : public IInterface
 };
 
 extern roxiemem_decl IRowManager *createRowManager(memsize_t memLimit, ITimeLimiter *tl, const IContextLogger &logctx, const IRowAllocatorCache *allocatorCache, bool ignoreLeaks = false, bool outputOOMReports = false);
+extern roxiemem_decl IRowManager *createGlobalRowManager(memsize_t memLimit, memsize_t globalLimit, unsigned numSlaves, ITimeLimiter *tl, const IContextLogger &logctx, const IRowAllocatorCache *allocatorCache, bool ignoreLeaks, bool outputOOMReports);
 
 // Fixed size aggregated link-counted zero-overhead data Buffer manager
 

rtl/eclrtl/CMakeLists.txt

@@ -52,9 +52,7 @@ set (    SRCS
          rtlread_imp.hpp
          rtlsize.hpp
          rtltype.hpp
-
          rtlbcdtest.cpp 
-
     )
 
 include_directories ( 
@@ -70,6 +68,10 @@ include_directories (
 
 ADD_DEFINITIONS( -D_USRDLL -DECLRTL_EXPORTS )
 
+if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX)
+    set_source_files_properties(eclrtl.cpp PROPERTIES COMPILE_FLAGS -std=c++98)
+endif ()
+
 if (WIN32)
 else ()
     ADD_DEFINITIONS( -DBOOST_DYN_LINK )

rtl/include/eclhelper.hpp

@@ -39,8 +39,8 @@ if the supplied pointer was not from the roxiemem heap. Usually an OwnedRoxieStr
 
 //Should be incremented whenever the virtuals in the context or a helper are changed, so
 //that a work unit can't be rerun.  Try as hard as possible to retain compatibility.
-#define ACTIVITY_INTERFACE_VERSION      159
-#define MIN_ACTIVITY_INTERFACE_VERSION  159             //minimum value that is compatible with current interface - without using selectInterface
+#define ACTIVITY_INTERFACE_VERSION      160
+#define MIN_ACTIVITY_INTERFACE_VERSION  160             //minimum value that is compatible with current interface - without using selectInterface
 
 typedef unsigned char byte;
 
@@ -254,7 +254,7 @@ interface IEngineRowAllocator : extends IInterface
     virtual void * finalizeRow(size32_t newSize, void * row, size32_t oldSize) = 0;
 
     virtual IOutputMetaData * queryOutputMeta() = 0;
-    virtual unsigned queryActivityId() = 0;
+    virtual unsigned queryActivityId() const = 0;
     virtual StringBuffer &getId(StringBuffer &) = 0;
     virtual IOutputRowSerializer *createDiskSerializer(ICodeContext *ctx = NULL) = 0;
     virtual IOutputRowDeserializer *createDiskDeserializer(ICodeContext *ctx) = 0;

system/jlib/jdebug.cpp

@@ -364,7 +364,9 @@ cycle_t jlib_decl get_cycles_now()
     if (useRDTSC)
         return getTSC();
 #endif
-#ifndef __APPLE__
+#ifdef __APPLE__
+    return mach_absolute_time();
+#elif defined(CLOCK_MONOTONIC)
     if (!use_gettimeofday) {
         timespec tm;
         if (clock_gettime(CLOCK_MONOTONIC, &tm)>=0)
@@ -383,6 +385,9 @@ __int64 jlib_decl cycle_to_nanosec(cycle_t cycles)
 #if defined(_ARCH_X86_) || defined(_ARCH_X86_64_)
     if (useRDTSC)
         return (__int64)((double)cycles * cycleToNanoScale);
+#ifdef __APPLE__
+    return cycles * (uint64_t) timebase_info.numer / (uint64_t)timebase_info.denom;
+#endif
 #endif
     return cycles;
 }

system/jlib/jlzw.cpp

@@ -173,8 +173,8 @@ static struct __initShiftArray {
 
 #define PUTCODE(code)                                       \
 {                                                            \
-  register unsigned inbits=code;                             \
-  register int shift=curShift;                               \
+  unsigned inbits=code;                                      \
+  int shift=curShift;                                        \
   int copyBits = dict.curbits - BITS_PER_UNIT;               \
                                                              \
   *(outbytes++) = (unsigned char)(inbits&0xff);              \
@@ -482,7 +482,7 @@ void CLZWExpander::expand(void *buf)
     unsigned char *outend = out+outlen;
     int oldcode ;
     GETCODE(oldcode);
-    register int ch=oldcode;
+    int ch=oldcode;
     *(out++)=(unsigned char)ch;
     while (out!=outend) {
         int newcode;

system/jlib/jstats.cpp

@@ -453,28 +453,14 @@ extern jlib_decl StatsMergeAction queryMergeMode(StatisticKind kind)
     BASE_TAGS(x, y) \
     "@TimeDelta" # y
 
-#define LEGACYTAGS(dft)    \
-        dft, \
-        NULL, \
-        NULL, \
-        NULL, \
-        NULL, \
-        NULL, \
-        NULL, \
-        NULL, \
-        NULL, \
-        NULL
-
 #define CORESTAT(x, y, m)     St##x##y, m, { NAMES(x, y) }, { TAGS(x, y) }
-#define STAT(x, y, m)         CORESTAT(x, y, m), { LEGACYTAGS(NULL) }
-#define TAGSTAT(x, y, m, dft) St##x##y, m, { NAMES(x, y) }, { TAGS(x, y) }, { LEGACYTAGS(dft) }
-
+#define STAT(x, y, m)         CORESTAT(x, y, m)
 
 //--------------------------------------------------------------------------------------------------------------------
 
 //These are the macros to use to define the different entries in the stats meta table
 #define TIMESTAT(y) STAT(Time, y, SMeasureTimeNs)
-#define WHENSTAT(y) St##When##y, SMeasureTimestampUs, { TIMENAMES(When, y) }, { TIMETAGS(When, y) }, { LEGACYTAGS(NULL) }
+#define WHENSTAT(y) St##When##y, SMeasureTimestampUs, { TIMENAMES(When, y) }, { TIMETAGS(When, y) }
 #define NUMSTAT(y) STAT(Num, y, SMeasureCount)
 #define SIZESTAT(y) STAT(Size, y, SMeasureSize)
 #define LOADSTAT(y) STAT(Load, y, SMeasureLoad)
@@ -483,10 +469,6 @@ extern jlib_decl StatsMergeAction queryMergeMode(StatisticKind kind)
 #define PERSTAT(y) STAT(Per, y, SMeasurePercent)
 #define IPV4STAT(y) STAT(IPV4, y, SMeasureIPV4)
 
-//The following variants are used where a different tag name is required
-#define TIMESTAT2(y, dft) TAGSTAT(Time, y, SMeasureTimeNs, dft)
-#define NUMSTAT2(y, dft) TAGSTAT(Num, y, SMeasureCount, dft)
-
 //--------------------------------------------------------------------------------------------------------------------
 
 class StatisticMeta
@@ -496,7 +478,6 @@ public:
     StatisticMeasure measure;
     const char * names[StNextModifier/StVariantScale];
     const char * tags[StNextModifier/StVariantScale];
-    const char * legacytags[StNextModifier/StVariantScale];
 };
 
 static const StatisticMeta statsMetaData[StMax] = {
@@ -511,38 +492,38 @@ static const StatisticMeta statsMetaData[StMax] = {
     { WHENSTAT(Compiled) },
     { WHENSTAT(WorkunitModified) },
     { TIMESTAT(Elapsed) },
-    { CORESTAT(Time, LocalExecute, SMeasureTimeNs), { "@localTime", "@timeMinMs", "@timeMaxMs" } },
-    { TIMESTAT2(TotalExecute, "@totalTime") },
+    { TIMESTAT(LocalExecute) },
+    { TIMESTAT(TotalExecute) },
     { TIMESTAT(Remaining) },
     { SIZESTAT(GeneratedCpp) },
     { SIZESTAT(PeakMemory) },
     { SIZESTAT(MaxRowSize) },
-    { CORESTAT(Num, RowsProcessed, SMeasureCount), { "@count", "@min", "@max", NULL, "skew", "minskew", "maxskew", NULL, NULL, NULL } },
-    { NUMSTAT2(Slaves, "@slaves") },
-    { NUMSTAT2(Started, "@started") },
-    { NUMSTAT2(Stopped, "@stopped") },
-    { NUMSTAT2(IndexSeeks, "@seeks") },
-    { NUMSTAT2(IndexScans, "@scans") },
-    { NUMSTAT2(IndexWildSeeks, "@wildscans") },
-    { NUMSTAT2(IndexSkips, "@skips") },
-    { NUMSTAT2(IndexNullSkips, "@nullskips") },
-    { NUMSTAT2(IndexMerges, "@merges") },
-    { NUMSTAT2(IndexMergeCompares, "@mergecompares") },
-    { NUMSTAT2(PreFiltered, "@prefiltered") },
-    { NUMSTAT2(PostFiltered, "@postfiltered") },
-    { NUMSTAT2(BlobCacheHits, "@blobhit") },
-    { NUMSTAT2(LeafCacheHits, "@leafhit") },
-    { NUMSTAT2(NodeCacheHits, "@nodehit") },
-    { NUMSTAT2(BlobCacheAdds, "@blobadd") },
-    { NUMSTAT2(LeafCacheAdds, "@leadadd") },
-    { NUMSTAT2(NodeCacheAdds, "@nodeadd") },
-    { NUMSTAT2(PreloadCacheHits, "@preloadhits") },
-    { NUMSTAT2(PreloadCacheAdds, "@preloadadds") },
-    { NUMSTAT2(ServerCacheHits, "@sschits") },
-    { NUMSTAT2(IndexAccepted, "@accepted") },
-    { NUMSTAT2(IndexRejected, "@rejected") },
-    { NUMSTAT2(AtmostTriggered, "@atmost") },
-    { NUMSTAT2(DiskSeeks, "@fseeks") },
+    { NUMSTAT(RowsProcessed) },
+    { NUMSTAT(Slaves) },
+    { NUMSTAT(Started) },
+    { NUMSTAT(Stopped) },
+    { NUMSTAT(IndexSeeks) },
+    { NUMSTAT(IndexScans) },
+    { NUMSTAT(IndexWildSeeks) },
+    { NUMSTAT(IndexSkips) },
+    { NUMSTAT(IndexNullSkips) },
+    { NUMSTAT(IndexMerges) },
+    { NUMSTAT(IndexMergeCompares) },
+    { NUMSTAT(PreFiltered) },
+    { NUMSTAT(PostFiltered) },
+    { NUMSTAT(BlobCacheHits) },
+    { NUMSTAT(LeafCacheHits) },
+    { NUMSTAT(NodeCacheHits) },
+    { NUMSTAT(BlobCacheAdds) },
+    { NUMSTAT(LeafCacheAdds) },
+    { NUMSTAT(NodeCacheAdds) },
+    { NUMSTAT(PreloadCacheHits) },
+    { NUMSTAT(PreloadCacheAdds) },
+    { NUMSTAT(ServerCacheHits) },
+    { NUMSTAT(IndexAccepted) },
+    { NUMSTAT(IndexRejected) },
+    { NUMSTAT(AtmostTriggered) },
+    { NUMSTAT(DiskSeeks) },
     { NUMSTAT(Iterations) },
     { LOADSTAT(WhileSorting) },
     { NUMSTAT(LeftRows) },
@@ -617,15 +598,6 @@ const char * queryTreeTag(StatisticKind kind)
     return statsMetaData[rawkind].tags[variant];
 }
 
-const char * queryLegacyTreeTag(StatisticKind kind)
-{
-    StatisticKind rawkind = (StatisticKind)(kind & StKindMask);
-    unsigned variant = (kind / StVariantScale);
-    dbgassertex(rawkind >= StKindNone && rawkind < StMax);
-    dbgassertex(variant < (StNextModifier/StVariantScale));
-    return statsMetaData[rawkind].legacytags[variant];
-}
-
 //--------------------------------------------------------------------------------------------------------------------
 
 StatisticKind queryStatisticKind(const char * search)

system/jlib/jstats.h

@@ -571,7 +571,6 @@ extern jlib_decl StatisticMeasure queryMeasure(StatisticKind kind);
 extern jlib_decl const char * queryStatisticName(StatisticKind kind);
 extern jlib_decl void queryLongStatisticName(StringBuffer & out, StatisticKind kind);
 extern jlib_decl const char * queryTreeTag(StatisticKind kind);
-extern jlib_decl const char * queryLegacyTreeTag(StatisticKind kind);
 extern jlib_decl const char * queryCreatorTypeName(StatisticCreatorType sct);
 extern jlib_decl const char * queryScopeTypeName(StatisticScopeType sst);
 extern jlib_decl const char * queryMeasureName(StatisticMeasure measure);

system/jlib/jutil.cpp

@@ -60,6 +60,8 @@ static CriticalSection * protectedGeneratorCs;
 
 #if defined (__APPLE__)
 #include <mach-o/dyld.h>
+#include <mach/mach_time.h> /* mach_absolute_time */
+mach_timebase_info_data_t timebase_info  = { 1,1 };
 #endif
 
 MODULE_INIT(INIT_PRIORITY_SYSTEM)
@@ -69,6 +71,10 @@ MODULE_INIT(INIT_PRIORITY_SYSTEM)
     protectedGenerator = createRandomNumberGenerator();
     protectedGeneratorCs = new CriticalSection;
 #endif
+#if defined (__APPLE__)
+    if (mach_timebase_info(&timebase_info) != KERN_SUCCESS)
+        return false;
+#endif
     return true;
 }
 
@@ -395,7 +401,7 @@ HINSTANCE LoadSharedObject(const char *name, bool isGlobal, bool raiseOnError)
         if (!streq(ext.str(), SharedObjectExtension))
         {
             // Assume if there's no .so, there may also be no lib at the beginning
-            if (strncmp(tail.str(), SharedObjectPrefix, strlen(SharedObjectPrefix) != 0))
+            if (strncmp(tail.str(), SharedObjectPrefix, strlen(SharedObjectPrefix)) != 0)
                 path.append(SharedObjectPrefix);
             path.append(tail).append(ext).append(SharedObjectExtension);
             name = path.str();
@@ -1483,6 +1489,20 @@ unsigned usTick()
     gettimeofday(&tm,NULL);
     return tm.tv_sec*1000000+tm.tv_usec; 
 }
+#elif __APPLE__
+
+unsigned usTick()
+{
+    __uint64 nano = mach_absolute_time() * (uint64_t)timebase_info.numer / (uint64_t)timebase_info.denom;
+    return nano / 1000;
+}
+
+unsigned msTick()
+{
+    __uint64 nano = mach_absolute_time() * (uint64_t)timebase_info.numer / (uint64_t)timebase_info.denom;
+    return nano / 1000000;
+}
+
 #else
 #warning "clock_gettime(CLOCK_MONOTONIC) not supported"
 unsigned msTick() 

system/jlib/jutil.hpp

@@ -24,6 +24,12 @@
 #include "jarray.hpp"
 #include "jbuff.hpp"
 
+#if defined (__APPLE__)
+#include <mach-o/dyld.h>
+#include <mach/mach_time.h>
+extern mach_timebase_info_data_t timebase_info;   // Calibration for nanosecond timer
+#endif
+
 //#define NAMEDCOUNTS
 
 interface IPropertyTree;

system/lzma/CMakeLists.txt

@@ -26,9 +26,9 @@
 project( lzma ) 
 
 set ( SRCS
-        LzFind.c
-        LzmaDec.c
-        LzmaEnc.c
+        LzFind.cpp
+        LzmaDec.cpp
+        LzmaEnc.cpp
 )
 
 ADD_DEFINITIONS( -D_LIB )

system/lzma/LzFind.c

@@ -1,4 +1,4 @@
-/* LzFind.c -- Match finder for LZ algorithms
+/* LzFind.cpp -- Match finder for LZ algorithms
 2008-10-04 : Igor Pavlov : Public domain */
 
 #include <string.h>

system/lzma/LzmaDec.c

@@ -1,4 +1,4 @@
-/* LzmaDec.c -- LZMA Decoder
+/* LzmaDec.cpp -- LZMA Decoder
 2008-11-06 : Igor Pavlov : Public domain */
 
 #include "LzmaDec.h"

system/lzma/LzmaEnc.c

@@ -1,4 +1,4 @@
-/* LzmaEnc.c -- LZMA Encoder
+/* LzmaEnc.cpp -- LZMA Encoder
 2009-02-02 : Igor Pavlov : Public domain */
 
 #include <string.h>

testing/regress/ecl/aaawriteresult.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+import Std.System.Workunit as Wu;
+
 namesRecord := 
             RECORD
 string10        forename;
@@ -22,5 +24,6 @@ string20        surname;
             END;
 
 ds := dataset([{'Jo','Smith'},{'Jim','Smithe'},{'Joe','Schmitt'}], namesRecord);
-    
+
+wu.setWorkunitAppValue('regress', 'writeresult', '1', true);
 output(ds,NAMED('ExportedNames'));

testing/regress/ecl/kafkatest.ecl

@@ -0,0 +1,75 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=embedded
+//class=3rdparty
+
+IMPORT kafka;
+IMPORT Std;
+
+/*******************************************************************************
+ * These tests assume a Kafka instance running on the local host with a default
+ * Kafka configuration.  Each iteration of the test creates a new Kafka topic
+ * named after the WUID of the test.  These should be periodically cleaned
+ * off Kafka (or the Kafka instance itself be refreshed).
+ ******************************************************************************/
+
+KAFKA_BROKER := '127.0.0.1';
+KAFKA_TEST_TOPIC := Std.System.Job.WUID() : INDEPENDENT;
+KAFKA_CONSUMER_GROUP := 'regress';
+
+p := kafka.Publisher(KAFKA_TEST_TOPIC, KAFKA_BROKER);
+c := kafka.Consumer(KAFKA_TEST_TOPIC, KAFKA_BROKER, KAFKA_CONSUMER_GROUP);
+
+SEQUENTIAL
+    (
+        // Action to prompt Kafka to create a new topic for us; this will
+        // will result in a partition count of zero, which is normal
+        OUTPUT(c.GetTopicPartitionCount(), NAMED('PingToCreateTopic'));
+
+        // Idle while Kafka prepares topic
+        Std.System.Debug.Sleep(1000);
+
+        OUTPUT(c.GetTopicPartitionCount(), NAMED('ConsumerGetTopicPartitionCount'));
+
+        OUTPUT(c.ResetMessageOffsets(), NAMED('ConsumerResetMessageOffsets1'));
+
+        OUTPUT(p.GetTopicPartitionCount(), NAMED('PublisherGetTopicPartitionCount'));
+
+        OUTPUT(p.PublishMessage('Regular message'), NAMED('PublishMessageUnkeyed'));
+
+        OUTPUT(p.PublishMessage('Keyed message'), NAMED('PublishMessageKeyed'));
+        
+        // Idle while Kafka publishes
+        Std.System.Debug.Sleep(1000);
+
+        OUTPUT(c.GetMessages(10), NAMED('GetMessages1'));
+
+        OUTPUT(c.GetMessages(10), NAMED('GetMessagesEmpty'));
+
+        OUTPUT(c.ResetMessageOffsets(), NAMED('ConsumerResetMessageOffsets2'));
+
+        OUTPUT(c.GetMessages(10), NAMED('GetMessages2'));
+
+        OUTPUT(c.SetMessageOffsets(DATASET([{0,0}], kafka.KafkaMessageOffset)), NAMED('ConsumerSetExplicitMessageOffsets'));
+
+        OUTPUT(c.GetMessages(10), NAMED('GetMessages3'));
+
+        OUTPUT(c.ResetMessageOffsets(), NAMED('ConsumerResetMessageOffsets3'));
+
+        OUTPUT(c.LastMessageOffsets(c.GetMessages(10)), NAMED('ConsumerLastMessageOffsets'));
+    );

testing/regress/ecl/key/aaawriteresult.xml

@@ -1,3 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
 <Dataset name='ExportedNames'>
  <Row><forename>Jo        </forename><surname>Smith               </surname></Row>
  <Row><forename>Jim       </forename><surname>Smithe              </surname></Row>

testing/regress/ecl/key/kafkatest.xml

@@ -0,0 +1,43 @@
+<Dataset name='PingToCreateTopic'>
+ <Row><PingToCreateTopic>0</PingToCreateTopic></Row>
+</Dataset>
+<Dataset name='ConsumerGetTopicPartitionCount'>
+ <Row><ConsumerGetTopicPartitionCount>1</ConsumerGetTopicPartitionCount></Row>
+</Dataset>
+<Dataset name='ConsumerResetMessageOffsets1'>
+ <Row><ConsumerResetMessageOffsets1>1</ConsumerResetMessageOffsets1></Row>
+</Dataset>
+<Dataset name='PublisherGetTopicPartitionCount'>
+ <Row><PublisherGetTopicPartitionCount>1</PublisherGetTopicPartitionCount></Row>
+</Dataset>
+<Dataset name='PublishMessageUnkeyed'>
+ <Row><PublishMessageUnkeyed>true</PublishMessageUnkeyed></Row>
+</Dataset>
+<Dataset name='PublishMessageKeyed'>
+ <Row><PublishMessageKeyed>true</PublishMessageKeyed></Row>
+</Dataset>
+<Dataset name='GetMessages1'>
+ <Row><partitionnum>0</partitionnum><offset>0</offset><message>Regular message</message></Row>
+ <Row><partitionnum>0</partitionnum><offset>1</offset><message>Keyed message</message></Row>
+</Dataset>
+<Dataset name='GetMessagesEmpty'>
+</Dataset>
+<Dataset name='ConsumerResetMessageOffsets2'>
+ <Row><ConsumerResetMessageOffsets2>1</ConsumerResetMessageOffsets2></Row>
+</Dataset>
+<Dataset name='GetMessages2'>
+ <Row><partitionnum>0</partitionnum><offset>0</offset><message>Regular message</message></Row>
+ <Row><partitionnum>0</partitionnum><offset>1</offset><message>Keyed message</message></Row>
+</Dataset>
+<Dataset name='ConsumerSetExplicitMessageOffsets'>
+ <Row><ConsumerSetExplicitMessageOffsets>1</ConsumerSetExplicitMessageOffsets></Row>
+</Dataset>
+<Dataset name='GetMessages3'>
+ <Row><partitionnum>0</partitionnum><offset>1</offset><message>Keyed message</message></Row>
+</Dataset>
+<Dataset name='ConsumerResetMessageOffsets3'>
+ <Row><ConsumerResetMessageOffsets3>1</ConsumerResetMessageOffsets3></Row>
+</Dataset>
+<Dataset name='ConsumerLastMessageOffsets'>
+ <Row><partitionnum>0</partitionnum><offset>1</offset></Row>
+</Dataset>

testing/regress/ecl/key/parse2.xml

@@ -0,0 +1,2 @@
+<Dataset name='Result 1'>
+</Dataset>

testing/regress/ecl/parse2.ecl

@@ -0,0 +1,24 @@
+rec := record
+  unsigned8 id;
+  unicode   searchText;
+end;
+
+cleansedFieldInline := dataset([{6420, ''}], rec);
+
+pattern words := pattern('[^,;]+');
+pattern sepchar := [',',';','AND'];
+rule termsRule := FIRST words sepchar |sepchar words LAST | sepchar words sepchar | FIRST words LAST;
+
+normalizeSeperators(unicode str) := regexreplace(u'AND',str,u',');
+
+termsDs := parse(NOFOLD(cleansedFieldInline),
+                 normalizeSeperators(searchText),
+                 termsRule,
+                 transform({rec, unicode terms},
+                           self.terms := trim(matchunicode(words),left,right),
+                           self := left),
+                 SCAN ALL);
+
+sequential (
+  output(termsDs);  // Test parsing an empty string with a pattern that has a minimum match length > 0
+);

testing/regress/ecl/readresult.ecl

@@ -22,13 +22,11 @@ string20        surname;
             END;
 
 
-//Horrible code - get a list of workunits that match the name of the job that creates the result
+//Horrible code - get a list of workunits that create the result
 //which needs to be inside a nothor.
 
 import Std.System.Workunit as Wu;
-myWuid := workunit;
-startOfDay := myWuid[1..9] + '-000000';
-writers := Wu.WorkunitList(lowWuid := startOfDay,jobname := 'aaawriteresult*');
+writers := Wu.WorkunitList(appvalues := 'regress/writeresult=1');
 
 //Now sort and extract the most recent wuid that matches the condition
 lastWriter := sort(nothor(writers), -wuid);

thorlcr/activities/filter/thfilterslave.cpp

@@ -249,6 +249,7 @@ public:
     CFilterGroupSlaveActivity(CGraphElementBase *container) : CFilterSlaveActivityBase(container), CThorSteppable(this)
     {
         groupLoader.setown(createThorRowLoader(*this, NULL, stableSort_none, rc_allMem));
+        helper = NULL;
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {

thorlcr/activities/group/thgroupslave.cpp

@@ -88,7 +88,7 @@ public:
 
         if (rolloverEnabled && !firstNode())  // 1st node can have nothing to send
         {
-            Owned<IThorRowCollector> collector = createThorRowCollector(*this, NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_SPILLABLE_STREAM);
+            Owned<IThorRowCollector> collector = createThorRowCollector(*this, this, NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_SPILLABLE_STREAM);
             Owned<IRowWriter> writer = collector->getWriter();
             if (next)
             {

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -2678,6 +2678,10 @@ class CBucketHandler : public CSimpleInterface, implements IInterface, implement
         {
             return SPILL_PRIORITY_HASHDEDUP_BUCKET_POSTSPILL;
         }
+        virtual unsigned getActivityId() const
+        {
+            return owner.getActivityId();
+        }
         virtual bool freeBufferedRows(bool critical)
         {
             if (NotFound == owner.nextSpilledBucketFlush)
@@ -2755,6 +2759,7 @@ public:
     {
         return SPILL_PRIORITY_HASHDEDUP;
     }
+    virtual unsigned getActivityId() const;
     virtual bool freeBufferedRows(bool critical)
     {
         return spillBucket(critical);
@@ -3259,6 +3264,8 @@ CBucketHandler::CBucketHandler(HashDedupSlaveActivityBase &_owner, IRowInterface
     nextToSpill = NotFound;
     peakKeyCount = RCIDXMAX;
     nextSpilledBucketFlush = NotFound;
+    numBuckets = 0;
+    buckets = NULL;
 }
 
 CBucketHandler::~CBucketHandler()
@@ -3356,6 +3363,11 @@ unsigned CBucketHandler::getBucketEstimate(rowcount_t totalRows) const
     return retBuckets;
 }
 
+unsigned CBucketHandler::getActivityId() const
+{
+    return owner.queryActivityId();
+}
+
 void CBucketHandler::init(unsigned _numBuckets, IRowStream *keyStream)
 {
     numBuckets = _numBuckets;

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -2152,6 +2152,10 @@ public:
     {
         return SPILL_PRIORITY_LOOKUPJOIN;
     }
+    virtual unsigned getActivityId() const
+    {
+        return this->queryActivityId();
+    }
     virtual bool freeBufferedRows(bool critical)
     {
         // NB: only installed if lookup join and global

thorlcr/activities/rollup/throllupslave.cpp

@@ -87,6 +87,10 @@ public:
         in = NULL;
         helper = NULL;
         abort = NULL;
+        dedupIdx = dedupCount = 0;
+        dedupArray = NULL;
+        iStopInput = NULL;
+        keepLeft = true;
         rowLoader.setown(createThorRowLoader(*activity, NULL, stableSort_none, rc_allMem));
     }
 
@@ -568,6 +572,9 @@ public:
 
     CRollupGroupSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), CThorDataLink(this)
     {
+        eoi = false;
+        input = NULL;
+        helper = NULL;
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {

thorlcr/graph/thgraph.hpp

@@ -1048,7 +1048,7 @@ public:
     virtual IOutputRowSerializer * queryRowSerializer(); 
     virtual IOutputRowDeserializer * queryRowDeserializer(); 
     virtual IOutputMetaData *queryRowMetaData() { return baseHelper->queryOutputMeta(); }
-    virtual unsigned queryActivityId() { return (unsigned)queryId(); }
+    virtual unsigned queryActivityId() const { return (unsigned)queryId(); }
     virtual ICodeContext *queryCodeContext() { return container.queryCodeContext(); }
 
     StringBuffer &getOpt(const char *prop, StringBuffer &out) const { return container.getOpt(prop, out); }

thorlcr/thorutil/thmem.cpp

@@ -205,13 +205,14 @@ protected:
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CSpillableStreamBase(CActivityBase &_activity, CThorSpillableRowArray &inRows, IRowInterfaces *_rowIf, bool _preserveNulls, unsigned _spillPriorirty)
-        : activity(_activity), rowIf(_rowIf), rows(_activity, _rowIf, _preserveNulls), preserveNulls(_preserveNulls), spillPriority(_spillPriorirty)
+    CSpillableStreamBase(CActivityBase &_activity, CThorSpillableRowArray &inRows, IRowInterfaces *_rowIf, bool _preserveNulls, unsigned _spillPriority)
+        : activity(_activity), rowIf(_rowIf), rows(_activity, _rowIf, _preserveNulls), preserveNulls(_preserveNulls), spillPriority(_spillPriority)
     {
         assertex(inRows.isFlushed());
         rows.swap(inRows);
         useCompression = false;
         mmRegistered = false;
+        ownsRows = false;
     }
     ~CSpillableStreamBase()
     {
@@ -225,6 +226,10 @@ public:
     {
         return spillPriority;
     }
+    virtual unsigned getActivityId() const
+    {
+        return activity.queryActivityId();
+    }
     virtual bool freeBufferedRows(bool critical)
     {
         if (spillFile) // i.e. if spilt already. NB: this is thread-safe, as 'spillFile' only set by spillRows() call below and can't be called on multiple threads concurrently.
@@ -1755,6 +1760,10 @@ public:
     {
         return spillPriority;
     }
+    virtual unsigned getActivityId() const
+    {
+        return activity.queryActivityId();
+    }
     virtual bool freeBufferedRows(bool critical)
     {
         if (!spillingEnabled())
@@ -1910,12 +1919,6 @@ IThorRowCollector *createThorRowCollector(CActivityBase &activity, IRowInterface
     return collector.getClear();
 }
 
-IThorRowCollector *createThorRowCollector(CActivityBase &activity, ICompare *iCompare, StableSortFlag stableSort, RowCollectorSpillFlags diskMemMix, unsigned spillPriority, bool preserveGrouping)
-{
-    return createThorRowCollector(activity, &activity, iCompare, stableSort, diskMemMix, spillPriority, preserveGrouping);
-}
-
-
 void setThorInABox(unsigned num)
 {
 }

thorlcr/thorutil/thmem.hpp

@@ -157,7 +157,7 @@ interface IThorAllocator : extends IInterface
     virtual bool queryCrc() const = 0;
 };
 
-IThorAllocator *createThorAllocator(memsize_t memSize, unsigned memorySpillAt, IContextLogger &logctx, bool crcChecking, bool usePacked);
+extern graph_decl IThorAllocator *createThorAllocator(memsize_t memSize, unsigned memorySpillAt, IContextLogger &logctx, bool crcChecking, bool usePacked);
 
 extern graph_decl IOutputMetaData *createOutputMetaDataWithExtra(IOutputMetaData *meta, size32_t sz);
 extern graph_decl IOutputMetaData *createOutputMetaDataWithChildRow(IEngineRowAllocator *childAllocator, size32_t extraSz);
@@ -536,7 +536,6 @@ interface IThorRowCollector : extends IThorRowCollectorCommon
 extern graph_decl IThorRowLoader *createThorRowLoader(CActivityBase &activity, IRowInterfaces *rowIf, ICompare *iCompare=NULL, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=SPILL_PRIORITY_DEFAULT);
 extern graph_decl IThorRowLoader *createThorRowLoader(CActivityBase &activity, ICompare *iCompare=NULL, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=SPILL_PRIORITY_DEFAULT);
 extern graph_decl IThorRowCollector *createThorRowCollector(CActivityBase &activity, IRowInterfaces *rowIf, ICompare *iCompare=NULL, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=SPILL_PRIORITY_DEFAULT, bool preserveGrouping=false);
-extern graph_decl IThorRowCollector *createThorRowCollector(CActivityBase &activity, ICompare *iCompare=NULL, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=SPILL_PRIORITY_DEFAULT, bool preserveGrouping=false);