Merge branch 'candidate-7.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -8619,8 +8619,12 @@ IConstWUScopeIterator & CLocalWorkUnit::getScopeIterator(const WuScopeFilter & f
 
     if (sources & SSFsearchWorkflow)
     {
-        Owned<IConstWUScopeIterator> workflowIter(new WorkflowStatisticsScopeIterator(getWorkflowItems()));
-        compoundIter->addIter(workflowIter);
+        Owned<IConstWorkflowItemIterator> iter = getWorkflowItems();
+        if (iter)
+        {
+            Owned<IConstWUScopeIterator> workflowIter(new WorkflowStatisticsScopeIterator(iter));
+            compoundIter->addIter(workflowIter);
+        }
     }
 
 
@@ -12407,29 +12411,38 @@ extern WORKUNIT_API bool secDebugWorkunit(const char * wuid, ISecManager &secmgr
         Owned<IConstWorkUnit> wu = factory->openWorkUnit(wuid, &secmgr, &secuser);
         SCMStringBuffer ip;
         unsigned port;
-        port = wu->getDebugAgentListenerPort();
-        wu->getDebugAgentListenerIP(ip);
-        SocketEndpoint debugEP(ip.str(), port);
-        Owned<ISocket> socket = ISocket::connect_timeout(debugEP, 1000);
-        unsigned len = (size32_t)strlen(command);
-        unsigned revlen = len;
-        _WINREV(revlen);
-        socket->write(&revlen, sizeof(revlen));
-        socket->write(command, len);
-        for (;;)
+        try
         {
-            socket->read(&len, sizeof(len));
-            _WINREV(len);                    
-            if (len == 0)
-                break;
-            if (len & 0x80000000)
+            port = wu->getDebugAgentListenerPort();
+            wu->getDebugAgentListenerIP(ip);
+            SocketEndpoint debugEP(ip.str(), port);
+            Owned<ISocket> socket = ISocket::connect_timeout(debugEP, 1000);
+            unsigned len = (size32_t)strlen(command);
+            unsigned revlen = len;
+            _WINREV(revlen);
+            socket->write(&revlen, sizeof(revlen));
+            socket->write(command, len);
+            for (;;)
             {
-                throwUnexpected();
+                socket->read(&len, sizeof(len));
+                _WINREV(len);
+                if (len == 0)
+                    break;
+                if (len & 0x80000000)
+                {
+                    throwUnexpected();
+                }
+                char * mem = (char*) response.reserve(len);
+                socket->read(mem, len);
             }
-            char * mem = (char*) response.reserve(len);
-            socket->read(mem, len);
+            return true;
+        }
+        catch (IException *E)
+        {
+            VStringBuffer msg("In secDebugWorkunit wuid %s port %d ip %s command %s", wuid, port, ip.str(), command);
+            EXCLOG(E, msg.str());
+            throw;
         }
-        return true;
     }
     return false;
 }

docs/EN_US/ECLWatch/ECLWa_mods/ECLWatchSrc.xml

@@ -679,11 +679,15 @@
           <title>Visualizations</title>
 
           <para>You can see visual representations of select workunits.
-          Visualizations are available from the workunit <emphasis
-          role="bold">Outputs</emphasis> tab. The Visualize tab provides a
-          number of chart types you can generate if you have included
-          additional resources in your ECL code via the enhanced manifest
-          mechanism (such as an index web page). <figure>
+          Visualizations are accessible from the workunit details page. On the
+          workunit details page select the <emphasis
+          role="bold">Outputs</emphasis> tab. Check the box to select the
+          result and press the <emphasis role="bold">Visualize</emphasis>
+          action button. <graphic
+          fileref="../../images/ECLWAViz01.jpg" /></para>
+
+          <para>The Visualize tab provides a number of chart types you can
+          generate from your result.<figure>
               <title>Visualization</title>
 
               <mediaobject>
@@ -693,12 +697,10 @@
               </mediaobject>
             </figure></para>
 
-          <para>To access Visualizations, click on the Outputs tab from the
-          selected workunit details page, then select the <emphasis
-          role="bold">Visualize</emphasis> tab. You can view different
-          visualization types by clicking on the drop list on the Visualize
-          tab. Click on the <emphasis role="bold">Mappings</emphasis> drop
-          menu, to change the parameters.</para>
+          <para>You can view different visualization types by clicking on the
+          drop list on the Visualize tab. Click on the <emphasis
+          role="bold">Mappings</emphasis> drop menu, to change the
+          parameters.</para>
         </sect3>
 
         <sect3 id="ECLWatch_ECLWorkunitsInputsTab" role="brk">

docs/PT_BR/CMakeLists.txt

@@ -0,0 +1,105 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+###
+#  This CMakeLists intended to be used in docs/PT_BR
+#    Refactored for internationalization
+###
+#    Last Mod GP 5/2019
+#  Removed ECL Plugin For Eclipse
+####################
+#
+# Doc dirs to include.
+#
+
+
+cmake_minimum_required(VERSION 2.8)
+PROJECT(docs_pt_br)
+
+#define_property(GLOBAL PROPERTY DOC_TARGETS BRIEF_DOCS "docs_pt_br" FULL_DOCS "docs_pt_br")
+#mark_as_advanced(DOC_TARGETS)
+
+#include(${CMAKE_MODULE_PATH}/docMacros.cmake)
+
+set (DOC_LANG PT_BR)
+#set (DOCBOOK_RESOURCES ${CMAKE_CURRENT_BINARY_DIR}/../resources)
+#set (DOCBOOK_XSL ${CMAKE_CURRENT_BINARY_DIR}/../resources/docbook-xsl)
+set (DOC_IMAGES ${CMAKE_CURRENT_SOURCE_DIR}/)
+#set (XML_CATALOG ${CMAKE_CURRENT_BINARY_DIR}/../BuildTools/catalog.xml)
+#set (DOC_VERSION "${HPCC_MAJOR}.${HPCC_MINOR}.${HPCC_POINT}")
+#set (FO_XSL ${CMAKE_CURRENT_BINARY_DIR}/../BuildTools/fo.xsl)
+#set (VERSION_DIR ${CMAKE_CURRENT_BINARY_DIR}/)
+#set (ECLIPSE_HTML_XSL ${CMAKE_CURRENT_BINARY_DIR}/../BuildTools/EclipseHelp.xsl)
+set (ECL_REFERENCE_XML ${CMAKE_CURRENT_BINARY_DIR}/ECLReference/ECLReference.xml)
+#set (HTML_HELP_XSL ${CMAKE_CURRENT_BINARY_DIR}/../resources/docbook-xsl/htmlhelp/htmlhelp.xsl)
+
+set (PORTAL_HTML_DIR "${CMAKE_CURRENT_BINARY_DIR}/PortalHTML")
+
+# Build image file list to add to source watch.
+file(GLOB_RECURSE DOC_IMAGES_LIST images/*)
+
+# Required directories to add.
+#add_subdirectory(../common common)
+#add_subdirectory(../BuildTools BuildTools)
+#add_subdirectory(../resources resources)
+
+
+add_subdirectory(DynamicESDL)
+add_subdirectory(ECLLanguageReference)
+add_subdirectory(ECLProgrammersGuide)
+add_subdirectory(ECLStandardLibraryReference)
+##add_subdirectory(ECLReference)
+
+##add_subdirectory(EclipseHelp)
+##add_subdirectory(HTMLHelp)
+add_subdirectory(ECLPlayground)
+add_subdirectory(HPCCClientTools)
+add_subdirectory(HPCCCertify)
+add_subdirectory(HPCCDataHandling)
+add_subdirectory(HPCCDataTutorial)
+add_subdirectory(HPCCMonitoring)
+add_subdirectory(HPCCSystemAdmin)
+##7.2#add_subdirectory(HPCCSpark)
+add_subdirectory(IMDB)
+add_subdirectory(InstantCloud)
+add_subdirectory(Installing_and_RunningTheHPCCPlatform)
+add_subdirectory(RoxieReference)
+add_subdirectory(RunningHPCCinaVirtualMachine)
+
+#add_subdirectory(ECLPluginForEclipse)
+add_subdirectory(ECLScheduler)
+add_subdirectory(ECLWatch)
+add_subdirectory(VisualizingECL)
+##7.2#add_subdirectory(WsSQLUG)
+#add_subdirectory(PortalHTML)
+
+
+
+
+# Docs automation
+##CommentedOut4PT_BR_TESTING
+##if (DOCS_AUTO)
+#   add_subdirectory(XMLGeneration)
+##   add_subdirectory(ConfiguringHPCC)
+##endif()
+
+
+#WIP -  Docs
+#add_subdirectory(RuningHPCCinAmazonWebServicesEC2)
+
+
+#GET_PROPERTY(_targets GLOBAL PROPERTY DOC_TARGETS)
+#add_custom_target(docs_en_us ALL DEPENDS ${_targets})

docs/PT_BR/DynamicESDL/CMakeLists.txt

@@ -1,5 +1,5 @@
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems(r).
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,18 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} DynamicESDL_Includer.xml "DynamicESDL" "DESDL-Mods")
-DOCBOOK_TO_PDF( ${FO_XSL} ESDL_LangRef_Includer.xml "ESDL_LangRef" "DESDL-Mods")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*ESDL_LangRef.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-    ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/ESDL_LangRef_Includer.xml" ${PORTAL_HTML_DIR}/ESDL_LangRef  "esdl_langref_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/ECLLanguageReference/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,17 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} ECLR-includer.xml "ECLLanguageReference" "ECLR_mods")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*ECLLanguageReference.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-    ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/ECLR-includer.xml" ${PORTAL_HTML_DIR}/ECLR  "lang_ref_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/ECLPlayground/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,18 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} ECLPlayground-includer.xml "ECL_Playground" "ECLPlay-Mods")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*RoxieReference.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-    ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/ECLPlayground-includer.xml" ${PORTAL_HTML_DIR}/ECLPlayground  "play_ground_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/ECLPluginForEclipse/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

docs/PT_BR/ECLProgrammersGuide/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -14,29 +14,5 @@
 #    limitations under the License.
 ################################################################################
 
-
-DOCBOOK_TO_PDF( ${FO_XSL} PrGd-Includer.xml "ECLProgrammersGuide" "PRG_Mods")
-
-IF(MAKE_DOCS)
-  set(zip_out_dir ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs)
-  ADD_CUSTOM_COMMAND(
-	COMMAND mkdir -p ${zip_out_dir}
-	COMMAND mkdir -p ECL_Code_Files
-	COMMAND rm -rf ECL_Code_Files/*
-        COMMAND cp ${HPCC_SOURCE_DIR}/docs/ECLProgrammersGuide/PRG_Mods/"\"ECL Code Files\""/*.ecl ECL_Code_Files/
-        COMMAND zip -r  ${zip_out_dir}/ECL_Code_Files.zip ECL_Code_Files
-        OUTPUT ${zip_out_dir}/ECL_Code_Files.zip
-	)
-
-  ADD_CUSTOM_TARGET(ECL_Code_Files ALL DEPENDS  ${zip_out_dir}/ECL_Code_Files.zip)
-
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*ECLProgrammersGuide.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/PrGd-Includer.xml" ${PORTAL_HTML_DIR}/ProgrammersGuide  "programmers_guide_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/ECLScheduler/CMakeLists.txt

@@ -1,5 +1,5 @@
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,17 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} ECLScheduler.xml "TheECLScheduler" "ECLSched-Mods")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*TheECLScheduler.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/ECLScheduler.xml" ${PORTAL_HTML_DIR}/Scheduler  "scheduler_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/ECLStandardLibraryReference/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,18 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} SLR-includer.xml "ECLStandardLibraryReference" "SLR-Mods")
-
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*ECLStandardLibraryReference.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-    ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/SLR-includer.xml" ${PORTAL_HTML_DIR}/SLR  "stdlib_ref_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/ECLWatch/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,19 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-#DOCBOOK_TO_PDF( ${FO_XSL} ECLWTechPrev.xml "ECLWatch" "ECLWa_mods")
-
-DOCBOOK_TO_PDF( ${FO_XSL} TheECLWatchMan.xml "The_ECL_Watch_Manual" "ECLWa_mods")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*The_ECL_Watch_Manual.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/TheECLWatchMan.xml" ${PORTAL_HTML_DIR}/ECLWatchMan  "eclwatch_man_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/HPCCCertify/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,18 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} certify.xml "HPCCCertification")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*HPCCCertification.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/certify.xml" ${PORTAL_HTML_DIR}/Certification  "certify_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
-
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/HPCCClientTools/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,31 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} ClientTools.xml "HPCCClientTools" "CT_Mods")
-
-DOCBOOK_TO_PDF( ${FO_XSL} TheECLIDEandHPCCClientTools.xml "TheECLIDEandHPCCClientTools" "CT_Mods")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*HPCCClientTools.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/ClientTools.xml" ${PORTAL_HTML_DIR}/ClientTools  "clienttools_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*TheECLIDEandHPCCClientTools.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/TheECLIDEandHPCCClientTools.xml" ${PORTAL_HTML_DIR}/ECLIDEandClientTools "eclide_clienttools_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-
-
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/HPCCDataHandling/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,17 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} DataHandling.xml "HPCCDataHandling" "DH-Mods")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*HPCCDataHandling.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/DataHandling.xml" ${PORTAL_HTML_DIR}/DataHandling "datahandling_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/HPCCDataTutorial/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

docs/PT_BR/HPCCMonitoring/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,16 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} HPCCMonitoringAndReporting.xml "HPCC_Monitoring_and_Reporting")
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*Monitoring_and_Reporting.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/HPCCMonitoringAndReporting.xml" ${PORTAL_HTML_DIR}/Monitoring_and_Reporting  "monitor_reporting_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/HPCCSystemAdmin/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,29 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} HPCCSystemAdministratorsGuide.xml "HPCCSystemAdministratorsGuide" "SA-Mods")
-
-DOCBOOK_TO_PDF( ${FO_XSL} SecMgrInc.xml "HPCCSecurityManagerGuide" "SA-Mods")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*SystemAdministratorsGuide.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/HPCCSystemAdministratorsGuide.xml" ${PORTAL_HTML_DIR}/SystemAdministratorsGuide  "system_administration_guide_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*SecurityManagerGuide.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/SecMgrInc.xml" ${PORTAL_HTML_DIR}/SecurityManagerGuide  "security_manager_guide_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/IMDB/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

docs/PT_BR/Installing_and_RunningTheHPCCPlatform/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,17 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} Installing_and_RunningTheHPCCPlatform.xml "Installing_and_RunningTheHPCCPlatform")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*Installing_and_RunningTheHPCCPlatform.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/Installing_and_RunningTheHPCCPlatform.xml" ${PORTAL_HTML_DIR}/Installing_and_Running  "install_and_running_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/InstantCloud/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,17 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} InstantCloud.xml "InstantCloud_for_AWS")
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*InstantCloud_for_AWS.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/InstantCloud.xml" ${PORTAL_HTML_DIR}/InstantCloud_for_AWS  "instantcloud_for_aws_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/RoxieReference/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -13,17 +13,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
-
-DOCBOOK_TO_PDF( ${FO_XSL} RoxieReference.xml "RoxieReference" "RoxieRefMods" )
-
-IF(MAKE_DOCS)
-  SET(HELP_DEPENDENCIES)
-  GET_PROPERTY(Current_Targets GLOBAL PROPERTY DOC_TARGETS)
-  FOREACH(T ${Current_Targets})
-    IF("${T}" MATCHES ".*RoxieReference.*")
-      LIST(APPEND HELP_DEPENDENCIES  ${T})
-     ENDIF()
-  ENDFOREACH()
-
-  DOCBOOK_TO_HTML(${ECLIPSE_HTML_XSL} "${CMAKE_CURRENT_BINARY_DIR}/RoxieReference.xml" ${PORTAL_HTML_DIR}/RoxieReference  "roxie_reference_html" "${HPCC_SOURCE_DIR}/docs/common/eclipsehelp.css" "")
-ENDIF(MAKE_DOCS)
+get_filename_component(DOC_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../BuildTools/cmake_config/${DOC_DIR_NAME}.txt)

docs/PT_BR/RunningHPCCinaVirtualMachine/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

docs/PT_BR/VisualizingECL/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

ecl/eclagent/eclagent.cpp

@@ -1985,7 +1985,7 @@ void EclAgent::doProcess()
             unsigned errCode = 0;
             try
             {
-               if (!dir->remove())
+               if (dir->exists() && dir->isDirectory() && !dir->remove())
                     rmMsg.append("Failed to remove temporary directory: ").append(jobTempDir.str());
             }
             catch (IException *e)

ecl/hqlcpp/hqlwcpp.cpp

@@ -1491,7 +1491,7 @@ StringBuffer & HqlCppWriter::generateChildExpr(IHqlExpression * expr, unsigned c
             case no_add:
             case no_band:
             case no_bor:
-                needBra = false;
+                needBra = (childIndex != 0); // Operators are left associative, so use () on rhs to preserve order
                 break;
             }
         }

ecl/hthor/hthor.cpp

@@ -1173,6 +1173,7 @@ void CHThorIndexWriteActivity::execute()
 
     if(clusterHandler)
         clusterHandler->copyPhysical(file, agent.queryWorkUnit()->getDebugValueBool("__output_cluster_no_copy_physical", false));
+    clearKeyStoreCacheEntry(file->queryFilename());
 
     // Now publish to name services
     StringBuffer dir,base;

esp/bindings/http/platform/httpservice.cpp

@@ -1734,14 +1734,13 @@ void CEspHttpServer::resetSessionTimeout(EspAuthRequest& authReq, unsigned sessi
 
 void CEspHttpServer::sendSessionReloadHTMLPage(IEspContext* ctx, EspAuthRequest& authReq, const char* errMsg)
 {
-    StringBuffer espURL, ip;
-    short port = 0;
-    ctx->getServAddress(ip, port);
+    StringBuffer espURL;
     if (isSSL)
         espURL.set("https://");
     else
         espURL.set("http://");
-    espURL.append(ip).append(":").append(port);
+    m_request->getHost(espURL);
+    espURL.append(":").append(m_request->getPort());
 
     StringBuffer content(
         "<!DOCTYPE html>"

esp/scm/ws_store.ecm

@@ -142,7 +142,7 @@ ESPrequest CreateStoreRequest
     string Name;
     string Type;
     string Description;
-    bool   UserSpecific(false);
+    [depr_ver("1.01")] bool UserSpecific(false);
 };
 
 ESPresponse [exceptions_inline] CreateStoreResponse
@@ -151,9 +151,10 @@ ESPresponse [exceptions_inline] CreateStoreResponse
     string Type;
     string Description;
     string Owner;
+    [min_ver("1.01")] bool Success;
 };
 
-ESPservice [version("1.00"), default_client_version("1.00"), auth_feature("WsStoreAccess:READ"), exceptions_inline("./smc_xslt/exceptions.xslt")] wsstore
+ESPservice [version("1.01"), default_client_version("1.01"), auth_feature("WsStoreAccess:READ"), exceptions_inline("./smc_xslt/exceptions.xslt")] wsstore
 {
     ESPmethod [auth_feature("WsStoreAccess:FULL")] Delete(DeleteRequest, DeleteResponse);
     ESPmethod [auth_feature("WsStoreAccess:FULL")] DeleteNamespace(DeleteNamespaceRequest, DeleteNamespaceResponse);

esp/scm/ws_workunits.ecm

@@ -163,6 +163,18 @@ ESPStruct [nil_remove] ECLQuery
 
 //  ===========================================================================
 
+ESPStruct [nil_remove] QueryStatsRecord
+{
+    string StartTime;
+    int64 ElapsedTimeMs;
+    int64 MemoryUsed;
+    int64 BytesOut;
+    int SlavesReplyLen;
+    bool Failed;
+};
+
+//  ===========================================================================
+
 ESPStruct [nil_remove] QuerySummaryStats
 {
     string Endpoint;
@@ -183,6 +195,24 @@ ESPStruct [nil_remove] QuerySummaryStats
 
 //  ===========================================================================
 
+ESPStruct [nil_remove] QueryStats
+{
+    string ID; //QueryID or Global
+    ESParray<ESPstruct QuerySummaryStats> AggregateQueryStatsList;
+    ESParray<ESPstruct QueryStatsRecord> QueryStatsRecordList;
+};
+
+//  ===========================================================================
+
+ESPStruct [nil_remove] EndpointQueryStats
+{
+    string Endpoint;
+    string Status;
+    ESParray<ESPstruct QueryStats> QueryStatsList;
+};
+
+//  ===========================================================================
+
 ESPStruct [nil_remove] DebugValue
 {
     string Name;
@@ -1187,11 +1217,13 @@ ESPrequest WUQueryGetSummaryStatsRequest
     string QueryId;
     string FromTime; //YYYY-MM-DDTHH:MM:SS
     string ToTime;
+    [min_ver("1.75")] bool IncludeRawStats(false);
 };
 
 ESPresponse [exceptions_inline] WUQueryGetSummaryStatsResponse
 {
     ESParray<ESPstruct QuerySummaryStats> StatsList;
+    [min_ver("1.75")] ESParray<ESPstruct EndpointQueryStats> QueryStatsList;
 };
 
 ESPrequest WUExportRequest
@@ -2265,7 +2297,7 @@ ESPresponse [exceptions_inline] WUEclDefinitionActionResponse
 // ----------------------------------------------------------------------------------
 ESPservice [
     auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
-    version("1.74"), default_client_version("1.74"), cache_group("ESPWsWUs"),
+    version("1.75"), default_client_version("1.75"), cache_group("ESPWsWUs"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/ws_store/ws_storeService.cpp

@@ -120,7 +120,11 @@ IEspStore* CwsstoreEx::loadStoreProvider(const char* instanceName, const char* l
 bool CwsstoreEx::onCreateStore(IEspContext &context, IEspCreateStoreRequest &req, IEspCreateStoreResponse &resp)
 {
     const char *user = context.queryUserId();
-    m_storeProvider->createStore(req.getType(), req.getName(), req.getDescription(), new CSecureUser(user, nullptr));
+    double version = context.getClientVersion();
+    bool success = m_storeProvider->createStore(req.getType(), req.getName(), req.getDescription(), new CSecureUser(user, nullptr));
+    if (version > 1)
+      resp.setSuccess(success);
+
     resp.setName(req.getName());
     resp.setType(req.getType());
     resp.setDescription(req.getDescription());
@@ -139,7 +143,9 @@ bool CwsstoreEx::onDelete(IEspContext &context, IEspDeleteRequest &req, IEspDele
         if (!m_defaultStore.isEmpty())
             storename = m_defaultStore.get();
     }
-    return m_storeProvider->deletekey(storename, req.getNamespace(), req.getKey(), new CSecureUser(user, nullptr), !req.getUserSpecific());
+
+    resp.setSuccess( m_storeProvider->deletekey(storename, req.getNamespace(), req.getKey(), new CSecureUser(user, nullptr), !req.getUserSpecific()));
+    return true;
 }
 
 bool CwsstoreEx::onDeleteNamespace(IEspContext &context, IEspDeleteNamespaceRequest &req, IEspDeleteNamespaceResponse &resp)
@@ -151,7 +157,7 @@ bool CwsstoreEx::onDeleteNamespace(IEspContext &context, IEspDeleteNamespaceRequ
 
     if (!global && !isEmptyString(targetUser))
     {
-        ESPLOG(LogMin, "CwsstoreEx::onDeleteNamespace: '%s' requesting to delete namespace on behalve of '%s'", user, targetUser);
+        ESPLOG(LogMin, "CwsstoreEx::onDeleteNamespace: '%s' requesting to delete namespace on behalf of '%s'", user, targetUser);
         user = targetUser;
     }
 
@@ -161,7 +167,8 @@ bool CwsstoreEx::onDeleteNamespace(IEspContext &context, IEspDeleteNamespaceRequ
             storename = m_defaultStore.get();
     }
 
-    return m_storeProvider->deleteNamespace(storename, req.getNamespace(), new CSecureUser(user, nullptr), !req.getUserSpecific());
+    resp.setSuccess(m_storeProvider->deleteNamespace(storename, req.getNamespace(), new CSecureUser(user, nullptr), !req.getUserSpecific()));
+    return true;
 }
 
 bool CwsstoreEx::onListNamespaces(IEspContext &context, IEspListNamespacesRequest &req, IEspListNamespacesResponse &resp)
@@ -178,6 +185,7 @@ bool CwsstoreEx::onListNamespaces(IEspContext &context, IEspListNamespacesReques
     StringArray namespaces;
     m_storeProvider->fetchAllNamespaces(namespaces, storename, new CSecureUser(user, nullptr), !req.getUserSpecific());
     resp.setNamespaces(namespaces);
+    resp.setStoreName(storename);
     return true;
 }
 
@@ -197,6 +205,7 @@ bool CwsstoreEx::onListKeys(IEspContext &context, IEspListKeysRequest &req, IEsp
     m_storeProvider->fetchKeySet(keys, storename, ns, new CSecureUser(user, nullptr), !req.getUserSpecific());
     resp.setKeySet(keys);
     resp.setNamespace(ns);
+    resp.setStoreName(storename);
 
     return true;
 }
@@ -215,7 +224,8 @@ bool CwsstoreEx::onSet(IEspContext &context, IEspSetRequest &req, IEspSetRespons
     }
 
     const char *user = context.queryUserId();
-    m_storeProvider->set(storename, ns, key, value, new CSecureUser(user, nullptr), !req.getUserSpecific());
+    resp.setSuccess(m_storeProvider->set(storename, ns, key, value, new CSecureUser(user, nullptr), !req.getUserSpecific()));
+
     return true;
 }
 
@@ -233,6 +243,7 @@ bool CwsstoreEx::onFetch(IEspContext &context, IEspFetchRequest &req, IEspFetchR
 
     m_storeProvider->fetch(storename, req.getNamespace(), req.getKey(), value, new CSecureUser(user, nullptr), !req.getUserSpecific());
     resp.setValue(value.str());
+
     return true;
 }
 
@@ -270,6 +281,7 @@ bool CwsstoreEx::onFetchKeyMetadata(IEspContext &context, IEspFetchKeyMDRequest
     resp.setStoreName(storename);
     resp.setNamespace(req.getNamespace());
     resp.setKey(req.getKey());
+
     return true;
 }
 

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -2980,7 +2980,11 @@ int WUSchedule::run()
     unsigned int waitTimeMillies = 1000*60;
     while(!stopping)
     {
-        if (!detached)
+        if (!m_container)
+        {
+            DBGLOG("ECLWorkunit WUSchedule Thread is waiting for container to be set.");
+        }
+        else if (!detached)
         {
             try
             {

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -3332,99 +3332,230 @@ bool CWsWorkunitsEx::onWUGetNumFileToCopy(IEspContext& context, IEspWUGetNumFile
     return true;
 }
 
-void getSummaryStatsByQueryId(const char *target, const char *queryId, const char *fromTime, const char *toTime, IArrayOf<IEspQuerySummaryStats>& querySummaryStatsList)
+bool CWsWorkunitsEx::onWUQueryGetSummaryStats(IEspContext& context, IEspWUQueryGetSummaryStatsRequest& req, IEspWUQueryGetSummaryStatsResponse& resp)
 {
-    if (!target || !*target)
-        throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Target name required");
-    if (!queryId || !*queryId)
-        throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Query Id required");
+    try
+    {
+        const char *target = req.getTarget();
+        if (isEmptyString(target))
+            throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Target name required");
 
-    Owned<IConstWUClusterInfo> info = getTargetClusterInfo(target);
-    if (!info || (info->getPlatform()!=RoxieCluster)) //Only support roxie for now
-        throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid Roxie name");
+        Owned<IConstWUClusterInfo> info = getTargetClusterInfo(target);
+        if (!info || (info->getPlatform()!=RoxieCluster)) //Only support roxie for now
+            throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Roxie name not found");
 
-    PROGLOG("getSummaryStatsByQueryId: target %s, query %s", target, queryId);
+        double version = context.getClientVersion();
+        const char *queryId = req.getQueryId();
+        if (!isEmptyString(queryId))
+            PROGLOG("WUQueryGetSummaryStats: target %s, query %s", target, queryId);
+        else
+            PROGLOG("WUQueryGetSummaryStats: target %s", target);
 
-    const SocketEndpointArray &eps = info->getRoxieServers();
-    if (eps.empty())
-        return;
+        const SocketEndpointArray &eps = info->getRoxieServers();
+        if (eps.empty())
+        {
+            IERRLOG("WUQueryGetSummaryStats: Failed to getRoxieServers for %s", target);
+            return true;
+        }
 
-    VStringBuffer control("<control:queryAggregates");
-    if (!isEmpty(fromTime))
-        control.appendf(" from='%s'", fromTime);
-    if (!isEmpty(toTime))
-        control.appendf(" to='%s'", toTime);
-    control.appendf("><Query id='%s'/></control:queryAggregates>", queryId);
-    Owned<IPropertyTree> queryAggregates = sendRoxieControlAllNodes(eps.item(0), control.str(), false, ROXIELOCKCONNECTIONTIMEOUT);
-    if (!queryAggregates)
-        return;
+        bool includeRawStats = req.getIncludeRawStats();
+        const char *fromTime = req.getFromTime();
+        const char *toTime = req.getToTime();
+        VStringBuffer control("<control:queryAggregates");
+        if (!isEmpty(fromTime))
+            control.appendf(" from='%s'", fromTime);
+        if (!isEmpty(toTime))
+            control.appendf(" to='%s'", toTime);
+        if (includeRawStats)
+        {
+            if (!isEmptyString(queryId))
+                control.appendf(" rawStats='1'><Query id='%s'/></control:queryAggregates>", queryId);
+            else
+                control.append(" all='1' rawStats='1' />");
+        }
+        else if (!isEmptyString(queryId))
+        {
+            control.appendf("><Query id='%s'/></control:queryAggregates>", queryId);
+        }
+        else
+            control.append(" />");
+
+        Owned<IPropertyTree> queryAggregates = sendRoxieControlAllNodes(eps.item(0), control.str(), false, ROXIELOCKCONNECTIONTIMEOUT);
+        if (!queryAggregates)
+        {
+            PROGLOG("WUQueryGetSummaryStats: %s returns empty for %s", control.str(), target);
+            return true;
+        }
+
+        if (getEspLogLevel() >= LogMax)
+        {
+            StringBuffer sb;
+            toXML(queryAggregates, sb);
+            DBGLOG("getQueryStats(): '%s' => '%s'", control.str(), sb.str());
+        }
+
+        IArrayOf<IEspQuerySummaryStats> querySummaryStatsList;
+        IArrayOf<IEspEndpointQueryStats> queryStatsList;
+        Owned<IPropertyTreeIterator> queryStatsOnEndpointItr = queryAggregates->getElements("Endpoint");
+        ForEach(*queryStatsOnEndpointItr)
+        {
+            IPropertyTree &queryStatsOnEndpoint = queryStatsOnEndpointItr->query();
+            const char *status = queryStatsOnEndpoint.queryProp("Status");
+            const char *ep = queryStatsOnEndpoint.queryProp("@ep");
+            if (isEmptyString(ep))
+                continue;
+
+            if (version >= 1.75)
+            {
+                if (includeRawStats)
+                    readQueryStatsList(&queryStatsOnEndpoint, status, ep, includeRawStats, queryStatsList);
+                else
+                    readQueryAggregateStats(queryStatsOnEndpoint.queryPropTree("Query"), status, ep, querySummaryStatsList);
+            }
+            else if (!isEmptyString(queryId))
+                readQueryAggregateStats(queryStatsOnEndpoint.queryPropTree("Query"), status, ep, querySummaryStatsList);
+        }
 
-    if (getEspLogLevel() >= LogMax)
+        resp.setStatsList(querySummaryStatsList);
+        resp.setQueryStatsList(queryStatsList);
+    }
+    catch(IException* e)
     {
-        StringBuffer sb;
-        toXML(queryAggregates, sb);
-        DBGLOG("getSummaryStatsByQueryId(): '%s' => '%s'", control.str(), sb.str());
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
     }
+    return true;
+}
 
-    //Parse queryAggregates and build querySummaryStatsList.
-    Owned<IPropertyTreeIterator> aggregates = queryAggregates->getElements("Endpoint");
-    ForEach(*aggregates)
-    {
-        IPropertyTree &aggregate = aggregates->query();
-        const char *status = aggregate.queryProp("Status");
-        const char *ep = aggregate.queryProp("@ep");
-        if (isEmptyString(ep))
-            continue;
+void CWsWorkunitsEx::readQueryAggregateStats(IPropertyTree *queryStats, const char *status, const char *ep,
+    IArrayOf<IEspQuerySummaryStats> &querySummaryStatsList)
+{
+    if (!queryStats)
+        return;
 
-        IPropertyTree *query = aggregate.queryPropTree("Query");
-        Owned<IEspQuerySummaryStats> querySummaryStats = createQuerySummaryStats();
+    Owned<IEspQuerySummaryStats> querySummaryStats = createQuerySummaryStats();
+    if (!isEmptyString(ep))
         querySummaryStats->setEndpoint(ep);
-        if (query->hasProp("countFailed"))
-            querySummaryStats->setCountFailed(query->getPropInt("countFailed"));
-        if (query->hasProp("countTotal"))
-            querySummaryStats->setCountTotal(query->getPropInt("countTotal"));
-        if (query->hasProp("averageBytesOut"))
-            querySummaryStats->setAverageBytesOut(query->getPropInt64("averageBytesOut"));
-        if (query->hasProp("averageMemUsed"))
-            querySummaryStats->setSizeAvgPeakMemory(query->getPropInt64("averageMemUsed"));
-        if (query->hasProp("averageSlavesReplyLen"))
-            querySummaryStats->setAverageSlavesReplyLen(query->getPropInt("averageSlavesReplyLen"));
-        if (query->hasProp("averageTimeMs"))
-            querySummaryStats->setTimeAvgTotalExecuteMinutes(query->getPropInt64("averageTimeMs"));
-        if (query->hasProp("minTimeMs"))
-            querySummaryStats->setTimeMinTotalExecuteMinutes(query->getPropInt64("minTimeMs"));
-        if (query->hasProp("maxTimeMs"))
-            querySummaryStats->setTimeMaxTotalExecuteMinutes(query->getPropInt64("maxTimeMs"));
-        if (query->hasProp("percentile97"))
-        {
-            querySummaryStats->setPercentile97(query->getPropInt("percentile97"));
-            if (query->hasProp("percentile97/@estimate"))
-                querySummaryStats->setPercentile97Estimate(query->getPropBool("percentile97/@estimate"));
-        }
-        const char *startTime = query->queryProp("startTime");
-        const char *endTime = query->queryProp("endTime");
-        if (!isEmptyString(startTime))
-            querySummaryStats->setStartTime(startTime);
-        if (!isEmptyString(endTime))
-            querySummaryStats->setEndTime(endTime);
-        if (!isEmptyString(status))
-            querySummaryStats->setStatus(status);
-        querySummaryStatsList.append(*querySummaryStats.getLink());
+    if (queryStats->hasProp("countFailed"))
+        querySummaryStats->setCountFailed(queryStats->getPropInt("countFailed"));
+    if (queryStats->hasProp("countTotal"))
+        querySummaryStats->setCountTotal(queryStats->getPropInt("countTotal"));
+    if (queryStats->hasProp("averageBytesOut"))
+        querySummaryStats->setAverageBytesOut(queryStats->getPropInt64("averageBytesOut"));
+    if (queryStats->hasProp("averageMemUsed"))
+        querySummaryStats->setSizeAvgPeakMemory(queryStats->getPropInt64("averageMemUsed"));
+    if (queryStats->hasProp("averageSlavesReplyLen"))
+        querySummaryStats->setAverageSlavesReplyLen(queryStats->getPropInt("averageSlavesReplyLen"));
+    if (queryStats->hasProp("averageTimeMs"))
+        querySummaryStats->setTimeAvgTotalExecuteMinutes(queryStats->getPropInt64("averageTimeMs"));
+    if (queryStats->hasProp("minTimeMs"))
+        querySummaryStats->setTimeMinTotalExecuteMinutes(queryStats->getPropInt64("minTimeMs"));
+    if (queryStats->hasProp("maxTimeMs"))
+        querySummaryStats->setTimeMaxTotalExecuteMinutes(queryStats->getPropInt64("maxTimeMs"));
+    if (queryStats->hasProp("percentile97"))
+    {
+        querySummaryStats->setPercentile97(queryStats->getPropInt("percentile97"));
+        if (queryStats->hasProp("percentile97/@estimate"))
+            querySummaryStats->setPercentile97Estimate(queryStats->getPropBool("percentile97/@estimate"));
+    }
+    const char *startTime = queryStats->queryProp("startTime");
+    const char *endTime = queryStats->queryProp("endTime");
+    if (!isEmptyString(startTime))
+        querySummaryStats->setStartTime(startTime);
+    if (!isEmptyString(endTime))
+        querySummaryStats->setEndTime(endTime);
+    if (!isEmptyString(status))
+        querySummaryStats->setStatus(status);
+    querySummaryStatsList.append(*querySummaryStats.getLink());
+}
+
+void CWsWorkunitsEx::readQueryStatsList(IPropertyTree *queryStatsTree, const char *status, const char *ep,
+    bool includeRawStats, IArrayOf<IEspEndpointQueryStats> &endpointQueryStatsList)
+{
+    if (!queryStatsTree)
+        return;
+
+    IArrayOf<IEspQueryStats> queryStatsList;
+    Owned<IPropertyTreeIterator> queryItr = queryStatsTree->getElements("QueryStats/Query");
+    ForEach(*queryItr)
+    {
+        IPropertyTree &query = queryItr->query();
+        readQueryStats(&query, query.queryProp("@id"), includeRawStats, queryStatsList);
     }
-    return;
+
+    IPropertyTree *globalStats = queryStatsTree->queryPropTree("QueryStats/Global");
+    if (globalStats)
+        readQueryStats(globalStats, "Global", includeRawStats, queryStatsList);
+
+    if (queryStatsList.ordinality() == 0)
+        return;
+
+    Owned<IEspEndpointQueryStats> endpointQueryStats = createEndpointQueryStats();
+    endpointQueryStats->setEndpoint(ep);
+    if (!isEmptyString(status))
+        endpointQueryStats->setStatus(status);
+
+    endpointQueryStats->setQueryStatsList(queryStatsList);
+    endpointQueryStatsList.append(*endpointQueryStats.getLink());
 }
 
-bool CWsWorkunitsEx::onWUQueryGetSummaryStats(IEspContext& context, IEspWUQueryGetSummaryStatsRequest& req, IEspWUQueryGetSummaryStatsResponse& resp)
+void CWsWorkunitsEx::readQueryStats(IPropertyTree *queryStatsTree, const char *id, bool includeRawStats,
+    IArrayOf<IEspQueryStats> &queryStatsList)
 {
-    try
+    if (!queryStatsTree)
+        return;
+
+    Owned<IEspQueryStats> queryStats = createQueryStats();
+    if (!isEmptyString(id))
+        queryStats->setID(id);
+
+    if (!includeRawStats)
     {
-        IArrayOf<IEspQuerySummaryStats> querySummaryStatsList;
-        getSummaryStatsByQueryId(req.getTarget(), req.getQueryId(), req.getFromTime(), req.getToTime(), querySummaryStatsList);
-        resp.setStatsList(querySummaryStatsList);
+        IArrayOf<IEspQuerySummaryStats> aggregateQueryStatsList;
+        readQueryAggregateStats(queryStatsTree, nullptr, nullptr, aggregateQueryStatsList);
+        queryStats->setAggregateQueryStatsList(aggregateQueryStatsList);
+        queryStatsList.append(*queryStats.getLink());
+        return;
     }
-    catch(IException* e)
+
+    IArrayOf<IEspQuerySummaryStats> aggregateQueryStatsList;
+    Owned<IPropertyTreeIterator> aggregateRecordItr = queryStatsTree->getElements("QueryStatsAggregateRecord");
+    ForEach(*aggregateRecordItr)
     {
-        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+        IPropertyTree &query = aggregateRecordItr->query();
+        readQueryAggregateStats(&query, nullptr, nullptr, aggregateQueryStatsList);
     }
-    return true;
+    queryStats->setAggregateQueryStatsList(aggregateQueryStatsList);
+
+    IArrayOf<IEspQueryStatsRecord> recordList;
+    Owned<IPropertyTreeIterator> recordItr = queryStatsTree->getElements("QueryStatsRecord");
+    ForEach(*recordItr)
+    {
+        IPropertyTree &query = recordItr->query();
+        readQueryStatsRecord(&query, recordList);
+    }
+    queryStats->setQueryStatsRecordList(recordList);
+
+    queryStatsList.append(*queryStats.getLink());
+}
+
+void CWsWorkunitsEx::readQueryStatsRecord(IPropertyTree *queryRecord, IArrayOf<IEspQueryStatsRecord> &recordList)
+{
+    if (!queryRecord)
+        return;
+
+    Owned<IEspQueryStatsRecord> record = createQueryStatsRecord();
+    const char *startTime = queryRecord->queryProp("@startTime");
+    if (!isEmptyString(startTime))
+        record->setStartTime(startTime);
+
+    if (queryRecord->hasProp("elapsedTimeMs"))
+        record->setElapsedTimeMs(queryRecord->getPropInt64("elapsedTimeMs"));
+    if (queryRecord->hasProp("memUsed"))
+        record->setMemoryUsed(queryRecord->getPropInt64("memUsed"));
+    if (queryRecord->hasProp("bytesOut"))
+        record->setBytesOut(queryRecord->getPropInt64("bytesOut"));
+    if (queryRecord->hasProp("slavesReplyLen"))
+        record->setSlavesReplyLen(queryRecord->getPropInt("slavesReplyLen"));
+
+    recordList.append(*record.getLink());
 }

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -337,6 +337,13 @@ private:
         const char *cluster, IArrayOf<IEspECLJob> &eclJobList, IArrayOf<IEspECLJob> &inProgressECLJobList);
     bool getThorJobsFromAuditLog(double version, CDateTime &queryAuditLogFrom, CDateTime &queryAuditLogTo,
         const char *cluster, unsigned maxJobsToReturn, IArrayOf<IEspECLJob> &eclJobList);
+    void readQueryAggregateStats(IPropertyTree *queryStats, const char *status, const char *ep,
+        IArrayOf<IEspQuerySummaryStats> &querySummaryStatsList);
+    void readQueryStatsRecord(IPropertyTree *queryRecord, IArrayOf<IEspQueryStatsRecord> &recordList);
+    void readQueryStats(IPropertyTree *queryStatsTree, const char *id, bool all,
+        IArrayOf<IEspQueryStats> &queryStatsList);
+    void readQueryStatsList(IPropertyTree *queryStatsTree, const char *status, const char *ep,
+        bool all, IArrayOf<IEspEndpointQueryStats> &endpointQueryStatsList);
 
     unsigned awusCacheMinutes;
     StringBuffer queryDirectory;

esp/src/eclwatch/ActivityWidget.js

@@ -530,6 +530,14 @@ define([
                 return retVal;
             },
 
+            getDetailID: function (row, params) {
+                var retVal = "Detail" + row[this.idProperty];
+                if (params && params.OpenMode === "Graph") {
+                    retVal += "Graph";
+                }
+                return retVal;
+            },
+
             createDetail: function (id, row, params) {
                 if (params && params.usage) {
                     return new DelayLoadWidget({

esp/src/eclwatch/ResourcesWidget.js

@@ -99,6 +99,14 @@ define([
                 });
             },
 
+            getDetailID: function (row, params) {
+                var retVal = "Detail" + row[this.idProperty];
+                if (params && params.showSource) {
+                    retVal += "Source";
+                }
+                return retVal;
+            },
+
             createDetail: function (id, row, params) {
                 if (params && params.showSource) {
                     return new DelayLoadWidget({

esp/src/eclwatch/ResultsWidget.js

@@ -144,12 +144,14 @@ define([
 
             getDetailID: function (row, params) {
                 var retVal = "Detail" + row[this.idProperty];
-                if (params) {
-                    if (params.vizMode) {
-                        retVal += "Viz";
-                    } else if (params.legacyMode) {
-                        retVal += "Legacy";
-                    }
+                if (params && params.vizMode) {
+                    retVal += "Viz";
+                } else if (params && params.legacyMode) {
+                    retVal += "Legacy";
+                } else if (row.FileName && params && params.logicalFile) {
+                    retVal += "LogicalFile";
+                } else if (params && params.resultView && params.viewName) {
+                    retVal += "View";
                 }
                 return retVal;
             },

esp/tools/soapplus/http.cpp

@@ -495,7 +495,19 @@ HttpClient::HttpClient(IProperties* globals, const char* url, const char* inname
         {
 #ifdef _USE_OPENSSL
             if(m_ssctx.get() == NULL)
-                m_ssctx.setown(createSecureSocketContext(ClientSocket));
+            {
+                Owned<IPropertyTree> cfgtree = nullptr;
+                if (globals->hasProp("cfg"))
+                {
+                    const char* cfg = globals->queryProp("cfg");
+                    if (cfg && *cfg)
+                        cfgtree.setown(createPTreeFromXMLFile(cfg));
+                }
+                if (cfgtree)
+                    m_ssctx.setown(createSecureSocketContextEx2(cfgtree, ClientSocket));
+                else
+                    m_ssctx.setown(createSecureSocketContext(ClientSocket));
+            }
 #else
         throw MakeStringException(-1, "HttpClient: failure to create SSL socket - OpenSSL not enabled in build");
 #endif

initfiles/bin/init_thorslave.in

@@ -66,14 +66,13 @@ stop_slaves()
 
 start_slaves()
 {
-    # insuring dafilesrv is running on the machine as it is a prerequisite
-    dafilesrv_owner=$(stat -c "%U" ${INIT_PATH}/dafilesrv 2>/dev/null || stat -f %Su ${INIT_PATH}/dafilesrv 2>/dev/null)
-    # use full path as with CentOS devtoolset another sudo is found in path
-    # which does not work properly with changes in sudoers.d hpcc file
-    [[ "${dafilesrv_owner}" != "${user}" ]] && cmd_prefix="/usr/bin/sudo"
-    # is sudo required for checking status ?
-    ${cmd_prefix} ${INIT_PATH}/dafilesrv status > /dev/null
+    killall -0 dafilesrv > /dev/null 2>&1
     if [[ $? -ne 0 ]];then
+        # insuring dafilesrv is running on the machine as it is a prerequisite
+        dafilesrv_owner=$(stat -c "%U" ${INIT_PATH}/dafilesrv 2>/dev/null || stat -f %Su ${INIT_PATH}/dafilesrv 2>/dev/null)
+        # use full path as with CentOS devtoolset another sudo is found in path
+        # which does not work properly with changes in sudoers.d hpcc file
+        [[ "${dafilesrv_owner}" != "${user}" ]] && cmd_prefix="/usr/bin/sudo"
         ${cmd_prefix} ${INIT_PATH}/dafilesrv start > /dev/null
         if [[ $? -ne 0 ]];then
             exit 1

initfiles/componentfiles/configschema/xsd/esp.xsd

@@ -144,7 +144,6 @@
                                 <xs:simpleType>
                                     <xs:restriction base="xs:string">
                                         <xs:enumeration value="none" hpcc:description=""/>
-                                        <xs:enumeration value="local" hpcc:description=""/>
                                         <xs:enumeration value="ldap"
                                                         hpcc:requiredAttributes="ldapServer,ldapAuthMethod,ldapConnections,passwordExpirationWarningDays,checkViewPermissions"
                                                         hpcc:description=""/>

initfiles/componentfiles/configxml/@temp/esp_service.xsl

@@ -173,13 +173,6 @@ xmlns:seisint="http://seisint.com"  xmlns:set="http://exslt.org/sets" exclude-re
                     </xsl:for-each>
                 </Authenticate>
             </xsl:when>
-            <xsl:when test="$authMethod='local'">
-                <Authenticate method="Local">
-                    <xsl:for-each select="$bindingNode/Authenticate[string(@path) != '']">
-                        <Location path="{@path}" resource="{@resource}" required="{@access}" description="{@description}"/>
-                    </xsl:for-each>
-                </Authenticate>
-            </xsl:when>
             <xsl:when test="$authMethod='ldap' or $authMethod='ldaps'">
                 <Authenticate method="LdapSecurity" config="ldapserver">
                     <xsl:copy-of select="$bindingNode/@resourcesBasedn"/>

initfiles/componentfiles/configxml/@temp/esp_service_WsSMC.xsl

@@ -712,13 +712,6 @@ This is required by its binding with ESP service '<xsl:value-of select="$espServ
             </xsl:for-each>
          </Authenticate>
       </xsl:when>
-      <xsl:when test="$authMethod='local'">
-         <Authenticate method="Local">
-            <xsl:for-each select="$bindingNode/Authenticate[string(@path) != '']">
-               <Location path="{@path}" resource="{@resource}" required="{@access}" description="{@description}"/>
-            </xsl:for-each>
-         </Authenticate>
-      </xsl:when>
       <xsl:when test="$authMethod='ldap' or $authMethod='ldaps'">
          <Authenticate method="LdapSecurity" config="ldapserver">
             <xsl:copy-of select="$bindingNode/@resourcesBasedn"/> <!--if binding has an ldap resourcebasedn specified then copy it out -->

initfiles/componentfiles/configxml/@temp/esp_service_wsecl.xsl

@@ -194,13 +194,6 @@
                </xsl:for-each>
             </Authenticate>
          </xsl:when>
-         <xsl:when test="$authMethod='local'">
-            <Authenticate method="Local">
-               <xsl:for-each select="$bindingNode/Authenticate[string(@path) != '']">
-                  <Location path="{@path}" resource="{@resource}" required="{@access}" description="{@description}"/>
-               </xsl:for-each>
-            </Authenticate>
-         </xsl:when>
          <xsl:when test="$authMethod='ldap' or $authMethod='ldaps'">
             <Authenticate method="LdapSecurity" config="ldapserver">
                <xsl:copy-of select="$bindingNode/@resourcesBasedn"/> <!--if binding has an ldap resourcebasedn specified then copy it out -->

initfiles/componentfiles/configxml/@temp/esp_service_wslogging.xsl

@@ -65,13 +65,6 @@ xmlns:seisint="http://seisint.com"  xmlns:set="http://exslt.org/sets" exclude-re
                </xsl:for-each>
             </Authenticate>
          </xsl:when>
-         <xsl:when test="$authMethod='local'">
-            <Authenticate method="Local">
-               <xsl:for-each select="$bindingNode/Authenticate[string(@path) != '']">
-                  <Location path="{@path}" resource="{@resource}" required="{@access}" description="{@description}"/>
-               </xsl:for-each>
-            </Authenticate>
-         </xsl:when>
          <xsl:when test="$authMethod='ldap' or $authMethod='ldaps'">
             <Authenticate method="LdapSecurity" config="ldapserver">
             <xsl:copy-of select="$bindingNode/@resourcesBasedn"/> <!--if binding has an ldap resourcebasedn specified then copy it out -->

initfiles/componentfiles/configxml/esp.xsd.in

@@ -363,7 +363,6 @@
                             <xs:simpleType>
                                 <xs:restriction base="xs:string">
                                     <xs:enumeration value="none"/>
-                                    <xs:enumeration value="local"/>
                                     <xs:enumeration value="userNameOnly"/>
                                     <xs:enumeration value="ldap"/>
                                     <xs:enumeration value="ldaps"/>

initfiles/componentfiles/configxml/esp.xsl

@@ -674,13 +674,6 @@
                  </xsl:for-each>
               </Authenticate>
            </xsl:when>
-           <xsl:when test="$authMethod='local'">
-              <Authenticate method="Local">
-                 <xsl:for-each select="$bindingNode/Authenticate[string(@path) != '']">
-                    <Location path="{@path}" resource="{@resource}" required="{@access}" description="{@description}"/>
-                 </xsl:for-each>
-              </Authenticate>
-           </xsl:when>
            <xsl:when test="$authMethod='ldap' or $authMethod='ldaps'">
               <Authenticate method="LdapSecurity" config="ldapserver">
                  <xsl:copy-of select="$bindingNode/@resourcesBasedn"/>

initfiles/examples/embed/javaembed-stream.ecl

@@ -0,0 +1,339 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=embedded
+//class=3rdparty
+
+import java;
+
+/*
+ Similar to java-stream example, but using inline Java 
+ */
+
+// Passing and returning records and datasets
+// When passing/returning a record, the corresponding Java function should take/return an object as a parameter whose fields
+// can be mapped by name to the ECL record fields
+
+nrec := record
+  utf8 ufield;
+end;
+
+jret := RECORD
+  boolean bfield;
+  integer4 ifield;
+  integer8 lfield;
+  real8 dfield;
+  real4 ffield;
+  string1 cfield1;
+  string1 cfield2;
+  string sfield;
+  nrec n;
+  set of boolean bset;
+  set of data dset;
+  set of string sset;
+  LINKCOUNTED DATASET(nrec) sub;
+end;
+
+jret returnrec(boolean b, integer i, real8 d) := EMBED(java)
+import java.util.*;
+public class Test1
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test1(boolean b, int i, double d)
+  {
+    bfield = b;
+    ifield = i;
+    lfield = i * 100000000;
+    dfield = d;
+    ffield = (float) d;
+    sfield = "Yoohoo";
+    cfield1 = 'X';
+    cfield2 = "Z";
+    n = new NestedClass("nest");
+    bset = new boolean [5];
+    bset[3] = b;
+    dset = new byte[1][];
+    dset[0] = new byte[1];
+    dset[0][0] = 14;
+    sset = new String[1];
+    sset[0] = "Hello";
+    sub = new NestedClass[1];
+    sub[0] = new NestedClass("subnest");
+  }
+
+  public Test1()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static Test1 returnrec(boolean b, int i, double d)
+  {
+    return new Test1(b,i,d);
+  }
+}
+ENDEMBED;
+
+
+STRING passrec(jret r) := EMBED(java)
+import java.util.*;
+public class Test2
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test2()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static String passrec(Test2 j)
+  {
+    return j.n.ufield;
+  }
+}
+ENDEMBED;
+
+ret := returnrec(false, 10, 2.345);
+
+OUTPUT(ret);            // Calls a Java function that returns an ECL record
+OUTPUT(passrec(ret));  // Passes an ECL record to a Java function
+
+// When passing a dataset to a Java function, the Java function should take either an array or an iterator of objects,
+// where the fields of the object in question are mapped by name to the fields in the ECL record.
+//
+// To return a dataset, an iterator must be returned.
+
+INTEGER passDataset(LINKCOUNTED DATASET(jret) d) := EMBED(Java)
+import java.util.*;
+public class Test3
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test3()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static int passDataset(Iterator<Test3> d)
+  {
+    int sum = 0;
+    while (d.hasNext())
+    {
+      Test3 r = d.next();
+      sum += r.lfield;
+    }
+    return sum;
+  }
+
+}
+ENDEMBED;
+
+DATASET(jret) passDataset2(LINKCOUNTED DATASET(jret) d) := EMBED(Java)
+import java.util.*;
+public class Test4
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test4()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static Iterator<Test4> passDataset2(Test4 d[])
+  {
+    return Arrays.asList(d).iterator();
+  }
+}
+
+ENDEMBED;
+
+ds := DATASET(
+  [
+     {true, 1,2,3,4,'a', 'b', 'cd', u'ef', [true,false], [], ['Hello from ECL'], [{'1'},{'2'},{'3'},{'4'},{'5'}]}
+    ,{true, 2,4,3,4,'a', 'b', 'cd', u'ef', [true,false], [], [], []}
+    ,{true, 3,6,3,4,'a', 'b', 'cd', u'ef', [true,false], [], [], []}
+    ,{true, 8,8,3,4,'a', 'b', 'cd', u'ef', [true,false], [d'AA55'], [], []}
+  ], jret);
+
+output(passDataset(ds));  // Using an iterator
+output(passDataset2(ds)); // using an array, and illustrating the return of a dataset
+
+// It is also possible to code a traonsform function in Java - both the parameter and the return type should be a
+// Java object type that maps the fields of the ECL record by name.
+
+transform(jret) testTransform(jret in, integer lim) := EMBED(java)
+import java.util.*;
+public class Test5
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test5(boolean b, int i, double d)
+  {
+    bfield = b;
+    ifield = i;
+    lfield = i * 100000000;
+    dfield = d;
+    ffield = (float) d;
+    sfield = "Yoohoo";
+    cfield1 = 'X';
+    cfield2 = "Z";
+    n = new NestedClass("nest");
+    bset = new boolean [5];
+    bset[3] = b;
+    dset = new byte[1][];
+    dset[0] = new byte[1];
+    dset[0][0] = 14;
+    sset = new String[1];
+    sset[0] = "Hello";
+    sub = new NestedClass[1];
+    sub[0] = new NestedClass("subnest");
+  }
+
+  public Test5()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static Test5 testTransform(Test5 in, int lim)
+  {
+    return new Test5(in.bfield, lim, in.dfield);
+  }
+
+}
+ENDEMBED;
+
+output(project(ds, testTransform(LEFT, COUNTER)));

plugins/javaembed/HpccClassLoader.java

@@ -71,16 +71,17 @@ public class HpccClassLoader extends java.lang.ClassLoader
     }
     public synchronized Class<?> findClass(String className) throws ClassNotFoundException
     {
-        Class<?> result = classes.get(className);
+        String luName = className.replace(".","/");
+        Class<?> result = classes.get(luName);
         if (result == null)
         {
             if (bytecodeLen != 0)
-                result = defineClassForEmbed(bytecodeLen, bytecode, className.replace(".","/"));
+                result = defineClassForEmbed(bytecodeLen, bytecode, luName);
             if ( result == null && pathLoader != null)
                 result = pathLoader.loadClass(className);
             if (result == null)
                 return super.findClass(className);
-            classes.put(className, result);
+            classes.put(luName, result);
         }
         return result; 
     }

plugins/javaembed/javaembed.cpp

@@ -363,7 +363,9 @@ public:
     jobject ToReflectedField(jclass cls, jfieldID fieldID, jboolean isStatic) {
         return checkException(functions->ToReflectedField(this,cls,fieldID,isStatic));
     }
-
+    jboolean IsAssignableFrom(jclass clazz1, jclass clazz2) {
+        return checkException(functions->IsAssignableFrom(this,clazz1,clazz2));
+    }
     jclass GetObjectClass(jobject obj)
     {
         return checkException(JNIEnv::GetObjectClass(obj));
@@ -650,6 +652,7 @@ static jclass customLoaderClass;
 static jmethodID clc_newInstance;
 static jmethodID clc_getSignature;
 static jclass hpccIteratorClass;
+static jclass utilIteratorClass;
 static jmethodID hi_constructor;
 
 static jclass systemClass;
@@ -706,6 +709,7 @@ static void setupGlobals(CheckedJNIEnv *J)
         arrayList_toArray = J->GetMethodID(arrayListClass, "toArray", "()[Ljava/lang/Object;" );
 
         langStringClass = J->FindGlobalClass("java/lang/String");
+        utilIteratorClass = J->FindGlobalClass("java/util/Iterator");
 
         langIllegalArgumentExceptionClass = J->FindGlobalClass("java/lang/IllegalArgumentException");
     }
@@ -1809,14 +1813,13 @@ public:
       javaBuilder(JNIenv, &dummyField, className)
     {
         nextRead = false;
-        nextPending = NULL;
     }
 
     bool hasNext()
     {
         if (!nextRead)
         {
-            nextPending = (const byte *) val->ungroupedNextRow();
+            nextPending.setown(val->ungroupedNextRow());
             nextRead = true;
             if (!nextPending)
                 val->stop();
@@ -1828,7 +1831,7 @@ public:
     {
         if (!hasNext())
             return NULL;
-        typeInfo->process(nextPending, nextPending, &dummyField, javaBuilder); // Creates a java object from the incoming ECL row
+        typeInfo->process((const byte *) nextPending.get(), (const byte *) nextPending.get(), &dummyField, javaBuilder); // Creates a java object from the incoming ECL row
         nextRead = false;
         return javaBuilder.getObject();
     }
@@ -1837,7 +1840,7 @@ protected:
     Linked<IRowStream> val;
     RtlFieldStrInfo dummyField;
     JavaObjectBuilder javaBuilder;
-    const byte *nextPending;
+    roxiemem::OwnedConstRoxieRow nextPending;
     bool nextRead;
 };
 
@@ -1867,8 +1870,12 @@ public:
     JavaRowStream(jobject _iterator, IEngineRowAllocator *_resultAllocator)
     : resultAllocator(_resultAllocator)
     {
-        iterator = queryJNIEnv()->NewGlobalRef(_iterator, "iterator");
-        // Note that we can't cache the JNIEnv, iterClass, or methodIds here - calls may be made on different threads (though not at the same time).
+        CheckedJNIEnv *JNIenv = queryJNIEnv();
+        iterator = JNIenv->NewGlobalRef(_iterator, "iterator");
+        iterClass = (jclass) JNIenv->NewGlobalRef(JNIenv->GetObjectClass(iterator), "iterClass");
+        hasNextMethod = JNIenv->GetMethodID(iterClass, "hasNext", "()Z" );
+        nextMethod = JNIenv->GetMethodID(iterClass, "next", "()Ljava/lang/Object;" );
+        // Note that we can't save the JNIEnv value here - calls may be made on different threads (though not at the same time).
     }
     ~JavaRowStream()
     {
@@ -1887,15 +1894,12 @@ public:
         //    return NULL;
         // }
         // result = iterator.next();
-        jclass iterClass =JNIenv->GetObjectClass(iterator);
-        jmethodID hasNextMethod = JNIenv->GetMethodID(iterClass, "hasNext", "()Z" );
         jboolean hasNext = JNIenv->CallBooleanMethod(iterator, hasNextMethod);
         if (!hasNext)
         {
             stop();
             return NULL;
         }
-        jmethodID nextMethod = JNIenv->GetMethodID(iterClass, "next", "()Ljava/lang/Object;" );
         jobject result = JNIenv->CallObjectMethod(iterator, nextMethod);
         RtlDynamicRowBuilder rowBuilder(resultAllocator);
         const RtlTypeInfo *typeInfo = resultAllocator->queryOutputMeta()->queryTypeInfo();
@@ -1916,12 +1920,22 @@ public:
                 JNIenv->DeleteGlobalRef(iterator);
                 iterator = NULL;
             }
+            if (iterClass)
+            {
+                JNIenv->DeleteGlobalRef(iterClass);
+                iterClass = NULL;
+            }
+            hasNextMethod = nullptr;
+            nextMethod = nullptr;
         }
     }
 
 protected:
     Linked<IEngineRowAllocator> resultAllocator;
-    jobject iterator;
+    jobject iterator = nullptr;
+    jclass iterClass = nullptr;
+    jmethodID hasNextMethod = nullptr;
+    jmethodID nextMethod = nullptr;
 };
 
 const char *esdl2JavaSig(IEsdlDefinition &esdl, const char *esdlType)
@@ -2874,10 +2888,27 @@ public:
             b.read(attr_count);
             for (unsigned j = 0; j < attr_count; j++)
             {
-                b.skip(2);
+                uint16_t attr_name_idx; b.read(attr_name_idx);
+                StringAttr attrName;
+                readUtf(attrName, attr_name_idx);
                 uint32_t attr_length;
                 b.read(attr_length);
-                b.skip(attr_length);
+                if (streq(attrName, "Signature") && attr_length==2)
+                {
+                    uint16_t ext_sig_idx; b.read(ext_sig_idx);
+                    StringAttr extSig;
+                    readUtf(extSig, ext_sig_idx);
+#ifdef TRACE_CLASSFILE
+                    DBGLOG("Seen extended signature %s", extSig.str());
+#endif
+                    if (flags & ACC_PUBLIC)
+                    {
+                        methodSigs.pop();
+                        methodSigs.append(extSig);
+                    }
+                }
+                else
+                    b.skip(attr_length);
             }
         }
         /* Don't bother reading attributes as they are not really interesting to us
@@ -3050,7 +3081,13 @@ public:
         if (codeCtx)
         {
             engine = codeCtx->queryEngineContext();
-            nodeNum = codeCtx->getNodeNum();
+            try {
+                nodeNum = codeCtx->getNodeNum();
+            }
+            catch (IException *E)
+            {
+                E->Release();  // We may get an error if calling on the master - we just want to ignore it
+            }
         }
         StringBuffer lclassPath;
         if (engine)
@@ -3423,6 +3460,13 @@ public:
     }
     virtual IRowStream *getDatasetResult(IEngineRowAllocator * _resultAllocator)
     {
+        jclass iterClass =JNIenv->GetObjectClass(result.l);
+        if (!JNIenv->IsAssignableFrom(iterClass, utilIteratorClass))
+        {
+            StringBuffer s;
+            throw MakeStringException(0, "javaembed: In method %s: Java code should return an iterator", getReportName(s).str());
+            // MORE - perhaps we should also support Iterable?
+        }
         return new JavaRowStream(result.l, _resultAllocator);
     }
     virtual byte * getRowResult(IEngineRowAllocator * _resultAllocator)
@@ -4075,6 +4119,7 @@ public:
         if (persistMode==persistNone)
             instance = 0;  // otherwise we leave it for next call as it saves a lot of time looking it up
         JNIenv->PopLocalFrame(nullptr);
+        iterators.kill();
 #ifdef FORCE_GC
         forceGC(JNIenv);
 #endif

roxie/ccd/ccdcontext.cpp

@@ -2786,8 +2786,8 @@ protected:
 
     void initDebugMode(bool breakAtStart, const char *debugUID)
     {
-        if (!debugPermitted || !ownEP.port || !nativeProtocol)
-            throw MakeStringException(ROXIE_ACCESS_ERROR, "Debug queries are not permitted on this system");
+        if (!debugPermitted || !ownEP.port || nativeProtocol)
+            throw MakeStringException(ROXIE_ACCESS_ERROR, "Debug query not permitted here");
         debugContext.setown(new CRoxieServerDebugContext(this, logctx, factory->cloneQueryXGMML()));
         debugContext->debugInitialize(debugUID, factory->queryQueryName(), breakAtStart);
         if (workUnit)

roxie/ccd/ccdprotocol.cpp

@@ -1889,7 +1889,8 @@ readAnother:
                     }
                 }
 
-                msgctx->initQuery(querySetName, queryName); //needed here to allow checking hash options
+                if (!streq(queryPrefix.str(), "debug"))
+                    msgctx->initQuery(querySetName, queryName); //needed here to allow checking hash options
 
                 if (whitespace == WhiteSpaceHandling::Default) //value in the request wins
                     whitespace = msgctx->getStripWhitespace() ? WhiteSpaceHandling::Strip : WhiteSpaceHandling::Preserve; //might be changed by hash option, returns default otherwise
@@ -1935,109 +1936,111 @@ readAnother:
                     sink->onDebugMsg(msgctx, uid, queryPT, out);
                     response.append(out.str());
                 }
-
-                Owned<IActiveQueryLimiter> l;
-                if (queryLimiterFactory)
-                    l.setown(queryLimiterFactory->create(listener));
-                if (l && !l->isAccepted())
+                else
                 {
-                    if (isHTTP)
+                    Owned<IActiveQueryLimiter> l;
+                    if (queryLimiterFactory)
+                        l.setown(queryLimiterFactory->create(listener));
+                    if (l && !l->isAccepted())
                     {
-                        sendHttpServerTooBusy(*client, logctx);
-                        logctx.CTXLOG("FAILED: %s", sanitizedText.str());
-                        logctx.CTXLOG("EXCEPTION: Too many active queries");
+                        if (isHTTP)
+                        {
+                            sendHttpServerTooBusy(*client, logctx);
+                            logctx.CTXLOG("FAILED: %s", sanitizedText.str());
+                            logctx.CTXLOG("EXCEPTION: Too many active queries");
+                        }
+                        else
+                        {
+                            IException *e = MakeStringException(ROXIE_TOO_MANY_QUERIES, "Too many active queries");
+                            if (msgctx->trapTooManyActiveQueries())
+                                logctx.logOperatorException(e, __FILE__, __LINE__, NULL);
+                            throw e;
+                        }
                     }
                     else
                     {
-                        IException *e = MakeStringException(ROXIE_TOO_MANY_QUERIES, "Too many active queries");
-                        if (msgctx->trapTooManyActiveQueries())
-                            logctx.logOperatorException(e, __FILE__, __LINE__, NULL);
-                        throw e;
-                    }
-                }
-                else
-                {
-                    int bindCores = queryPT->getPropInt("@bindCores", msgctx->getBindCores());
-                    if (bindCores > 0)
-                        listener->setThreadAffinity(bindCores);
-                    IArrayOf<IPropertyTree> requestArray;
-                    if (isHTTP)
-                    {
-                        if (isRequestArray)
+                        int bindCores = queryPT->getPropInt("@bindCores", msgctx->getBindCores());
+                        if (bindCores > 0)
+                            listener->setThreadAffinity(bindCores);
+                        IArrayOf<IPropertyTree> requestArray;
+                        if (isHTTP)
                         {
-                            StringBuffer reqIterString;
-                            reqIterString.append(queryName).append("Request");
+                            if (isRequestArray)
+                            {
+                                StringBuffer reqIterString;
+                                reqIterString.append(queryName).append("Request");
 
-                            Owned<IPropertyTreeIterator> reqIter = queryPT->getElements(reqIterString.str());
-                            ForEach(*reqIter)
+                                Owned<IPropertyTreeIterator> reqIter = queryPT->getElements(reqIterString.str());
+                                ForEach(*reqIter)
+                                {
+                                    IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive|ipt_fast);
+                                    Owned<IPropertyTreeIterator> iter = reqIter->query().getElements("*");
+                                    ForEach(*iter)
+                                    {
+                                        fixedreq->addPropTree(iter->query().queryName(), LINK(&iter->query()));
+                                    }
+                                    requestArray.append(*fixedreq);
+                                }
+                            }
+                            else
                             {
                                 IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive|ipt_fast);
-                                Owned<IPropertyTreeIterator> iter = reqIter->query().getElements("*");
+                                Owned<IPropertyTreeIterator> iter = queryPT->getElements("*");
                                 ForEach(*iter)
                                 {
                                     fixedreq->addPropTree(iter->query().queryName(), LINK(&iter->query()));
                                 }
                                 requestArray.append(*fixedreq);
+
+                                msgctx->setIntercept(queryPT->getPropBool("@log", false));
+                                msgctx->setTraceLevel(queryPT->getPropInt("@traceLevel", logctx.queryTraceLevel()));
                             }
+                            if (httpHelper.getTrim())
+                                protocolFlags |= HPCC_PROTOCOL_TRIM;
                         }
                         else
                         {
-                            IPropertyTree *fixedreq = createPTree(queryName, ipt_caseInsensitive|ipt_fast);
-                            Owned<IPropertyTreeIterator> iter = queryPT->getElements("*");
-                            ForEach(*iter)
+                            const char *format = queryPT->queryProp("@format");
+                            if (format)
                             {
-                                fixedreq->addPropTree(iter->query().queryName(), LINK(&iter->query()));
+                                if (stricmp(format, "raw") == 0)
+                                {
+                                    protocolFlags |= HPCC_PROTOCOL_NATIVE_RAW;
+                                    if (client) //not stand alone roxie exe
+                                        protocolFlags |= HPCC_PROTOCOL_BLOCKED;
+                                    mlResponseFmt = MarkupFmt_Unknown;
+                                }
+                                else if (stricmp(format, "bxml") == 0)
+                                {
+                                    protocolFlags |= HPCC_PROTOCOL_BLOCKED;
+                                    mlResponseFmt = MarkupFmt_XML;
+                                }
+                                else if (stricmp(format, "ascii") == 0)
+                                {
+                                    protocolFlags |= HPCC_PROTOCOL_NATIVE_ASCII;
+                                    mlResponseFmt = MarkupFmt_Unknown;
+                                }
+                                else if (stricmp(format, "xml") != 0) // xml is the default
+                                    throw MakeStringException(ROXIE_INVALID_INPUT, "Unsupported format specified: %s", format);
                             }
-                            requestArray.append(*fixedreq);
-
+                            if (queryPT->getPropBool("@trim", false))
+                                protocolFlags |= HPCC_PROTOCOL_TRIM;
                             msgctx->setIntercept(queryPT->getPropBool("@log", false));
                             msgctx->setTraceLevel(queryPT->getPropInt("@traceLevel", logctx.queryTraceLevel()));
                         }
-                        if (httpHelper.getTrim())
-                            protocolFlags |= HPCC_PROTOCOL_TRIM;
-                    }
-                    else
-                    {
-                        const char *format = queryPT->queryProp("@format");
-                        if (format)
-                        {
-                            if (stricmp(format, "raw") == 0)
-                            {
-                                protocolFlags |= HPCC_PROTOCOL_NATIVE_RAW;
-                                if (client) //not stand alone roxie exe
-                                    protocolFlags |= HPCC_PROTOCOL_BLOCKED;
-                                mlResponseFmt = MarkupFmt_Unknown;
-                            }
-                            else if (stricmp(format, "bxml") == 0)
-                            {
-                                protocolFlags |= HPCC_PROTOCOL_BLOCKED;
-                                mlResponseFmt = MarkupFmt_XML;
-                            }
-                            else if (stricmp(format, "ascii") == 0)
-                            {
-                                protocolFlags |= HPCC_PROTOCOL_NATIVE_ASCII;
-                                mlResponseFmt = MarkupFmt_Unknown;
-                            }
-                            else if (stricmp(format, "xml") != 0) // xml is the default
-                                throw MakeStringException(ROXIE_INVALID_INPUT, "Unsupported format specified: %s", format);
-                        }
-                        if (queryPT->getPropBool("@trim", false))
-                            protocolFlags |= HPCC_PROTOCOL_TRIM;
-                        msgctx->setIntercept(queryPT->getPropBool("@log", false));
-                        msgctx->setTraceLevel(queryPT->getPropInt("@traceLevel", logctx.queryTraceLevel()));
-                    }
 
-                    msgctx->noteQueryActive();
+                        msgctx->noteQueryActive();
 
-                    if (isHTTP)
-                    {
-                        CHttpRequestAsyncFor af(queryName, sink, msgctx, requestArray, *client, httpHelper, protocolFlags, memused, slavesReplyLen, sanitizedText, logctx, (PTreeReaderOptions)readFlags, querySetName);
-                        af.For(requestArray.length(), global->numRequestArrayThreads);
-                    }
-                    else
-                    {
-                        Owned<IHpccProtocolResponse> protocol = createProtocolResponse(queryPT->queryName(), client, httpHelper, logctx, protocolFlags, (PTreeReaderOptions)readFlags);
-                        sink->onQueryMsg(msgctx, queryPT, protocol, protocolFlags, (PTreeReaderOptions)readFlags, querySetName, 0, memused, slavesReplyLen);
+                        if (isHTTP)
+                        {
+                            CHttpRequestAsyncFor af(queryName, sink, msgctx, requestArray, *client, httpHelper, protocolFlags, memused, slavesReplyLen, sanitizedText, logctx, (PTreeReaderOptions)readFlags, querySetName);
+                            af.For(requestArray.length(), global->numRequestArrayThreads);
+                        }
+                        else
+                        {
+                            Owned<IHpccProtocolResponse> protocol = createProtocolResponse(queryPT->queryName(), client, httpHelper, logctx, protocolFlags, (PTreeReaderOptions)readFlags);
+                            sink->onQueryMsg(msgctx, queryPT, protocol, protocolFlags, (PTreeReaderOptions)readFlags, querySetName, 0, memused, slavesReplyLen);
+                        }
                     }
                 }
             }

roxie/ccd/ccdserver.cpp

@@ -1716,7 +1716,7 @@ public:
     }
     virtual void stop()
     {
-        if (!prefiltered)
+        if (!prefiltered && inputStream)
         {
             inputStream->stop();
         }
@@ -12290,6 +12290,7 @@ public:
             duplicateKeyCount = builder->getDuplicateCount();
             cummulativeDuplicateKeyCount += duplicateKeyCount;
             builder->finish(metadata, &fileCrc);
+            clearKeyStoreCache(false);
         }
     }
 

roxie/ccd/ccdsnmp.cpp

@@ -940,6 +940,28 @@ public:
         return result.getClear();
     }
 
+    const char *getQueryName()
+    {
+        return queryName.get();
+    }
+    static IPropertyTree *getQueryRawStats(const char *queryID, time_t from, time_t to)
+    {
+        ReadLockBlock b(queryStatsLock);
+
+        Owned<IPTree> result = createPTree("QueryStats", ipt_fast);
+        ForEachItemIn(idx, queryStatsAggregators)
+        {
+            CQueryStatsAggregator &thisQuery = queryStatsAggregators.item(idx);
+            const char *queryName = thisQuery.getQueryName();
+            if (queryName && strieq(queryName, queryID))
+            {
+                result->addPropTree("Query", thisQuery.getRawStats(from, to));
+                break;
+            }
+        }
+        return result.getClear();
+    }
+
     virtual void noteQuery(time_t startTime, bool failed, unsigned elapsedTimeMs, unsigned memUsed, unsigned slavesReplyLen, unsigned bytesOut)
     {
         time_t timeNow;
@@ -1055,6 +1077,11 @@ IPropertyTree *getAllQueryStats(bool includeQueries, bool rawStats, time_t from,
      return CQueryStatsAggregator::getAllQueryStats(includeQueries, rawStats, from, to);
 }
 
+IPropertyTree *getQueryRawStats(const char *queryID, time_t from, time_t to)
+{
+     return CQueryStatsAggregator::getQueryRawStats(queryID, from, to);
+}
+
 //=======================================================================================================
 
 #ifdef _USE_CPPUNIT

roxie/ccd/ccdsnmp.hpp

@@ -76,6 +76,7 @@ interface IQueryStatsAggregator : public IInterface
 extern IQueryStatsAggregator *queryGlobalQueryStatsAggregator();
 extern IQueryStatsAggregator *createQueryStatsAggregator(const char *queryName, unsigned expirySeconds);
 extern IPropertyTree *getAllQueryStats(bool includeQueries, bool rawStats, time_t from, time_t to);
+extern IPropertyTree *getQueryRawStats(const char *queryID, time_t from, time_t to);
 
 extern RelaxedAtomic<unsigned> queryCount;
 extern RoxieQueryStats unknownQueryStats;

roxie/ccd/ccdstate.cpp

@@ -2464,21 +2464,29 @@ private:
                 else
                     time(&to);
                 const char *id = control->queryProp("Query/@id");
+                bool rawStats = control->getPropBool("@rawStats", false);
                 if (id)
                 {
-                    Owned<IQueryFactory> f = getQuery(id, NULL, NULL, logctx);
-                    if (f)
+                    if (!rawStats)
                     {
-                        Owned<const IPropertyTree> stats = f->getQueryStats(from, to);
-                        toXML(stats, reply);
+                        Owned<IQueryFactory> f = getQuery(id, NULL, NULL, logctx);
+                        if (f)
+                        {
+                            Owned<const IPropertyTree> stats = f->getQueryStats(from, to);
+                            toXML(stats, reply);
+                        }
+                        else
+                            throw MakeStringException(ROXIE_CONTROL_MSG_ERROR, "Unknown query %s", id);
                     }
                     else
-                        throw MakeStringException(ROXIE_CONTROL_MSG_ERROR, "Unknown query %s", id);
+                    {
+                        Owned<const IPropertyTree> stats = getQueryRawStats(id, from, to);
+                        toXML(stats, reply);
+                    }
                 }
                 else
                 {
                     bool includeAllQueries = control->getPropBool("@all", true);
-                    bool rawStats = control->getPropBool("@rawStats", false);
                     Owned<const IPropertyTree> stats = getAllQueryStats(includeAllQueries, rawStats, from, to);
                     toXML(stats, reply);
                 }

system/security/LdapSecurity/ldapsecurity.cpp

@@ -1234,10 +1234,11 @@ IAuthMap * CLdapSecManager::createAuthMap(IPropertyTree * authconfig)
                 location->getProp("@required", required);
                 location->getProp("@description", description);
                 
-                if(pathstr.length() == 0)
-                    throw MakeStringException(-1, "path empty in Authenticate/Location");
                 if(rstr.length() == 0)
                     throw MakeStringException(-1, "resource empty in Authenticate/Location");
+                if(pathstr.length() == 0)
+                    throw MakeStringException(-1, "path empty in Authenticate/Location for resource '%s'", rstr.str());
+
 
                 ISecResourceList* rlist = authmap->queryResourceList(pathstr.str());
                 if(rlist == NULL)
@@ -1285,6 +1286,10 @@ IAuthMap * CLdapSecManager::createFeatureMap(IPropertyTree * authconfig)
                 ISecResourceList* rlist = feature_authmap->queryResourceList(pathstr.str());
                 if(rlist == NULL)
                 {
+                    if(rstr.length() == 0)
+                        throw MakeStringException(-1, "resource empty in Feature Map");
+                    if(pathstr.length() == 0)
+                        throw MakeStringException(-1, "path empty in Feature Map for resource '%s'", rstr.str());
                     rlist = createResourceList(pathstr.str());                      
                     feature_authmap->add(pathstr.str(), rlist);
                 }
@@ -1587,8 +1592,14 @@ LDAPSECURITY_API IAuthMap *newDefaultAuthMap(IPropertyTree* config)
             location = &loc_iter->query();
             if (location)
             {
-                StringBuffer pathstr, rstr;
+                StringBuffer pathstr;
                 location->getProp("@path", pathstr);
+                if (pathstr.isEmpty())
+                {
+                    StringBuffer rstr;
+                    location->getProp("@resource", rstr);
+                    throw MakeStringException(-1, "path empty in DefaultAuthMap for resource '%s'", rstr.isEmpty() ? "unspecified" : rstr.str());
+                }
                 authmap->add(pathstr.str(), NULL);
             }
             loc_iter->next();

testing/regress/ecl/indexcachebug.ecl

@@ -0,0 +1,43 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT STD;
+
+#onwarning (4523, ignore);
+
+rec := RECORD
+ string15 fname;
+ string15 lname;
+ unsigned age;
+END;
+
+inds1 := DATASET([{ 'Aaron', 'Jones', 100}, {'Adam', 'Smith', 90}, {'Bob', 'Brown', 80}, {'Brian', 'Brown', 70 }, {'Charles', 'Dance', 60}, {'Christopher', 'Gould', 50},  {'David', 'Brokenshire', 40}, {'Edward', 'Green', 30}, {'Egbert', 'Sillyname', 20}, {'Freddy', 'Peters', 10} ], rec, DISTRIBUTED);
+inds2 := DATASET(10, TRANSFORM(rec, SELF.fname := (string)COUNTER; SELF.lname := (string)HASH(COUNTER); SELF.age := 1000-COUNTER));
+
+i := INDEX(inds1, { fname }, { lname, age }, '~myindex');
+i2 := INDEX(inds2, { fname }, { lname, age }, '~myindex');
+
+
+
+SEQUENTIAL(
+ OUTPUT(inds1, , '~ds1', OVERWRITE);
+ OUTPUT(inds2, , '~ds2', OVERWRITE);
+ BUILDINDEX(i, SORTED, OVERWRITE);
+ OUTPUT(i);
+ BUILDINDEX(i2, OVERWRITE);
+ OUTPUT(i2);
+);

testing/regress/ecl/java-stream.ecl

@@ -0,0 +1,98 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=embedded
+//class=3rdparty
+
+import java;
+
+/*
+ This example illustrates various calls to Java functions defined in the Java module JavaCat.
+ The source of JavaCat can be found in the examples directory - it can be compiled to JavaCat.class
+ using
+
+   javac JavaCat
+
+ and the resulting file JavaCat.class should be placed in /opt/HPCCSystems/classes (or somewhere else
+ where it can be located via the standard Java CLASSPATH environment variable.
+ */
+
+// Passing and returning records and datasets
+// When passing/returning a record, the corresponding Java function should take/return an object as a parameter whose fields
+// can be mapped by name to the ECL record fields
+
+nrec := record
+  utf8 ufield;
+end;
+
+jret := RECORD
+  boolean bfield;
+  integer4 ifield;
+  integer8 lfield;
+  real8 dfield;
+  real4 ffield;
+  string1 cfield1;
+  string1 cfield2;
+  string sfield;
+  nrec n;
+  set of boolean bset;
+  set of data dset;
+  set of string sset;
+  LINKCOUNTED DATASET(nrec) sub;
+end;
+
+jret jreturnrec(boolean b, integer i, real8 d) := IMPORT(java, 'JavaCat.returnrec:(ZID)LJavaCat;');
+STRING jpassrec(jret r) := IMPORT(java, 'JavaCat.passrec:(LJavaCat;)Ljava/lang/String;');
+
+ret := jreturnrec(false, 10, 2.345);
+
+OUTPUT(ret);            // Calls a Java function that returns an ECL record
+OUTPUT(jpassrec(ret));  // Passes an ECL record to a Java function
+
+// When passing a dataset to a Java function, the Java function should take either an array or an iterator of objects,
+// where the fields of the object in question are mapped by name to the fields in the ECL record.
+// When passing an iterator, we use a modified form of the function signature in the IMPORT statement, using a < to indicate "Iterator of"
+// followed by the name of the class of the objects that the iterator is to return. This is the one case where the output of javap -s cannot be used
+// directly to provide the signature of the java function being called. We can also use the "extended" signature of the method that is output by
+// javap -s -v, of the form 'JavaCat.passDataset:(Ljava/util/Iterator<LJavaCat;>;)I'
+//
+// To return a dataset, an iterator must be returned.
+
+INTEGER passDataset(LINKCOUNTED DATASET(jret) d) :=
+  IMPORT(java, 'JavaCat.passDataset:(<LJavaCat;)I'); // Calls int passDataset(Iterator<JavaCat> d)
+// Note we could also use 'Ljava/util/Iterator<LJavaCat;>;)I' as the signature, but not 'Ljava/util/Iterator;)I' which is the signature output by javap -s
+
+DATASET(jret) passDataset2(LINKCOUNTED DATASET(jret) d) :=
+  IMPORT(java, 'JavaCat.passDataset2:([LJavaCat;)Ljava/util/Iterator;'); // Calls Iterator<JavaCat> passDataset2(JavaCat d[])
+
+ds := DATASET(
+  [
+     {true, 1,2,3,4,'a', 'b', 'cd', u'ef', [true,false], [], ['Hello from ECL'], [{'1'},{'2'},{'3'},{'4'},{'5'}]}
+    ,{true, 2,4,3,4,'a', 'b', 'cd', u'ef', [true,false], [], [], []}
+    ,{true, 3,6,3,4,'a', 'b', 'cd', u'ef', [true,false], [], [], []}
+    ,{true, 8,8,3,4,'a', 'b', 'cd', u'ef', [true,false], [d'AA55'], [], []}
+  ], jret);
+
+output(passDataset(ds));  // Using an iterator
+output(passDataset2(ds)); // using an array, and illustrating the return of a dataset
+
+// It is also possible to code a traonsform function in Java - both the parameter and the return type should be a
+// Java object type that maps the fields of the ECL record by name.
+
+transform(jret) testTransform(jret in, integer lim) := IMPORT(java, 'JavaCat.transform:(LJavaCat;I)LJavaCat;');
+
+output(project(ds, testTransform(LEFT, COUNTER)));

testing/regress/ecl/javaembed-stream.ecl

@@ -0,0 +1,339 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=embedded
+//class=3rdparty
+
+import java;
+
+/*
+ Similar to java-stream example, but using inline Java 
+ */
+
+// Passing and returning records and datasets
+// When passing/returning a record, the corresponding Java function should take/return an object as a parameter whose fields
+// can be mapped by name to the ECL record fields
+
+nrec := record
+  utf8 ufield;
+end;
+
+jret := RECORD
+  boolean bfield;
+  integer4 ifield;
+  integer8 lfield;
+  real8 dfield;
+  real4 ffield;
+  string1 cfield1;
+  string1 cfield2;
+  string sfield;
+  nrec n;
+  set of boolean bset;
+  set of data dset;
+  set of string sset;
+  LINKCOUNTED DATASET(nrec) sub;
+end;
+
+jret returnrec(boolean b, integer i, real8 d) := EMBED(java)
+import java.util.*;
+public class Test1
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test1(boolean b, int i, double d)
+  {
+    bfield = b;
+    ifield = i;
+    lfield = i * 100000000;
+    dfield = d;
+    ffield = (float) d;
+    sfield = "Yoohoo";
+    cfield1 = 'X';
+    cfield2 = "Z";
+    n = new NestedClass("nest");
+    bset = new boolean [5];
+    bset[3] = b;
+    dset = new byte[1][];
+    dset[0] = new byte[1];
+    dset[0][0] = 14;
+    sset = new String[1];
+    sset[0] = "Hello";
+    sub = new NestedClass[1];
+    sub[0] = new NestedClass("subnest");
+  }
+
+  public Test1()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static Test1 returnrec(boolean b, int i, double d)
+  {
+    return new Test1(b,i,d);
+  }
+}
+ENDEMBED;
+
+
+STRING passrec(jret r) := EMBED(java)
+import java.util.*;
+public class Test2
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test2()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static String passrec(Test2 j)
+  {
+    return j.n.ufield;
+  }
+}
+ENDEMBED;
+
+ret := returnrec(false, 10, 2.345);
+
+OUTPUT(ret);            // Calls a Java function that returns an ECL record
+OUTPUT(passrec(ret));  // Passes an ECL record to a Java function
+
+// When passing a dataset to a Java function, the Java function should take either an array or an iterator of objects,
+// where the fields of the object in question are mapped by name to the fields in the ECL record.
+//
+// To return a dataset, an iterator must be returned.
+
+INTEGER passDataset(LINKCOUNTED DATASET(jret) d) := EMBED(Java)
+import java.util.*;
+public class Test3
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test3()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static int passDataset(Iterator<Test3> d)
+  {
+    int sum = 0;
+    while (d.hasNext())
+    {
+      Test3 r = d.next();
+      sum += r.lfield;
+    }
+    return sum;
+  }
+
+}
+ENDEMBED;
+
+DATASET(jret) passDataset2(LINKCOUNTED DATASET(jret) d) := EMBED(Java)
+import java.util.*;
+public class Test4
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test4()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static Iterator<Test4> passDataset2(Test4 d[])
+  {
+    return Arrays.asList(d).iterator();
+  }
+}
+
+ENDEMBED;
+
+ds := DATASET(
+  [
+     {true, 1,2,3,4,'a', 'b', 'cd', u'ef', [true,false], [], ['Hello from ECL'], [{'1'},{'2'},{'3'},{'4'},{'5'}]}
+    ,{true, 2,4,3,4,'a', 'b', 'cd', u'ef', [true,false], [], [], []}
+    ,{true, 3,6,3,4,'a', 'b', 'cd', u'ef', [true,false], [], [], []}
+    ,{true, 8,8,3,4,'a', 'b', 'cd', u'ef', [true,false], [d'AA55'], [], []}
+  ], jret);
+
+output(passDataset(ds));  // Using an iterator
+output(passDataset2(ds)); // using an array, and illustrating the return of a dataset
+
+// It is also possible to code a traonsform function in Java - both the parameter and the return type should be a
+// Java object type that maps the fields of the ECL record by name.
+
+transform(jret) testTransform(jret in, integer lim) := EMBED(java)
+import java.util.*;
+public class Test5
+{
+  public static class NestedClass
+  {
+    String ufield;
+    public NestedClass(String s)
+    {
+      ufield = s;
+    }
+    public NestedClass()
+    {
+    }
+  }
+
+  boolean bfield;
+  int ifield;
+  long lfield;
+  double dfield;
+  float ffield;
+  String sfield;
+  char cfield1;
+  String cfield2;
+  NestedClass n;
+  boolean bset[];
+  byte [] dset[];
+  String sset[];
+  NestedClass sub[];
+
+  public Test5(boolean b, int i, double d)
+  {
+    bfield = b;
+    ifield = i;
+    lfield = i * 100000000;
+    dfield = d;
+    ffield = (float) d;
+    sfield = "Yoohoo";
+    cfield1 = 'X';
+    cfield2 = "Z";
+    n = new NestedClass("nest");
+    bset = new boolean [5];
+    bset[3] = b;
+    dset = new byte[1][];
+    dset[0] = new byte[1];
+    dset[0][0] = 14;
+    sset = new String[1];
+    sset[0] = "Hello";
+    sub = new NestedClass[1];
+    sub[0] = new NestedClass("subnest");
+  }
+
+  public Test5()
+  {
+    n = new NestedClass("nest2");
+  }
+
+  public static Test5 testTransform(Test5 in, int lim)
+  {
+    return new Test5(in.bfield, lim, in.dfield);
+  }
+
+}
+ENDEMBED;
+
+output(project(ds, testTransform(LEFT, COUNTER)));

testing/regress/ecl/key/indexcachebug.xml

@@ -0,0 +1,32 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><fname>Aaron          </fname><lname>Jones          </lname><age>100</age></Row>
+ <Row><fname>Adam           </fname><lname>Smith          </lname><age>90</age></Row>
+ <Row><fname>Bob            </fname><lname>Brown          </lname><age>80</age></Row>
+ <Row><fname>Brian          </fname><lname>Brown          </lname><age>70</age></Row>
+ <Row><fname>Charles        </fname><lname>Dance          </lname><age>60</age></Row>
+ <Row><fname>Christopher    </fname><lname>Gould          </lname><age>50</age></Row>
+ <Row><fname>David          </fname><lname>Brokenshire    </lname><age>40</age></Row>
+ <Row><fname>Edward         </fname><lname>Green          </lname><age>30</age></Row>
+ <Row><fname>Egbert         </fname><lname>Sillyname      </lname><age>20</age></Row>
+ <Row><fname>Freddy         </fname><lname>Peters         </lname><age>10</age></Row>
+</Dataset>
+<Dataset name='Result 5'>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><fname>1              </fname><lname>4280804635     </lname><age>999</age></Row>
+ <Row><fname>10             </fname><lname>4248528966     </lname><age>990</age></Row>
+ <Row><fname>2              </fname><lname>3497243773     </lname><age>998</age></Row>
+ <Row><fname>3              </fname><lname>3720055187     </lname><age>997</age></Row>
+ <Row><fname>4              </fname><lname>1626808383     </lname><age>996</age></Row>
+ <Row><fname>5              </fname><lname>2551174723     </lname><age>995</age></Row>
+ <Row><fname>6              </fname><lname>1985572737     </lname><age>994</age></Row>
+ <Row><fname>7              </fname><lname>647687799      </lname><age>993</age></Row>
+ <Row><fname>8              </fname><lname>2548245530     </lname><age>992</age></Row>
+ <Row><fname>9              </fname><lname>704836619      </lname><age>991</age></Row>
+</Dataset>

testing/regress/ecl/key/java-stream.xml

@@ -0,0 +1,21 @@
+<Dataset name='Result 1'>
+ <Row><bfield>false</bfield><ifield>10</ifield><lfield>1000000000</lfield><dfield>2.345</dfield><ffield>2.345000028610229</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>false</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>nest</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>20</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><bfield>true</bfield><ifield>1</ifield><lfield>2</lfield><dfield>3.0</dfield><ffield>4.0</ffield><cfield1>a</cfield1><cfield2>b</cfield2><sfield>cd</sfield><n><ufield>ef</ufield></n><bset><Item>true</Item><Item>false</Item></bset><dset></dset><sset><Item>Hello from ECL</Item></sset><sub><Row><ufield>1</ufield></Row><Row><ufield>2</ufield></Row><Row><ufield>3</ufield></Row><Row><ufield>4</ufield></Row><Row><ufield>5</ufield></Row></sub></Row>
+ <Row><bfield>true</bfield><ifield>2</ifield><lfield>4</lfield><dfield>3.0</dfield><ffield>4.0</ffield><cfield1>a</cfield1><cfield2>b</cfield2><sfield>cd</sfield><n><ufield>ef</ufield></n><bset><Item>true</Item><Item>false</Item></bset><dset></dset><sset></sset><sub></sub></Row>
+ <Row><bfield>true</bfield><ifield>3</ifield><lfield>6</lfield><dfield>3.0</dfield><ffield>4.0</ffield><cfield1>a</cfield1><cfield2>b</cfield2><sfield>cd</sfield><n><ufield>ef</ufield></n><bset><Item>true</Item><Item>false</Item></bset><dset></dset><sset></sset><sub></sub></Row>
+ <Row><bfield>true</bfield><ifield>8</ifield><lfield>8</lfield><dfield>3.0</dfield><ffield>4.0</ffield><cfield1>a</cfield1><cfield2>b</cfield2><sfield>cd</sfield><n><ufield>ef</ufield></n><bset><Item>true</Item><Item>false</Item></bset><dset><Item>41413535</Item></dset><sset></sset><sub></sub></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><bfield>true</bfield><ifield>1</ifield><lfield>100000000</lfield><dfield>3.0</dfield><ffield>3.0</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>true</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+ <Row><bfield>true</bfield><ifield>2</ifield><lfield>200000000</lfield><dfield>3.0</dfield><ffield>3.0</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>true</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+ <Row><bfield>true</bfield><ifield>3</ifield><lfield>300000000</lfield><dfield>3.0</dfield><ffield>3.0</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>true</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+ <Row><bfield>true</bfield><ifield>4</ifield><lfield>400000000</lfield><dfield>3.0</dfield><ffield>3.0</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>true</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+</Dataset>

testing/regress/ecl/key/javaembed-stream.xml

@@ -0,0 +1,21 @@
+<Dataset name='Result 1'>
+ <Row><bfield>false</bfield><ifield>10</ifield><lfield>1000000000</lfield><dfield>2.345</dfield><ffield>2.345000028610229</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>false</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>nest</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>20</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><bfield>true</bfield><ifield>1</ifield><lfield>2</lfield><dfield>3.0</dfield><ffield>4.0</ffield><cfield1>a</cfield1><cfield2>b</cfield2><sfield>cd</sfield><n><ufield>ef</ufield></n><bset><Item>true</Item><Item>false</Item></bset><dset></dset><sset><Item>Hello from ECL</Item></sset><sub><Row><ufield>1</ufield></Row><Row><ufield>2</ufield></Row><Row><ufield>3</ufield></Row><Row><ufield>4</ufield></Row><Row><ufield>5</ufield></Row></sub></Row>
+ <Row><bfield>true</bfield><ifield>2</ifield><lfield>4</lfield><dfield>3.0</dfield><ffield>4.0</ffield><cfield1>a</cfield1><cfield2>b</cfield2><sfield>cd</sfield><n><ufield>ef</ufield></n><bset><Item>true</Item><Item>false</Item></bset><dset></dset><sset></sset><sub></sub></Row>
+ <Row><bfield>true</bfield><ifield>3</ifield><lfield>6</lfield><dfield>3.0</dfield><ffield>4.0</ffield><cfield1>a</cfield1><cfield2>b</cfield2><sfield>cd</sfield><n><ufield>ef</ufield></n><bset><Item>true</Item><Item>false</Item></bset><dset></dset><sset></sset><sub></sub></Row>
+ <Row><bfield>true</bfield><ifield>8</ifield><lfield>8</lfield><dfield>3.0</dfield><ffield>4.0</ffield><cfield1>a</cfield1><cfield2>b</cfield2><sfield>cd</sfield><n><ufield>ef</ufield></n><bset><Item>true</Item><Item>false</Item></bset><dset><Item>41413535</Item></dset><sset></sset><sub></sub></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><bfield>true</bfield><ifield>1</ifield><lfield>100000000</lfield><dfield>3.0</dfield><ffield>3.0</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>true</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+ <Row><bfield>true</bfield><ifield>2</ifield><lfield>200000000</lfield><dfield>3.0</dfield><ffield>3.0</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>true</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+ <Row><bfield>true</bfield><ifield>3</ifield><lfield>300000000</lfield><dfield>3.0</dfield><ffield>3.0</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>true</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+ <Row><bfield>true</bfield><ifield>4</ifield><lfield>400000000</lfield><dfield>3.0</dfield><ffield>3.0</ffield><cfield1>X</cfield1><cfield2>Z</cfield2><sfield>Yoohoo</sfield><n><ufield>nest</ufield></n><bset><Item>false</Item><Item>false</Item><Item>false</Item><Item>true</Item><Item>false</Item></bset><dset><Item>0E</Item></dset><sset><Item>Hello</Item></sset><sub><Row><ufield>subnest</ufield></Row></sub></Row>
+</Dataset>