Merge branch 'candidate-6.2.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.gitmodules

@@ -40,3 +40,6 @@
 [submodule "plugins/couchbase/libcouchbase-cxx"]
 	path = plugins/couchbase/libcouchbase-cxx
 	url = https://github.com/hpcc-systems/libcouchbase-cxx.git
+[submodule "libcouchbase"]
+	path = plugins/couchbase/libcouchbase
+	url = https://github.com/hpcc-systems/libcouchbase.git

CMakeLists.txt

@@ -189,6 +189,10 @@ if(APPLE OR WIN32)
     HPCC_ADD_SUBDIRECTORY(lib2)
 endif(APPLE OR WIN32)
 
+if(APPLE)
+    HPCC_ADD_SUBDIRECTORY(package)
+endif(APPLE)
+
 ###
 ## CPack install and packaging setup.
 ###

cmake_modules/commonSetup.cmake

@@ -225,6 +225,11 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   set(CMAKE_MODULE_PATH "${HPCC_SOURCE_DIR}/cmake_modules/")
 
   if(UNIX AND SIGN_MODULES)
+      execute_process(COMMAND bash "-c" "gpg --version | awk 'NR==1{print $3}'"
+        OUTPUT_VARIABLE GPG_VERSION
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET)
+    message(STATUS "gpg version ${GPG_VERSION}")
     #export gpg public key used for signing to new installation
     add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/pub.key
       COMMAND gpg --output=${CMAKE_BINARY_DIR}/pub.key --batch --no-tty --export ${SIGN_MODULES_KEYID}
@@ -948,9 +953,13 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       if(DEFINED SIGN_MODULES_KEYID)
         set(GPG_DEFAULT_KEY_OPTION --default-key)
       endif()
+      set(GPG_BATCH_OPTIONS --batch --no-tty)
+      if("${GPG_VERSION}" VERSION_GREATER "2.1")
+          set(GPG_BATCH_OPTIONS --pinentry-mode=loopback ${GPG_BATCH_OPTIONS})
+      endif()
       add_custom_command(
         OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${module}
-        COMMAND gpg --output ${CMAKE_CURRENT_BINARY_DIR}/${module} ${GPG_DEFAULT_KEY_OPTION} ${SIGN_MODULES_KEYID}  --clearsign ${GPG_PASSPHRASE_OPTION} ${SIGN_MODULES_PASSPHRASE} --batch --no-tty ${module}
+        COMMAND gpg ${GPG_BATCH_OPTIONS} --output ${CMAKE_CURRENT_BINARY_DIR}/${module} ${GPG_PASSPHRASE_OPTION} ${SIGN_MODULES_PASSPHRASE} ${GPG_DEFAULT_KEY_OPTION} ${SIGN_MODULES_KEYID} --clearsign ${module} </dev/null
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
         COMMENT "Adding signed ${module} to project"
         )

cmake_modules/dependencies/yakkety.cmake

@@ -0,0 +1 @@
+SET_DEPENDENCIES ( CPACK_DEBIAN_PACKAGE_DEPENDS g++ openssh-client openssh-server expect rsync libapr1 python psmisc )

docs/HPCCClientTools/CT_Mods/CT_ECL_IDE.xml

@@ -344,7 +344,7 @@
                 <row>
                   <entry><emphasis role="bold">Use Spaces</emphasis></entry>
 
-                  <entry>Specify to use spaces for a tab. </entry>
+                  <entry>Specify to use spaces for a tab.</entry>
                 </row>
 
                 <row>
@@ -1001,7 +1001,8 @@
                   </row>
 
                   <row>
-                    <entry><emphasis role="bold">Check out</emphasis></entry>
+                    <entry><emphasis role="bold">Checked
+                    out</emphasis></entry>
 
                     <entry>Check this box to only find ECL files that have
                     been checked out.</entry>
@@ -3064,6 +3065,49 @@ ENDMACRO;
         <para>There is also a right-click pop-up menu with the all the options
         found on the <emphasis role="bold">Edit</emphasis> menu.</para>
 
+        <sect3 role="brk">
+          <title>Builder Tab Options</title>
+
+          <para>When working in the IDE builder window you may find that you
+          have numerous tabs open. Right-click on any of the open tabs to
+          display tab options. <graphic
+          fileref="../../images/IDE0101.jpg" /></para>
+
+          <para>Select the desired action to perform on the tabs.</para>
+
+          <para><itemizedlist>
+              <listitem>
+                <para><emphasis role="bold">Save</emphasis> Saves the selected
+                tab.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">Close</emphasis> Closes the
+                selected tab.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">Close All</emphasis> Closes all
+                open tabs.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">Close All Others</emphasis> Closes
+                all open tabs except the selected tab.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">Close All to Right</emphasis>
+                Closes all open tabs to the right of the selected tab.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">Close All to Left</emphasis>
+                Closes all open tabs to the left of the selected tab.</para>
+              </listitem>
+            </itemizedlist></para>
+        </sect3>
+
         <sect3 id="CT_ECLIDE_AdvancedBuilderWindowOptions" role="brk">
           <title>Advanced Builder Window Options</title>
 
@@ -3136,6 +3180,47 @@ ENDMACRO;
         found on the <emphasis role="bold">Edit</emphasis> menu.</para>
       </sect2>
 
+      <sect2>
+        <title>Bookmarks</title>
+
+        <para>The ECL IDE supports internal bookmarks. Bookmarks provide a way
+        to mark a place that you can easily return to any time in the
+        future.</para>
+
+        <para>To create a <emphasis role="bold">bookmark</emphasis>, a
+        <emphasis role="bold">todo</emphasis>, or <emphasis
+        role="bold">hack</emphasis>, simply enter the appropriate code into
+        the place you want to mark, and save it. The syntax is in the form of
+        an ECL comment. Note that bookmarks ARE case-sensitive.</para>
+
+        <para>Bookmark syntax. Note the square brackets and that it is all
+        upper case.</para>
+
+        <programlisting>//[BM]</programlisting>
+
+        <para>To do syntax. Note that it is all upper case and terminated with
+        a colon.</para>
+
+        <programlisting>//TODO:</programlisting>
+
+        <para>Hack syntax. Note that it is all upper case and terminated with
+        a colon.</para>
+
+        <programlisting>//HACK:</programlisting>
+
+        <para>Enter the code into your workspace in the builder window then
+        save.</para>
+
+        <para><graphic fileref="../../images/IDE-BM01.jpg" />Your bookmark
+        appears in the bookmarks window right below your builder
+        window.</para>
+
+        <para><graphic fileref="../../images/IDE-BM02.jpg" /></para>
+
+        <para>To return to your bookmarked place, double-click on the
+        corresponding bookmark in the bookmarks panel.</para>
+      </sect2>
+
       <sect2 id="Results_Viewer">
         <title><emphasis>Results Viewer</emphasis></title>
 

ecl/hqlcpp/hqlcpp.cpp

@@ -8028,9 +8028,8 @@ void HqlCppTranslator::buildIteratorNext(BuildCtx & ctx, IHqlExpression * iter,
 void HqlCppTranslator::doBuildAssignCompareRow(BuildCtx & ctx, EvaluateCompareInfo & info, IHqlExpression * left, IHqlExpression * right)
 {
     HqlExprArray leftValues, rightValues;
-    IHqlExpression * record = left->queryRecord();
-    expandRowOrder(left->queryNormalizedSelector(), record, leftValues, false);
-    expandRowOrder(right->queryNormalizedSelector(), record, rightValues, false);
+    expandRowOrder(left->queryNormalizedSelector(), left->queryRecord(), leftValues, false);
+    expandRowOrder(right->queryNormalizedSelector(), right->queryRecord(), rightValues, false);
     optimizeOrderValues(leftValues, rightValues, false);
 
     doBuildAssignCompare(ctx, info, leftValues, rightValues, true, false);      //MORE: ,no_break,true

ecl/regress/sortds.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := RECORD
+    unsigned id;
+END;
+
+childRec := RECORD
+    unsigned id;
+    dataset(idRec) ids;
+END;
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+dataset(childRec) children;
+            END;
+
+namesTable := dataset('x', namesRecord, thor);
+
+sort(namesTable, surname, children);

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -70,7 +70,7 @@ IPropertyTree * fetchESDLDefinitionFromDaliById(const char *id)
 
     DBGLOG("ESDL Binding: Fetching ESDL Definition from Dali: %s ", id);
 
-    Owned<IRemoteConnection> conn = querySDS().connect(ESDL_DEFS_ROOT_PATH, myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_DESDL);
+    Owned<IRemoteConnection> conn = querySDS().connect(ESDL_DEFS_ROOT_PATH, myProcessSession(), RTM_LOCK_READ | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT_DESDL);
     if (!conn)
        throw MakeStringException(-1, "Unable to connect to ESDL Service definition information in dali '%s'", ESDL_DEFS_ROOT_PATH);
 
@@ -105,7 +105,7 @@ IPropertyTree * fetchESDLBindingFromDali(const char *process, const char *bindin
     Owned<IRemoteConnection> conn;
     try
     {
-        conn.set(querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_DESDL));
+        conn.set(querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_READ|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT_DESDL));
         if (!conn)
         {
             DBGLOG("Unable to connect to ESDL Service binding information in dali %s", ESDL_BINDINGS_ROOT_PATH);

esp/services/ws_esdlconfig/ws_esdlconfigservice.cpp

@@ -173,7 +173,7 @@ bool ensureSDSSubPath(const char * sdsPath)
     if (!sdsPath)
         return false;
 
-    Owned<IRemoteConnection> conn = querySDS().connect(sdsPath, myProcessSession(), RTM_LOCK_READ, 4000);
+    Owned<IRemoteConnection> conn = querySDS().connect(sdsPath, myProcessSession(), RTM_LOCK_READ|RTM_CREATE_QUERY, 4000);
     if (!conn)
     {
         conn.setown(querySDS().connect("/", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_DESDL));
@@ -267,7 +267,7 @@ void CWsESDLConfigEx::init(IPropertyTree *cfg, const char *process, const char *
 
 IPropertyTree * CWsESDLConfigEx::getESDLDefinitionRegistry(const char * wsEclId, bool readonly)
 {
-    if (!ensureSDSPath("ESDL/Definitions"))
+    if (!ensureSDSPath(ESDL_DEFS_ROOT_PATH))
         throw MakeStringException(-1, "Unexpected error while attempting to access ESDL definition dali registry.");
 
     Owned<IRemoteConnection> conn = querySDS().connect(ESDL_DEFS_ROOT_PATH, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_DESDL);
@@ -398,7 +398,7 @@ void CWsESDLConfigEx::ensureESDLServiceBindingRegistry(const char * espProcName,
     if (!espBindingName || !*espBindingName)
         throw MakeStringException(-1, "Unable to ensure ESDL service binding registry in dali, esp binding name not available");
 
-    if (!ensureSDSPath("ESDL/Bindings"))
+    if (!ensureSDSPath(ESDL_BINDINGS_ROOT_PATH))
         throw MakeStringException(-1, "Unable to connect to ESDL Service binding information in dali %s", ESDL_BINDINGS_ROOT_PATH);
 
     Owned<IRemoteConnection> globalLock = querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_DESDL);
@@ -644,8 +644,8 @@ int CWsESDLConfigEx::publishESDLBinding(const char * bindingName,
         return -1;
     }
 
-    if (!ensureSDSPath("ESDL/Bindings"))
-           throw MakeStringException(-1, "Unexpected error while attempting to access ESDL definition dali registry.");
+    if (!ensureSDSPath(ESDL_BINDINGS_ROOT_PATH))
+        throw MakeStringException(-1, "Unexpected error while attempting to access ESDL definition dali registry.");
 
     Owned<IRemoteConnection> conn = querySDS().connect(ESDL_BINDINGS_ROOT_PATH, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_DESDL);
     if (!conn)

esp/src/eclwatch/DynamicESDLDetailsWidget.js

@@ -249,6 +249,13 @@ define([
                     context.addBindingButton.set("disabled", false);
                     context.deleteBindingButton.set("disabled", true);
                 }
+            }  else {
+                domConstruct.destroy(this.id + "serviceInformation");
+                domConstruct.create("p", {});
+                context.details.setContent(context.i18n.PleaseSelectADynamicESDLService);
+                context.widget._Binding.set("disabled", true);
+                context.deleteBindingButton.set("disabled", true);
+                context.addBindingButton.set("disabled", true);
             }
 
             this.inherited(arguments);
@@ -260,7 +267,7 @@ define([
             var currSel = this.getSelectedChild();
             if (currSel.id == this.widget._Summary.id && !this.widget._Summary.__hpcc_initalized) {
                 this.widget._Summary.__hpcc_initalized = true;
-                var table = domConstruct.create("table", {});
+                var table = domConstruct.create("table", {id: this.id + "serviceInformation"});
                 if (this.params.__hpcc_parentName) {
                     for (var key in this.params) {
                         if (this.params.hasOwnProperty(key) && !(this.params[key] instanceof Object)) {

esp/src/eclwatch/DynamicESDLQueryWidget.js

@@ -93,11 +93,13 @@ define([
 
             retVal.on("dgrid-select", function (event) {
                 var selection = context.grid.getSelected();
-                if (selection.length) {
+                if (selection[0].__hpcc_parentName) {
                     lang.mixin(selection[0],{
                         Owner: context
                     });
                     context.detailsWidget.init(selection[0]);
+                } else {
+                    context.detailsWidget.init({0: context.i18n.PleaseSelectADynamicESDLService});
                 }
             });
             return retVal;

package/CMakeLists.txt

@@ -0,0 +1,2 @@
+configure_file("packaging_osx.sh" ${CMAKE_CURRENT_BINARY_DIR} COPYONLY)
+configure_file("distribution_template.xml" ${CMAKE_CURRENT_BINARY_DIR} COPYONLY)

package/distribution_template.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<installer-gui-script minSpecVersion="2">
+    <title>${TITLE}</title>
+    <welcome file="Welcome.txt"/>
+    <license file="License.txt"/>
+    <readme file="ReadMe.txt"/>
+    <options allow-external-scripts="no" customize="allow" rootVolumeOnly="false"/>
+    <pkg-ref id="hpccsystems.product"/>
+    <options customize="never" require-scripts="false"/>
+    <choices-outline>
+        <line choice="default">
+            <line choice="hpccsystems.product"/>
+        </line>
+    </choices-outline>
+    <choice id="default"/>
+    <choice id="hpccsystems.product" visible="false">
+        <pkg-ref id="hpccsystems.product"/>
+    </choice>
+    <pkg-ref id="hpccsystems.product" version="0" onConclusion="none">${PACKAGE_NAME}.pkg</pkg-ref>
+    <product version="${FULL_VERSION}"/>
+</installer-gui-script>

package/packaging_osx.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+WORK_ROOT=$(dirname $0)
+cd $WORK_ROOT
+WORK_ROOT=$(pwd)
+
+ARCH=$(uname -m)
+BUILD_ROOT=${WORK_ROOT}/..
+cd ${BUILD_ROOT}/_CPack_Packages/Darwin-x86_64/productbuild/hpccsystems-clienttools-community*${ARCH}
+PACKAGE_SRC=$(pwd)
+cd $WORK_ROOT
+
+#-----------------------------------------------
+#
+# Get variables
+#
+#-----------------------------------------------
+#get os "Darwin"
+OSTYPE=Darwin
+
+#ID=$(grep "^SET\(CPACK_PACKAGE_NAME \"hpccsystems-clienttools-6.2\"" ${BUILD_ROOT}/CPackConfig.cmake)
+ID=$(grep "CPACK_PACKAGE_NAME " ${BUILD_ROOT}/CPackConfig.cmake | \
+     cut -d' ' -f2 | cut -d'"' -f2)
+echo "Title: $ID"
+
+PACKAGE_NAME=$(grep "CPACK_SOURCE_PACKAGE_FILE_NAME " \
+       ${BUILD_ROOT}/CPackConfig.cmake | cut -d' ' -f2 | cut -d'"' -f2)
+PACKAGE_NAME="${PACKAGE_NAME}${OSTYPE}-${ARCH}"
+echo "CPACK_PACKAGE_NAME: $PACKAGE_NAME"
+
+FULL_VERSION=$(grep "CPACK_PACKAGE_VERSION " ${BUILD_ROOT}/CPackConfig.cmake | \
+     cut -d' ' -f2 | cut -d'"' -f2)
+echo "FULL_VERSION: $FULL_VERSION"
+
+#-----------------------------------------------
+#
+# Create package with pkgbuilds
+#
+#-----------------------------------------------
+echo ""
+echo "Create ${PACKAGE_NAME}.pkg with pkgbuild"
+[ -d "${PACKAGE_NAME}" ] && rm -rf ${PACKAGE_NAME}
+mkdir -p ${PACKAGE_NAME}
+cd ${PACKAGE_NAME}
+echo "pkgbuild --root ${PACKAGE_SRC}/opt --install-location "/opt" --identifier $ID ${PACKAGE_NAME}.pkg"
+pkgbuild --root ${PACKAGE_SRC}/opt --install-location "/opt" --identifier $ID ${PACKAGE_NAME}.pkg
+if [ $? -ne 0 ]
+then
+  echo "Error to run pkgbuild"
+  exit 1
+fi
+if [ ! -e ${PACKAGE_NAME}.pkg ]
+then
+  echo "Failed to generate ${PACKAGE_NAME}.pkg"
+  exit 1
+fi
+cd ..
+
+#-----------------------------------------------
+#
+# Creaate distribution.xml from template
+#
+#-----------------------------------------------
+echo ""
+echo "Create Distribution.xml"
+
+sed "s/\${TITLE}/${ID}/g; \
+     s/\${PACKAGE_NAME}/${PACKAGE_NAME}/g; \
+     s/\${FULL_VERSION}/${FULL_VERSION}/g" \
+     ${WORK_ROOT}/distribution_template.xml > distribution.xml
+
+#-----------------------------------------------
+#
+# Add Welcome/License/ReadMe with productbuild
+#
+#-----------------------------------------------
+echo ""
+echo "Add Welcome/License/ReadMe to ${PACKAGE_NAME}.pkg with productbuild"
+[ -d resources ] && rm -rf resources
+mkdir -p resources
+cp ${PACKAGE_SRC}/Contents/*.txt resources/
+echo "productbuild --distribution ./distribution.xml --package-path ./${PACKAGE_NAME} --resources ./resources  ${PACKAGE_NAME}.pkg"
+productbuild --distribution ./distribution.xml --package-path ./${PACKAGE_NAME} --resources ./resources  ${PACKAGE_NAME}.pkg
+
+#-----------------------------------------------
+#
+# Create DMG file
+#
+#-----------------------------------------------
+echo ""
+mv ${PACKAGE_NAME}.pkg ./${PACKAGE_NAME}/
+echo "hdiutil create -volname ${PACKAGE_NAME} -srcfolder ./${PACKAGE_NAME} -ov ${PACKAGE_NAME}.dmg"
+hdiutil create -volname ${PACKAGE_NAME} -srcfolder ./${PACKAGE_NAME} -ov ${PACKAGE_NAME}.dmg
+
+echo ""
+echo "DONE"

plugins/couchbase/CMakeLists.txt

@@ -25,37 +25,68 @@
 project(couchbaseembed)
 
 IF ( COUCHBASEEMBED )
-    ADD_PLUGIN(COUCHBASEEMBED PACKAGES COUCHBASE)
-    IF ( MAKE_COUCHBASEEMBED )
-        set(
-            SRCS
-            couchbaseembed.cpp
+
+    if(APPLE)
+        set(LIBCOUCHBASE_LIB ${CMAKE_CURRENT_BINARY_DIR}/libcouchbase-build/lib/libcouchbase.so)
+        set(LIBCOUCHBASE_LIB_SONAME ${CMAKE_CURRENT_BINARY_DIR}/libcouchbase-build/lib/libcouchbase.so.2)
+        set(LIBCOUCHBASE_LIB_REAL ${CMAKE_CURRENT_BINARY_DIR}/libcouchbase-build/lib/libcouchbase.so.2.0.38)
+    else()
+        set(LIBCOUCHBASE_LIB ${CMAKE_CURRENT_BINARY_DIR}/libcouchbase-build/lib/libcouchbase.so)
+        set(LIBCOUCHBASE_LIB_SONAME ${CMAKE_CURRENT_BINARY_DIR}/libcouchbase-build/lib/libcouchbase.so.2)
+        set(LIBCOUCHBASE_LIB_REAL ${CMAKE_CURRENT_BINARY_DIR}/libcouchbase-build/lib/libcouchbase.so.2.0.38)
+    endif()
+    add_custom_command(
+        OUTPUT ${LIBCOUCHBASE_LIB}
+        COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/libcouchbase-build
+        COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/libcouchbase-build && ${CMAKE_COMMAND} ${CMAKE_CURRENT_SOURCE_DIR}/libcouchbase && ${CMAKE_MAKE_PROGRAM}
+        COMMENT Building libcouchbase for import)
+    add_custom_target(libcouchbase-build ALL DEPENDS ${LIBCOUCHBASE_LIB})
+    add_library(libcouchbase SHARED IMPORTED)
+    set_property(TARGET libcouchbase PROPERTY IMPORTED_LOCATION ${LIBCOUCHBASE_LIB})
+    add_dependencies(libcouchbase libcouchbase-build)
+
+    set(
+        SRCS
+        couchbaseembed.cpp
         )
 
-        INCLUDE_DIRECTORIES (
-            ${PROJECT_SOURCE_DIR}/libcouchbase-cxx/include
-            ./../../esp/platform #for token serialize/deserialize
-            ./../../system/include
-            ./../../rtl/eclrtl
-            ./../../rtl/include
-            ./../../rtl/nbcd
-            ./../../common/deftype
-            ./../../system/jlib
-            ./../../roxie/roxiemem
+    INCLUDE_DIRECTORIES (
+        ${PROJECT_SOURCE_DIR}/libcouchbase/include
+        ${PROJECT_BINARY_DIR}/libcouchbase-build/generated
+        ${PROJECT_SOURCE_DIR}/libcouchbase-cxx/include
+        ${HPCC_SOURCE_DIR}/esp/platform #for token serialize/deserialize
+        ${HPCC_SOURCE_DIR}/system/include
+        ${HPCC_SOURCE_DIR}/rtl/eclrtl
+        ${HPCC_SOURCE_DIR}/rtl/include
+        ${HPCC_SOURCE_DIR}/rtl/nbcd
+        ${HPCC_SOURCE_DIR}/common/deftype
+        ${HPCC_SOURCE_DIR}/system/jlib
+        ${HPCC_SOURCE_DIR}/roxie/roxiemem
         )
 
-        ADD_DEFINITIONS(-D_USRDLL -DCOUCHBASEEMBED_EXPORTS)
-        HPCC_ADD_LIBRARY(couchbaseembed SHARED ${SRCS})
-        install(
-            TARGETS couchbaseembed
-            DESTINATION plugins)
+    ADD_DEFINITIONS(-D_USRDLL -DCOUCHBASEEMBED_EXPORTS)
+    HPCC_ADD_LIBRARY(couchbaseembed SHARED ${SRCS})
+    install(CODE "set(ENV{LD_LIBRARY_PATH} \"\$ENV{LD_LIBRARY_PATH}:${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/libcouchbase-build/lib\")")
+    install(
+        TARGETS couchbaseembed
+        DESTINATION plugins)
 
-        target_link_libraries(
-            couchbaseembed
-            ${LIBCOUCHBASE_LIBRARIES}
-            eclrtl
-            jlib)
-    endif()
+    install(
+        FILES ${LIBCOUCHBASE_LIB_REAL}
+        DESTINATION ${LIB_DIR}
+        PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
+        COMPONENT Runtime)
+
+    install(
+        FILES ${LIBCOUCHBASE_LIB} ${LIBCOUCHBASE_LIB_SONAME}
+        DESTINATION ${LIB_DIR}
+        COMPONENT Runtime)
+
+    target_link_libraries(
+        couchbaseembed
+        libcouchbase
+        eclrtl
+        jlib)
 endif()
 
 if(PLATFORM OR CLIENTTOOLS_ONLY)

plugins/couchbase/libcouchbase

@@ -0,0 +1 @@
+Subproject commit 822615ccbeecb9107bc4c02616b41aed6d9938c0

roxie/roxiemem/roxiemem.cpp

@@ -566,16 +566,16 @@ static StringBuffer &memmap(StringBuffer &stats)
     return stats.appendf("\nHeap size %u pages, %u free, largest block %u", heapTotalPages, freePages, maxBlock);
 }
 
-static void throwHeapExhausted(unsigned allocatorId, unsigned pages)
+static void throwGlobalHeapExhausted(unsigned allocatorId, unsigned pages)
 {
-    VStringBuffer msg("Memory pool exhausted: pool id %u (%u pages) exhausted, requested %u active(%u) heap(%u/%u)", allocatorId, heapTotalPages, pages, activeRowManagers.load(), heapAllocated, heapTotalPages);
+    VStringBuffer msg("Global memory exhausted: pool id %u (%u pages) exhausted, requested %u active(%u) heap(%u/%u)", allocatorId, heapTotalPages, pages, activeRowManagers.load(), heapAllocated, heapTotalPages);
     DBGLOG("%s", msg.str());
     throw MakeStringExceptionDirect(ROXIEMM_MEMORY_POOL_EXHAUSTED, msg.str());
 }
 
-static void throwHeapExhausted(unsigned allocatorId, unsigned newPages, unsigned oldPages)
+static void throwGlobalHeapExhausted(unsigned allocatorId, unsigned newPages, unsigned oldPages)
 {
-    VStringBuffer msg("Memory pool exhausted: pool id %u (%u pages) exhausted, requested %u, had %u active(%u) heap(%u/%u)", allocatorId, heapTotalPages, newPages, oldPages, activeRowManagers.load(), heapAllocated, heapTotalPages);
+    VStringBuffer msg("Global memory exhausted: pool id %u (%u pages) exhausted, requested %u, had %u active(%u) heap(%u/%u)", allocatorId, heapTotalPages, newPages, oldPages, activeRowManagers.load(), heapAllocated, heapTotalPages);
     DBGLOG("%s", msg.str());
     throw MakeStringExceptionDirect(ROXIEMM_MEMORY_POOL_EXHAUSTED, msg.str());
 }
@@ -605,7 +605,7 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
     if (heapAllocated + pages > heapTotalPages) {
         if (returnNullWhenExhausted)
             return NULL;
-        throwHeapExhausted(0, pages);
+        throwGlobalHeapExhausted(0, pages);
     }
     if (heapLargeBlockGranularity)
     {
@@ -733,7 +733,7 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
     }
     if (returnNullWhenExhausted)
         return NULL;
-    throwHeapExhausted(0, pages);
+    throwGlobalHeapExhausted(0, pages);
     return NULL;
 }
 
@@ -2890,6 +2890,8 @@ protected:
     inline void internalLock() { heapletLock.enter(); }
     inline void internalUnlock() { heapletLock.leave(); }
 
+    void throwHeapExhausted(unsigned allocatorId, unsigned pages) const;
+
 protected:
     unsigned flags; // before the pointer so it packs better in 64bit.
     Heaplet * activeHeaplet = nullptr; // which block is the current candidate for adding rows.
@@ -4410,6 +4412,12 @@ public:
 
     inline unsigned getActiveHeapPages() const { return totalHeapPages.load(std::memory_order_relaxed); }
 
+    virtual void throwHeapExhausted(unsigned allocatorId, unsigned pages)
+    {
+        VStringBuffer msg("Pool memory exhausted: pool id %u exhausted, requested %u heap(%u/%u) global(%u/%u)", allocatorId, pages, getActiveHeapPages(), getPageLimit(), heapAllocated, heapTotalPages);
+        DBGLOG("%s", msg.str());
+        throw MakeStringExceptionDirect(ROXIEMM_MEMORY_POOL_EXHAUSTED, msg.str());
+    }
 
 protected:
     void checkTraceAllocationSize(memsize_t _size, unsigned activityId);
@@ -4663,6 +4671,7 @@ public:
     virtual void setReleaseWhenModifyCallback(bool value, bool critical) { throwUnexpected(); }
     virtual unsigned querySlaveId() const { return slaveId; }
     virtual void reportMemoryUsage(bool peak) const;
+    virtual void throwHeapExhausted(unsigned allocatorId, unsigned pages);
 
 protected:
     virtual unsigned getPageLimit() const;
@@ -4765,12 +4774,32 @@ protected:
 
 //================================================================================
 
+/*
+ * The Global Memory Manager allows a single block of global memory to be shared by multiple slave channels.
+ *
+ * memLimit is the total memory that is available globally and to all the slave channels
+ * globalLimit is the maximum memory that can be allocated globally
+ *
+ * It has the following requirements:
+ * - Global row manager running out of memory needs to be able to spill any slave callbacks.
+ * - Slave row manager will need to be able to call global callbacks, but not other slaves.
+ * - Once an activity spills, other slaves should also preferentially spill that activity. (Ideally on other nodes as well).
+ *
+ * The memory available to the global row manager should be:
+ *      totalMemory - sum(slave-allocated) limited to globalLimit.
+ *
+ * The memory available to each slave should be:
+ *      (totalMemory - globallyAllocatedMemory) / numSlaves
+ *
+ * Note: This currently does not allow slave channels to use more than their fair share of memory.
+ */
 class CGlobalRowManager : public CCallbackRowManager
 {
 public:
     CGlobalRowManager(memsize_t _memLimit, memsize_t _globalLimit, unsigned _numSlaves, ITimeLimiter *_tl, const IContextLogger &_logctx, const IRowAllocatorCache *_allocatorCache, const IRowAllocatorCache **slaveAllocatorCaches, bool _ignoreLeaks, bool _outputOOMReports)
         : CCallbackRowManager(_memLimit, _tl, _logctx, _allocatorCache, _ignoreLeaks, _outputOOMReports), numSlaves(_numSlaves)
     {
+        DBGLOG("Create Global/Slave Row Manager %uMB total global can use max %uMB", (unsigned)(_memLimit / 0x100000), (unsigned)(_globalLimit / 0x100000));
         assertex(_globalLimit <= _memLimit);
         globalPageLimit = (unsigned) PAGES(_globalLimit, HEAP_ALIGNMENT_SIZE);
         slaveRowManagers = new CChunkingRowManager * [numSlaves];
@@ -4822,12 +4851,16 @@ public:
     virtual unsigned getPageLimit() const
     {
         //Sum the total pages allocated by the slaves.  Not called very frequently.
-        //Do this in preference to maintaining a slaveTotal sine that would involve an atomic add for
+        //Do this in preference to maintaining a slaveTotal since that would involve an atomic add for
         //each slave page allocation.
         unsigned slavePages = 0;
         for (unsigned i=0; i < numSlaves; i++)
             slavePages += slaveRowManagers[i]->getActiveHeapPages();
-        return globalPageLimit - slavePages;
+        unsigned available = maxPageLimit - slavePages;
+        if (available > globalPageLimit)
+            return globalPageLimit;
+        else
+            return available;
     }
 
     virtual IRowManager * querySlaveRowManager(unsigned slave)
@@ -4836,6 +4869,13 @@ public:
         return slaveRowManagers[slave];
     }
 
+    void throwHeapExhausted(unsigned allocatorId, unsigned pages)
+    {
+        VStringBuffer msg("Shared global memory exhausted: pool id %u exhausted, requested %u heap(%u/%u/%u/%u)) global(%u/%u)", allocatorId, pages, getActiveHeapPages(), getPageLimit(), globalPageLimit, maxPageLimit, heapAllocated, heapTotalPages);
+        DBGLOG("%s", msg.str());
+        throw MakeStringExceptionDirect(ROXIEMM_MEMORY_POOL_EXHAUSTED, msg.str());
+    }
+
 public:
     unsigned getSlavePageLimit(unsigned slaveId) const
     {
@@ -4887,6 +4927,14 @@ void CSlaveRowManager::reportMemoryUsage(bool peak) const
     globalManager->reportMemoryUsage(peak);
 }
 
+void CSlaveRowManager::throwHeapExhausted(unsigned allocatorId, unsigned pages)
+{
+    VStringBuffer msg("Channel memory exhausted: pool id %u exhausted, requested %u heap(%u/%u/%u)) global(%u/%u)", allocatorId, pages, getActiveHeapPages(), getPageLimit(), maxPageLimit, heapAllocated, heapTotalPages);
+    DBGLOG("%s", msg.str());
+    throw MakeStringExceptionDirect(ROXIEMM_MEMORY_POOL_EXHAUSTED, msg.str());
+}
+
+
 //================================================================================
 
 void CRoxieFixedRowHeapBase::beforeDispose()
@@ -4931,6 +4979,11 @@ void * CRoxieVariableRowHeap::finalizeRow(void *final, memsize_t originalSize, m
 
 //================================================================================
 
+void CHeap::throwHeapExhausted(unsigned allocatorId, unsigned pages) const
+{
+    rowManager->throwHeapExhausted(allocatorId, pages);
+}
+
 //MORE: Make this a nested class??
 HugeHeaplet * CHugeHeap::allocateHeaplet(memsize_t _size, unsigned allocatorId, unsigned maxSpillCost)
 {
@@ -5061,7 +5114,7 @@ void CHugeHeap::expandHeap(void * original, memsize_t copysize, memsize_t oldcap
         {
             if (maxSpillCost == SpillAllCost)
                 rowManager->reportMemoryUsage(false);
-            throwHeapExhausted(activityId, newPages, oldPages);
+            throwGlobalHeapExhausted(activityId, newPages, oldPages);
         }
     }
 }

system/CMakeLists.txt

@@ -14,7 +14,7 @@
 #    limitations under the License.
 ################################################################################
 HPCC_ADD_SUBDIRECTORY (hrpc)
-HPCC_ADD_SUBDIRECTORY (include "PLATFORM")
+HPCC_ADD_SUBDIRECTORY (include)
 HPCC_ADD_SUBDIRECTORY (jhtree)
 HPCC_ADD_SUBDIRECTORY (jlib)
 HPCC_ADD_SUBDIRECTORY (mp)

system/jlib/CMakeLists.txt

@@ -40,7 +40,7 @@ endif(NOT TARGET lz4)
 
 
 if (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
-  SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-switch -Wno-unused-parameter -Werror")
+  SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-switch -Wno-unused-parameter -Werror -Wno-error=delete-non-virtual-dtor")
 endif()
 
 set (    SRCS 

system/jlib/jdebug.cpp

@@ -1481,7 +1481,7 @@ class CProcessMonitor
     CIArrayOf<CProcInfo> processes;
     unsigned tot_time;
     bool busy;
-
+    CriticalSection sect;
 
     static int compare(CInterface * const *i1, CInterface * const *i2)
     {
@@ -1502,9 +1502,8 @@ public:
             processes.item(i1).active = false;
         DIR *dir = opendir("/proc");
         loop {
-            struct dirent *ent;
-            struct dirent entryMem;
-            readdir_r(dir, &entryMem, &ent);
+            CriticalBlock b(sect);
+            struct dirent *ent = readdir(dir);
             if (!ent)
                 break;
             if ((ent->d_name[0]>='0')&&(ent->d_name[0]<='9')) {

system/jlib/jfile.cpp

@@ -3557,6 +3557,7 @@ class CLinuxDirectoryIterator : public CDirectoryIterator
     DIR *           handle;
     struct stat     st;
     bool            gotst;
+    CriticalSection sect;
 
     bool loadst()
     {
@@ -3617,11 +3618,11 @@ public:
     bool next()
     {
         loop {
-            struct dirent dirEntry;
             struct dirent *entry;
             loop {
                 gotst = false;
-                readdir_r(handle, &dirEntry, &entry);
+                CriticalBlock b(sect);
+                entry = readdir(handle);
                 // need better checking here?
                 if (!entry)
                     break;

system/jlib/jregexp.cpp

@@ -1344,10 +1344,11 @@ static bool WildMatchNreplace ( const char *src, int srclen, int srcidx,
                     (toupper(src[srcidx])!=toupper(next_char)))
                     goto Fail;
             }
-            else
+            else {
                 if ((srcidx == srclen) || (src[srcidx]!=next_char))
                     goto Fail;
-                srcidx++;
+            }
+            srcidx++;
         }
         else {
             loop {

system/jlib/jthread.cpp

@@ -1644,20 +1644,24 @@ class CIgnoreSIGPIPE
 public:
     CIgnoreSIGPIPE()
     {
+        oact.sa_handler = SIG_IGN;
         struct sigaction act;
         sigset_t blockset;
         sigemptyset(&blockset);
         act.sa_mask = blockset;
         act.sa_handler = SIG_IGN;
         act.sa_flags = 0;
-        sigaction(SIGPIPE, &act, NULL);
+        sigaction(SIGPIPE, &act, &oact);
     }
 
     ~CIgnoreSIGPIPE()
     {
-        signal(SIGPIPE, SIG_DFL);
+        if (oact.sa_handler != SIG_IGN)
+            sigaction(SIGPIPE, &oact, NULL);
     }
 
+private:
+    struct sigaction oact;
 };
 
 #define WHITESPACE " \t\n\r"

system/security/LdapSecurity/ldaputils.cpp

@@ -23,6 +23,9 @@
 
 #include "ldaputils.hpp"
 
+#ifndef _WIN32
+# include <signal.h>
+#endif
 
 //------------------------------------
 // LdapUtils implementation
@@ -107,7 +110,12 @@ int LdapUtils::LdapSimpleBind(LDAP* ld, char* userdn, char* password)
     ldap_set_option(ld, LDAP_OPT_TIMEOUT, &timeout);
     ldap_set_option(ld, LDAP_OPT_NETWORK_TIMEOUT, &timeout);
 #endif
-    return ldap_bind_s(ld, userdn, password, LDAP_AUTH_SIMPLE);
+    int srtn = ldap_bind_s(ld, userdn, password, LDAP_AUTH_SIMPLE);
+#ifndef _WIN32
+    // secure ldap tls might overwrite SIGPIPE handler
+    signal(SIGPIPE, SIG_IGN);
+#endif
+    return srtn;
 }
 
 // userdn is required for ldap_simple_bind_s, not really necessary for ldap_bind_s.

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -1536,9 +1536,9 @@ public:
         {
             clearRHS();
             clearHT();
-            if (right)
-                stopInput(1, "(R)");
         }
+        if (right)
+            stopInput(1, "(R)");
         if (broadcaster)
             broadcaster->reset();
         stopInput(0, "(L)");

thorlcr/graph/thgraphmaster.cpp

@@ -237,6 +237,8 @@ void CSlaveMessageHandler::main()
                             size32_t startCtxLen;
                             msg.read(startCtxLen);
                             element->doCreateActivity(parentExtractSz, parentExtract, startCtxLen ? &msg : nullptr);
+                            if (element->queryActivity())
+                                element->preStart(parentExtractSz, parentExtract);
                         }
                         catch (IException *e)
                         {