Merge latest changes from candidate-3.6.x to master

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

CMakeLists.txt

@@ -213,32 +213,23 @@ set(CPACK_SOURCE_IGNORE_FILES
 ###
 set( BUILD_TAG "${CPACK_RPM_PACKAGE_VERSION}_${version}-${stagever}")
 if (USE_GIT_DESCRIBE OR CHECK_GIT_TAG)
-    execute_process(COMMAND "${GIT_COMMAND}" describe --exact --tags --dirty --match ${CPACK_RPM_PACKAGE_VERSION}* 
-        WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
-        OUTPUT_VARIABLE GIT_BUILD_TAG
-        ERROR_QUIET
-        OUTPUT_STRIP_TRAILING_WHITESPACE)
-    if (CHECK_GIT_TAG)
-        if(NOT "${GIT_BUILD_TAG}" STREQUAL "${BUILD_TAG}")
-            message(FATAL_ERROR "Git tag '${GIT_BUILD_TAG}' does not match source version '${BUILD_TAG}'" )
-        endif()
-    else()
-        if ("${GIT_BUILD_TAG}" STREQUAL "")
-            execute_process(COMMAND "${GIT_COMMAND}" describe --always --tags --all --abbrev=6 --dirty --long
-                WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
-                OUTPUT_VARIABLE GIT_BUILD_TAG
-                ERROR_QUIET
-                OUTPUT_STRIP_TRAILING_WHITESPACE)
-        endif()
-    endif()
+    FETCH_GIT_TAG (${CMAKE_SOURCE_DIR} ${CPACK_RPM_PACKAGE_VERSION} GIT_BUILD_TAG)
     message ("-- Git tag is '${GIT_BUILD_TAG}'")
-    if(NOT "${GIT_BUILD_TAG}" STREQUAL "${BUILD_TAG}")
-        if(NOT "${GIT_BUILD_TAG}" STREQUAL "") # probably means being built from a tarball...
-            set( BUILD_TAG "${BUILD_TAG}[${GIT_BUILD_TAG}]")
+    if (NOT "${GIT_BUILD_TAG}" STREQUAL "${BUILD_TAG}")
+        if (CHECK_GIT_TAG)
+            message(FATAL_ERROR "Git tag '${GIT_BUILD_TAG}' does not match source version '${BUILD_TAG}'" )
+        else()
+            if(NOT "${GIT_BUILD_TAG}" STREQUAL "") # probably means being built from a tarball...
+                set( BUILD_TAG "${BUILD_TAG}[${GIT_BUILD_TAG}]")
+            endif()
         endif()
     endif()
 endif()
 message ("-- Build tag is '${BUILD_TAG}'")
+if (NOT "${BASE_BUILD_TAG}" STREQUAL "")
+    set(BASE_BUILD_TAG "${BUILD_TAG}")
+endif()
+message ("-- Base build tag is '${BASE_BUILD_TAG}'")
 configure_file(${HPCC_SOURCE_DIR}/build-config.h.cmake "build-config.h" )
 
 #set( CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON )

build-config.h.cmake

@@ -82,6 +82,10 @@
     #cmakedefine BUILD_TAG "${BUILD_TAG}"
 #endif
 
+#ifndef BASE_BUILD_TAG
+    #cmakedefine BASE_BUILD_TAG "${BASE_BUILD_TAG}"
+#endif
+
 #ifndef BUILD_LEVEL
     #cmakedefine BUILD_LEVEL "${BUILD_LEVEL}"
 #endif

cmake_modules/commonSetup.cmake

@@ -395,4 +395,19 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${OSSDIR}/lib")
   SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
+  MACRO (FETCH_GIT_TAG workdir edition result)
+      execute_process(COMMAND "${GIT_COMMAND}" describe --tags --dirty --abbrev=6 --match ${edition}*
+        WORKING_DIRECTORY "${workdir}"
+        OUTPUT_VARIABLE ${result}
+        ERROR_QUIET
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+        if ("${${result}}" STREQUAL "")
+            execute_process(COMMAND "${GIT_COMMAND}" describe --always --tags --all --abbrev=6 --dirty --long
+                WORKING_DIRECTORY "${workdir}"
+                OUTPUT_VARIABLE ${result}
+                ERROR_QUIET
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+        endif()
+  ENDMACRO()
+
 endif ("${COMMONSETUP_DONE}" STREQUAL "")

esp/eclwatch/ws_XSLT/CMakeLists.txt

@@ -107,6 +107,7 @@ FOREACH ( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/targetclusters.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/showresult.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/table.xslt
+    ${CMAKE_CURRENT_SOURCE_DIR}/thor_status.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/topology.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/tplog.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/tplogdisplay.xslt

esp/eclwatch/ws_XSLT/index.xslt

@@ -532,7 +532,7 @@
                                     </xsl:when>
                                     <xsl:otherwise>
                                         <h3>
-                                            Existing Activity on Servers:<xsl:value-of select="SortBy"/>
+                                            Existing Activity on Servers:
                                         </h3>
                                     </xsl:otherwise>
                                 </xsl:choose>

esp/files/scripts/configmgr/navtree.js

@@ -2265,6 +2265,7 @@ function promptVerifyPwd(category, params, attrName, oldValue, newValue, recordI
       var xmlArgs = "<XmlArgs><Setting category=\"" + category;
       xmlArgs += "\" params=\"" + params;
       xmlArgs += "\" attrName=\"" + attrName;
+      xmlArgs += "\" viewType=\"password";
       xmlArgs += "\" rowIndex=\"" + recordIndex;
       xmlArgs += "\" oldValue=\"" + escape(oldValue);
       xmlArgs += "\" newValue=\"" + escape(pwd) + "\"/></XmlArgs>";

esp/services/WsDeploy/WsDeployService.cpp

@@ -929,6 +929,7 @@ bool CWsDeployFileInfo::saveSetting(IEspContext &context, IEspSaveSettingRequest
       const char* pszOldValue = pSetting->queryProp("@oldValue");
       const char* pszNewValue = pSetting->queryProp("@newValue");
       const char* pszOnChange = pSetting->queryProp("@onChange");
+      const char* pszViewType = pSetting->queryProp("@viewType");
 
       StringBuffer xpath;
       xpath.clear().appendf("%s[@name='%s']", pszCompType, pszCompName);
@@ -1293,8 +1294,7 @@ bool CWsDeployFileInfo::saveSetting(IEspContext &context, IEspSaveSettingRequest
         {
           xpath.clear().appendf("@%s", pszAttrName);
 
-          String strAttrName(pszAttrName);
-          if (strAttrName.toLowerCase()->endsWith("password"))
+          if (pszViewType && *pszViewType && !strcmp(pszViewType, "password"))
           {
             encrypt(encryptedText, pszNewValue);
             pszNewValue = encryptedText.str();

plugins/datastream/CMakeLists.txt

@@ -1,2 +1,147 @@
-project (datastream)
+project (hpccsystems-datastream)
+cmake_minimum_required (VERSION 2.6)
+
+set ( HPCC_DATASTREAM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set ( HPCC_SOURCE_DIR ${HPCC_DATASTREAM_SOURCE_DIR}/../../)
+
+set ( CMAKE_MODULE_PATH "${HPCC_SOURCE_DIR}/cmake_modules")
+set ( EXECUTABLE_OUTPUT_PATH "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bin" )
+set ( PRODUCT_PREFIX "hpccsystems" )
+
+SET(CMAKE_SKIP_BUILD_RPATH  FALSE)
+SET(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
+SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${OSSDIR}/lib")
+SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
+#include(${HPCC_SOURCE_DIR}/cmake_modules/optionDefaults.cmake)
+
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+    set ( CMAKE_BUILD_TYPE "Release" )
+elseif (NOT "${CMAKE_BUILD_TYPE}" MATCHES "Debug|Release")
+    message (FATAL_ERROR "Unknown build type $ENV{CMAKE_BUILD_TYPE}")
+endif ()
+message ("-- Making ${CMAKE_BUILD_TYPE} system")
+
+if (CMAKE_SIZEOF_VOID_P EQUAL 8)
+   set ( ARCH64BIT 1 )
+else ()
+   set ( ARCH64BIT 0 )
+endif ()
+message ("-- 64bit architecture is ${ARCH64BIT}")
+
+set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_DEBUG -DDEBUG")
+
+###
+## Version Information keep in synch with top level hpcc versioning
+###
+set ( HPCC_PROJECT "community" )
+set ( HPCC_MAJOR 3 )
+set ( HPCC_MINOR 6 )
+set ( HPCC_POINT 2 )
+set ( HPCC_MATURITY "rc" )
+set ( HPCC_SEQUENCE 3 )
+###
+
+include(${HPCC_SOURCE_DIR}/cmake_modules/optionDefaults.cmake)
+include(${HPCC_SOURCE_DIR}/cmake_modules/commonSetup.cmake)
+
+set ( CPACK_PACKAGE_CONTACT "HPCCSystems <ossdevelopment@lexisnexis.com>" )
+set ( CPACK_SOURCE_GENERATOR TGZ )
+set ( CPACK_RPM_PACKAGE_VERSION "${projname}" )
+
+if ( ${ARCH64BIT} EQUAL 1 )
+    set ( CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
+else( ${ARCH64BIT} EQUAL 1 )
+    set ( CPACK_RPM_PACKAGE_ARCHITECTURE "i386")
+endif ( ${ARCH64BIT} EQUAL 1 )
+
+set(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CPACK_RPM_PACKAGE_ARCHITECTURE}")
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
+    set(CPACK_STRIP_FILES TRUE)
+endif()
+
+set ( CPACK_INSTALL_CMAKE_PROJECTS "${CMAKE_CURRENT_BINARY_DIR};hdfsstream;ALL;/")
+
+if ( CMAKE_SYSTEM MATCHES Linux )
+    EXECUTE_PROCESS (
+                COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/distrocheck.sh
+                    OUTPUT_VARIABLE packageManagement
+                        ERROR_VARIABLE  packageManagement
+                )
+    EXECUTE_PROCESS (
+                COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/getpackagerevisionarch.sh
+                    OUTPUT_VARIABLE packageRevisionArch
+                        ERROR_VARIABLE  packageRevisionArch
+                )
+    EXECUTE_PROCESS (
+                COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/getpackagerevisionarch.sh --noarch
+                    OUTPUT_VARIABLE packageRevision
+                        ERROR_VARIABLE  packageRevision
+                )
+
+    message ( "-- Auto Detecting Packaging type")
+    message ( "-- distro uses ${packageManagement}, revision is ${packageRevisionArch}" )
+
+if ( ${packageManagement} STREQUAL "DEB" )
+        set(CPACK_PACKAGE_FILE_NAME     "${CMAKE_PROJECT_NAME}_${CPACK_RPM_PACKAGE_VERSION}-${version}-${stagever}${packageRevisionArch}")
+    elseif ( ${packageManagement} STREQUAL "RPM" )
+        set(CPACK_PACKAGE_FILE_NAME     "${CMAKE_PROJECT_NAME}_${CPACK_RPM_PACKAGE_VERSION}-${version}-${stagever}.${packageRevisionArch}")
+        else()
+        set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}_${CPACK_RPM_PACKAGE_VERSION}_${version}-${stagever}${CPACK_SYSTEM_NAME}")
+    endif ()
+endif ( CMAKE_SYSTEM MATCHES Linux )
+
+MESSAGE ("-- Current release version is ${CPACK_PACKAGE_FILE_NAME}")
+
+set( CPACK_SOURCE_GENERATOR TGZ )
+
+###
+## CPack commands in this section require cpack 2.8.1 to function.
+## When using cpack 2.8.1, the command "make package" will create
+## an RPM.
+###
+
+if (NOT "${CMAKE_VERSION}" VERSION_LESS "2.8.1")
+    if ( CMAKE_SYSTEM MATCHES Linux )
+        if ( ${packageManagement} STREQUAL "DEB" )
+            if ("${CMAKE_VERSION}" VERSION_EQUAL "2.8.2")
+                message("WARNING: CMAKE 2.8.2  would not build DEB package")
+            else ()
+                set ( CPACK_GENERATOR "${packageManagement}" )
+                message("-- Will build DEB package")
+                ###
+                ## CPack instruction required for Debian
+                ###
+                message ("-- Packing BASH installation files")
+                set ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postinst;${CMAKE_CURRENT_BINARY_DIR}/initfiles/sbin/prerm;${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postrm" )
+            endif ()
+
+        elseif ( ${packageManagement} STREQUAL "RPM" )
+            set ( CPACK_GENERATOR "${packageManagement}" )
+            ###
+            ## CPack instruction required for RPM
+            ###
+            message("-- Will build RPM package")
+            message ("-- Packing BASH installation files")
+            set ( CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postinst" )
+
+            set ( CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/sbin/prerm" )
+            set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postrm" )
+                else()
+            message("WARNING: Unsupported package ${packageManagement}.")
+        endif ()
+
+    endif ( CMAKE_SYSTEM MATCHES Linux )
+    if ( EXISTS ${HPCC_SOURCE_DIR}/cmake_modules/dependencies/${packageRevision}.cmake )
+        include( ${HPCC_SOURCE_DIR}/cmake_modules/dependencies/${packageRevision}.cmake )
+    else()
+        message("-- WARNING: DEPENDENCY FILE FOR ${packageRevision} NOT FOUND, Using deps template.")
+        include( ${HPCC_SOURCE_DIR}/cmake_modules/dependencies/template.cmake )
+    endif()
+else()
+    message("WARNING: CMAKE 2.8.1 or later required to create RPMs from this project")
+endif()
+
 add_subdirectory (hdfsstream)
+
+INCLUDE(CPack)

plugins/datastream/hdfsstream/CMakeLists.txt

@@ -2,6 +2,7 @@ project(hdfsstream)
 
 option(USE_HDFSSTREAM "Configure use of hdstream plugin" OFF)
 if ( USE_HDFSSTREAM )
+add_subdirectory (ecl)
 	option(HADOOP_PATH "Set the Hadoop path.")
 	if( NOT HADOOP_PATH )
 		set(HADOOP_PATH "/usr/local/hadoop")
@@ -28,10 +29,11 @@ if ( USE_HDFSSTREAM )
 
 	add_executable( hdfsstream ${SRC} )
 
-#	Install ( TARGETS hdfsstream DESTINATION ${DIR_NAME}/bin COMPONENT Runtime)
-#	Install ( PROGRAMS hdfspipe DESTINATION ${DIR_NAME}/bin COMPONENT Runtime )
-#	Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/hdfsstreamconfig DESTINATION ${CONFIG_DIR}/rpmnew COMPONENT Runtime )
-#	Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/hdfsstreamconfig DESTINATION ${DIR_NAME}/bin COMPONENT Runtime )
+	set ( INSTALLDIR "${OSSDIR}/bin")
+	Install ( TARGETS hdfsstream DESTINATION ${INSTALLDIR} COMPONENT Runtime)
+	Install ( PROGRAMS hdfspipe DESTINATION ${INSTALLDIR} COMPONENT Runtime )
+	Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/hdfsstreamconfig DESTINATION ${INSTALLDIR} COMPONENT Runtime )
+
 	target_link_libraries ( hdfsstream
 					${JAVA_JVM_LIBRARY}
 					${LIBHDFS_LIBRARIES})

plugins/datastream/hdfsstream/ecl/CMakeLists.txt

@@ -0,0 +1 @@
+Install ( FILES HDFSPipe.ecl DESTINATION "${OSSDIR}/share/My Files/DataStream" COMPONENT Runtime )

plugins/datastream/hdfsstream/hdfspipe

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-source hadoopConfig
+source hdfsstreamconfig 
 
 CLASSPATH=$CLASSPATH:$HADOOP_LOCATION/conf
 
@@ -13,16 +13,32 @@ for f in $HADOOP_LOCATION/lib/*.jar ; do
 done
 
 export CLASSPATH=$CLASSPATH
-PID=$$
-
-LOG=/tmp/HPCC-HadoopStream.log.$PID
 
-echo "Script starting"  		>> $LOG
-echo "Running as user: $USER" 	>> $LOG
-echo "Running mode: $run_mode" 	>> $LOG
-echo "Incoming params: ${@}" 	>> $LOG
+PID=$$
 
-HPCCTMPFILE=/tmp/HPCCTMPFILE;
+idfound=0;
+nodeid=0;
+
+for p in $*;
+ do
+#   echo "[$p]" >> $LOG;
+   if [ "$idfound" = "1" ];
+   then
+        nodeid=$p;
+        idfound=0;
+   elif [ "$p" = "-nodeid" ];
+   then
+	idfound=1;
+   fi
+done;
+
+LOG=/tmp/HPCC-HadoopStream.log.$nodeid.$PID
+
+echo "Script starting"		>> $LOG
+echo "Running as user: $USER"   >> $LOG
+echo "Running mode: $run_mode"  >> $LOG
+echo "Incoming params: ${@}"    >> $LOG
+echo "nodeid: $nodeid" 		>> $LOG
 
 if [ "$1" = "" ];
 then
@@ -36,6 +52,8 @@ then
 	/opt/HPCCSystems/bin/hdfsstream  "${@}" 2>> $LOG;
 elif [ $1 = "-so" ];
 then
+
+	HPCCTMPFILE=/tmp/HPCCTMPFILE_$nodeid;
 	if [ -f $HPCCTMPFILE ]
 	then
 		rm "$HPCCTMPFILE" 			2>> $LOG
@@ -51,12 +69,12 @@ then
 
 	echo "calling hdfsstream..." 		>> $LOG
 
-	/opt/HPCCSystems/bin/hdfsstream "${@}" 	2>> $LOG
+	/opt/HPCCSystems/bin/hdfsstream "${@}" -pipepath $HPCCTMPFILE  	2>> $LOG
 
 	echo "write exited with: $?" 			>> $LOG
 elif [ $1 = "-sop" ];
 then
-	pipepath=/tmp/HPCCPIPE;
+	pipepath=/tmp/HPCCPIPE_$nodeid;
 	mkfifo $pipepath 2> /tmp/HPCC-FIFO.err.$PID;
 	chmod 666 $pipepath 2> /tmp/HPCC-FIFO.err.$PID;
 
@@ -68,7 +86,7 @@ then
 	else
 		echo "  WARNING (hdfsstream mkfifo) error registered in file: /tmp/HPCC-FIFO.err.$PID " >> $LOG
 	fi
-	/opt/HPCCSystems/bin/hdfsstream  "${@}" 	2>> $LOG &
+	/opt/HPCCSystems/bin/hdfsstream  "${@}" -pipepath $pipepath	2>> $LOG &
 	echo "redirecting stdin to named pipe ... " 	>> $LOG
 	cat < /dev/stdin > "$pipepath"			2>> $LOG
 

thorlcr/msort/tsortm.cpp

@@ -1234,20 +1234,24 @@ public:
 #ifdef USE_SAMPLE_PARTITIONING
         bool usesampling = true;        
 #endif
+        bool useAux = false; // JCSMORE using existing partioning and auxillary rowIf (only used if overflow)
         loop {
             OwnedMalloc<rowmap_t> splitMap, splitMapUpper;
             CTimer timer;
             if (numnodes>1) {
                 timer.start();
                 if (cosortfilenames) {
+                    useAux = true;
                     CalcExtPartition();
                     canoptimizenullcolumns = false;
                 }
                 if (usepartitionrow) {
+                    useAux = true;
                     CalcPreviousPartition();
                     canoptimizenullcolumns = false;
                 }
                 if (partitioninfo->IsOK()) {
+                    useAux = true;
                     splitMap.setown(UsePartitionInfo(*partitioninfo, betweensort));
                     if (betweensort) {
                         splitMapUpper.setown(UsePartitionInfo(*partitioninfo, false));
@@ -1329,7 +1333,7 @@ public:
                     for (i=0;i<numnodes;i++) {
                         CSortNode &slave = slaves.item(i);
                         if (slave.overflow) 
-                            slave.OverflowAdjustMapStart(numnodes,splitMap+i*numnodes,mbsk.length(),(const byte *)mbsk.bufferBase(),CMPFN_COLLATE);
+                            slave.OverflowAdjustMapStart(numnodes,splitMap+i*numnodes,mbsk.length(),(const byte *)mbsk.bufferBase(),CMPFN_COLLATE,useAux);
                     }
                     for (i=0;i<numnodes;i++) {
                         CSortNode &slave = slaves.item(i);
@@ -1340,7 +1344,7 @@ public:
                         for (i=0;i<numnodes;i++) {
                             CSortNode &slave = slaves.item(i);
                             if (slave.overflow) 
-                                slave.OverflowAdjustMapStart(numnodes,splitMapUpper+i*numnodes,mbsk.length(),(const byte *)mbsk.bufferBase(),CMPFN_UPPER);
+                                slave.OverflowAdjustMapStart(numnodes,splitMapUpper+i*numnodes,mbsk.length(),(const byte *)mbsk.bufferBase(),CMPFN_UPPER,useAux);
                         }
                         for (i=0;i<numnodes;i++) {
                             CSortNode &slave = slaves.item(i);

thorlcr/msort/tsortmp.cpp

@@ -207,12 +207,12 @@ void SortSlaveMP::MultiBinChopStop(unsigned num,rowmap_t *pos)
     mb.read(num*sizeof(rowmap_t),pos);
 }
 
-void SortSlaveMP::OverflowAdjustMapStart( unsigned mapsize,rowmap_t *map,size32_t keybuffsize,const byte *keybuff, byte cmpfn) /* async */
+void SortSlaveMP::OverflowAdjustMapStart( unsigned mapsize,rowmap_t *map,size32_t keybuffsize,const byte *keybuff, byte cmpfn,bool useaux) /* async */
 {
     CMessageBuffer mb;
     mb.append((byte)FN_OverflowAdjustMapStart);
     mb.append(mapsize).append(mapsize*sizeof(rowmap_t),map);
-    serializeblk(mb,keybuffsize,keybuff).append(cmpfn);
+    serializeblk(mb,keybuffsize,keybuff).append(cmpfn).append(useaux);
     sendRecv(mb);
 
 }
@@ -477,7 +477,9 @@ bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag
                 deserializeblk(mb,keybuffsize,keybuff);
                 byte cmpfn;
                 mb.read(cmpfn);
-                slave.OverflowAdjustMapStart(mapsize,(rowmap_t *)map,keybuffsize,(const byte *)keybuff,cmpfn);
+                bool useaux;
+                mb.read(useaux);
+                slave.OverflowAdjustMapStart(mapsize,(rowmap_t *)map,keybuffsize,(const byte *)keybuff,cmpfn,useaux);
                 free(keybuff);
             }
             break;

thorlcr/msort/tsortmp.hpp

@@ -25,7 +25,7 @@ interface ISortSlaveMP
     virtual void MultiBinChop(size32_t keybuffsize,const byte *keybuff, unsigned num,rowmap_t *pos,byte cmpfn,bool useaux)=0;
     virtual void MultiBinChopStart(size32_t keybuffsize,const byte *keybuff, byte cmpfn)=0; /* async */
     virtual void MultiBinChopStop(unsigned num,rowmap_t *pos)=0;
-    virtual void OverflowAdjustMapStart( unsigned mapsize,rowmap_t *map,size32_t keybuffsize,const byte *keybuff, byte cmpfn)=0; /* async */
+    virtual void OverflowAdjustMapStart( unsigned mapsize,rowmap_t *map,size32_t keybuffsize,const byte *keybuff, byte cmpfn, bool useaux)=0; /* async */
     virtual rowmap_t OverflowAdjustMapStop( unsigned mapsize, rowmap_t *map)=0;
     virtual void MultiMerge(unsigned mapsize,rowmap_t *map,unsigned num,SocketEndpoint* endpoints)=0; /* async */
     virtual void MultiMergeBetween(unsigned mapsize,rowmap_t *map,rowmap_t *mapupper,unsigned num,SocketEndpoint* endpoints)=0; /* async */
@@ -62,7 +62,7 @@ public:
     void MultiBinChop(size32_t keybuffsize,const byte *keybuff, unsigned num,rowmap_t *pos,byte cmpfn,bool useaux);
     void MultiBinChopStart(size32_t keybuffsize,const byte *keybuff, byte cmpfn); /* async */
     void MultiBinChopStop(unsigned num,rowmap_t *pos);
-    void OverflowAdjustMapStart( unsigned mapsize,rowmap_t *map,size32_t keybuffsize,const byte *keybuff, byte cmpfn); /* async */
+    void OverflowAdjustMapStart( unsigned mapsize,rowmap_t *map,size32_t keybuffsize,const byte *keybuff, byte cmpfn,bool useaux); /* async */
     rowmap_t OverflowAdjustMapStop( unsigned mapsize, rowmap_t *map);
     void MultiMerge(unsigned mapsize,rowmap_t *map,unsigned num,SocketEndpoint* endpoints); /* async */
     void MultiMergeBetween(unsigned mapsize,rowmap_t *map,rowmap_t *mapupper,unsigned num,SocketEndpoint* endpoints); /* async */

thorlcr/msort/tsorts.cpp

@@ -623,7 +623,7 @@ public:
 
 
     void OverflowAdjustMapStart(unsigned mapsize, rowmap_t * map,
-                               size32_t keybufsize, const byte * keybuf, byte cmpfn)
+                               size32_t keybufsize, const byte * keybuf, byte cmpfn, bool useaux)
     {
         assertex(overflowfile);
         overflowmap = (rowmap_t *)malloc(mapsize*sizeof(rowmap_t));
@@ -636,7 +636,7 @@ public:
         for (i=0;i<mapsize;i++)
             ActPrintLog(activity, "%"RMF"d ",overflowmap[i]);
 #endif
-        VarElemArray keys(rowif,NULL);
+        VarElemArray keys(useaux?auxrowif:rowif,NULL);
         keys.deserialize(keybuf,keybufsize,false);
         for (i=0;i<mapsize-1;i++)
             AdjustOverflow(overflowmap[i],keys.item(i),cmpfn);