Merge branch 'candidate-7.8.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/server/daserver.cpp

@@ -618,8 +618,6 @@ int main(int argc, const char* argv[])
         mpServer->installWhiteListCallback(whiteListHandler);
 #ifndef _CONTAINERIZED
         setMsgLevel(fileMsgHandler, serverConfig->getPropInt("SDS/@msgLevel", 100));
-#else
-        setupContainerizedLogMsgHandler();
 #endif
         startLogMsgChildReceiver(); 
         startLogMsgParentReceiver();

dockerfiles/platform-build/Dockerfile

@@ -48,7 +48,7 @@ RUN mkdir build
 WORKDIR /hpcc-dev/build
 
 ARG BUILD_TYPE=RelWithDebInfo
-RUN cmake /hpcc-dev/HPCC-Platform -Wno-dev -DCONTAINERIZED=1 -DINCLUDE_PLUGINS=1 -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
+RUN cmake /hpcc-dev/HPCC-Platform -Wno-dev -DCONTAINERIZED=1 -DINCLUDE_PLUGINS=1 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DUSE_PYTHON2=0
 
 ARG BUILD_THREADS
 RUN if [ -n "${BUILD_THREADS}" ] ; then echo ${BUILD_THREADS} > ~/build_threads; else echo $(nproc) > ~/build_threads ; fi

dockerfiles/startall.sh

@@ -20,11 +20,38 @@
 # Utility script for starting a local cluster corresponding to current git branch
 # Usage startup.sh [<image.version>] [<helm-install-options>]
 
-LABEL=$1
-[[ -z ${LABEL} ]] && LABEL=$(docker image ls | fgrep 'hpccsystems/platform-core' | head -n 1 | awk '{print $2}')
-shift
+DOCKER_REPO=hpccsystems
+restArgs=()
+
+while [ "$#" -gt 0 ]; do
+  arg=$1
+  if [[ ${arg:0:1} == '-' ]]; then
+    case "${arg:1:1}" in
+      l) shift
+         LABEL=$1
+         ;;
+      d) shift;
+         INPUT_DOCKER_REPO=$1
+         ;;
+      h) echo "Usage: startall.sh [options]"
+         echo "    -d <docker-repo>   Docker repository to fetch images from"
+         echo "    -l                 Build image label to use"
+         exit
+         ;;
+      *) restArgs+=(${arg})
+         ;;
+    esac
+  else
+    restArgs+=(${arg})
+  fi
+  shift
+done
+
+[[ -n ${INPUT_DOCKER_REPO} ]] && DOCKER_REPO=${INPUT_DOCKER_REPO}
+[[ -z ${LABEL} ]] && LABEL=$(docker image ls | fgrep "${DOCKER_REPO}/platform-core" | head -n 1 | awk '{print $2}')
+
+helm install mycluster hpcc/ --set global.image.root="${DOCKER_REPO}" --set global.image.version=$LABEL --set global.privileged=true ${restArgs[@]}
 
-helm install mycluster hpcc/ --set global.image.version=$LABEL --set global.privileged=true $*
 sleep 1
 kubectl get pods
 

ecl/hthor/hthor.cpp

@@ -8401,9 +8401,11 @@ bool CHThorDiskReadBaseActivity::openNext()
     actualFilter.clear();
     if (translators)
     {
-        /* if previous part was remotely accessed, the format used (actualDiskMeta), became the projected meta
-         * reset here, in case this next part is access locally, in which case the published or expect meta is needed
+        /* If previous part was remotely accessed, the format used (actualDiskMeta), became the projected meta.
+         * Reset for local/direct access.
          */
+        translator = &translators->queryTranslator();
+        keyedTranslator = translators->queryKeyedTranslator();
         actualDiskMeta.set(&translators->queryActualFormat());
     }
 

plugins/py3embed/CMakeLists.txt

@@ -53,9 +53,15 @@ elseif(NOT APPLE)
     set_target_properties(py3embed PROPERTIES NO_SONAME 1)
 endif()
 
+if (CONTAINERIZED)
+install(
+    TARGETS py3embed
+    DESTINATION plugins)
+else()
 install(
     TARGETS py3embed
     DESTINATION versioned/python3)
+endif()
 
 target_link_libraries(py3embed ${Python3_LIBRARIES})
 

plugins/pyembed/CMakeLists.txt

@@ -59,9 +59,16 @@ elseif(NOT APPLE)
     set_target_properties(py2embed PROPERTIES NO_SONAME 1)
 endif()
 
+if (CONTAINERIZED)
+install(
+    TARGETS py2embed
+    DESTINATION plugins)
+else()
 install(
     TARGETS py2embed
     DESTINATION versioned/python2)
+endif()
+
 if(debug_python)
     target_link_libraries(py2embed ${DEBUG_PYTHON2_LIBRARY})
 else()

system/jlib/jlog.cpp

@@ -2379,7 +2379,8 @@ void setupContainerizedLogMsgHandler()
             if (!isEmptyString(logAudiences))
                 msgAudiences = logMsgAudsFromAbbrevs(logAudiences);
 
-            Owned<ILogMsgFilter> filter = getCategoryLogMsgFilter(msgAudiences, msgClasses, logDetail);
+            const bool local = true; // Do not include remote messages from other components
+            Owned<ILogMsgFilter> filter = getCategoryLogMsgFilter(msgAudiences, msgClasses, logDetail, local);
             theManager->changeMonitorFilter(theStderrHandler, filter);
         }
 

thorlcr/activities/indexwrite/thindexwriteslave.cpp

@@ -158,7 +158,7 @@ public:
         getPartFilename(partDesc, 0, partFname);
         bool compress=false;
         ActPrintLog("INDEXWRITE: created fixed output stream %s", partFname.str());
-        bool needsSeek = true;
+        //bool needsSeek = true;
         unsigned flags = COL_PREFIX;
         if (TIWrowcompress & helper->getFlags())
             flags |= HTREE_COMPRESSED_KEY|HTREE_QUICK_COMPRESSED_KEY;
@@ -177,12 +177,12 @@ public:
         if (metadata->getPropBool("_noSeek", defaultNoSeek))
         {
             flags |= TRAILING_HEADER_ONLY;
-            needsSeek = false;
+            //needsSeek = false;
         }
         if (metadata->getPropBool("_useTrailingHeader", true))
             flags |= USE_TRAILING_HEADER;
         OwnedIFileIO iFileIO = createMultipleWrite(this, partDesc, 0, TW_RenameToPrimary, compress, NULL, this, &abortSoon);
-        Owned<IFileIOStream> out = createBufferedIOStream(iFileIO, needsSeek);
+        Owned<IFileIOStream> out = createBufferedIOStream(iFileIO); //, needsSeek);
         builder.setown(createKeyBuilder(out, flags, maxDiskRecordSize, nodeSize, helper->getKeyedSize(), isTopLevel ? 0 : totalCount, helper, !isTlk, isTlk));
     }
     void buildUserMetadata(Owned<IPropertyTree> & metadata)