Merge branch 'candidate-7.8.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dockerfiles/hpcc/templates/thor.yaml

@@ -92,6 +92,9 @@ data:
 {{- include "hpcc.generateLoggingConfig" (dict "root" $ "me" $thorScope) | indent 6 }}
     {{ $agentAppType }}:
 {{ toYaml (omit $hthorScope "logging") | indent 6 }}
+      platform:
+        type: "thor"
+        width: {{ $thor.numSlaves }}
 {{- include "hpcc.generateLoggingConfig" (dict "root" $ "me" $hthorScope) | indent 6 }}
     eclagent: # main agent Q handler
 {{ toYaml (omit $eclAgentScope "logging") | indent 6 }}

dockerfiles/incr.sh

@@ -62,8 +62,8 @@ if [[ -z "$FORCE" ]] ; then
   # If not found above, look for latest tagged
   if [[ -z ${PREV} ]] ; then
     PREV=$(git describe --abbrev=0 --tags)-Debug
-    IMAGE_FOUND=$(docker images ${DOCKER_REPO}/platform-build --format {{.Tag}} | fgrep "$PREV" | head -n 1)
-    if [[ -z "$IMAGE_FOUND" ]] ; then
+    docker pull ${DOCKER_REPO}/platform-build:${PREV}
+    if [ $? -ne 0 ]; then
       echo "Could not locate docker image based on PREV tag: ${PREV} for docker user: ${DOCKER_REPO}"
       exit
     fi
@@ -72,7 +72,7 @@ if [[ -z "$FORCE" ]] ; then
   PREV_COMMIT=$(echo "${PREV}" | sed -e "s/-Debug.*$//")
   # create empty patch file
   echo -n > platform-build-incremental/hpcc.gitpatch
-  if [[ -n "$(git status -uno --porcelain)" ]] ; then
+  if [[ -n "$(git status -uno --porcelain)" || "${HEAD}" != "${PREV_COMMIT}" ]] ; then
     git diff --binary ${PREV_COMMIT} ':!./' > platform-build-incremental/hpcc.gitpatch
     # PATCH_MD5 is an ARG of the docker file, which ensures that if different from cached version, image will rebuild from that stage
     PATCH_MD5=$(md5sum platform-build-incremental/hpcc.gitpatch  | awk '{print $1}')

ecl/eclagent/eclagent.cpp

@@ -1558,11 +1558,25 @@ char *EclAgent::getPlatform()
 {
     if (!isStandAloneExe)
     {
+#ifdef _CONTAINERIZED
+        /* NB: platform specs. are defined if agent is running in the context of
+         * another engine, e.g. query has been submitted to Thor, but some code is
+        * executing outside of it.
+        * 
+        * If not defined then assumed to be executing in hthor context,
+        * where platform() defaults to "hthor".
+        */
+        StringBuffer type;
+        if (!agentTopology->getProp("platform/@type", type))
+            type.set("hthor"); // default
+        return type.detach();
+#else            
         const char * cluster = clusterNames.tos();
         Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(cluster);
         if (!clusterInfo)
             throw MakeStringException(-1, "Unknown Cluster '%s'", cluster);
         return strdup(clusterTypeString(clusterInfo->getPlatform(), false));
+#endif
     }
     else
         return strdup("standalone");
@@ -1610,12 +1624,23 @@ unsigned EclAgent::getNodes()//retrieve node count for current cluster
     {
         if (!isStandAloneExe)
         {
+#ifdef _CONTAINERIZED
+            /* NB: platform specs. are defined if agent is running in the context of
+             * another engine, e.g. query has been submitted to Thor, but some code is
+             * executing outside of it.
+             * 
+             * If not defined then assumed to be executing in hthor context,
+             * where getNodes() defaults to 1.
+             */
+            clusterWidth = agentTopology->getPropInt("platform/@width", 1);
+#else            
             const char * cluster = clusterNames.tos();
             Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(cluster);
             if (!clusterInfo) 
                 throw MakeStringException(-1, "Unknown cluster '%s'", cluster);
             clusterWidth = clusterInfo->getSize();
             assertex(clusterWidth != 0);
+#endif
         }
         else
             clusterWidth = 1;
@@ -2951,6 +2976,10 @@ char * EclAgent::getClusterName()
 
 char * EclAgent::getGroupName()
 {
+#ifdef _CONTAINERIZED
+    // in a containerized setup, the group is moving..
+    return strdup("unknown");
+#endif
     StringBuffer groupName;
     if (!isStandAloneExe)
     {

roxie/ccd/ccdcontext.cpp

@@ -3857,6 +3857,10 @@ public:
     }
     virtual char *getGroupName()
     {
+#ifdef _CONTAINERIZED
+        // in a containerized setup, the group is moving..
+        return strdup("unknown");
+#endif
         StringBuffer groupName;
         if (workUnit && clusterNames.length())
         {
@@ -3921,6 +3925,19 @@ public:
     }
     virtual char *getPlatform()
     {
+#ifdef _CONTAINERIZED
+        /* NB: platform specs. are defined if agent is running in the context of
+         * another engine, e.g. query has been submitted to Thor, but some code is
+        * executing outside of it.
+        * 
+        * If not defined then assumed to be executing in roxie context,
+        * where platform() defaults to "roxie".
+        */
+        StringBuffer type;
+        if (!topology->getProp("platform/@type", type))
+            type.set("roxie");
+        return type.detach();
+#else            
         if (clusterNames.length())
         {
             const char * cluster = clusterNames.tos();
@@ -3931,6 +3948,7 @@ public:
         }
         else
             return strdup("roxie");
+#endif
     }
     virtual char *getWuid()
     {
@@ -4053,6 +4071,16 @@ public:
         {
             if (clusterWidth == -1)
             {
+#ifdef _CONTAINERIZED
+                /* NB: platform specs. are defined if agent is running in the context of
+                 * another engine, e.g. query has been submitted to Thor, but some code is
+                 * executing outside of it.
+                 * 
+                 * If not defined then assumed to be executing in roxie context,
+                 * where getNodes() defaults to 'numChannels'.
+                 */
+                clusterWidth = topology->getPropInt("platform/@width", numChannels);
+#else            
                 const char * cluster = clusterNames.tos();
                 Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(cluster);
                 if (!clusterInfo)
@@ -4061,6 +4089,7 @@ public:
                     clusterWidth = numChannels;  // We assume it's the current roxie - that's ok so long as roxie's don't call other roxies.
                 else
                     clusterWidth = clusterInfo->getSize();
+#endif            
             }
             return clusterWidth;
         }

roxie/ccd/ccdmain.cpp

@@ -422,6 +422,11 @@ void readStaticTopology()
     std::vector<RoxieEndpointInfo> allRoles;
     IpAddressArray nodeTable;
     unsigned numNodes = topology->getCount("./RoxieServerProcess");
+    if (!numNodes && localSlave)
+    {
+        topology->addPropTree("RoxieServerProcess")->setProp("@netAddress", ".");
+        numNodes = 1;
+    }
     Owned<IPropertyTreeIterator> roxieServers = topology->getElements("./RoxieServerProcess");
 
     bool myNodeSet = false;
@@ -695,6 +700,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         {
             if (topology->getPropBool("@server", false))
             {
+#ifdef _CONTAINERIZED
                 if (!topology->getCount("services"))
                 {
                     // Makes debugging easier...
@@ -702,6 +708,14 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
                     service->setProp("@name", "query");
                     service->setPropInt("@port", 9876);
                 }
+#else
+                if (!topology->getCount("RoxieFarmProcess"))
+                {
+                    // Makes debugging easier...
+                    IPropertyTree *service = topology->addPropTree("RoxieFarmProcess");
+                    service->setPropInt("@port", 9876);
+                }
+#endif
                 runOnce = false;
             }
             else
@@ -1350,6 +1364,8 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
     {
         StringBuffer x;
         IERRLOG("EXCEPTION: (%d): %s", E->errorCode(), E->errorMessage(x).str());
+        if (!queryLogMsgManager()->isActiveMonitor(queryStderrLogMsgHandler()))
+            fprintf(stderr, "EXCEPTION: (%d): %s\n", E->errorCode(), x.str());
         E->Release();
     }