HPCC-23900 Roxie query port should be exposed as a service under k8s

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dockerfiles/hpcc/templates/localroxie.yaml

@@ -2,6 +2,7 @@
 {{- if not $roxie.disabled  -}}
 {{- if $roxie.localSlave -}}
 {{- $name := $roxie.name -}}
+{{- $servername := printf "%s-server" $roxie.name -}}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -16,6 +17,7 @@ spec:
     metadata:
       labels:
         run: {{ $roxie.name | quote }}
+        roxie-server: {{ $servername | quote }}
     spec:
       {{- include "hpcc.checkDataStorageHostMount" (dict "root" $) | indent 6 }}
       containers:
@@ -23,10 +25,21 @@ spec:
         args: [
                 {{ include "hpcc.configArg" $roxie }},
                 {{ include "hpcc.daliArg" $ }},
-                "--serverPorts={{ template "hpcc.portListToCommas" $roxie.ports }}", 
+                "--server=true", 
                 "--localSlave=true",
                 "--resolveLocally=false"
               ]
+{{- $local := dict "first" true }}
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- if $local.first }}
+{{- $_ := set $local "first" false }}
+        ports:
+{{- end }}
+        - name: {{ $service.name }}
+          containerPort: {{ $service.port }}
+{{- end }}
+{{- end }}
 {{- include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
 {{ include "hpcc.addImageAttrs" (dict "root" $ "me" . "imagename" "roxie") | indent 8 }}
         volumeMounts:
@@ -38,6 +51,24 @@ spec:
 {{ include "hpcc.addDataVolume" $ | indent 6 }}
 {{ include "hpcc.addDllserverVolume" $ | indent 6 }}
 ---
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- $name := printf "%s-%s" $roxie.name $service.name }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $name | quote }}
+spec:
+  ports:
+  - port: {{ $service.port }}
+    protocol: TCP
+    targetPort: {{ $service.port }}
+  selector:
+    roxie-server: {{ $servername | quote }}
+  type: {{ if $service.external -}} LoadBalancer {{- else -}} ClusterIP {{- end }}
+{{- end }}
+{{- end }}
+---
 kind: ConfigMap 
 apiVersion: v1 
 metadata:

dockerfiles/hpcc/templates/roxie.yaml

@@ -4,6 +4,7 @@
 {{- $toponame := printf "%s-toposerver" $roxie.name -}}
 {{- $numChannels := $roxie.numChannels | int | default 1 -}}
 {{- $topoport := $roxie.topoport | int | default 9004 -}}
+{{- $servername := printf "%s-server" $roxie.name -}}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -32,6 +33,26 @@ spec:
           name: {{ .name }}-configmap
 ---
 
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- $name := printf "%s-%s" $roxie.name $service.name }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $name | quote }}
+spec:
+  ports:
+  - port: {{ $service.port }}
+    protocol: TCP
+    targetPort: {{ $service.port }}
+  selector:
+    roxie-server: {{ $servername | quote }}
+  type: {{ if $service.external -}} LoadBalancer {{- else -}} ClusterIP {{- end }}
+{{- end }}
+{{- end }}
+
+---
+
 apiVersion: v1
 kind: Service
 metadata:
@@ -60,7 +81,6 @@ data:
 ---
 
 {{ if $roxie.serverReplicas -}}
-{{- $servername := printf "%s-server" $roxie.name -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -74,6 +94,7 @@ spec:
     metadata:
       labels:
         run: {{ $servername | quote }}
+        roxie-server: {{ $servername | quote }}
     spec:
       {{- include "hpcc.checkDataStorageHostMount" (dict "root" $) | indent 6 }}
       containers:
@@ -82,10 +103,21 @@ spec:
                 {{ include "hpcc.configArg" . }},
                 {{ include "hpcc.daliArg" $ }},
                 "--numChannels={{ $numChannels }}",
-                "--serverPorts={{ template "hpcc.portListToCommas" $roxie.ports }}", 
+                "--server=true", 
                 "--topologyServers={{ $toponame }}:{{ $roxie.topoport }}",
                 "--resolveLocally=false"
               ]
+{{- $local := dict "first" true }}
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- if $local.first }}
+{{- $_ := set $local "first" false }}
+        ports:
+{{- end }}
+        - name: {{ $service.name }}
+          containerPort: {{ $service.port }}
+{{- end }}
+{{- end }}
 {{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
 {{ include "hpcc.addImageAttrs" (dict "root" $ "me" $roxie "imagename" "roxie") | indent 8 }}
         volumeMounts:
@@ -108,7 +140,7 @@ kind: Deployment
 metadata:
   name: {{ $name | quote}}
 spec:
-  replicas: {{ $roxie.channelReplicas | default 2 }}
+  replicas: {{ $roxie.channelReplicas | default 1 }}
   selector:
     matchLabels:
       run: {{ $name | quote}}
@@ -116,6 +148,9 @@ spec:
     metadata:
       labels:
         run: {{ $name | quote}}
+{{- if not $roxie.serverReplicas }}        
+        roxie-server: {{ $servername | quote }}
+{{- end }}
     spec:
       {{- include "hpcc.checkDataStorageHostMount" (dict "root" $) | indent 6 }}
       containers:
@@ -124,10 +159,24 @@ spec:
                 {{ include "hpcc.configArg" $roxie }},
                 {{ include "hpcc.daliArg" $ }},
                 "--channels={{ $channel }}", 
-                "--serverPorts={{ if not $roxie.serverReplicas }}{{ template "hpcc.portListToCommas" $roxie.ports }}{{ end }}",
+                "--server={{ not $roxie.serverReplicas }}",
                 "--numChannels={{ $numChannels }}",
                 "--topologyServers={{ $toponame }}:{{ $roxie.topoport }}",
+                "--resolveLocally=false"
               ]
+{{- if not $roxie.serverReplicas }}
+{{- $local := dict "first" true }}
+{{- range $service := $roxie.services }}
+{{- if ne (int $service.port)  0 }}
+{{- if $local.first }}
+{{- $_ := set $local "first" false }}
+        ports:
+{{- end }}
+        - name: {{ $service.name }}
+          containerPort: {{ $service.port }}
+{{- end }}
+{{- end }}
+{{- end }}
 {{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
 {{ include "hpcc.addImageAttrs" (dict "root" $ "me" $roxie "imagename" "roxie") | indent 8 }}
         volumeMounts:

dockerfiles/hpcc/values.schema.json

@@ -210,16 +210,38 @@
           "type": "string",
           "description": "The (optional) file prefix to add to relative filenames"
         },
-        "ports": {
-          "type": "array",
-          "description": "The ports to listen on",
-          "items": { "type": "integer" }
-        },
         "image": {
           "$ref": "#/definitions/image"
-        }
+        },
+        "services": {
+          "description": "Roxie query services",
+          "type": "array",
+          "items": { "$ref": "#/definitions/roxieservice" }
+        }    
       }
     },
+    "roxieservice": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "port": {
+          "type": "integer"
+        },
+        "numThreads": {
+          "type": "integer"
+        },
+        "listenQueue": {
+          "type": "integer"
+        },
+        "external": {
+          "type": "boolean"
+        }
+      },
+      "required": [ "name", "port" ],
+      "additionalProperties": false
+    },
     "thor": {
       "type": "object",
       "required": [ "name" ],

dockerfiles/hpcc/values.yaml

@@ -83,14 +83,24 @@ esp:
 
 roxie:
 - name: roxie-cluster
-  disabled: true
+  disabled: false
   prefix: roxiecluster
-  ports: [9876]
+  services:
+  - name: query
+    port: 9876
+    listenQueue: 200
+    numThreads: 0
+    external: true
+  - name: on-demand
+    port: 0
   numChannels: 2
-  serverReplicas: 1
+  ## Set serverReplicas to indicate a separate replicaSet of roxie servers, with slave nodes not acting as servers
+  serverReplicas: 0
   topoReplicas: 1
   topoport: 9004
-  useAeron: true
+  ## Set localSlave to true for a scalable cluster of "single-node" roxie servers
+  localSlave: false
+  useAeron: false
 
 thor:
 - name: thor

dockerfiles/roxie/Dockerfile

@@ -23,6 +23,5 @@ FROM hpccsystems/platform-core:${BUILD_LABEL}
 USER hpcc
 RUN mkdir -p /var/lib/HPCCSystems/roxie
 WORKDIR /var/lib/HPCCSystems/roxie
-RUN roxie --init > roxie.yaml
-ENTRYPOINT ["roxie", "--config=roxie.yaml"]
+ENTRYPOINT ["roxie"]
 

roxie/ccd/ccd.hpp

@@ -297,7 +297,11 @@ extern bool simpleLocalKeyedJoins;
 extern bool enableKeyDiff;
 extern PTreeReaderOptions defaultXmlReadFlags;
 extern bool mergeSlaveStatistics;
+#ifdef _CONTAINERIZED
+static constexpr bool roxieMulticastEnabled = false;
+#else
 extern bool roxieMulticastEnabled;   // enable use of multicast for sending requests to slaves
+#endif
 extern bool preloadOnceData;
 extern bool reloadRetriesFailed;
 extern bool selfTestMode;

roxie/ccd/ccdmain.cpp

@@ -95,7 +95,9 @@ PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
 bool runOnce = false;
 
 unsigned udpMulticastBufferSize = 262142;
+#ifndef _CONTAINERIZED
 bool roxieMulticastEnabled = true;
+#endif
 
 IPropertyTree *topology;
 MapStringTo<int> *preferredClusters;
@@ -410,9 +412,9 @@ public:
 };
 
 static std::vector<RoxieEndpointInfo> myRoles;
-static std::vector<unsigned> farmerPorts;
 static std::vector<std::pair<unsigned, unsigned>> slaveChannels;
 
+#ifndef _CONTAINERIZED
 void readStaticTopology()
 {
     // If dynamicServers not set, we read a list of all servers form the topology file, and deduce which ones are on which channel
@@ -523,6 +525,7 @@ void readStaticTopology()
     numChannels = calcNumChannels;
     createStaticTopology(allRoles, traceLevel);
 }
+#endif
 
 int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
 {
@@ -654,16 +657,6 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         StringArray topoValues;
         if (topos)
             topoValues.appendList(topos, ",", true);
-        const char *serverPorts = topology->queryProp("@serverPorts");
-        if (serverPorts)
-        {
-            StringArray values;
-            values.appendList(serverPorts, ",", true);
-            ForEachItemIn(idx, values)
-            {
-                farmerPorts.push_back(atoi(values.item(idx)));
-            }
-        }
 
         if (topology->hasProp("PreferredCluster"))
         {
@@ -700,13 +693,11 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         }
         if (standAloneDll || wuid)
         {
-            unsigned port = topology->getPropInt("@port", 0);
-            runOnce = port == 0;
-            if (port)
-            {
-                farmerPorts.clear();
-                farmerPorts.push_back(port);
-            }
+            IPropertyTree *services = topology->queryPropTree("services");
+            if (topology->getPropBool("@server", false) && services && services->hasChildren())
+                runOnce = false;
+            else
+                runOnce = true;
         }
 
         if (!topology->hasProp("@resolveLocally"))
@@ -914,8 +905,9 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         udpMulticastBufferSize = topology->getPropInt("@udpMulticastBufferSize", 262142);
         udpFlowSocketsSize = topology->getPropInt("@udpFlowSocketsSize", 131072);
         udpLocalWriteSocketSize = topology->getPropInt("@udpLocalWriteSocketSize", 1024000);
-        
+#ifndef _CONTAINERIZED
         roxieMulticastEnabled = topology->getPropBool("@roxieMulticastEnabled", true) && !useAeron;   // enable use of multicast for sending requests to slaves
+#endif
         if (udpSnifferEnabled && !roxieMulticastEnabled)
         {
             DBGLOG("WARNING: ignoring udpSnifferEnabled setting as multicast not enabled");
@@ -1093,13 +1085,39 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
 #else
         topology->addPropBool("@linuxOS", true);
 #endif
+        if (useAeron)
+            setAeronProperties(topology);
+
 #ifdef _CONTAINERIZED
         allQuerySetNames.append(roxieName);
 #else
         allQuerySetNames.appendListUniq(topology->queryProp("@querySets"), ",");
 #endif
-        // Set multicast base addresses - must be done before generating slave channels
 
+#ifdef _CONTAINERIZED
+        if (!numChannels)
+            throw makeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numChannels not set");
+        IpAddress myIP(".");
+        myNode.setIp(myIP);
+        if (topology->getPropBool("@server", true))
+        {
+            Owned<IPropertyTreeIterator> roxieFarms = topology->getElements("./services");
+            ForEach(*roxieFarms)
+            {
+                IPropertyTree &roxieFarm = roxieFarms->query();
+                unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
+                RoxieEndpointInfo me = {RoxieEndpointInfo::RoxieServer, 0, { (unsigned short) port, myIP }, 0};
+                myRoles.push_back(me);
+            }
+        }
+        for (std::pair<unsigned, unsigned> channel: slaveChannels)
+        {
+            mySlaveEP.set(ccdMulticastPort, myIP);
+            RoxieEndpointInfo me = { RoxieEndpointInfo::RoxieSlave, channel.first, mySlaveEP, channel.second };
+            myRoles.push_back(me);
+        }
+#else
+        // Set multicast base addresses - must be done before generating slave channels
         if (roxieMulticastEnabled && !localSlave)
         {
             if (topology->queryProp("@multicastBase"))
@@ -1111,34 +1129,8 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
             else
                 throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - multicastLast not set");
         }
-        if (useAeron)
-            setAeronProperties(topology);
-
-        if (useDynamicServers)
-        {
-            if (!numChannels)
-                throw makeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numChannels not set");
-            IpAddress myIP(".");
-            myNode.setIp(myIP);
-            for (unsigned port: farmerPorts)
-            {
-                VStringBuffer xpath("RoxieFarmProcess[@port='%u']", port);
-                if (!topology->hasProp(xpath))
-                    topology->addPropTree("RoxieFarmProcess")->setPropInt("@port", port);
-                RoxieEndpointInfo me = {RoxieEndpointInfo::RoxieServer, 0, { (unsigned short) port, myIP }, 0};
-                myRoles.push_back(me);
-            }
-            for (std::pair<unsigned, unsigned> channel: slaveChannels)
-            {
-                mySlaveEP.set(ccdMulticastPort, myIP);
-                RoxieEndpointInfo me = { RoxieEndpointInfo::RoxieSlave, channel.first, mySlaveEP, channel.second };
-                myRoles.push_back(me);
-            }
-        }
-        else
-        {
-            readStaticTopology();
-        }
+        readStaticTopology();
+#endif
         // Now we know all the channels, we can open and subscribe the multicast channels
         if (!localSlave)
         {
@@ -1219,7 +1211,11 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         {
             try
             {
+#ifdef _CONTAINERIZED
+                Owned<IPropertyTreeIterator> roxieFarms = topology->getElements("./services");
+#else
                 Owned<IPropertyTreeIterator> roxieFarms = topology->getElements("./RoxieFarmProcess");
+#endif
                 ForEach(*roxieFarms)
                 {
                     IPropertyTree &roxieFarm = roxieFarms->query();
@@ -1228,11 +1224,6 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
                     if (!numThreads)
                         numThreads = numServerThreads;
                     unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
-                    if (useDynamicServers)
-                    {
-                        if (std::find(std::begin(farmerPorts), std::end(farmerPorts), port) == std::end(farmerPorts))
-                            continue;
-                    }
                     //unsigned requestArrayThreads = roxieFarm.getPropInt("@requestArrayThreads", 5);
                     // NOTE: farmer name [@name=] is not copied into topology
                     const IpAddress &ip = myNode.getNodeAddress();
@@ -1401,19 +1392,9 @@ roxie:
   allFilesDynamic: true
   localSlave: true
   numChannels: 1
-  numServerThreads: 30
   queueNames: roxie.roxie
-  roxieMulticastEnabled: false
   traceLevel: 0
-  useAeron: false
-  RoxieFarmProcess:
-    - name: default
-      port: 9876
-      listenQueue: 200
-      numThreads: 0
-    - name: workunit
-      port: 0
-      numThreads: 0
+  server: false
   logging:
     disabled: true
 )!!";

roxie/roxie/roxie.cpp

@@ -50,21 +50,14 @@ roxie:
   allFilesDynamic: true
   localSlave: true
   numChannels: 1
-  numServerThreads: 30
   queueNames: roxie.roxie
-  serverPorts: "9876,0"
-  roxieMulticastEnabled: false
-  useAeron: false
-  RoxieFarmProcess:
-    - name: default
+  services:
+    - name: query
       port: 9876
-      listenQueue: 200
-      numThreads: 0
     - name: workunit
       port: 0
-      numThreads: 0
   logging:
-      detail: 100
+    detail: 100
 )!!";
 
 int main(int argc, const char *argv[])

roxie/udplib/udptopo.cpp

@@ -345,10 +345,12 @@ extern UDPLIB_API unsigned getNumSlaves(unsigned channel)
     return topology->querySlaves(channel).ordinality();
 }
 
+#ifndef _CONTAINERIZED
 extern UDPLIB_API void createStaticTopology(const std::vector<RoxieEndpointInfo> &allRoles, unsigned traceLevel)
 {
     topologyManager.setRoles(allRoles);
 }
+#endif
 
 static std::thread topoThread;
 static Semaphore abortTopo;

roxie/udplib/udptopo.hpp

@@ -116,6 +116,7 @@ struct RoxieEndpointInfo
 };
 
 extern UDPLIB_API void startTopoThread(const StringArray &topoServers, const std::vector<RoxieEndpointInfo> &myRoles, unsigned traceLevel);
+#ifndef _CONTAINERIZED
 extern UDPLIB_API void createStaticTopology(const std::vector<RoxieEndpointInfo> &allRoles, unsigned traceLevel);
-
+#endif
 #endif

system/jlib/jptree.cpp

@@ -8046,12 +8046,6 @@ jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char
         applyEnvironmentConfig(*config, envPrefix, *cur);
     }
 
-#ifdef _DEBUG
-    // NB: don't re-hold, if CLI --hold already held.
-    if (!held && config->getPropBool("@hold"))
-        holdLoop();
-#endif
-
     if (outputConfig)
     {
         loadArgsIntoConfiguration(config, argv, { "config", "outputconfig" });
@@ -8069,6 +8063,12 @@ jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char
     else
         loadArgsIntoConfiguration(config, argv);
 
+#ifdef _DEBUG
+    // NB: don't re-hold, if CLI --hold already held.
+    if (!held && config->getPropBool("@hold"))
+        holdLoop();
+#endif
+
     if (!globalConfiguration)
         globalConfiguration.setown(createPTree("global"));