HPCC-25722 Allow annotations on services and pods

Also common-up a lot of service-related handling, and modify the structure of
service-related information in the values.yaml file to be more consistent
between different components.

Allow user-specified service visibilities

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

helm/examples/loggingservice/loggingservice-values.yaml

@@ -5,5 +5,6 @@ esp:
   auth: none
   tls: off
   replicas: 1
-  servicePort: 8123 # This is the service port your logging service will be accessible on.
-  public: false # generally an ESP logging service would be used by internal components, not accessed by external public
+  service:
+    servicePort: 8123 # This is the service port your logging service will be accessible on.
+    visibility: cluster # generally an ESP logging service would be used by internal components, not accessed by external public

helm/hpcc/docs/changes.md

@@ -0,0 +1,119 @@
+#Changes in 8.2.0
+
+There are a few changes in the way the values.yaml needs to be specified for HPCC clusters
+starting with 8.2.0. These were necessary to implement required functionality. If you have a
+customized values file that you have been using with 8.0.x builds, you will need to make a
+few simple changes to be able to use it with 8.2.x. The required changes are all enforced
+by the schema, so you will see schema errors if you try to use an uncorrected values file.
+
+Service changes
+
+In order to support annotations on services, and thus control whether public services are
+connected to internal subnets or published to the internet, we have changed the way that
+services are specified. We have also adjusted some names of fields within service definitions
+so that Roxie, Sasha and ESP services are more consistent in their syntax.
+
+The following changes should be noted:
+1. For ESP and Sasha services, the settings related to the service have been moved into a new 
+   service: section within the component settings.
+2. For Roxie services, the name of the setting used to define the external port has been
+   changed to servicePort
+3. The “public” setting on ESP services and the “external” setting on Roxie services have
+   been replaced by a new “visibility” setting specifying a user-settable preset. We provide
+   pre-populated presets “cluster”, “local” and “global” but others can be added if needed.
+4. Annotations and labels can be specified on any service, or for any visibility preset, or
+   for any component pod.
+
+If you try to launch with an unmodified 8.0.x values.yaml, you may see errors like this:
+
+```code
+- esp.1: service is required
+- esp.2: service is required
+- esp.3: service is required
+- esp.4: service is required
+- roxie.0.services.0: servicePort is required
+- roxie.0.services.0: Additional property external is not allowed
+- sasha: Must validate one and only one schema (oneOf)
+- sasha.dfuwu-archiver: Must validate one and only one schema (oneOf)
+- sasha.dfuwu-archiver: Additional property servicePort is not allowed
+- sasha.dfuwu-archiver: Must validate all the schemas (allOf)
+- sasha.wu-archiver: Must validate one and only one schema (oneOf)
+- sasha.wu-archiver: Additional property servicePort is not allowed
+- sasha.wu-archiver: Must validate all the schemas (allOf)
+```
+
+
For the ESP “service is required” errors, you will need to change a spec that looks like (for example)
+
+```code
+esp:
+- name: eclwatch
+  application: eclwatch
+  auth: none
+  replicas: 1
+  port: 8888
+  servicePort: 8010
+  public: true
+```
+
+to

+
+```code
+- name: eclwatch
+  application: eclwatch
+  auth: none
+  replicas: 1
+  service:
+    port: 8888
+    servicePort: 8010
+    visibility: local  # or global if you want to be open to the internet
+```
+
+For the roxie errors, change something that looks like (for example)
+
+```code
+roxie:
+- name: roxie
+  disabled: false
+  prefix: roxie
+  services:
+  - name: roxie
+    port: 9876
+    listenQueue: 200
+    numThreads: 30
+    external: true
+  ...
+```
+
+
to
+
+```code
+roxie:
+- name: roxie
+  disabled: false
+  prefix: roxie
+  services:
+  - name: roxie
+    servicePort: 9876
+    listenQueue: 200
+    numThreads: 30
+    visibility: local
+```
+
+The sasha errors are not quite so descriptive but are addressed in the same way as the ESP ones, i.e. change something that looks like:


+
+```code
+sasha:
+  wu-archiver:
+    servicePort: 8877
+```
+
+to
+
+```code
+sasha:
+  wu-archiver:
+    service:
+       servicePort: 8877
+    storage:
+```
+

helm/hpcc/templates/_helpers.tpl

@@ -840,13 +840,13 @@ Generate list of available services
 {{- range $roxie := $.Values.roxie -}}
  {{- if not $roxie.disabled -}}
   {{- range $service := $roxie.services -}}
-   {{- if ne (int $service.port) 0 -}}
+   {{- if ne (int $service.servicePort) 0 -}}
 - name: {{ $service.name }}
   class: roxie
   type: roxie
-  port: {{ $service.port }}
+  port: {{ $service.servicePort }}
   target: {{ $roxie.name }}
-  public: {{ $service.external }}
+  public: {{ (ne ( include "hpcc.isVisibilityPublic" (dict "root" $ "visibility" $service.visibility)) "") | ternary "true" "false" }}
    {{- end -}}
   {{- end }}
 {{ end -}}
@@ -855,24 +855,26 @@ Generate list of available services
 - name: {{ $esp.name }}
   class: esp
   type: {{ $esp.application }}
-  port: {{ $esp.servicePort }}
+  port: {{ $esp.service.servicePort }}
   {{- if hasKey $esp "tls" }}
   tls: {{ $esp.tls }}
   {{- else }}
   tls: {{ ($.Values.certificates | default dict).enabled }}
   {{- end }}
-  public: {{ $esp.public }}
+  public: {{ (ne ( include "hpcc.isVisibilityPublic" (dict "root" $ "visibility" $esp.service.visibility))  "") | ternary "true" "false" }}
 {{ end -}}
 {{- range $dali := $.Values.dali -}}
 {{- $sashaServices := $dali.services | default dict -}}
 {{- if not $sashaServices.disabled -}}
 {{- range $sashaName, $_sasha := $sashaServices -}}
 {{- $sasha := ($_sasha | default dict) -}}
-{{- if and (not $sasha.disabled) ($sasha.servicePort) -}}
+{{- if hasKey $sasha "service" -}}
+{{- if and (not $sasha.disabled) ($sasha.service.servicePort) -}}
 - name: {{ printf "sasha-%s" $sashaName }}
   class: sasha
   type: {{ $sashaName }}
-  port: {{ $sasha.servicePort }}
+  port: {{ $sasha.service.servicePort }}
+{{ end -}}
 {{ end -}}
 {{ end -}}
 {{ end -}}
@@ -881,11 +883,13 @@ Generate list of available services
 {{- if not $sashaServices.disabled -}}
 {{- range $sashaName, $_sasha := $sashaServices -}}
 {{- $sasha := ($_sasha | default dict) -}}
-{{- if and (not $sasha.disabled) ($sasha.servicePort) -}}
+{{- if and (not $sasha.disabled) (hasKey $sasha "service") -}}
+{{- if $sasha.service.servicePort -}}
 - name: {{ printf "sasha-%s" $sashaName }}
   class: sasha
   type: {{ $sashaName }}
-  port: {{ $sasha.servicePort }}
+  port: {{ $sasha.service.servicePort }}
+{{ end -}}
 {{ end -}}
 {{ end -}}
 {{- end -}}
@@ -1023,23 +1027,71 @@ Pass in dict with root and me
 {{- end -}}
 
 {{/*
-A template to generate Sasha service
-Pass in dict me
+A template to generate the type of a service based on the visibility setting
+Pass in dict with .root, .visibility defined
 */}}
-{{- define "hpcc.addSashaService" }}
-{{- $serviceName := printf "sasha-%s" .me.name }}
+{{- define "hpcc.isVisibilityPublic" }}
+{{- if and (hasKey . "visibility") .visibility -}}
+ {{- if hasKey .root.Values.global "visibilities" -}}
+  {{- if hasKey .root.Values.global.visibilities .visibility -}}
+   {{- $globalServiceInfo := get .root.Values.global.visibilities .visibility -}}
+   {{- if (not (eq $globalServiceInfo.type "ClusterIP")) -}}
+   true
+   {{- end -}}
+  {{- else -}}
+   {{- required (printf "Specified service visibility %s not found in global visibilities section" .visibility) nil -}}
+  {{- end -}}
+ {{- else -}}
+  {{- required "global visibilities section not found" nil -}}
+ {{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+A template to generate a service
+Pass in dict with .root, .name, .service, .defaultPort, .selector defined
+*/}}
+{{- define "hpcc.addService" }}
+{{- $lvars := dict "type" "ClusterIP" "labels" dict "annotations" dict -}}
+{{- if hasKey . "service" -}}
+ {{- if hasKey .service "labels" -}}{{- $_ := set $lvars "labels" (merge $lvars.labels .service.labels) -}}{{- end -}}
+ {{- if hasKey .service "annotations" -}}{{- $_ := set $lvars "annotations" (merge $lvars.annotations .service.annotations) -}}{{- end -}}
+ {{- if hasKey .service "visibility" -}}
+  {{- if hasKey .root.Values.global "visibilities" -}}
+   {{- if hasKey .root.Values.global.visibilities .service.visibility -}}
+    {{- $globalServiceInfo := get .root.Values.global.visibilities .service.visibility -}}
+    {{- if hasKey $globalServiceInfo "labels" -}}{{- $_ := set $lvars "labels" (merge $lvars.labels $globalServiceInfo.labels) -}}{{- end -}}
+    {{- if hasKey $globalServiceInfo "annotations" -}}{{- $_ := set $lvars "annotations" (merge $lvars.annotations $globalServiceInfo.annotations) -}}{{- end -}}
+    {{- $_ := set $lvars "type" $globalServiceInfo.type -}}
+   {{- else -}}
+    {{- required (printf "Specified service visibility %s not found in global visibilities section" .service.visibility) nil -}}
+   {{- end -}}
+  {{- else -}}
+   {{- required "global visibilities section not found" nil -}}
+  {{- end -}}
+ {{- end -}}
+{{- end -}}
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ $serviceName | quote }}
+  name: {{ .name | quote }}
+  labels:
+    helmVersion: 8.2.0-closedown
+{{- if $lvars.labels }}
+{{ toYaml $lvars.labels | indent 4 }}
+{{- end }}
+{{- if $lvars.annotations }}
+  annotations:
+{{ toYaml $lvars.annotations | indent 4 }}
+{{- end }}
 spec:
   ports:
-  - port: {{ .me.servicePort }}
+  - port: {{ required "servicePort must be specified" .service.servicePort }}
     protocol: TCP
-    targetPort: {{ .port | default 8877 }}
+    targetPort: {{ .service.port | default .defaultPort }}
   selector:
-    run: {{ .selectorName | default $serviceName | quote }}
-  type: ClusterIP
+    server: {{ .selector | quote }}
+  type: {{ $lvars.type }}
 {{- end -}}
 
 
@@ -1193,7 +1245,7 @@ so that we can recognize the signature of our own CA.
 */}}
 {{- define "hpcc.addCertificate" }}
 {{- if (.root.Values.certificates | default dict).enabled -}}
-{{- $externalCert := and (hasKey . "external") .external -}}
+{{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}}
 {{- $issuer := ternary .root.Values.certificates.issuers.public .root.Values.certificates.issuers.local $externalCert -}}
 {{- if $issuer -}}
 {{- $namespace := .root.Release.Namespace -}}
@@ -1230,7 +1282,7 @@ spec:
   - {{ .servicename }}.{{ $domain }}
  {{- /* if service parameter is passed in we are using the component config as a service config entry */ -}}
  {{- else if .service -}}
-   {{- $public := and (hasKey .service "public") .service.public -}}
+   {{- $public := and (hasKey .service "visibility") (not (eq .service.visibility "cluster")) -}}
    {{- if eq $public $externalCert }}
   - {{ .service.name }}.{{ $domain }}
    {{- end }}
@@ -1311,7 +1363,7 @@ spec:
 Add a certficate volume mount for a component
 */}}
 {{- define "hpcc.addCertificateVolumeMount" -}}
-{{- $externalCert := and (hasKey . "external") .external -}}
+{{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}}
 {{- $exposure := ternary "public" "local" $externalCert }}
 {{- /*
     A .certificate parameter means the user explictly configured a certificate to use
@@ -1333,7 +1385,7 @@ Add a certficate volume mount for a component
 Add a secret volume for a certificate
 */}}
 {{- define "hpcc.addCertificateVolume" -}}
-{{- $externalCert := and (hasKey . "external") .external -}}
+{{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}}
 {{- $exposure := ternary "public" "local" $externalCert -}}
 {{- /*
     A .certificate parameter means the user explictly configured a certificate to use

helm/hpcc/templates/dali.yaml

@@ -37,13 +37,22 @@ spec:
   selector:
     matchLabels:
       run: {{ $dali.name | quote }}
+      server: {{ $dali.name | quote }}
       app: dali
   template:
     metadata:
       labels:
         run: {{ $dali.name | quote }}
+        server: {{ $dali.name | quote }}
         app: dali
         helmVersion: 8.2.0-closedown
+{{- if hasKey $dali "labels" }}
+{{ toYaml $dali.labels | indent 8 }}
+{{- end }}
+{{- if hasKey $dali "annotations" }}
+      annotations:
+{{ toYaml $dali.annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $dali.name "type" "dali") | indent 6 }}
       serviceAccountName: "hpcc-default"
@@ -144,10 +153,12 @@ kind: ConfigMap
 {{ include "hpcc.generateConfig" (dict "root" $ "me" . "secretsCategories" $sashaSecretsCategories "configMapHelper" "hpcc.sashaConfigMap") }}
 {{- end }}
 ---
-{{- if $sasha.servicePort -}}
-{{ include "hpcc.addSashaService" (dict "root" $ "selectorName" $dali.name "me" $sasha) }}
+{{- if hasKey $sasha "service" -}}
+{{- if $sasha.service.servicePort -}}
+{{ include "hpcc.addService" ( dict "root" $ "name" (printf "sasha-%s" $sasha.name) "service" $sasha.service "selector" $dali.name "defaultPort" 8877 ) }}
 ---
 {{- end }}
+{{- end }}
 
 {{- if $sasha.storage }}
 {{- if and (not $sasha.storage.existingClaim) (not $sasha.storage.plane) }}
@@ -168,7 +179,7 @@ spec:
     protocol: TCP
     targetPort: 7070
   selector:
-    run: {{ .name | quote }}
+    server: {{ .name | quote }}
   type: ClusterIP
 ---
 {{ include "hpcc.addCertificate" (dict "root" $ "name" .name "service" . "component" "dali" "external" false) }}

helm/hpcc/templates/dfuserver.yaml

@@ -56,8 +56,14 @@ spec:
         run: {{ .name | quote }}
         accessDali: "yes"
         helmVersion: 8.2.0-closedown
+{{- if hasKey . "labels" }}
+{{ toYaml .labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey . "annotations" }}
+{{ toYaml .annotations | indent 8 }}
+{{- end }}
     spec:
       serviceAccountName: "hpcc-default"
       initContainers:

helm/hpcc/templates/eclagent.yaml

@@ -58,6 +58,13 @@ data:
             accessDali: "yes"
             accessEsp: "yes"
             helmVersion: 8.2.0-closedown
+{{- if hasKey .me "labels" }}
+{{ toYaml .me.labels | indent 12 }}
+{{- end }}
+{{- if hasKey .me "annotations" }}
+          annotations:
+{{ toYaml .me.annotations | indent 12 }}
+{{- end }}
         spec:
           {{- include "hpcc.placementsByJobTargetType" (dict "root" .root "job" $appJobName "target" .me.name "type" "eclagent") | indent 10 }}
           serviceAccountName: "hpcc-default"
@@ -126,8 +133,14 @@ spec:
         accessDali: "yes"
         accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }}
         helmVersion: 8.2.0-closedown
+{{- if hasKey . "labels" }}
+{{ toYaml .labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey . "annotations" }}
+{{ toYaml .annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" .name "target" .name "type" "eclagent") | indent 6 }}
       serviceAccountName: {{ .useChildProcesses | default false | ternary "hpcc-default" "hpcc-agent" }}

helm/hpcc/templates/eclccserver.yaml

@@ -57,6 +57,13 @@ data:
             accessDali: "yes"
             accessEsp: "yes"
             helmVersion: 8.2.0-closedown
+{{- if hasKey .me "labels" }}
+{{ toYaml .me.labels | indent 12 }}
+{{- end }}
+{{- if hasKey .me "annotations" }}
+          annotations:
+{{ toYaml .me.annotations | indent 12 }}
+{{- end }}
         spec:
           {{- include "hpcc.placementsByJobTargetType" (dict "root" .root "job" $compileJobName "target" .me.name "type" "eclccserver") | indent 10 }}
           serviceAccountName: "hpcc-default"
@@ -126,8 +133,14 @@ spec:
         accessDali: "yes"
         accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }}
         helmVersion: 8.2.0-closedown
+{{- if hasKey . "labels" }}
+{{ toYaml .labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey . "annotations" }}
+{{ toYaml .annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" .name "target" .name "type" "eclccserver") | indent 6 }}
       serviceAccountName: {{ .useChildProcesses | default false | ternary "hpcc-default" "hpcc-agent" }}

helm/hpcc/templates/eclscheduler.yaml

@@ -65,8 +65,14 @@ spec:
         accessDali: "yes"
         accessEsp: "no"
         helmVersion: 8.2.0-closedown
+{{- if hasKey . "labels" }}
+{{ toYaml .labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey . "annotations" }}
+{{ toYaml .annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" .name "target" .name "type" "eclscheduler") | indent 6 }}
       serviceAccountName: "hpcc-default"

helm/hpcc/templates/esp.yaml

@@ -41,7 +41,7 @@ data:
       tls: true
  {{- end }}
       tls_config:
- {{- if .me.public }}
+ {{- if (not (eq .me.service.visibility "cluster")) }}
         certificate: /opt/HPCCSystems/secrets/certificates/public/tls.crt
         privatekey: /opt/HPCCSystems/secrets/certificates/public/tls.key
  {{- else }}
@@ -89,11 +89,18 @@ spec:
     metadata:
       labels:
         run: {{ .name | quote }}
+        server: {{ .name | quote }}
         accessDali: "yes"
         app: {{ $application }}
         helmVersion: 8.2.0-closedown
+{{- if hasKey . "labels" }}
+{{ toYaml .labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey . "annotations" }}
+{{ toYaml .annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" .name "type" "esp") | indent 6 }}
       serviceAccountName: "hpcc-default"
@@ -119,30 +126,24 @@ spec:
 {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}
 {{ include "hpcc.addDllVolumeMount" $commonCtx | indent 8 }}
 {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }}
-{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name  "certificate" .certificate "external" (and (hasKey . "public") .public)) | indent 8 }}
+{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name  "certificate" .certificate "visibility" .service.visibility) | indent 8 }}
       volumes:
 {{ include "hpcc.addConfigMapVolume" . | indent 6 }}
 {{ include "hpcc.addVolumes" $commonCtx | indent 6 }}
 {{ include "hpcc.addDllVolume" $commonCtx | indent 6 }}
 {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }}
-{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "certificate" .certificate "external" (and (hasKey . "public") .public)) | indent 6 }}
+{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility) | indent 6 }}
 ---
 kind: ConfigMap
 {{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.espConfigMap")) }}
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ .name | quote }}
-spec:
-  ports:
-  - port: {{ required "servicePort must be specified" .servicePort }}
-    protocol: TCP
-    targetPort: {{ .port | default 8880 }}
-  selector:
-    run: {{ .name | quote }}
-  type: {{ .public | ternary "LoadBalancer" "ClusterIP" }}
+{{- if hasKey . "service" -}}
+{{- if .service.servicePort -}}
+{{ include "hpcc.addService" ( dict "root" $ "name" .name "service" .service "selector" .name "defaultPort" 8880 ) }}
 ---
-{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "service" . "component" $application "external" (and (hasKey . "public") .public)) }}
+{{- end }}
+{{- end }}
+
+{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "service" . "component" $application "visibility" .service.visibility) }}
 {{- end }}
 {{- end }}

helm/hpcc/templates/localroxie.yaml

@@ -60,16 +60,23 @@ spec:
   selector:
     matchLabels:
       run: {{ $roxie.name | quote }}
+      server: {{ $servername | quote }}
   template:
     metadata:
       labels:
         run: {{ $roxie.name | quote }}
-        roxie-server: {{ $servername | quote }}
+        server: {{ $servername | quote }}
         accessDali: "yes"
         accessEsp: "yes"
         helmVersion: 8.2.0-closedown
+{{- if hasKey . "labels" }}
+{{ toYaml .labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey . "annotations" }}
+{{ toYaml .annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $roxie.name "target" $roxie.name "type" "roxie") | indent 6 }}
       serviceAccountName: "hpcc-default"
@@ -92,13 +99,13 @@ spec:
           value: "/tmp/{{ $roxie.name }}.sentinel"
 {{- $local := dict "first" true }}
 {{- range $service := $roxie.services }}
-{{- if ne (int $service.port)  0 }}
+{{- if ne (int $service.servicePort)  0 }}
 {{- if $local.first }}
 {{- $_ := set $local "first" false }}
         ports:
 {{- end }}
         - name: {{ $service.name }}
-          containerPort: {{ $service.port }}
+          containerPort: {{ $service.servicePort }}
 {{- end }}
 {{- end }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
@@ -122,22 +129,8 @@ spec:
 {{ include "hpcc.addUDPCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localudpkey" ) | indent 6 }}
 ---
 {{- range $service := $roxie.services }}
-{{- if ne (int $service.port)  0 }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ $service.name | quote }}
-  labels:
-    type: roxie-service
-    helmVersion: 8.2.0-closedown
-spec:
-  ports:
-  - port: {{ $service.port }}
-    protocol: TCP
-    targetPort: {{ $service.port }}
-  selector:
-    roxie-server: {{ $servername | quote }}
-  type: {{ if $service.external -}} LoadBalancer {{- else -}} ClusterIP {{- end }}
+{{- if ne (int $service.servicePort)  0 }}
+{{ include "hpcc.addService" ( dict "root" $ "name" $service.name "service" $service "selector" $servername "defaultPort" $service.servicePort ) }}
 ---
 {{- end }}
 {{- end }}

helm/hpcc/templates/roxie.yaml

@@ -85,6 +85,11 @@ data:
 {{- $_ := set $roxie "localAgent" false -}}
 {{- $servername := printf "%s-server" $roxie.name -}}
 {{- $udpkeyname := $roxie.name -}}
+{{- range $service := $roxie.services }}
+ {{- if ne (int $service.servicePort)  0 }}
+  {{- $_ := set $service "port" $service.servicePort }} 
+ {{- end }}
+{{- end }}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -101,8 +106,14 @@ spec:
         run: {{ $commonCtx.toponame | quote }}
         roxie-cluster: {{ $roxie.name | quote }}
         helmVersion: 8.2.0-closedown
+{{- if hasKey $toposerver "labels" }}
+{{ toYaml $toposerver.labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $topoconfigSHA }}
+{{- if hasKey $toposerver "annotations" }}
+{{ toYaml $toposerver.annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $commonCtx.toponame "target" $roxie.name "type" "roxie") | indent 6 }}
       serviceAccountName: "hpcc-default"
@@ -136,22 +147,8 @@ spec:
 
 ---
 {{- range $service := $roxie.services }}
-{{- if ne (int $service.port)  0 }}
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ $service.name | quote }}
-  labels:
-    type: roxie-service
-    helmVersion: 8.2.0-closedown
-spec:
-  ports:
-  - port: {{ $service.port }}
-    protocol: TCP
-    targetPort: {{ $service.port }}
-  selector:
-    roxie-server: {{ $servername | quote }}
-  type: {{ if $service.external -}} LoadBalancer {{- else -}} ClusterIP {{- end }}
+{{- if ne (int $service.servicePort)  0 }}
+{{ include "hpcc.addService" ( dict "root" $ "name" $service.name "service" $service "selector" $servername "defaultPort" $service.servicePort ) }}
 ---
 {{- end }}
 {{- end }}
@@ -214,13 +211,19 @@ spec:
     metadata:
       labels:
         run: {{ $servername | quote }}
-        roxie-server: {{ $servername | quote }}
+        server: {{ $roxie.name | quote }}
         roxie-cluster: {{ $roxie.name | quote }}
         accessDali: "yes"
         accessEsp: "yes"
         helmVersion: 8.2.0-closedown
+{{- if hasKey $roxie "labels" }}
+{{ toYaml $roxie.labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey $roxie "annotations" }}
+{{ toYaml $roxie.annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $servername "target" $roxie.name "type" "roxie") | indent 6 }}
       serviceAccountName: "hpcc-default"
@@ -241,13 +244,13 @@ spec:
           value: "/tmp/{{ $roxie.name }}.sentinel"
 {{- $local := dict "first" true }}
 {{- range $service := $roxie.services }}
-{{- if ne (int $service.port)  0 }}
+{{- if ne (int $service.servicePort)  0 }}
 {{- if $local.first }}
 {{- $_ := set $local "first" false }}
         ports:
 {{- end }}
         - name: {{ $service.name }}
-          containerPort: {{ $service.port }}
+          containerPort: {{ $service.servicePort }}
 {{- end }}
 {{- end }}
 {{ include "hpcc.addSentinelProbes" $roxie | indent 8 }}
@@ -298,14 +301,20 @@ spec:
       labels:
         run: {{ $name | quote}}
 {{- if not $roxie.serverReplicas }}        
-        roxie-server: {{ $servername | quote }}
+        server: {{ $servername | quote }}
 {{- end }}
         roxie-cluster: {{ $roxie.name | quote }}
         accessDali: "yes"
         accessEsp: "yes"
         helmVersion: 8.2.0-closedown
+{{- if hasKey $roxie "labels" }}
+{{ toYaml $roxie.labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey $roxie "annotations" }}
+{{ toYaml $roxie.annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $name "target" $roxie.name "type" "roxie") | indent 6 }}
       serviceAccountName: "hpcc-default"
@@ -328,13 +337,13 @@ spec:
 {{- if not $roxie.serverReplicas }}
 {{- $local := dict "first" true }}
 {{- range $service := $roxie.services }}
-{{- if ne (int $service.port)  0 }}
+{{- if ne (int $service.servicePort)  0 }}
 {{- if $local.first }}
 {{- $_ := set $local "first" false }}
         ports:
 {{- end }}
         - name: {{ $service.name }}
-          containerPort: {{ $service.port }}
+          containerPort: {{ $service.servicePort }}
 {{- end }}
 {{- end }}
 {{ include "hpcc.addSentinelProbes" $roxie | indent 8 }}

helm/hpcc/templates/sasha.yaml

@@ -41,14 +41,22 @@ spec:
   selector:
     matchLabels:
       run: {{ $serviceName | quote }}
+      server: {{ $serviceName | quote }}
   template:
     metadata:
       labels:
         run: {{ $serviceName | quote }}
+        server: {{ $serviceName | quote }}
         accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }}
         helmVersion: 8.2.0-closedown
+{{- if hasKey $sasha "labels" }}
+{{ toYaml $sasha.labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey $sasha "annotations" }}
+{{ toYaml $sasha.annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $serviceName "type" "sasha") | indent 6 }}
       serviceAccountName: "hpcc-default"
@@ -65,10 +73,12 @@ spec:
 {{- include "hpcc.addSashaVolumes" $commonCtx | indent 6 }}
 {{- include "hpcc.addSecretVolumes"  $commonCtx | indent 6 }}
 ---
-{{- if and (not $sasha.disabled) ($sasha.servicePort) -}}
-{{ include "hpcc.addSashaService" $commonCtx }}
+{{- if and (not $sasha.disabled) (hasKey $sasha "service") -}}
+{{- if $sasha.service.servicePort -}}
+{{ include "hpcc.addService" ( dict "root" $ "name" $serviceName "service" $sasha.service "selector" $serviceName "defaultPort" 8877 ) }}
 ---
 {{- end }}
+{{- end }}
 kind: ConfigMap 
 {{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.sashaConfigMap")) }}
 ---

helm/hpcc/templates/thor.yaml

@@ -79,6 +79,13 @@ data:
             accessDali: "yes"
             accessEsp: "yes"
             helmVersion: 8.2.0-closedown
+{{- if hasKey .me "labels" }}
+{{ toYaml .me.labels | indent 12 }}
+{{- end }}
+{{- if hasKey .me "annotations" }}
+          annotations:
+{{ toYaml .me.annotations | indent 12 }}
+{{- end }}
         spec:
           {{- include "hpcc.placementsByJobTargetType" (dict "root" .root "job" $eclAgentJobName "target" .me.name "type" "thor") | indent 10 }}
           serviceAccountName: "hpcc-agent"
@@ -139,6 +146,13 @@ data:
             accessEsp: "yes"
             job: "_HPCC_JOBNAME_"
             helmVersion: 8.2.0-closedown
+{{- if hasKey $thorScope "labels" }}
+{{ toYaml $thorScope.labels | indent 12 }}
+{{- end }}
+{{- if hasKey $thorScope "annotations" }}
+          annotations:
+{{ toYaml $thorScope.annotations | indent 12 }}
+{{- end }}
         spec:
           {{- include "hpcc.placementsByJobTargetType" (dict "root" .root "job" $thorManagerJobName "target" .me.name "type" "thor") | indent 10 }}
           serviceAccountName: hpcc-agent
@@ -198,6 +212,13 @@ data:
             accessEsp: "true"
             job: "_HPCC_JOBNAME_"
             helmVersion: 8.2.0-closedown
+{{- if hasKey $thorScope "labels" }}
+{{ toYaml $thorScope.labels | indent 12 }}
+{{- end }}
+{{- if hasKey $thorScope "annotations" }}
+          annotations:
+{{ toYaml $thorScope.annotations | indent 12 }}
+{{- end }}
         spec:
           {{- include "hpcc.placementsByJobTargetType" (dict "root" .root "job" $thorWorkerJobName "target" .me.name "type" "thor") | indent 10 }}
           serviceAccountName: hpcc-default
@@ -293,8 +314,14 @@ spec:
         accessDali: "yes"
         accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }}
         helmVersion: 8.2.0-closedown
+{{- if hasKey $commonCtx.me "labels" }}
+{{ toYaml $commonCtx.me.labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey $commonCtx.me "annotations" }}
+{{ toYaml $commonCtx.me.annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $commonCtx.eclAgentName "target" .name "type" "thor") | indent 6 }}
       serviceAccountName: {{ $commonCtx.eclAgentUseChildProcesses | ternary "hpcc-default" "hpcc-agent" }}
@@ -347,8 +374,14 @@ spec:
         accessDali: "yes"
         accessEsp: "no"
         helmVersion: 8.2.0-closedown
+{{- if hasKey $commonCtx.me "labels" }}
+{{ toYaml $commonCtx.me.labels | indent 8 }}
+{{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey $commonCtx.me "annotations" }}
+{{ toYaml $commonCtx.me.annotations | indent 8 }}
+{{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $commonCtx.thorAgentName "target" .name "type" "thor") | indent 6 }}
       serviceAccountName: "hpcc-thoragent"

helm/hpcc/values.schema.json

@@ -135,6 +135,8 @@
     "dali": {
       "description": "dali process",
       "type": "array",
+      "minItems": 1,
+      "maxItems": 1,
       "items": { "$ref": "#/definitions/dali" }
     },
     "eclccserver": {
@@ -184,25 +186,33 @@
       "description": "dfuserver process",
       "type": "array",
       "items": {
-        "allOf": [
-          {
-            "name": {
-              "type": "string",
-              "description": "The name of the dfuserver process"
-            },
-            "disabled": {
-              "type": "boolean"
-            },
-            "maxJobs": {
-              "type": "integer"
-            },
-            "required": [ "name" ]
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "The name of the dfuserver process"
+          },
+          "disabled": {
+            "type": "boolean"
+          },
+          "maxJobs": {
+            "type": "integer"
+          },
+          "annotations": {
+            "type": "object",
+            "additionalProperties": { "type": "string" }
+          },
+          "labels": {
+            "type": "object",
+            "additionalProperties": { "type": "string" }
           }
-        ]
+        },
+        "additionalProperties": { "type": ["integer", "string", "boolean"] },
+        "required": [ "name" ]
       }
     },
     "roxie": {
-      "description": "eclagent process",
+      "description": "roxie process",
       "type": "array",
       "items": { "$ref": "#/definitions/roxie" }
     },
@@ -292,6 +302,29 @@
               "$ref" : "#/definitions/sinks"
             }
           }
+        },
+        "visibilities": {
+          "type": "object",
+          "description" : "Global visibilities configuration",
+          "additionalProperties": {
+            "type": "object",
+            "required": [ "type" ],
+            "properties":
+            {
+              "annotations": {
+                "type": "object",
+                "additionalProperties": { "type": "string" }
+              },
+              "labels": {
+                "type": "object",
+                "additionalProperties": { "type": "string" }
+              },
+              "type": {
+                "type": "string"
+              }
+            },
+            "additionalProperties": false
+          }
         }
       },
       "additionalProperties": false
@@ -637,6 +670,14 @@
         "logging": {
           "$ref": "#/definitions/logging"
         },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
         "resources": {
           "$ref": "#/definitions/resources"
         },
@@ -673,6 +714,14 @@
         "logging": {
           "$ref": "#/definitions/logging"
         },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
         "resources": {
           "$ref": "#/definitions/resources"
         }
@@ -693,6 +742,14 @@
         "logging": {
           "$ref": "#/definitions/logging"
         },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
         "resources": {
           "$ref": "#/definitions/resources"
         }
@@ -700,7 +757,7 @@
     },
     "esp": {
       "type": "object",
-      "required": [ "name" ],
+      "required": [ "name", "service" ],
       "additionalProperties": { "type": ["integer", "string", "boolean"] },
       "properties": {
         "name": {
@@ -718,6 +775,42 @@
         },
         "resources": {
           "$ref": "#/definitions/resources"
+        },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "service": {
+          "description": "Service properties",
+          "type": "object",
+          "required": [ "servicePort", "visibility" ],
+          "properties": {
+            "port": {
+              "type": "integer",
+              "description": "The local port used by the pod",
+              "default": 8880
+            },
+            "servicePort": {
+              "type": "integer",
+              "description": "The port that this service will be exposed on"
+            },
+            "visibility": {
+              "type": "string",
+              "description": "Should this service be exposed outside the cluster, locally or to the internet"
+            },
+            "annotations": {
+              "type": "object",
+              "additionalProperties": { "type": "string" }
+            },
+            "labels": {
+              "type": "object",
+              "additionalProperties": { "type": "string" }
+            }
+          }
         }
       }
     },
@@ -756,6 +849,14 @@
           "description": "The default storage plane to write data files to",
           "type": "string"
         },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
         "resources": {
           "$ref": "#/definitions/resources"
         }
@@ -811,11 +912,20 @@
         },
         "channelResources": {
           "$ref": "#/definitions/resources"
+        },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
         }
       }
     },
     "toposerver": {
       "type": "object",
+      "additionalProperties": { "type": ["integer", "string", "boolean"] },
       "properties": {
         "port": {
           "type": "integer"
@@ -828,6 +938,14 @@
         },
         "logging": {
           "$ref": "#/definitions/logging"
+        },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
         }
       }
     },
@@ -838,7 +956,12 @@
           "type": "string"
         },
         "port": {
-          "type": "integer"
+          "type": "integer",
+          "description": "The local port used by the pod (same as servicePort if not specified)"
+        },
+        "servicePort": {
+          "type": "integer",
+          "description": "The port that this service will be exposed on"
         },
         "numThreads": {
           "type": "integer"
@@ -846,11 +969,20 @@
         "listenQueue": {
           "type": "integer"
         },
-        "external": {
-          "type": "boolean"
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "visibility": {
+          "type": "string",
+          "description": "Should this service be exposed outside the cluster, locally or to the internet"
         }
       },
-      "required": [ "name", "port" ],
+      "required": [ "name", "servicePort" ],
       "additionalProperties": false
     },
     "bundle": {
@@ -937,6 +1069,14 @@
           "description": "The storage plane to write spill files to",
           "type": "string"
         },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
         "managerResources": {
           "$ref": "#/definitions/resources"
         },
@@ -962,11 +1102,37 @@
           "type": "integer",
           "default": 1
         },
-        "servicePort": {
-          "type": "integer"
+        "annotations": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
         },
-        "port": {
-          "type": "integer"
+        "labels": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        },
+        "service": {
+          "description": "Service properties",
+          "type": "object",
+          "required": [ "servicePort" ],
+          "properties": {
+            "port": {
+              "type": "integer",
+              "description": "The local port used by the pod",
+              "default": 8880
+            },
+            "servicePort": {
+              "type": "integer",
+              "description": "The port that this service will be exposed on"
+            },
+            "annotations": {
+              "type": "object",
+              "additionalProperties": { "type": "string" }
+            },
+            "labels": {
+              "type": "object",
+              "additionalProperties": { "type": "string" }
+            }
+          }
         },
         "at" : {
           "type": "string",
@@ -1012,17 +1178,18 @@
               "properties": {
                 "minDeltaSize": {
                   "type": "integer",
-                  "description": "Coalescing will only begin, if the delta size is above this threashold (K)"
+                  "description": "Coalescing will only begin, if the delta size is above this threshold (K)"
                 },
                 "disabled": {},
                 "interval": {},
-                "servicePort": {},
-                "port": {},
+                "service": {},
                 "at" : {},
                 "throttle": {},
                 "image": {},
                 "storage": {},
-                "resources": {}    
+                "resources": {},
+                "annotations": {},
+                "labels": {}
               },
               "additionalProperties": false
             }
@@ -1065,13 +1232,14 @@
                 },
                 "disabled": {},
                 "interval": {},
-                "servicePort": {},
-                "port": {},
+                "service": {},
                 "at" : {},
                 "throttle": {},
                 "image": {},
                 "storage": {},
                 "resources": {},
+                "annotations": {},
+                "labels": {},
                 "limit": {},
                 "cutoff": {}    
               },
@@ -1096,13 +1264,14 @@
               {
                 "disabled": {},
                 "interval": {},
-                "servicePort": {},
-                "port": {},
+                "service": {},
                 "at" : {},
                 "throttle": {},
                 "image": {},
                 "storage": {},
                 "resources": {},
+                "annotations": {},
+                "labels": {},
                 "limit": {},
                 "cutoff": {}
               },
@@ -1126,13 +1295,14 @@
               "properties": {
                 "disabled": {},
                 "interval": {},
-                "servicePort": {},
-                "port": {},
+                "service": {},
                 "at" : {},
                 "throttle": {},
                 "image": {},
                 "storage": {},
                 "resources": {},
+                "annotations": {},
+                "labels": {},
                 "limit": {},
                 "cutoff": {}    
               },
@@ -1167,13 +1337,14 @@
             },
             "disabled": {},
             "interval": {},
-            "servicePort": {},
-            "port": {},
+            "service": {},
             "at" : {},
             "throttle": {},
             "image": {},
             "storage": {},
-            "resources": {}
+            "resources": {},
+            "annotations": {},
+            "labels": {}
           },
           "additionalProperties": false
         },

helm/hpcc/values.yaml

@@ -63,6 +63,23 @@ global:
   #   postJobCommand: "kill $(pgrep linkerd2-proxy)"
   #   postJobCommandViaSidecar: true
 
+  ## visibilities section can be used to set labels, annotations and service type for any service with the specified visibility
+  visibilities:
+    cluster:
+      type: ClusterIP
+    local:
+      annotations:
+        # This annotation will make azure load balancer use an internal rather than an internet-visible address
+        # May want different values on different cloud providers or use-cases. For example on AWS you may want to use
+        #service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+        service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+      type: LoadBalancer
+    global:
+      #labels:
+      #  mylabel: "4"
+      type: LoadBalancer
+      
+      
 
   # For pod placement instruction and examples please reference docs/placements.md
 
@@ -262,7 +279,8 @@ dali:
 - name: mydali
   services: # internal house keeping services
     coalescer:
-      #servicePort: 8877
+      service:
+        servicePort: 8877
       #interval: 2 # (hours)
       #at: "* * * * *" # cron type schedule, i.e. Min(0-59) Hour(0-23) DayOfMonth(1-31) Month(1-12) DayOfWeek(0-6)
       #minDeltaSize: 50 # (Kb) will not start coalescing until delta log is above this threshold
@@ -280,7 +298,8 @@ sasha:
   #disabled: true # disable all services. Alternatively set sasha to null (sasha: null)
   wu-archiver:
     #disabled: true
-    servicePort: 8877
+    service:
+      servicePort: 8877
     storage:
       #existingClaim: pvc
       storageSize: 1Gi
@@ -298,7 +317,8 @@ sasha:
 
   dfuwu-archiver:
     #disabled: true
-    servicePort: 8877
+    service:
+      servicePort: 8877
     storage:
       #existingClaim: pvc
       storageSize: 1Gi
@@ -377,12 +397,19 @@ esp:
   application: eclwatch
   auth: none
   replicas: 1
-  ## port can be used to change the local port used by the pod. If omitted, the default port (8880) is used
-  port: 8888
-  ## servicePort controls the port that this service will be exposed on, either internally to the cluster, or externally
-  servicePort: 8010
-  ## Specify public: true if you want the service available from outside the cluster. Typically, eclwatch and wsecl are published externally, while eclservices is designed for internal use.
-  public: true
+  service:
+    ## port can be used to change the local port used by the pod. If omitted, the default port (8880) is used
+    port: 8888
+    ## servicePort controls the port that this service will be exposed on, either internally to the cluster, or externally
+    servicePort: 8010
+    ## Specify visibility: local (or global) if you want the service available from outside the cluster. Typically, eclwatch and wsecl are published externally, while eclservices is designed for internal use.
+    visibility: local
+    ## Annotations can be specified on a service - for example to specify provider-specific information such as service.beta.kubernetes.io/azure-load-balancer-internal-subnet
+    #annotations:
+    #  service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "mysubnet"
+    ## You can also specify labels on a service
+    #labels:
+    #  mylabel: "3"
   #resources:
   #  cpu: "1"
   #  memory: "2G"
@@ -390,8 +417,9 @@ esp:
   application: eclservices
   auth: none
   replicas: 1
-  servicePort: 8010
-  public: false
+  service:
+    servicePort: 8010
+    visibility: cluster
   #resources:
   #  cpu: "250m"
   #  memory: "1G"
@@ -399,8 +427,9 @@ esp:
   application: eclqueries
   auth: none
   replicas: 1
-  public: true
-  servicePort: 8002
+  service:
+    visibility: local
+    servicePort: 8002
   #resources:
   #  cpu: "250m"
   #  memory: "1G"
@@ -408,8 +437,9 @@ esp:
   application: esdl-sandbox
   auth: none
   replicas: 1
-  public: true
-  servicePort: 8899
+  service:
+    visibility: local
+    servicePort: 8899
   #resources:
   #  cpu: "250m"
   #  memory: "1G"
@@ -417,8 +447,9 @@ esp:
   application: sql2ecl
   auth: none
   replicas: 1
-  public: true
-  servicePort: 8510
+  service:
+    visibility: local
+    servicePort: 8510
   #domain: hpccsql.com
   #resources:
   #  cpu: "250m"
@@ -430,10 +461,10 @@ roxie:
   prefix: roxie
   services:
   - name: roxie
-    port: 9876
+    servicePort: 9876
     listenQueue: 200
     numThreads: 30
-    external: true
+    visibility: local
   ## replicas indicates the number of replicas per channel
   replicas: 2  
   numChannels: 2

roxie/ccd/ccdmain.cpp

@@ -739,7 +739,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
                     // Makes debugging easier...
                     IPropertyTree *service = topology->addPropTree("services");
                     service->setProp("@name", "query");
-                    service->setPropInt("@port", 9876);
+                    service->setPropInt("@port", ROXIE_SERVER_PORT);
                 }
 #else
                 if (!topology->getCount("RoxieFarmProcess"))
@@ -1202,7 +1202,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
             ForEach(*roxieFarms)
             {
                 IPropertyTree &roxieFarm = roxieFarms->query();
-                unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
+                unsigned port = roxieFarm.getPropInt("@port", roxieFarm.getPropInt("@servicePort", ROXIE_SERVER_PORT));
                 RoxieEndpointInfo me = {RoxieEndpointInfo::RoxieServer, 0, { (unsigned short) port, myIP }, 0};
                 myRoles.push_back(me);
             }
@@ -1326,7 +1326,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
                     unsigned numThreads = roxieFarm.getPropInt("@numThreads", 0);
                     if (!numThreads)
                         numThreads = numServerThreads;
-                    unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
+                    unsigned port = roxieFarm.getPropInt("@port", roxieFarm.getPropInt("@servicePort", ROXIE_SERVER_PORT));
                     //unsigned requestArrayThreads = roxieFarm.getPropInt("@requestArrayThreads", 5);
                     // NOTE: farmer name [@name=] is not copied into topology
                     const IpAddress ip = myNode.getIpAddress();

testing/helm/tests/labels.yaml

@@ -0,0 +1,92 @@
+esp:
+- name: eclwatch
+  application: eclwatch
+  service:
+    servicePort: 8880
+    visibility: cluster
+  annotations:
+    ea1: eav1
+  labels:
+    el1: elv1
+    
+roxie:
+- name: roxie
+  services:
+  - name: roxie
+    servicePort: 9876
+  annotations:
+    ra1: rav1
+  labels:
+    rl1: rlv1
+  topoServer:
+    replicas: 1
+    annotations:
+      rta1: rtav1
+    labels:
+      rtl1: rtlv1
+- name: roxiel
+  services:
+  - name: roxiel
+    servicePort: 9876
+  annotations:
+    ra1: rav1
+  labels:
+    rl1: rlv1
+  localAgent: true
+
+
+dfuserver:
+- name: dfuserver
+  annotations:
+    da1: dav1
+  labels:
+    dl1: dlv1
+    
+eclagent:
+- name: eclagent
+  annotations:
+    eaa1: eaav1
+  labels:
+    eal1: ealv1
+
+eclscheduler: 
+- name: eclscheduler
+  annotations:
+    esa1: esav1
+  labels:
+    esl1: esalv1
+
+thor:
+- name: thor
+  maxJobs: 3
+  maxGraphs: 4
+  annotations:
+    ta1: tv1
+  labels:
+    tl1: tv1
+
+eclccserver: 
+- name: eclccserver
+  annotations:
+    esa1: esav1
+  labels:
+    esl1: esalv1
+
+sasha:
+  wu-archiver:
+    annotations:
+      sa1: sav1
+    labels:
+      sl1: salv1
+    service:
+      servicePort: 8877
+
+dali:
+- name: mydali
+  annotations:
+    da1: dav1
+  labels:
+    dl1: dlv1
+  services: # internal house keeping services
+    coalescer:
+