Merge pull request #15250 from rpastrana/HPCC-26381-metricserviceannotation

HPCC-26381 Enable Prometheus metric service autodiscovery

Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

helm/examples/metrics/README.md

@@ -20,3 +20,36 @@ For example, the following adds the Prometheus configuration as the collection s
 ```code
 helm install mycluster ./hpcc -f <path>/prometheus_metrics.yml
 ```
+### Prometheus Collection System
+When enabled and properly configured, the HPCC metrics for Prometheus feature exposes an HTTP service on each pod of any HPCC component
+which reports metrics. It also annotates all HPCC pods to be discoverable by Prometheus servers and provides the connectivity inforation required, such as
+the port to liston on, and the url path on which to serve Prometheus formatted metrics. Autodiscovery annotations can be disabled via metrics.sinks[type=prometheus].settings.autodiscovery
+
+####Prometheus Metric Service Discovery
+On the Prometheus server side, an "additionalScrapeConfigs" entry is necessary to link the HPCC metrics services to Prometheus
+
+For example, the following Prometheus scrape job can be applied to Prometheus deployments as part of the 'additionalScrapeConfigs' configuration:
+
+```code
+    - job_name: 'hpcc-per-pod-metrics'
+      scrape_interval: 5s
+      metrics_path: /metrics
+      kubernetes_sd_configs:
+      - role: pod
+      relabel_configs:
+      - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_scrape ]
+        action: keep
+        regex: true
+      - source_labels: [ __meta_kubernetes_pod_annotation_prometheus_io_path ]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ]
+        action: replace
+        regex: ([^:]+)(:\d+)?;(\d+)
+        replacement: ${1}:${3}
+        target_label: __address__
+```
+
+ More information on the "additionalScrapeConfigs" options provided by Prometheus here: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config
+ 3rd Party write up on the relabel_configs used in the additionalscrapeconfigs here: https://gist.github.com/reachlin/a98b90afcbff4604c90c183a0169474f

helm/examples/metrics/prometheus_metrics.yaml

@@ -12,6 +12,8 @@ global:
     - type: prometheus
       name: prometheussink
       settings:
+        autodiscovery: true #disable if Prometheus cannot access hpcc pods
         port: 8767
         serviceName: metrics
         verbose: true
+        path: "/metrics"

helm/hpcc/templates/_helpers.tpl

@@ -429,6 +429,28 @@ Pass in dict with root and secretsCategories
 {{- end -}}
 
 {{/*
+Generate Prometheus scrape annotations
+Enables selfdiscovery of metrics service on configured path/port
+Requires sinks[type=prometheus]
+Pass in dict with sinks
+*/}}
+{{- define "hpcc.addPrometheusScrapeAnnotations" -}}
+{{- if hasKey . "sinks" }}
+ {{ range $sink := .sinks -}}
+  {{- if eq (get $sink "type") "prometheus" }}
+   {{- if and (hasKey $sink "settings") ( hasKey $sink.settings "autodiscovery") }}
+    {{- if (eq $sink.settings.autodiscovery true ) }}
+prometheus.io/scrape: 'true'
+prometheus.io/path: {{ $sink.settings.path | default "/metrics" }}
+prometheus.io/port: {{ $sink.settings.port | default 8767 | quote }}
+    {{ end }}
+   {{ end }}
+  {{ end }}
+ {{ end}}
+{{ end}}
+{{- end -}}
+
+{{/*
 Add Secret volume for a component
 Pass in dict with root and secretsCategories
 */}}
@@ -1023,7 +1045,22 @@ spec:
   type: {{ $lvars.type }}
 {{- end -}}
 
-
+{{/*
+Generate prometheusMetricsReporter label if metrics.sinks[type=prometheus].
+Ranges over metric sinks map
+Pass in dict with sinks
+*/}}
+{{- define "hpcc.generateMetricsReporterLabel" }}
+ {{ range $sink := .sinks -}}
+  {{- if eq (get $sink "type") "prometheus" }}
+   {{- if and (hasKey $sink "settings") ( hasKey $sink.settings "autodiscovery") }}
+    {{- if (eq $sink.settings.autodiscovery true ) }}
+prometheusMetricsReporter: "yes"
+    {{ end }}
+   {{ end }}
+  {{ end }}
+ {{ end }}
+{{- end -}}
 {{/*
 Return access permssions for a given service
 */}}

helm/hpcc/templates/dali.yaml

@@ -46,12 +46,20 @@ spec:
         server: {{ $dali.name | quote }}
         app: dali
         helmVersion: 8.3.0-trunk0
+{{- if hasKey $.Values.global "metrics" }}
+ {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }}
+{{- end }}
 {{- if hasKey $dali "labels" }}
 {{ toYaml $dali.labels | indent 8 }}
 {{- end }}
-{{- if hasKey $dali "annotations" }}
+{{- if or (hasKey $dali "annotations") (hasKey $.Values.global "metrics") }}
       annotations:
-{{ toYaml $dali.annotations | indent 8 }}
+ {{- if hasKey $.Values.global "metrics" }}
+  {{- include "hpcc.addPrometheusScrapeAnnotations" $.Values.global.metrics | nindent 8 }}
+ {{- end }}
+ {{- if hasKey $dali "annotations" }}
+  {{- toYaml $dali.annotations | nindent 8 }}
+ {{- end }}
 {{- end }}
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $dali.name "type" "dali") | indent 6 }}

helm/hpcc/templates/esp.yaml

@@ -102,11 +102,17 @@ spec:
         accessDali: "yes"
         app: {{ $application }}
         helmVersion: 8.3.0-trunk0
+{{- if hasKey $.Values.global "metrics" }}
+ {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }}
+{{- end }}
 {{- if hasKey . "labels" }}
 {{ toYaml .labels | indent 8 }}
 {{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey  $.Values.global "metrics" }}
+ {{- include "hpcc.addPrometheusScrapeAnnotations" $.Values.global.metrics | nindent 8 }}
+{{- end }}
 {{- if hasKey . "annotations" }}
 {{ toYaml .annotations | indent 8 }}
 {{- end }}

helm/hpcc/templates/network.yaml

@@ -23,6 +23,7 @@
 */}}
 {{- $egress := (.Values.global.egress | default dict) -}}
 {{- $egressRestricted := $egress.restricted | default false -}}
+{{- $metrics := .Values.global.metrics -}}
 ---
 ##
 ## Default to no ingress, and egress only to dns servers
@@ -155,3 +156,29 @@ spec:
     - podSelector:
         matchLabels:
           app: eclservices
+---
+##
+## Provides access to pods labeled prometheusMetricsReporter: "yes"
+## onto port declared in global.metrics.sinks[type=prometheus].configuration.port
+## Only created if global.metrics.sinks[type=prometheus].settings.autodiscovery true
+{{- if hasKey $metrics "sinks" }}
+ {{ range $sink := $metrics.sinks -}}
+  {{- if eq (get $sink "type") "prometheus" }}
+   {{- if and (hasKey $sink "settings") (hasKey $sink.settings "autodiscovery") }}
+    {{- if (eq $sink.settings.autodiscovery true ) }}
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hpcc-metrics-prometheus
+spec:
+  podSelector:
+    matchLabels:
+      prometheusMetricsReporter: "yes"
+  ingress:
+  - ports:
+    - port: {{ $sink.settings.port | default 8767 }}
+     {{ end }}
+    {{ end }}
+   {{ end }}
+  {{ end }}
+{{ end}}

helm/hpcc/templates/roxie.yaml

@@ -105,11 +105,17 @@ spec:
         run: {{ $commonCtx.toponame | quote }}
         roxie-cluster: {{ $roxie.name | quote }}
         helmVersion: 8.3.0-trunk0
+{{- if hasKey $.Values.global "metrics" }}
+ {{- include "hpcc.generateMetricsReporterLabel"  $.Values.global.metrics | nindent 8}}
+{{- end }}
 {{- if hasKey $toposerver "labels" }}
 {{ toYaml $toposerver.labels | indent 8 }}
 {{- end }}
       annotations:
         checksum/config: {{ $topoconfigSHA }}
+{{- if hasKey $.Values.global "metrics" }}
+ {{- include "hpcc.addPrometheusScrapeAnnotations"  $.Values.global.metrics | nindent 8 }}
+{{- end }}
 {{- if hasKey $toposerver "annotations" }}
 {{ toYaml $toposerver.annotations | indent 8 }}
 {{- end }}
@@ -215,11 +221,17 @@ spec:
         accessDali: "yes"
         accessEsp: "yes"
         helmVersion: 8.3.0-trunk0
+{{- if hasKey $.Values.global "metrics" }}
+ {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}}
+{{- end }}
 {{- if hasKey $roxie "labels" }}
 {{ toYaml $roxie.labels | indent 8 }}
 {{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey $.Values.global "metrics" }}
+ {{- include "hpcc.addPrometheusScrapeAnnotations" $.Values.global.metrics | nindent 8 }}
+{{- end }}
 {{- if hasKey $roxie "annotations" }}
 {{ toYaml $roxie.annotations | indent 8 }}
 {{- end }}
@@ -317,11 +329,17 @@ spec:
         accessDali: "yes"
         accessEsp: "yes"
         helmVersion: 8.3.0-trunk0
+{{- if hasKey $.Values.global "metrics" }}
+ {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}}
+{{- end }}
 {{- if hasKey $roxie "labels" }}
 {{ toYaml $roxie.labels | indent 8 }}
 {{- end }}
       annotations:
         checksum/config: {{ $configSHA }}
+{{- if hasKey $.Values.global "metrics" }}
+ {{- include "hpcc.addPrometheusScrapeAnnotations" $.Values.global.metrics | nindent 8 }}
+{{- end }}
 {{- if hasKey $roxie "annotations" }}
 {{ toYaml $roxie.annotations | indent 8 }}
 {{- end }}