Merge pull request #725 from abhishek-sa1/devel

Issue #724: Grafana Loki Installation and Configuration

control_plane/roles/control_plane_monitoring/files/Dockerfile_promtail

@@ -0,0 +1,3 @@
+FROM docker.io/grafana/promtail:2.4.1
+
+COPY promtail_config.yml /etc/promtail/config.yml

control_plane/roles/control_plane_monitoring/files/loki_dashboard.json

@@ -0,0 +1,135 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "control-plane-loki",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "control-plane-loki"
+          },
+          "expr": "{filename=\"/var/log/omnia.log\",job=\"Omnia\"}",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "title": "omnia.log",
+      "type": "logs"
+    },
+    {
+      "datasource": "control-plane-loki",
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "id": 2,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "control-plane-loki"
+          },
+          "expr": "{filename=\"/var/log/messages\",job=\"syslog\"}",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "title": "syslog",
+      "type": "logs"
+    }
+  ],
+  "schemaVersion": 30,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "loki",
+  "uid": "1k9NY90nz",
+  "version": 1
+}

control_plane/roles/control_plane_monitoring/files/promtail_config.yml

@@ -0,0 +1,138 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: varlogs
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: varlogs
+      __path__: /var/log/*log
+
+- job_name: Directory Server Utilization
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: Directory Server Utilization
+      __path__: /var/log/dirsrv/slapd-OMNIA-TEST/access
+
+- job_name: Directory Server Errors
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: Directory Server Errors
+      __path__: /var/log/dirsrv/slapd-OMNIA-TEST/errors
+
+- job_name: PKI Transactions
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: PKI Transactions
+      __path__: /var/log/pki/pki-tomcat/ca/transactions
+
+- job_name: KDC Utilization
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: KDC Utilization
+      __path__: /var/log/krb5kdc.log
+
+- job_name: FreeIPA API Call
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: FreeIPA API Call
+      __path__: /var/log/httpd/*
+
+- job_name: audit
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: audit
+      __path__: /var/log/audit/*log
+
+- job_name: SSSD
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: SSSD
+      __path__: /var/log/sssd/*log
+
+- job_name: secure
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: secure
+      __path__: /var/log/secure
+
+- job_name: Omnia
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: Omnia
+      __path__: /var/log/omnia.log
+
+- job_name: syslog
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: syslog
+      __path__: /var/log/messages
+
+- job_name: cron
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: cron
+      __path__: /var/log/cron
+
+- job_name: IPA Server and Client
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: IPA Server and Client
+      __path__: /var/log/ipa*log
+
+- job_name: Package Rocky
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: Package Rocky
+      __path__: /var/log/dnf.log
+
+- job_name: Package Leap
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: Package Leap
+      __path__: /var/log/zypper.log
+
+- job_name: k8s pods
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: k8s pods
+      __path__: /var/log/pods/*/*/*log

control_plane/roles/control_plane_monitoring/tasks/configure_k8s_grafana.yml

@@ -1,73 +1,59 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
----
-
-- name: Get grafana service IP
-  command: kubectl get svc -n {{ grafana_namespace }} -o=jsonpath='{.items[0].spec.clusterIP}'
-  changed_when: false
-  register: grafana_svc_ip
-
-- name: Get grafana service port
-  command: kubectl get svc -n {{ grafana_namespace }} -o=jsonpath='{.items[0].spec.ports[0].port}'
-  changed_when: false
-  register: grafana_svc_port
-
-- name: Get kube-prometheus svc IP
-  command: kubectl get svc -n {{ monitoring_namespace }} -l app=kube-prometheus-stack-prometheus -o=jsonpath='{.items[0].spec.clusterIP}'
-  changed_when: false
-  register: kube_prom_svc_ip
-
-- name: Get kube-prometheus svc port
-  command: kubectl get svc -n {{ monitoring_namespace }} -l app=kube-prometheus-stack-prometheus -o=jsonpath='{.items[0].spec.ports[0].port}'
-  changed_when: false
-  register: kube_prom_svc_port
-
-- name: Install community.grafana collection
-  command: ansible-galaxy collection install community.grafana
-  changed_when: True
-
-- name: Create prometheus datasource
-  community.grafana.grafana_datasource:
-    name: control-plane-prometheus
-    grafana_url: "http://{{ grafana_svc_ip.stdout }}:{{ grafana_svc_port.stdout }}"
-    grafana_user: "{{ grafana_username }}"
-    grafana_password: "{{ grafana_password }}"
-    ds_type: prometheus
-    ds_url: "http://{{ kube_prom_svc_ip.stdout }}:{{ kube_prom_svc_port.stdout }}"
-    access: proxy
-  no_log: true
-
-- name: Import K8s Grafana dashboards
-  community.grafana.grafana_dashboard:
-    grafana_url: "http://{{ grafana_svc_ip.stdout }}:{{ grafana_svc_port.stdout }}"
-    grafana_user: "{{ grafana_username }}"
-    grafana_password: "{{ grafana_password }}"
-    state: present
-    commit_message: Updated by ansible
-    overwrite: yes
-    path: "{{ role_path }}/files/{{ item }}"
-  with_items: "{{ grafana_dashboard_json_files }}"
-  no_log: true
-
-- name: Save grafana svc ip
-  replace:
-    path: "{{ role_path }}/../../../roles/k8s_start_services/files/grafana_svc_details.ini"
-    regexp: "ip=.*"
-    replace: "ip={{ grafana_svc_ip.stdout }}"
-
-- name: Save grafana svc ip
-  replace:
-    path: "{{ role_path }}/../../../roles/k8s_start_services/files/grafana_svc_details.ini"
-    regexp: "port=.*"
-    replace: "port={{ grafana_svc_port.stdout }}"
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Get kube-prometheus svc IP
+  command: kubectl get svc -n {{ monitoring_namespace }} -l app=kube-prometheus-stack-prometheus -o=jsonpath='{.items[0].spec.clusterIP}'
+  changed_when: false
+  register: kube_prom_svc_ip
+
+- name: Get kube-prometheus svc port
+  command: kubectl get svc -n {{ monitoring_namespace }} -l app=kube-prometheus-stack-prometheus -o=jsonpath='{.items[0].spec.ports[0].port}'
+  changed_when: false
+  register: kube_prom_svc_port
+
+- name: Create prometheus datasource in grafana
+  community.grafana.grafana_datasource:
+    name: control-plane-prometheus
+    grafana_url: "http://{{ grafana_svc_ip.stdout }}:{{ grafana_svc_port.stdout }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    ds_type: prometheus
+    ds_url: "http://{{ kube_prom_svc_ip.stdout }}:{{ kube_prom_svc_port.stdout }}"
+    access: proxy
+  no_log: true
+
+- name: Import K8s grafana dashboards
+  community.grafana.grafana_dashboard:
+    grafana_url: "http://{{ grafana_svc_ip.stdout }}:{{ grafana_svc_port.stdout }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    state: present
+    commit_message: Updated by ansible
+    overwrite: yes
+    path: "{{ role_path }}/files/{{ item }}"
+  with_items: "{{ grafana_dashboard_json_files }}"
+  no_log: true
+
+- name: Save grafana svc ip
+  replace:
+    path: "{{ role_path }}/../../../roles/k8s_start_services/files/grafana_svc_details.ini"
+    regexp: "ip=.*"
+    replace: "ip={{ grafana_svc_ip.stdout }}"
+
+- name: Save grafana port
+  replace:
+    path: "{{ role_path }}/../../../roles/k8s_start_services/files/grafana_svc_details.ini"
+    regexp: "port=.*"
+    replace: "port={{ grafana_svc_port.stdout }}"

control_plane/roles/control_plane_monitoring/tasks/configure_loki_grafana.yml

@@ -0,0 +1,49 @@
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Wait for loki pod to come to ready state
+  command: kubectl wait --for=condition=ready --timeout=10m -n "{{ grafana_namespace }}" pod -l app="{{ loki_k8s_name }}"
+  changed_when: false
+
+- name: Get loki service ip
+  command: kubectl get svc loki -n {{ grafana_namespace }} -o=jsonpath='{.spec.clusterIP}'
+  changed_when: false
+  register: loki_svc_ip
+
+- name: Get loki service port
+  command: kubectl get svc loki -n {{ grafana_namespace }} -o=jsonpath='{.spec.ports[0].port}'
+  changed_when: false
+  register: loki_svc_port
+
+- name: Create loki datasource in grafana
+  community.grafana.grafana_datasource:
+    name: control-plane-loki
+    grafana_url: "http://{{ grafana_svc_ip.stdout }}:{{ grafana_svc_port.stdout }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    ds_type: loki
+    ds_url: "http://{{ loki_svc_ip.stdout }}:{{ loki_svc_port.stdout }}"
+  no_log: true
+
+- name: Import loki dashboard in grafana
+  community.grafana.grafana_dashboard:
+    grafana_url: "http://{{ grafana_svc_ip.stdout }}:{{ grafana_svc_port.stdout }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    state: present
+    commit_message: Updated by ansible
+    overwrite: yes
+    path: "{{ role_path }}/files/loki_dashboard.json"
+  no_log: true

control_plane/roles/control_plane_monitoring/tasks/install_k8s_prom_stack.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,21 +13,17 @@
 #  limitations under the License.
 ---
 
-- name: Install kubernetes.core collection
-  command: ansible-galaxy collection install kubernetes.core
-  changed_when: True
-
 - name: Add prometheus-community chart
   kubernetes.core.helm_repository:
     name: prometheus-community
-    repo_url: "https://kubernetes.github.io/ingress-nginx"
+    repo_url: "{{ helm_repo_prometheus }}"
   failed_when: False
 
 - name: Install kube-prom-stack
   block:
     - name: Install kube-prom-stack
       kubernetes.core.helm:
-        name: monitoring
+        name: "{{ monitoring_app_name }}"
         chart_ref: prometheus-community/kube-prometheus-stack
         chart_version: "{{ k8s_prom_stack_chart_version }}"
         update_repo_cache: True
@@ -38,14 +34,20 @@
           grafana:
             enabled: False
   rescue:
+    - name: Delete helm kube-prom-stack deployments
+      command: helm delete {{ monitoring_app_name }} -n {{ monitoring_namespace }}
+      changed_when: True
+      failed_when: False
+
     - name: Delete existing kube-prom-stack crd
       command: "kubectl delete crd {{ item }}"
       changed_when: True
+      failed_when: False
       with_items: "{{ k8s_prom_stack_crd }}"
 
     - name: Install kube-prom-stack
       kubernetes.core.helm:
-        name: monitoring
+        name: "{{ monitoring_app_name }}"
         chart_ref: prometheus-community/kube-prometheus-stack
         chart_version: "{{ k8s_prom_stack_chart_version }}"
         update_repo_cache: True
@@ -54,4 +56,8 @@
         wait: true
         values:
           grafana:
-            enabled: False
+            enabled: False
+
+- name: Wait for kube-prom-stack pod to come to ready state
+  command: kubectl wait --for=condition=ready --timeout=10m -n "{{ monitoring_namespace }}" pod -l app=kube-prometheus-stack-operator
+  changed_when: false

control_plane/roles/control_plane_monitoring/tasks/k8s_loki_pod.yml

@@ -0,0 +1,63 @@
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Deploy loki pod
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+          name: "{{ loki_k8s_name }}"
+          namespace: "{{ grafana_namespace }}"
+          labels:
+              app: "{{ loki_k8s_name }}"
+      spec:
+          selector:
+              matchLabels:
+                  app: "{{ loki_k8s_name }}"
+          replicas: 1
+          strategy:
+              type: RollingUpdate
+          template:
+              metadata:
+                  labels:
+                      app: "{{ loki_k8s_name }}"
+              spec:
+                  volumes:
+                    - name: varlog
+                      hostPath:
+                        path: /var/log
+                        type: Directory
+
+                  hostAliases:
+                    - ip: "127.0.0.1"
+                      hostnames:
+                        - "loki"
+                        - "promtail"
+
+                  containers:
+                    - name: loki
+                      image: docker.io/grafana/loki:2.4.1
+                      imagePullPolicy: "Always"
+                      ports:
+                        - containerPort: "{{ loki_port }}"
+
+                    - name: promtail
+                      image: localhost/promtail:latest
+                      imagePullPolicy: "Never"
+                      volumeMounts:
+                        - name: varlog
+                          mountPath: /var/log

control_plane/roles/control_plane_monitoring/tasks/loki_service.yml

@@ -0,0 +1,33 @@
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Service for loki
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: "{{ loki_k8s_name }}"
+        namespace: "{{ grafana_namespace }}"
+        labels:
+          app: "{{ loki_k8s_name }}"
+      spec:
+        type: ClusterIP
+        ports:
+          - name: loki
+            port: "{{ loki_port }}"
+        selector:
+          app: "{{ loki_k8s_name }}"

control_plane/roles/control_plane_monitoring/tasks/main.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,11 +13,20 @@
 #  limitations under the License.
 ---
 
-#- name: Install Grafana
-#  import_tasks: install_grafana.yml
-
 - name: Install K8s prometheus stack
   import_tasks: install_k8s_prom_stack.yml
 
-- name: Configure K8s grafana
-  import_tasks: configure_k8s_grafana.yml
+- name: Configure K8s prometheus in grafana
+  import_tasks: configure_k8s_prom_grafana.yml
+
+- name: Build promtail image
+  import_tasks: promtail_image.yml
+
+- name: Deploy loki pod
+  import_tasks: k8s_loki_pod.yml
+
+- name: Deploy loki service
+  import_tasks: loki_service.yml
+
+- name: Configure loki in grafana
+  import_tasks: configure_loki_grafana.yml

control_plane/roles/control_plane_monitoring/tasks/install_grafana.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,3 +13,15 @@
 #  limitations under the License.
 ---
 
+- name: Update realm_name in promtail_config.yml
+  replace:
+    path: "{{ role_path }}/files/promtail_config.yml"
+    regexp: 'OMNIA-TEST'
+    replace: "{{ realm_name | replace('.','-') }}"
+  when: enable_security_support
+
+- name: Promtail image creation
+  command: "buildah bud -f Dockerfile_promtail -t {{ promtail_image_name }}:{{ promtail_image_tag }} ."
+  changed_when: true
+  args:
+    chdir: "{{ role_path }}/files/"

control_plane/roles/control_plane_monitoring/vars/main.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,11 +13,10 @@
 #  limitations under the License.
 ---
 
-grafana_username: ""
-grafana_password: ""
-grafana_namespace: grafana
-
+# Usage: install_k8s_prom_stack.yml
+helm_repo_prometheus: https://prometheus-community.github.io/helm-charts
 monitoring_namespace: monitoring
+monitoring_app_name: monitoring
 k8s_prom_stack_chart_version: 25.0.0
 k8s_prom_stack_crd:
   - alertmanagerconfigs.monitoring.coreos.com
@@ -29,6 +28,8 @@ k8s_prom_stack_crd:
   - servicemonitors.monitoring.coreos.com
   - thanosrulers.monitoring.coreos.com
 
+# Usage: configure_k8s_prom_grafana.yml
+grafana_namespace: grafana
 grafana_dashboard_json_files:
   - CoreDNS.json
   - Kubernetes_API_server.json
@@ -44,4 +45,12 @@ grafana_dashboard_json_files:
   - Kubernetes_Networking_Pod.json
   - Kubernetes_Networking_Workload.json
   - Kubernetes_Scheduler.json
-  - Prometheus_Overview.json
+  - Prometheus_Overview.json
+
+# Usage: promtail_image.yml
+promtail_image_name: "promtail"
+promtail_image_tag: "latest"
+
+# Usage: k8s_loki_pod.yml
+loki_k8s_name: loki
+loki_port: 3100