Issue#689: Need to update documentation for 1.2

Signed-off-by: cgoveas <cassandra.goveas@dell.com>

control_plane/roles/control_plane_customiso/files/temp_leap15.xml

@@ -0,0 +1,84 @@
+<?xml version="1.0"?>
+<!DOCTYPE profile>
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
+  <install>
+    <init>
+      <info_file>
+        <![CDATA[
+          textmode: 1]]>
+      </info_file>
+    </init>
+  </install>
+  <general>
+    <mode>
+      <confirm config:type="boolean">false</confirm>
+      <final_reboot config:type="boolean">true</final_reboot>
+    </mode>
+  </general>
+  <bootloader t="map">
+    <loader_type>default</loader_type>
+  </bootloader>
+  <host t="map">
+    <hosts t="list">
+      <hosts_entry t="map">
+        <host_address>127.0.0.1</host_address>
+        <names t="list">
+          <name>localhost</name>
+        </names>
+      </hosts_entry>
+    </hosts>
+  </host>
+  <networking t="map">
+    <dns t="map">
+      <dhcp_hostname t="boolean">false</dhcp_hostname>
+      <hostname>localhost.localdomain</hostname>
+      <resolv_conf_policy>auto</resolv_conf_policy>
+    </dns>
+  </networking>
+  <services-manager t="map">
+    <services t="map">
+      <enable t="list">
+        <service>chronyd</service>
+        <service>sshd</service>
+      </enable>
+    </services>
+  </services-manager>
+  <software t="map">
+    <install_recommended t="boolean">true</install_recommended>
+    <instsource/>
+    <packages t="list">
+      <package>openssh</package>
+      <package>firewalld</package>
+      <package>chrony</package>
+    </packages>
+  </software>
+  <ssh_import t="map">
+    <copy_config t="boolean">false</copy_config>
+    <import t="boolean">false</import>
+  </ssh_import>
+  <timezone t="map">
+    <timezone>ks_timezone</timezone>
+  </timezone>
+  <users t="list">
+    <user t="map">
+      <authorized_keys t="list"/>
+      <encrypted t="boolean">true</encrypted>
+      <fullname>root</fullname>
+      <gid>0</gid>
+      <home>/root</home>
+      <home_btrfs_subvolume t="boolean">false</home_btrfs_subvolume>
+      <password_settings t="map">
+        <expire/>
+        <flag/>
+        <inact/>
+        <max/>
+        <min/>
+        <warn/>
+      </password_settings>
+      <shell>/bin/bash</shell>
+      <uid>0</uid>
+      <user_password>ks_password</user_password>
+      <username>root</username>
+    </user>
+  </users>
+</profile>

control_plane/roles/control_plane_customiso/tasks/check_prerequisites.yml

@@ -1,4 +1,4 @@
-# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,15 +13,26 @@
 # limitations under the License.
 ---
 
-- name: Install genisoimage package
+- name: Install xorriso package - rocky
   package:
-    name: genisoimage
+    name: xorriso
     state: present
   tags: install
+  when:
+    - ansible_distribution | lower == os_supported_centos or
+      ansible_distribution | lower == os_supported_rocky
+
+- name: Install xorriso package - leap
+  zypper:
+    name: xorriso
+    state: present
+  tags: install
+  when: os_supported_leap in ansible_distribution | lower
 
 - name: Install ansible-galaxy modules
   command: ansible-galaxy collection install {{ item }}
   changed_when: true
+  tags: install
   with_items:
    - community.general
    - dellemc.openmanage
@@ -34,7 +45,7 @@
 
 - name: Check iso mount folder
   stat:
-    path: "{{ iso_mount_path }}{{ isolinux_cfg_path }}"
+    path: "{{ iso_mount_path }}{{ grub_cfg_path }}"
   register: check_mount_iso
   tags: install
 
@@ -50,4 +61,5 @@
     path: "{{ role_path }}/../provision_idrac/files/{{ management_station_ip_file }}"
     line: "{{ mngmnt_network_ip }}"
     mode: "{{ file_permission }}"
-    create: yes
+    create: yes
+  tags: install

control_plane/roles/control_plane_customiso/tasks/create_unattended_iso.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
   block:
     - name: Create custom ISO
       command: >-
-        mkisofs -o /tmp/{{ centos_iso_filename }} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4
+        xorriso -as mkisofs -o {{ nfs_share_offline_repo }}/{{ centos_iso_filename }} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4
         -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot -J -R -V "CentOS 7 x86_64"  {{ tmp_iso_dir }}
       changed_when: true
       register: centos_iso_status
@@ -28,18 +28,10 @@
     - name: Custom ISO creation status check
       assert:
         that:
-          - "'Total directory bytes:' in centos_iso_status.stderr"
-          - "'Path table size(bytes):' in centos_iso_status.stderr"
-          - "'Max brk space used' in centos_iso_status.stderr"
-          - "'extents written' in centos_iso_status.stderr"
+          - custom_iso_success_keyword1 in centos_iso_status.stderr
+          - custom_iso_success_keyword2 in centos_iso_status.stderr
         success_msg: "{{ custom_iso_success_msg }}"
         fail_msg: "{{ custom_iso_fail_msg }}"
-
-    - name: Copy ISO file to nfs share
-      copy:
-        src: "/tmp/{{ centos_iso_filename }}"
-        dest: "{{ nfs_share_offline_repo }}/{{ centos_iso_filename }}"
-        mode: preserve
       tags: install
   when: provision_os == os_supported_centos
 
@@ -47,7 +39,7 @@
   block:
     - name: Create custom ISO
       command: >-
-        mkisofs -o /tmp/{{ rocky_iso_filename }} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4
+        xorriso -as mkisofs -o {{ nfs_share_offline_repo }}/{{ rocky_iso_filename }} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4
         -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot -J -R -V "Rocky-8-4-x86_64-dvd"  {{ tmp_iso_dir }}
       changed_when: true
       register: rocky_iso_status
@@ -58,23 +50,37 @@
     - name: Custom ISO creation status check
       assert:
         that:
-          - "'Total directory bytes:' in rocky_iso_status.stderr"
-          - "'Path table size(bytes):' in rocky_iso_status.stderr"
-          - "'Max brk space used' in rocky_iso_status.stderr"
-          - "'extents written' in rocky_iso_status.stderr"
+          - custom_iso_success_keyword1 in rocky_iso_status.stderr
+          - custom_iso_success_keyword2 in rocky_iso_status.stderr
         success_msg: "{{ custom_iso_success_msg }}"
         fail_msg: "{{ custom_iso_fail_msg }}"
-
-    - name: Copy ISO file to nfs share
-      copy:
-        src: "/tmp/{{ rocky_iso_filename }}"
-        dest: "{{ nfs_share_offline_repo }}/{{ rocky_iso_filename }}"
-        mode: preserve
       tags: install
   when: provision_os == os_supported_rocky
 
+- name: ISO creation - leap
+  block:
+    - name: Create custom ISO
+      command: >-
+        xorriso -as mkisofs -no-emul-boot -boot-load-size 4 -boot-info-table -iso-level 4 -b boot/x86_64/loader/isolinux.bin 
+        -c boot/x86_64/loader/boot.cat -eltorito-alt-boot -e boot/x86_64/efi -no-emul-boot -o {{ nfs_share_offline_repo }}/{{ leap_iso_filename }} {{ tmp_iso_dir }}
+      changed_when: true
+      register: leap_iso_status
+      tags: install
+      args:
+        chdir: "{{ tmp_iso_dir }}"
+
+    - name: Custom ISO creation status check
+      assert:
+        that:
+          - custom_iso_success_keyword1 in leap_iso_status.stderr
+          - custom_iso_success_keyword2 in leap_iso_status.stderr
+        success_msg: "{{ custom_iso_success_msg }}"
+        fail_msg: "{{ custom_iso_fail_msg }}"
+      tags: install
+  when: provision_os == os_supported_leap
+
 - name: Remove the kickstart file
   file:
     path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
     state: absent
-  tags: install
+  tags: install

control_plane/roles/control_plane_customiso/tasks/edit_iso_config.yml

@@ -1,4 +1,4 @@
-# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -15,68 +15,11 @@
 
 - name: Include control_plane_common vars
   include_vars: ../../control_plane_common/vars/main.yml
+  tags: install
 
 - name: Include provision_cobbler vars
   include_vars: ../../provision_cobbler/vars/main.yml
-
-- name: Copy files to tmp folder
-  command: cp -r {{ iso_mount_path }} /tmp/
-  changed_when: true
-  tags: install
-
-- name: Set centos kickstart file name
-  set_fact:
-    idrac_kickstart_file: "{{ idrac_centos_ks }}"
-  when: provision_os == os_supported_centos
-
-- name: Set rocky kickstart file name
-  set_fact:
-    idrac_kickstart_file: "{{ idrac_rocky_ks }}"
-  when: provision_os == os_supported_rocky
-
-- name: Edit isolinux.cfg
-  replace:
-    path: "{{ tmp_iso_dir }}{{ isolinux_cfg_path }}"
-    regexp: "{{ item.regexp }}"
-    replace: "{{ item.replace }}"
-  with_items:
-    - { regexp: "append initrd=initrd.img", replace: "append initrd=initrd.img ks=cdrom:/{{ idrac_kickstart_file }}" }
-    - { regexp: "rd.live.check quiet", replace: "" }
-  tags: install
-
-- name: Edit grub.cfg
-  replace:
-    path: "{{ tmp_iso_dir }}{{ grub_cfg_path }}"
-    regexp: "{{ item.regexp }}"
-    replace: "{{ item.replace }}"
-  with_items:
-    - { regexp: "timeout=60", replace: "timeout=5" }
-    - { regexp: "kernel /images/pxeboot/vmlinuz", replace: "kernel /images/pxeboot/vmlinuz ks=cdrom:/{{ idrac_kickstart_file }}" }
-    - { regexp: "linuxefi /images/pxeboot/vmlinuz", replace: "linuxefi /images/pxeboot/vmlinuz ks=cdrom:/{{ idrac_kickstart_file }}" }
-    - { regexp: "rd.live.check quiet", replace: "" }
-  tags: install
-
-- name: Remove the kickstart file if exists
-  file:
-    path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
-    state: absent
-  tags: install
-
-- name: Create the centos kickstart file
-  copy:
-    src: "{{ role_path }}/files/temp_centos7.cfg"
-    dest: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
-    mode: "{{ file_permission }}"
-  tags: install
-  when: provision_os == os_supported_centos
-  
-- name: Create the rocky kickstart file
-  copy:
-    src: "{{ role_path }}/files/temp_rocky8.cfg"
-    dest: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
-    mode: "{{ file_permission }}"
   tags: install
-  when: provision_os == os_supported_rocky
 
 - name: Random phrase generation
   command: openssl rand -base64 12
@@ -92,54 +35,177 @@
   register: encrypt_login_pass
   tags: install
 
-- name: Configure kickstart file - Password
-  replace:
-    path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
-    regexp: '^rootpw --iscrypted ks_password'
-    replace: 'rootpw --iscrypted {{ encrypt_login_pass.stdout }}'
-  no_log: true
-  tags: install
-
-- name: Configure kickstart file centos - nic
-  lineinfile:
-    path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
-    insertafter: '^network  --bootproto=dhcp --device=link --onboot=on --activate'
-    line: 'network  --bootproto=dhcp --device={{ item }} --onboot=on --activate'
-  tags: install
-  with_items: "{{ centos_host_nic }}"
-  when: provision_os == os_supported_centos
-
-- name: Configure kickstart file rocky - nic
-  lineinfile:
-    path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
-    insertafter: '^network  --bootproto=dhcp --device=link --onboot=on --activate'
-    line: 'network  --bootproto=dhcp --device={{ item }} --onboot=on --activate'
-  tags: install
-  with_items: "{{ rocky_host_nic }}"
-  when: provision_os == os_supported_rocky
-
-- name: Configure kickstart file - timezone
-  replace:
-    path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
-    regexp: '^timezone --utc ks_timezone'
-    replace: 'timezone --utc {{ timezone }}'
-  tags: install
-
-- name: Configure kickstart file - language
-  replace:
-    path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
-    regexp: '^lang ks_language'
-    replace: 'lang {{ language }}'
+- name: Create a tmp iso directory
+  file:
+    path: "{{ tmp_iso_dir }}"
+    state: directory
+    mode: "{{ file_permission }}"
   tags: install
 
-- name: Copy kickstart file to iso mount path
-  copy:
-    src: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
-    dest: "{{ tmp_iso_dir }}{{ idrac_kickstart_file }}"
-    mode: preserve
+- name: Copy files to tmp folder
+  command:  rsync -AHPSXav {{ iso_mount_path }} {{ tmp_iso_dir }}
+  changed_when: true
+  args:
+    warn: false
   tags: install
 
-- name: Remove ^M characters
-  command: dos2unix {{ tmp_iso_dir }}{{ idrac_kickstart_file }}
-  changed_when: false
-  failed_when: false
+- block:
+    - block:
+        - name: Set centos kickstart file name
+          set_fact:
+            idrac_kickstart_file: "{{ idrac_centos_ks }}"
+          tags: install
+
+        - name: Remove the kickstart file if exists
+          file:
+            path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+            state: absent
+          tags: install
+
+        - name: Create the centos kickstart file
+          copy:
+            src: "{{ role_path }}/files/temp_centos7.cfg"
+            dest: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+            mode: "{{ file_permission }}"
+          tags: install
+
+        - name: Configure kickstart file centos - nic
+          lineinfile:
+            path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+            insertafter: '^network  --bootproto=dhcp --device=link --onboot=on --activate'
+            line: 'network  --bootproto=dhcp --device={{ item }} --onboot=on --activate'
+          tags: install
+          with_items: "{{ centos_host_nic }}"
+      when: provision_os == os_supported_centos
+
+    - block:
+        - name: Set rocky kickstart file name
+          set_fact:
+            idrac_kickstart_file: "{{ idrac_rocky_ks }}"
+          tags: install
+
+        - name: Remove the kickstart file if exists
+          file:
+            path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+            state: absent
+          tags: install
+
+        - name: Create the rocky kickstart file
+          copy:
+            src: "{{ role_path }}/files/temp_rocky8.cfg"
+            dest: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+            mode: "{{ file_permission }}"
+          tags: install
+
+        - name: Configure kickstart file rocky - nic
+          lineinfile:
+            path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+            insertafter: '^network  --bootproto=dhcp --device=link --onboot=on --activate'
+            line: 'network  --bootproto=dhcp --device={{ item }} --onboot=on --activate'
+          tags: install
+          with_items: "{{ rocky_host_nic }}"
+      when: provision_os == os_supported_rocky
+
+    - name: Configure kickstart file - Password
+      replace:
+        path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+        regexp: '^rootpw --iscrypted ks_password'
+        replace: 'rootpw --iscrypted {{ encrypt_login_pass.stdout }}'
+      no_log: true
+      tags: install
+
+    - name: Configure kickstart file - timezone
+      replace:
+        path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+        regexp: '^timezone --utc ks_timezone'
+        replace: 'timezone --utc {{ timezone }}'
+      tags: install
+
+    - name: Configure kickstart file - language
+      replace:
+        path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+        regexp: '^lang ks_language'
+        replace: 'lang {{ language }}'
+      tags: install
+
+    - name: Copy kickstart file to iso mount path
+      copy:
+        src: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+        dest: "{{ tmp_iso_dir }}{{ idrac_kickstart_file }}"
+        mode: preserve
+      tags: install
+
+    - name: Remove ^M characters
+      command: dos2unix {{ tmp_iso_dir }}{{ idrac_kickstart_file }}
+      changed_when: false
+      failed_when: false
+
+    - name: Edit isolinux.cfg
+      replace:
+        path: "{{ tmp_iso_dir }}{{ isolinux_cfg_path }}"
+        regexp: "{{ item.regexp }}"
+        replace: "{{ item.replace }}"
+      with_items:
+        - { regexp: "append initrd=initrd.img", replace: "append initrd=initrd.img ks=cdrom:/{{ idrac_kickstart_file }}" }
+        - { regexp: "rd.live.check quiet", replace: "" }
+      tags: install
+
+    - name: Edit grub.cfg
+      replace:
+        path: "{{ tmp_iso_dir }}{{ grub_cfg_path }}"
+        regexp: "{{ item.regexp }}"
+        replace: "{{ item.replace }}"
+      with_items:
+        - { regexp: "timeout=60", replace: "timeout=5" }
+        - { regexp: "kernel /images/pxeboot/vmlinuz", replace: "kernel /images/pxeboot/vmlinuz ks=cdrom:/{{ idrac_kickstart_file }}" }
+        - { regexp: "linuxefi /images/pxeboot/vmlinuz", replace: "linuxefi /images/pxeboot/vmlinuz ks=cdrom:/{{ idrac_kickstart_file }}" }
+        - { regexp: "rd.live.check quiet", replace: "" }
+      tags: install
+  when:
+    - provision_os == os_supported_rocky or
+      provision_os == os_supported_centos
+
+- block:
+    - name: Set leap kickstart file name
+      set_fact:
+        idrac_kickstart_file: "{{ idrac_leap_ks }}"
+      tags: install
+  
+    - name: Create the leap kickstart file
+      copy:
+        src: "{{ role_path }}/files/temp_leap15.xml"
+        dest: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+        mode: "{{ file_permission }}"
+      tags: install
+
+    - name: Configure kickstart file - Password
+      replace:
+        path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+        regexp: '      <user_password>ks_password</user_password>'
+        replace: '      <user_password>{{ encrypt_login_pass.stdout }}</user_password>'
+      no_log: true
+      tags: install
+
+    - name: Configure kickstart file - timezone
+      replace:
+        path: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+        regexp: '    <timezone>ks_timezone</timezone>'
+        replace: '    <timezone>{{ timezone }}</timezone>'
+      tags: install
+
+    - name: Copy kickstart file to iso mount path
+      copy:
+        src: "{{ role_path }}/files/{{ idrac_kickstart_file }}"
+        dest: "/tmp/leap/{{ idrac_kickstart_file }}"
+        mode: preserve
+      tags: install
+
+    - name: Edit grub.cfg
+      replace:
+        path: "{{ tmp_iso_dir }}{{ grub_cfg_path }}"
+        regexp: "{{ item.regexp }}"
+        replace: "{{ item.replace }}"
+      with_items:
+        - { regexp: "  timeout=60", replace: "  timeout=5" }
+      tags: install
+  when: provision_os == os_supported_leap

control_plane/roles/control_plane_customiso/tasks/main.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -22,4 +22,4 @@
   include_tasks: edit_iso_config.yml
 
 - name: Create unattended iso file
-  include_tasks: create_unattended_iso.yml
+  include_tasks: create_unattended_iso.yml

control_plane/roles/control_plane_customiso/vars/main.yml

@@ -1,4 +1,4 @@
-# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,11 +18,13 @@
 #Usage: check_prerequisites.yml
 iso_mount_path: /mnt/iso/
 iso_mount_check_fail_msg: "ISO file not mounted. Ensure /mnt/iso path is mounted with {{ provision_os }} ISO file."
+management_station_ip_file: "management_station_ip.txt"
 
 #Usage: edit_iso_config.yml
-tmp_iso_dir: /tmp/iso/
+tmp_iso_dir: /tmp/{{ provision_os }}/
 idrac_centos_ks: centos7.cfg
 idrac_rocky_ks: rocky8.cfg
+idrac_leap_ks: autoinst.xml
 file_permission: 0744
 isolinux_cfg_path: isolinux/isolinux.cfg
 grub_cfg_path: EFI/BOOT/grub.cfg
@@ -30,6 +32,8 @@ grub_cfg_path: EFI/BOOT/grub.cfg
 #Usage: create_unattended_iso.yml
 centos_iso_filename: unattended_centos7.iso
 rocky_iso_filename: unattended_rocky8.iso
+leap_iso_filename: unattended_leap15.iso
 custom_iso_success_msg: "Unattended ISO file created successfully"
 custom_iso_fail_msg: "Unattended ISO file creation failed. Ensure /mnt/iso path is mounted with valid centos minimal ISO file."
-management_station_ip_file: "management_station_ip.txt"
+custom_iso_success_keyword1: "ISO image produced:"
+custom_iso_success_keyword2: "completed successfully"

control_plane/roles/control_plane_monitoring/files/Kubernetes_Compute_Resources_Node_Pods.json

@@ -0,0 +1,918 @@
+{
+  "__inputs": [],
+  "__elements": [],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "8.3.2"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph (old)",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "table-old",
+      "name": "Table (old)",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1640151600624,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 5,
+      "panels": [],
+      "title": "CPU Usage",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 10,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 1
+      },
+      "hiddenSeries": false,
+      "id": 1,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 0,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{pod}}",
+          "refId": "A",
+          "step": 10
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "CPU Usage",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 8
+      },
+      "id": 6,
+      "panels": [],
+      "title": "CPU Quota",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "columns": [],
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 9
+      },
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "showHeader": true,
+      "sort": {
+        "col": 0,
+        "desc": true
+      },
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "styles": [
+        {
+          "alias": "Time",
+          "align": "auto",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "hidden"
+        },
+        {
+          "alias": "CPU Usage",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #A",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        },
+        {
+          "alias": "CPU Requests",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #B",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        },
+        {
+          "alias": "CPU Requests %",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #C",
+          "thresholds": [],
+          "type": "number",
+          "unit": "percentunit"
+        },
+        {
+          "alias": "CPU Limits",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #D",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        },
+        {
+          "alias": "CPU Limits %",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #E",
+          "thresholds": [],
+          "type": "number",
+          "unit": "percentunit"
+        },
+        {
+          "alias": "Pod",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "pod",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        },
+        {
+          "alias": "",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "string",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A",
+          "step": 10
+        },
+        {
+          "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "B",
+          "step": 10
+        },
+        {
+          "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "C",
+          "step": 10
+        },
+        {
+          "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "D",
+          "step": 10
+        },
+        {
+          "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "E",
+          "step": 10
+        }
+      ],
+      "thresholds": [],
+      "title": "CPU Quota",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "transform": "table",
+      "type": "table-old",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ]
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "id": 7,
+      "panels": [],
+      "title": "Memory Usage",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 10,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 17
+      },
+      "hiddenSeries": false,
+      "id": 3,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 0,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{pod}}",
+          "refId": "A",
+          "step": 10
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Memory Usage (w/o cache)",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "id": 8,
+      "panels": [],
+      "title": "Memory Quota",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "columns": [],
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 25
+      },
+      "id": 4,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "showHeader": true,
+      "sort": {
+        "col": 0,
+        "desc": true
+      },
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "styles": [
+        {
+          "alias": "Time",
+          "align": "auto",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "hidden"
+        },
+        {
+          "alias": "Memory Usage",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #A",
+          "thresholds": [],
+          "type": "number",
+          "unit": "bytes"
+        },
+        {
+          "alias": "Memory Requests",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #B",
+          "thresholds": [],
+          "type": "number",
+          "unit": "bytes"
+        },
+        {
+          "alias": "Memory Requests %",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #C",
+          "thresholds": [],
+          "type": "number",
+          "unit": "percentunit"
+        },
+        {
+          "alias": "Memory Limits",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #D",
+          "thresholds": [],
+          "type": "number",
+          "unit": "bytes"
+        },
+        {
+          "alias": "Memory Limits %",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #E",
+          "thresholds": [],
+          "type": "number",
+          "unit": "percentunit"
+        },
+        {
+          "alias": "Memory Usage (RSS)",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #F",
+          "thresholds": [],
+          "type": "number",
+          "unit": "bytes"
+        },
+        {
+          "alias": "Memory Usage (Cache)",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #G",
+          "thresholds": [],
+          "type": "number",
+          "unit": "bytes"
+        },
+        {
+          "alias": "Memory Usage (Swap)",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "Value #H",
+          "thresholds": [],
+          "type": "number",
+          "unit": "bytes"
+        },
+        {
+          "alias": "Pod",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "link": false,
+          "linkTargetBlank": false,
+          "linkTooltip": "Drill down",
+          "linkUrl": "",
+          "pattern": "pod",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        },
+        {
+          "alias": "",
+          "align": "auto",
+          "colors": [],
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "decimals": 2,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "string",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A",
+          "step": 10
+        },
+        {
+          "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", node=~\"$node\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "B",
+          "step": 10
+        },
+        {
+          "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node=~\"$node\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "C",
+          "step": 10
+        },
+        {
+          "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", node=~\"$node\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "D",
+          "step": 10
+        },
+        {
+          "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node=~\"$node\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "E",
+          "step": 10
+        },
+        {
+          "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "F",
+          "step": 10
+        },
+        {
+          "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "G",
+          "step": 10
+        },
+        {
+          "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)",
+          "format": "table",
+          "instant": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "H",
+          "step": 10
+        }
+      ],
+      "thresholds": [],
+      "title": "Memory Quota",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "transform": "table",
+      "type": "table-old",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ]
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 33,
+  "style": "dark",
+  "tags": [
+    "kubernetes-mixin"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "default",
+          "value": "default"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "uid": "$datasource"
+        },
+        "definition": "",
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(kube_pod_info, cluster)",
+          "refId": "Prometheus-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {},
+        "datasource": {
+          "uid": "$datasource"
+        },
+        "definition": "",
+        "hide": 0,
+        "includeAll": false,
+        "multi": true,
+        "name": "node",
+        "options": [],
+        "query": {
+          "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)",
+          "refId": "Prometheus-node-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "UTC",
+  "title": "Kubernetes / Compute Resources / Node (Pods)",
+  "uid": "200ac8fdbfbb74b39aff88118e4d1c2c",
+  "version": 1,
+  "weekStart": ""
+}

control_plane/roles/control_plane_monitoring/files/Kubernetes_Scheduler.json

@@ -0,0 +1,963 @@
+{
+  "__inputs": [],
+  "__elements": [],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "8.3.2"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph (old)",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1640159989547,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "min"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "expr": "sum(up{job=\"kube-scheduler\"})",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Up",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 10,
+        "x": 4,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 3,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}} e2e",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(rate(scheduler_binding_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}} binding",
+          "refId": "B"
+        },
+        {
+          "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}} scheduling algorithm",
+          "refId": "C"
+        },
+        {
+          "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}} volume",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Scheduling Rate",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "ops",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "ops",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 10,
+        "x": 14,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}} e2e",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}} binding",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}} scheduling algorithm",
+          "refId": "C"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}} volume",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Scheduling latency 99th Quantile",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "s",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 0,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 5,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "2xx",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "3xx",
+          "refId": "B"
+        },
+        {
+          "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "4xx",
+          "refId": "C"
+        },
+        {
+          "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "5xx",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Kube API Request Rate",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "ops",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "ops",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 16,
+        "x": 8,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{verb}} {{url}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Post Request Latency 99th Quantile",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "s",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 14
+      },
+      "hiddenSeries": false,
+      "id": 7,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{verb}} {{url}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Get Request Latency 99th Quantile",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "s",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 0,
+        "y": 21
+      },
+      "hiddenSeries": false,
+      "id": 8,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "process_resident_memory_bytes{job=\"kube-scheduler\", instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Memory",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 8,
+        "y": 21
+      },
+      "hiddenSeries": false,
+      "id": 9,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rate(process_cpu_seconds_total{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "CPU usage",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 16,
+        "y": 21
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "go_goroutines{job=\"kube-scheduler\",instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Goroutines",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 33,
+  "style": "dark",
+  "tags": [
+    "kubernetes-mixin"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "uid": "$datasource"
+        },
+        "definition": "",
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "name": "instance",
+        "options": [],
+        "query": {
+          "query": "label_values(process_cpu_seconds_total{job=\"kube-scheduler\"}, instance)",
+          "refId": "Prometheus-instance-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "UTC",
+  "title": "Kubernetes / Scheduler",
+  "uid": "2e6b6a3b4bddf1427b3a55aa1311c656",
+  "version": 1,
+  "weekStart": ""
+}

control_plane/roles/control_plane_monitoring/files/Kubernetes_StatefulSets.json

@@ -0,0 +1,820 @@
+{
+  "__inputs": [],
+  "__elements": [],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "8.3.2"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph (old)",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1640160026561,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "rgb(31, 120, 193)",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m]))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "CPU",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "rgb(31, 120, 193)",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 8,
+        "y": 0
+      },
+      "id": 3,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "expr": "sum(container_memory_usage_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}) / 1024^3",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Memory",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "rgb(31, 120, 193)",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 16,
+        "y": 0
+      },
+      "id": 4,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\",pod=~\"$statefulset.*\"}[3m]))",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Network",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 6,
+        "x": 0,
+        "y": 7
+      },
+      "id": 5,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Desired Replicas",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 6,
+        "x": 6,
+        "y": 7
+      },
+      "id": 6,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Replicas of current version",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 6,
+        "x": 12,
+        "y": 7
+      },
+      "id": 7,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Observed Generation",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "0"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 6,
+        "x": 18,
+        "y": 7
+      },
+      "id": 8,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.3.2",
+      "targets": [
+        {
+          "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Metadata Generation",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 0,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 9,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.3.2",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "replicas specified",
+          "refId": "A"
+        },
+        {
+          "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "replicas created",
+          "refId": "B"
+        },
+        {
+          "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "ready",
+          "refId": "C"
+        },
+        {
+          "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "replicas of current version",
+          "refId": "D"
+        },
+        {
+          "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "updated",
+          "refId": "E"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Replicas",
+      "tooltip": {
+        "shared": false,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 33,
+  "style": "dark",
+  "tags": [
+    "kubernetes-mixin"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "default",
+          "value": "default"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "uid": "$datasource"
+        },
+        "definition": "",
+        "hide": 2,
+        "includeAll": false,
+        "label": "cluster",
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(kube_statefulset_metadata_generation, cluster)",
+          "refId": "Prometheus-cluster-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {},
+        "datasource": {
+          "uid": "$datasource"
+        },
+        "definition": "",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Namespace",
+        "multi": false,
+        "name": "namespace",
+        "options": [],
+        "query": {
+          "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)",
+          "refId": "Prometheus-namespace-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {},
+        "datasource": {
+          "uid": "$datasource"
+        },
+        "definition": "",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Name",
+        "multi": false,
+        "name": "statefulset",
+        "options": [],
+        "query": {
+          "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, statefulset)",
+          "refId": "Prometheus-statefulset-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "UTC",
+  "title": "Kubernetes / StatefulSets",
+  "uid": "a31c1f46e6f727cb37c0d731a7245005",
+  "version": 1,
+  "weekStart": ""
+}

control_plane/roles/control_plane_monitoring/tasks/configure_k8s_grafana.yml

@@ -0,0 +1,73 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Get grafana service IP
+  command: kubectl get svc -n {{ grafana_namespace }} -o=jsonpath='{.items[0].spec.clusterIP}'
+  changed_when: false
+  register: grafana_svc_ip
+
+- name: Get grafana service port
+  command: kubectl get svc -n {{ grafana_namespace }} -o=jsonpath='{.items[0].spec.ports[0].port}'
+  changed_when: false
+  register: grafana_svc_port
+
+- name: Get kube-prometheus svc IP
+  command: kubectl get svc -n {{ monitoring_namespace }} -l app=kube-prometheus-stack-prometheus -o=jsonpath='{.items[0].spec.clusterIP}'
+  changed_when: false
+  register: kube_prom_svc_ip
+
+- name: Get kube-prometheus svc port
+  command: kubectl get svc -n {{ monitoring_namespace }} -l app=kube-prometheus-stack-prometheus -o=jsonpath='{.items[0].spec.ports[0].port}'
+  changed_when: false
+  register: kube_prom_svc_port
+
+- name: Install community.grafana collection
+  command: ansible-galaxy collection install community.grafana
+  changed_when: True
+
+- name: Create prometheus datasource
+  community.grafana.grafana_datasource:
+    name: control-plane-prometheus
+    grafana_url: "http://{{ grafana_svc_ip.stdout }}:{{ grafana_svc_port.stdout }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    ds_type: prometheus
+    ds_url: "http://{{ kube_prom_svc_ip.stdout }}:{{ kube_prom_svc_port.stdout }}"
+    access: proxy
+  no_log: true
+
+- name: Import K8s Grafana dashboards
+  community.grafana.grafana_dashboard:
+    grafana_url: "http://{{ grafana_svc_ip.stdout }}:{{ grafana_svc_port.stdout }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    state: present
+    commit_message: Updated by ansible
+    overwrite: yes
+    path: "{{ role_path }}/files/{{ item }}"
+  with_items: "{{ grafana_dashboard_json_files }}"
+  no_log: true
+
+- name: Save grafana svc ip
+  replace:
+    path: "{{ role_path }}/../../../roles/k8s_start_services/files/grafana_svc_details.ini"
+    regexp: "ip=.*"
+    replace: "ip={{ grafana_svc_ip.stdout }}"
+
+- name: Save grafana svc ip
+  replace:
+    path: "{{ role_path }}/../../../roles/k8s_start_services/files/grafana_svc_details.ini"
+    regexp: "port=.*"
+    replace: "port={{ grafana_svc_port.stdout }}"

control_plane/roles/control_plane_monitoring/tasks/install_grafana.yml

@@ -0,0 +1,15 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+

control_plane/roles/control_plane_monitoring/tasks/install_k8s_prom_stack.yml

@@ -0,0 +1,57 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Install kubernetes.core collection
+  command: ansible-galaxy collection install kubernetes.core
+  changed_when: True
+
+- name: Add prometheus-community chart
+  kubernetes.core.helm_repository:
+    name: prometheus-community
+    repo_url: "https://kubernetes.github.io/ingress-nginx"
+  failed_when: False
+
+- name: Install kube-prom-stack
+  block:
+    - name: Install kube-prom-stack
+      kubernetes.core.helm:
+        name: monitoring
+        chart_ref: prometheus-community/kube-prometheus-stack
+        chart_version: "{{ k8s_prom_stack_chart_version }}"
+        update_repo_cache: True
+        release_namespace: "{{ monitoring_namespace }}"
+        create_namespace : yes
+        wait: true
+        values:
+          grafana:
+            enabled: False
+  rescue:
+    - name: Delete existing kube-prom-stack crd
+      command: "kubectl delete crd {{ item }}"
+      changed_when: True
+      with_items: "{{ k8s_prom_stack_crd }}"
+
+    - name: Install kube-prom-stack
+      kubernetes.core.helm:
+        name: monitoring
+        chart_ref: prometheus-community/kube-prometheus-stack
+        chart_version: "{{ k8s_prom_stack_chart_version }}"
+        update_repo_cache: True
+        release_namespace: "{{ monitoring_namespace }}"
+        create_namespace : yes
+        wait: true
+        values:
+          grafana:
+            enabled: False

control_plane/roles/control_plane_monitoring/tasks/main.yml

@@ -0,0 +1,23 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+#- name: Install Grafana
+#  import_tasks: install_grafana.yml
+
+- name: Install K8s prometheus stack
+  import_tasks: install_k8s_prom_stack.yml
+
+- name: Configure K8s grafana
+  import_tasks: configure_k8s_grafana.yml

control_plane/roles/control_plane_monitoring/vars/main.yml

@@ -0,0 +1,47 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+grafana_username: ""
+grafana_password: ""
+grafana_namespace: grafana
+
+monitoring_namespace: monitoring
+k8s_prom_stack_chart_version: 25.0.0
+k8s_prom_stack_crd:
+  - alertmanagerconfigs.monitoring.coreos.com
+  - alertmanagers.monitoring.coreos.com
+  - podmonitors.monitoring.coreos.com
+  - probes.monitoring.coreos.com
+  - prometheuses.monitoring.coreos.com
+  - prometheusrules.monitoring.coreos.com
+  - servicemonitors.monitoring.coreos.com
+  - thanosrulers.monitoring.coreos.com
+
+grafana_dashboard_json_files:
+  - CoreDNS.json
+  - Kubernetes_API_server.json
+  - Kubernetes_Compute_Resources_Cluster.json
+  - Kubernetes_Compute_Resources_Namespace_Pods.json
+  - Kubernetes_Compute_Resources_Node_Pods.json
+  - Kubernetes_Compute_Resources_Pod.json
+  - Kubernetes_Compute_Resources_Workload.json
+  - Kubernetes_Kubelet.json
+  - Kubernetes_Networking_Cluster.json
+  - Kubernetes_Networking_Namespace_Pods.json
+  - Kubernetes_Networking_Namespace_Workload.json
+  - Kubernetes_Networking_Pod.json
+  - Kubernetes_Networking_Workload.json
+  - Kubernetes_Scheduler.json
+  - Prometheus_Overview.json

control_plane/roles/provision_cobbler/files/Dockerfile_leap

@@ -0,0 +1,70 @@
+FROM registry.opensuse.org/opensuse/leap:15.3
+
+#Enable all repose
+RUN zypper mr -ea
+
+# Refresh and update the repo
+RUN zypper --gpg-auto-import-keys refresh
+RUN zypper --gpg-auto-import-keys up -y
+
+#Enable systemd
+RUN zypper in -y dbus-1 systemd-sysvinit
+RUN cp /usr/lib/systemd/system/dbus.service /etc/systemd/system/; \
+    sed -i 's/OOMScoreAdjust=-900//' /etc/systemd/system/dbus.service
+
+VOLUME ["/sys/fs/cgroup", "/run"]
+
+RUN zypper in -y git-core
+RUN git clone -b v3.2.2 https://github.com/cobbler/cobbler.git
+RUN cd cobbler/
+
+RUN mkdir /root/omnia
+
+RUN zypper in --no-confirm python3-librepo \
+                      python3-schema \
+                      syslinux \
+                      wget \
+                      dhcp-server \
+                      python3-pykickstart \
+                      dnf-plugins-core \
+                      cronie \
+                      xinetd \
+                      python3-coverage \
+                      python3-Cheetah3 \
+                      python3-netaddr \
+                      python3-devel \
+                      python3-future \
+                      apache2-mod_wsgi-python3 \
+                      gcc \
+                      python-rpm-macros \
+                      rpm-build \
+                      ansible \
+                      grub2-x86_64-efi \
+                      shim \
+                      yum-utils \
+                      python3-Sphinx \
+                      python3-wheel \
+                      bash-completion \
+                      tftp \
+                      vim
+
+RUN useradd -ms /bin/bash dhcpd
+
+#Copy Configuration files
+COPY settings.yaml /etc/cobbler/settings.yaml
+COPY dhcp.template  /etc/cobbler/dhcp.template
+COPY modules.conf  /etc/cobbler/modules.conf
+COPY tftp /etc/xinetd.d/tftp
+COPY .users.digest /etc/cobbler/users.digest
+COPY cobbler_configurations_leap.yml /root
+COPY tftp.yml /root
+COPY inventory_creation.yml /root
+
+EXPOSE 69 80 443 25151
+
+VOLUME [ "/srv/www/cobbler", "/var/lib/cobbler/backup", "/mnt" ]
+
+RUN systemctl enable apache2
+RUN systemctl enable dhcpd
+
+CMD ["sbin/init"]

control_plane/roles/provision_cobbler/files/cobbler_configurations.yml

@@ -110,7 +110,7 @@
       mode: 0775
     tags: install
     when: name_iso == "rocky"
-
+    
   - name: Pxe menu
     copy:
       src: "/root/omnia/control_plane/roles/provision_cobbler/files/menu.yml"
@@ -121,8 +121,8 @@
   - name: Assign default grub option
     replace:
       path: "/var/lib/cobbler/grub_config/grub/grub.cfg"
-      regexp: "^set default=\'local\'"
-      replace: "set default=\'1\'"
+      regexp: "^set default='local'"
+      replace: "set default='1'"
     tags: install
 
   - name: Assign default grub timeout
@@ -155,10 +155,10 @@
     cron:
       name: Start tftp service
       minute: "*"
-      job: "{{ ansible_playbook_path.stdout.split(' ')[1] }} /root/tftp.yml"
+      job: "{{ ansible_playbook_path.stdout.split(' ')[1] }} /root/tftp.yml -e provision_os={{ name_iso }}"
 
   - name: Add inventory cron job
     cron:
       name: Create inventory
       minute: "*/5"
-      job: "{{ ansible_playbook_path.stdout.split(' ')[1] }} /root/inventory_creation.yml"
+      job: "{{ ansible_playbook_path.stdout.split(' ')[1] }} /root/inventory_creation.yml -e dhcpd_lease_file=\"/var/lib/dhcpd/dhcpd.leases\""

control_plane/roles/provision_cobbler/files/cobbler_configurations_leap.yml

@@ -0,0 +1,152 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Initial cobbler setup
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Inside cobbler container
+    debug:
+      msg: "Hiii! I am cobbler"
+
+  - name: Make
+    command: make rpms
+    args:
+      chdir: /cobbler
+    changed_when: false
+
+  - name: Install cobbler and cobbler-web
+    command: zypper in -y --allow-unsigned-rpm cobbler-3.2.2-1.noarch.rpm cobbler-web-3.2.2-1.noarch.rpm
+    args:
+      chdir: /cobbler/rpm-build
+    changed_when: false
+    
+  - name: Run script
+    shell: sh ./mkgrub.sh
+    args:
+      chdir: /usr/share/cobbler/bin
+    changed_when: false
+  
+  - name: Add settings to http
+    shell: echo "LoadModule wsgi_module modules/mod_wsgi_python3.so" >/etc/apache2/conf.d/wsgi.conf
+    changed_when: false
+
+  - name: Add interface to the /etc/sysconfig/dhcpd
+    replace:
+      path: "/etc/sysconfig/dhcpd"
+      regexp: "^DHCPD_INTERFACE=\"\""
+      replace: "DHCPD_INTERFACE={{ host_network_nic }}"
+
+  - name: Enable cobbler
+    command: systemctl enable {{ item }}
+    with_items:
+      - cobblerd
+      - tftp
+    changed_when: false
+
+  - name: Restart httpd
+    service:
+      name: "{{ item }}"
+      state: restarted
+    loop:
+      - apache2
+      - cobblerd
+
+  - name: Start services
+    service:
+      name: "{{ item }}"
+      state: started
+    loop:
+      - cobblerd
+      - tftp
+
+  - name: Adding curl
+    shell: export PATH="/usr/bin/curl:$PATH"
+
+  - name: Run import command
+    command: cobbler import --arch=x86_64 --path=/mnt --name="{{ name_iso }}"
+    changed_when: false
+
+  - name: Kickstart profile - leap
+    copy:
+      src: "/root/leap15.xml"
+      dest: "/var/lib/cobbler/templates/sample_autoyast.xml"
+      mode: 0775
+    tags: install
+
+  - name: Pxe menu
+    copy:
+      src: "/root/omnia/control_plane/roles/provision_cobbler/files/menu.yml"
+      dest: "/etc/cobbler/boot_loader_conf/pxedefault.template"
+      mode: 0775
+    tags: install
+
+  - name: Assign default grub option
+    replace:
+      path: "/var/lib/cobbler/grub_config/grub/grub.cfg"
+      regexp: "^set default='local'"
+      replace: "set default='1'"
+    tags: install
+
+  - name: Assign default grub timeout
+    replace:
+      path: "/var/lib/cobbler/grub_config/grub/grub.cfg"
+      regexp: '^set timeout=80'
+      replace: 'set timeout=10'
+    tags: install
+
+  - name: Syncing of cobbler
+    command: cobbler sync
+    changed_when: false 
+
+  - name: Restart cobblerd, http, xinetd and dhcpd
+    service:
+      name: "{{ item }}"
+      state: restarted
+    loop:
+      - cobblerd
+      - apache2
+      - xinetd
+      - dhcpd
+
+  - name: Fetch ansible-playbook path
+    command: whereis ansible-playbook
+    changed_when: false
+    register: ansible_playbook_path
+
+#  - name: Set dhcpd_lease_file variable for inventory creation
+#    cron:
+#      env: yes
+#      name: dhcpd_lease_file
+#      value: "/var/lib/dhcp/db/dhcpd.leases"
+
+#  - name: Set provision_os variable for inventory creation
+#    cron:
+#      env: yes
+#      name: provision_os
+#      value: "{{ name_iso }}"
+
+  - name: Add tftp cron job
+    cron:
+      name: Start tftp service
+      minute: "*"
+      job: "{{ ansible_playbook_path.stdout.split(' ')[1] }} /root/tftp.yml -e provision_os={{ name_iso }}"
+
+  - name: Add inventory cron job
+    cron:
+      name: Create inventory
+      minute: "*/5"
+      job: "{{ ansible_playbook_path.stdout.split(' ')[1] }} /root/inventory_creation.yml -e dhcpd_lease_file=\"/var/lib/dhcp/db/dhcpd.leases\""

control_plane/roles/provision_cobbler/files/inventory_creation.yml

@@ -4,8 +4,8 @@
   tasks:
     - name: Read dhcp file
       set_fact:
-        var: "{{ lookup('file', '/var/lib/dhcpd/dhcpd.leases').split()| unique | select| list }}"
-
+        var: "{{ lookup('file', '{{ dhcpd_lease_file }}').split()| unique | select| list }}"
+    
     - name: Filter the ip
       set_fact:
         vars_new: "{{ var| ipv4('address')| to_nice_yaml}}"
@@ -35,4 +35,4 @@
 
     - name: New line at end of file
       shell: echo "">> omnia/control_plane/roles/collect_node_info/files/provisioned_hosts.yml
-      changed_when: false
+      changed_when: false

control_plane/roles/provision_cobbler/files/temp_leap15.xml

@@ -0,0 +1,86 @@
+<?xml version="1.0"?>
+<!DOCTYPE profile>
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
+  <install>
+    <init>
+      <info_file>
+        <![CDATA[
+          install: http://ip/cblr/links/leap-x86_64/
+          textmode: 1]]>
+      </info_file>
+    </init>
+  </install>
+  <general>
+    <mode>
+      <confirm config:type="boolean">false</confirm>
+      <final_reboot config:type="boolean">true</final_reboot>
+    </mode>
+  </general>
+  <bootloader t="map">
+    <loader_type>default</loader_type>
+  </bootloader>
+  <host t="map">
+    <hosts t="list">
+      <hosts_entry t="map">
+        <host_address>127.0.0.1</host_address>
+        <names t="list">
+          <name>localhost</name>
+        </names>
+      </hosts_entry>
+    </hosts>
+  </host>
+  <networking t="map">
+    <dns t="map">
+      <dhcp_hostname t="boolean">false</dhcp_hostname>
+      <hostname>localhost.localdomain</hostname>
+      <resolv_conf_policy>auto</resolv_conf_policy>
+    </dns>
+  </networking>
+  <services-manager t="map">
+    <default_target>multi-user</default_target>
+    <services t="map">
+      <enable t="list">
+        <service>chronyd</service>
+        <service>sshd</service>
+      </enable>
+    </services>
+  </services-manager>
+  <software t="map">
+    <install_recommended t="boolean">true</install_recommended>
+    <instsource/>
+    <packages t="list">
+      <package>openssh</package>
+      <package>firewalld</package>
+      <package>chrony</package>
+    </packages>
+  </software>
+  <ssh_import t="map">
+    <copy_config t="boolean">false</copy_config>
+    <import t="boolean">false</import>
+  </ssh_import>
+  <timezone t="map">
+    <timezone>yast_timezone</timezone>
+  </timezone>
+  <users t="list">
+    <user t="map">
+      <authorized_keys t="list"/>
+      <encrypted t="boolean">true</encrypted>
+      <fullname>root</fullname>
+      <gid>0</gid>
+      <home>/root</home>
+      <home_btrfs_subvolume t="boolean">false</home_btrfs_subvolume>
+      <password_settings t="map">
+        <expire/>
+        <flag/>
+        <inact/>
+        <max/>
+        <min/>
+        <warn/>
+      </password_settings>
+      <shell>/bin/bash</shell>
+      <uid>0</uid>
+      <user_password>yast_password</user_password>
+      <username>root</username>
+    </user>
+  </users>
+</profile>

control_plane/roles/provision_cobbler/files/tftp.yml

@@ -17,6 +17,27 @@
   hosts: localhost
   connection: local
   tasks:
+    - name: Configure tftp for leap
+      block:
+        - name: Stop the tftp.socket
+          command: systemctl stop tftp.socket
+
+        - name: Modify tftp.socket to listsen on IPv4
+          replace:
+            path: "/usr/lib/systemd/system/tftp.socket"
+            regexp: 'ListenDatagram=69'
+            replace: 'ListenDatagram=0.0.0.0:69'
+        
+        - name: Reload the configurations
+          command: systemctl daemon-reload
+          
+        - name: Enable tftp.socket
+          command: systemctl enable tftp.socket
+
+        - name: Start tftp.socket
+          command: systemctl start tftp.socket
+      when: provision_os == "leap"
+
     - name: Fetch tftp status
       command: systemctl is-active tftp
       args:

control_plane/roles/provision_cobbler/tasks/check_prerequisites.yml

@@ -32,6 +32,11 @@
     cobbler_kickstart_file: "{{ cobbler_rocky_ks }}"
   when: provision_os == os_supported_rocky
 
+- name: Set rocky kickstart file name
+  set_fact:
+    cobbler_kickstart_file: "{{ cobbler_leap_ks }}"
+  when: provision_os == os_supported_leap
+
 - name: Check if any backup file exists
   block:
   - name: Check status of backup file

control_plane/roles/provision_cobbler/tasks/cobbler_image.yml

@@ -18,7 +18,19 @@
   changed_when: true
   args:
     chdir: "{{ role_path }}/files/"
-  when: not cobbler_image_status
+  when: 
+    - not cobbler_image_status
+    - provision_os != os_supported_leap
+  tags: install
+
+- name: Image creation (It may take 5-10 mins)
+  command: "buildah bud -f Dockerfile_leap -t {{ cobbler_image_name }}:{{ cobbler_image_tag }} --network host ."
+  changed_when: true
+  args:
+    chdir: "{{ role_path }}/files/"
+  when: 
+    - not cobbler_image_status
+    - provision_os == os_supported_leap
   tags: install
 
 - name: Update image name in k8s_cobbler.yml

control_plane/roles/provision_cobbler/tasks/configure_cobbler.yml

@@ -34,7 +34,7 @@
   wait_for:
     timeout: 30
 
-- name: Copy dhcpd.leases from cobbler
+- name: Copy kickstart file inside cobbler container
   command: kubectl cp {{ role_path }}/files/{{ cobbler_kickstart_file }} {{ cobbler_pod_name.stdout }}:/root/{{ cobbler_kickstart_file }} -n {{ cobbler_namespace }}
   changed_when: true
   when: not cobbler_config_status
@@ -43,7 +43,17 @@
   command: "kubectl exec --stdin --tty -n {{ cobbler_namespace }} {{ cobbler_pod_name.stdout }} -- ansible-playbook /root/cobbler_configurations.yml -e name_iso={{ provision_os }}"
   changed_when: true
   tags: install
-  when: not cobbler_config_status
+  when: 
+    - not cobbler_config_status
+    - provision_os != os_supported_leap
+      
+- name: Configuring cobbler inside container (It may take 5-10 mins)
+  command: "kubectl exec --stdin --tty -n {{ cobbler_namespace }} {{ cobbler_pod_name.stdout }} -- ansible-playbook /root/cobbler_configurations_leap.yml -e name_iso={{ provision_os }} -e host_network_nic={{ host_network_nic }}"
+  changed_when: true
+  tags: install
+  when: 
+    - not cobbler_config_status
+    - provision_os == os_supported_leap
 
 - name: Schedule task
   cron:
@@ -67,4 +77,4 @@
     - "{{ role_path }}/files/dhcp.template"
     - "{{ role_path }}/files/settings"
     - "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
-    - "{{ role_path }}/files/temp_host_mapping_file.csv.bak"
+    - "{{ role_path }}/files/temp_host_mapping_file.csv.bak"

control_plane/roles/provision_cobbler/tasks/dhcp_configure.yml

@@ -69,3 +69,32 @@
     path: "{{ role_path }}/files/settings.yaml"
     regexp: '^next_server: 127.0.0.1'
     replace: 'next_server: {{ hpc_ip }}'
+
+- name: Set the server_args for {{ provision_os }} in tftp file
+  replace:
+    path: "{{ role_path }}/files/tftp"
+    regexp: '^        server_args             = -s /srv/tftpboot'
+    replace: '        server_args             = -s /var/lib/tftpboot'
+  when:
+    - provision_os != os_supported_leap
+      
+- name: Set the parameters for {{ provision_os }}
+  block:
+    - name: Set the tftpboot_location for {{ provision_os }}
+      replace:
+        path: "{{ role_path }}/files/settings.yaml"
+        regexp: '^tftpboot_location: "/var/lib/tftpboot"'
+        replace: 'tftpboot_location: "/srv/tftpboot"'
+
+    - name: Set the webdir for {{ provision_os }}
+      replace:
+        path: "{{ role_path }}/files/settings.yaml"
+        regexp: '^webdir: "/var/www/cobbler"'
+        replace: 'webdir: "/srv/www/cobbler"'
+
+    - name: Set the server_args for {{ provision_os }} in tftp file
+      replace:
+        path: "{{ role_path }}/files/tftp"
+        regexp: '^        server_args             = -s /var/lib/tftpboot'
+        replace: '        server_args             = -s /srv/tftpboot'
+  when: provision_os == os_supported_leap

control_plane/roles/provision_cobbler/tasks/provision_password.yml

@@ -88,6 +88,23 @@
       with_items: "{{ rocky_host_nic }}"
   when: provision_os == os_supported_rocky
 
+- name: Kickstart configuration - leap
+  block:
+    - name: Create the kickstart file
+      copy:
+        src: "{{ role_path }}/files/temp_leap15.xml"
+        dest: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
+        mode: 0775
+      tags: install
+
+    - name: Configure kickstart file - IP
+      replace:
+        path: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
+        regexp: '^          install: http://ip/cblr/links/leap-x86_64/'
+        replace: '          install: http://{{ hpc_ip }}/cblr/links/leap-x86_64/'
+      tags: install
+  when: provision_os == os_supported_leap
+
 - name: Random phrase generation
   command: openssl rand -base64 12
   changed_when: false
@@ -116,29 +133,51 @@
   no_log: true
   tags: install
 
-- name: Configure kickstart file - Password
-  replace:
-    path: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
-    regexp: '^rootpw --iscrypted ks_password'
-    replace: 'rootpw --iscrypted {{ login_pass.stdout }}'
-  no_log: true
-  tags: install
+- name: Configure kickstart file for {{ provision_os }}
+  block:
+    - name: Configure kickstart file - Password
+      replace:
+        path: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
+        regexp: '^rootpw --iscrypted ks_password'
+        replace: 'rootpw --iscrypted {{ login_pass.stdout }}'
+      no_log: true
+      tags: install
 
-- name: Configure kickstart file - timezone
-  replace:
-    path: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
-    regexp: '^timezone --utc ks_timezone'
-    replace: 'timezone --utc {{ timezone }}'
-  tags: install
+    - name: Configure kickstart file - timezone
+      replace:
+        path: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
+        regexp: '^timezone --utc ks_timezone'
+        replace: 'timezone --utc {{ timezone }}'
+      tags: install
 
-- name: Configure kickstart file - language
-  replace:
-    path: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
-    regexp: '^lang ks_language'
-    replace: 'lang {{ language }}'
-  tags: install
+    - name: Configure kickstart file - language
+      replace:
+        path: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
+        regexp: '^lang ks_language'
+        replace: 'lang {{ language }}'
+      tags: install
+  when: 
+    - provision_os != os_supported_leap
+
+- name: Configure kickstart file for {{ provision_os }}
+  block:
+    - name: Configure kickstart file - Password
+      replace:
+        path: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
+        regexp: '^      <user_password>ks_password</user_password>'
+        replace: '      <user_password>{{ login_pass.stdout }}</user_password>'
+      no_log: true
+      tags: install
+
+    - name: Configure kickstart file - timezone
+      replace:
+        path: "{{ role_path }}/files/{{ cobbler_kickstart_file }}"
+        regexp: '^    <timezone>ks_timezone</timezone>'
+        replace: '    <timezone>{{ timezone }}</timezone>'
+      tags: install
+  when: provision_os == os_supported_leap
 
 - name: Remove ^M characters
   command: dos2unix {{ role_path }}/files/{{ cobbler_kickstart_file }}
   changed_when: false
-  failed_when: false
+  failed_when: false

control_plane/roles/provision_cobbler/vars/main.yml

@@ -30,6 +30,7 @@ username: cobbler
 user_mode: 0644
 cobbler_centos_ks: centos7.ks
 cobbler_rocky_ks: rocky8.ks
+cobbler_leap_ks: leap15.xml
 centos_host_nic:
  - em1
  - em2

control_plane/roles/provision_idrac/tasks/deploy_os.yml

@@ -1,4 +1,4 @@
-# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -44,6 +44,11 @@
         unattended_iso_filename: "{{ rocky_iso_filename }}"
       when: provision_os == os_supported_rocky
 
+    - name: Set unattended_iso_filename - leap
+      set_fact:
+        unattended_iso_filename: "{{ leap_iso_filename }}"
+      when: provision_os == os_supported_leap
+
     - name: Install OS using iDRAC
       dellemc.openmanage.idrac_os_deployment:
         idrac_ip: "{{ inventory_hostname }}"

control_plane/roles/provision_idrac/vars/main.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -55,7 +55,7 @@ provision_method_pxe: "PXE"
 # Usage: update_firmware.yml
 idrac_port: 443
 idrac_error_message: "Unable to complete the operation because the catalog name entered has either unsupported firmware packages or same version installed on the server"
-firmware_job_fail_msg: "Failed. Error occured while updating firmware"
+firmware_job_fail_msg: "Failed. Error occurred while updating firmware"
 firmware_job_success_msg: "Firmware update job compeleted successfully"
 
 # Usage: import_scp.yml
@@ -71,4 +71,4 @@ file_permission: 0644
 provision_os_msg: "OS provisioning is initiated. Wait for installation to complete for all servers."
 idrac_provisioning_fail_msg: "OS provisioning using iDRAC is failed. This could be due to older firmware or some internal issues with server. Re-run idrac_template after fixing the issue"
 pxe_provisioning_fail_msg: "OS provisioning using PXE is failed. This could be due some internal issues with server. Re-run idrac_template after fixing the issue"
-add_inventory_fail_msg: "Failed. Unable to add provisioned host to provisioned_idrac_inventory."
+add_inventory_fail_msg: "Failed. Unable to add provisioned host to provisioned_idrac_inventory."

docs/README.md

@@ -1,10 +1,10 @@
 **Omnia** (Latin: all or everything) is a deployment tool to configure Dell EMC PowerEdge servers running standard RPM-based Linux OS images into clusters capable of supporting HPC, AI, and data analytics workloads. It uses Slurm, Kubernetes, and other packages to manage jobs and run diverse workloads on the same converged solution. It is a collection of [Ansible](https://ansible.com) playbooks, is open source, and is constantly being extended to enable comprehensive workloads.
 
 #### Current release version
-1.1.1
+1.2
 
 #### Previous release version
-1.1  
+1.1.1
 
 ## Blogs about Omnia
 - [Introduction to Omnia](https://infohub.delltechnologies.com/p/omnia-open-source-deployment-of-high-performance-clusters-to-run-simulation-ai-and-data-analytics-workloads/)
@@ -27,6 +27,8 @@ Omnia can install Kubernetes or Slurm (or both), along with additional drivers,
 ![Omnia Slurm Stack](images/omnia-slurm.png)  
 
 ## What's new in this release
+* Extended support of Leap OS on Management station, login, compute and NFS nodes.
+* Omnia now supports Powervault configurations with 2 network interfaces.
 * Provisioning of Rocky custom ISO on supported PowerEdge servers using iDRAC.
 * Configuring Dell EMC networking switches, Mellanox InfiniBand switches, and PowerVault storage devices in the cluster. 
 * An option to configure a login node with the same configurations as the compute nodes in the cluster. With appropriate user privileges provided by the cluster administrator, users can log in to the login node and schedule Slurm jobs. The authentication mechanism in the login node uses the FreeIPA solution.
@@ -46,8 +48,8 @@ The following table lists the software and operating system requirements on the
 
 Requirements  |   Version
 ----------------------------------  |   -------
-OS pre-installed on the management station  |  CentOS 8.4/ Rocky 8.4
-OS deployed by Omnia on bare-metal Dell EMC PowerEdge Servers | CentOS 7.9 2009 Minimal Edition/ Rocky 8.4 Minimal Edition
+OS pre-installed on the management station  |  CentOS 8.4/ Rocky 8.5/ Leap 15.3
+OS deployed by Omnia on bare-metal Dell EMC PowerEdge Servers | Rocky 8.5 Minimal Edition/ Leap 15.3
 Cobbler  |  3.2.2
 Ansible AWX  |  19.1.0
 Slurm Workload Manager  |  20.11.2
@@ -55,6 +57,9 @@ Kubernetes on the management station  |  1.21.0
 Kubernetes on the manager and compute nodes	|	1.16.7 or 1.19.3
 Kubeflow  |  1
 Prometheus  |  2.23.0
+Ansible  |  2.9.21
+Python  |  3.6.15
+CRI-O  |  1.17.3
 
 ## Hardware managed by Omnia
 The following table lists the supported devices managed by Omnia. Other devices than those listed in the following table will be discovered by Omnia, but features offered by Omnia will not be applicable.
@@ -72,10 +77,11 @@ The following table lists the software and its compatible version managed by Omn
 
 Software	|	License	|	Compatible Version	|	Description
 -----------	|	-------	|	----------------	|	-----------------
+LeapOS 15.3	|	-	|	15.3|	Operating system on entire cluster
 CentOS Linux release 7.9.2009 (Core)	|	-	|	7.9	|	Operating system on entire cluster except for management station
-Rocky 8.4	|	-	|	8.4	|	Operating system on entire cluster except for management station
+Rocky 8.5	|	-	|	8.5	|	Operating system on entire cluster except for management station
 CentOS Linux release 8.4.2105	|	-	|	8.4	|	Operating system on the management station	
-Rocky 8.4	|	-	|	8.4	|	Operating system on the management station
+Rocky 8.5	|	-	|	8.5	|	Operating system on the management station
 MariaDB	|	GPL 2.0	|	5.5.68	|	Relational database used by Slurm
 Slurm	|	GNU General Public	|	20.11.7	|	HPC Workload Manager
 Docker CE	|	Apache-2.0	|	20.10.2	|	Docker Service
@@ -195,9 +201,6 @@ If hosts are listed, then an IP address has been assigned to them by DHCP. Howev
 * **Issue**: Hosts are not automatically deleted from awx UI when redeploying the cluster.  
 	**Resolution**: Before re-deploying the cluster, ensure that the user manually deletes all hosts from the awx UI.
 	
-* **Issue**: Decomissioned compute nodes do not get deleted automatically from the awx UI.
-	**Resolution**: Once a node is decommisioned, ensure that the user manually deletes decomissioned hosts from the awx UI.
-
 
 # [Frequently asked questions](FAQ.md)
 
@@ -210,7 +213,7 @@ If hosts are listed, then an IP address has been assigned to them by DHCP. Howev
 * To change the Kubernetes version from 1.16 to 1.19 or 1.19 to 1.16, you must redeploy the entire cluster.  
 * The Kubernetes pods will not be able to access the Internet or start when firewalld is enabled on the node. This is a limitation in Kubernetes. So, the firewalld daemon will be disabled on all the nodes as part of omnia.yml execution.
 * Only one storage instance (Powervault) is currently supported in the HPC cluster.
-* With the latest `catalog.xml` file, firmware updates of a few components might fail for server models: R640 and R740. Note that Omnia doesn't halt or get interrupted despite these failures. (Fix Expected by 17th December 2021)
+* Cobbler web support has been discontinued from Omnia 1.2 onwards.
 
 
 # Contributing to Omnia

roles/k8s_start_services/files/grafana_svc_details.ini

@@ -0,0 +1,3 @@
+[grafana_svc]
+ip=
+port=

roles/k8s_start_services/tasks/configure_nginx_prom_grafana.yml

@@ -0,0 +1,82 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Include control_plane_monitoring variables
+  include_vars: "{{ role_path }}/../../control_plane/roles/control_plane_monitoring/vars/main.yml"
+
+- name: Install Nginx
+  package:
+    name: nginx
+    state: present
+  when: ansible_facts['distribution'] != opensuse_os_name
+
+- name: Install Nginx
+  zypper:
+    name: nginx
+    state: present
+  when: ansible_facts['distribution'] == opensuse_os_name
+
+- name: Start and enable nginx service
+  service:
+    name: nginx
+    state: restarted
+    enabled: yes
+
+- name: Get prometheus service IP
+  command: kubectl get svc -l app=prometheus,component=server -o=jsonpath='{.items[0].spec.clusterIP}'
+  changed_when: false
+  register: prometheus_svc_ip
+
+- name: Configure nginx.conf (1/2)
+  replace:
+    path: "{{ role_path }}/templates/nginx.conf.j2"
+    regexp: '        server_name  .*'
+    replace: "        server_name  {{ ansible_default_ipv4.address }};"
+  delegate_to: localhost
+
+- name: Configure nginx.conf (2/2)
+  replace:
+    path: "{{ role_path }}/templates/nginx.conf.j2"
+    regexp: '          proxy_pass http://.*'
+    replace: "          proxy_pass http://{{ prometheus_svc_ip.stdout }};"
+  delegate_to: localhost
+
+- name: Load nginx conf
+  template:
+    src: nginx.conf.j2
+    dest: "{{ nginx_conf_file_path }}"
+    mode: "{{ nginx_conf_file_mode }}"
+
+- name: Validate nginx conf file
+  command: nginx -t
+  changed_when: false
+
+- name: Start and enable nginx service
+  service:
+    name: nginx
+    state: restarted
+    enabled: yes
+
+- name: Create prometheus datasource in grafana
+  community.grafana.grafana_datasource:
+    name: "hpc-prometheus-{{ ansible_default_ipv4.address }}"
+    grafana_url: "http://{{ grafana_svc_ip }}:{{ grafana_svc_port }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    ds_type: prometheus
+    ds_url: "http://{{ ansible_default_ipv4.address }}"
+    access: direct
+  delegate_to: localhost
+  no_log: true

roles/k8s_start_services/tasks/main.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -20,4 +20,12 @@
 
 - name: Check K8s pods
   include_tasks: check_k8s_pods.yml
-  tags: install
+  tags: install
+
+- name: Configure Nginx/Prometheus/Grafana
+  include_tasks: configure_nginx_prom_grafana.yml
+  when:
+    - "'manager' in group_names"
+    - grafana_svc_ip != ""
+    - grafana_svc_port != ""
+  tags: install

roles/k8s_start_services/templates/nginx.conf.j2

@@ -0,0 +1,90 @@
+# For more information on configuration, see:
+#   * Official English Documentation: http://nginx.org/en/docs/
+#   * Official Russian Documentation: http://nginx.org/ru/docs/
+
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log;
+pid /run/nginx.pid;
+
+# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
+include /usr/share/nginx/modules/*.conf;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile            on;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    keepalive_timeout   65;
+    types_hash_max_size 2048;
+
+    include             /etc/nginx/mime.types;
+    default_type        application/octet-stream;
+
+    # Load modular configuration files from the /etc/nginx/conf.d directory.
+    # See http://nginx.org/en/docs/ngx_core_module.html#include
+    # for more information.
+    include /etc/nginx/conf.d/*.conf;
+
+    server {
+        listen       80 default_server;
+        listen       [::]:80 default_server;
+        server_name  server_ip;
+        root         /usr/share/nginx/html;
+
+        # Load configuration files for the default server block.
+        include /etc/nginx/default.d/*.conf;
+
+        location / {
+          proxy_pass http://prometheus_svc_ip;
+        }
+
+        error_page 404 /404.html;
+            location = /40x.html {
+        }
+
+        error_page 500 502 503 504 /50x.html;
+            location = /50x.html {
+        }
+    }
+
+# Settings for a TLS enabled server.
+#
+#    server {
+#        listen       443 ssl http2 default_server;
+#        listen       [::]:443 ssl http2 default_server;
+#        server_name  _;
+#        root         /usr/share/nginx/html;
+#
+#        ssl_certificate "/etc/pki/nginx/server.crt";
+#        ssl_certificate_key "/etc/pki/nginx/private/server.key";
+#        ssl_session_cache shared:SSL:1m;
+#        ssl_session_timeout  10m;
+#        ssl_ciphers PROFILE=SYSTEM;
+#        ssl_prefer_server_ciphers on;
+#
+#        # Load configuration files for the default server block.
+#        include /etc/nginx/default.d/*.conf;
+#
+#        location / {
+#        }
+#
+#        error_page 404 /404.html;
+#            location = /40x.html {
+#        }
+#
+#        error_page 500 502 503 504 /50x.html;
+#            location = /50x.html {
+#        }
+#    }
+
+}

roles/k8s_start_services/vars/main.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -94,3 +94,13 @@ spark_operator_repo: https://googlecloudplatform.github.io/spark-on-k8s-operator
 operator_image_tag: v1beta2-1.2.3-3.1.1
 
 volcano_scheduling_yaml_url: https://raw.githubusercontent.com/volcano-sh/volcano/v1.3.0/installer/volcano-development.yaml
+
+nginx_conf_file_path: /etc/nginx/nginx.conf
+
+nginx_conf_file_mode: 2534
+
+grafana_svc_ip: "{{ lookup('ini', 'ip section=grafana_svc file={{ role_path }}/files/grafana_svc_details.ini') }}"
+
+grafana_svc_port: "{{ lookup('ini', 'port section=grafana_svc file={{ role_path }}/files/grafana_svc_details.ini') }}"
+
+opensuse_os_name: "openSUSE Leap"

roles/slurm_exporter/tasks/configure_grafana.yml

@@ -0,0 +1,64 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Include control_plane_monitoring variables
+  include_vars: "{{ role_path }}/../../control_plane/roles/control_plane_monitoring/vars/main.yml"
+
+- name: Create prometheus datasource in grafana
+  community.grafana.grafana_datasource:
+    name: "hpc-prometheus-{{ ansible_default_ipv4.address }}"
+    grafana_url: "http://{{ grafana_svc_ip }}:{{ grafana_svc_port }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    ds_type: prometheus
+    ds_url: "http://{{ ansible_default_ipv4.address }}"
+    access: direct
+  delegate_to: localhost
+  no_log: true
+
+- name: Replace data source in slurm dashboard
+  replace:
+    path: "{{ role_path }}/files/{{ item }}"
+    regexp: '"uid": "hpc.*'
+    replace: '"uid": "hpc-prometheus-{{ ansible_default_ipv4.address }}"'
+  with_items: "{{ slurm_dashboard_json_files }}"
+  delegate_to: localhost
+
+- name: Replace title in slurm dashboard (1/2)
+  replace:
+    path: "{{ role_path }}/files/slurm-dashboard-node-exporter.json"
+    regexp: '"title": "SLURM - Node Exporter Server Metrics.*'
+    replace: '"title": "SLURM - Node Exporter Server Metrics - ({{ ansible_default_ipv4.address }})",'
+  delegate_to: localhost
+
+- name: Replace title in slurm dashboard (2/2)
+  replace:
+    path: "{{ role_path }}/files/slurm-dashboard.json"
+    regexp: '"title": "SLURM - CPUs/GPUs, Nodes, Jobs, Scheduler.*'
+    replace: '"title": "SLURM - CPUs/GPUs, Nodes, Jobs, Scheduler ({{ ansible_default_ipv4.address }})",'
+  delegate_to: localhost
+
+- name: Import Slurm Grafana dashboards
+  community.grafana.grafana_dashboard:
+    grafana_url: "http://{{ grafana_svc_ip }}:{{ grafana_svc_port }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    state: present
+    commit_message: Updated by ansible
+    overwrite: yes
+    path: "{{ role_path }}/files/{{ item }}"
+  with_items: "{{ slurm_dashboard_json_files }}"
+  delegate_to: localhost
+  no_log: true

roles/slurm_exporter/tasks/install_prometheus.yml

@@ -1,4 +1,4 @@
-#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -37,4 +37,57 @@
           scrape_interval:  30s
           scrape_timeout:   30s
           static_configs:
-            - targets: ['localhost:{{ slurm_exporter_port }}']
+            - targets: ['localhost:{{ slurm_exporter_port }}']
+
+- name: Install & configure Nginx
+  block:
+    - name: Install Nginx
+      package:
+        name: nginx
+        state: present
+      when: ansible_facts['distribution'] != opensuse_os_name
+
+    - name: Install Nginx
+      zypper:
+        name: nginx
+        state: present
+      when: ansible_facts['distribution'] == opensuse_os_name
+
+    - name: Start and enable nginx service
+      service:
+        name: nginx
+        state: restarted
+        enabled: yes
+
+    - name: Configure nginx.conf (1/2)
+      replace:
+        path: "../../k8s_start_services/templates/nginx.conf.j2"
+        regexp: '        server_name  .*'
+        replace: "        server_name  {{ ansible_default_ipv4.address }};"
+      delegate_to: localhost
+
+    - name: Configure nginx.conf (2/2)
+      replace:
+        path: "../../k8s_start_services/templates/nginx.conf.j2"
+        regexp: '          proxy_pass http://.*'
+        replace: "          proxy_pass {{ prometheus_ip }};"
+      delegate_to: localhost
+
+    - name: Load nginx conf
+      template:
+        src: "../../k8s_start_services/templates/nginx.conf.j2"
+        dest: "{{ nginx_conf_file_path }}"
+        mode: "{{ nginx_conf_file_mode }}"
+
+    - name: Validate nginx conf file
+      command: nginx -t
+      changed_when: false
+
+    - name: Start and enable nginx service
+      service:
+        name: nginx
+        state: restarted
+        enabled: yes
+  when:
+    - grafana_svc_ip != ""
+    - grafana_svc_port != ""

roles/slurm_exporter/tasks/main.yml

@@ -1,4 +1,4 @@
-# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -37,3 +37,9 @@
   when:
     - "'kubernetes' in ansible_skip_tags"
     - "'No such file' in k8s_installation_status.stderr"
+
+- name: Configure grafana dashboard
+  include_tasks: configure_grafana.yml
+  when:
+    - grafana_svc_ip != ""
+    - grafana_svc_port != ""

roles/slurm_exporter/vars/main.yml

@@ -1,4 +1,4 @@
-# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -38,4 +38,14 @@ systemd_path_dest: "/etc/systemd/system/"
 slurm_exporter_port: "8081"
 
 #Usage: configure_prometheus_pod.yml
-slurm_config_file: "slurm_exporter_config.yaml"
+slurm_config_file: "slurm_exporter_config.yaml"
+
+#Usage: configure_grafana.yml
+prometheus_ip: http://localhost:9090
+nginx_conf_file_path: /etc/nginx/nginx.conf
+nginx_conf_file_mode: 2534
+grafana_svc_ip: "{{ lookup('ini', 'ip section=grafana_svc file=../../k8s_start_services/files/grafana_svc_details.ini') }}"
+grafana_svc_port: "{{ lookup('ini', 'port section=grafana_svc file=../../k8s_start_services/files/grafana_svc_details.ini') }}"
+slurm_dashboard_json_files:
+  - slurm-dashboard.json
+  - slurm-dashboard-node-exporter.json