Merge pull request #728 from blesson-james/slurm_telemetry

Issue #722: Adding slurm_telemetry role for MonSter integration

roles/cluster_validation/tasks/fetch_grafana_cred.yml

@@ -0,0 +1,63 @@
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Check login_vars file is encrypted
+  command: cat "{{ role_path }}/../../control_plane/{{ login_vars_filename }}"
+  changed_when: false
+  register: config_content
+  no_log: true
+
+- name: Decrpyt login_vars.yml
+  command: >-
+    ansible-vault decrypt "{{ role_path }}/../../control_plane/{{ login_vars_filename }}"
+    --vault-password-file "{{ role_path }}/../../control_plane/{{ vault_filename }}"
+  changed_when: false
+  when: "'$ANSIBLE_VAULT;' in config_content.stdout"
+
+- name: Include variable file login_vars.yml
+  include_vars: "{{ role_path }}/../../control_plane/{{ login_vars_filename }}"
+  no_log: true
+
+- name: Save grafana variables
+  set_fact:
+    grafana_username: '{{ grafana_username }}'
+    grafana_password: '{{ grafana_password }}'
+  no_log: True
+
+- name: Create ansible vault key
+  set_fact:
+    vault_key: "{{ lookup('password', '/dev/null chars=ascii_letters') }}"
+  when: "'$ANSIBLE_VAULT;' not in config_content.stdout"
+
+- name: Save vault key
+  copy:
+    dest: "{{ role_path }}/../../control_plane/{{ vault_filename }}"
+    content: |
+      {{ vault_key }}
+    owner: root
+    force: yes
+    mode: "{{ vault_file_perm }}"
+  when: "'$ANSIBLE_VAULT;' not in config_content.stdout"
+
+- name: Encrypt input config file
+  command: >-
+    ansible-vault encrypt "{{ role_path }}/../../control_plane/{{ login_vars_filename }}"
+    --vault-password-file "{{ role_path }}/../../control_plane/{{ vault_filename }}"
+  changed_when: false
+
+- name: Update login_vars.yml permission
+  file:
+    path: "{{ role_path }}/../../control_plane/{{ login_vars_filename }}"
+    mode: "{{ vault_file_perm }}"

roles/cluster_validation/tasks/main.yml

@@ -93,4 +93,8 @@
         regexp: '#log_path = /var/log/ansible.log'
         replace: 'log_path = /var/log/omnia.log'
       when: ansible_conf_exists.stat.exists
-  when: not control_plane_status
+  when: not control_plane_status
+
+- name: Fetch grafana credentials
+  include_tasks: fetch_grafana_cred.yml
+  when: control_plane_status

roles/k8s_start_services/tasks/configure_nginx_prom_grafana.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,9 +13,6 @@
 #  limitations under the License.
 ---
 
-- name: Include control_plane_monitoring variables
-  include_vars: "{{ role_path }}/../../control_plane/roles/control_plane_monitoring/vars/main.yml"
-
 - name: Install Nginx
   package:
     name: nginx
@@ -73,10 +70,10 @@
   community.grafana.grafana_datasource:
     name: "hpc-prometheus-{{ ansible_default_ipv4.address }}"
     grafana_url: "http://{{ grafana_svc_ip }}:{{ grafana_svc_port }}"
-    grafana_user: "{{ grafana_username }}"
-    grafana_password: "{{ grafana_password }}"
+    grafana_user: "{{ hostvars['127.0.0.1']['grafana_username'] }}"
+    grafana_password: "{{ hostvars['127.0.0.1']['grafana_password'] }}"
     ds_type: prometheus
     ds_url: "http://{{ ansible_default_ipv4.address }}"
     access: direct
   delegate_to: localhost
-  no_log: true
+  no_log: true

roles/k8s_start_services/tasks/main.yml

@@ -22,6 +22,13 @@
   include_tasks: check_k8s_pods.yml
   tags: install
 
+- name: Install community.grafana collection
+  command: ansible-galaxy collection install community.grafana
+  changed_when: True
+  tags: install
+  delegate_to: localhost
+  when: hostvars['127.0.0.1']['control_plane_status']
+
 - name: Configure Nginx/Prometheus/Grafana
   include_tasks: configure_nginx_prom_grafana.yml
   when:

roles/slurm_exporter/tasks/configure_grafana.yml

@@ -1,4 +1,4 @@
-#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,15 +13,12 @@
 #  limitations under the License.
 ---
 
-- name: Include control_plane_monitoring variables
-  include_vars: "{{ role_path }}/../../control_plane/roles/control_plane_monitoring/vars/main.yml"
-
 - name: Create prometheus datasource in grafana
   community.grafana.grafana_datasource:
     name: "hpc-prometheus-{{ ansible_default_ipv4.address }}"
     grafana_url: "http://{{ grafana_svc_ip }}:{{ grafana_svc_port }}"
-    grafana_user: "{{ grafana_username }}"
-    grafana_password: "{{ grafana_password }}"
+    grafana_user: "{{ hostvars['127.0.0.1']['grafana_username'] }}"
+    grafana_password: "{{ hostvars['127.0.0.1']['grafana_password'] }}"
     ds_type: prometheus
     ds_url: "http://{{ ansible_default_ipv4.address }}"
     access: direct
@@ -53,12 +50,12 @@
 - name: Import Slurm Grafana dashboards
   community.grafana.grafana_dashboard:
     grafana_url: "http://{{ grafana_svc_ip }}:{{ grafana_svc_port }}"
-    grafana_user: "{{ grafana_username }}"
-    grafana_password: "{{ grafana_password }}"
+    grafana_user: "{{ hostvars['127.0.0.1']['grafana_username'] }}"
+    grafana_password: "{{ hostvars['127.0.0.1']['grafana_password'] }}"
     state: present
     commit_message: Updated by ansible
     overwrite: yes
     path: "{{ role_path }}/files/{{ item }}"
   with_items: "{{ slurm_dashboard_json_files }}"
   delegate_to: localhost
-  no_log: true
+  no_log: true

roles/slurm_exporter/tasks/main.yml

@@ -38,6 +38,13 @@
     - "'kubernetes' in ansible_skip_tags"
     - "'No such file' in k8s_installation_status.stderr"
 
+- name: Install community.grafana collection
+  command: ansible-galaxy collection install community.grafana
+  changed_when: True
+  tags: install
+  delegate_to: localhost
+  when: hostvars['127.0.0.1']['control_plane_status']
+
 - name: Configure grafana dashboard
   include_tasks: configure_grafana.yml
   when:

telemetry/roles/slurm_telemetry/files/Dockerfile

@@ -0,0 +1,20 @@
+FROM rockylinux:8.5
+
+RUN yum install epel-release git gcc -y
+RUN yum -y install openssl-devel bzip2-devel libffi-devel xz-devel
+RUN yum install python3.8 -y
+RUN echo 1 | update-alternatives --config python3
+RUN dnf -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
+RUN dnf module disable postgresql -y
+RUN dnf install postgresql13-devel -y
+RUN yum install python38-devel libpq-devel -y
+
+COPY requirements.txt requirements.txt
+RUN ln -s /usr/pgsql-13/bin/pg_config /usr/bin/pg_config
+
+RUN pip3 install psycopg2-binary
+RUN pip3 install -r requirements.txt
+RUN mkdir /log/
+RUN touch /log/monster.log
+
+WORKDIR /MonSter/

telemetry/roles/slurm_telemetry/files/input_config.yml

@@ -0,0 +1,23 @@
+# TimeScaleDB Configuration
+timescaledb:
+  host:
+  port:
+  username:
+  password:
+  database:
+
+# Slurm REST API Configuration
+slurm_rest_api:
+  ip:
+  port: 6820
+  user: root
+  headnode:
+  slurm_jobs: /slurm/v0.0.36/jobs/
+  slurm_nodes: /slurm/v0.0.36/nodes/
+  openapi: /openapi/v3
+
+# Mapping of OS hostnames and OS IP addresses
+hostnames:
+
+# Mapping of service tag and OS IP addresses
+clusternodes:

telemetry/roles/slurm_telemetry/files/requirements.txt

@@ -0,0 +1,15 @@
+sphinx
+pytest
+PyYAML
+tqdm
+requests
+psycopg2
+pgcopy
+aiohttp
+schedule
+async_retrying
+python-hostlist
+python-dateutil
+flask
+flask_cors
+sqlalchemy

telemetry/roles/slurm_telemetry/files/update_service_tags.yml

@@ -0,0 +1,90 @@
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.​0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+- name: Create inventory in awx
+  hosts: manager, compute
+  tasks:
+    - name: Check slurmctld service
+      systemd:
+        name: slurmctld
+      register: slurm_service_status
+
+    - name: Set fact slurm_service
+      set_fact:
+        slurm_service: True
+      delegate_to: localhost
+      when: "slurm_service_status.status.ActiveState == 'active'"
+
+    - name: Set fact slurm_service
+      set_fact:
+        slurm_service: False
+      delegate_to: localhost
+      when: "slurm_service_status.status.ActiveState == 'inactive'"
+
+    - name: Replace input file
+      copy:
+        src: "input_config.yml"
+        dest: /mnt/omnia/slurm/monster/config.yml
+        mode: 0644
+      delegate_to: localhost
+      when: "slurm_service_status.status.ActiveState == 'active'"
+
+    - name: Prepare input config file
+      block:
+        - name: Get service tag
+          shell: >
+            set -o pipefail && \
+            dmidecode -t 1 | grep Serial
+          changed_when: false
+          register: service_tag_details
+
+        - name: Set fact service tag
+          set_fact:
+            service_tag: "{{ service_tag_details.stdout.split(':')[1].strip() }}"
+
+        - name: Get the hostname
+          command: hostname
+          register: machine_hostname
+          changed_when: false
+
+        - name: Update Head Node IP
+          replace:
+            path: /mnt/omnia/slurm/monster/config.yml
+            regexp: '  ip:.*'
+            replace: "  ip: {{ groups['manager'][0] }}"
+          delegate_to: localhost
+
+        - name: Update Head Node hostname
+          replace:
+            path: /mnt/omnia/slurm/monster/config.yml
+            regexp: '  headnode:.*'
+            replace: "  headnode: {{ hostvars[groups['manager'][0]]['machine_hostname'].stdout }}"
+          delegate_to: localhost
+
+        - name: Update nodes hostnames
+          lineinfile:
+            path: /mnt/omnia/slurm/monster/config.yml
+            line: "  {{ machine_hostname.stdout }}: {{ ansible_default_ipv4.address }}"
+            insertafter: "hostnames:"
+          delegate_to: localhost
+
+        - name: Update service tag info
+          lineinfile:
+            path: /mnt/omnia/slurm/monster/config.yml
+            line: "  - Servicetag: {{ service_tag }}\n    Os_Ip_Addr: {{ ansible_default_ipv4.address }}"
+            insertafter: "clusternodes:"
+          delegate_to: localhost
+      when: hostvars[groups['manager'][0]]['slurm_service']

telemetry/roles/slurm_telemetry/tasks/get_node_inventory.yml

@@ -0,0 +1,82 @@
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Copy slurm telemetry code
+  copy:
+    src: "{{ role_path }}/files/monster"
+    dest: "{{ slurm_telemetry_code_dir }}"
+    mode: "{{ slurm_telemetry_code_dir_mode }}"
+
+- name: Get AWX service IP
+  command: kubectl get svc awx-ui -n {{ awx_namespace }} -o=jsonpath='{.spec.clusterIP}'
+  changed_when: false
+  failed_when: false
+  register: awx_svc_ip
+
+- name: AWX needs to be installed
+  fail:
+    msg: "{{ awx_fail_msg }}"
+  when: not awx_svc_ip.stdout
+
+- name: Get AWX service port
+  command: kubectl get svc awx-ui -n {{ awx_namespace }} -o=jsonpath='{.spec.ports[0].port}'
+  changed_when: false
+  register: awx_svc_port
+
+- name: Get AWX secret
+  shell: >
+    set -o pipefail && \
+    kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath="{.data.password}" | base64 --decode
+  changed_when: false
+  register: awx_secret
+
+- name: Get node_inventory id
+  shell: >
+    set -o pipefail && \
+    awx --conf.host http://{{ awx_svc_ip.stdout }}:{{ awx_svc_port.stdout }} --conf.username {{ awx_username }} \
+    --conf.password {{ awx_secret.stdout }} --conf.insecure inventory list -f human | grep node_inventory
+  changed_when: false
+  register: inventory_id
+
+- name: Node inventory not found in AWX
+  fail:
+    msg: "{{ node_inventory_fail_msg }}"
+  when: not inventory_id.stdout
+
+- name: Get node_inventory
+  command: awx --conf.host http://{{ awx_svc_ip.stdout }}:{{ awx_svc_port.stdout }} --conf.username {{ awx_username }} \
+    --conf.password {{ awx_secret.stdout }} --conf.insecure hosts list --inventory {{ inventory_id.stdout[0] }}
+  changed_when: false
+  register: node_inventory_output
+
+- name: Save the json data
+  set_fact:
+    node_inventory_jsondata: "{{ node_inventory_output.stdout | from_json }}"
+
+- name: Add temporary hosts
+  add_host:
+    name: "{{ item.name }}"
+    groups: "{{ item.summary_fields.groups.results[0].name }}"
+  with_items: "{{ node_inventory_jsondata | json_query('results') }}"
+  no_log: true
+
+- name: Update slurm telemetry code path
+  replace:
+    path: "{{ role_path }}/files/update_service_tags.yml"
+    regexp: '{{ item }}.*'
+    replace: "{{ item }} {{ slurm_telemetry_code_dir }}/monster/config.yml"
+  with_items:
+    - "dest:"
+    - "path:"

telemetry/roles/slurm_telemetry/tasks/main.yml

@@ -0,0 +1,24 @@
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Include common variables
+  include_vars: ../../common/vars/main.yml
+
+- name: Include timescaledb variables
+  include_vars: ../../timescaledb/vars/main.yml
+
+- name: Prepare MonSter input file
+  include_tasks: update_timescaledb_details.yml
+  when: hostvars[groups['manager'][0]]['slurm_service']

telemetry/roles/slurm_telemetry/tasks/update_timescaledb_details.yml

@@ -0,0 +1,55 @@
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Get timescaledb service IP
+  command: kubectl get svc {{ timescaledb_k8s_name }} -n {{ namespace }} -o=jsonpath='{.spec.clusterIP}'
+  changed_when: false
+  register: timescaledb_svc_ip
+
+- name: Get timescaledb service port
+  command: kubectl get svc {{ timescaledb_k8s_name }} -n {{ namespace }} -o=jsonpath='{.spec.ports[0].port}'
+  changed_when: false
+  register: timescaledb_svc_port
+
+- name: Update timescaledb service IP
+  replace:
+    path: "{{ slurm_telemetry_code_dir }}/monster/config.yml"
+    regexp: "  host:.*"
+    replace: "  host: {{ timescaledb_svc_ip.stdout }}"
+
+- name: Update timescaledb service port
+  replace:
+    path: "{{ slurm_telemetry_code_dir }}/monster/config.yml"
+    regexp: "  port:.*"
+    replace: "  port: {{ timescaledb_svc_port.stdout }}"
+    before: "# Slurm REST API Configuration"
+
+- name: Update timescaledb username
+  replace:
+    path: "{{ slurm_telemetry_code_dir }}/monster/config.yml"
+    regexp: "  username:.*"
+    replace: "  username: {{ timescaledb_user }}"
+
+- name: Update timescaledb password
+  replace:
+    path: "{{ slurm_telemetry_code_dir }}/monster/config.yml"
+    regexp: "  password:.*"
+    replace: "  password: {{ timescaledb_password }}"
+
+- name: Update timescaledb database
+  replace:
+    path: "{{ slurm_telemetry_code_dir }}/monster/config.yml"
+    regexp: "  database:.*"
+    replace: "  database: {{ timescaledb_name }}"

telemetry/roles/slurm_telemetry/vars/main.yml

@@ -0,0 +1,22 @@
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# usage: get_node_inventory.yml, update_timescaledb_details.yml
+slurm_telemetry_code_dir: "{{ mount_location }}slurm"
+slurm_telemetry_code_dir_mode: 0755
+awx_namespace: awx
+awx_username: admin
+awx_fail_msg: "AWX service not found. AWX needs to be installed"
+node_inventory_fail_msg: "AWX node inventory not found. Node inventory needs be created in AWX"

telemetry/telemetry.yml

@@ -21,3 +21,25 @@
     - common
     - timescaledb
     - grafana_config
+
+- name: Get node inventory
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Get node inventory
+      include_role:
+        name: slurm_telemetry
+        tasks_from: get_node_inventory.yml
+      tags: slurm_telemetry
+
+- name: Update slurm node IPs and service tags
+  import_playbook: "{{ playbook_dir }}/roles/slurm_telemetry/files/update_service_tags.yml"
+  tags: slurm_telemetry
+
+- name: Slurm Telemetry
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  roles:
+   - slurm_telemetry
+  tags: slurm_telemetry