Merge pull request #186 from DeepikaKrishnaiah/devel

Issue #174: Prometheus slurm exporter and test framework

roles/slurm_exporter/files/prometheus-slurm-exporter.service

@@ -0,0 +1,10 @@
+[Unit]
+Description = Start prometheus slurm exporter
+
+[Service]
+ExecStart = /usr/bin/prometheus-slurm-exporter
+Restart = always
+RestartSec = 15
+
+[Install]
+WantedBy = multi-user.target

roles/slurm_exporter/files/slurm_exporter_config.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus-slurmexporter-metrics-2
+  namespace: default
+  annotations:
+      prometheus.io/scrape: 'true'
+  labels:
+    app: prometheus
+    app.kubernetes.io/managed-by: Helm
+    chart: prometheus-11.12.1
+    component: server
+spec:
+  ports:
+  - name: metrics
+    port: 8080
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    app: prometheus
+    component: server
+  additionalScrapeConfigs:
+    name: prometheus-config
+    key: prometheus-config.yaml
+    job_name: 'prometheus-slurm-exporter'
+    scrape_interval: 15s
+    static_configs:
+      - targets:
+        - http:"{{ inventory_hostname }}":8080/metrics
+  serviceMonitorSelector: {}

roles/slurm_exporter/tasks/configure_prometheus_pod.yml

@@ -0,0 +1,18 @@
+# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Apply slurm exporter configuration to prometheus
+  command: kubectl apply -f "{{ role_path }}/files/{{ slurm_config_file }}" --validate=false
+  changed_when: False

roles/slurm_exporter/tasks/install_prometheus.yml

@@ -0,0 +1,40 @@
+#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Download and untar prometheus stable version
+  unarchive:
+    src: "{{ prometheus_git_repo }}"
+    dest: "{{ installation_dir }}"
+    remote_src: yes
+
+- name: Copy prometheus executable to /usr/local/bin
+  copy:
+    src: "{{ prometheus_exec_path }}"
+    dest: "{{ system_local_path }}"
+    remote_src: yes
+    mode: "{{ file_permission }}"
+
+- name: Configure prometheus for slurm exporter
+  blockinfile:
+    path: "{{ prometheus_config_file }}"
+    insertafter: EOF
+    mode: "{{ file_permission }}"
+    block: |
+      # SLURM resource manager:
+        - job_name: 'my_slurm_exporter'
+          scrape_interval:  30s
+          scrape_timeout:   30s
+          static_configs:
+            - targets: ['localhost:8080']

roles/slurm_exporter/tasks/install_slurm_exporter.yml

@@ -0,0 +1,65 @@
+#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Download and untar go package
+  unarchive:
+    src: "{{ go_pack_url }}"
+    dest: "{{ installation_dir }}"
+    remote_src: yes
+
+- name: Add to the linux path
+  shell: echo $PATH
+  environment:
+    PATH: "{{ extended_path }}:{{ ansible_env.PATH }}"
+  changed_when: False
+
+- name: Clone the source code
+  git:
+    repo: "{{ slurm_exporter_git_repo }}"
+    dest: "{{ slurm_exporter_inst_dir }}"
+    version: "master"
+
+- name: export GOPATH
+  shell: echo $GOPATH
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/var/lib/go/bin/"
+    GOPATH: "{{ go_modules_path }}"
+  changed_when: False
+
+- name: Download dependencies
+  command: "{{ go_exec_path }} mod download"
+  args:
+    chdir: "{{ slurm_exporter_inst_dir }}"
+  changed_when: False
+
+- name: Build the exporter
+  shell: "{{ go_exec_path }} build -o bin/prometheus-slurm-exporter {main,accounts,cpus,nodes,partitions,queue,scheduler,users}.go"
+  args:
+    chdir: "{{ slurm_exporter_inst_dir }}"
+  changed_when: False
+
+- name: Run all tests included in _test.go files
+  shell: "{{ go_exec_path }} test -v *.go"
+  args:
+    chdir: "{{ slurm_exporter_inst_dir }}"
+  changed_when: False
+  ignore_errors: yes
+
+- name: Copy executable to /usr/bin
+  copy:
+    src: "{{ slurm_exporter_exec }}"
+    dest: "{{ system_path }}"
+    remote_src: yes
+    mode: "{{ file_permission }}"

roles/slurm_exporter/tasks/main.yml

@@ -0,0 +1,28 @@
+# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Install slurm exporter
+  include_tasks: install_slurm_exporter.yml
+
+- name: Start slurm exporter services
+  include_tasks: start_services.yml
+
+- name: Install prometheus on host
+  include_tasks: install_prometheus.yml
+  when: "'kubernetes' in ansible_skip_tags"
+
+- name: Apply slurm exporter config to prometheus pod
+  include_tasks: configure_prometheus_pod.yml
+  tags: kubernetes

roles/slurm_exporter/tasks/start_services.yml

@@ -0,0 +1,26 @@
+# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Create systemd unit file
+  copy:
+    src: "{{ role_path }}/files/prometheus-slurm-exporter.service"
+    dest: "{{ systemd_path_dest }}"
+    remote_src: no
+    mode: "{{ file_permission }}"
+
+- name: Start services
+  systemd:
+    name: prometheus-slurm-exporter
+    state: started

roles/slurm_exporter/vars/main.yml

@@ -0,0 +1,39 @@
+# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# Usage: install_slurm_exporter.yml
+installation_dir: "/var/lib/"
+slurm_exporter_inst_dir: "/var/lib/slurm-exporter"
+go_pack_url: "https://dl.google.com/go/go1.15.linux-amd64.tar.gz"
+extended_path: "{{ installation_dir }}/go/bin"
+go_exec_path: "{{ installation_dir }}/go/bin/go"
+slurm_exporter_git_repo: "https://github.com/vpenso/prometheus-slurm-exporter.git"
+go_modules_path: "{{ slurm_exporter_inst_dir }}/go/modules"
+slurm_exporter_exec: "{{ slurm_exporter_inst_dir }}/bin/prometheus-slurm-exporter"
+system_path: "/usr/bin"
+
+#Usage: install_prometheus.yml
+prometheus_git_repo: "https://github.com/prometheus/prometheus/releases/download/v2.23.0/prometheus-2.23.0.linux-amd64.tar.gz"
+prometheus_inst_path: "/var/lib/prometheus-2.23.0.linux-amd64/"
+prometheus_exec_path: "{{ prometheus_inst_path }}/prometheus"
+system_local_path: "/usr/local/bin"
+prometheus_config_file: "{{ prometheus_inst_path }}/prometheus.yml"
+
+#Usage: start_service.yml
+file_permission: "0755"
+systemd_path_dest: "/etc/systemd/system/"
+
+#Usage: configure_prometheus_pod.yml
+slurm_config_file: "slurm_exporter_config.yaml"

test/test_slurm_exporter_inst_host.yml

@@ -0,0 +1,56 @@
+# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+# OMNIA_PSE_TC_002
+# Install prometheus on host when kubernetes is not installed
+- name: OMNIA_PSE_TC_002
+  hosts: manager
+  vars_files:
+    - test_vars/test_slurmexporter_vars.yml
+  tasks:
+    - block:
+        - name: Call install slurm exporter role
+          include_role:
+            name: ../roles/slurm_exporter
+      tags: TC_002
+
+    - name: Verify slurm exporter status
+      systemd:
+        name: prometheus-slurm-exporter
+      register: slurm_exporter_status
+      tags: TC_002, VERIFY_002
+
+    - name: Validate slurm exporter service status
+      assert:
+        that:
+          - slurm_exporter_status.status.ActiveState == 'active'
+        fail_msg: "{{ slurm_exporter_service_fail_msg }}"
+        success_msg: "{{ slurm_exporter_service_success_msg }}"
+      tags: TC_002, VERIFY_002
+
+    - name: Verify prometheus installation status
+      command: prometheus --version
+      register: prometheus_status
+      tags: TC_002, VERIFY_002
+      ignore_errors: yes
+      changed_when: False
+
+    - name: Validate prometheus version command
+      assert:
+        that:
+          - "'Command not found' not in prometheus_status.stdout"
+        fail_msg: "{{ prometheus_installation_fail_msg }}"
+        success_msg: "{{ prometheus_installation_success_msg }}"
+      tags: TC_002, VERIFY_002

test/test_slurm_exporter_inst_k8s.yml

@@ -0,0 +1,55 @@
+# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+# OMNIA_PSE_TC_001
+# Successful installation of slurm exporter on the host when both slurm and kubernetes is installed
+- name: OMNIA_PSE_TC_001
+  hosts: manager
+  vars_files:
+    - test_vars/test_slurmexporter_vars.yml
+  tasks:
+    - block:
+        - name: Call install slurm exporter role
+          include_role:
+            name: ../roles/slurm_exporter
+      tags: TC_001
+
+    - name: Verify slurm exporter status
+      systemd:
+        name: prometheus-slurm-exporter
+      register: slurm_exporter_status
+      tags: TC_001, VERIFY_001
+
+    - name: Validate slurm exporter service status
+      assert:
+        that:
+          - slurm_exporter_status.status.ActiveState == 'active'
+        fail_msg: "{{ slurm_exporter_service_fail_msg }}"
+        success_msg: "{{ slurm_exporter_service_success_msg }}"
+      tags: TC_001, VERIFY_001
+
+    - name: Verify slurm exporter job in k8s services
+      command: kubectl get service prometheus-slurmexporter-metrics-1
+      register: slurm_exporter_service_status
+      tags: TC_001, VERIFY_001
+      changed_when: False
+
+    - name: Validate slurm exporter job in k8s services
+      assert:
+        that:
+          - "'Error from server' not in slurm_exporter_service_status.stdout"
+        fail_msg: "{{ slurm_exporter_job_fail_msg }}"
+        success_msg: "{{ slurm_exporter_job_success_msg }}"
+      tags: TC_001, VERIFY_001

test/test_vars/test_slurm_common_vars.yml

@@ -27,6 +27,5 @@ common_packages:
    - munge-devel
    - mariadb-server
    - mariadb-devel
-   - python3
    - man2html
    - MySQL-python

test/test_vars/test_slurmexporter_vars.yml

@@ -0,0 +1,26 @@
+#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+slurm_exporter_service_fail_msg: "Slurm exporter service is not running"
+
+slurm_exporter_service_success_msg: "Slurm exporter service is active and running"
+
+slurm_exporter_job_fail_msg: "Slurm-exporter-metrics not configured as k8s service"
+
+slurm_exporter_job_success_msg: "Slurm-exporter-metrics successfully configured as k8s service"
+
+prometheus_installation_fail_msg: "Prometheus not installed"
+
+prometheus_installation_success_msg: "Prometheus is installed"