Dell-Omnia-Clusters-HPC-AI/test/test_compute_gpu.yml

#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
---

# OMNIA_UKP_US_VFKP_TC_003
# Execute compute_gpu role in compute nodes with os installed centos 7.8
- name: OMNIA_UKP_US_VFKP_TC_003
  hosts: gpus
  vars_files:
    - test_vars/test_compute_gpu_vars.yml
  tasks:
    - block:
        - name: Call compute_gpu role
          include_role:
            name: ../roles/compute_gpu
      tags: TC_003

    - name: Checking Nvidia drivers installation
      command: whereis nvidia
      register: nvidia_drivers
      changed_when: false
      ignore_errors: True
      tags: TC_003, VERIFY_003

    - name: Checking nvidia-docker2 installation status
      command: nvidia-docker version
      register: nvidia_docker_version
      changed_when: false
      ignore_errors: True
      tags: TC_003, VERIFY_003

    - name: Checking docker service status
      systemd:
        name: docker
      register: docker_service
      tags: TC_003, VERIFY_003

    - name: Checking K8s service status
      systemd:
        name: kubelet
      register: kubelet_service
      tags: TC_003, VERIFY_003

    - name: Validating Nvidia drivers
      assert:
        that:
          - "'/usr/lib/nvidia' in nvidia_drivers.stdout"
        fail_msg: "{{ nvidia_drivers_fail_msg }}"
        success_msg: "{{ nvidia_drivers_success_msg }}"
      tags: TC_003, VERIFY_003

    - name: Validating nvidia-docker2 installation
      assert:
        that:
          - "'command not found' not in nvidia_docker_version.stdout"
        fail_msg: "{{ nvidia_docker_fail_msg }}"
        success_msg: "{{ nvidia_docker_success_msg }}"
      tags: TC_003, VERIFY_003

    - name: Validating docker service status
      assert:
        that:
          - docker_service.status.ActiveState == 'active'
        fail_msg: "{{ docker_service_fail_msg }}"
        success_msg: "{{ docker_service_success_msg }}"
      tags: TC_003, VERIFY_003

    - name: Validating K8s service status
      assert:
        that:
          - kubelet_service.status.ActiveState == 'active'
        fail_msg: "{{ kubelet_service_fail_msg }}"
        success_msg: "{{ kubelet_service_success_msg }}"
      tags: TC_003, VERIFY_003

# OMNIA_UKP_US_VFKP_TC_004
# Execute compute_gpu role in compute nodes with NVIDIA kmod/docker drivers already installed
- name: OMNIA_UKP_US_VFKP_TC_004
  hosts: gpus
  vars_files:
    - test_vars/test_compute_gpu_vars.yml
  tasks:
    - block:
        - name: Call compute_gpu role
          include_role:
            name: ../roles/compute_gpu
      tags: TC_004

    - name: Checking Nvidia drivers installation
      command: whereis nvidia
      register: nvidia_drivers
      changed_when: false
      ignore_errors: True
      tags: TC_004, VERIFY_004

    - name: Checking nvidia-docker2 installation status
      command: nvidia-docker version
      register: nvidia_docker_version
      changed_when: false
      ignore_errors: True
      tags: TC_004, VERIFY_004

    - name: Checking docker service status
      systemd:
        name: docker
      register: docker_service
      tags: TC_004, VERIFY_0042

    - name: Checking K8s service status
      systemd:
        name: kubelet
      register: kubelet_service
      tags: TC_004, VERIFY_004

    - name: Validating Nvidia drivers
      assert:
        that:
          - "'/usr/lib/nvidia' in nvidia_drivers.stdout"
        fail_msg: "{{ nvidia_drivers_fail_msg }}"
        success_msg: "{{ nvidia_drivers_success_msg }}"
      tags: TC_004, VERIFY_004

    - name: Validating nvidia-docker2 installation
      assert:
        that:
          - "'command not found' not in nvidia_docker_version.stdout"
        fail_msg: "{{ nvidia_docker_fail_msg }}"
        success_msg: "{{ nvidia_docker_success_msg }}"
      tags: TC_004, VERIFY_004

    - name: Validating docker service status
      assert:
        that:
          - docker_service.status.ActiveState == 'active'
        fail_msg: "{{ docker_service_fail_msg }}"
        success_msg: "{{ docker_service_success_msg }}"
      tags: TC_004, VERIFY_004

    - name: Validating K8s service status
      assert:
        that:
          - kubelet_service.status.ActiveState == 'active'
        fail_msg: "{{ kubelet_service_fail_msg }}"
        success_msg: "{{ kubelet_service_success_msg }}"
      tags: TC_004, VERIFY_004