Merge branch 'devel' into infiniband

control_plane/input_params/idrac_tools_vars.yml

@@ -35,18 +35,21 @@ ipv4_static_dns2: ""
 # Mandatory value required
 smtp_server_ip: ""
 
-# Username used for SMTP
+# Email address used for enabling 2FA
+# Mandatory value required
+use_email_address_2fa: ""
+
+# SMTP authentication disabled by default
+# If enabled provide smtp username and password
 # Mandatory value required
+smtp_authentication: "disabled"
+
+# Username used for SMTP
 smtp_username: ""
    
 # Password used for SMTP
-# Mandatory value required    
 smtp_password: ""
 
-# Email address used for enabling 2FA
-# Mandatory value required
-use_email_address_2fa: ""
-
 
 ### Usage: idrac_ldap ###
 
@@ -108,4 +111,4 @@ role_group1_dn: ""
 # Supported options are Administrator, Operator, ReadOnly
 # By default role_group1_privilege will be Administrator
 # Mandatory value required
-role_group1_privilege: "Administrator"
+role_group1_privilege: "Administrator"

control_plane/input_params/idrac_vars.yml

@@ -23,9 +23,7 @@ idrac_system_profile: "Performance"
 # Boolean value indicating whether OMNIA should perform firmware update or not
 # It takes values "true" or "false" indicating required and not required cases respectively.
 # Default value is "true"
-# firmware_update_required should be 'false' now as there is bug in DSU & OMAM modules and firmware updates dependent on that.
-# It will be updated to 'true' once DSU and OMAM fix the bugs
-firmware_update_required: false
+firmware_update_required: true
 
 # This is the list of poweredge server models
 # The firmware updates will be downloaded only for the below list of models
@@ -71,4 +69,4 @@ two_factor_authentication: "disabled"
 # If required it can be "enabled"
 # Update 2FA input parameters in idrac_tools_vars.yml if two_factor_authentication is enabled
 # Command to edit idrac_tools_vars.yml: ansible-vault edit idrac_tools_vars.yml --vault-password-file .idrac_vault_key
-ldap_directory_services: "disabled"
+ldap_directory_services: "disabled"

control_plane/roles/deploy_job_templates/tasks/group_inventory.yml

@@ -0,0 +1,32 @@
+# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Get the hosts in node_inventory
+  command: >-
+    awx --conf.host {{ awx_host }} --conf.username {{ awx_admin_user }} --conf.password {{ awx_password }}
+    --conf.insecure hosts list --inventory {{ node_inventory }} -f human --filter "name"
+  changed_when: false
+  no_log: true
+  register: hosts_list
+
+- name: Add the host to the group in node_inventory if present
+  awx.awx.tower_group:
+    name: "{{ item.split(',')[3] }}"
+    inventory: "{{ node_inventory }}"
+    hosts:
+      - "{{ item.split(',')[2] }}"
+    tower_config_file: "{{ tower_config_file }}"
+  when:
+    - item.split(',')[2] != "IP"
+    - item.split(',')[2] in hosts_list.stdout

control_plane/roles/deploy_job_templates/tasks/main.yml

@@ -0,0 +1,232 @@
+# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Check if {{ tower_config_file }} file is encrypted
+  command: cat {{ tower_config_file }}
+  changed_when: false
+  no_log: true
+  register: config_content
+
+- name: Decrpyt {{ tower_config_file }}
+  command: >-
+    ansible-vault decrypt {{ tower_config_file }}
+    --vault-password-file {{ tower_vault_file }}
+  when: "'$ANSIBLE_VAULT;' in config_content.stdout"
+  changed_when: false
+
+- name: Change file permissions
+  file:
+    path: "{{ tower_config_file }}"
+    mode: "{{ file_perm }}"
+
+- name: Fetch awx host
+  command: grep "host:" "{{ tower_config_file }}"
+  changed_when: false
+  register: fetch_awx_host
+
+- name: Fetch awx password
+  command: grep "password:" "{{ tower_config_file }}"
+  changed_when: false
+  no_log: true
+  register: fetch_awx_password
+
+- name: Set awx variables
+  set_fact:
+    awx_host: "{{ fetch_awx_host.stdout | regex_replace('host: ','') }}"
+    awx_password: "{{ fetch_awx_password.stdout | regex_replace('password: ','') }}"
+  no_log: true
+
+- name: Launch dynamic inventory
+  block:
+    - name: Launch device inventory job template
+      awx.awx.tower_job_launch:
+        job_template: "{{ device_inventory_template }}"
+        tower_config_file: "{{ tower_config_file }}"
+        wait: yes
+        timeout: "{{ awx_max_wait_time }}"
+      register: inventory_job_status
+  rescue:
+    - name: Restart awx pod
+      command: kubectl rollout restart deployment awx -n awx
+      changed_when: false
+      when:
+        - inventory_job_status.status is defined
+        - '"pending" in inventory_job_status.status'
+
+    - name: Wait for the awx pod to be up and running
+      wait_for:
+        timeout: "{{ pod_restart_time }}"
+      when:
+        - inventory_job_status.status is defined
+        - '"pending" in inventory_job_status.status'
+
+    - name: Launch device inventory job template
+      awx.awx.tower_job_launch:
+        job_template: "{{ device_inventory_template }}"
+        tower_config_file: "{{ tower_config_file }}"
+        wait: yes
+        timeout: "{{ awx_max_wait_time }}"
+      when:
+        - inventory_job_status.status is defined
+        - '"pending" in inventory_job_status.status'
+
+    - name: Warning message for device inventory template
+      debug:
+        msg: "{{ device_inventory_template_warn_msg }}"
+      when:
+        - inventory_job_status.status is defined
+        - '"pending" not in inventory_job_status.status'
+
+- name: Configure TOR Switches
+  block:
+    - name: Launch ethernet job template for TOR switches
+      awx.awx.tower_job_launch:
+        job_template: "{{ ethernet_job_template }}"
+        tower_config_file: "{{ tower_config_file }}"
+        wait: yes
+        timeout: "{{ awx_max_wait_time }}"
+      register: ethernet_job_status
+  rescue:
+    - name: Warning message for ethernet template
+      debug:
+        msg: "{{ ethernet_template_warn_msg }}"
+  when: ethernet_switch_support
+
+- name: Wait for 15 mins for DHCP to assign IP to devices
+  wait_for:
+    timeout: "{{ dhcp_wait_time }}"
+
+- name: Launch device inventory job template
+  awx.awx.tower_job_launch:
+    job_template: "{{ device_inventory_template }}"
+    tower_config_file: "{{ tower_config_file }}"
+    wait: yes
+    timeout: "{{ awx_max_wait_time }}"
+  register: inventory_job_status
+
+- name: Execute ethernet template
+  block:
+    - name: Launch ethernet job template for all switches
+      awx.awx.tower_job_launch:
+        job_template: "{{ ethernet_job_template }}"
+        tower_config_file: "{{ tower_config_file }}"
+        wait: yes
+        timeout: "{{ awx_max_wait_time }}"
+      register: ethernet_job_status
+  rescue:
+    - name: Warning message for ethernet template
+      debug:
+        msg: "{{ ethernet_template_warn_msg }}"
+  when: ethernet_switch_support
+
+- name: Execute infiniband template
+  block:
+    - name: Launch infiniband job template
+      awx.awx.tower_job_launch:
+        job_template: "{{ infiniband_job_template }}"
+        tower_config_file: "{{ tower_config_file }}"
+        wait: yes
+        timeout: "{{ awx_max_wait_time }}"
+      register: ib_job_status
+  rescue:
+    - name: Warning message for infiniband template
+      debug:
+        msg: "{{ infiniband_template_warn_msg }}"
+  when: ib_switch_support
+
+- name: Execute powervault_me4 template
+  block:
+    - name: Launch powervault_me4 job template
+      awx.awx.tower_job_launch:
+        job_template: "{{ powervault_me4_job_template }}"
+        tower_config_file: "{{ tower_config_file }}"
+        wait: yes
+        timeout: "{{ awx_max_wait_time }}"
+      register: powervault_job_status
+  rescue:
+    - name: Warning message for powervault_me4 template
+      debug:
+        msg: "{{ powervault_template_warn_msg }}"
+  when: powervault_support
+
+- name: Execute idrac template
+  block:
+    - name: Launch idrac job template
+      awx.awx.tower_job_launch:
+        job_template: "{{ idrac_job_template }}"
+        tower_config_file: "{{ tower_config_file }}"
+        wait: yes
+      register: idrac_job_status
+  rescue:
+    - name: Warning message for idrac template
+      debug:
+        msg: "{{ idrac_template_warn_msg }}"
+
+- name: Wait for 30 mins for idrac provisioning to be completed and inventory to be updated in AWX
+  wait_for:
+    timeout: "{{ provisioning_wait_time }}"
+  when: host_mapping_file
+
+- name: Check the host_mapping_file_path output
+  command: cat {{ host_mapping_file_path }}
+  changed_when: false
+  register: mapping_file
+  when: host_mapping_file
+
+- name: Group the hosts in node_inventory when mapping file is present
+  include_tasks: "{{ role_path }}/tasks/group_inventory.yml"
+  with_items: "{{ mapping_file.stdout_lines }}"
+  when: host_mapping_file and component_role_support
+
+- name: Launch deploy_omnia job template
+  awx.awx.tower_job_launch:
+    job_template: "{{ component_role_job_template }}"
+    tower_config_file: "{{ tower_config_file }}"
+    wait: yes
+  register: component_role_job_status
+  when: host_mapping_file and component_role_support
+
+- name: Create awx job template for configuring new devices
+  awx.awx.tower_job_template:
+    name: "{{ item.name }}"
+    job_type: "run"
+    organization: "{{ awx_organization }}"
+    inventory: "{{ item.inventory }}"
+    project: "{{ project_name }}"
+    playbook: "{{ item.playbook }}"
+    credentials:
+     - "{{ item.credential }}"
+    state: present
+    tower_config_file: "{{ tower_config_file }}"
+  loop: "{{ job_template_details }}"
+
+- name: Build a schedule for configure new devices
+  awx.awx.tower_schedule:
+    name: "{{ item.name }}"
+    unified_job_template: "{{ item.template }}"
+    rrule: "{{ item.rrule }}"
+    state: present
+    tower_config_file: "{{ tower_config_file }}"
+  loop: "{{ scheduled_template }}"
+
+- name: Encrypt {{ tower_config_file }}
+  command: >-
+    ansible-vault encrypt {{ tower_config_file }}
+    --vault-password-file {{ tower_vault_file }}
+  changed_when: false
+
+- name: Change file permissions
+  file:
+    path: "{{ tower_config_file }}"
+    mode: "{{ file_perm }}"

control_plane/roles/deploy_job_templates/vars/main.yml

@@ -0,0 +1,44 @@
+# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+base_vars_file: "{{ role_path }}/../../input_params/base_vars.yml"
+awx_namespace: awx
+awx_admin_user: admin
+awx_port: 8052
+tower_config_file: "{{ role_path }}/../../roles/webui_awx/files/.tower_cli.cfg"
+tower_vault_file: "{{ role_path }}/../../roles/webui_awx/files/.tower_vault_key"
+ethernet_inventory: "ethernet_inventory"
+node_inventory: "node_inventory"
+device_inventory_template: "device_inventory_job"
+idrac_job_template: "idrac_template"
+ethernet_job_template: "ethernet_template"
+infiniband_job_template: "infiniband_template"
+powervault_me4_job_template: "powervault_me4_template"
+component_role_job_template: "deploy_omnia_template"
+job_execution_success_msg: "Job execution is successful"
+job_execution_failure_msg: "Job execution is failed!"
+device_inventory_template_warn_msg: "Device inventory template execution didn't exit successfully. Detailed output can be viewed on AWX UI"
+ethernet_template_warn_msg: "Ethernet template execution didn't exit successfully. Detailed output can be viewed on AWX UI"
+infiniband_template_warn_msg: "Infiniband template execution didn't exit successfully. Detailed output can be viewed on AWX UI"
+powervault_template_warn_msg: "Powervault_me4 template execution didn't exit successfully. Detailed output can be viewed on AWX UI"
+idrac_template_warn_msg: "idrac template execution didn't exit successfully. Detailed output can be viewed on AWX UI"
+provisioning_wait_time: 1800
+dhcp_wait_time: 900
+awx_max_wait_time: 900
+pod_restart_time: 90
+project_name: 'omnia'
+job_template_details:
+  - { name: configure_new_devices_job, inventory: idrac_inventory, playbook: control_plane/tools/configure_new_devices.yml, credential: idrac_credential }
+scheduled_template:
+  - { name: ConfigureNewDeviceSchedule, template: configure_new_devices_job, rrule: "DTSTART:20210729T063000Z RRULE:FREQ=DAILY;INTERVAL=1" }

control_plane/roles/webui_awx/files/awx_ee.yml

@@ -5,14 +5,16 @@ USER root
 # add Ansible galaxy dependencies
 ADD requirements.yml /tmp/requirements.yml
 
-# install omsdk
-RUN pip install omsdk --upgrade
+# install packages
+RUN pip install --upgrade \
+omsdk \
+netaddr
 
 # install Ansible Galaxy collections
 RUN ansible-galaxy collection install -r /tmp/requirements.yml --collections-path /usr/share/ansible/collections
 
 # add certificates
 RUN update-ca-trust force-enable
-RUN chmod -R 0777 /usr/share/ansible/collections
+RUN chmod -R 0777 /usr/share/ansible/collections 
 
 USER 1000

control_plane/roles/webui_awx/tasks/awx_configuration.yml

@@ -12,6 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
+- name: Check if {{ tower_config_file }} file is encrypted
+  command: cat {{ tower_config_file }}
+  changed_when: false
+  no_log: true
+  register: config_content
+  run_once: true
+
+- name: Decrpyt {{ tower_config_file }}
+  command: >-
+    ansible-vault decrypt {{ tower_config_file }}
+    --vault-password-file {{ tower_vault_file }}
+  when: "'$ANSIBLE_VAULT;' in config_content.stdout"
+  changed_when: false
+  run_once: true
 
 # Deleting the defaults
 - name: Delete machine credential
@@ -19,25 +33,29 @@
     name: "{{ default_credential }}"
     credential_type: "{{ default_credential_type }}"
     state: absent
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
+  register: delete_credential
+  until: not delete_credential.failed
+  retries: "{{ max_retries }}"
+  delay: "{{ max_delay }}"
 
 - name: Delete job template
   awx.awx.tower_job_template:
     name: "{{ default_template }}"
     state: absent
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
     
 - name: Delete project
   awx.awx.tower_project:
     name: "{{ default_project }}"
     state: absent
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
 
 - name: Delete organization
   awx.awx.tower_organization:
     name: "{{ default_org }}"
     state: absent
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
 
 # Configuration begins
 - name: Create organization
@@ -45,7 +63,7 @@
     name: "{{ awx_organization }}"
     description: "Name of organization using this product"
     state: present
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
 
 - name: Create awx inventories
   awx.awx.tower_inventory:
@@ -53,7 +71,7 @@
     description: "{{ item.description }}"
     organization: "{{ awx_organization }}"
     state: present
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
   loop: "{{ inventory_names }}"
   when: item.flag
 
@@ -63,7 +81,7 @@
     description: "{{ item.description }}"
     inventory: "node_inventory"
     state: present
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
   loop: "{{ group_names }}"
 
 - name: Add project
@@ -73,9 +91,9 @@
     organization: "{{ awx_organization }}"
     scm_type: manual
     local_path: "{{ role_path.split('/')[-4] }}"
-    default_environment: "custom-awx-ee"
+    default_environment: custom-awx-ee
     state: present
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
 
 - name: Add awx credentials
   awx.awx.tower_credential:
@@ -86,12 +104,13 @@
       username: "{{ item.username }}"
       password: "{{ item.password }}"
     state: present
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
   loop: "{{ credential_details }}"
+  no_log: true
   changed_when: true
   when: item.flag
 
-- name: Create awx job templates
+- name: Create awx job templates for network devices, inventories, storage and idrac
   awx.awx.tower_job_template:
     name: "{{ item.name }}"
     job_type: "run"
@@ -102,11 +121,11 @@
     credentials:
       - "{{ item.credential }}"
     state: present
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
   loop: "{{ job_template_details }}"
   when: item.flag
 
-- name: Create deploy_omnia_template
+- name: Create awx job template for deploying omnia
   awx.awx.tower_job_template:
     name: "{{ item.name }}"
     job_type: "run"
@@ -114,19 +133,30 @@
     inventory: "{{ item.inventory }}"
     project: "{{ project_name }}"
     playbook: "{{ item.playbook }}"
+    ask_skip_tags_on_launch: true
     credentials:
       - "{{ item.credential }}"
-    ask_skip_tags_on_launch: true
     state: present
-    tower_config_file: "~/.tower_cli.cfg"
-  loop: "{{ deploy_omnia_details }}"
+    tower_config_file: "{{ tower_config_file }}"
+  loop: "{{ omnia_job_template_details }}"
 
 - name: Build a schedule for idrac job template
   awx.awx.tower_schedule:
     name: "{{ item.name }}"
     unified_job_template: "{{ item.template }}"
-    rrule: "{{ schedule_rule }}"
+    rrule: "{{ item.schedule_rule }}"
     state: present
-    tower_config_file: "~/.tower_cli.cfg"
+    tower_config_file: "{{ tower_config_file }}"
   register: result
-  loop: "{{ scheduled_templates}}"
+  loop: "{{ scheduled_templates }}"
+
+- name: Encrypt {{ tower_config_file }}
+  command: >-
+    ansible-vault encrypt {{ tower_config_file }}
+    --vault-password-file {{ tower_vault_file }}
+  changed_when: false
+
+- name: Change file permissions
+  file:
+    path: "{{ tower_config_file }}"
+    mode: "{{ file_perm }}"

control_plane/roles/webui_awx/tasks/configure_settings.yml

@@ -12,60 +12,125 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
-
 # Get Current AWX configuration
 
+- name: Get the awx services
+  command: "kubectl get svc -n {{ awx_namespace }}"
+  changed_when: false
+  register: awx_services
+  
+- name: Expose the service for awx deployment on 8052 port
+  command: "kubectl expose deployment awx --type=NodePort --name={{ awx_service_name }} --port={{ awx_port }} -n {{ awx_namespace }}"
+  changed_when: false
+  when: awx_service_name not in awx_services.stdout
+
 - name: Get awx-service Cluster-IP
-  command: "kubectl get svc awx-service -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+  command: "kubectl get svc {{ awx_service_name }} -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
   register: awx_cluster_ip
   changed_when: false
 
 - name: Get AWX admin password
-  shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+  shell: >
+    set -o pipefail && \
+    kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode
   register: awx_admin_password
   changed_when: false
 
 - name: Check if config file exists
   stat:
-    path: "~/.tower_cli.cfg"
-  register: config_file_status
+    path: "{{ tower_config_file }}"
+  register: config_file
 
-- name: Create config file
+- name: Create tower config file
   copy:
-    dest:  "~/.tower_cli.cfg"
+    dest:  "{{ tower_config_file }}"
     content: |
       [general]
-      host: http://{{ awx_cluster_ip.stdout }}
+      host: http://{{ awx_cluster_ip.stdout }}:{{ awx_port }}
       username: admin
       password: {{ awx_admin_password.stdout }}
       verify_ssl: false
       use_token: false
     owner: root
     mode: "{{ file_perm }}"
+  when: not config_file.stat.exists
+
+- name: Check if tower_vault_key exists
+  stat:
+    path: "{{ tower_vault_file }}"
+  register: tower_vault
+
+- name: Create ansible vault key if it does not exist
+  set_fact:
+    tower_vault_key: "{{ lookup('password', '/dev/null chars=ascii_letters') }}"
+  when: not tower_vault.stat.exists
+
+- name: Save vault key
+  copy:
+    dest: "{{ tower_vault_file }}"
+    content: |
+      {{ tower_vault_key }}
+    owner: root
+    force: yes
+    mode: "{{ vault_file_perm }}"
+  when: not tower_vault.stat.exists
+
+- name: Check if {{ tower_config_file }} file is encrypted
+  command: cat {{ tower_config_file }}
+  changed_when: false
+  no_log: true
+  register: config_content
+  run_once: true
+
+- name: Encrypt {{ tower_config_file }}
+  command: >-
+    ansible-vault encrypt {{ tower_config_file }}
+    --vault-password-file {{ tower_vault_file }}
+  changed_when: false
+  when: "'$ANSIBLE_VAULT;' not in config_content.stdout"
+  run_once: true
+
+- name: Change file permissions
+  file:
+    path: "{{ tower_config_file }}"
+    mode: "{{ file_perm }}"
+
+- name: Open awx TCP ports on the firewall
+  firewalld:
+    port: "{{ item }}/tcp"
+    permanent: yes
+    state: enabled
+  with_items: "{{ awx_tcp_ports }}"
+
+- name: Masquerade the firewall
+  firewalld:
+    masquerade: yes
+    permanent: yes
+    state: enabled
+    zone: public
 
-- name: Stop and disable firewalld
-  service:
+- name: Reload firewalld service
+  systemd:
     name: firewalld
-    state: stopped
-    enabled: no
+    state: reloaded
 
 - name: Waiting for the AWX UI to be up
   uri:
-    url: "http://{{ awx_cluster_ip.stdout }}"
+    url: "http://{{ awx_cluster_ip.stdout }}:{{ awx_port }}"
     status_code: "{{ return_status }}"
   register: display
   until: display.status == 200
-  retries: 20
-  delay: 15
+  retries: "{{ max_retries }}"
+  delay: "{{ max_delay }}"
   changed_when: false
 
 - name: Waiting for the AWX UI to be in running state
   uri:
-    url: "http://{{ awx_cluster_ip.stdout }}"
+    url: "http://{{ awx_cluster_ip.stdout }}:{{ awx_port }}"
     status_code: "{{ return_status }}"
     return_content: true
   register: display
   until: awx_ui_msg not in display.content
-  retries: 20
-  delay: 15
+  retries: "{{ max_retries }}"
+  delay: "{{ max_delay }}"
   changed_when: false

control_plane/roles/webui_awx/tasks/install_awx.yml

@@ -62,13 +62,13 @@
   changed_when: false
   register: docker_images
 
-- name: Build the custom-awx-ee image from the docker file
+- name: Build the custom-awx-ee image from the docker file (It may take 5-10min)
   command: "buildah bud -t custom-awx-ee {{ awx_ee_docker_file }}"
   changed_when: false
   when: "'custom-awx-ee' not in docker_images.stdout"
 
 - name: Waiting for awx-operator deployment to be up and running
-  shell: "kubectl wait --for=condition=available deployment/awx-operator -n {{ awx_namespace }} --timeout=600s"
+  command: kubectl wait --for=condition=available deployment awx-operator -n {{ awx_namespace }} --timeout=600s
   changed_when: false
 
 - name: Deploy awx
@@ -91,6 +91,13 @@
     timeout: "{{ awx_wait_time }}"
   when: not k8s_pods.stdout | regex_search('awx-([A-Za-z0-9]{10})-([A-Za-z0-9]{5})')
 
-- name: Waiting for awx deployment to be up and running
-  shell: "kubectl wait --for=condition=available deployment/awx -n {{ awx_namespace }} --timeout=1200s"
-  changed_when: false
+- name: Fails if the pods go into ImagePullBackOff state
+  block:
+    - name: Waiting for awx deployment to be up and running
+      command: kubectl wait --for=condition=available deployment awx -n {{ awx_namespace }} --timeout=1200s
+      changed_when: false
+
+  rescue:
+    - name: Display failure message
+      debug:
+        msg: "{{ deployment_failure_msg }}"

control_plane/roles/webui_awx/vars/main.yml

@@ -18,19 +18,33 @@
 awx_namespace: awx
 awx_pv_yml_file_path: "{{ role_path }}/files/awx_projects_pv.yml"
 awx_postgres_pv_file_path: "{{ role_path }}/files/awx_postgres_pv.yml"
-awx_pvc_yml_file_path: "{{ role_path }}/files/awx_projects_pvclaim.yml"
 awx_operator_yml_file_path: "{{ role_path }}/files/awx_operator.yml"
 awx_ee_docker_file: "{{ role_path }}/files/awx_ee.yml"
 awx_yml_file_path: "{{ role_path }}/files/awx.yml"
 postgres_storage_class: nfs-client
 projects_existing_claim: awx-pv-claim
 awx_version: 19.1.0
-file_perm: 644
+file_perm: '0644'
 awx_wait_time: 90
+deployment_failure_msg: "Deployment failed due to docker pull rate limit"
 
 # Usage: configure_settings.yml
+awx_port: 8052
+awx_service_name: awx-ui
+tower_config_file: "{{ role_path }}/files/.tower_cli.cfg"
+tower_vault_file: "{{ role_path }}/files/.tower_vault_key"
+vault_file_perm: '0644'
+awx_tcp_ports:
+  - 80
+  - 8383
+  - 8686
+  - 5432
+  - 8052
+  - 30000-32767
 awx_ui_msg: "AWX Upgrading"
 return_status: 200
+max_retries: 20
+max_delay: 15
 
 # Usage: awx_configuration.yml
 default_org: Default
@@ -50,14 +64,14 @@ inventory_names:
 group_names:
   - { name: manager, description: "Group to store IP of head node" }
   - { name: compute, description: "Group to store IPs of compute nodes" }
-  - { name: login, description: "Group to store IP of login node" }
-  - { name: nfs, description: "Group to store IP of NFS node" }
+  - { name: login_node, description: "Group to store IP of login node" }
+  - { name: nfs_node, description: "Group to store IP of NFS node" }
 credential_details:
   - { name: idrac_credential, type: Network, username: "{{ idrac_username }}", password: "{{ idrac_password }}", flag: true }
   - { name: ethernet_credential, type: Machine, username: "{{ ethernet_switch_username }}", password: "{{ ethernet_switch_password }}", flag: "{{ ethernet_switch_support }}" }
   - { name: infiniband_credential, type: Network, username: "{{ ib_username }}", password: "{{ ib_password }}", flag: "{{ ib_switch_support }}" }
   - { name: powervault_me4_credential, type: Network, username: "{{ powervault_me4_username }}", password: "{{ powervault_me4_password }}", flag: "{{ powervault_support }}" }
-  - { name: node_credential, type: Machine, username: root, password: "{{ provision_password }}", flag: true }
+  - { name: node_credential, type: Machine, username: root, password: omnia@123, flag: true }
 job_template_details:
   - { name: idrac_template, inventory: idrac_inventory, playbook: control_plane/idrac.yml, credential: idrac_credential, flag: true }
   - { name: ethernet_template, inventory: ethernet_inventory, playbook: control_plane/ethernet.yml, credential: ethernet_credential, flag: "{{ ethernet_switch_support }}" }
@@ -65,9 +79,8 @@ job_template_details:
   - { name: powervault_me4_template, inventory: powervault_me4_inventory, playbook: control_plane/powervault_me4.yml, credential: powervault_me4_credential, flag: "{{ powervault_support }}" }
   - { name: node_inventory_job, inventory: node_inventory, playbook: control_plane/collect_node_info.yml, credential: node_credential, flag: true }
   - { name: device_inventory_job, inventory: node_inventory, playbook: control_plane/collect_device_info.yml, credential: node_credential, flag: true }
-deploy_omnia_details:
-  - { name: deploy_omnia_template, inventory: node_inventory, playbook: omnia.yml, credential: node_credential }
-schedule_rule: "DTSTART:20210608T120000Z RRULE:FREQ=MINUTELY;INTERVAL=10"
+omnia_job_template_details:
+  - { name: deploy_omnia_template, inventory: node_inventory, playbook: omnia.yml, credential: node_credential }  
 scheduled_templates:
-  - { name: NodeInventorySchedule, template: node_inventory_job }
-  - { name: DeviceInventorySchedule, template: device_inventory_job }
+  - { name: NodeInventorySchedule, template: node_inventory_job, schedule_rule: "DTSTART:20210815T120000Z RRULE:FREQ=MINUTELY;INTERVAL=10" }
+  - { name: DeviceInventorySchedule, template: device_inventory_job, schedule_rule: "DTSTART:20210815T060000Z RRULE:FREQ=DAILY;INTERVAL=1"}

control_plane/test/test_ib_inventory

@@ -0,0 +1,2 @@
+[infiniband]
+1.2.3.4

control_plane/test/test_ib_mtu.yml

@@ -0,0 +1,101 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+---
+- name: Get MTU of mgmt5
+  hosts: infiniband
+  gather_facts: no
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+  tasks:
+    - name: Set credentials and variables
+      set_fact:
+        username: "{{ username }}"
+        password: "{{ password }}"
+        filtered_dict: {}
+      no_log: true
+      tags: reboot,mtu
+
+    - name: Authenticate
+      block:
+        - name: Authenticate to switch- "{{ inventory_hostname }}"
+          uri:
+            url: http://{{ inventory_hostname }}/admin/launch?script=rh&template=login&action=login
+            method: POST
+            body_format: form-urlencoded
+            body:
+              f_user_id: "{{ username }}"
+              f_password: "{{ password }}"
+              enter: Sign in
+            status_code: 302
+          register: login
+          no_log: true
+ 
+        - name: Verify authentication status
+          fail:
+            msg: "Authentication failed"
+          when: login.set_cookie is undefined
+      rescue:
+        - name: Filtered response creation
+          set_fact:
+            filtered_dict: "{{filtered_dict |combine({item.key: item.value})}}"
+          when: item.key not in 'invocation'
+          with_dict: "{{ login }}"
+          no_log: true
+
+        - name: Authentication failure response
+          fail: 
+            msg: "{{ filtered_dict }}"
+      tags: reboot,mtu
+
+    - name: get MTU from ib {{ validation_port }}
+      uri:
+        url: http://{{ inventory_hostname }}/admin/launch?script=json
+        method: POST
+        body_format: json
+        headers:
+          Cookie: "{{ login.set_cookie.split(';')[0] }}"
+        body:
+          {
+          "commands": 
+           [
+             "show interfaces ib {{ validation_port }}"
+           ]
+          }
+        return_content: yes
+      register: value
+      tags: mtu
+
+    - name: print reistered o/p
+      debug:
+        msg: "{{ value }}"
+      tags: mtu
+
+    - name: reboot IB switch
+      uri:
+        url: http://{{ inventory_hostname }}/admin/launch?script=json
+        method: POST
+        body_format: json
+        headers:
+          Cookie: "{{ login.set_cookie.split(';')[0] }}"
+        body:
+          {
+          "commands":
+           [
+             "reload"
+           ]
+          }
+        return_content: yes
+      tags: reboot

control_plane/test/test_infiniband_config.yml

@@ -0,0 +1,333 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+# Testcase OMNIA_1.1_IB_TC_007
+# Execute infiniband.yml with both valid Global and valid interface configs in ib_config.yml
+- name: OMNIA_1.1_IB_TC_007
+  hosts: infiniband
+  gather_facts: false
+  tags: VERIFY_OMNIA_01
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml    
+    - ../input_params/base_vars.yml
+  tasks:
+     - name: Executing network_ib role
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ template_value }}"
+        job_template_name: "{{ job_name }}"
+        playbook_path: "{{ ib_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+     
+     - block:
+        - name: Validate default flow
+          assert:
+            that:
+              - ib_success_msg in job_status.status
+            success_msg: "{{ success_message }}"
+            fail_msg: "{{ fail_case }}"
+          changed_when: false
+   
+     - name: Backup of ib_vars.yml
+       copy:
+         src: "{{ ib_vars_dir }}"
+         dest: "{{ ib_vars_backup_dir }}"
+         mode: "{{ file_perm }}"
+
+     - name: Set MTU of port {{ port_num }}
+       lineinfile:
+        dest: "{{ ib_vars_dir }}"
+        insertbefore: "{{ search_line }}"
+        line: "{{ line_to_add }}"
+
+     - name: Execute network_ib role after setting MTU
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ template_value }}"
+        job_template_name: "{{ job_name }}"
+        playbook_path: "{{ ib_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+
+     - name: Get MTU of port {{ port_num }}
+       command: ansible-playbook -i "{{ inventory_dir }}" "{{ ib_mtu_path }}" --tags 'mtu'
+       register: mtuvalue
+       changed_when: false
+
+     - block:
+        - name: Validate MTU
+          assert:
+            that:
+              - 'var_check in mtuvalue.stdout'
+            success_msg: "{{ success_message }}"
+            fail_msg: "{{ fail_case }}"
+            
+# Testcase OMNIA_1.1_IB_TC_005
+# set save_changes_to_startup to false
+- name: OMNIA_1.1_IB_TC_005
+  hosts: infiniband
+  gather_facts: false
+  tags: TC_005
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml    
+    - ../input_params/base_vars.yml
+  tasks:        
+     - name: Reload IB switch 
+       command: ansible-playbook -i "{{ inventory_dir }}" "{{ ib_mtu_path }}" --tags 'reboot'
+       changed_when: false
+
+     - name: Pausing for IB switch to come up
+       pause:
+         minutes: "{{ time_to_pause }}"
+   
+     - name: Get MTU of port {{ port_num }}
+       command: ansible-playbook -i "{{ inventory_dir }}" "{{ ib_mtu_path }}" --tags 'mtu'
+       register: mtuvalue
+       changed_when: false
+          
+     - block:
+        - name: Validate that MTU is changed
+          assert:
+            that:
+              - 'var_check not in mtuvalue.stdout'
+            success_msg: "{{ success_message }}"
+            fail_msg: "{{ fail_case }}"
+            
+# Testcase OMNIA_1.1_IB_TC_006
+# set save_changes_to_startup to True            
+- name: OMNIA_1.1_IB_TC_006
+  hosts: infiniband
+  gather_facts: false
+  tags: TC_006
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml    
+    - ../input_params/base_vars.yml
+  tasks:        
+     - name: Set save_changes_to_startup to True
+       ansible.builtin.replace:
+         dest: "{{ ib_vars_dir }}"
+         regexp: 'save_changes_to_startup: false'
+         replace: 'save_changes_to_startup: True'
+                          
+     - name: Execute network_ib role as port {{ port_num }} has mtu set in ib_Vars
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ template_value }}"
+        job_template_name: "{{ job_name }}"
+        playbook_path: "{{ ib_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+       
+     - name: Reload IB switch 
+       command: ansible-playbook -i "{{ inventory_dir }}" "{{ ib_mtu_path }}" --tags 'reboot'
+       changed_when: false
+       
+     - name: Pausing for IB switch to come up
+       pause:
+         minutes: "{{ time_to_pause }}"
+  
+     - name: Get MTU of port {{ port_num }}
+       command: ansible-playbook -i "{{ inventory_dir }}" "{{ ib_mtu_path }}" --tags 'mtu'
+       register: mtuvalue
+       changed_when: false
+       
+     - block:
+        - name: Validate that MTU is not changed
+          assert:
+            that:
+              - 'var_check in mtuvalue.stdout'
+            success_msg: "{{ success_message }}"
+            fail_msg: "{{ fail_case }}"
+       
+# Testcase OMNIA_1.1_IB_TC_010
+# Execute infiniband.yml with valid interface and incorrect Global configs in ib_config.yml
+- name: OMNIA_1.1_IB_TC_010
+  hosts: infiniband
+  gather_facts: false
+  tags: TC_010
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml    
+    - ../input_params/base_vars.yml    
+  tasks:
+     - name: Making gobal config incorrect
+       lineinfile:
+        dest: "{{ ib_vars_dir }}"
+        insertafter: 'mellanox_switch_config:'
+        line: "gibberish inserted"
+        
+     - name: Executing network_ib role
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ template_value }}"
+        job_template_name: "{{ job_name }}"
+        playbook_path: "{{ ib_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+
+     - block:
+        - name: Validate role exec output
+          assert:
+            that:
+              - ib_fail_msg in job_status.status
+            success_msg: "{{ success_message }}"
+            fail_msg: "{{ fail_case }}"
+      
+# Testcase OMNIA_1.1_IB_TC_009
+# Execute infiniband.yml with only interface and no Global configs in ib_config.yml
+- name: OMNIA_1.1_IB_TC_009
+  hosts: infiniband
+  gather_facts: false
+  tags: TC_009
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml    
+    - ../input_params/base_vars.yml
+  tasks:
+     - name: Removing global config from ib_vars.yml
+       lineinfile:
+        dest: "{{ ib_vars_dir }}"
+        state: absent
+        regexp: "^gibberish inserted"
+        
+     - name: Executing network_ib role
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ template_value }}"
+        job_template_name: "{{ job_name }}"
+        playbook_path: "{{ ib_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+
+     - name: Validate role exec output
+       assert:
+         that:
+           - ib_success_msg in job_status.status
+         success_msg: "{{ success_message }}"
+         fail_msg: "{{ fail_case }}"
+       changed_when: false
+       
+# Testcase OMNIA_1.1_IB_TC_011
+# Execute infiniband.yml with valid Global and incorrect interface configs in ib_config.yml
+- name: OMNIA_1.1_IB_TC_011
+  hosts: infiniband
+  gather_facts: false
+  tags: TC_011
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml    
+    - ../input_params/base_vars.yml
+  tasks:
+     - name: Make interface config incorrect
+       lineinfile:
+        dest: "{{ ib_vars_dir }}"
+        insertafter: "{{ line_to_search }}"
+        line: "gibberish inserted"
+        
+     - name: Executing network_ib role
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ template_value }}"
+        job_template_name: "{{ job_name }}"
+        playbook_path: "{{ ib_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+ 
+     - name: Validate role exec output
+       assert:
+         that:
+           - ib_fail_msg in job_status.status
+         success_msg: "{{ success_message }}"
+         fail_msg: "{{ fail_case }}"
+      
+# Testcase OMNIA_1.1_IB_TC_008
+# Execute infiniband.yml with only Global and no interface configs in ib_config.yml
+- name: OMNIA_1.1_IB_TC_008
+  hosts: infiniband
+  gather_facts: false
+  tags: TC_008
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml    
+    - ../input_params/base_vars.yml
+  tasks:
+     - name: Removing interface config
+       ansible.builtin.command: sed -i '49,196d' "{{ ib_vars_dir }}"
+       args:
+        warn: no
+       changed_when: false
+     
+     - name: Executing network_ib role
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ template_value }}"
+        job_template_name: "{{ job_name }}"
+        playbook_path: "{{ ib_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+
+     - name: Validate role exec output
+       assert:
+         that:
+           - ib_success_msg in job_status.status
+         success_msg: "{{ success_message }}"
+         fail_msg: "{{ fail_case }}"
+       changed_when: false
+
+     - name: Restore orginal ib_vars file
+       copy:
+         src: "{{ ib_vars_backup_dir }}"
+         dest: "{{ ib_vars_dir }}"
+         mode: "{{ file_perm }}"
+         
+     - name: Set save_changes_to_startup to True
+       ansible.builtin.replace:
+         dest: "{{ ib_vars_dir }}"
+         regexp: 'save_changes_to_startup: false'
+         replace: 'save_changes_to_startup: True'
+                          
+     - name: Execute network_ib role to set default IB config as is
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ template_value }}"
+        job_template_name: "{{ job_name }}"
+        playbook_path: "{{ ib_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+       
+     - name: Set save_changes_to_startup back to false
+       ansible.builtin.replace:
+         dest: "{{ ib_vars_dir }}"
+         regexp: 'save_changes_to_startup: True'
+         replace: 'save_changes_to_startup: false'

control_plane/test/test_infiniband_facts.yml

@@ -0,0 +1,156 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+# Testcase OMNIA_1.1_IB_TC_002
+# Execute ib_facts.yml with valid IP and valid credentials
+- name: OMNIA_1.1_IB_TC_002
+  hosts: infiniband
+  gather_facts: false
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml
+    - ../input_params/base_vars.yml
+  tasks: 
+     - name: Execute ib_facts
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ fact_template_value }}"
+        job_template_name: "{{ fact_job_name }}"
+        playbook_path: "{{ ib_facts_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+       
+     - block:
+        - name: Validate default flow with valid IP and valid credentials
+          assert:
+            that:
+              - ib_success_msg in job_status.status
+            success_msg: "{{ success_message }}"
+            fail_msg: "{{ fail_case }}"
+          changed_when: false
+
+# Testcase OMNIA_1.1_IB_TC_003
+# Execute ib_facts.yml with Invalid IP in inventory group
+- name: OMNIA_1.1_IB_TC_003
+  hosts: infiniband
+  gather_facts: false
+  tags: TC_003
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml
+    - ../input_params/base_vars.yml
+  tasks:
+     - name: setting ip
+       set_fact:
+         ib_host_name: "{{ random_ip }}"
+    
+     - name: Execute ib_facts
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ fact_template_value }}"
+        job_template_name: "{{ fact_job_name }}"
+        playbook_path: "{{ ib_facts_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+
+     - block:
+        - name: Validate invalid IP and valid credentials
+          assert:
+            that:
+              - ib_fail_msg in job_status.status
+            success_msg: "{{ success_message }}"
+            fail_msg: "{{ fail_case }}"
+          changed_when: false
+          failed_when: false
+      
+# Testcase OMNIA_1.1_IB_TC_001
+# Execute ib_facts.yml with no hosts in inventory
+- name: OMNIA_1.1_IB_TC_001
+  hosts: infiniband
+  gather_facts: false
+  tags: TC_001
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml
+    - ../input_params/base_vars.yml
+  tasks:
+     - name: Execute ib_facts with no host details
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        template_name: "{{ fact_template_value }}"
+        job_template_name: "{{ fact_job_name }}"
+        playbook_path: "{{ ib_facts_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+  
+     - block:
+        - name: Validate no hosts and valid credentials
+          assert:
+            that:
+              - ib_success_msg in job_status.status
+            success_msg: "{{ success_message }}"
+            fail_msg: "{{ fail_case }}"
+          changed_when: false
+
+# Testcase OMNIA_1.1_IB_TC_004
+# Execute ib_facts.yml with valid IP and incorrect credentials
+- name: OMNIA_1.1_IB_TC_004
+  hosts: infiniband
+  gather_facts: false
+  tags: TC_004
+  connection: local
+  vars_files:
+    - test_vars/test_infiniband_vars.yml
+    - ../roles/webui_awx/vars/main.yml
+    - ../input_params/base_vars.yml
+  tasks:
+     - name: Making infiniband_credentials invalid
+       tower_credential:
+         name: "infiniband_credential"
+         credential_type: "Network"
+         inputs:
+           username: "{{ invalid_username }}"
+         
+     - name: Execute ib_facts
+       vars:
+        inventory_name: "{{ ib_inventory_name }}"
+        host_name: "{{ ib_host_name }}"
+        template_name: "{{ fact_template_value }}"
+        job_template_name: "{{ fact_job_name }}"
+        playbook_path: "{{ ib_facts_playbook_path }}"
+        delete_status: true
+       include_tasks: "{{ awx_script_path }}"
+
+     - block:
+        - name: Validate valid IP and invalid credentials
+          assert:
+            that:
+              - ib_fail_msg in job_status.status
+            success_msg: "{{ success_message }}"
+            fail_msg: "{{ fail_case }}"
+          changed_when: false
+  
+     - name: Set credentials back to default
+       tower_credential:
+         name: "infiniband_credential"
+         credential_type: "Network"
+         inputs:
+           username: "{{ username }}"
+           password: "{{ password }}"

control_plane/test/test_inventory.yml

@@ -0,0 +1,754 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---         
+# Test case to verify the prerequisites are installed and execute the AWX deployment
+- name: OMNIA_1.1_AWX_TC_001    
+  hosts: localhost
+  connection: local
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+  tasks:
+     
+   - name: Check login_vars file is encrypted
+     command: cat "{{ login_vars_path }}"
+     changed_when: false
+     register: config_content
+     tags: always
+      
+   - name: Decrpyt login_vars.yml
+     command: >-
+       ansible-vault decrypt {{ login_vars_path }}
+       --vault-password-file {{ login_vars_vault_path }}
+     changed_when: false
+     when: "'$ANSIBLE_VAULT;' in config_content.stdout"
+     tags: always
+
+   - name: Include variable file login_vars.yml
+     include_vars: "{{ login_vars_path }}"
+     tags: always
+       
+   - name: Encypt login file
+     command: >-
+       ansible-vault encrypt {{ login_vars_path }}
+       --vault-password-file {{ login_vars_vault_path }}
+     changed_when: false
+     tags: always
+                     
+   - name: Execute awx command
+     command: "kubectl get pods -n {{ awx_namespace }}"
+     changed_when: true
+     register: k8s_pods
+     run_once: true
+     ignore_errors: true
+     tags: TC_001,VERIFY_OMNIA_01    
+     
+   - name: Validate awx operator containers
+     assert:
+      that:  
+       -  k8s_pods.stdout | regex_search("{{ item }}")
+      fail_msg: "{{ awx_fail_msg }}"
+      success_msg: "{{ awx_success_msg }}"
+     loop: 
+       - "awx-([A-Za-z0-9]{10})-([A-Za-z0-9]{5})"
+       - "awx-operator-([A-Za-z0-9]{10})-([A-Za-z0-9]{5})"
+       - "awx-postgres-([A-Za-z0-9]{1})"
+     run_once: true
+     tags: TC_001,VERIFY_OMNIA_01       
+            
+# Test case to verify inventory groups are present in AWX UI  (idrac, ethernet, inifiniband, rbod)  
+- name: OMNIA_1.1_AWX_TC_003   
+  hosts: localhost
+  connection: local
+  
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+  tasks:                 
+
+   - name: Execute get pods command
+     command: "kubectl get pods -n {{ awx_namespace }}"
+     changed_when: true
+     register: k8s_pods
+     run_once: true
+     ignore_errors: true
+     tags: TC_003     
+     
+   - name: Get awx pod 
+     set_fact:
+      awx_pods: "{{ item | regex_search(awx_pod_regex) | trim  }}"
+      idrac_status: true
+     with_items: 
+       - "{{ k8s_pods.stdout_lines }}"
+     run_once: true
+     when: item | regex_search(awx_pod_item_regex)
+     tags: TC_003
+
+   - name: Get awx cluster ip
+     shell: "kubectl get svc awx-ui -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+     register: awx_cluster_ip
+     changed_when: false
+     ignore_errors: true
+     tags: TC_003
+
+   - name: Get AWX admin password
+     shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+     register: awx_admin_password
+     changed_when: false
+     ignore_errors: true
+     tags: TC_003
+          
+   - name: Execute awx get inventory hosts command
+     command: "awx --conf.host http://{{ awx_cluster_ip.stdout }}:8052 --conf.username admin --conf.password {{ awx_admin_password.stdout }} --conf.insecure hosts list --inventory {{ item }} -f human --filter 'name'"
+     register: idrac_hosts
+     with_items:
+      - "idrac_inventory"
+      - "infiniband_inventory"
+      - "ethernet_inventory"
+      - "powervault_me4_inventory"
+     run_once: true
+     changed_when: false
+     tags: TC_003        
+       
+   - name: Verify  inventory are present in AWX UI  
+     assert:
+      that: 
+       - item.stdout_lines[0] | regex_search("name")
+      fail_msg: "{{ item.item }} - {{ inventory_fail_msg }}"
+      success_msg: "{{ item.item }} - {{ inventory_success_msg }}"
+     with_items:
+      - "{{ idrac_hosts.results }}"
+     changed_when: false
+     tags: TC_003
+              
+# Test case to validate ip of idrac     
+- name: OMNIA_1.1_AWX_TC_004    
+  hosts: localhost
+  connection: local
+  
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+   - ../input_params/base_vars.yml  
+  tasks:       
+      
+   - name: Execute get pods command
+     command: "kubectl get pods -n {{ awx_namespace }}"
+     changed_when: true
+     register: k8s_pods
+     run_once: true
+     ignore_errors: true
+     tags: TC_004     
+     
+   - name: Get awx pod 
+     set_fact:
+      awx_pods: "{{ item | regex_search(awx_pod_regex) | trim  }}"
+     with_items: 
+       - "{{ k8s_pods.stdout_lines }}"
+     run_once: true
+     when: item | regex_search(awx_pod_item_regex)
+     changed_when: false
+     tags: TC_004
+
+   - name: Get awx cluster ip
+     shell: "kubectl get svc awx-ui -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+     register: awx_cluster_ip
+     changed_when: false
+     tags: TC_004
+
+   - name: Get AWX admin password
+     shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+     register: awx_admin_password
+     changed_when: false
+     ignore_errors: true
+     tags: TC_004
+          
+   - name: Execute awx get inventory hosts command
+     command: "awx --conf.host http://{{ awx_cluster_ip.stdout }}:8052 --conf.username admin --conf.password {{ awx_admin_password.stdout }} --conf.insecure hosts list --inventory {{ idrac_inventory_name }} -f human --filter 'name'"
+     changed_when: true
+     register: idrac_hosts
+     run_once: true
+     tags: TC_004     
+     
+   - name: List of iDRAC host
+     include_tasks: "{{ validation_script_path }}"
+     with_items:
+      - "{{ idrac_hosts.stdout_lines[2:] }}"
+     when: idrac_hosts.stdout_lines | length > 2
+     ignore_errors: true
+     tags: TC_004
+     
+   - name: Empty iDRAC hosts
+     debug:
+      msg: "{{ empty_host_err }}"
+     when: idrac_hosts.stdout_lines | length < 3
+     failed_when: false
+     tags: TC_004     
+
+# Test case to validate ip of infiniband
+- name: OMNIA_1.1_AWX_TC_005    
+  hosts: localhost
+  connection: local
+  
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+   - ../input_params/base_vars.yml  
+  tasks:                 
+
+   - name: Execute get pods command
+     command: "kubectl get pods -n {{ awx_namespace }}"
+     changed_when: true
+     register: k8s_pods
+     run_once: true
+     ignore_errors: true
+     tags: TC_005     
+     
+   - name: Get awx pod 
+     set_fact:
+      awx_pods: "{{ item | regex_search(awx_pod_regex) | trim  }}"
+     with_items: 
+       - "{{ k8s_pods.stdout_lines }}"
+     run_once: true
+     when: item | regex_search(awx_pod_item_regex)
+     failed_when: false
+     tags: TC_005
+
+   - name: Get awx cluster ip
+     shell: "kubectl get svc awx-ui -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+     register: awx_cluster_ip
+     changed_when: false
+     tags: TC_005
+
+   - name: Get AWX admin password
+     shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+     register: awx_admin_password
+     changed_when: false
+     ignore_errors: true
+     tags: TC_005
+          
+   - name: Execute awx get inventory hosts command
+     command: "awx --conf.host http://{{ awx_cluster_ip.stdout }}:8052 --conf.username admin --conf.password {{ awx_admin_password.stdout }} --conf.insecure hosts list --inventory {{ ib_inventory_name }} -f human --filter 'name'"
+     changed_when: true
+     register: infiniband_hosts
+     run_once: true
+     ignore_errors: true
+     tags: TC_005     
+     
+   - name: List of infiniband hosts
+     include_tasks: "{{ validation_script_path }}"  
+     with_items:
+      - "{{ infiniband_hosts.stdout_lines[2:] }}"
+     when: infiniband_hosts.stdout_lines | length > 2
+     ignore_errors: true
+     tags: TC_005
+     
+   - name: Empty infiniband hosts
+     debug:
+      msg: "{{ empty_host_err }}"
+     when: infiniband_hosts.stdout_lines | length < 3
+     failed_when: false
+     tags: TC_005 
+
+# Test case to validate ip of ethernet
+- name: OMNIA_1.1_AWX_TC_006    
+  hosts: localhost
+  connection: local
+  
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+   - ../input_params/base_vars.yml  
+  tasks:                 
+
+   - name: Execute get pods command
+     command: "kubectl get pods -n {{ awx_namespace }}"
+     changed_when: true
+     register: k8s_pods
+     run_once: true
+     ignore_errors: true
+     tags: TC_006     
+     
+   - name: Get awx pod 
+     set_fact:
+      awx_pods: "{{ item | regex_search(awx_pod_regex) | trim  }}"
+     with_items: 
+       - "{{ k8s_pods.stdout_lines }}"
+     run_once: true
+     when: item | regex_search(awx_pod_item_regex)
+     failed_when: false
+     tags: TC_006
+
+   - name: Get awx cluster ip
+     shell: "kubectl get svc awx-ui -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+     register: awx_cluster_ip
+     changed_when: false
+     tags: TC_006
+
+   - name: Get AWX admin password
+     shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+     register: awx_admin_password
+     changed_when: false
+     ignore_errors: true
+     tags: TC_006
+          
+   - name: Execute awx get inventory hosts command
+     command: "awx --conf.host http://{{ awx_cluster_ip.stdout }}:8052 --conf.username admin --conf.password {{ awx_admin_password.stdout }} --conf.insecure hosts list --inventory {{ ethernet_inventory_name }} -f human --filter 'name'"
+     changed_when: true
+     register: ethernet_hosts
+     run_once: true
+     ignore_errors: true
+     tags: TC_006     
+     
+   - name: List of ethernet hosts   
+     include_tasks: "{{ validation_script_path }}" 
+     with_items:
+      - "{{ ethernet_hosts.stdout_lines[2:] }}"
+     when: ethernet_hosts.stdout_lines | length > 2
+     ignore_errors: true
+     tags: TC_006
+     
+   - name: Empty ethernet hosts
+     debug:
+      msg: "{{ empty_host_err }}"
+     when: ethernet_hosts.stdout_lines | length < 3
+     failed_when: false
+     tags: TC_006
+      
+# Test case to validate ip of powervault      
+- name: OMNIA_1.1_AWX_TC_007    
+  hosts: localhost
+  connection: local
+  
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+   - ../input_params/base_vars.yml  
+  tasks:                 
+
+   - name: Execute get pods command
+     command: "kubectl get pods -n {{ awx_namespace }}"
+     changed_when: true
+     register: k8s_pods
+     run_once: true
+     ignore_errors: true
+     tags: TC_007     
+     
+   - name: Get awx pod 
+     set_fact:
+      awx_pods: "{{ item | regex_search(awx_pod_regex) | trim  }}"
+      idrac_status: true
+     with_items: 
+       - "{{ k8s_pods.stdout_lines }}"
+     run_once: true
+     when: item | regex_search(awx_pod_item_regex)
+     failed_when: false
+     tags: TC_007
+
+   - name: Get awx cluster ip
+     shell: "kubectl get svc awx-ui -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+     register: awx_cluster_ip
+     changed_when: false
+     tags: TC_007
+
+   - name: Get AWX admin password
+     shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+     register: awx_admin_password
+     changed_when: false
+     ignore_errors: true
+     tags: TC_007
+          
+   - name: Execute awx get inventory hosts command
+     command: "awx --conf.host http://{{ awx_cluster_ip.stdout }}:8052 --conf.username admin --conf.password {{ awx_admin_password.stdout }} --conf.insecure hosts list --inventory {{ pv_inventory_name }} -f human --filter 'name'"
+     changed_when: true
+     register: powervault_hosts
+     run_once: true
+     ignore_errors: true
+     tags: TC_007     
+     
+   - name: List of powervault hosts
+     include_tasks: "{{ validation_script_path }}"  
+     with_items:
+      - "{{ powervault_hosts.stdout_lines[2:] }}"
+     when: powervault_hosts.stdout_lines | length > 2
+     ignore_errors: true
+     tags: TC_007
+     
+   - name: Empty powervault hosts
+     debug:
+      msg: "{{ empty_host_err }}"
+     when: powervault_hosts.stdout_lines | length < 3
+     failed_when: false
+     tags: TC_007
+
+# Test case to verify omnia inventory groups (manager, compute, login, nfs)                        
+- name: OMNIA_1.1_AWX_TC_008   
+  hosts: localhost
+  connection: local
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+   - ../roles/webui_awx/vars/main.yml
+    
+  tasks:                 
+
+    - name: Get awx-service Cluster-IP
+      command: "kubectl get svc awx-service -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+      register: awx_cluster_ip
+      changed_when: false
+      run_once: true
+      ignore_errors: true
+      tags: TC_008
+    
+    - name: Get AWX admin password
+      shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+      register: awx_admin_password
+      changed_when: false
+      run_once: true
+      ignore_errors: true
+      tags: TC_008
+         
+    - name: Set IP and password
+      set_fact:
+        awx_ip: 'http://{{ awx_cluster_ip.stdout }}'
+        admin_password: "{{ awx_admin_password.stdout }}"
+      run_once: true
+      failed_when: false
+      tags: TC_008
+       
+    - name: Get omnia inventory groups
+      awx.awx.tower_group:
+        name: "{{ item.name }}"
+        description: "{{ item.description }}"
+        inventory: "node_inventory"
+        state: present
+      loop: "{{ group_names }}"
+      register: awx_group
+      run_once: true
+      ignore_errors: true
+      tags: TC_008
+     
+    - name: Verify omnia inventory groups
+      assert:
+       that: 
+         - item.changed == false
+         - item.item.name == "{{ manager_group }}" or 
+           item.item.name =="{{ compute_group }}" or 
+           item.item.name == "{{ login_group }}" or 
+           item.item.name == "{{ nfs_group }}"
+       fail_msg: "{{ item .item.name }}{{ group_fail_msg }}"
+       success_msg: "{{ item .item.name }}{{ group_success_msg }}"
+      with_items:
+       - "{{ awx_group.results }}"
+      failed_when: false
+      tags: TC_008
+      
+# Test case to verify AWX configuration is done properly with job_templates, schedules in place      
+- name: OMNIA_1.1_AWX_TC_009   
+  hosts: localhost
+  connection: local
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+   - ../roles/webui_awx/vars/main.yml
+    
+  tasks:                 
+
+    - name: Get awx-service Cluster-IP
+      command: "kubectl get svc awx-service -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+      register: awx_cluster_ip
+      changed_when: false
+      ignore_errors: true
+      run_once: true
+      tags: TC_009
+    
+    - name: Get AWX admin password
+      shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+      register: awx_admin_password
+      changed_when: false
+      ignore_errors: true
+      run_once: true
+      tags: TC_009
+           
+    - name: Set IP and password
+      set_fact:
+        awx_ip: 'http://{{ awx_cluster_ip.stdout }}'
+        admin_password: "{{ awx_admin_password.stdout }}"
+      failed_when: false
+      run_once: true
+      tags: TC_009   
+      
+    - name: Get job template details
+      awx.awx.tower_job_template:
+        name: "{{ item.name }}"
+        job_type: "run"
+        organization: "{{ organization_name }}"
+        inventory: "{{ item.inventory }}"
+        project: "{{ project_name }}"
+        playbook: "{{ item.playbook }}"
+        credentials:
+          - "{{ item.credential }}"
+        state: present
+        tower_config_file: "{{ tower_config_file_path }}"
+      loop: "{{ job_template_details }}"
+      register: job_template
+      when: item.flag
+      ignore_errors: true
+      tags: TC_009  
+
+    - name: Validate job template 
+      assert:
+       that: 
+         - item.changed == false
+       fail_msg: "{{ item.item.name }}{{ job_template_fail_msg }}"
+       success_msg: " {{ item.item.name }} {{ job_template_success_msg }}"
+      with_items:
+        - "{{ job_template.results }}"
+      failed_when: false
+      when: item.item.flag
+      tags: TC_009  
+      
+    - name: Build a schedule for job template
+      awx.awx.tower_schedule:
+        name: "{{ item.name }}"
+        unified_job_template: "{{ item.template }}"
+        rrule: "{{ schedule_rule }}"
+        state: present
+        tower_config_file: "{{ tower_config_file_path }}"
+      register: schedule
+      loop: "{{ scheduled_templates }}"
+      failed_when: false
+      run_once: true
+      tags: TC_009
+      
+    - name: Validate schedule status
+      assert:
+       that: 
+         - schedule.changed == false
+       fail_msg: "{{ schedule_fail_msg }}"
+       success_msg: "{{ schedule_success_msg }}"
+      failed_when: false
+      tags: TC_009 
+
+
+# Test case to verify updation of new node in omnia inventory
+- name: OMNIA_1.1_AWX_TC_010   
+  hosts: localhost
+  connection: local
+  
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+   - ../input_params/base_vars.yml  
+  tasks:                 
+
+    - name: Execute get pods command
+      command: "kubectl get pods -n {{ awx_namespace }}"
+      changed_when: true
+      register: k8s_pods
+      run_once: true
+      ignore_errors: true
+      tags: TC_010
+          
+    - name: Get awx pod 
+      set_fact:
+       awx_pods: "{{ item | regex_search(awx_pod_regex) | trim  }}"
+      with_items: 
+        - "{{ k8s_pods.stdout_lines }}"
+      run_once: true
+      when: item | regex_search(awx_pod_item_regex)
+      failed_when: false
+      tags: TC_010
+
+    - name: Get AWX admin password
+      shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+      register: awx_admin_password
+      changed_when: false
+      ignore_errors: true
+      tags: TC_010
+          
+    - name: Execute awx get inventory hosts command
+      command: "awx --conf.host {{ awx_host }} --conf.username admin --conf.password {{ awx_admin_password.stdout }} --conf.insecure hosts list --inventory {{ node_inventory_name }} -f human --filter 'name'"
+      changed_when: true
+      register: node_hosts
+      run_once: true
+      failed_when: false
+      tags: TC_010
+         
+    - name: Get node_inventory hosts
+      command: ping -c1 {{ item }}
+      delegate_to: localhost
+      register: ping_result
+      ignore_errors: yes
+      changed_when: false
+      with_items:
+      - "{{ node_hosts.stdout_lines[2:] }}"
+      when: node_hosts.stdout_lines | length > 2
+      tags: TC_010
+            
+    - name: Verify updation of new node
+      assert:
+       that: 
+         - "'100% packet loss' not in item.stdout"
+       fail_msg: "{{ node_fail_msg }}"
+       success_msg: "{{ node_success_msg }}"
+      with_items:
+       - "{{ ping_result.results }}"
+      when: node_hosts.stdout_lines | length > 2
+      failed_when: false
+      tags: TC_010
+
+    - name: Empty node hosts
+      debug:
+       msg: "{{ empty_host_err }}"
+      when: node_hosts.stdout_lines | length < 3
+      tags: TC_010
+          
+# Test case to verify AWX configuration is done properly with all items in place      
+- name: OMNIA_1.1_AWX_TC_011   
+  hosts: localhost
+  connection: local
+  vars_files:
+   - test_vars/test_inventory_vars.yml
+   - ../roles/webui_awx/vars/main.yml
+    
+  tasks:                 
+
+    - name: Get awx-service Cluster-IP
+      command: "kubectl get svc awx-service -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+      register: awx_cluster_ip
+      changed_when: false
+      ignore_errors: true
+      run_once: true
+      tags: TC_011
+    
+    - name: Get AWX admin password
+      shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+      register: awx_admin_password
+      changed_when: false
+      ignore_errors: true
+      run_once: true
+      tags: TC_011
+           
+    - name: Set IP and password
+      set_fact:
+        awx_ip: 'http://{{ awx_cluster_ip.stdout }}'
+        admin_password: "{{ awx_admin_password.stdout }}"
+      run_once: true
+      tags: TC_011
+
+    - name: Get organization details
+      awx.awx.tower_organization:
+        name: "{{ organization_name }}"
+        description: "{{ org_description }}"
+        state: present
+      register: organization
+      ignore_errors: true
+      run_once: true
+      tags: TC_011
+      
+    - name: Validate an organization
+      assert:
+       that: 
+         - organization.changed == false
+       fail_msg: "{{ organization_fail_msg }}"
+       success_msg: "{{ organization_success_msg }}"
+      failed_when: false
+      tags: TC_011
+             
+    - name: Get tower inventory details
+      awx.awx.tower_inventory:
+        name: "{{ item.name }}"
+        description: "{{ item.description }}"
+        organization: "{{ organization_name }}"
+        state: present
+      loop: "{{ inventory_names }}"  
+      register: inventory 
+      when: item.flag
+      ignore_errors: true
+      run_once: true
+      tags: TC_011 
+
+    - name: Validate inventory status
+      assert:
+       that: 
+         - item.changed == false
+       fail_msg: "{{ inventory_fail_msg }}"
+       success_msg: "{{ inventory_success_msg }}"
+      with_items:
+       - "{{ inventory.results }}"
+      failed_when: false
+      tags: TC_011   
+      
+    - name: Get job template details
+      awx.awx.tower_job_template:
+        name: "{{ item.name }}"
+        job_type: "run"
+        organization: "{{ organization_name }}"
+        inventory: "{{ item.inventory }}"
+        project: "{{ project_name }}"
+        playbook: "{{ item.playbook }}"
+        credentials:
+          - "{{ item.credential }}"
+        state: present
+      loop: "{{ job_template_details }}"
+      register: job_template
+      when: item.flag
+      ignore_errors: true
+      run_once: true
+      tags: TC_011  
+
+    - name: Validate job template 
+      assert:
+       that: 
+         - item.changed == false
+       fail_msg: "{{ item.item.name }} {{ job_template_fail_msg }}"
+       success_msg: "{{ item.item.name }} {{ job_template_success_msg }}"
+      with_items:
+        - "{{ job_template.results }}"
+      failed_when: false
+      when: item.item.flag
+      tags: TC_011 
+      
+    - name: Get project details
+      awx.awx.tower_project:
+        name: "{{ project_name }}"
+        description: "{{ project_description }}"
+        organization: "{{ organization_name }}"
+        state: present
+      register: project
+      ignore_errors: true
+      run_once: true
+      tags: TC_011 
+            
+    - name: Verify project 
+      assert:
+       that: 
+         - project.changed == false
+       fail_msg: "{{ project_fail_msg }}"
+       success_msg: "{{ project_success_msg }}"
+      failed_when: false
+      tags: TC_011 
+      
+    - name: Build a schedule for job template
+      awx.awx.tower_schedule:
+        name: "{{ item.name }}"
+        unified_job_template: "{{ item.template }}"
+        rrule: "{{ schedule_rule }}"
+        state: present
+      register: schedule
+      loop: "{{ scheduled_templates }}"
+      failed_when: false
+      run_once: true
+      tags: TC_011
+      
+    - name: Validate schedule status
+      assert:
+       that: 
+         - schedule.changed == false
+       fail_msg: "{{ schedule_fail_msg }}"
+       success_msg: "{{ schedule_success_msg }}"
+      failed_when: false
+      tags: TC_011

control_plane/test/test_inventory_validation.yml

@@ -0,0 +1,147 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+- block: 
+      
+   - name: Get iDRAC IP details
+     dellemc.openmanage.idrac_system_info:
+      idrac_ip: "{{ item | trim }}"
+      idrac_user: "{{ idrac_username }}"
+      idrac_password: "{{ idrac_password }}"
+     register: idrac_ip
+     run_once: true
+     ignore_errors: yes 
+     tags: "TC_004"   
+
+   - name: Validate iDRAC IP
+     assert:
+      that: 
+        - idrac_ip.system_info.iDRACNIC[0].ProductInfo == idrac_search_key
+      fail_msg: "{{ item | trim }} {{ idrac_fail_msg }}"
+      success_msg: "{{ item | trim }} {{ idrac_success_msg }}"
+     tags: "TC_004" 
+
+   - name: Authenticate infiniband Switch
+     uri:
+       url: http://{{ item | trim }}/admin/launch?script=rh&template=login&action=login
+       method: POST
+       body_format: form-urlencoded
+       body:
+         f_user_id: "{{ ib_username }}"
+         f_password: "{{ ib_password }}"
+         enter: Sign in
+       status_code: "{{ infiniband_status_code }}"
+     register: login
+     ignore_errors: yes
+     run_once: true
+     tags: "TC_005"  
+          
+   - name: Assert infiniband switch
+     uri:
+       url: http://{{ item | trim }}/admin/launch?script=json
+       method: POST
+       body_format: json
+       headers:
+         Cookie: "{{ login.set_cookie.split(';')[0] }}"
+       body:
+         {
+         "commands":
+          [
+            "show version"
+          ]
+         }
+     register: infinibandswitch_info
+     when: login.failed == false  or 'set_cookie' not in login
+     ignore_errors: yes
+     run_once: true
+     tags: "TC_005"
+              
+   - name: Validate infiniband  IP
+     assert:
+      that: 
+        - infinibandswitch_info.json.data['Product name'] == infiniband_search_key
+      fail_msg: "{{ infiniband_fail_msg }}"
+      success_msg: "{{ infiniband_success_msg }}"
+     ignore_errors: yes
+     tags: "TC_005"
+          
+   - name: Get ethernet IP details
+     dellos10_command:
+       provider:
+         host: "{{ item | trim }}"
+         username: "{{ ethernet_switch_username }}"
+         password: "{{ ethernet_switch_password }}"
+       commands: ['show version']
+     register: ethernet_info
+     ignore_errors: yes
+     run_once: true
+     when: ethernet_switch_support
+     tags: "TC_006"
+
+   - name: Validate ethernet  IP
+     assert:
+      that: 
+        - ethernet_info.stdout | regex_search(ethernet_search_key)
+      fail_msg: " {{ item | trim }} {{ ethernet_fail_msg }}"
+      success_msg: "{{ item | trim }} {{ ethernet_success_msg }}"
+     tags: "TC_006"
+           
+   - name: Get auth string for powervault
+     shell: echo -n {{ powervault_me4_username }}_{{powervault_me4_password}} | sha256sum
+     changed_when: false
+     when: powervault_support 
+     register: auth_string
+     ignore_errors: yes
+     run_once: true
+     tags: "TC_007"
+         
+   - name: Get session key for powervault
+     uri:
+       url: https://{{ item | trim }}/api/login/{{ auth_string.stdout | replace(" -", "") }}
+       method: GET
+       headers:
+         {'datatype': 'json'}
+       validate_certs: no
+     when: powervault_support 
+     register: session_key
+     ignore_errors: yes
+     run_once: true
+     tags: "TC_007"
+        
+   - name: Assert me4_powervault
+     uri:
+       url: https://{{ item | trim }}/api/show/system
+       method: GET
+       body_format: json
+       validate_certs: no
+       use_proxy: no
+       headers:
+         {'sessionKey': "{{ session_key.json.status[0].response }}", 'datatype':'json'} 
+     register: system_info
+     ignore_errors: yes
+     run_once: true
+     tags: "TC_007"
+
+   - name: Validate me4_powervault  IP
+     assert:
+      that: 
+        - "'{{ me4_powervault_search_key }}' in system_info.json.system[0]['scsi-product-id']"
+      fail_msg: "{{ item | trim }} {{ powervault_fail_msg }}"
+      success_msg: "{{ item | trim }} {{ powervault_success_msg }}"
+     ignore_errors: yes
+     tags: "TC_007"
+       
+  rescue:
+    - debug:
+       msg: "{{ failed_msg }}"

control_plane/test/test_vars/test_infiniband_vars.yml

@@ -0,0 +1,56 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+# Usage : test_infiniband_config.yml
+ib_inventory_name: "infiniband_inventory"
+template_value: 'infiniband_template'
+ib_host_name: 100.96.28.140
+job_name: "test_infiniband_template"
+ib_playbook_path: "control_plane/infiniband.yml"
+success_message: "Execution Successful"
+fail_case: "Failed. please check input parameters and try again!"
+awx_script_path: "test_prepare.yml"
+ib_vars_dir: ../input_params/ib_vars.yml
+ib_vars_backup_dir: ib_vars_backup.yml
+inventory_dir: test_ib_inventory
+ib_mtu_path: test_ib_mtu.yml
+login_vars_path: "../input_params/login_vars.yml"
+login_vars_vault_path: "../input_params/.login_vault_key"
+tower_config_file_path: "../roles/webui_awx/files/.tower_cli.cfg"
+tower_vault_file_path: "../roles/webui_awx/files/.tower_vault_key"
+file_perm: '0644'
+
+# Usage: test_ib_mtu.yml
+username: admin
+password: admin
+var_check: '"MTU": "2048"'
+
+# Usage: test_infiniband_facts.yml
+fact_template_value: 'infiniband_template'
+fact_job_name: 'test_ib_fact_job'
+ib_facts_playbook_path: "control_plane/tools/ib_facts.yml"
+test_infiniband_vars_dir: "test_vars/test_infiniband_vars.yml"
+random_ip: 100.100.100.100
+invalid_username: 'invalid_username'
+
+# Usage: test_infiniband_facts.yml
+validation_port: 1/5
+port_num: 5
+search_line: "  ib 1/6:"
+line_to_add: '      - "mtu 2K"'
+time_to_pause: 2
+line_to_search: "^(.*)port 8"
+ib_success_msg: "successful"
+ib_fail_msg: "failed"

control_plane/test/test_vars/test_inventory_vars.yml

@@ -0,0 +1,70 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+# Usage: test_inventory_validation.yml
+idrac_search_key: "Integrated Dell Remote Access Controller"
+ethernet_search_key: "Dell EMC Networking OS10-Enterprise"
+infiniband_search_key: "MLNX-OS"
+me4_powervault_search_key: "ME4"
+idrac_fail_msg: "iDRAC IP validation is failed"
+idrac_success_msg: "iDRAC IP validation is success"
+ethernet_fail_msg: "Ethernet IP validation is failed"
+ethernet_success_msg: "Ethernet IP validation is success"
+infiniband_fail_msg: "Infiniband IP validation is failed"
+infiniband_success_msg: "Infiniband IP validation is success"
+powervault_fail_msg: "Powervault IP validation is failed"
+powervault_success_msg: "Powervault IP validation is success"
+failed_msg: "Failed. Please check input parameters and try again!"
+
+# Usage: test_inventory.yml
+login_vars_path: "../input_params/login_vars.yml"
+login_vars_vault_path: "../input_params/.login_vault_key"
+tower_config_file_path: "../roles/webui_awx/files/.tower_cli.cfg"
+tower_vault_file_path: "../roles/webui_awx/files/.tower_vault_key"
+awx_namespace: "awx"
+awx_pod_regex: 'awx-([A-Za-z0-9]{10})-([A-Za-z0-9]{5})'
+awx_pod_item_regex: "awx-([A-Za-z0-9]{10})-([A-Za-z0-9]{5})"
+ethernet_switch_support: true
+ib_switch_support: true
+powervault_support: true
+idrac_inventory_name: "idrac_inventory"
+ib_inventory_name: "infiniband_inventory"
+ethernet_inventory_name: "ethernet_inventory"
+pv_inventory_name: "powervault_me4_inventory"
+node_inventory_name: "node_inventory"
+validation_script_path: "test_inventory_validation.yml" 
+infiniband_status_code: 302
+org_description: "Name of organization using this product"
+manager_group: "manager" 
+compute_group: "compute" 
+login_group: "login" 
+nfs_group: "nfs" 
+group_fail_msg: "Group verification is failed"
+group_success_msg: " Group verification is successful"
+awx_fail_msg: "awx-operator containers creation failed"
+awx_success_msg: "awx-operator containers creation is successful"
+empty_host_err: "No hosts available"
+node_fail_msg: "Updation of new node is failed"
+node_success_msg: "Updation of new node is successful"
+inventory_fail_msg: "Inventory creation is failed"
+inventory_success_msg: "Inventory creation is successful"
+job_template_fail_msg: "Template creation is failed"
+job_template_success_msg: "Template creation is successful"
+project_fail_msg: " Project creation is failed"
+project_success_msg: "Project creation is successful"
+organization_fail_msg: "Organization is not created"
+organization_success_msg: "Organization is created"
+schedule_fail_msg: "Schedules are not created"
+schedule_success_msg: "Schedules are created"

control_plane/tools/provision_report.yml

@@ -1,4 +1,4 @@
-# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@
 - name: Fetch provision_password
   hosts: localhost
   connection: local
-  gather_facts: no
+  gather_facts: false
   roles:
     - fetch_password
 
@@ -50,7 +50,7 @@
 
 - name: Find reachable hosts using ssh
   hosts: reachable
-  gather_facts: False
+  gather_facts: false
   ignore_unreachable: true
   remote_user: "root"
   vars:

control_plane/tools/roles/fetch_password/tasks/main.yml

@@ -1,4 +1,4 @@
-#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -42,4 +42,9 @@
   command: >-
     ansible-vault encrypt {{ role_path }}/../../../{{ login_vars_filename }}
     --vault-password-file {{ role_path }}/../../../{{ vault_filename }}
-  changed_when: false
+  changed_when: false
+
+- name: Update login_vars.yml permission
+  file:
+    path: "{{ role_path }}/../../../{{ login_vars_filename }}"
+    mode: "{{ file_perm }}"

control_plane/tools/roles/hpc_cluster_report/tasks/main.yml

@@ -1,4 +1,4 @@
-# Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,9 +42,14 @@
   file:
     path: "{{ role_path}}/files"
     state: directory
-    
+
+- name: Fetch cobbler pod name
+  command: kubectl get pods -n cobbler -o jsonpath="{.items[0].metadata.name}"
+  register: cobbler_pod 
+  changed_when: false
+
 - name: Copy dhcpd.leases from cobbler
-  command: docker cp cobbler:/var/lib/dhcpd/dhcpd.leases {{ role_path}}/files/dhcpd.leases
+  command: kubectl cp {{ cobbler_pod.stdout }}:/var/lib/dhcpd/dhcpd.leases {{ role_path}}/files/dhcpd.leases -n cobbler
   changed_when: true
 
 - name: Fetch ethernet details of unreachable hosts
@@ -88,4 +93,4 @@
 
 - name: Display provision host report
   debug:
-    var: host_report.stdout_lines
+    var: host_report.stdout_lines

control_plane/tools/roles/hpc_cluster_report/templates/provision_host_report.j2

@@ -1,12 +1,10 @@
 HPC Cluster
 -----------
 Reachable Hosts:
-{% if reachable_host_number > 0 %}
+{% if reachable_host_number | int > 0 %}
 {% for host in groups['reachable_ssh'] %}
-{% if reachable_host_number == 1 %}
-  inet={{ host }}, link/ether={{ ethernet_detail_reachable.stdout | replace(';','')}}
-{% elif reachable_host_number > 1 %}
-{% if ethernet_detail_reachable.results[loop.index|int - 1].stdout | length > 1 %}
+{% if reachable_host_number | int >= 1 %}
+{% if ethernet_detail_reachable.results[loop.index|int - 1].stdout | length | int > 1 %}
   inet={{ host }}, link/ether={{ ethernet_detail_reachable.results[loop.index|int - 1].stdout | replace(';','')}}
 {% else %}
   inet={{ host }}, link/ether=Refer to mapping file provided
@@ -17,22 +15,18 @@ Reachable Hosts:
 Total reachable hosts: {{ reachable_host_number }}
 
 Unreachable Hosts:
-{% if unreachable_ping_host_number > 0 %}
+{% if unreachable_ping_host_number | int > 0 %}
 {% for host in groups['ungrouped'] %}
-{% if unreachable_ping_host_number == 1 %}
-  inet={{ host }}, link/ether={{ ethernet_detail_unreachable_ping.stdout | replace(';','')}}
-{% elif unreachable_ping_host_number > 1 %}
+{% if unreachable_ping_host_number | int >=  1 %}
   inet={{ host }}, link/ether={{ ethernet_detail_unreachable_ping.results[loop.index|int - 1].stdout | replace(';','')}}
 {% endif %}
 {% endfor %}
 {% endif %}
-{% if unreachable_ssh_host_number > 0 %}
+{% if unreachable_ssh_host_number | int  > 0 %}
 {% for host in groups['unreachable_ssh'] %}
-{% if unreachable_ssh_host_number == 1 %}
-  inet={{ host }}, link/ether={{ ethernet_detail_unreachable_ssh.stdout | replace(';','')}}
-{% elif unreachable_ssh_host_number > 1 %}
+{% if unreachable_ssh_host_number | int >= 1 %}
   inet={{ host }}, link/ether={{ ethernet_detail_unreachable_ssh.results[loop.index|int - 1].stdout | replace(';','')}}
 {% endif %}
 {% endfor %}
 {% endif %}
-Total unreachable hosts: {{ unreachable_host_number }}
+Total unreachable hosts: {{ unreachable_host_number }}

control_plane/tools/roles/idrac_2fa/tasks/configure_smtp.yml

@@ -35,7 +35,7 @@
     manager_attributes:
       RemoteHosts.1.SMTPServerIPAddress: "{{ smtp_server_ip }}"
       RemoteHosts.1.SMTPPort: 25
-      RemoteHosts.1.SMTPAuthentication: "Enabled"
+      RemoteHosts.1.SMTPAuthentication: "{{ smtp_authentication }}"
       RemoteHosts.1.SMTPUserName: "{{ smtp_username }}"
       RemoteHosts.1.SMTPPassword: "{{ smtp_password }}"
       EmailAlert.1.Address: "{{ use_email_address_2fa }}"

control_plane/tools/roles/idrac_2fa/tasks/validate_2fa_vars.yml

@@ -76,10 +76,16 @@
           ipv4_static_dns1 | length < 1 or
           ipv4_static_dns2 | length < 1 or
           smtp_server_ip | length < 1 or
-          smtp_username | length < 1 or
-          smtp_password | length < 1 or
           use_email_address_2fa | length < 1 
 
+    - name: Validate SMTP parameters if smtp_authentication is enabled
+      fail:
+        msg: "{{ smtp_input_fail_msg }} when smtp_authentication is enabled"
+      when:
+        - smtp_authentication | lower == "enabled"
+        - smtp_username | length < 1 or
+          smtp_password | length < 1
+
     - name: Assert use_email_address_2fa value
       assert:
         that: '"@" in use_email_address_2fa'
@@ -93,4 +99,4 @@
       changed_when: false
       run_once: true
       when: "'$ANSIBLE_VAULT;' in config_content.stdout"
-  when: two_factor_authentication == "enabled"
+  when: two_factor_authentication == "enabled"