Merge branch 'devel' into pv

.all-contributorsrc

@@ -18,7 +18,8 @@
         "ideas",
         "maintenance",
         "mentoring",
-        "design"
+        "design",
+        "review"
       ]
     },
     {
@@ -122,7 +123,8 @@
       "avatar_url": "https://avatars.githubusercontent.com/u/73212230?v=4",
       "profile": "https://github.com/abhishek-s-a",
       "contributions": [
-        "code"
+        "code",
+        "doc"
       ]
     },
     {
@@ -243,6 +245,46 @@
       "contributions": [
         "ideas"
       ]
+    },
+    {
+      "login": "Kavyabr23",
+      "name": "Kavyabr23",
+      "avatar_url": "https://avatars.githubusercontent.com/u/90390587?v=4",
+      "profile": "https://github.com/Kavyabr23",
+      "contributions": [
+        "code",
+        "test"
+      ]
+    },
+    {
+      "login": "vedaprakashanp",
+      "name": "vedaprakashanp",
+      "avatar_url": "https://avatars.githubusercontent.com/u/90596073?v=4",
+      "profile": "https://github.com/vedaprakashanp",
+      "contributions": [
+        "test",
+        "code"
+      ]
+    },
+    {
+      "login": "Bhagyashree-shetty",
+      "name": "Bhagyashree-shetty",
+      "avatar_url": "https://avatars.githubusercontent.com/u/90620926?v=4",
+      "profile": "https://github.com/Bhagyashree-shetty",
+      "contributions": [
+        "test",
+        "code"
+      ]
+    },
+    {
+      "login": "nihalranjan-hpc",
+      "name": "Nihal Ranjan",
+      "avatar_url": "https://avatars.githubusercontent.com/u/84398828?v=4",
+      "profile": "https://github.com/nihalranjan-hpc",
+      "contributions": [
+        "test",
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

control_plane/input_params/base_vars.yml

@@ -68,6 +68,13 @@ awx_organization: "DellEMC"
 
 ### Usage: provision_cobbler, provision_idrac ###
 
+# This variable is used to set node provisioning method
+# It accepts values: idrac, pxe
+# Default value is "idrac"
+# If provisioning needs to be done through cobbler, set it to "pxe"
+# If idrac license is not present, provisioning mode will be set to "pxe"
+provision_method: "idrac"
+
 # This is the timezone that will be set during provisioning of OS
 # Available timezones are provided in control_plane/common/files/timezone.txt
 # Default timezone will be "GMT"
@@ -142,4 +149,4 @@ ib_network_dhcp_end_range: ""
 # Eg: xx:yy:zz:aa:bb,server,172.17.0.5
 # A template for mapping file exists in omnia/examples and is named as mapping_file.csv.
 # This depicts the path where user has kept the mapping file for DHCP configurations.
-ib_mapping_file_path: ""
+ib_mapping_file_path: ""

control_plane/roles/control_plane_common/tasks/fetch_base_inputs.yml

@@ -34,7 +34,8 @@
       mngmnt_network_dhcp_end_range | length < 1 or
       host_network_nic | length < 1 or
       host_network_dhcp_start_range | length < 1 or
-      host_network_dhcp_end_range | length < 1
+      host_network_dhcp_end_range | length < 1 or
+      provision_method | length < 1
 
 - name: Validate infiniband base_vars are not empty
   assert:
@@ -47,7 +48,6 @@
   register: ib_check
   when: ib_switch_support
 
-
 - name: Set facts to validate snmp support
   set_fact:
     snmp_enabled: false   
@@ -159,6 +159,13 @@
     success_msg: "{{ success_awx_organization }}"
     fail_msg: "{{ fail_awx_organization }}"
 
+- name: Assert provisioning method
+  assert:
+    that:
+      - provision_method == "pxe" or provision_method == "idrac"
+    success_msg: "{{ success_provision_method }}"
+    fail_msg: "{{ fail_provision_method }}"
+    
 - name: Check timezone file
   command: grep -Fx "{{ timezone }}" {{ role_path }}/files/timezone.txt
   ignore_errors: yes
@@ -452,4 +459,4 @@
       - ib_network_nic != host_network_nic
     success_msg: "{{ success_msg_different_nics_ib }}"
     fail_msg: "{{ fail_msg_different_nics_ib }}"
-  when: ib_switch_support
+  when: ib_switch_support

control_plane/roles/control_plane_common/vars/main.yml

@@ -89,6 +89,8 @@ success_msg_k8s_pod_network_cidr: "Appliance k8s pod network cidr validated"
 fail_msg_k8s_pod_network_cidr: "Failed. Incorrect appliance k8s pod network cidr provided in base_vars.yml"
 success_awx_organization: "awx organization validated"
 fail_awx_organization: "Failed. Incorrect format in awx organization"
+success_provision_method: "Provision method validated"
+fail_provision_method: "Failed. Provision method can either be set to idrac or pxe"
 success_timezone_msg: "timezone validated"
 fail_timezone_msg: "Failed. Incorrect timezone provided. Please check the file timezone.txt in control_plane/roles/control_plane_common/files/ folder"
 fail_language: "Failed. Only en-US(english) language supported"
@@ -137,4 +139,4 @@ exports_file_path: /etc/exports
 nfs_services:
   - mountd
   - rpc-bind
-  - nfs
+  - nfs

control_plane/roles/control_plane_customiso/files/temp_centos7.cfg

@@ -53,4 +53,9 @@ reboot
 %packages
 @core
 net-tools
-%end
+%end
+
+%post --log=/root/ks-post.log
+yum groupinstall "Infiniband Support" -y
+yum install infiniband-diags perftest qperf -y
+%end

control_plane/roles/control_plane_customiso/vars/main.yml

@@ -32,9 +32,15 @@ host_nic:
  - em4
  - p4p1
  - p4p2
+ - p3p1
+ - p3p2
+ - p2p1
+ - p2p2
+ - p1p2
+ - p1p1
 
 #Usage: create_unattended_iso.yml
 unattended_iso_filename: unattended_centos7.iso
 custom_iso_success_msg: "Unattended ISO file created successfully"
 custom_iso_fail_msg: "Unattended ISO file creation failed. Ensure /mnt/iso path is mounted with valid centos minimal ISO file."
-management_station_ip_file: "management_station_ip.txt"
+management_station_ip_file: "management_station_ip.txt"

control_plane/roles/provision_cobbler/files/cobbler_configurations.yml

@@ -105,6 +105,27 @@
       mode: 0775
     tags: install
 
+  - name: Pxe menu
+    copy:
+      src: "/root/omnia/control_plane/roles/provision_cobbler/files/menu.yml"
+      dest: "/etc/cobbler/boot_loader_conf/pxedefault.template"
+      mode: 0775
+    tags: install
+
+  - name: Assign default grub option
+    replace:
+      path: "/var/lib/cobbler/grub_config/grub/grub.cfg"
+      regexp: "^set default=\'local\'"
+      replace: "set default=\'1\'"
+    tags: install
+
+  - name: Assign default grub timeout
+    replace:
+      path: "/var/lib/cobbler/grub_config/grub/grub.cfg"
+      regexp: '^set timeout=80'
+      replace: 'set timeout=10'
+    tags: install
+
   - name: Syncing of cobbler
     command: cobbler sync
     changed_when: false 
@@ -144,4 +165,4 @@
     cron:
       name: Create inventory
       minute: "*/5"
-      job: "{{ ansible_playbook_path.stdout.split(' ')[1] }} /root/inventory_creation.yml"
+      job: "{{ ansible_playbook_path.stdout.split(' ')[1] }} /root/inventory_creation.yml"

control_plane/roles/provision_idrac/tasks/check_prerequisites.yml

@@ -42,30 +42,6 @@
 - name: Set management_station_ip
   set_fact:
     management_station_ip: "{{ fetch_ip.stdout }}"
-    
-- name: Check NFS share access
-  idrac_server_config_profile:
-    idrac_ip: "{{ inventory_hostname }}"
-    idrac_user: "{{ idrac_username }}"
-    idrac_password: "{{ idrac_password }}"
-    share_name: "{{ management_station_ip }}:{{ nfs_share_offline_repo }}"
-    command: "export"
-    scp_components: "BIOS"
-    scp_file: "{{ nfs_check_file }}"
-    export_format: XML
-    export_use: Default
-    job_wait: true
-  register: nfs_check
-  ignore_errors: true
-  until: not nfs_check.failed
-  retries: 3
-
-- name: Missing entries in nfs exports
-  fail:
-    msg: "{{ missing_exports_fail_msg }}"
-  when:
-    - nfs_check_key in nfs_check.msg or
-      nfs_check_key in nfs_check.scp_status.Status
 
 - name: Initialize variables
   set_fact:
@@ -77,6 +53,8 @@
     datacenter_license: false
     provision_status: false
     model_status: false
+    idrac_license_name: ""
+    deploy_os_status: false
 
 - block:
     - name: Check tower_cli.cfg is encrypted
@@ -192,6 +170,7 @@
     - name: Set enterprise license status
       set_fact:
         enterprise_license: true
+        idrac_license_name: "{{ idrac_info.system_info.License[my_idx1].LicenseDescription }}"
       with_items: "{{ idrac_info.system_info.License }}"
       when:
         - '"iDRAC" in idrac_info.system_info.License[my_idx1].LicenseDescription'
@@ -204,6 +183,7 @@
     - name: Set datacenter license status
       set_fact:
         datacenter_license: true
+        idrac_license_name: "{{ idrac_info.system_info.License[my_idx1].LicenseDescription }}"
       with_items: "{{ idrac_info.system_info.License }}"
       when:
         - '"iDRAC" in idrac_info.system_info.License[my_idx2].LicenseDescription'
@@ -212,4 +192,47 @@
         - '"Healthy" in idrac_info.system_info.License[my_idx2].PrimaryStatus'
       loop_control:
         index_var: my_idx2
-  when: not provision_status
+
+    - name: Change provision mode in absence of license
+      set_fact:
+        provision_method: "pxe"
+      when: not (enterprise_license or datacenter_license)
+
+    - name: Firmware version of iDRAC9 not supported
+      debug:
+        msg: "{{ idrac9_firmware_not_supported_msg }}"
+      when:
+        - '"iDRAC9" in idrac_license_name'
+        - idrac_info.system_info.iDRAC[0].FirmwareVersion < idrac9_supported_version
+
+    - name: Firmware version of iDRAC8 not supported
+      debug:
+        msg: "{{ idrac8_firmware_not_supported_msg }}"
+      when:
+        - '"iDRAC8" in idrac_license_name'
+        - idrac_info.system_info.iDRAC[0].FirmwareVersion < idrac8_supported_version
+
+    - name: Check NFS share access
+      dellemc.openmanage.idrac_server_config_profile:
+        idrac_ip: "{{ inventory_hostname }}"
+        idrac_user: "{{ idrac_username }}"
+        idrac_password: "{{ idrac_password }}"
+        share_name: "{{ management_station_ip }}:{{ nfs_share_offline_repo }}"
+        command: "export"
+        scp_components: "BIOS"
+        scp_file: "{{ nfs_check_file }}"
+        export_format: XML
+        export_use: Default
+        job_wait: true
+      register: nfs_check
+      ignore_errors: true
+      until: not nfs_check.failed
+      retries: "{{ retries_count }}"
+
+    - name: Missing entries in nfs exports
+      fail:
+        msg: "{{ missing_exports_fail_msg }}"
+      when:
+        - nfs_check_key in nfs_check.msg or
+          nfs_check_key in nfs_check.scp_status.Status
+  when: not provision_status

control_plane/roles/provision_idrac/tasks/deploy_os.yml

@@ -13,24 +13,21 @@
 # limitations under the License.
 ---
 
-- name: Set one-time boot device to PXE
-  community.general.redfish_command:
-    category: Systems
-    command: SetOneTimeBoot
-    bootdevice: "Pxe"
-    baseuri: "{{ inventory_hostname }}"
-    username: "{{ idrac_username }}"
-    password: "{{ idrac_password }}"
-  when: not (enterprise_license or datacenter_license)
+- name: Configure boot order for PXE booting
+  dellemc.openmanage.idrac_bios:
+    idrac_ip: "{{ inventory_hostname }}"
+    idrac_user: "{{ idrac_username }}"
+    idrac_password: "{{ idrac_password }}"
+    attributes:
+      SetBootOrderEn: NIC.PxeDevice.1-1,NIC.PxeDevice.2-1,NIC.PxeDevice.3-1,NIC.PxeDevice.4-1
+      UefiBootSeq: NIC.PxeDevice.1-1,NIC.PxeDevice.2-1,NIC.PxeDevice.3-1,NIC.PxeDevice.4-1
+  register: deploy_os_pxe
+  when: provision_method == "pxe"
 
-- name: Reboot server
-  dellemc.openmanage.redfish_powerstate:
-    baseuri: "{{ inventory_hostname }}"
-    username: "{{ idrac_username }}"
-    password: "{{ idrac_password }}"
-    reset_type: ForceRestart
-  when: not (enterprise_license or datacenter_license)
-  register: deploy_os
+- name: Set deploy_os_status when provision_method == pxe
+  set_fact:
+    deploy_os_status: "{{ not deploy_os_pxe.failed }}"
+  when: provision_method == "pxe"
 
 - name: Install OS using iDRAC
   dellemc.openmanage.idrac_os_deployment:
@@ -40,12 +37,17 @@
     share_name: "{{ management_station_ip }}:{{ nfs_share_offline_repo }}"
     iso_image: "{{ unattended_iso_filename }}"
     expose_duration: "{{ expose_duration }}"
-  register: deploy_os
-  when: enterprise_license or datacenter_license
+  register: deploy_os_idrac
+  when: provision_method == "idrac"
+
+- name: Set deploy_os_status when provision_method == idrac
+  set_fact:
+    deploy_os_status: "{{ not deploy_os_idrac.failed }}"
+  when: provision_method == "idrac"
 
 - name: Add to provisioned_hosts to inventory
   command: >-
-    awx --conf.host {{ awx_host }} --conf.username {{ awx_username }} --conf.password {{ awx_password }} 
+    awx --conf.host {{ awx_host }} --conf.username {{ awx_username }} --conf.password {{ awx_password }}
     hosts create --name {{ inventory_hostname }} --inventory "{{ provisioned_idrac_inventory_name }}"
   register: update_inventory
   changed_when: true
@@ -53,9 +55,9 @@
   when:
     - awx_search_key in hostname.stdout
     - inventory_hostname not in fetch_inventory.stdout
-    - not deploy_os.failed
+    - deploy_os_status
 
 - name: Provision OS status
   debug:
     msg: "{{ provision_os_msg }}"
-  when: not deploy_os.failed
+  when: deploy_os_status

control_plane/roles/provision_idrac/tasks/import_scp.yml

@@ -44,22 +44,29 @@
     path: "{{ role_path }}/files/{{ scp_filename }}"
     line: "{{ item }}"
     insertafter: '^(.*)SysProfile'
-    with_items:
-      - '  <Attribute Name="PxeDev1VlanPriority">0</Attribute>'
-      - '  <Attribute Name="PxeDev1Interface">NIC.Integrated.1-1-1</Attribute>'
-      - '  <Attribute Name="PxeDev1VlanId">1</Attribute>'
-      - '  <Attribute Name="PxeDev1VlanEnDis">Enabled</Attribute>'
-      - '  <Attribute Name="PxeDev1Protocol">IPv4</Attribute>'
-      - '  <Attribute Name="PxeDev1EnDis">Enabled</Attribute>'
-  when: not (enterprise_license or datacenter_license)
+  with_items:
+    - '  <Attribute Name="PxeDev1EnDis">Enabled</Attribute>'
+    - '  <Attribute Name="PxeDev2EnDis">Enabled</Attribute>'
+    - '  <Attribute Name="PxeDev3EnDis">Enabled</Attribute>'
+    - '  <Attribute Name="PxeDev4EnDis">Enabled</Attribute>'
+    - '  <Attribute Name="HttpDev1EnDis">Disabled</Attribute>'
+    - '  <Attribute Name="HttpDev2EnDis">Disabled</Attribute>'
+    - '  <Attribute Name="HttpDev3EnDis">Disabled</Attribute>'
+    - '  <Attribute Name="HttpDev4EnDis">Disabled</Attribute>'
+  when: provision_method == "pxe"
   run_once: true
 
 - name: Disable PXE attributes to SCP file
   lineinfile:
     path: "{{ role_path }}/files/{{ scp_filename }}"
-    line: '  <Attribute Name="PxeDev1EnDis">Disabled</Attribute>'
+    line: "{{ item }}"
     insertafter: '^(.*)SysProfile'
-  when: enterprise_license or datacenter_license
+  with_items:
+    - '  <Attribute Name="PxeDev1EnDis">Disabled</Attribute>'
+    - '  <Attribute Name="PxeDev2EnDis">Disabled</Attribute>'
+    - '  <Attribute Name="PxeDev3EnDis">Disabled</Attribute>'
+    - '  <Attribute Name="PxeDev4EnDis">Disabled</Attribute>'
+  when: provision_method == "idrac"
   run_once: true
 
 - name: Add SNMP community name attribute to SCP file
@@ -89,12 +96,14 @@
     command: "import"
     scp_file: "{{ scp_filename }}"
     scp_components: "ALL"
-    shutdown_type: "Graceful"
+    shutdown_type: "Forced"
     job_wait: "True"
   register: import_scp_status
-
+  until: not import_scp_status.failed
+  retries: "{{ retries_count }}"
+  
 - name: Remove the SCP file
   file:
     path: "{{ role_path }}/files/{{ scp_filename }}"
     state: absent
-  run_once: true
+  run_once: true

control_plane/roles/provision_idrac/vars/main.yml

@@ -44,6 +44,11 @@ nfs_check_file: "nfs_check.xml"
 provisioned_idrac_inventory_name: "provisioned_idrac_inventory"
 awx_vars_filename: ".tower_cli.cfg"
 awx_vaultname: ".tower_vault_key"
+idrac9_supported_version: "4.40.40.00"
+idrac9_firmware_not_supported_msg: "[WARNING]Firmware version of iDRAC9 less than 4.40.40.00 is not supported for provisioning. Following tasks can be failed due to older firmware version. In case of failure, update firmware manually and re-run the idrac_template"
+idrac8_supported_version: "2.75.75.75"
+idrac8_firmware_not_supported_msg: "[WARNING]Firmware version of iDRAC8 less than 2.75.75.75 is not supported for provisioning. Following tasks can be failed due to older firmware version. In case of failure, update firmware manually and re-run the idrac_template"
+retries_count: 5
 
 # Usage: update_firmware.yml
 idrac_port: 443
@@ -60,4 +65,4 @@ raid_level: "RAID 0"
 # Usage: deploy_os.yml
 expose_duration: 60
 file_permission: 0644
-provision_os_msg: "OS provisioning is initiated. Wait for installation to complete for all servers."
+provision_os_msg: "OS provisioning is initiated. Wait for installation to complete for all servers."

control_plane/test/temp_scp.xml

@@ -0,0 +1,10 @@
+<SystemConfiguration>
+<Component FQDD="BIOS.Setup.1-1">    
+<Attribute Name="BootMode">Uefi</Attribute>
+<Attribute Name="PxeDev1EnDis">Enabled</Attribute>
+<Attribute Name="SysProfile">PerfOptimized</Attribute>
+</Component>
+<Component FQDD="iDRAC.Embedded.1">
+<Attribute Name="SNMPAlert.1#State">Enabled</Attribute>
+</Component>
+</SystemConfiguration>

control_plane/test/test_idrac_inventory

@@ -0,0 +1,2 @@
+[all]
+10.10.10.10

control_plane/test/test_idrac_validation.yml

@@ -0,0 +1,197 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---  
+- block: 
+
+   - name: Include idrac vars
+     include_vars: "../input_params/idrac_vars.yml"
+  
+   - name: Export SCP configuration to local file
+     idrac_server_config_profile:
+      idrac_ip: "{{ inventory_hostname }}"
+      idrac_user: "{{ idrac_username }}"
+      idrac_password: "{{ idrac_password }}"
+      share_name: "{{ playbook_dir }}"
+      command: "export"
+      scp_file: "{{ inventory_hostname }}_scp.xml"
+      job_wait: true
+     run_once: true
+         
+   - name: Fetch the SCP configuration
+     command: cat "{{ inventory_hostname }}_scp.xml"
+     register: config_file
+           
+   - name: Testcase to verify the boot mode
+     assert:
+      that: 
+        - config_file.stdout | regex_search( bootmode_regex1 )
+      fail_msg: " {{ bootmode_check_fail_msg }}"
+      success_msg: "{{ bootmode_check_success_msg }}"
+     ignore_errors: true
+         
+   - name: Testcase to verify the SNMP configuration
+     assert:
+      that: 
+       - config_file.stdout | regex_search(snmp_regex1)
+      fail_msg: " {{ snmp_check_fail_msg }}"
+      success_msg: "{{ snmp_check_success_msg }}"
+     ignore_errors: true
+     
+   - name: Testcase to verify the system profile value Performance
+     assert:
+      that: 
+       - config_file.stdout | regex_search(sysprofile_regex1)
+      fail_msg: " {{ sysprofile_check_fail_msg }}"
+      success_msg: "{{ sysprofile_check_success_msg }}"
+     ignore_errors: true
+     when: idrac_system_profile == sysprofile_value1
+          
+   - name: Testcase to verify the system profile value PerformancePerWatt(DAPC)
+     assert:
+      that: 
+       - config_file.stdout | regex_search(sysprofile_regex2)
+      fail_msg: " {{ sysprofile_check_fail_msg }}"
+      success_msg: "{{ sysprofile_check_success_msg }}"
+     ignore_errors: true
+     when: idrac_system_profile == sysprofile_value2
+
+   - name: Testcase to verify the system profile value WorkstationPerformance
+     assert:
+      that: 
+       - config_file.stdout | regex_search(sysprofile_regex3)
+      fail_msg: " {{ sysprofile_check_fail_msg }}"
+      success_msg: "{{ sysprofile_check_success_msg }}"
+     ignore_errors: true
+     when: idrac_system_profile == sysprofile_value3
+
+   - name: Testcase to verify the system profile value PerformancePerWatt(OS)
+     assert:
+      that: 
+       - config_file.stdout | regex_search(sysprofile_regex4)
+      fail_msg: " {{ sysprofile_check_fail_msg }}"
+      success_msg: "{{ sysprofile_check_success_msg }}"
+     ignore_errors: true
+     when: idrac_system_profile == sysprofile_value4  
+
+   - name: Testcase to verify the pxe device status
+     assert:
+      that: 
+       - config_file.stdout | regex_search(pxedevice)
+      fail_msg: " {{ pxedevice_check_fail_msg }}"
+      success_msg: "{{ pxedevice_check_success_msg }}"
+     ignore_errors: true
+     when: not (enterprise_license or datacenter_license)
+     
+   - name: Initialized RAID status
+     set_fact:
+       raid_type: false
+               
+   - name: Get iDRAC info details
+     idrac_system_info:
+      idrac_ip: "{{ inventory_hostname }}"
+      idrac_user: "{{ idrac_username }}"
+      idrac_password: "{{ idrac_password }}"
+     register: idrac_info 
+
+   - name: Set RAID status
+     set_fact:
+       raid_type: true
+     with_items: "{{ idrac_info.system_info.Controller }}"
+     loop_control:
+       index_var: my_idx3
+     when: '"RAID" in idrac_info.system_info.ControllerSensor[my_idx3].FQDD'
+                
+   - name: Testcase to verify virtual disk creation status
+     assert:
+      that:
+       - " 'omnia_vd' in idrac_info.system_info.VirtualDisk[0].Name "
+      fail_msg: "{{ vd_fail_msg }}"
+      success_msg: " {{ vd_success_msg }}"
+     ignore_errors: true
+     when: raid_type
+
+   - name: Execute get pods command
+     command: "kubectl get pods -n {{ awx_namespace }}"
+     changed_when: true
+     register: k8s_pods
+     ignore_errors: true
+     run_once: true
+     
+   - name: Get awx pod 
+     set_fact:
+      awx_pods: "{{ item | regex_search(awx_pod_regex) | trim  }}"
+     with_items: 
+       - "{{ k8s_pods.stdout_lines }}"
+     run_once: true
+     ignore_errors: true
+     when: item | regex_search(awx_pod_item_regex)
+
+   - name: Get awx cluster ip
+     shell: "kubectl get svc awx-ui -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+     register: awx_cluster_ip
+     changed_when: false
+     ignore_errors: true
+
+   - name: Get AWX admin password
+     shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+     register: awx_admin_password
+     changed_when: false
+     ignore_errors: true
+          
+   - name: Execute awx get inventory hosts command
+     shell: "awx --conf.host http://{{ awx_cluster_ip.stdout }}:8052 --conf.username admin --conf.password {{ awx_admin_password.stdout }} --conf.insecure hosts list --inventory {{ provisioned_inventory_name }} -f human --filter 'name'"
+     changed_when: true
+     register: idrac_hosts
+     run_once: true
+     ignore_errors: true
+               
+   - name: Testcase to verify IP added to provisioned_idrac_inventory
+     assert:
+      that: 
+        - idrac_hosts.stdout | regex_search( inventory_hostname )
+      fail_msg: "{{ provisioned_ip_fail_msg }}"
+      success_msg: "{{ provisioned_ip_success_msg }}"
+     ignore_errors: true 
+          
+   - name: Update firmware
+     idrac_firmware:
+      idrac_ip: "{{ inventory_hostname }}"
+      idrac_user: "{{ idrac_username }}"
+      idrac_password: "{{ idrac_password }}"
+      share_name: "/var/nfs_repo/dellupdates"
+      reboot: false
+      job_wait: true
+      apply_update: False
+      catalog_file_name: "Catalog.xml"
+     register: idrac_firmware
+     when: firmware_update_required
+
+   - name: Checking firmware update status
+     assert:
+      that:
+        - idrac_firmware.msg in firmware_status
+      fail_msg: "{{ firmware_fail_msg }}"
+      success_msg: "{{ firmware_success_msg }}"
+     ignore_errors: true
+     when: firmware_update_required  
+     
+   - name: Remove the scp file
+     file:
+       path: "{{ inventory_hostname }}_scp.xml"
+       state: absent
+     run_once: true             
+      
+  rescue:
+    - debug:
+       msg: "{{ failed_msg }}"

control_plane/test/test_prepare.yml

@@ -0,0 +1,135 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+- block:
+
+    - name: Get awx-service Cluster-IP
+      command: "kubectl get svc awx-service -n {{ awx_namespace }} -o jsonpath='{.spec.clusterIP}'"
+      register: awx_cluster_ip
+      changed_when: false
+
+    - name: Get AWX admin password
+      shell: "kubectl get secret awx-admin-password -n {{ awx_namespace }} -o jsonpath='{.data.password}' | base64 --decode"
+      register: awx_admin_password
+      changed_when: false
+
+    - name: Set IP and password
+      set_fact:
+        awx_ip: 'http://{{ awx_cluster_ip.stdout }}'
+        admin_password: "{{ awx_admin_password.stdout }}"
+      no_log: true
+
+    - name: Check if {{ tower_config_file_path }} file is encrypted
+      command: cat {{ tower_config_file_path }}
+      changed_when: false
+      no_log: true
+      register: config_content
+      run_once: true
+
+    - name: Decrpyt {{ tower_config_file_path }}
+      command: >-
+        ansible-vault decrypt {{ tower_config_file_path }}
+        --vault-password-file {{ tower_vault_file_path }}
+      when: "'$ANSIBLE_VAULT;' in config_content.stdout"
+      no_log: true
+      changed_when: false
+      run_once: true
+
+    - name: Change tower_config file permissions
+      file:
+        path: "{{ tower_config_file_path }}"
+        mode: "{{ file_perm }}"
+
+    - name: Create an inventory
+      tower_inventory:
+        name: "{{ inventory_name }}"
+        organization: "{{ organization_name }}"
+        tower_config_file: "{{ tower_config_file_path }}"
+        state: present
+
+    - name: Create a host with inventory_hostname
+      tower_host:
+        name: "{{ inventory_hostname }}"
+        inventory: "{{ inventory_name }}"
+        state: present
+        tower_config_file: "{{ tower_config_file_path }}"
+      when: inventory_hostname is defined
+
+    - name: Create a job template
+      awx.awx.tower_job_template:
+        name: "{{ job_template_name }}"
+        job_type: "run"
+        organization: "{{ organization_name }}"
+        inventory: "{{ inventory_name }}"
+        project: "{{ project_name }}"
+        playbook: "{{ playbook_path }}"
+        credentials:
+          - "{{ item.credential }}"
+        state: present
+        tower_config_file: "{{ tower_config_file_path }}"
+      loop: "{{ job_template_details }}"
+      when: item.name == template_name
+
+    - name: Change file permissions
+      file:
+        path: "../input_params/login_vars.yml"
+        owner: root
+        mode: "{{ file_perm }}"
+
+    - name: Launch a job
+      awx.awx.tower_job_launch:
+        job_template: "{{ job_template_name }}"
+        wait: yes
+        tower_config_file: "{{ tower_config_file_path }}"
+      failed_when: false
+      register: job_status
+
+    - name: Delete host created with inventory_hostname
+      tower_host:
+        name: "{{ inventory_hostname }}"
+        inventory: "{{ inventory_name }}"
+        state: absent
+        tower_config_file: "{{ tower_config_file_path }}"
+      when: inventory_hostname is defined
+
+    - name: Delete an inventory
+      tower_inventory:
+        name: "{{ inventory_name }}"
+        organization: "{{ organization_name }}"
+        state: absent
+        tower_config_file: "{{ tower_config_file_path }}"
+      when: delete_status
+
+    - name: Delete  a job template
+      awx.awx.tower_job_template:
+        name: "{{ job_template_name }}"
+        state: absent
+        tower_config_file: "{{ tower_config_file_path }}"
+      when: delete_status
+
+    - name: Encrypt {{ tower_config_file_path }}
+      command: >-
+        ansible-vault encrypt {{ tower_config_file_path }}
+        --vault-password-file {{ tower_vault_file_path }}
+      changed_when: false
+
+    - name: Change file permissions
+      file:
+        path: "{{ tower_config_file_path }}"
+        owner: root
+        mode: "{{ file_perm }}"      
+
+  rescue:
+    - debug:
+       msg: "{{ failed_msg }}"

control_plane/test/test_pv_inventory

@@ -0,0 +1,2 @@
+[powervault]
+192.26.0.1

control_plane/test/test_vars/test_idrac_vars.yml

@@ -0,0 +1,75 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+# Usage: test_prepare.yml
+idrac_inventory_name: "test_idrac_inventory"
+template_value: 'idrac_template'
+job_name: "test_idrac_template"
+idrac_playbook_path: "control_plane/idrac.yml"
+tower_config_file_path: "../roles/webui_awx/files/.tower_cli.cfg"
+tower_vault_file_path: "../roles/webui_awx/files/.tower_vault_key"
+file_perm: '0644'
+
+# Usage: test_idrac.yml
+validation_script_path: "test_idrac_validation.yml"
+awx_script_path: "test_prepare.yml"
+temp_scp_path: "temp_scp.xml"
+idrac_var_path: "../input_params/idrac_vars.yml"
+login_vars_path: "../input_params/login_vars.yml"
+login_vars_vault_path: "../input_params/.login_vault_key"
+vd_name: "omnia_vd"
+bootmode_attr_name: 'Name="BootMode"'
+sysprofile_attr_name: 'Name="SysProfile"'
+snmp_attr_name: 'Name="SNMPAlert.1#State"'
+sysprofile: "idrac_system_profile:"
+sysprofile_params1: 'idrac_system_profile: "Performance"'
+sysprofile_params2: 'idrac_system_profile: "PerformancePerWatt(DAPC)"'
+sysprofile_params3: 'idrac_system_profile: "WorkstationPerformance"'
+sysprofile_params4: 'idrac_system_profile: "PerformancePerWatt(OS)"'
+sysprofile_value1: "Performance"
+sysprofile_value2: "PerformancePerWatt(DAPC)"
+sysprofile_value3: "WorkstationPerformance"
+sysprofile_value4: "PerformancePerWatt(OS)"
+
+# Usage: test_idrac_validation.yml
+provisioned_inventory_name: "provisioned_idrac_inventory"
+awx_namespace: "awx"
+awx_pod_regex: 'awx-([A-Za-z0-9]{10})-([A-Za-z0-9]{5})'
+awx_pod_item_regex: "awx-([A-Za-z0-9]{10})-([A-Za-z0-9]{5})"
+bootmode_regex1: '<Attribute Name="BootMode">Uefi</Attribute>'
+bootmode_regex2: '<Attribute Name="BootMode">Bios</Attribute>'
+snmp_regex1: '<Attribute Name="SNMPAlert.1#State">Enabled</Attribute>'
+snmp_regex2: '<Attribute Name="SNMPAlert.1#State">Disabled</Attribute>'
+sysprofile_regex1: '<Attribute Name="SysProfile">PerfOptimized</Attribute>'
+sysprofile_regex2: '<Attribute Name="SysProfile">PerfPerWattOptimizedDapc</Attribute>'
+sysprofile_regex3: '<Attribute Name="SysProfile">PerfWorkStationOptimized</Attribute>'
+sysprofile_regex4: '<Attribute Name="SysProfile">PerfPerWattOptimizedOs</Attribute>'
+pxedevice: '<Attribute Name="PxeDev1EnDis">Enabled</Attribute>'
+firmware_status: "The catalog in the repository specified in the operation has the same firmware versions as currently present on the server."
+bootmode_check_fail_msg: "Boot mode configuration is failed"
+bootmode_check_success_msg: "Boot mode configuration is successful"
+snmp_check_fail_msg: "SNMP configuration failed"
+snmp_check_success_msg: "SNMP configuration is successful"
+sysprofile_check_fail_msg: "System profile configuration failed"
+sysprofile_check_success_msg: "System profile configuration is successful"
+pxedevice_check_fail_msg: "Pxe device configuration is failed"
+pxedevice_check_success_msg: "Pxe device configuration is successful"
+vd_success_msg: "Virtual disk created successfully"
+vd_fail_msg: "Virtual disk is not present"
+provisioned_ip_success_msg: "IP added to provisioned_idrac_inventory  is successfully"
+provisioned_ip_fail_msg: "IP is not added to provisioned_idrac_inventory"
+failed_msg: "Failed. Please check input parameters and try again!"
+firmware_fail_msg: "Firmware update is failed"
+firmware_success_msg: "Firmware updated is success"

control_plane/test/test_vars/test_powervault_vars.yml

@@ -0,0 +1,88 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+# vars file for test_powervault.yml file
+connectivity_success_msg: "Connection was successful"
+connectivity_failure_msg: "Connectivity failed"
+product_id_success_msg: "Product id matches the supported powervault versions"
+product_id_failure_msg: "Product id doesn't match the supported powervault versions"
+ip_validation_success_msg: "IP validation was successful"
+ip_validation_failure_msg: "IP validation failed"
+user_validation_success_msg: "Users validation was successful"
+user_validation_failure_msg: "Users validation failed"
+pool_diskgroup_config_success_msg: "Pools and disk groups addition were successful"
+pool_diskgroup_config_failure_msg: "Pools and disk groups addition failed"
+volumes_creation_success_msg: "Volumes creation was successful"
+volumes_creation_failure_msg: "Volumes creation failed"
+snmp_trap_success_msg: "SNMP trap notifications are disabled"
+snmp_trap_enable_success_msg: "SNMP trap notifications are enabled"
+snmp_trap_failure_msg: "Error in setting up the snmp parameters"
+scsi_product_id: "ME4"
+user_roles: "manage"
+user_type: "Standard"
+user_locale: "English"
+pool_a: "A"
+pool_b: "B"
+pool_type: "Virtual"
+raid1_level: "RAID1"
+raid5_level: "RAID5"
+raid6_level: "RAID6"
+raid10_level: "RAID10"
+raid_adapt_level: "ADAPT"
+disk_type_ssdsas: "SSD SAS"
+disk_type_sasmdl: "SAS MDL"
+disk_group_name_omnia_dgA01: "omnia_dgA01"
+disk_group_name_omnia_dgA02: "omnia_dgA02"
+disk_group_name_omnia_dgB01: "omnia_dgB01"
+volume_name_k8s_v1: "k8s_volume"
+volume_name_slurm_v1: "slurm_volume"
+volume_name_k8s_v2: "k8s_V2"
+volume_name_slurm_v2: "slurm_V2"
+volume_size_input_100gb: "100GB"
+volume_size_100gb: "99.9GB"
+disk_range_5_6: "0.5-6"
+disk_range_3_4: "0.3-4"
+disk_range_7_10: "0.7-10"
+disk_range_8_11: "0.8-9:0.10-11"
+disk_range_0_2: "0.0-2"
+disk_range_2_3: "0.2-3"
+disk_range_0_11: "0.0-11"
+dhcp_start_range: "192.168.0.0"
+dhcp_end_range: "192.168.0.100"
+ip_range: "192.168"
+snmp_notify_level_crit: "crit"
+snmp_notify_level_none: "none"
+snmp_destination: "100.96.22.199"
+snmp_notify_status_enable: "Enabled"
+snmp_notify_status_disable: "Disabled"
+
+login_vars_path: "../input_params/login_vars.yml"
+login_vars_vault_path: "../input_params/.login_vault_key"
+powervault_me4_var_path: "../input_params/powervault_me4_vars.yml"
+base_var_path: "../input_params/base_vars.yml"
+powervault_inventory_name: "powervault_me4_inventory"
+template_value: "powervault_me4_template"
+job_name: "test_powervault_me4_template"
+powervault_playbook_path: "control_plane/powervault_me4.yml"
+awx_script_path: "test_prepare.yml"
+failed_job_status: "failed"
+success_job_status: "successful"
+playbook_exec_success_msg: "Powervault playbook execution completed successfully"
+playbook_exec_fail_msg: "Job execution failed. Please check input parameters and try again!"
+failed_job_status_success_msg: "Execution failed. Please check AWX console for appropriate error message"
+failed_msg: "Failed. Please check input parameters and try again!"
+tower_config_file_path: "../roles/webui_awx/files/.tower_cli.cfg"
+tower_vault_file_path: "../roles/webui_awx/files/.tower_vault_key"
+file_perm: '0644'

docs/FAQ.md

@@ -112,7 +112,7 @@ Resolution:
 It is recommended that the ansible-vault view or edit commands are used and not the ansible-vault decrypt or encrypt commands.
 
 ## What to do if the LC is not ready?
-* Ensure the LC is in a ready state for all the servers.
+* Verify the state of the LC in all servers by running `racadm getremoteservicesstatus`
 * Launch iDRAC template.
 
 ## What to do if the network CIDR entry of iDRAC IP in /etc/exports file is missing?
@@ -127,6 +127,15 @@ It is recommended that the ansible-vault view or edit commands are used and not
 ## Is Disabling 2FA supported by Omnia?
 * Disabling 2FA is not supported by Omnia and must be manually disabled.
 
+## Is provisioning server using BOSS controller supported by Omnia?
+* Provisioning server using BOSS controller is not supported by Omnia. It will be supported in upcoming releases.
+
 ## The provisioning of PowerEdge servers failed. How do I clean up before starting over?
 1. Delete the respective iDRAC IP addresses from the *provisioned_idrac_inventory* on the AWX UI or delete the *provisioned_idrac_inventory* to delete the iDRAC IP addresses of all the servers in the cluster.
-2. Launch the iDRAC template from the AWX UI.
+2. Launch the iDRAC template from the AWX UI.
+
+## What to do when WARNING message regarding older firmware displayed during idrac_template execution and idrac_template task failed?
+Potential Cause: Older firmware version in PowerEdge servers. Omnia supports only iDRAC 8 based Dell EMC PowerEdge Servers with firmware versions 2.75.75.75 and above and iDRAC 9 based Dell EMC PowerEdge Servers with Firmware versions 4.40.40.00 and above.
+
+1. Update idrac firmware version in PowerEdge servers manually to the supported version.
+2. Re-run idrac_template.

roles/login_server/tasks/install_packages.yml

@@ -24,6 +24,7 @@
     src: "{{ resolv_conf_path }}"
     dest: "{{ temp_resolv_conf_path }}"
     mode: "{{ resolv_file_mode }}"
+    remote_src: yes
 
 - name: Add the domain name in /etc/resolv.conf
   replace:
@@ -39,4 +40,4 @@
     regexp: "# Generated by NetworkManager"
     replace: "# Generated by NetworkManager\nsearch {{ hostvars['127.0.0.1']['domain_name'] }}"
   when:
-    replace_output.msg | length == 0
+    replace_output.msg | length == 0

roles/slurm_workers/tasks/main.yml

@@ -1,4 +1,4 @@
-#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -114,6 +114,9 @@
   with_items:
     - "{{ groups['compute'] }}"
   when: '"compute" in group_names'
+  delegate_to: "{{ item }}"
+  with_items:
+    - "{{ play_hosts }}"
 
 - name: Add login node core & socket info in slurm config file
   lineinfile:
@@ -127,6 +130,9 @@
   when:
     - hostvars["127.0.0.1"]["login_node_required"]
     - '"login_node" in group_names'
+  delegate_to: "{{ item }}"
+  with_items:
+    - "{{ play_hosts }}"
 
 - name: Update hostnames of compute node when ALL in partition nodes
   replace:
@@ -152,4 +158,4 @@
   fetch:
     src: "{{ slurm_confpth }}"
     dest: "{{ buffer_path }}"
-    flat: true
+    flat: true