Merge pull request #859 from abhishek-sa1/idrac

Issue #858: Failover mechanism of idrac_template failure

control_plane/roles/provision_idrac/tasks/deploy_os.yml

@@ -33,7 +33,8 @@
         deploy_os_status: "{{ not deploy_os_pxe.failed }}"
   when: provision_method == provision_method_pxe
 
-- block:
+- name: Deploy OS using iDRAC
+  block:
     - name: Set unattended_iso_filename - centos
       set_fact:
         unattended_iso_filename: "{{ centos_iso_filename }}"
@@ -59,9 +60,47 @@
         expose_duration: "{{ expose_duration }}"
       register: deploy_os_idrac
   rescue:
-    - name: OS provisioning failed using iDRAC
-      fail:
-        msg: "{{ idrac_provisioning_fail_msg }}"
+    - name: Retry deploy OS using iDRAC after reset
+      block:
+        - name: Reset iDRAC
+          dellemc.openmanage.idrac_reset:
+            idrac_ip: "{{ inventory_hostname }}"
+            idrac_user: "{{ idrac_username }}"
+            idrac_password: "{{ idrac_password }}"
+
+        - name: Wait for iDRAC Port to become active
+          wait_for:
+            host: "{{ inventory_hostname }}"
+            port: "{{ idrac_port }}"
+            delay: "{{ max_delay }}"
+            timeout: "{{ max_timeout }}"
+            state: started
+          register: wait_for_idrac
+          failed_when: wait_for_idrac.elapsed < idrac_reset_time
+
+        - name: Get LC status
+          dellemc.openmanage.idrac_lifecycle_controller_status_info:
+            idrac_ip: "{{ inventory_hostname }}"
+            idrac_user: "{{ idrac_username }}"
+            idrac_password: "{{ idrac_password }}"
+          register: lc_status
+          until: lc_status.lc_status_info.LCReady
+          retries: "{{ max_retries }}"
+          delay: "{{ max_delay }}"
+
+        - name: Retry installing OS using iDRAC
+          dellemc.openmanage.idrac_os_deployment:
+            idrac_ip: "{{ inventory_hostname }}"
+            idrac_user: "{{ idrac_username }}"
+            idrac_password: "{{ idrac_password }}"
+            share_name: "{{ management_station_ip }}:{{ nfs_share_offline_repo }}"
+            iso_image: "{{ unattended_iso_filename }}"
+            expose_duration: "{{ expose_duration }}"
+          register: deploy_os_idrac
+      rescue:
+        - name: OS provisioning failed using iDRAC
+          fail:
+            msg: "{{ idrac_provisioning_fail_msg }}"
   always:
     - name: Set deploy_os_status when provision_method == idrac
       set_fact:

control_plane/roles/provision_idrac/tasks/update_firmware.yml

@@ -1,4 +1,4 @@
-# Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,7 +32,7 @@
         catalog_file_name: "Catalog.xml"
       register: update_firmware
       when: model_status
-
+    
     - name: Wait for one minute to complete firmware update
       wait_for:
         timeout: 60
@@ -43,6 +43,8 @@
       wait_for:
         host: "{{ inventory_hostname }}"
         port: "{{ idrac_port }}"
+        delay: "{{ max_delay }}"
+        timeout: "{{ max_timeout }}"
         state: started
 
     - name: Firmware update status

control_plane/roles/provision_idrac/vars/main.yml

@@ -44,10 +44,10 @@ nfs_check_file: "nfs_check.xml"
 provisioned_idrac_inventory_name: "provisioned_idrac_inventory"
 awx_vars_filename: ".tower_cli.cfg"
 awx_vaultname: ".tower_vault_key"
-idrac9_supported_version: "4.40.40.00"
-idrac9_firmware_not_supported_msg: "[WARNING]Firmware version of iDRAC9 less than 4.40.40.00 is not supported for provisioning. Following tasks can be failed due to older firmware version. In case of failure, update firmware manually and re-run the idrac_template"
-idrac8_supported_version: "2.75.75.75"
-idrac8_firmware_not_supported_msg: "[WARNING]Firmware version of iDRAC8 less than 2.75.75.75 is not supported for provisioning. Following tasks can be failed due to older firmware version. In case of failure, update firmware manually and re-run the idrac_template"
+idrac9_supported_version: "5.00.10.20"
+idrac9_firmware_not_supported_msg: "[WARNING]Firmware version of iDRAC9 less than 5.00.10.20 is not supported for provisioning. Following tasks can be failed due to older firmware version. In case of failure, update firmware manually and re-run the idrac_template"
+idrac8_supported_version: "2.81.81.81"
+idrac8_firmware_not_supported_msg: "[WARNING]Firmware version of iDRAC8 less than 2.81.81.81 is not supported for provisioning. Following tasks can be failed due to older firmware version. In case of failure, update firmware manually and re-run the idrac_template"
 retries_count: 5
 provision_method_idrac: "idrac"
 provision_method_pxe: "PXE"
@@ -72,3 +72,7 @@ provision_os_msg: "OS provisioning is initiated. Wait for installation to comple
 idrac_provisioning_fail_msg: "OS provisioning using iDRAC is failed. This could be due to older firmware or some internal issues with server. Re-run idrac_template after fixing the issue"
 pxe_provisioning_fail_msg: "OS provisioning using PXE is failed. This could be due some internal issues with server. Re-run idrac_template after fixing the issue"
 add_inventory_fail_msg: "Failed. Unable to add provisioned host to provisioned_idrac_inventory."
+max_delay: 40
+max_retries: 30
+max_timeout: 500
+idrac_reset_time: 30