Issue #242: Added checks to join compute nodes without redeploying cluster

Signed-off-by: blesson-james <blesson_james@Dellteam>

roles/k8s_start_manager/tasks/main.yml

@@ -22,17 +22,25 @@
   setup:
     filter: ansible_default_ipv4.address
 
-- name: Reset kubeadm
-  command: kubeadm reset -f
-  changed_when: true
+- name: Get K8s nodes status
+  command: kubectl get nodes
+  changed_when: false
   ignore_errors: True
+  register: k8s_nodes
   tags: init
 
-- name: Check K8s nodes status
-  command: kubectl get nodes
+- name: Get K8s ready compute nodes
+  shell: kubectl get node --selector='!node-role.kubernetes.io/master' | grep -v 'NotReady'
   changed_when: false
   ignore_errors: True
-  register: k8s_nodes
+  register: k8s_nodes_ready
+  tags: init
+
+- name: Get K8s not ready compute nodes
+  shell: kubectl get node --selector='!node-role.kubernetes.io/master' | grep 'NotReady'
+  changed_when: false
+  ignore_errors: True
+  register: k8s_nodes_not_ready
   tags: init
 
 - name: Initialize kubeadm
@@ -89,6 +97,8 @@
     hash:   "{{ K8S_MANAGER_CA_HASH.stdout }}"
     ip:     "{{ ansible_default_ipv4.address }}"
     k8s_nodes:  "{{ k8s_nodes.stdout }}"
+    k8s_nodes_ready:  "{{ k8s_nodes_ready.stdout }}"
+    k8s_nodes_not_ready: "{{ k8s_nodes_not_ready.stdout }}"
   tags: init
 
 - name: Print k8s token
@@ -172,4 +182,4 @@
 - name: Edge / Workstation Install allows pods to scheudle on manager
   command: kubectl taint nodes --all node-role.kubernetes.io/master-
   when: groups['manager'][0] == groups['compute'][0] and groups['compute']|length == 1
-  tags: init
+  tags: init

roles/k8s_start_workers/tasks/main.yml

@@ -18,22 +18,29 @@
   changed_when: true
   tags: init
 
-- name: Reset kubeadm
-  command: kubeadm reset -f
-  changed_when: true
-  ignore_errors: True
-  tags: init
-
 - name: Get hostname
   command: hostname
   changed_when: true
   register: node_hostname
   tags: init
 
+- name: Reset kubeadm
+  command: kubeadm reset -f
+  changed_when: true
+  ignore_errors: True
+  when:
+    - groups['manager'][0] != groups['compute'][0]
+    - groups['compute']|length >= 1
+    - node_hostname.stdout in hostvars['K8S_TOKEN_HOLDER']['k8s_nodes_not_ready']
+  tags: init
+
 - name: Execute kubeadm join command
   shell: >
     kubeadm join --token={{ hostvars['K8S_TOKEN_HOLDER']['token'] }}
     --discovery-token-ca-cert-hash sha256:{{ hostvars['K8S_TOKEN_HOLDER']['hash'] }}
     {{ hostvars['K8S_TOKEN_HOLDER']['ip'] }}:{{ apiserver_bind_port }}
-  when: groups['manager'][0] != groups['compute'][0] and groups['compute']|length >= 1 and node_hostname.stdout not in hostvars['K8S_TOKEN_HOLDER']['k8s_nodes']
-  tags: init
+  when:
+    - groups['manager'][0] != groups['compute'][0]
+    - groups['compute']|length >= 1
+    - node_hostname.stdout not in hostvars['K8S_TOKEN_HOLDER']['k8s_nodes_ready']
+  tags: init

roles/slurm_exporter/tasks/start_services.yml

@@ -23,4 +23,5 @@
 - name: Start services
   systemd:
     name: prometheus-slurm-exporter
-    state: restarted
+    state: restarted
+    enabled: yes

roles/slurm_start_services/tasks/main.yml

@@ -36,15 +36,17 @@
     - "{{ groups['compute'] }}"
 
 - name: Enable slurmdbd on manager
-  service:
+  systemd:
     name: slurmdbd
     state: restarted
+    enabled: yes
   tags: install
 
 - name: Start slurmctld on manager
-  service:
+  systemd:
     name: slurmctld
     state: restarted
+    enabled: yes
   tags: install
 
 - name: Show cluster if exists

roles/slurm_workers/tasks/main.yml

@@ -147,4 +147,5 @@
   systemd:
     name: slurmd.service
     state: started
+    enabled: yes
   tags: install