Issue #521: Integrated K8s with NFS Node

Signed-off-by: blesson-james <blesson_james@Dellteam.com>

.all-contributorsrc

@@ -18,7 +18,8 @@
         "ideas",
         "maintenance",
         "mentoring",
-        "design"
+        "design",
+        "review"
       ]
     },
     {

roles/k8s_nfs_client_setup/tasks/main.yml

@@ -21,12 +21,14 @@
   tags: nfs_client
 
 - name: Check mounted share
-  shell: mount | grep nfs
+  shell: >
+    set -o pipefail && \
+    mount | grep nfs
   changed_when: false
   args:
     warn: false
   register: mounted_share
-  ignore_errors: True
+  failed_when: false
   tags: nfs_client
 
 - name: Creating directory to mount NFS Share

roles/k8s_nfs_client_setup/vars/main.yml

@@ -19,6 +19,6 @@ nfs_share_dir: /home/k8snfs
 
 mounthost: "{{ groups['manager'][0] }}"
 
-nfs_mnt_dir_mode: 0755
+nfs_mnt_dir_mode: 0777
 
 fstab_file_path: /etc/fstab

roles/powervault_me4_nfs/tasks/main.yml

@@ -0,0 +1,21 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Include variable file for powervault
+  include_vars: "{{ pv_nfs_file }}"
+  no_log: true
+
+- name: Configure the server
+  include_tasks: nfs_node_configure.yml

roles/powervault_me4_nfs/tasks/me4_nfs_server_setup.yml

@@ -0,0 +1,79 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Install nfs-utils
+  package:
+    name: nfs-utils
+    state: present
+
+- name: Install firewalld
+  package:
+    name: firewalld
+    state: present
+
+- name: Start and enable firewalld
+  service:
+    name: firewalld
+    state: started
+    enabled: yes
+
+- name: Start and enable rpcbind and nfs-server service
+  service:
+    name: "{{ item }}"
+    state: restarted
+    enabled: yes
+  with_items:
+    - rpcbind
+    - nfs-server
+
+- name: Adding K8s NFS share entries in /etc/exports
+  lineinfile:
+    path: "{{ exports_file_path }}"
+    line: "{{ me4_nfs_share_k8s }} {{ item }}(rw,sync,no_root_squash)"
+  with_items:
+    - "{{ groups['manager'] }}"
+    - "{{ groups['compute'] }}"
+
+- name: Adding K8s NFS share entries in /etc/exports
+  lineinfile:
+    path: "{{ exports_file_path }}"
+    line: "{{ me4_nfs_share_k8s }} {{ item }}(rw,sync,no_root_squash)"
+  with_items:
+    - "{{ groups['manager'] }}"
+    - "{{ groups['compute'] }}"
+
+- name: Adding Slurm NFS share entries in /etc/exports
+  lineinfile:
+    path: "{{ exports_file_path }}"
+    line: "{{ me4_nfs_share_slurm }} {{ item }}(rw,sync,no_root_squash)"
+  with_items:
+    - "{{ groups['manager'] }}"
+    - "{{ groups['compute'] }}"
+
+- name: Exporting the shared directories
+  command: exportfs -ra
+  changed_when: true
+
+- name: Configuring firewall
+  firewalld:
+    service: "{{ item }}"
+    permanent: true
+    state: enabled
+  with_items:
+    - "{{ nfs_services }}"
+
+- name: Reload firewalld
+  command: firewall-cmd --reload
+  changed_when: true

roles/powervault_me4_nfs/tasks/mount_me4_partitions.yml

@@ -0,0 +1,111 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Get ME4 volume
+  shell: >
+    set -o pipefail && \
+    lsscsi -s | grep ME4
+  changed_when: false
+  register: me4_output
+  failed_when: false
+
+- name: ME4 volume check
+  fail:
+    msg: "{{ me4_volume_fail_msg }}"
+  when: me4_output is failed or (me4_output.stdout | regex_findall('ME4') | length) != 2
+
+- name: Set ME4 data facts
+  set_fact:
+    me4_k8s_volume_data: "{{ me4_output.stdout.split('\n')[0].split(' ') | select() }}"
+    me4_slurm_volume_data: "{{ me4_output.stdout.split('\n')[1].split(' ') | select() }}"
+
+- name: Add ME4 volume data to dummy host
+  add_host:
+    name:   "NFS_NODE_TOKEN_HOLDER"
+    me4_k8s_volume: "{{ me4_k8s_volume_data[-2] }}"
+    me4_slurm_volume: "{{ me4_slurm_volume_data[-2] }}"
+
+- name: Get all mounted partitions
+  command: df -h
+  changed_when: false
+  register: mounted_partitions
+
+- name: Create partition on ME4 volumes
+  command: "parted -a optimal {{ item }} --script -- mklabel gpt mkpart primary 0% {{ powervault_me4_disk_partition_size }}"
+  changed_when: true
+  with_items:
+    - "{{ hostvars['NFS_NODE_TOKEN_HOLDER']['me4_k8s_volume'] }}"
+    - "{{ hostvars['NFS_NODE_TOKEN_HOLDER']['me4_slurm_volume'] }}"
+  when:
+    - hostvars['NFS_NODE_TOKEN_HOLDER']['me4_k8s_volume'] not in mounted_partitions.stdout
+    - hostvars['NFS_NODE_TOKEN_HOLDER']['me4_slurm_volume'] not in mounted_partitions.stdout
+
+- name: Update kernel with new partition changes
+  command: partprobe
+  changed_when: false
+
+- name: Check ME4 mounted partitions
+  shell: >
+    set -o pipefail && \
+    mount | grep me4
+  failed_when: false
+  changed_when: false
+  args:
+    warn: false
+  register: me4_mounted_partitions
+
+- name: Set file system on partition
+  shell: >
+    set -o pipefail && \
+    echo y | mkfs -t ext4 {{ item }}1
+  with_items:
+    - "{{ hostvars['NFS_NODE_TOKEN_HOLDER']['me4_k8s_volume'] }}"
+    - "{{ hostvars['NFS_NODE_TOKEN_HOLDER']['me4_slurm_volume'] }}"
+  when:
+    - me4_nfs_share_k8s not in me4_mounted_partitions.stdout
+    - me4_nfs_share_slurm not in me4_mounted_partitions.stdout
+
+- name: Creating NFS share directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "{{ nfs_share_dir_mode }}"
+  with_items:
+    - "{{ me4_nfs_share_k8s }}"
+    - "{{ me4_nfs_share_slurm }}"
+
+- name: Mount K8s partition on K8s NFS share
+  command: "mount {{ hostvars['NFS_NODE_TOKEN_HOLDER']['me4_k8s_volume'] }}1 {{ me4_nfs_share_k8s }}"
+  changed_when: true
+  args:
+    warn: false
+  when: me4_nfs_share_k8s not in me4_mounted_partitions.stdout
+
+- name: Mount Slurm partition on Slurm NFS share
+  command: "mount {{ hostvars['NFS_NODE_TOKEN_HOLDER']['me4_slurm_volume'] }}1 {{ me4_nfs_share_slurm }}"
+  changed_when: true
+  args:
+    warn: false
+  when: me4_nfs_share_slurm not in me4_mounted_partitions.stdout
+
+- name: Configuring auto mount K8s partition on reboot
+  lineinfile:
+    path: "{{ fstab_file_path }}"
+    line: "{{ hostvars['NFS_NODE_TOKEN_HOLDER']['me4_k8s_volume'] }}1            {{ me4_nfs_share_k8s }}      ext4            defaults        0     0"
+
+- name: Configuring auto mount Slurm partition on reboot
+  lineinfile:
+    path: "{{ fstab_file_path }}"
+    line: "{{ hostvars['NFS_NODE_TOKEN_HOLDER']['me4_slurm_volume'] }}1            {{ me4_nfs_share_slurm }}      ext4            defaults        0     0"

roles/powervault_me4_nfs/tasks/nfs_node_configure.yml

@@ -0,0 +1,137 @@
+
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Include
+  include_tasks: "{{ playbook_dir }}/control_plane/roles/powervault_me4/tasks/ports.yml"
+
+- name: Refresh ssh keys
+  command: ssh-keygen -R {{ groups['powervault_me4'][0] }}
+  changed_when: false
+  tags: install
+  failed_when: false
+
+- name: Validate authentication of username and password
+  command: ping -c1 {{ groups['powervault_me4'][0] }}
+  register: validate_login
+  changed_when: false
+  failed_when: false
+
+- name: Install packages
+  package:
+    name: iscsi-initiator-utils
+    state: present
+  tags: install
+
+- name: Install packages
+  package:
+    name: sg3_utils
+    state: present
+  tags: install
+
+- name: Set bootproto value
+  lineinfile:
+    path: "{{ nic_path }}"
+    regexp: '^BOOTPROTO='
+    line: 'BOOTPROTO=none'
+  register: result
+
+- name: Set onboot value
+  lineinfile:
+    path: "{{ nic_path }}"
+    regexp: '^ONBOOT='
+    line: 'ONBOOT=yes'
+
+- name: Add ip address
+  lineinfile:
+    path: "{{ nic_path }}"
+    insertafter: '^ONBOOT=yes'
+    line: 'IPADDR={{ pv_nic_ip }}'
+
+- name: Add netmask address
+  lineinfile:
+    path: "{{ nic_path }}"
+    insertafter: '^IPADDR={{ pv_nic_ip }}'
+    line: NETMASK=255.255.255.0
+
+- name: Down the nic
+  command: ifdown {{ pv_nic }}
+  changed_when: true
+  failed_when: false
+  tags: install
+
+- name: Up the nic
+  command: ifup {{ pv_nic }}
+  changed_when: true
+  tags: install
+
+- name: Show ip
+  shell: >
+    set -o pipefail && \
+    ifconfig {{ pv_nic }} | grep 'inet' |cut -d: -f2 |  awk '{ print $2}'
+  changed_when: false
+
+- name: Discover nodes
+  command: iscsiadm -m discovery -t sendtargets -p {{ item }}
+  with_items: "{{ set_port_ip }}"
+  register: ports_available
+  failed_when: false
+  changed_when: false
+  tags: install
+
+- name: Pv port ip
+  add_host:
+    name: pv
+    map_ip: "{{ item.item }}"
+  with_items: "{{ ports_available.results }}"
+  when: item.rc == 0
+
+- name: Pv port ip
+  set_fact:
+    map_ip_output: "{{ item.stdout_lines }}"
+  with_items: "{{ ports_available.results }}"
+  when: item.rc == 0
+
+- name: Find feasible port ip
+  set_fact:
+    discover: "{{ item }}"
+  with_items: "{{ map_ip_output }}"
+  when: hostvars['pv']['map_ip'] in item
+
+- name: Split on comma
+  set_fact:
+    ip_port: "{{ discover.split(',')[0] }}"
+
+- name: Pv name
+  set_fact:
+    pv_name: "{{ discover.split(',')[1].split()[1] }}"
+
+- name: IQDN id
+  shell: >
+    set -o pipefail && \
+    cat /etc/iscsi/initiatorname.iscsi | cut -f2 -d"="
+  register: iqdn_id
+  changed_when: false
+  tags: install
+
+- name: Add ME4 volume data to dummy host
+  add_host:
+    name:   "server_iqdn_id"
+    server_iqdn: "{{ iqdn_id.stdout }}"
+
+- name: Login to the powervault
+  command: iscsiadm -m node --login {{ pv_name }} -p {{ ip_port }}
+  changed_when: true
+  tags: install

roles/powervault_me4_nfs/tasks/nfs_volume.yml

@@ -0,0 +1,40 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Refresh ssh keys
+  command: ssh-keygen -R {{ groups['powervault_me4'][0] }}
+  changed_when: false
+  tags: install
+  failed_when: false
+
+- name: Validate authentication of username and password
+  command: ping -c1 {{ groups['powervault_me4'][0] }}
+  register: validate_login
+  changed_when: false
+  failed_when: false
+
+- name: Scan for getting the volume
+  command: rescan-scsi-bus.sh --forcerescan
+  changed_when: false
+  register: volume_pv
+  tags: install
+
+- name: Assert if volume created or not
+  assert:
+    that:
+      - "' Model: ME4' in volume_pv.stdout"
+    success_msg: "Volume is created"
+    fail_msg: "Volume is not created properly."
+  tags: install

roles/powervault_me4_nfs/vars/main.yml

@@ -0,0 +1,38 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+# vars file for powervault_me4_nfs
+
+# Usage: mount_me4_partitions.yml
+me4_volume_fail_msg: "ME4 volumes not found!"
+me4_nfs_share_k8s: "/me4_k8s_nfs"
+me4_nfs_share_slurm: "/me4_slurm_nfs"
+fstab_file_path: "/etc/fstab"
+
+# Usage: me4_nfs_server_setup.yml
+exports_file_path: /etc/exports
+nfs_services:
+  - mountd
+  - rpc-bind
+  - nfs
+nfs_share_dir_mode: 0777
+
+# Usage: nfs_node_configure.yml
+pv_nic: "{{ powervault_me4_server_nic }}"
+pv_nic_ip: 192.168.25.3
+pv_nic_gateway: 192.168.25.1
+pv_port_ip: 192.168.25.5
+pv_nfs_file: "{{ role_path }}/../../control_plane/input_params/powervault_me4_vars.yml"
+nic_path: "/etc/sysconfig/network-scripts/ifcfg-{{ powervault_me4_server_nic }}"