Dell-Omnia-Clusters-HPC-AI/roles/k8s_start_manager/tasks/main.yml

#  Copyright 2022 Dell Inc. or its subsidiaries. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
---

- name: Disable SWAP (1/2)
  command: /usr/sbin/swapoff -a
  changed_when: true
  tags: init

- name: Disable SWAP in fstab (2/2)
  replace:
    path: /etc/fstab
    regexp: '^([^#].*?\sswap\s+.*)$'
    replace: '# \1'

- name: Get netaddr
  setup:
    filter: ansible_default_ipv4.address

- name: Get K8s nodes status
  command: kubectl get nodes
  changed_when: false
  failed_when: false
  register: k8s_nodes
  tags: init

- name: Get K8s ready compute nodes
  shell: >
    set -o pipefail && \
    kubectl get node --selector='!node-role.kubernetes.io/master' | grep -v 'NotReady'
  changed_when: false
  failed_when: false
  register: k8s_nodes_ready
  tags: init

- name: Get K8s not ready compute nodes
  shell: >
    set -o pipefail && \
    kubectl get node --selector='!node-role.kubernetes.io/master' | grep 'NotReady'
  changed_when: false
  failed_when: false
  register: k8s_nodes_not_ready
  tags: init

- name: Docker login
  command: docker login -u {{ hostvars['127.0.0.1']['docker_username'] }} -p {{ hostvars['127.0.0.1']['docker_password'] }}
  changed_when: true
  register: docker_login_output
  failed_when: false
  when: hostvars['127.0.0.1']['docker_username'] or hostvars['127.0.0.1']['docker_password']
  no_log: true

- name: Docker login check
  fail:
    msg: "{{ docker_login_fail_msg }}"
  when: docker_login_output is failed

- name: Initialize kubeadm (This process may take 5-10min)
  block:
    - name: Initialize kubeadm (This process may take 5-10min)
      command: "kubeadm init --pod-network-cidr='{{ hostvars['127.0.0.1']['k8s_pod_network_cidr'] }}' \
        --apiserver-advertise-address='{{ ansible_default_ipv4.address }}'"
      changed_when: true
      register: init_output
      tags: init
  rescue:
    - name: Reset kubeadm
      command: "kubeadm reset -f"
      changed_when: true

    - name: Initialize kubeadm (This process may take 5-10min)
      command: "kubeadm init --pod-network-cidr='{{ hostvars['127.0.0.1']['k8s_pod_network_cidr'] }}' \
        --apiserver-advertise-address='{{ ansible_default_ipv4.address }}'"
      changed_when: true
      register: init_output
      tags: init
  when: "'master' not in k8s_nodes.stdout"

- name: Setup directory for Kubernetes environment for root
  file:
    path: "{{ k8s_root_directory }}"
    state: directory
    mode: "{{ k8s_root_directory_mode }}"
  tags: init

- name: Copy Kubernetes config for root
  copy:
    src: "{{ k8s_config_src }}"
    dest: "{{ k8s_config_dest }}"
    owner: root
    group: root
    mode: "{{ k8s_config_file_mode }}"
    remote_src: yes
  tags: init

- name: Update the kubernetes config file permissions
  shell: "chown $(id -u):$(id -g) '{{ k8s_config_dest }}'"
  args:
    warn: false
  changed_when: true
  tags: init

- name: Cluster token
  shell: >
    set -o pipefail && \
      kubeadm token list | cut -d ' ' -f1 | sed -n '2p'
  changed_when: false
  register: k8s_token
  tags: init

- name: CA Hash
  shell: >
    set -o pipefail && \
      openssl x509 -pubkey -in {{ k8s_cert_path }} | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
  changed_when: false
  register: k8s_manager_ca_hash
  tags: init

- name: Add K8S Manager IP, Token, and Hash to dummy host
  add_host:
    name:   "K8S_TOKEN_HOLDER"
    token:  "{{ k8s_token.stdout }}"
    hash:   "{{ k8s_manager_ca_hash.stdout }}"
    ip:     "{{ ansible_default_ipv4.address }}"
    k8s_nodes:  "{{ k8s_nodes.stdout }}"
    k8s_nodes_ready:  "{{ k8s_nodes_ready.stdout }}"
    k8s_nodes_not_ready: "{{ k8s_nodes_not_ready.stdout }}"
  tags: init

- name: Print k8s token
  debug:
    msg: "[Manager] K8S_TOKEN_HOLDER K8S token is {{ hostvars['K8S_TOKEN_HOLDER']['token'] }}"
    verbosity: 2
  tags: init

- name: Print k8s hash
  debug:
    msg: "[Manager] K8S_TOKEN_HOLDER K8S Hash is  {{ hostvars['K8S_TOKEN_HOLDER']['hash'] }}"
    verbosity: 2
  tags: init

- name: Print k8s ansible_default_ipv4.address
  debug:
    msg: "[Manager] K8S_MANAGER_IP is  {{ ansible_default_ipv4.address }}"
    verbosity: 2
  tags: init

- name: Get K8s secrets
  command: kubectl get secrets
  changed_when: false
  register: k8s_secret
  when: hostvars['127.0.0.1']['docker_username'] and hostvars['127.0.0.1']['docker_password']

- name: Create docker secret
  command: kubectl create secret docker-registry dockerregcred --docker-username={{ hostvars['127.0.0.1']['docker_username'] }} \
    --docker-password={{ hostvars['127.0.0.1']['docker_password'] }}
  when:
    - hostvars['127.0.0.1']['docker_username'] and hostvars['127.0.0.1']['docker_password']
    - "'dockerregcred' not in k8s_secret.stdout"

- name: Add docker secret to default service account
  shell: >
    kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "dockerregcred"}]}'
  register: patch_service_account
  when: hostvars['127.0.0.1']['docker_username'] and hostvars['127.0.0.1']['docker_password']
  until: patch_service_account is not failed
  retries: 10
  tags: install

- name: Create yaml repo for setup
  file:
    path: "{{ yaml_repo_dir_path }}"
    state: directory
    mode: "{{ yaml_repo_dir_mode }}"
  tags: init

- name: Delete Calico yaml file if exists
  file:
    path: "{{ calico_yml_file_path }}"
    state: absent
  when: hostvars['127.0.0.1']['k8s_cni'] == "calico"
  tags: init

- name: Copy Calico yaml file
  copy:
    src: kube-calico.yaml
    dest: "{{ calico_yml_file_path }}"
    owner: root
    group: root
    mode: "{{ calico_yml_file_mode }}"
  when: hostvars['127.0.0.1']['k8s_cni'] == "calico"
  tags: init

- name: Setup Calico SDN network
  command: "kubectl apply -f '{{ calico_yml_file_path }}'"
  when: hostvars['127.0.0.1']['k8s_cni'] == "calico"
  tags: init

- name: Delete Flannel yaml file if exists
  file:
    path: "{{ flannel_yml_file_path }}"
    state: absent
  when: hostvars['127.0.0.1']['k8s_cni'] == "flannel"
  tags: init

- name: Copy Flannel yaml file
  copy:
    src: kube-flannel.yaml
    dest: "{{ flannel_yml_file_path }}"
    owner: root
    group: root
    mode: "{{ flannel_yml_file_mode }}"
  when: hostvars['127.0.0.1']['k8s_cni'] == "flannel"
  tags: init

- name: Replace flannel network cidr
  replace:
    path: "{{ flannel_yml_file_path }}"
    regexp: "10.244.0.0/16"
    replace: "{{ hostvars['127.0.0.1']['k8s_pod_network_cidr'] }}"
  when: hostvars['127.0.0.1']['k8s_cni'] == "flannel"
  tags: init

- name: Setup Flannel SDN network
  command: "kubectl apply -f '{{ flannel_yml_file_path }}'"
  when: hostvars['127.0.0.1']['k8s_cni'] == "flannel"
  tags: init

- name: Create service account (K8s dashboard) files
  copy:
    src: create_admin_user.yaml
    dest: "{{ k8s_service_account_file_dest }}"
    owner: root
    group: root
    mode: "{{ k8s_service_account_file_mode }}"
  tags: init

- name: Check K8s service accounts status
  command: "kubectl get serviceaccounts"
  changed_when: false
  register: k8s_service_accounts
  tags: init

- name: Create service account (K8s dashboard)
  command: "kubectl create -f '{{ k8s_service_account_file_dest }}'"
  changed_when: true
  when: "'default' not in k8s_service_accounts.stdout"
  tags: init

- name: Create clusterRoleBinding (K8s dashboard) files
  copy:
    src: create_clusterRoleBinding.yaml
    dest: "{{ cluster_role_binding_file_dest }}"
    owner: root
    group: root
    mode: "{{ cluster_role_binding_file_mode }}"
  tags: init

- name: Create clusterRoleBinding (K8s dashboard)
  command: "kubectl create -f '{{ cluster_role_binding_file_dest }}'"
  changed_when: true
  failed_when: false
  tags: init

- name: Dump bearer token for K8s dashboard login
  shell: >
    set -o pipefail && \
      kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') > /root/k8s/token
  changed_when: true
  tags: init

- name: Edge / Workstation Install allows pods to scheudle on manager
  command: kubectl taint nodes --all node-role.kubernetes.io/master-
  when: groups['manager'][0] == groups['compute'][0] and groups['compute']|length == 1
  failed_when: false
  tags: init