Merge pull request #642 from blesson-james/kubeflow

Issue #502: Updated Kubeflow firewalld ports config

control_plane/roles/control_plane_k8s/tasks/k8s_firewalld.yml

@@ -26,7 +26,7 @@
 
 - name: Configure firewalld on master nodes
   firewalld:
-    port: "{{ item }}/tcp"
+    port: "{{ item }}"
     permanent: yes
     state: enabled
   with_items: '{{ k8s_master_ports }}'
@@ -45,6 +45,11 @@
     state: enabled
   with_items: "{{ calico_tcp_ports }}"
 
+- name: Masquerade the firewall
+  command: firewall-cmd --add-masquerade --permanent
+  changed_when: true
+  tags: firewalld
+
 - name: Reload firewalld
   command: firewall-cmd --reload
   changed_when: true
@@ -53,4 +58,4 @@
   service:
     name: firewalld
     state: stopped
-    enabled: no
+    enabled: no

control_plane/roles/control_plane_k8s/vars/main.yml

@@ -38,11 +38,14 @@ docker_repo_dest: /etc/yum.repos.d/docker-ce.repo
 
 # Usage: k8s_firewalld.yml
 k8s_master_ports:
-  - 6443
-  - 2379-2380
-  - 10250
-  - 10251
-  - 10252
+  - 6443/tcp
+  - 2379-2380/tcp
+  - 10250/tcp
+  - 10251/tcp
+  - 10252/tcp
+  - 10255/tcp
+  - 8472/udp
+  - 30000-32767/tcp
 calico_udp_ports:
   - 4789
 calico_tcp_ports:
@@ -95,4 +98,4 @@ metallb_run_as_user_port: "65534"
 k8s_dashboard_yaml_url: https://raw.githubusercontent.com/kubernetes/dashboard/v2.2.0/aio/deploy/recommended.yaml
 k8s_dashboard_admin_file_dest: /root/k8s/k8s_dashboard_admin.yaml
 k8s_dashboard_admin_file_mode: 0655
-nfs_path: /var/nfs_awx
+nfs_path: /var/nfs_awx

platforms/roles/kubeflow/tasks/deploy_kubeflow.yml

@@ -0,0 +1,141 @@
+#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Download kfctl release from the Kubeflow releases page
+  unarchive:
+    src: "{{ kfctl_download_url }}"
+    dest: "{{ kfctl_download_dest_path }}"
+    mode: "{{ kfctl_download_file_mode }}"
+    remote_src: yes
+
+- name: Delete omnia kubeflow directory if exists
+  file:
+    path: "{{ omnia_kubeflow_dir_path }}"
+    state: absent
+
+- name: Create omnia kubeflow directory
+  file:
+    path: "{{ omnia_kubeflow_dir_path }}"
+    state: directory
+    mode: "{{ omnia_kubeflow_dir_mode }}"
+    recurse: yes
+
+- name: Build kubeflow configuration
+  command:
+    cmd: /usr/bin/kfctl build -V -f "{{ kubeflow_config_yaml_url }}"
+    chdir: "{{ omnia_kubeflow_dir_path }}"
+  changed_when: true
+
+- name: Modify CPU limit for istio-ingressgateway-service-account
+  replace:
+    path: "{{ istio_noauth_yaml_file_path }}"
+    after: 'serviceAccountName: istio-ingressgateway-service-account'
+    before: '---'
+    regexp: 'cpu: 100m'
+    replace: 'cpu: 2'
+
+- name: Modify memory limit for istio-ingressgateway-service-account
+  replace:
+    path: "{{ istio_noauth_yaml_file_path }}"
+    after: 'serviceAccountName: istio-ingressgateway-service-account'
+    before: '---'
+    regexp: 'memory: 128Mi'
+    replace: 'memory: 512Mi'
+
+- name: Modify CPU request for istio-ingressgateway-service-account
+  replace:
+    path: "{{ istio_noauth_yaml_file_path }}"
+    after: 'serviceAccountName: istio-ingressgateway-service-account'
+    before: '---'
+    regexp: 'cpu: 10m'
+    replace: 'cpu: 1'
+
+- name: Modify memory request for istio-ingressgateway-service-account
+  replace:
+    path: "{{ istio_noauth_yaml_file_path }}"
+    after: 'serviceAccountName: istio-ingressgateway-service-account'
+    before: '---'
+    regexp: 'memory: 40Mi'
+    replace: 'memory: 256Mi'
+
+- name: Modify memory request for istio-engressgateway-service-account
+  replace:
+    path: "{{ istio_noauth_yaml_file_path }}"
+    after: 'serviceAccountName: istio-egressgateway-service-account'
+    before: '---'
+    regexp: 'memory: 128Mi'
+    replace: 'memory: 256Mi'
+
+- name: Modify memory request for istio-engressgateway-service-account
+  replace:
+    path: "{{ istio_noauth_yaml_file_path }}"
+    after: 'serviceAccountName: istio-egressgateway-service-account'
+    before: '---'
+    regexp: 'memory: 40Mi'
+    replace: 'memory: 128Mi'
+
+- name: Modify CPU limit for kfserving-gateway
+  replace:
+    path: "{{ kfserving_gateway_yaml_file_path }}"
+    after: 'serviceAccountName: istio-ingressgateway-service-account'
+    before: 'env:'
+    regexp: 'cpu: 100m'
+    replace: 'cpu: 2'
+
+- name: Modify memory limit for kfserving-gateway
+  replace:
+    path: "{{ kfserving_gateway_yaml_file_path }}"
+    after: 'serviceAccountName: istio-ingressgateway-service-account'
+    before: 'env:'
+    regexp: 'memory: 128Mi'
+    replace: 'memory: 512Mi'
+
+- name: Modify CPU request for kfserving-gateway
+  replace:
+    path: "{{ kfserving_gateway_yaml_file_path }}"
+    after: 'serviceAccountName: istio-ingressgateway-service-account'
+    before: 'env:'
+    regexp: 'cpu: 10m'
+    replace: 'cpu: 1'
+
+- name: Modify memory request for kfserving-gateway
+  replace:
+    path: "{{ kfserving_gateway_yaml_file_path }}"
+    after: 'serviceAccountName: istio-ingressgateway-service-account'
+    before: 'env:'
+    regexp: 'memory: 40Mi'
+    replace: 'memory: 256Mi'
+
+- name: Change argo base service from NodePort to LoadBalancer
+  replace:
+    path: "{{ argo_yaml_file_path }}"
+    regexp: 'NodePort'
+    replace: 'LoadBalancer'
+
+- name: Change istio-install base istio-noauth service from NodePort to LoadBalancer
+  replace:
+    path: "{{ istio_noauth_yaml_file_path }}"
+    regexp: 'NodePort'
+    replace: 'LoadBalancer'
+
+- name: Apply kubeflow configuration
+  command:
+    cmd: "/usr/bin/kfctl apply -V -f '{{ kubeflow_config_file }}'"
+    chdir: "{{ omnia_kubeflow_dir_path }}"
+  changed_when: true
+  register: apply_kubeflow_config
+  until: apply_kubeflow_config is not failed
+  retries: 20
+  delay: 10

platforms/roles/kubeflow/tasks/firewalld_config.yml

@@ -0,0 +1,45 @@
+#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+
+- name: Install firewalld
+  package:
+    name: firewalld
+    state: present
+  tags: firewalld
+
+- name: Start and enable firewalld
+  service:
+    name: firewalld
+    state: started
+    enabled: yes
+  tags: firewalld
+
+- name: Configure firewalld on master nodes
+  firewalld:
+    port: "{{ item }}/tcp"
+    permanent: yes
+    state: enabled
+  with_items: '{{ kubeflow_firewalld_ports }}'
+  tags: firewalld
+
+- name: Masquerade the firewall
+  command: firewall-cmd --add-masquerade --permanent
+  changed_when: true
+  tags: firewalld
+
+- name: Reload firewalld
+  command: firewall-cmd --reload
+  changed_when: true
+  tags: firewalld

platforms/roles/kubeflow/tasks/main.yml

@@ -13,129 +13,8 @@
 #  limitations under the License.
 ---
 
-- name: Download kfctl release from the Kubeflow releases page
-  unarchive:
-    src: "{{ kfctl_download_url }}"
-    dest: "{{ kfctl_download_dest_path }}"
-    mode: "{{ kfctl_download_file_mode }}"
-    remote_src: yes
+- name: Configure firewalld ports
+  import_tasks: firewalld_config.yml
 
-- name: Delete omnia kubeflow directory if exists
-  file:
-    path: "{{ omnia_kubeflow_dir_path }}"
-    state: absent
-
-- name: Create omnia kubeflow directory
-  file:
-    path: "{{ omnia_kubeflow_dir_path }}"
-    state: directory
-    mode: "{{ omnia_kubeflow_dir_mode }}"
-    recurse: yes
-
-- name: Build kubeflow configuration
-  command:
-    cmd: /usr/bin/kfctl build -V -f "{{ kubeflow_config_yaml_url }}"
-    chdir: "{{ omnia_kubeflow_dir_path }}"
-  changed_when: true
-
-- name: Modify CPU limit for istio-ingressgateway-service-account
-  replace:
-    path: "{{ istio_noauth_yaml_file_path }}"
-    after: 'serviceAccountName: istio-ingressgateway-service-account'
-    before: '---'
-    regexp: 'cpu: 100m'
-    replace: 'cpu: 2'
-
-- name: Modify memory limit for istio-ingressgateway-service-account
-  replace:
-    path: "{{ istio_noauth_yaml_file_path }}"
-    after: 'serviceAccountName: istio-ingressgateway-service-account'
-    before: '---'
-    regexp: 'memory: 128Mi'
-    replace: 'memory: 512Mi'
-
-- name: Modify CPU request for istio-ingressgateway-service-account
-  replace:
-    path: "{{ istio_noauth_yaml_file_path }}"
-    after: 'serviceAccountName: istio-ingressgateway-service-account'
-    before: '---'
-    regexp: 'cpu: 10m'
-    replace: 'cpu: 1'
-
-- name: Modify memory request for istio-ingressgateway-service-account
-  replace:
-    path: "{{ istio_noauth_yaml_file_path }}"
-    after: 'serviceAccountName: istio-ingressgateway-service-account'
-    before: '---'
-    regexp: 'memory: 40Mi'
-    replace: 'memory: 256Mi'
-
-- name: Modify memory request for istio-engressgateway-service-account
-  replace:
-    path: "{{ istio_noauth_yaml_file_path }}"
-    after: 'serviceAccountName: istio-egressgateway-service-account'
-    before: '---'
-    regexp: 'memory: 128Mi'
-    replace: 'memory: 256Mi'
-
-- name: Modify memory request for istio-engressgateway-service-account
-  replace:
-    path: "{{ istio_noauth_yaml_file_path }}"
-    after: 'serviceAccountName: istio-egressgateway-service-account'
-    before: '---'
-    regexp: 'memory: 40Mi'
-    replace: 'memory: 128Mi'
-
-- name: Modify CPU limit for kfserving-gateway
-  replace:
-    path: "{{ kfserving_gateway_yaml_file_path }}"
-    after: 'serviceAccountName: istio-ingressgateway-service-account'
-    before: 'env:'
-    regexp: 'cpu: 100m'
-    replace: 'cpu: 2'
-
-- name: Modify memory limit for kfserving-gateway
-  replace:
-    path: "{{ kfserving_gateway_yaml_file_path }}"
-    after: 'serviceAccountName: istio-ingressgateway-service-account'
-    before: 'env:'
-    regexp: 'memory: 128Mi'
-    replace: 'memory: 512Mi'
-
-- name: Modify CPU request for kfserving-gateway
-  replace:
-    path: "{{ kfserving_gateway_yaml_file_path }}"
-    after: 'serviceAccountName: istio-ingressgateway-service-account'
-    before: 'env:'
-    regexp: 'cpu: 10m'
-    replace: 'cpu: 1'
-
-- name: Modify memory request for kfserving-gateway
-  replace:
-    path: "{{ kfserving_gateway_yaml_file_path }}"
-    after: 'serviceAccountName: istio-ingressgateway-service-account'
-    before: 'env:'
-    regexp: 'memory: 40Mi'
-    replace: 'memory: 256Mi'
-
-- name: Change argo base service from NodePort to LoadBalancer
-  replace:
-    path: "{{ argo_yaml_file_path }}"
-    regexp: 'NodePort'
-    replace: 'LoadBalancer'
-
-- name: Change istio-install base istio-noauth service from NodePort to LoadBalancer
-  replace:
-    path: "{{ istio_noauth_yaml_file_path }}"
-    regexp: 'NodePort'
-    replace: 'LoadBalancer'
-
-- name: Apply kubeflow configuration
-  command:
-    cmd: "/usr/bin/kfctl apply -V -f '{{ kubeflow_config_file }}'"
-    chdir: "{{ omnia_kubeflow_dir_path }}"
-  changed_when: true
-  register: apply_kubeflow_config
-  until: apply_kubeflow_config is not failed
-  retries: 20
-  delay: 10
+- name: Deploy kubeflow
+  import_tasks: deploy_kubeflow.yml

platforms/roles/kubeflow/vars/main.yml

@@ -13,6 +13,20 @@
 #  limitations under the License.
 ---
 
+kubeflow_firewalld_ports:
+  - 15020
+  - 80
+  - 443
+  - 31400
+  - 15011
+  - 8060
+  - 853
+  - 15029
+  - 15030
+  - 15031
+  - 15032
+  - 15443
+
 kfctl_download_url: https://github.com/kubeflow/kfctl/releases/download/v1.0.2/kfctl_v1.0.2-0-ga476281_linux.tar.gz
 
 kfctl_download_dest_path: /usr/bin/

roles/k8s_firewalld/tasks/main.yml

@@ -28,7 +28,7 @@
 
 - name: Configure firewalld on master nodes
   firewalld:
-    port: "{{ item }}/tcp"
+    port: "{{ item }}"
     permanent: yes
     state: enabled
   with_items: '{{ k8s_master_ports }}'
@@ -71,6 +71,11 @@
   when: hostvars['127.0.0.1']['k8s_cni'] == "calico"
   tags: firewalld
 
+- name: Masquerade the firewall
+  command: firewall-cmd --add-masquerade --permanent
+  changed_when: true
+  tags: firewalld
+
 - name: Reload firewalld
   command: firewall-cmd --reload
   changed_when: true
@@ -81,4 +86,4 @@
     name: firewalld
     state: stopped
     enabled: no
-  tags: firewalld
+  tags: firewalld

roles/k8s_firewalld/vars/main.yml

@@ -15,11 +15,14 @@
 
 # Master nodes firewall ports
 k8s_master_ports:
-  - 6443
-  - 2379-2380
-  - 10250
-  - 10251
-  - 10252
+  - 6443/tcp
+  - 2379-2380/tcp
+  - 10250/tcp
+  - 10251/tcp
+  - 10252/tcp
+  - 10255/tcp
+  - 30000-32767/tcp
+  - 8472/udp
 
 # Worker nodes firewall ports
 k8s_compute_ports: