From 7ffb711f0302d1136652cf710d9cbd1cad989690 Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Wed, 1 Sep 2021 01:11:15 +0700 Subject: [PATCH 1/8] Split metal group into masters and workers --- metal/hosts.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/metal/hosts.yml b/metal/hosts.yml index c4bdec1a..20309674 100644 --- a/metal/hosts.yml +++ b/metal/hosts.yml @@ -1,9 +1,13 @@ metal: - hosts: - # metal0: {ansible_host: 192.168.1.110, mac: '00:23:24:d1:f3:f0'} - metal1: {ansible_host: 192.168.1.111, mac: '00:23:24:d1:f4:d6'} - metal2: {ansible_host: 192.168.1.112, mac: '00:23:24:e7:04:60'} - metal3: {ansible_host: 192.168.1.113, mac: '00:23:24:d1:f5:69'} + children: + masters: + hosts: + # metal0: {ansible_host: 192.168.1.110, mac: '00:23:24:d1:f3:f0'} + metal1: {ansible_host: 192.168.1.111, mac: '00:23:24:d1:f4:d6'} + metal2: {ansible_host: 192.168.1.112, mac: '00:23:24:e7:04:60'} + workers: + hosts: + metal3: {ansible_host: 192.168.1.113, mac: '00:23:24:d1:f5:69'} vars: ansible_user: root ansible_ssh_private_key_file: ~/.ssh/id_ed25519 From 92387435c8b714e835c493d8267802e18a02541c Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Wed, 1 Sep 2021 04:19:05 +0700 Subject: [PATCH 2/8] Add k3s role --- metal/main.yml | 5 ++ metal/roles/k3s/defaults/main.yml | 4 ++ metal/roles/k3s/tasks/main.yml | 68 ++++++++++++++++++++++++ metal/roles/k3s/templates/config.yaml.j2 | 1 + metal/roles/k3s/templates/k3s.service.j2 | 24 +++++++++ 5 files changed, 102 insertions(+) create mode 100644 metal/roles/k3s/defaults/main.yml create mode 100644 metal/roles/k3s/tasks/main.yml create mode 100644 metal/roles/k3s/templates/config.yaml.j2 create mode 100644 metal/roles/k3s/templates/k3s.service.j2 diff --git a/metal/main.yml b/metal/main.yml index 4bc6a240..d0162afb 100644 --- a/metal/main.yml +++ b/metal/main.yml @@ -8,3 +8,8 @@ gather_facts: no roles: - wake + +- name: Create Kubernetes cluster + hosts: metal + roles: + - k3s diff --git a/metal/roles/k3s/defaults/main.yml b/metal/roles/k3s/defaults/main.yml new file mode 100644 index 00000000..1d911c20 --- /dev/null +++ b/metal/roles/k3s/defaults/main.yml @@ -0,0 +1,4 @@ +k3s_version: v1.21.4+k3s1 +k3s_config_file: /etc/rancher/k3s/config.yaml +k3s_token_file: /etc/rancher/node/password +k3s_service_file: /etc/systemd/system/k3s.service diff --git a/metal/roles/k3s/tasks/main.yml b/metal/roles/k3s/tasks/main.yml new file mode 100644 index 00000000..04f9b474 --- /dev/null +++ b/metal/roles/k3s/tasks/main.yml @@ -0,0 +1,68 @@ +- name: Download k3s binary + get_url: + url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s + checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt + dest: /usr/local/bin/k3s + owner: root + group: root + mode: 0755 + +- name: Ensure config directories exist + file: + path: "{{ item }}" + state: directory + loop: + - /etc/rancher/k3s + - /etc/rancher/node + +- name: Check if k3s token file exists on the first node + run_once: yes + stat: + path: "{{ k3s_token_file }}" + register: k3s_token_file_stat + +- name: Generate k3s token file on the first node if not exist yet + run_once: yes + when: not k3s_token_file_stat.stat.exists + copy: + content: lookup('community.general.random_string', length=32) + dest: "{{ k3s_token_file }}" + +- name: Get k3s token from the first node + run_once: yes + slurp: + src: "{{ k3s_token_file }}" + register: k3s_token_base64 + +- name: Ensure all nodes has the same token + copy: + content: "{{ k3s_token_base64.content | b64decode }}" + dest: "{{ k3s_token_file }}" + +- name: Copy k3s config files to master nodes + when: "'masters' in group_names" + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + vars: + node_role: server + loop: + - src: config.yaml.j2 + dest: "{{ k3s_config_file }}" + - src: k3s.service.j2 + dest: "{{ k3s_service_file }}" + +- name: Copy k3s config files to worker nodes + when: "'workers' in group_names" + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + vars: + node_role: agent + loop: + - src: config.yaml.j2 + dest: "{{ k3s_config_file }}" + - src: k3s.service.j2 + dest: "{{ k3s_service_file }}" + +# Get kubeconfig /etc/rancher/k3s/k3s.yaml diff --git a/metal/roles/k3s/templates/config.yaml.j2 b/metal/roles/k3s/templates/config.yaml.j2 new file mode 100644 index 00000000..eda6f511 --- /dev/null +++ b/metal/roles/k3s/templates/config.yaml.j2 @@ -0,0 +1 @@ +{{ node_role }} diff --git a/metal/roles/k3s/templates/k3s.service.j2 b/metal/roles/k3s/templates/k3s.service.j2 new file mode 100644 index 00000000..9fb43616 --- /dev/null +++ b/metal/roles/k3s/templates/k3s.service.j2 @@ -0,0 +1,24 @@ +[Unit] +Description=Lightweight Kubernetes +Documentation=https://k3s.io +After=network-online.target + +[Service] +Type=notify +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/k3s {{ node_role }} +KillMode=process +Delegate=yes +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s + +[Install] +WantedBy=multi-user.target From 713631587c01f1d4f0c6c44c6cdbb01e21f8cc03 Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Wed, 1 Sep 2021 04:58:47 +0700 Subject: [PATCH 3/8] Improve k3s config template rendering --- metal/roles/k3s/tasks/main.yml | 17 +---------------- metal/roles/k3s/templates/config.yaml.j2 | 2 +- metal/roles/k3s/templates/k3s.service.j2 | 2 +- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/metal/roles/k3s/tasks/main.yml b/metal/roles/k3s/tasks/main.yml index 04f9b474..ec46c4eb 100644 --- a/metal/roles/k3s/tasks/main.yml +++ b/metal/roles/k3s/tasks/main.yml @@ -39,26 +39,11 @@ content: "{{ k3s_token_base64.content | b64decode }}" dest: "{{ k3s_token_file }}" -- name: Copy k3s config files to master nodes +- name: Copy k3s config files when: "'masters' in group_names" template: src: "{{ item.src }}" dest: "{{ item.dest }}" - vars: - node_role: server - loop: - - src: config.yaml.j2 - dest: "{{ k3s_config_file }}" - - src: k3s.service.j2 - dest: "{{ k3s_service_file }}" - -- name: Copy k3s config files to worker nodes - when: "'workers' in group_names" - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - vars: - node_role: agent loop: - src: config.yaml.j2 dest: "{{ k3s_config_file }}" diff --git a/metal/roles/k3s/templates/config.yaml.j2 b/metal/roles/k3s/templates/config.yaml.j2 index eda6f511..adbf009c 100644 --- a/metal/roles/k3s/templates/config.yaml.j2 +++ b/metal/roles/k3s/templates/config.yaml.j2 @@ -1 +1 @@ -{{ node_role }} +{{ 'server' if 'masters' in group_names else 'agent' }} diff --git a/metal/roles/k3s/templates/k3s.service.j2 b/metal/roles/k3s/templates/k3s.service.j2 index 9fb43616..ee1c15ef 100644 --- a/metal/roles/k3s/templates/k3s.service.j2 +++ b/metal/roles/k3s/templates/k3s.service.j2 @@ -7,7 +7,7 @@ After=network-online.target Type=notify ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s {{ node_role }} +ExecStart=/usr/local/bin/k3s {{ 'server' if 'masters' in group_names else 'agent' }} KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead From 8a8d4d7ea2c0be6d1ab451f3b891bce4daf7cb69 Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Wed, 1 Sep 2021 05:35:35 +0700 Subject: [PATCH 4/8] Working k3s service and config --- metal/roles/k3s/handlers/main.yml | 6 ++++++ metal/roles/k3s/tasks/main.yml | 2 +- metal/roles/k3s/templates/config.yaml.j2 | 7 ++++++- 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 metal/roles/k3s/handlers/main.yml diff --git a/metal/roles/k3s/handlers/main.yml b/metal/roles/k3s/handlers/main.yml new file mode 100644 index 00000000..8491971c --- /dev/null +++ b/metal/roles/k3s/handlers/main.yml @@ -0,0 +1,6 @@ +- name: Restart k3s service + systemd: + name: k3s + daemon_reload: yes + enabled: yes + state: restarted diff --git a/metal/roles/k3s/tasks/main.yml b/metal/roles/k3s/tasks/main.yml index ec46c4eb..2b439ed2 100644 --- a/metal/roles/k3s/tasks/main.yml +++ b/metal/roles/k3s/tasks/main.yml @@ -40,7 +40,6 @@ dest: "{{ k3s_token_file }}" - name: Copy k3s config files - when: "'masters' in group_names" template: src: "{{ item.src }}" dest: "{{ item.dest }}" @@ -49,5 +48,6 @@ dest: "{{ k3s_config_file }}" - src: k3s.service.j2 dest: "{{ k3s_service_file }}" + notify: Restart k3s service # Get kubeconfig /etc/rancher/k3s/k3s.yaml diff --git a/metal/roles/k3s/templates/config.yaml.j2 b/metal/roles/k3s/templates/config.yaml.j2 index adbf009c..06169e9b 100644 --- a/metal/roles/k3s/templates/config.yaml.j2 +++ b/metal/roles/k3s/templates/config.yaml.j2 @@ -1 +1,6 @@ -{{ 'server' if 'masters' in group_names else 'agent' }} +{% if inventory_hostname == groups['masters'][0] %} +cluster-init: true +{% else %} +server: https://{{ hostvars[groups['masters'][0]].ansible_host }}:6443 +{% endif %} +token-file: {{ k3s_token_file }} From 84b8404d17b9383dbe11f44ccb7b9a82145e5ecc Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Wed, 1 Sep 2021 05:39:04 +0700 Subject: [PATCH 5/8] Add k3s systemd service file credit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4b10192c..78768a25 100644 --- a/README.md +++ b/README.md @@ -125,3 +125,4 @@ Distributed under the GPLv3 License. See `LICENSE` for more information. - [README template](https://github.com/othneildrew/Best-README-Template) - [Run the same Cloudflare Tunnel across many `cloudflared` processes](https://developers.cloudflare.com/cloudflare-one/tutorials/many-cfd-one-tunnel) - [MAC address environment variable in GRUB config](https://askubuntu.com/questions/1272400/how-do-i-automate-network-installation-of-many-ubuntu-18-04-systems-with-efi-and) +- [Official k3s systemd service file](https://github.com/k3s-io/k3s/blob/master/k3s.service) From 6f5ef180d3299251da2ffcba06697735b6ae63ae Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Wed, 1 Sep 2021 05:43:35 +0700 Subject: [PATCH 6/8] Fetch kubeconfig to local --- metal/roles/k3s/tasks/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/metal/roles/k3s/tasks/main.yml b/metal/roles/k3s/tasks/main.yml index 2b439ed2..b5c1fd6e 100644 --- a/metal/roles/k3s/tasks/main.yml +++ b/metal/roles/k3s/tasks/main.yml @@ -50,4 +50,8 @@ dest: "{{ k3s_service_file }}" notify: Restart k3s service -# Get kubeconfig /etc/rancher/k3s/k3s.yaml +- name: Get Kubernetes config file + run_once: yes + fetch: + src: /etc/rancher/k3s/k3s.yaml + dest: "{{ playbook_dir }}/kubeconfig.yaml" From dc80ff21f9d5b410cd0e945dd371b8cca0609c37 Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Wed, 1 Sep 2021 05:46:12 +0700 Subject: [PATCH 7/8] Fix incorrect kubeconfig path --- metal/roles/k3s/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/metal/roles/k3s/tasks/main.yml b/metal/roles/k3s/tasks/main.yml index b5c1fd6e..48e5fd79 100644 --- a/metal/roles/k3s/tasks/main.yml +++ b/metal/roles/k3s/tasks/main.yml @@ -55,3 +55,4 @@ fetch: src: /etc/rancher/k3s/k3s.yaml dest: "{{ playbook_dir }}/kubeconfig.yaml" + flat: yes From e028ca7bc2f9148724989462a7890e0351f874b4 Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Wed, 1 Sep 2021 16:26:10 +0700 Subject: [PATCH 8/8] Move k3s systemd service from handler to task --- metal/roles/k3s/handlers/main.yml | 6 ------ metal/roles/k3s/tasks/main.yml | 10 +++++++++- 2 files changed, 9 insertions(+), 7 deletions(-) delete mode 100644 metal/roles/k3s/handlers/main.yml diff --git a/metal/roles/k3s/handlers/main.yml b/metal/roles/k3s/handlers/main.yml deleted file mode 100644 index 8491971c..00000000 --- a/metal/roles/k3s/handlers/main.yml +++ /dev/null @@ -1,6 +0,0 @@ -- name: Restart k3s service - systemd: - name: k3s - daemon_reload: yes - enabled: yes - state: restarted diff --git a/metal/roles/k3s/tasks/main.yml b/metal/roles/k3s/tasks/main.yml index 48e5fd79..7fca1a49 100644 --- a/metal/roles/k3s/tasks/main.yml +++ b/metal/roles/k3s/tasks/main.yml @@ -48,7 +48,15 @@ dest: "{{ k3s_config_file }}" - src: k3s.service.j2 dest: "{{ k3s_service_file }}" - notify: Restart k3s service + +- name: Enable k3s service + systemd: + name: k3s + enabled: yes + state: started + register: k3s_service + until: k3s_service is succeeded + retries: 5 - name: Get Kubernetes config file run_once: yes