From 23f695b8fbe355c6461326f3649dab6591ca0e27 Mon Sep 17 00:00:00 2001
From: Khue Doan <mail@khuedoan.com>
Date: Tue, 16 Jan 2024 12:34:22 +0700
Subject: [PATCH] feat(k3s): install kube-vip for HA control plane

---
 metal/group_vars/all.yml                   |  2 --
 metal/inventories/prod.yml                 |  5 +++
 metal/roles/k3s/defaults/main.yml          |  4 +++
 metal/roles/k3s/tasks/main.yml             | 15 +++++++--
 metal/roles/k3s/templates/config.yaml.j2   |  2 +-
 metal/roles/k3s/templates/kube-vip.yaml.j2 | 39 ++++++++++++++++++++++
 6 files changed, 62 insertions(+), 5 deletions(-)
 create mode 100644 metal/roles/k3s/templates/kube-vip.yaml.j2

diff --git a/metal/group_vars/all.yml b/metal/group_vars/all.yml
index 0f702090..39fb7737 100644
--- a/metal/group_vars/all.yml
+++ b/metal/group_vars/all.yml
@@ -2,5 +2,3 @@ ansible_user: root
 ansible_ssh_private_key_file: ~/.ssh/id_ed25519
 ssh_public_key: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
 dns_server: "8.8.8.8"
-load_balancer_ip_pool:
-  - 192.168.1.224/27
diff --git a/metal/inventories/prod.yml b/metal/inventories/prod.yml
index 224941df..775c86a9 100644
--- a/metal/inventories/prod.yml
+++ b/metal/inventories/prod.yml
@@ -1,3 +1,8 @@
+all:
+  vars:
+    control_plane_endpoint: 192.168.1.100
+    load_balancer_ip_pool:
+      - 192.168.1.224/27
 metal:
   children:
     masters:
diff --git a/metal/roles/k3s/defaults/main.yml b/metal/roles/k3s/defaults/main.yml
index fa6abb43..d7acdec2 100644
--- a/metal/roles/k3s/defaults/main.yml
+++ b/metal/roles/k3s/defaults/main.yml
@@ -2,7 +2,11 @@ k3s_version: v1.28.3+k3s2
 k3s_config_file: /etc/rancher/k3s/config.yaml
 k3s_token_file: /etc/rancher/node/password
 k3s_service_file: /etc/systemd/system/k3s.service
+k3s_data_dir: /var/lib/rancher/k3s
+k3s_kubeconfig_file: /etc/rancher/k3s/k3s.yaml
 k3s_server_config:
+  tls-san:
+    - "{{ control_plane_endpoint }}"
   disable:
     - local-storage
     - servicelb
diff --git a/metal/roles/k3s/tasks/main.yml b/metal/roles/k3s/tasks/main.yml
index 5e53b708..21cd85c0 100644
--- a/metal/roles/k3s/tasks/main.yml
+++ b/metal/roles/k3s/tasks/main.yml
@@ -24,6 +24,7 @@
   loop:
     - /etc/rancher/k3s
     - /etc/rancher/node
+    - "{{ k3s_data_dir }}/agent/pod-manifests"
 
 - name: Check if k3s token file exists on the first node
   run_once: true
@@ -62,6 +63,16 @@
     - src: k3s.service.j2
       dest: "{{ k3s_service_file }}"
 
+- name: Copy kube-vip manifests
+  when: "'masters' in group_names"
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0644
+  loop:
+    - src: kube-vip.yaml.j2
+      dest: "{{ k3s_data_dir }}/agent/pod-manifests/kube-vip.yaml"
+
 - name: Enable k3s service
   ansible.builtin.systemd:
     name: k3s
@@ -74,12 +85,12 @@
 - name: Get Kubernetes config file
   run_once: true
   ansible.builtin.slurp:
-    src: /etc/rancher/k3s/k3s.yaml
+    src: "{{ k3s_kubeconfig_file }}"
   register: kubeconfig_base64
 
 - name: Write Kubernetes config file with the correct cluster address
   ansible.builtin.copy:
-    content: "{{ kubeconfig_base64.content | b64decode | replace('127.0.0.1', hostvars[groups['masters'][0]].ansible_host) }}"
+    content: "{{ kubeconfig_base64.content | b64decode | replace('127.0.0.1', control_plane_endpoint) }}"
     dest: "{{ playbook_dir }}/kubeconfig.yaml"
     mode: 0600
   delegate_to: localhost
diff --git a/metal/roles/k3s/templates/config.yaml.j2 b/metal/roles/k3s/templates/config.yaml.j2
index ec8b0d59..50ab4195 100644
--- a/metal/roles/k3s/templates/config.yaml.j2
+++ b/metal/roles/k3s/templates/config.yaml.j2
@@ -1,7 +1,7 @@
 {% if inventory_hostname == groups['masters'][0] %}
 cluster-init: true
 {% else %}
-server: https://{{ hostvars[groups['masters'][0]].ansible_host }}:6443
+server: https://{{ control_plane_endpoint }}:6443
 {% endif %}
 token-file: {{ k3s_token_file }}
 {% if 'masters' in group_names %}
diff --git a/metal/roles/k3s/templates/kube-vip.yaml.j2 b/metal/roles/k3s/templates/kube-vip.yaml.j2
new file mode 100644
index 00000000..53131498
--- /dev/null
+++ b/metal/roles/k3s/templates/kube-vip.yaml.j2
@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-vip
+  namespace: kube-system
+spec:
+  containers:
+    - name: kube-vip
+      image: ghcr.io/kube-vip/kube-vip:v0.6.4
+      args:
+        - manager
+      env:
+        - name: address
+          value: {{ control_plane_endpoint }}
+        - name: vip_arp
+          value: "true"
+        - name: cp_enable
+          value: "true"
+        - name: vip_leaderelection
+          value: "true"
+        - name: lb_enable
+          value: "true"
+      securityContext:
+        capabilities:
+          add:
+            - NET_ADMIN
+            - NET_RAW
+      volumeMounts:
+        - mountPath: /etc/kubernetes/admin.conf
+          name: kubeconfig
+  hostAliases:
+    - hostnames:
+        - kubernetes
+      ip: 127.0.0.1
+  hostNetwork: true
+  volumes:
+    - hostPath:
+        path: {{ k3s_kubeconfig_file }}
+      name: kubeconfig