feat!: install Cilium

Installed using Ansible instead of ArgoCD because Cilium replaces
the default CNI, so ArgoCD pod cannot be scheduled before Cilium
is installed.

metal/Makefile

@@ -1,6 +1,7 @@
 .POSIX:
 
-env ?= "prod"
+env ?= prod
+export KUBECONFIG = $(shell pwd)/kubeconfig.yaml
 
 default: boot cluster
 

metal/cluster.yml

@@ -3,3 +3,8 @@
   roles:
     - k3s
     - automatic_upgrade
+
+- name: Install Kubernetes addons
+  hosts: localhost
+  roles:
+    - cilium

metal/group_vars/all.yml

@@ -2,3 +2,5 @@ ansible_user: root
 ansible_ssh_private_key_file: ~/.ssh/id_ed25519
 ssh_public_key: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
 dns_server: "8.8.8.8"
+load_balancer_ip_pool:
+  - 192.168.1.224/27

metal/roles/cilium/defaults/main.yml

@@ -0,0 +1,20 @@
+cilium_repo_url: https://helm.cilium.io
+cilium_version: 1.14.4
+cilium_namespace: kube-system
+cilium_values:
+  operator:
+    replicas: 1
+  kubeProxyReplacement: true
+  l2announcements:
+    enabled: true
+  ingressController:
+    enabled: true
+    default: true
+    loadbalancerMode: shared
+    service:
+      allocateLoadBalancerNodePorts: false
+  # TODO the host and port are k3s-specific, generic solution is in progress
+  # https://github.com/cilium/cilium/issues/19038
+  # https://github.com/cilium/cilium/pull/28741
+  k8sServiceHost: 127.0.0.1
+  k8sServicePort: 6444

metal/roles/cilium/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: Install Cilium
+  kubernetes.core.helm:
+    name: cilium
+    chart_ref: cilium
+    chart_repo_url: "{{ cilium_repo_url }}"
+    chart_version: "{{ cilium_version }}"
+    release_namespace: "{{ cilium_namespace }}"
+    values: "{{ cilium_values }}"
+
+- name: Wait for Cilium CRDs
+  kubernetes.core.k8s_info:
+    kind: CustomResourceDefinition
+    name: "{{ item }}"
+  loop:
+    - ciliuml2announcementpolicies.cilium.io
+    - ciliumloadbalancerippools.cilium.io
+  register: crd
+  until: crd.resources | length > 0
+  retries: 5
+  delay: 10
+
+- name: Apply Cilium resources
+  kubernetes.core.k8s:
+    template: "{{ item }}"
+  loop:
+    - ciliuml2announcementpolicy.yaml
+    - ciliumloadbalancerippool.yaml

metal/roles/cilium/templates/ciliuml2announcementpolicy.yaml

@@ -0,0 +1,7 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: default
+spec:
+  externalIPs: true
+  loadBalancerIPs: true

metal/roles/cilium/templates/ciliumloadbalancerippool.yaml

@@ -0,0 +1,9 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: default
+spec:
+  cidrs:
+    {% for cidr in load_balancer_ip_pool %}
+    - cidr: {{ cidr }}
+    {% endfor %}

metal/roles/k3s/defaults/main.yml

@@ -8,5 +8,8 @@ k3s_server_config:
     - servicelb
     - traefik
   disable-cloud-controller: true
+  disable-helm-controller: true
+  disable-kube-proxy: true
+  disable-network-policy: true
+  flannel-backend: none
   secrets-encryption: true
-  snapshotter: stargz

scripts/configure

@@ -87,7 +87,8 @@ def main() -> None:
         pattern=upstream_config['loadbalancer_ip_range'],
         replacement=loadbalancer_ip_range,
         paths=[
-            "system/metallb-system/templates/ipaddresspool.yaml",
+            "metal/group_vars/all.yml",
+            "external/main.tf",
         ]
     )