From 8d00d55eb1faa214b29ab73fb0405351d47b079b Mon Sep 17 00:00:00 2001 From: Khue Doan Date: Wed, 17 Apr 2024 15:21:10 +0700 Subject: [PATCH] refactor(argocd)!: merge bootstrap and system This is a breaking change and requires cluster rebuild (carefully replacing the ApplicationSets may should work but I didn't bother at the current alpha stage): - ApplicationSets are merged into a single root one to use the progressive sync feature when it's ready. - Switched to server side apply to avoid CRDs not ready issues. Also replace the apply script with Ansible, since the Ansible Helm dependency update feature was released. --- Makefile | 10 +-- bootstrap/Makefile | 15 ---- bootstrap/argocd/apply.sh | 18 ---- bootstrap/argocd/values-seed.yaml | 12 --- bootstrap/argocd/values.yaml | 31 ------- bootstrap/main.yml | 1 - bootstrap/root/Chart.yaml | 3 - bootstrap/root/apply.sh | 13 --- bootstrap/root/templates/stack.yaml | 41 --------- bootstrap/root/values-seed.yaml | 2 - bootstrap/root/values.yaml | 9 -- docs/reference/architecture/overview.md | 106 ++---------------------- scripts/configure | 3 +- system/Makefile | 8 ++ {bootstrap => system}/argocd/Chart.yaml | 3 + system/argocd/values-seed.yaml | 28 +++++++ system/argocd/values.yaml | 69 +++++++++++++++ system/bootstrap.yml | 36 ++++++++ 18 files changed, 157 insertions(+), 251 deletions(-) delete mode 100644 bootstrap/Makefile delete mode 100755 bootstrap/argocd/apply.sh delete mode 100644 bootstrap/argocd/values-seed.yaml delete mode 100644 bootstrap/argocd/values.yaml delete mode 100644 bootstrap/main.yml delete mode 100644 bootstrap/root/Chart.yaml delete mode 100755 bootstrap/root/apply.sh delete mode 100644 bootstrap/root/templates/stack.yaml delete mode 100644 bootstrap/root/values-seed.yaml delete mode 100644 bootstrap/root/values.yaml create mode 100644 system/Makefile rename {bootstrap => system}/argocd/Chart.yaml (60%) create mode 100644 system/argocd/values-seed.yaml create mode 100644 system/argocd/values.yaml create mode 100644 system/bootstrap.yml diff --git a/Makefile b/Makefile index f6cbf526..536672c4 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ KUBECONFIG = $(shell pwd)/metal/kubeconfig.yaml KUBE_CONFIG_PATH = $(KUBECONFIG) -default: metal bootstrap external smoke-test post-install clean +default: metal system external smoke-test post-install clean configure: ./scripts/configure @@ -14,8 +14,8 @@ configure: metal: make -C metal -bootstrap: - make -C bootstrap +system: + make -C system external: make -C external @@ -48,10 +48,6 @@ test: clean: docker compose --project-directory ./metal/roles/pxe_server/files down -dev: - make -C metal cluster env=dev - make -C bootstrap - docs: mkdocs serve diff --git a/bootstrap/Makefile b/bootstrap/Makefile deleted file mode 100644 index fa29123f..00000000 --- a/bootstrap/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -.POSIX: - -default: namespace argocd root - -namespace: - kubectl create namespace argocd --dry-run=client --output=yaml \ - | kubectl apply -f - - -.PHONY: argocd -argocd: - cd argocd && ./apply.sh - -.PHONY: root -root: - cd root && ./apply.sh diff --git a/bootstrap/argocd/apply.sh b/bootstrap/argocd/apply.sh deleted file mode 100755 index 702f22b8..00000000 --- a/bootstrap/argocd/apply.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -VALUES="values.yaml" - -kubectl get ingress argocd-server --namespace argocd \ - || VALUES="values-seed.yaml" - -helm template \ - --dependency-update \ - --include-crds \ - --namespace argocd \ - --values "${VALUES}" \ - argocd . \ - | kubectl apply -n argocd -f - - -kubectl -n argocd wait --timeout=60s --for condition=Established \ - crd/applications.argoproj.io \ - crd/applicationsets.argoproj.io diff --git a/bootstrap/argocd/values-seed.yaml b/bootstrap/argocd/values-seed.yaml deleted file mode 100644 index f8ead7da..00000000 --- a/bootstrap/argocd/values-seed.yaml +++ /dev/null @@ -1,12 +0,0 @@ -argo-cd: - server: - metrics: &metrics - enabled: false - serviceMonitor: - enabled: false - controller: - metrics: *metrics - repoServer: - metrics: *metrics - redis: - metrics: *metrics diff --git a/bootstrap/argocd/values.yaml b/bootstrap/argocd/values.yaml deleted file mode 100644 index 14e5f887..00000000 --- a/bootstrap/argocd/values.yaml +++ /dev/null @@ -1,31 +0,0 @@ -argo-cd: - global: - domain: argocd.khuedoan.com - configs: - params: - server.insecure: true - controller.diff.server.side: true - cm: - resource.ignoreResourceUpdatesEnabled: true - resource.customizations.ignoreResourceUpdates.all: | - jsonPointers: - - /status - server: - ingress: - enabled: true - ingressClassName: nginx - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - tls: true - metrics: &metrics - enabled: true - serviceMonitor: - enabled: true - dex: - enabled: false - controller: - metrics: *metrics - repoServer: - metrics: *metrics - redis: - metrics: *metrics diff --git a/bootstrap/main.yml b/bootstrap/main.yml deleted file mode 100644 index 847ad96b..00000000 --- a/bootstrap/main.yml +++ /dev/null @@ -1 +0,0 @@ -# TODO convert ./argocd/apply.sh and ./root/apply.sh to Ansible playbook diff --git a/bootstrap/root/Chart.yaml b/bootstrap/root/Chart.yaml deleted file mode 100644 index 563e5408..00000000 --- a/bootstrap/root/Chart.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: v2 -name: root -version: 0.0.0 diff --git a/bootstrap/root/apply.sh b/bootstrap/root/apply.sh deleted file mode 100755 index 2dd4465f..00000000 --- a/bootstrap/root/apply.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -VALUES="values.yaml" - -kubectl get ingress gitea --namespace gitea \ - || VALUES="values-seed.yaml" - -helm template \ - --include-crds \ - --namespace argocd \ - --values "${VALUES}" \ - argocd . \ - | kubectl apply -n argocd -f - diff --git a/bootstrap/root/templates/stack.yaml b/bootstrap/root/templates/stack.yaml deleted file mode 100644 index b16dbd1f..00000000 --- a/bootstrap/root/templates/stack.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- range $index, $stack := .Values.stacks }} ---- -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: {{ $stack.name }} - namespace: {{ $.Release.Namespace }} -spec: - generators: - - git: - repoURL: {{ $.Values.gitops.repo }} - revision: {{ $.Values.gitops.revision }} - directories: - - path: {{ $stack.name }}/* - template: - metadata: - name: '{{ `{{path.basename}}` }}' - spec: - destination: - name: in-cluster - namespace: '{{ default `{{path.basename}}` $stack.namespace }}' - project: default # TODO - source: - repoURL: {{ $.Values.gitops.repo }} - path: '{{ `{{path}}` }}' - targetRevision: {{ $.Values.gitops.revision }} - syncPolicy: - automated: - prune: true - selfHeal: true - retry: - limit: 10 - backoff: - duration: 1m - factor: 2 - maxDuration: 16m - syncOptions: - - CreateNamespace=true - - ApplyOutOfSyncOnly=true - - ServerSideApply=true -{{- end }} diff --git a/bootstrap/root/values-seed.yaml b/bootstrap/root/values-seed.yaml deleted file mode 100644 index ae729830..00000000 --- a/bootstrap/root/values-seed.yaml +++ /dev/null @@ -1,2 +0,0 @@ -gitops: - repo: https://github.com/khuedoan/homelab diff --git a/bootstrap/root/values.yaml b/bootstrap/root/values.yaml deleted file mode 100644 index 37e96eec..00000000 --- a/bootstrap/root/values.yaml +++ /dev/null @@ -1,9 +0,0 @@ -gitops: - repo: http://gitea-http.gitea:3000/ops/homelab - revision: master -stacks: - - name: bootstrap - namespace: argocd # Override default value - - name: system - - name: platform - - name: apps diff --git a/docs/reference/architecture/overview.md b/docs/reference/architecture/overview.md index 55352557..e0159d1b 100644 --- a/docs/reference/architecture/overview.md +++ b/docs/reference/architecture/overview.md @@ -10,8 +10,6 @@ |--------------| +------------+ | ./system |- - - -| ./external | |--------------| +------------+ -| ./bootstrap | -|--------------| | ./metal | |--------------| | HARDWARE | @@ -21,7 +19,6 @@ Main components: - `./metal`: bare metal management, install Linux and Kubernetes -- `./bootstrap`: GitOps bootstrap with ArgoCD - `./system`: critical system components for the cluster (load balancer, storage, ingress, operation tools...) - `./platform`: essential components for service hosting platform (git, build runners, dashboards...) - `./apps`: user facing applications @@ -41,15 +38,11 @@ Everything is automated, after you edit the configuration files, you just need t - Create an ephemeral, stateless PXE server - Install Linux on all servers in parallel - Build a Kubernetes cluster (based on k3s) -- (2) Build the `./bootstrap` layer: - - Install ArgoCD - - Configure the root app to manage other layers (and also manage itself) - -From now on, ArgoCD will do the rest: - -- (3) Build the `./system` layer (storage, networking, monitoring, etc) -- (4) Build the `./platform` layer (Gitea, Grafana, SSO, etc) -- (5) Build the `./apps` layer: (Syncthing, Jellyfin, etc) +- (2) Bootstrap the `./system` layer: + - Install ArgoCD and the root app to manage itself and other layers, from now on ArgoCD will do the rest + - Install the remaining components (storage, monitoring, etc) +- (3) Build the `./platform` layer (Gitea, Grafana, SSO, etc) +- (4) Deploy applications in the `./apps` layer ```mermaid flowchart TD @@ -57,11 +50,8 @@ flowchart TD pxe[PXE Server] -.-> linux[Fedora Server] --> k3s end - subgraph bootstrap[./bootstrap] - argocd[ArgoCD] --> rootapp[Root app] - end - subgraph system[./system] + argocd[ArgoCD and root app] nginx[NGINX] rook-ceph[Rook Ceph] cert-manager @@ -79,96 +69,18 @@ flowchart TD cloudflare -.-> external-dns cloudflare -.-> cloudflared - subgraph platform + subgraph platform[./platform] Gitea Woodpecker Grafana end - subgraph apps + subgraph apps[./apps] homepage[Homepage] jellyfin[Jellyfin] matrix[Matrix] paperless[Paperless] end - make[Run make] -- 1 --> metal -- 2 --> bootstrap -. 3 .-> system -. 4 .-> platform -. 5 .-> apps + make[Run make] -- 1 --> metal -- 2 --> system -. 3 .-> platform -. 4 .-> apps ``` - -Below is the pseudo code for the entire process, you don't have to read it right now, but it will be handy for debugging. - -??? detailed "Detailed provisioning flow" - - ``` - Human run make: - build ./metal: - install the OS: - download the installer image and extract it - create a PXE server on the controller using Docker Compose: - DHCP server - TFTP server - HTTP server - create init config for each machine - turn the machines on via WoL - the machines boot: - select network boot automatically - broadcast DHCP request - DHCP server reply: - machine IP - TFTP server (next-server) IP - get boot files from TFTP server - GRUB - GRUB config with URL to init config based on MAC address - kernel - initrd - boot to the kernel - download from HTTP server: - init config from the URL in GRUB config - remaining files required to boot - install the OS based on the init config: - configure the system - remaining files required to install - reboot to the new OS - controller see the machines are ready - build a Kubernetes cluster: - download k3s binary - generate cluster token - copy k3s config files - enable k3s service and form a cluster - create KUBECONFIG file - install Cilium - build ./bootstrap: - install ArgoCD: - apply helm chart - wait for status - install root app: - select values file: - if Gitea unreachable (first install): - get data from GitHub - else: - get data from Gitea - apply helm chart - wait for status - ArgoCD apply the rest: - clone git repo - install components based on directories: - ./bootstrap (it manages itself): - argocd - root - ./system: - storage - loadbalancer - ingress - etc - ./platform (depends on ./system): - git: - migrate the homelab repository from GitHub - ArgoCD switch the source from GitHub to Gitea - ci - dashboards - etc - ./apps (depends on ./system and ./platform): - homepage - jellyfin - etc - ``` diff --git a/scripts/configure b/scripts/configure index a53a6f09..a93ba3dd 100755 --- a/scripts/configure +++ b/scripts/configure @@ -57,7 +57,6 @@ def main() -> None: paths=[ ".ci", "apps", - "bootstrap", "platform", "system", "external" @@ -68,7 +67,7 @@ def main() -> None: pattern=upstream_config['seed_repo'], replacement=seed_repo, paths=[ - "bootstrap", + "system", "platform" ] ) diff --git a/system/Makefile b/system/Makefile new file mode 100644 index 00000000..2fbf29f0 --- /dev/null +++ b/system/Makefile @@ -0,0 +1,8 @@ +.POSIX: + +export KUBECONFIG = $(shell pwd)/../metal/kubeconfig.yaml + +.PHONY: bootstrap +bootstrap: + ansible-playbook \ + bootstrap.yml diff --git a/bootstrap/argocd/Chart.yaml b/system/argocd/Chart.yaml similarity index 60% rename from bootstrap/argocd/Chart.yaml rename to system/argocd/Chart.yaml index e0b86840..c3b2c050 100644 --- a/bootstrap/argocd/Chart.yaml +++ b/system/argocd/Chart.yaml @@ -5,3 +5,6 @@ dependencies: - name: argo-cd version: 6.4.1 repository: https://argoproj.github.io/argo-helm + - name: argocd-apps + version: 2.0.0 + repository: https://argoproj.github.io/argo-helm diff --git a/system/argocd/values-seed.yaml b/system/argocd/values-seed.yaml new file mode 100644 index 00000000..4e8b3c90 --- /dev/null +++ b/system/argocd/values-seed.yaml @@ -0,0 +1,28 @@ +argo-cd: + server: + metrics: &metrics + enabled: false + serviceMonitor: + enabled: false + controller: + metrics: *metrics + repoServer: + metrics: *metrics + redis: + metrics: *metrics +argocd-apps: + applicationsets: + root: + generators: + - git: + repoURL: &repoURL https://github.com/khuedoan/homelab + revision: &revision master + directories: + - path: system/* + - path: platform/* + - path: apps/* + template: + spec: + source: + repoURL: *repoURL + targetRevision: *revision diff --git a/system/argocd/values.yaml b/system/argocd/values.yaml new file mode 100644 index 00000000..1797ae3d --- /dev/null +++ b/system/argocd/values.yaml @@ -0,0 +1,69 @@ +argo-cd: + global: + domain: argocd.khuedoan.com + configs: + params: + server.insecure: true + controller.diff.server.side: true + cm: + resource.ignoreResourceUpdatesEnabled: true + resource.customizations.ignoreResourceUpdates.all: | + jsonPointers: + - /status + server: + ingress: + enabled: true + ingressClassName: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + tls: true + metrics: &metrics + enabled: true + serviceMonitor: + enabled: true + dex: + enabled: false + controller: + metrics: *metrics + repoServer: + metrics: *metrics + redis: + metrics: *metrics +argocd-apps: + applicationsets: + root: + namespace: argocd + generators: + - git: + repoURL: &repoURL http://gitea-http.gitea:3000/ops/homelab + revision: &revision master + directories: + - path: system/* + - path: platform/* + - path: apps/* + template: + metadata: + name: '{{path.basename}}' + spec: + destination: + name: in-cluster + namespace: '{{path.basename}}' + project: default # TODO + source: + repoURL: *repoURL + path: '{{path}}' + targetRevision: *revision + syncPolicy: + automated: + prune: true + selfHeal: true + retry: + limit: 10 + backoff: + duration: 1m + factor: 2 + maxDuration: 16m + syncOptions: + - CreateNamespace=true + - ApplyOutOfSyncOnly=true + - ServerSideApply=true diff --git a/system/bootstrap.yml b/system/bootstrap.yml new file mode 100644 index 00000000..37d5c1b7 --- /dev/null +++ b/system/bootstrap.yml @@ -0,0 +1,36 @@ +- name: Bootstrapping the cluster + hosts: localhost + tasks: + - name: Create ArgoCD namespace + kubernetes.core.k8s: + api_version: v1 + kind: Namespace + name: argocd + state: present + + - name: Check if this is the first installation + kubernetes.core.k8s_info: + kind: Pod + label_selectors: + - app.kubernetes.io/instance=gitea + field_selectors: + - status.phase=Running + register: first_install + + - name: Render ArgoCD manifests from Helm chart + kubernetes.core.helm_template: + chart_ref: ./argocd + include_crds: true + release_name: argocd + release_namespace: argocd + dependency_update: true + values_files: + - "argocd/{{ (first_install.resources | length == 0) | ternary('values-seed.yaml', 'values.yaml') }}" + register: argocd_manifests + + - name: Apply ArgoCD manifests + kubernetes.core.k8s: + resource_definition: "{{ argocd_manifests.stdout }}" + apply: true + server_side_apply: + field_manager: argocd-controller