diff --git a/docs/todo.md b/docs/todo.md index 0b3c3f50..0549338f 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -3,8 +3,8 @@ - [Better Terraform provider inheritance](../infra/terraform.tf#L8) - [(bug) Apply LXD VMs in parallel](../infra/Makefile#L12) - [(bug) Investigate why --noconfirm is not working](../tools/Dockerfile#L3) -- [(bug) Should be posible to put it in the profile instead lxd_profile.kubenode.config, and make it a variable](../infra/modules/kubernetes-cluster/main.tf#L101) -- [(bug) Use containers instead of virtual machines for Kubernetes nodes https](../infra/modules/kubernetes-cluster/main.tf#L94) +- [(bug) Should be posible to put it in the profile instead lxd_profile.master_profile.config, and make it a variable](../infra/modules/kubernetes-cluster/main.tf#L145) +- [(bug) Use containers instead of virtual machines for Kubernetes nodes https](../infra/modules/kubernetes-cluster/main.tf#L155) - [(feature) Automatic ingress and tunnel for all services](../infra/modules/kubernetes-bootstrap/main.tf#L85) - [(feature) Enable etcd authentication and generate terraform backend config variables](../metal/roles/tfstate/tasks/main.yml#L43) - [(feature) Generate endpoint automatically (terragrunt for variable)](../infra/terraform.tf#L2) @@ -12,7 +12,7 @@ - [(optimize) Change to /var/lib/lxd/server.crt after https](../metal/roles/lxd/tasks/main.yml#L26) - [(optimize) Convert to YAML for Terraform yamldecode](../metal/hosts.ini#L1) - [(optimize) Decide if VPN should be inside Kubernetes](../infra/base.tf#L1) -- [(optimize) DRY master and worker definition](../infra/modules/kubernetes-cluster/main.tf#L89) +- [(optimize) DRY master and worker definition](../infra/modules/kubernetes-cluster/main.tf#L135) - [(optimize) HA Vault and auto unseal Vault](../infra/modules/kubernetes-bootstrap/main.tf#L82) - [(optimize) LXD node firewall](../metal/roles/lxd/tasks/main.yml#L6) - [(optimize) LXD node SELinux](../metal/roles/lxd/tasks/main.yml#L1) diff --git a/infra/README.md b/infra/README.md index bedb246e..5d7e9e31 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1 +1,27 @@ # Private cloud infrastructure + +Current state: + +`lxc list` + +``` ++----------+---------+-----------------------+------+-----------------+-----------+----------+ +| NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS | LOCATION | ++----------+---------+-----------------------+------+-----------------+-----------+----------+ +| tfstate | RUNNING | 192.168.1.25 (eth0) | | CONTAINER | 0 | metal0 | ++----------+---------+-----------------------+------+-----------------+-----------+----------+ +| master-0 | RUNNING | 192.168.1.49 (eth0) | | CONTAINER | 0 | metal3 | ++----------+---------+-----------------------+------+-----------------+-----------+----------+ +| master-1 | RUNNING | 192.168.1.48 (eth0) | | CONTAINER | 0 | metal1 | ++----------+---------+-----------------------+------+-----------------+-----------+----------+ +| master-2 | RUNNING | 192.168.1.50 (eth0) | | CONTAINER | 0 | metal2 | ++----------+---------+-----------------------+------+-----------------+-----------+----------+ +| worker-0 | RUNNING | 192.168.1.52 (enp5s0) | | VIRTUAL-MACHINE | 0 | metal1 | ++----------+---------+-----------------------+------+-----------------+-----------+----------+ +| worker-1 | RUNNING | 192.168.1.51 (enp5s0) | | VIRTUAL-MACHINE | 0 | metal0 | ++----------+---------+-----------------------+------+-----------------+-----------+----------+ +| worker-2 | RUNNING | 192.168.1.53 (enp5s0) | | VIRTUAL-MACHINE | 0 | metal3 | ++----------+---------+-----------------------+------+-----------------+-----------+----------+ +``` + +Currently, worker nodes are virtual machines instead of containers because of an ISCSI bug in the Linux kernel. diff --git a/infra/modules/kubernetes-cluster/main.tf b/infra/modules/kubernetes-cluster/main.tf index d1973a99..0dc16b46 100644 --- a/infra/modules/kubernetes-cluster/main.tf +++ b/infra/modules/kubernetes-cluster/main.tf @@ -9,22 +9,89 @@ resource "local_file" "ssh_private_key" { file_permission = "0600" } -resource "lxd_profile" "kubenode" { - name = "kubenode" +resource "lxd_profile" "master_profile" { + name = "kube-master" + + config = { + "limits.cpu" = 1 + "limits.memory" = "2GiB" + "limits.memory.swap" = false + "security.nesting" = true + "security.privileged" = true + "linux.kernel_modules" = "ip_tables,ip6_tables,nf_nat,overlay,br_netfilter" + "raw.lxc" = <<-EOT + lxc.apparmor.profile=unconfined + lxc.cap.drop= + lxc.cgroup.devices.allow=a + lxc.mount.auto=proc:rw sys:rw cgroup:rw + EOT + "user.user-data" = <<-EOT + #cloud-config + ssh_authorized_keys: + - ${tls_private_key.ssh.public_key_openssh} + disable_root: false + runcmd: + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + - add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - apt-get update -y + - apt-get install -y docker-ce docker-ce-cli containerd.io + - mkdir -p /etc/systemd/system/docker.service.d/ + - printf "[Service]\nMountFlags=shared" > /etc/systemd/system/docker.service.d/mount_flags.conf + - mount --make-rshared / + - systemctl enable --now docker + EOT + } + + # # # echo "262144" > /sys/module/nf_conntrack/parameters/hashsize + # device { + # type = "disk" + # name = "hashsize" + + # properties = { + # source = "/sys/module/nf_conntrack/parameters/hashsize" + # path = "/sys/module/nf_conntrack/parameters/hashsize" + # } + # } + + device { + type = "unix-char" + name = "kmsg" + + properties = { + source = "/dev/kmsg" + path = "/dev/kmsg" + } + } + + device { + name = "eth0" + type = "nic" + + properties = { + nictype = "macvlan" + parent = "eno1" + } + } + + device { + type = "disk" + name = "root" + + properties = { + pool = "default" + path = "/" + size = "8GiB" + } + } +} + +resource "lxd_profile" "worker_profile" { + name = "kube-worker" config = { "limits.cpu" = 2 "limits.memory" = "4GiB" "limits.memory.swap" = false - # "security.nesting" = true - # "security.privileged" = true - # "linux.kernel_modules" = "ip_tables,ip6_tables,nf_nat,overlay,br_netfilter" - # "raw.lxc" = <<-EOT - # lxc.apparmor.profile=unconfined - # lxc.cap.drop= - # lxc.cgroup.devices.allow=a - # lxc.mount.auto=proc:rw sys:rw cgroup:rw - # EOT "user.user-data" = <<-EOT #cloud-config ssh_authorized_keys: @@ -43,27 +110,6 @@ resource "lxd_profile" "kubenode" { EOT } - # # echo "262144" > /sys/module/nf_conntrack/parameters/hashsize - # device { - # type = "disk" - # name = "hashsize" - - # properties = { - # source = "/sys/module/nf_conntrack/parameters/hashsize" - # path = "/sys/module/nf_conntrack/parameters/hashsize" - # } - # } - - # device { - # type = "unix-char" - # name = "kmsg" - - # properties = { - # source = "/dev/kmsg" - # path = "/dev/kmsg" - # } - # } - device { name = "eth0" type = "nic" @@ -91,16 +137,14 @@ resource "lxd_container" "masters" { count = 3 name = "master-${count.index}" image = "ubuntu:20.04" - # TODO (bug) Use containers instead of virtual machines for Kubernetes nodes https://bugs.launchpad.net/ubuntu/+source/lxc/+bug/1226855 - type = "virtual-machine" ephemeral = false - profiles = [lxd_profile.kubenode.name] + profiles = [lxd_profile.master_profile.name] config = { - # TODO (bug) Should be posible to put it in the profile instead lxd_profile.kubenode.config, and make it a variable + # TODO (bug) Should be posible to put it in the profile instead lxd_profile.master_profile.config, and make it a variable # https://github.com/terraform-lxd/terraform-provider-lxd/blob/master/lxd/resource_lxd_container.go#L473 - "user.access_interface" = "enp5s0" + "user.access_interface" = "eth0" } } @@ -108,10 +152,11 @@ resource "lxd_container" "workers" { count = 3 name = "worker-${count.index}" image = "ubuntu:20.04" + # TODO (bug) Use containers instead of virtual machines for Kubernetes nodes https://bugs.launchpad.net/ubuntu/+source/lxc/+bug/1226855 type = "virtual-machine" ephemeral = false - profiles = [lxd_profile.kubenode.name] + profiles = [lxd_profile.worker_profile.name] config = { "user.access_interface" = "enp5s0"