diff --git a/metal/main.yml b/metal/main.yml index 956bb25c..4bc6a240 100644 --- a/metal/main.yml +++ b/metal/main.yml @@ -8,9 +8,3 @@ gather_facts: no roles: - wake - -- name: Create Terraform state storage - hosts: metal[0] - gather_facts: no - roles: - - tfstate diff --git a/metal/roles/pxe-server/defaults/main.yml b/metal/roles/pxe-server/defaults/main.yml index 7c5b0b81..0d54a00c 100644 --- a/metal/roles/pxe-server/defaults/main.yml +++ b/metal/roles/pxe-server/defaults/main.yml @@ -1,6 +1,3 @@ -iso_url: "https://builds.coreos.fedoraproject.org/prod/streams/testing/builds/34.20210808.2.0/x86_64/fedora-coreos-34.20210808.2.0-live.x86_64.iso" -iso_checksum: "sha256:8ec901fcb5bf9f05cf8046cbe7bce29e36600b7ec61165577c1b3c565b85425c" - -os_name: CoreOS -# TODO (optimize) Get timezone automatically from the controller +iso_url: "https://download.rockylinux.org/pub/rocky/8/isos/x86_64/Rocky-8.4-x86_64-minimal.iso" +iso_checksum: "sha256:0de5f12eba93e00fefc06cdb0aa4389a0972a4212977362ea18bde46a1a1aa4f" timezone: Asia/Ho_Chi_Minh diff --git a/metal/roles/pxe-server/files/data/init-config/.gitignore b/metal/roles/pxe-server/files/data/init-config/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/metal/roles/pxe-server/files/data/init-config/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/metal/roles/pxe-server/files/data/iso/.gitignore b/metal/roles/pxe-server/files/data/iso/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/metal/roles/pxe-server/files/data/iso/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/metal/roles/pxe-server/files/data/os/.gitignore b/metal/roles/pxe-server/files/data/os/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/metal/roles/pxe-server/files/data/os/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/metal/roles/pxe-server/files/data/pxe-config/.gitignore b/metal/roles/pxe-server/files/data/pxe-config/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/metal/roles/pxe-server/files/data/pxe-config/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/metal/roles/pxe-server/templates/dhcp/Dockerfile b/metal/roles/pxe-server/files/dhcp/Dockerfile similarity index 100% rename from metal/roles/pxe-server/templates/dhcp/Dockerfile rename to metal/roles/pxe-server/files/dhcp/Dockerfile diff --git a/metal/roles/pxe-server/files/dhcp/dhcpd.conf b/metal/roles/pxe-server/files/dhcp/dhcpd.conf new file mode 100644 index 00000000..6abb83d3 --- /dev/null +++ b/metal/roles/pxe-server/files/dhcp/dhcpd.conf @@ -0,0 +1,20 @@ +option space pxelinux; +option pxelinux.magic code 208 = string; +option pxelinux.configfile code 209 = text; +option pxelinux.pathprefix code 210 = text; +option pxelinux.reboottime code 211 = unsigned integer 32; +option architecture-type code 93 = unsigned integer 16; + +subnet 192.168.1.0 netmask 255.255.255.0 { + option routers 192.168.1.1; + range 192.168.1.2 192.168.1.254; + + class "pxeclients" { + match if substring (option vendor-class-identifier, 0, 9) = "PXEClient"; + next-server 192.168.1.12; + + if option architecture-type = 00:07 { + filename "grubx64.efi"; + } + } +} diff --git a/metal/roles/pxe-server/files/docker-compose.yml b/metal/roles/pxe-server/files/docker-compose.yml new file mode 100644 index 00000000..c8814857 --- /dev/null +++ b/metal/roles/pxe-server/files/docker-compose.yml @@ -0,0 +1,24 @@ +version: "3" + +services: + dhcp: + build: ./dhcp + volumes: + - ./data/pxe-config/dhcpd.conf:/etc/dhcp/dhcpd.conf + network_mode: host + tftp: + build: ./tftp + network_mode: host + volumes: + - ./data/pxe-config/grub.cfg:/var/lib/tftpboot/grub.cfg + - ./data/os/EFI/BOOT/grubx64.efi:/var/lib/tftpboot/grubx64.efi + - ./data/os/images/pxeboot/initrd.img:/var/lib/tftpboot/initrd.img + - ./data/os/images/pxeboot/vmlinuz:/var/lib/tftpboot/vmlinuz + http: + build: ./http + network_mode: host + volumes: + - ./data/os:/usr/share/nginx/html/os + - ./data/init-config/:/usr/share/nginx/html/init-config + environment: + NGINX_PORT: 80 diff --git a/metal/roles/pxe-server/templates/http/Dockerfile b/metal/roles/pxe-server/files/http/Dockerfile similarity index 100% rename from metal/roles/pxe-server/templates/http/Dockerfile rename to metal/roles/pxe-server/files/http/Dockerfile diff --git a/metal/roles/pxe-server/templates/tftp/Dockerfile b/metal/roles/pxe-server/files/tftp/Dockerfile similarity index 100% rename from metal/roles/pxe-server/templates/tftp/Dockerfile rename to metal/roles/pxe-server/files/tftp/Dockerfile diff --git a/metal/roles/pxe-server/files/tftp/grub.cfg b/metal/roles/pxe-server/files/tftp/grub.cfg new file mode 100644 index 00000000..d9ae55fa --- /dev/null +++ b/metal/roles/pxe-server/files/tftp/grub.cfg @@ -0,0 +1,8 @@ +set timeout=1 + +menuentry 'Rocky-8.4-x86_64-minimal (PXE)' { + linux vmlinuz \ + ip=dhcp \ + ks=http://192.168.1.12/init-config/${net_default_mac}.ks + initrd initrd.img +} diff --git a/metal/roles/pxe-server/tasks/main.yml b/metal/roles/pxe-server/tasks/main.yml index bb0f7684..c1d88e3f 100644 --- a/metal/roles/pxe-server/tasks/main.yml +++ b/metal/roles/pxe-server/tasks/main.yml @@ -1,77 +1,34 @@ -- name: Gather network facts - ansible.builtin.setup: - gather_subset: - - network - -- name: Create build directory - file: - path: "{{ item }}" - state: directory - loop: - - "{{ role_path }}/build" - - "{{ role_path }}/files/images" - - "{{ role_path }}/build/mnt" - - name: Download ISO get_url: url: "{{ iso_url }}" - dest: "{{ role_path }}/files/images/{{ iso_url | basename }}" + dest: "{{ role_path }}/files/data/iso/{{ iso_url | basename }}" checksum: "{{ iso_checksum }}" register: iso - name: Extract the ISO command: - cmd: "xorriso -osirrox on -indev {{ iso.dest }} -extract / {{ role_path }}/build/mnt" - creates: "{{ role_path }}/build/mnt/.treeinfo" - -- name: Extract bootloader - iso_extract: - image: "{{ role_path }}/build/mnt/images/efiboot.img" - dest: "{{ role_path }}/build/mnt/EFI/fedora" - files: - - EFI/fedora/grubx64.efi - -- name: Copy configs - copy: - src: "{{ role_path }}/templates/" - dest: "{{ role_path }}/build" + cmd: "xorriso -osirrox on -indev {{ iso.dest }} -extract / {{ role_path }}/files/data/os" + creates: "{{ role_path }}/files/data/os/.treeinfo" - name: Render DHCP config template: - src: dhcp/dhcpd.conf.j2 - dest: "{{ role_path }}/build/dhcp/dhcpd.conf" + src: dhcpd.conf.j2 + dest: "{{ role_path }}/files/data/pxe-config/dhcpd.conf" - name: Render GRUB config template: - src: tftp/tftpboot/grub.cfg.j2 - dest: "{{ role_path }}/build/tftp/tftpboot/grub.cfg" + src: grub.cfg.j2 + dest: "{{ role_path }}/files/data/pxe-config/grub.cfg" -- name: Render machine specific Butane config +- name: Render machine specific init config template: - src: http/ignition/ignition.yaml.j2 - dest: "{{ role_path }}/build/http/ignition/{{ hostvars[item]['mac'] }}.yaml" - loop: "{{ groups['metal'] }}" - -- name: Render Ignition config from Butane config - docker_container: - name: butane - image: quay.io/coreos/butane:release - auto_remove: yes - volumes: - - "{{ role_path }}/build/http/ignition:/local/src" - working_dir: /local/src - command: - - --pretty - - --strict - - "{{ hostvars[item]['mac'] }}.yaml" - - --output - - "{{ hostvars[item]['mac'] }}.json" + src: kickstart.ks.j2 + dest: "{{ role_path }}/files/data/init-config/{{ hostvars[item]['mac'] }}.ks" loop: "{{ groups['metal'] }}" - name: Start ephemeral PXE server docker_compose: - project_src: "{{ role_path }}/build" + project_src: "{{ role_path }}/files" state: present restarted: yes build: yes - recreate: always diff --git a/metal/roles/pxe-server/templates/dhcp/dhcpd.conf.j2 b/metal/roles/pxe-server/templates/dhcpd.conf.j2 similarity index 100% rename from metal/roles/pxe-server/templates/dhcp/dhcpd.conf.j2 rename to metal/roles/pxe-server/templates/dhcpd.conf.j2 diff --git a/metal/roles/pxe-server/templates/docker-compose.yml b/metal/roles/pxe-server/templates/docker-compose.yml deleted file mode 100644 index dcacf065..00000000 --- a/metal/roles/pxe-server/templates/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: "3" - -services: - dhcp: - build: ./dhcp - volumes: - - ./dhcp/dhcpd.conf:/etc/dhcp/dhcpd.conf - network_mode: host - tftp: - build: ./tftp - network_mode: host - volumes: - - ./tftp/tftpboot/grub.cfg:/var/lib/tftpboot/grub.cfg - - ./mnt/EFI/fedora/grubx64.efi:/var/lib/tftpboot/grubx64.efi - - ./mnt/images/ignition.img:/var/lib/tftpboot/ignition.img - - ./mnt/images/pxeboot/initrd.img:/var/lib/tftpboot/initrd.img - - ./mnt/images/pxeboot/vmlinuz:/var/lib/tftpboot/vmlinuz - http: - build: ./http - network_mode: host - volumes: - - ./mnt:/usr/share/nginx/html/CoreOS - - ./http/ignition/:/usr/share/nginx/html/ignition - environment: - NGINX_PORT: 80 diff --git a/metal/roles/pxe-server/templates/grub.cfg.j2 b/metal/roles/pxe-server/templates/grub.cfg.j2 new file mode 100644 index 00000000..bbd3c977 --- /dev/null +++ b/metal/roles/pxe-server/templates/grub.cfg.j2 @@ -0,0 +1,8 @@ +set timeout=1 + +menuentry '{{ iso_url | basename | splitext | first }} (PXE)' { + linux vmlinuz \ + ip=dhcp \ + ks=http://{{ ansible_default_ipv4.address }}/init-config/${net_default_mac}.ks + initrd initrd.img +} diff --git a/metal/roles/pxe-server/templates/http/ignition/ignition.yaml.j2 b/metal/roles/pxe-server/templates/http/ignition/ignition.yaml.j2 deleted file mode 100644 index 91d0ae67..00000000 --- a/metal/roles/pxe-server/templates/http/ignition/ignition.yaml.j2 +++ /dev/null @@ -1,79 +0,0 @@ -variant: fcos -version: 1.3.0 - -passwd: - users: - - name: root - ssh_authorized_keys: - - {{ ssh_public_key }} - -storage: - files: - # Set hostname - - path: /etc/hostname - mode: 0644 - contents: - inline: {{ hostvars[item]['inventory_hostname'] }} - # Set static IP - - path: /etc/NetworkManager/system-connections/{{ network_interface }}.nmconnection - mode: 0600 - contents: - inline: | - [connection] - id={{ network_interface }} - type=ethernet - interface-name={{ network_interface }} - permissions= - [ipv4] - address1={{ (hostvars[item]['ansible_host'] + '/' + ansible_default_ipv4.netmask) | ansible.netcommon.ipaddr('host/prefix') }},{{ ansible_default_ipv4.gateway }} - dns={{ dns_server }}; - dns-search= - method=manual - # Make audit logs less verbose - - path: /etc/sysctl.d/20-silence-audit.conf - contents: - inline: | - kernel.printk=4 - # Fix flannel support - - path: /etc/systemd/network/50-flannel.link - contents: - inline: | - [Match] - OriginalName=flannel* - [Link] - MACAddressPolicy=none - links: - # Set timezone - - path: /etc/localtime - target: /usr/share/zoneinfo/{{ timezone }} - -systemd: - units: - # iSCSI for Longhorn distributed block storage - - name: iscsid.service - enabled: true -{% if item == "metal0" %} - # Terraform state backend - - name: tfstate.service - enabled: true - contents: | - [Unit] - Description=Run etcd for Terraform state backend - After=network-online.target - Wants=network-online.target - - [Service] - ExecStartPre=-/usr/bin/docker kill tfstate - ExecStartPre=-/usr/bin/docker rm tfstate - ExecStart=/usr/bin/docker run --name tfstate \ - --volume tfstate:/bitnami/etcd/data \ - --env ALLOW_NONE_AUTHENTICATION=yes \ - --publish 23799:2379 \ - --restart always \ - bitnami/etcd - ExecStop=/usr/bin/docker stop tfstate - Restart=always - - [Install] - WantedBy=multi-user.target -{% endif %} diff --git a/metal/roles/pxe-server/templates/kickstart.ks.j2 b/metal/roles/pxe-server/templates/kickstart.ks.j2 new file mode 100644 index 00000000..4825c3ad --- /dev/null +++ b/metal/roles/pxe-server/templates/kickstart.ks.j2 @@ -0,0 +1,51 @@ +#version=RHEL8 + +# Do not use graphical install +text + +# Keyboard layouts +keyboard --xlayouts='us' +# System language +lang en_US.UTF-8 + +# Partition clearing information +clearpart --all --drives={{ disk }} +# Partitioning +ignoredisk --only-use={{ disk }} +autopart + +# Network information +network --bootproto=static --device={{ network_interface }} --ip={{ hostvars[item]['ansible_host'] }} --gateway={{ ansible_default_ipv4.gateway }} --nameserver={{ dns_server }} --netmask={{ ansible_default_ipv4.netmask }} --ipv6=auto --hostname={{ hostvars[item]['inventory_hostname'] }} --activate + +# Use network installation +repo --name="Minimal" --baseurl=http://{{ ansible_default_ipv4.address }}/os/Minimal +url --url="http://{{ ansible_default_ipv4.address }}/os" +# Disable Setup Agent on first boot +firstboot --disable +# Do not configure the X Window System +skipx +# Enable NTP +services --enabled="chronyd" +# System timezone +timezone {{ timezone }} --isUtc + +# Create user (locked by default) +user --groups=wheel --name=admin +# Add SSH key +sshkey --username=root "{{ ssh_public_key }}" + +# Disable SELinux +selinux --disabled + +# Disable firewall +firewall --disabled + +%packages +@^minimal-environment +iscsi-initiator-utils +%end + +# Enable iSCSI for Kubernetes storage +services --enable=iscsid + +reboot diff --git a/metal/roles/pxe-server/templates/tftp/tftpboot/grub.cfg.j2 b/metal/roles/pxe-server/templates/tftp/tftpboot/grub.cfg.j2 deleted file mode 100644 index 6a79aa9b..00000000 --- a/metal/roles/pxe-server/templates/tftp/tftpboot/grub.cfg.j2 +++ /dev/null @@ -1,11 +0,0 @@ -set timeout=1 - -menuentry '{{ os_name }} (Live)' { - linux vmlinuz \ - ip=dhcp \ - ignition.platform.id=metal \ - coreos.live.rootfs_url=http://{{ ansible_default_ipv4.address }}/{{ os_name }}/images/pxeboot/rootfs.img \ - coreos.inst.install_dev=/dev/{{ disk }} \ - coreos.inst.ignition_url=http://{{ ansible_default_ipv4.address }}/ignition/${net_default_mac}.json - initrd initrd.img ignition.img -} diff --git a/metal/roles/tfstate/defaults/main.yml b/metal/roles/tfstate/defaults/main.yml deleted file mode 100644 index a8f43f12..00000000 --- a/metal/roles/tfstate/defaults/main.yml +++ /dev/null @@ -1 +0,0 @@ -etcd_port: 23799 diff --git a/metal/roles/tfstate/tasks/main.yml b/metal/roles/tfstate/tasks/main.yml deleted file mode 100644 index 122e9149..00000000 --- a/metal/roles/tfstate/tasks/main.yml +++ /dev/null @@ -1,11 +0,0 @@ -- name: Wait for etcd - wait_for: - port: 23799 - host: '{{ ansible_ssh_host }}' - connection: local - -- name: Generate Terraform backend config - delegate_to: localhost - template: - src: backend.tfvars.j2 - dest: "{{ playbook_dir }}/../cluster/backend.tfvars" diff --git a/metal/roles/tfstate/templates/backend.tfvars.j2 b/metal/roles/tfstate/templates/backend.tfvars.j2 deleted file mode 100644 index 4a285b08..00000000 --- a/metal/roles/tfstate/templates/backend.tfvars.j2 +++ /dev/null @@ -1,5 +0,0 @@ -endpoints = [ -{% for host in ansible_play_hosts %} - "{{ hostvars[host].ansible_host }}:{{ etcd_port }}", -{% endfor %} -] diff --git a/metal/roles/wake/tasks/main.yml b/metal/roles/wake/tasks/main.yml index ac3098d2..ebd8e736 100644 --- a/metal/roles/wake/tasks/main.yml +++ b/metal/roles/wake/tasks/main.yml @@ -4,9 +4,5 @@ delegate_to: localhost - name: Wait for the servers to comes up - wait_for: - host: '{{ ansible_host }}' - port: 22 - search_regex: OpenSSH + wait_for_connection: timeout: 600 - delegate_to: localhost