From ce077ad5578854719957eafcc3aacc4253773111 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sun, 31 Jul 2022 00:14:09 -0500 Subject: [PATCH] setup: Use separate kicstarts for control/worker We're going to be using Longhorn for persistent storage. Longhorn allocates space on worker nodes and exposes iSCSI LUNs to other worker nodes. It creates sparse filesystem images under `/var/lib/longhorn` for each volume. Thus, we need to mount a large filesystem at that path on each worker node for Longhorn to use. Using two different kickstart scripts, one for the control plane nodes, and one for the worker nodes, we can properly mount the Longhorn data directory only on machines that will be running the Longhorn manager. Longhorn only supports *ext4* and *XFS* filesystem types. --- setup/README.md | 6 +- setup/{fedora-k8s.ks => fedora-k8s-ctrl.ks} | 0 setup/fedora-k8s-node.ks | 156 ++++++++++++++++++++ 3 files changed, 160 insertions(+), 2 deletions(-) rename setup/{fedora-k8s.ks => fedora-k8s-ctrl.ks} (100%) create mode 100644 setup/fedora-k8s-node.ks diff --git a/setup/README.md b/setup/README.md index 095723e..2841e9f 100644 --- a/setup/README.md +++ b/setup/README.md @@ -6,9 +6,11 @@ ## Installation -Use the [`fedora-k8s.ks`][0] kickstart file +For control plane nodes, use the [`fedora-k8s-ctrl.ks`][0] kickstart file. For +worker nodes, use [`fedora-k8s-node.ks`][1]. -[0]: fedora-k8s.ks +[0]: fedora-k8s-ctrl.ks +[0]: fedora-k8s-node.ks ## Machine Setup diff --git a/setup/fedora-k8s.ks b/setup/fedora-k8s-ctrl.ks similarity index 100% rename from setup/fedora-k8s.ks rename to setup/fedora-k8s-ctrl.ks diff --git a/setup/fedora-k8s-node.ks b/setup/fedora-k8s-node.ks new file mode 100644 index 0000000..c917eb2 --- /dev/null +++ b/setup/fedora-k8s-node.ks @@ -0,0 +1,156 @@ +# vim: set ft=sh : +text +url --metalink https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch +repo --name=updates --metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f$releasever&arch=$basearch +repo --name=fedora-modular --metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-modular-$releasever&arch=$basearch +repo --name=updates-modular --metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-modular-f$releasever&arch=$basearch +lang en_US.UTF-8 +keyboard us +timezone --utc UTC +rootpw --lock +reboot + +ignoredisk --only-use vda +bootloader --location mbr +clearpart --all --initlabel +reqpart +part /boot --fstype ext4 --size=1024 +part btrfs.0 --fstype btrfs --size 4096 +part btrfs.1 --fstype btrfs --grow +btrfs none --label fedora btrfs.0 +btrfs none --label data btrfs.1 +btrfs / --subvol --name root LABEL=fedora +btrfs /home --subvol --name home LABEL=data +btrfs /var --subvol --name var LABEL=data +btrfs /etc/cni/net.d --subvol --name cni-net LABEL=data +btrfs /usr/libexec/kubernetes/kubelet-plugins --subvol --name kubelet-plugins LABEL=data +btrfs /opt --subvol --name opt LABEL=data + +%pre +echo '%packages' > /tmp/packages.ks +sys_vendor=$(tr A-Z a-z < /sys/devices/virtual/dmi/id/sys_vendor) +case "${sys_vendor}" in +kvm|bochs|qemu) + install_qga=1 + ;; +esac +if [ ${install_qga:-0} -eq 1 ]; then + echo 'qemu-guest-agent' >> /tmp/packages.ks +fi +echo '%end' >> /tmp/packages.ks +%end +%include /tmp/packages.ks + +module --name cri-o --stream 1.22 + +%packages --exclude-weakdeps +-NetworkManager +-authconfig +-dhcp-client +-dnf-plugins-core +-dnf-yum +-dracut-config-rescue +-e2fsprogs +-firewalld +-man-db +-openssh-clients +-parted +-plymouth +-sssd-common +-sssd-kcm +-sudo +-yum +-zram-generator +-zram-generator-defaults +chrony +cri-o +cri-tools +dnf +dnf-command(system-upgrade) +e2fsprogs +ethtool +grubby +iproute-tc +iptables-nft +iscsi-initiator-utils +kitty-terminfo +kubernetes-client +kubernetes-kubeadm +kubernetes-node +openssh-server +rng-tools +selinux-policy-targeted +systemd-networkd +%end + +services --enabled crio,iscsid,kubelet,systemd-networkd,systemd-resolved + +%addon com_redhat_kdump --disable +%end + +%post --erroronfail +echo 'install_weak_deps=0' >> /etc/dnf/dnf.conf +echo 'deltarpm=0' >> /etc/dnf/dnf.conf +echo '%_excludedocs 1' >> /etc/rpm/macros + +systemctl mask systemd-journald-audit.socket + +sed -i \ + -e 's:.*AuthorizedKeysCommand .*:AuthorizedKeysCommand /usr/local/libexec/ssh-authorized-keys %u %t:' \ + -e 's:.*AuthorizedKeysCommandUser .*:AuthorizedKeysCommandUser nobody:' \ + /etc/ssh/sshd_config +cat > /usr/local/libexec/ssh-authorized-keys <<"EOF" +#!/bin/sh + +USER="${1}" +KEYTYPE="${2}" + +curl -gs http://sshkeys.pyrocufflink.blue/"${USER}"/"${KEYTYPE}".pub +EOF +chmod +x /usr/local/libexec/ssh-authorized-keys +chcon -t bin_t /usr/local/libexec/ssh-authorized-keys +setsebool -NP authlogin_yubikey on + +rm -rf /etc/sysconfig/network-scripts /etc/sysconfig/network + +cat > /etc/systemd/network/99-default.network < /etc/modules-load.d/k8s.conf <<'EOF' +br_netfilter +EOF + +cat > /etc/sysctl.d/k8s.conf <<'EOF' +# Required for Kubernetes +net.bridge.bridge-nf-call-iptables = 1 +net.bridge.bridge-nf-call-ip6tables = 1 +net.ipv4.ip_forward = 1 +EOF + +sed -i 's/^driver = .*/driver = "btrfs"/' /etc/containers/storage.conf + +# Anaconda always creates a partition on the disk and formats that, instead of +# just formatting the whole disk. This makes it difficult to extend the disk +# later. The only way to use the whole disk is to format it ourselves. +mkfs.ext4 -L longhorn /dev/vdb +mkdir -p /var/lib/longhorn +mount LABEL=longhorn /var/lib/longhorn +chcon -t container_var_lib_t /var/lib/longhorn +echo 'LABEL=longhorn /var/lib/longhorn ext4 defaults 0 0' >> /etc/fstab + +# Enable read-only rootfs. This cannot be done with part/logvol, as that would +# make Anaconda mount it read-only befor the installation starts. +sed -i -r '/\S+\s+\/\s+/s/subvol=root/ro,&/' /etc/fstab +%end