diff --git a/deploy.sh b/deploy.sh index 12a256b..e1315fe 100644 --- a/deploy.sh +++ b/deploy.sh @@ -12,5 +12,8 @@ set +a virsh uri || exit rbw unlock || exit sshca-cli user login || exit +kubectl get node > /dev/null || exit -. deploy/"${1}".sh +script=deploy/"${1}".sh +shift +. "${script}" diff --git a/deploy/k8s-longhorn.sh b/deploy/k8s-longhorn.sh new file mode 100644 index 0000000..d51ddce --- /dev/null +++ b/deploy/k8s-longhorn.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# vim: set ts=4 sw=4 noet : + +name=${1:-stor-$(diceware -n1 --no-caps)} +hostname=${name}.k8s.pyrocufflink.black + +if ! virsh list --all --name | grep -qF ${name}; then + ./newvm.sh ${name} \ + --fedora 40 \ + --memory 4096 \ + --vcpus 4 \ + --no-console \ + --network network=kube \ + -- \ + --disk pool=default,size=4,cache=none \ + --disk pool=default,size=8,cache=none \ + --disk pool=default,size=512,cache=none \ + || exit + sleep 15 +fi + +if ! grep -q "${hostname}" hosts; then + sed -i '/\[k8s-longhorn\]/a'"${hostname}" hosts +fi + +ansible-playbook \ + -l ${hostname} \ + wait-for-host.yml \ + || exit + +printf 'Waiting for SSH host certificate to be signed ... ' +until ssh-keyscan -c ${hostname} 2>/dev/null | grep -q cert; do + sleep 1 +done +echo done + +ansible-playbook \ + -l ${hostname} \ + bootstrap.yml \ + datavol.yml \ + kubernetes.yml \ + collectd.yml \ + btop.yml \ + -u root \ + || exit diff --git a/deploy/k8s-node.sh b/deploy/k8s-node.sh new file mode 100644 index 0000000..a613e7c --- /dev/null +++ b/deploy/k8s-node.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# vim: set ts=4 sw=4 noet : + +name=${1:-node-$(diceware -n1 --no-caps)} +hostname=${name}.k8s.pyrocufflink.black + +if ! virsh list --all --name | grep -qF ${name}; then + ./newvm.sh ${name} \ + --domain k8s.pyrocufflink.black \ + --fedora 40 \ + --memory 16384 \ + --vcpus 8 \ + --no-console \ + --network network=kube \ + -- \ + --network network=storage \ + --disk pool=default,size=32,cache=none \ + --disk pool=default,size=32,cache=none \ + || exit + sleep 15 +fi + +if ! grep -q "${hostname}" hosts; then + sed -i '/\[k8s-node\]/a'"${hostname}" hosts +fi + +ansible-playbook \ + -l ${hostname} \ + wait-for-host.yml \ + || exit + +printf 'Waiting for SSH host certificate to be signed ... ' +until ssh-keyscan -c ${hostname} 2>/dev/null | grep -q cert; do + sleep 1 || exit +done +echo done + +ansible-playbook \ + -l ${hostname} \ + bootstrap.yml \ + datavol.yml \ + users.yml \ + kubernetes.yml \ + collectd.yml \ + btop.yml \ + -u root \ + || exit diff --git a/group_vars/k8s-longhorn.yml b/group_vars/k8s-longhorn.yml new file mode 100644 index 0000000..caf7b33 --- /dev/null +++ b/group_vars/k8s-longhorn.yml @@ -0,0 +1,18 @@ +data_volumes: +- dev: /dev/vdb + fstype: ext4 + mountpoint: /var/lib/containers +- dev: /dev/vdc + fstype: ext4 + mountpoint: /var/lib/kubelet +- dev: /dev/vdd + fstype: ext4 + mountpoint: /var/lib/longhorn + +k8s_node_labels: + node-role.kubernetes.io/longhorn: '' + node.longhorn.io/create-default-disk: 'true' + +k8s_node_taints: +- key: node-role.kubernetes.io/longhorn + effect: NoSchedule diff --git a/group_vars/k8s-node.yml b/group_vars/k8s-node.yml index a3d2fc4..f50cfbb 100644 --- a/group_vars/k8s-node.yml +++ b/group_vars/k8s-node.yml @@ -1 +1,12 @@ collectd_processes: '{{ collectd_processes_kubelet }}' + +data_volumes: +- dev: /dev/vdb + fstype: ext4 + mountpoint: /var/lib/containers +- dev: /dev/vdc + fstype: ext4 + mountpoint: /var/lib/kubelet + +k8s_node_labels: + network.du5t1n.me/storage: 'true' diff --git a/hosts b/hosts index 3308c6d..8ffdef5 100644 --- a/hosts +++ b/hosts @@ -11,6 +11,9 @@ bw0.pyrocufflink.blue [btop] chromie.pyrocufflink.blue +[btop:children] +kubelet + [burp-client] bw0.pyrocufflink.blue cloud0.pyrocufflink.blue @@ -24,6 +27,7 @@ git0.pyrocufflink.blue [collectd] [collectd:children] +kubelet pyrocufflink [collectd-prometheus:children] @@ -84,8 +88,13 @@ burp-server [k8s-controller] k8s-ctrl0.pyrocufflink.blue +[k8s-longhorn] + [k8s-node] +[k8s-node:children] +k8s-longhorn + [kubelet:children] k8s-controller k8s-node @@ -105,6 +114,9 @@ minio-backups [nfs-client:children] k8s-node +[no-firewalld:children] +kubelet + [needproxy] nvr2.pyrocufflink.blue diff --git a/kubernetes.yml b/kubernetes.yml new file mode 100644 index 0000000..fee0002 --- /dev/null +++ b/kubernetes.yml @@ -0,0 +1,19 @@ +- hosts: k8s-node + roles: + - role: k8s-worker + tags: + - k8s-worker + tasks: + - name: ensure kubernetes node labels and taints are set + delegate_to: localhost + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Node + metadata: + name: '{{ inventory_hostname }}' + labels: '{{ k8s_node_labels }}' + spec: + taints: '{{ k8s_node_taints | d(omit) }}' + tags: + - node-labels diff --git a/roles/cri-o/files/crun.conf b/roles/cri-o/files/crun.conf new file mode 100644 index 0000000..bf873a9 --- /dev/null +++ b/roles/cri-o/files/crun.conf @@ -0,0 +1,7 @@ +[crio.runtime] +default_runtime = "crun" + +[crio.runtime.runtimes.crun] +runtime_path = "/usr/bin/crun" +runtime_type = "oci" +runtime_root = "/run/crun" diff --git a/roles/cri-o/handlers/main.yml b/roles/cri-o/handlers/main.yml new file mode 100644 index 0000000..0048c99 --- /dev/null +++ b/roles/cri-o/handlers/main.yml @@ -0,0 +1,4 @@ +- name: restart crio + service: + name: crio + state: restarted diff --git a/roles/cri-o/tasks/main.yml b/roles/cri-o/tasks/main.yml new file mode 100644 index 0000000..6d54902 --- /dev/null +++ b/roles/cri-o/tasks/main.yml @@ -0,0 +1,47 @@ +- name: ensure cri-o is installed + package: + name: + - container-selinux + - cri-o + - crun + state: present + tags: + - install + +- name: ensure crio configuration drop-in directory exists + file: + path: /etc/crio/crio.conf.d + owner: root + group: root + mode: u=rwx,go=rx + state: directory + tags: + - config + +- name: ensure cri-o is configured to use crun + copy: + src: crun.conf + dest: /etc/crio/crio.conf.d/10-crio-crun.conf + owner: root + group: root + mode: u=rw,go=r + notify: + - restart crio + tags: + - config + +- name: flush handlers + meta: flush_handlers + +- name: ensure cri-o service is enabled + service: + name: crio + enabled: true + tags: + - service +- name: ensure cri-o service is running + service: + name: crio + state: started + tags: + - service diff --git a/roles/k8s-worker/meta/main.yml b/roles/k8s-worker/meta/main.yml new file mode 100644 index 0000000..bf4c433 --- /dev/null +++ b/roles/k8s-worker/meta/main.yml @@ -0,0 +1,4 @@ +dependencies: +- role: kubelet + tags: + - kubelet diff --git a/roles/k8s-worker/tasks/main.yml b/roles/k8s-worker/tasks/main.yml new file mode 100644 index 0000000..9a1a075 --- /dev/null +++ b/roles/k8s-worker/tasks/main.yml @@ -0,0 +1,30 @@ +- name: stat /var/lib/kubelet/config.yaml + stat: + path: /var/lib/kubelet/config.yaml + ignore_errors: true + register: stat_kublet_config + tags: + - kubeadm-join + +- name: generate bootstrap token + delegate_to: '{{ groups["k8s-controller"][0] }}' + command: + kubeadm token create + --kubeconfig /etc/kubernetes/admin.conf + --ttl 1h + --print-join-command + when: + not stat_kublet_config.stat.exists + changed_when: true + register: kubeadm_token_create + tags: + - bootstrap-token + - kubeadm-join +- name: join the kubernetes cluster + command: >- + {{ kubeadm_token_create.stdout }} + when: + not stat_kublet_config.stat.exists + changed_when: true + tags: + - kubeadm-join diff --git a/roles/kubelet/files/sysctl.conf b/roles/kubelet/files/sysctl.conf new file mode 100644 index 0000000..4b884f8 --- /dev/null +++ b/roles/kubelet/files/sysctl.conf @@ -0,0 +1,4 @@ +# Required for Kubernetes +net.bridge.bridge-nf-call-iptables = 1 +net.bridge.bridge-nf-call-ip6tables = 1 +net.ipv4.ip_forward = 1 diff --git a/roles/kubelet/handlers/main.yml b/roles/kubelet/handlers/main.yml new file mode 100644 index 0000000..93535f4 --- /dev/null +++ b/roles/kubelet/handlers/main.yml @@ -0,0 +1,11 @@ +- name: load kernel modules + command: + /usr/lib/systemd/systemd-modules-load + +- name: set kernel tunables + command: + sysctl --system + +- name: swapoff -a + command: + swapoff -a diff --git a/roles/kubelet/meta/main.yml b/roles/kubelet/meta/main.yml new file mode 100644 index 0000000..4595a0b --- /dev/null +++ b/roles/kubelet/meta/main.yml @@ -0,0 +1,5 @@ +dependencies: +- systemd-base +- role: cri-o + tags: + - cri-o diff --git a/roles/kubelet/tasks/main.yml b/roles/kubelet/tasks/main.yml new file mode 100644 index 0000000..1722819 --- /dev/null +++ b/roles/kubelet/tasks/main.yml @@ -0,0 +1,92 @@ +- name: load os-specific values + include_vars: '{{ item }}' + with_first_found: + - '{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml' + - '{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml' + - '{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml' + - '{{ ansible_distribution }}.yml' + - '{{ ansible_os_family }}.yml' + - defaults.yml + tags: + - always + +- name: ensure required packages are installed + dnf: + name: '{{ kubernetes_packages }}' + install_weak_deps: false + state: present + tags: + - install + +- name: ensure firewalld service is stopped + service: + name: firewalld + state: stopped + enabled: false + ignore_errors: true + tags: + - firewalld + +- name: ensure kernel modules-load is configured for kubernetes + copy: + content: |+ + {{ kubernetes_kernel_modules | join('\n') }} + dest: /etc/modules-load.d/k8s.conf + owner: root + group: root + mode: u=rw,go=r + notify: + - load kernel modules + tags: + - kmod + +- name: ensure kernel tunables are set for kubernetes + copy: + src: sysctl.conf + dest: /etc/sysctl.d/60-k8s.conf + owner: root + group: root + mode: u=rw,go=r + notify: + - set kernel tunables + tags: + - sysctl + +- name: ensure zram generator defaults are disabled + copy: + content: '' + dest: /etc/systemd/zram-generator.conf + owner: root + group: root + mode: u=rw,go=r + notify: + - reload systemd + tags: + - zram-generator + +- name: ensure zram0 is stopped + systemd: + name: systemd-zram-setup@zram0 + state: stopped + ignore_errors: true + notify: + - swapoff -a + tags: + - zram-generator + +- name: ensure unneeded cni configuration files are removed + file: + path: /etc/cni/net.d/{{ item }} + state: absent + loop: + - 100-crio-bridge.conflist + - 200-loopback.conflist + tags: + - cni + +- name: ensure kubelet service is enabled + service: + name: kubelet + enabled: true + tags: + - service diff --git a/roles/kubelet/vars/Fedora-40.yml b/roles/kubelet/vars/Fedora-40.yml new file mode 100644 index 0000000..f89e0ca --- /dev/null +++ b/roles/kubelet/vars/Fedora-40.yml @@ -0,0 +1,6 @@ +kubernetes_distro_packages: +- kubernetes +- kubernetes-client +- kubernetes-kubeadm + +kubernetes_packages: '{{ kubernetes_common_packages + kubernetes_distro_packages }}' diff --git a/roles/kubelet/vars/main.yml b/roles/kubelet/vars/main.yml new file mode 100644 index 0000000..6dd7fe5 --- /dev/null +++ b/roles/kubelet/vars/main.yml @@ -0,0 +1,11 @@ +kubernetes_kernel_modules: +- br_netfilter + +kubernetes_common_packages: +- ebtables +- ethtool +- iproute-tc +# Required for Longhorn RWO volumes +- iscsi-initiator-utils +# Required for Longhorn RWX volumes +- nfs-utils