From cd1d472b74a89271009fe333a2ed44a6c7521c0a Mon Sep 17 00:00:00 2001
From: "Dustin C. Hatch" <dustin@hatch.name>
Date: Sat, 17 Aug 2024 11:05:25 -0500
Subject: [PATCH] scripts: Add VM host maintenance scripts

The `migrate-all.sh` script is used to migrate one or more VMs (default:
all) from one VM host to the other on demand.

The `shutdown-vmhost.sh` script prepares a VM host to shut down by
evicting Kubernetes Pods from the Nodes running on that host and then
shutting them down, followed by migrating the rest of the running VMs to
the other host.
---
 scripts/migrate-all.sh     | 28 ++++++++++++
 scripts/shutdown-vmhost.sh | 89 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 117 insertions(+)
 create mode 100755 scripts/migrate-all.sh
 create mode 100755 scripts/shutdown-vmhost.sh

diff --git a/scripts/migrate-all.sh b/scripts/migrate-all.sh
new file mode 100755
index 0000000..52b6848
--- /dev/null
+++ b/scripts/migrate-all.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+list_vms() {
+    if [ $# -gt 0 ]; then
+        echo "$@" | xargs -n1
+    else
+        virsh list --name
+    fi
+}
+
+src=$1
+shift 2>&-
+dest=$1
+shift 2>&-
+
+if [ -z "${src}" ] || [ -z "${dest}" ]; then
+    printf 'usage: %s source_uri dest_uri [name [...]]\n' "${0##*/}"
+    exit 2
+fi
+
+export LIBVIRT_DEFAULT_URI="${src}"
+
+list_vms "$@" | while read name; do
+    [ -n "${name}" ] || continue
+    printf 'Migrating %s ...\n' "${name}"
+    virsh migrate "${name}" "${dest}" --live --persistent --undefinesource --verbose
+done
diff --git a/scripts/shutdown-vmhost.sh b/scripts/shutdown-vmhost.sh
new file mode 100755
index 0000000..62fc70e
--- /dev/null
+++ b/scripts/shutdown-vmhost.sh
@@ -0,0 +1,89 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+cordon() {
+    printf 'Cordoning node %s ... ' "$1" >&2
+    if kubectl cordon "$1"; then
+        :
+    else
+        r=$?
+        echo FAILED >&2
+        return $r
+    fi
+}
+
+drain() {
+    printf 'Draining node %s ... ' "$1"
+    kubectl drain \
+        --ignore-daemonsets \
+        --pod-selector 'app!=csi-attacher,app!=csi-provisioner' \
+        --delete-emptydir-data \
+        "$1"
+}
+
+migrate() {
+    printf 'Migrating %s to %s ... \n' "$1" "$2" >&2
+    virsh migrate "$1" "$2" --live --persistent --undefinesource --verbose
+}
+
+shutdown() {
+    if [ "$(virsh domstate "$1")" = 'shut off' ]; then
+        printf 'VM %s is already shut off\n' "$1" >&2
+        return 0
+    fi
+    virsh shutdown "$1" || return
+    printf 'Waiting for %s to shut down ... ' "$1" >&2
+    if virsh event --event lifecycle "$1"; then
+        echo OK >&2
+    else
+        r=$?
+        echo FAILED >&2
+        return $r
+    fi
+}
+
+host="$1"
+export LIBVIRT_DEFAULT_URI=qemu+ssh://${host}/system
+
+case "${host}" in
+vmhost0.pyrocufflink.blue)
+    dest=qemu+ssh://vmhost1.pyrocufflink.blue/system
+    ;;
+vmhost1.pyrocufflink.blue)
+    dest=qemu+ssh://vmhost0.pyrocufflink.blue/system
+    ;;
+*)
+    printf 'Unknown VM host: %s\n' "${host}" >&2
+    exit 1
+    ;;
+esac
+
+printf 'Checking libvirt connection ... '
+virsh uri || exit
+
+set --
+nodes=$(
+    kubectl get node -o name -l '!node-role.kubernetes.io/control-plane' \
+    | sed s@node/@@
+)
+for node in ${nodes}; do
+    if virsh domuuid "${node%%.*}" >/dev/null 2>&1; then
+        set -- "$@" "${node}"
+        cordon "${node}" || exit
+    fi
+done
+
+for node; do
+    drain "${node}" || exit
+done
+
+for node; do
+    shutdown "${node%%.*}" || exit
+done
+
+virsh list --name | while read -r vm_name; do
+    [ -n "${vm_name}" ] || continue
+    if [ "$(virsh domstate "${vm_name}")" = running ]; then
+        migrate "${vm_name}" "${dest}" || exit
+    fi
+done