From 77ce7aa5e79980d2ae6f9be7d321156870edc53a Mon Sep 17 00:00:00 2001
From: "Dustin C. Hatch" <dustin@hatch.name>
Date: Sun, 1 Sep 2024 07:45:59 -0500
Subject: [PATCH] r/minio-backups-cert: Certbot for MinIO+nginx

The MinIO server for backups has special requirements for HTTPS.  I want
to use subdomains for bucket names, so the certificate must have a
wildcard name, which requires using the DNS-01 challenge.  Fortunately,
it is actually pretty easy to use `nsupdate` with GSS-TSIG
authentication to automate DNS record creation, and by default, all
domain-member machines can create any records.  Thus, using the `manual`
auth plugin for `certbot` and a script to run `nsupdate`, obtaining the
wildcard certificate is fairly straightforward.

The biggest issue I encountered while developing this feature was
caching of NXDOMAIN responses.  There doesn't seem to be a way to change
the TTL of the SOA record of the Active Directory DNS domain, which
defaults to 3600, meaning NXDOMAIN responses are always cached for an
hour.  When adding a record using `nsupdate -g`, the tool always
performs a SOA lookup of new name to find the target zone for it.  Since
the name does not exist yet, the domain controller responds with
NXDOMAIN, which gets cached by the main DNS server.  Thus, even after
adding the record, the ACME server will not be able to resolve the
name for up to an hour.  We can a void this by explicitly setting the
target zone.  That would not work in a multi-domain forest, but
fortunately, we do not have to worry about that.

This role borrows some logic from the *postgresql-cert* role.
Eventually, I probably want to combine some of the steps from both of
these roles, possibly replacing the old *certbot* role.
---
 roles/minio-backups-cert/files/deploy-hook.sh |  3 +
 .../minio-backups-cert/files/nsupdate-auth.sh | 19 +++++
 .../files/nsupdate-cleanup.sh                 | 11 +++
 roles/minio-backups-cert/handlers/main.yml    |  8 ++
 roles/minio-backups-cert/tasks/main.yml       | 80 +++++++++++++++++++
 .../templates/certbot-renew.timer.j2          |  3 +
 roles/minio-backups-cert/vars/main.yml        |  2 +
 7 files changed, 126 insertions(+)
 create mode 100644 roles/minio-backups-cert/files/deploy-hook.sh
 create mode 100755 roles/minio-backups-cert/files/nsupdate-auth.sh
 create mode 100755 roles/minio-backups-cert/files/nsupdate-cleanup.sh
 create mode 100644 roles/minio-backups-cert/handlers/main.yml
 create mode 100644 roles/minio-backups-cert/tasks/main.yml
 create mode 100644 roles/minio-backups-cert/templates/certbot-renew.timer.j2
 create mode 100644 roles/minio-backups-cert/vars/main.yml

diff --git a/roles/minio-backups-cert/files/deploy-hook.sh b/roles/minio-backups-cert/files/deploy-hook.sh
new file mode 100644
index 0000000..0964002
--- /dev/null
+++ b/roles/minio-backups-cert/files/deploy-hook.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+systemctl reload nginx
diff --git a/roles/minio-backups-cert/files/nsupdate-auth.sh b/roles/minio-backups-cert/files/nsupdate-auth.sh
new file mode 100755
index 0000000..720bfa2
--- /dev/null
+++ b/roles/minio-backups-cert/files/nsupdate-auth.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+export KRB5CCNAME=/run/certbot.krb5_ccache
+klist -s || net ads kerberos kinit -P || exit
+nsupdate -g <<EOF || exit
+zone $(dnsdomainname)
+update add _acme-challenge.${CERTBOT_DOMAIN} 10 TXT ${CERTBOT_VALIDATION}
+send
+EOF
+
+while :; do
+    t=$(dig +short -t txt _acme-challenge.${CERTBOT_DOMAIN})
+    case "$t" in
+    *\"${CERTBOT_VALIDATION}\"*)
+        break
+        ;;
+    esac
+    sleep 1
+done
diff --git a/roles/minio-backups-cert/files/nsupdate-cleanup.sh b/roles/minio-backups-cert/files/nsupdate-cleanup.sh
new file mode 100755
index 0000000..be88f1a
--- /dev/null
+++ b/roles/minio-backups-cert/files/nsupdate-cleanup.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+export KRB5CCNAME=/run/certbot.krb5_ccache
+klist -s || net ads kerberos kinit -P || exit
+
+nsupdate -g <<EOF
+update del _acme-challenge.${CERTBOT_DOMAIN} TXT
+send
+EOF
+
+kdestroy
diff --git a/roles/minio-backups-cert/handlers/main.yml b/roles/minio-backups-cert/handlers/main.yml
new file mode 100644
index 0000000..0cd6ed8
--- /dev/null
+++ b/roles/minio-backups-cert/handlers/main.yml
@@ -0,0 +1,8 @@
+- name: reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: restart certbot-renew timer
+  systemd:
+    name: certbot-renew.timer
+    state: restarted
diff --git a/roles/minio-backups-cert/tasks/main.yml b/roles/minio-backups-cert/tasks/main.yml
new file mode 100644
index 0000000..1c6a69c
--- /dev/null
+++ b/roles/minio-backups-cert/tasks/main.yml
@@ -0,0 +1,80 @@
+- name: ensure packages are installed
+  package:
+    name:
+    - bind-utils
+    - certbot
+  tags:
+  - install
+
+- name: ensure certbot nsupdate scripts are installed
+  copy:
+    src: '{{ item }}'
+    dest: /etc/letsencrypt/{{ item }}
+    owner: root
+    group: root
+    mode: u=rwx,go=rx
+  loop:
+  - nsupdate-auth.sh
+  - nsupdate-cleanup.sh
+  tags:
+  - scripts
+
+- name: ensure minio server certificate exists
+  command:
+    certbot certonly -n
+    --manual
+    --manual-auth-hook /etc/letsencrypt/nsupdate-auth.sh
+    --manual-cleanup-hook /etc/letsencrypt/nsupdate-cleanup.sh
+    --preferred-challenges dns
+    {% for domain in minio_cert_domains %}
+    -d {{ domain }}
+    {% endfor %}
+    --server {{ minio_cert_acme_server }}
+    --agree-tos
+    --email {{ minio_cert_acme_email }}
+  args:
+    creates: /etc/letsencrypt/live/{{ minio_cert_main_domain }}/fullchain.pem
+  tags:
+  - cert
+
+- name: ensure certbot deploy renewal hook script is installed
+  copy:
+    src: deploy-hook.sh
+    dest: /etc/letsencrypt/renewal-hooks/deploy/nginx.sh
+    owner: root
+    group: root
+    mode: u=rwx,go=rx
+  tags:
+  - deploy-hook
+
+- name: ensure certbot renewal period is configured for minio server cert
+  lineinfile:
+    line: renew_before_expiry = 8 hours
+    regexp: '^#?\s*renew_before_expiry\s*='
+    path: /etc/letsencrypt/renewal/{{ minio_cert_main_domain }}.conf
+    state: present
+  tags:
+  - config
+
+- name: ensure certbot-renew timer unit drop-in directory exists
+  file:
+    path: /etc/systemd/system/certbot-renew.timer.d
+    owner: root
+    group: root
+    mode: u=rwx,go=rx
+    state: directory
+  tags:
+  - systemd
+- name: ensure certbot-renew timer schedule is configured
+  template:
+    src: certbot-renew.timer.j2
+    dest: /etc/systemd/system/certbot-renew.timer.d/schedule.conf
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload systemd
+  - restart certbot-renew timer
+  tags:
+  - systemd
+
diff --git a/roles/minio-backups-cert/templates/certbot-renew.timer.j2 b/roles/minio-backups-cert/templates/certbot-renew.timer.j2
new file mode 100644
index 0000000..11ea320
--- /dev/null
+++ b/roles/minio-backups-cert/templates/certbot-renew.timer.j2
@@ -0,0 +1,3 @@
+[Timer]
+RandomizedDelaySec=15m
+OnCalendar=hourly
diff --git a/roles/minio-backups-cert/vars/main.yml b/roles/minio-backups-cert/vars/main.yml
new file mode 100644
index 0000000..c4dc3d8
--- /dev/null
+++ b/roles/minio-backups-cert/vars/main.yml
@@ -0,0 +1,2 @@
+minio_cert_main_domain: >-
+  {{ minio_cert_domains[0].replace('*.', '') }}