Revert "collectd: Run collectd in privileged container"

Unfortunately, running *collectd* in a container is not going to work.
Although containers can be configured to share some of the host's
namespaces, one notable exception is the mount namespace.  Naturally,
containers must have their own mount namespace, which prevents them from
seeing filesystems that are actually mounted on the host.  For
*collectd*, this effectively makes the `df` plugin useless, which
ultimately prevents us from monitoring disk space.

This reverts commit 4048e5cc0a.

.gitattributes

@@ -0,0 +1 @@
+*.gpg diff=gpg

.gitignore

@@ -1 +1,4 @@
 *.ign
+frigate.env
+*.token
+RPi4boot

65-apex.rules

@@ -0,0 +1 @@
+SUBSYSTEM=="apex", MODE="0666", TAG+="systemd"

Makefile

@@ -0,0 +1,28 @@
+.PHONY: \
+	all \
+	clean \
+	publish
+
+.DEFAULT_GOAL := all
+
+clean:
+	rm -f *.ign
+
+define genrules
+$(patsubst %.yaml,%.ign,$(1)): $(1) $$(shell sed -rn 's/.*local: (.*)/\1/p' $(1))
+	butane -d . $$< > $$@
+
+all: $(patsubst %.yaml,%.ign,$(1))
+endef
+
+$(foreach t,$(wildcard *.yaml),$(eval $(call genrules,$(t))))
+
+%.env: %.env.gpg
+	gpg2 --decrypt $< > $@
+
+%.token: %.token.gpg
+	gpg2 --decrypt $< > $@
+
+publish: \
+		nvr1.ign
+	rsync -rti $^ files.pyrocufflink.blue:public_html/

butane-watch.sh

@@ -2,8 +2,9 @@
 # vim: set sw=4 ts=4 sts=4 et :
 
 inotifywait -e CLOSE_WRITE -m . \
-    | stdbuf -o 0 grep -F .yaml \
+    | stdbuf -o 0 grep -v .ign \
     | while read _ _ f; do
-        printf 'Regenerating %s from %s ...\n' "${f%.yaml}.ign" "${f}"
+        [ -f "${f}" ] || continue
-        butane -d . ${f} > ${f%.yaml}.ign
+        printf '%s changed, running make ...\n' "${f}"
+        make && make publish
     done

collectd.yaml

@@ -0,0 +1,57 @@
+variant: fcos
+version: 1.4.0
+
+ignition:
+  config:
+    merge:
+    - local: packages.yaml
+
+storage:
+  files:
+  - path: /etc/ignition/packages.d/collectd
+    mode: 0644
+    contents:
+      inline: |
+        collectd
+        collectd-chrony
+        collectd-disk
+        collectd-sensors
+        collectd-write_prometheus
+  - path: /etc/collectd.d/df.conf
+    mode: 0644
+    contents:
+      inline: |
+        LoadPlugin df
+
+        <Plugin df>
+            FSType overlay
+            IgnoreSelected true
+        </Plugin>
+  - path: /etc/collectd.d/plugins.conf
+    mode: 0644
+    contents:
+      inline: |
+        LoadPlugin chrony
+        LoadPlugin cpufreq
+        LoadPlugin disk
+        LoadPlugin entropy
+        LoadPlugin processes
+        LoadPlugin swap
+        LoadPlugin tcpconns
+        LoadPlugin thermal
+        LoadPlugin uptime
+
+  - path: /etc/collectd.d/prometheus.conf
+    mode: 0644
+    contents:
+      inline: |
+        LoadPlugin write_prometheus
+        
+        <Plugin write_prometheus>
+            Port 9103
+        </Plugin>
+
+systemd:
+  units:
+  - name: collectd.service
+    enabled: true

common.yaml

@@ -0,0 +1,11 @@
+variant: fcos
+version: 1.4.0
+
+ignition:
+  config:
+    merge:
+    - local: sshkeys.ign
+    - local: collectd.ign
+    - local: local_exporter.ign
+    - local: notify-shutdown.ign
+    - local: step-ssh.ign

fetchcert.service

@@ -0,0 +1,36 @@
+[Unit]
+Description=Fetch HTTPS certificate from Kubernetes Secret API
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/bin/sh /etc/fetchcert/fetchcert.sh default pyrocufflink-cert
+ProtectSystem=strict
+ReadWritePaths=/etc/pki/nginx
+CapabilityBoundingSet=CAP_CHOWN
+DeviceAllow=
+DevicePolicy=closed
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+NoNewPrivileges=yes
+PrivateDevices=yes
+PrivateUsers=yes
+PrivateTmp=yes
+ProcSubset=pid
+ProtectClock=yes
+ProtectControlGroups=yes
+ProtectHome=yes
+ProtectHostname=yes
+ProtectKernelLogs=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+ProtectProc=invisible
+ProtectSystem=strict
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallFilter=~@privileged @resources

fetchcert.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+namespace=$2
+secret=$3
+
+keyout=/etc/pki/nginx/private/server.key
+crtout=/etc/pki/nginx/server.crt
+
+tmpdir=$(mktemp -d)
+trap 'rm -rf "${tmpdir}"' INT TERM QUIT EXIT
+
+cat > "${tmpdir}"/ca.crt <<EOF
+-----BEGIN CERTIFICATE-----
+MIIC/jCCAeagAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
+cm5ldGVzMB4XDTIyMDgwMTAyNTUzM1oXDTMyMDcyOTAyNTUzM1owFTETMBEGA1UE
+AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMs6
+2PUOzIClsAgPv1Mn9CTwzSFMntAn7OppwK5BQ4E5Vd1yMjz3p0uA1ZINv1ORorG0
+mLl95C7y+CWUGPx+stHKQr/40sLGyypbX+AfjoPzHiDbIcbZEff8X5RwKqzmT9V7
+Yt29KewADod0z+fqNYa62MJDaUunfwaV8kKFU/WJM8IKv2eJxAtWzvK3iHAFhx0j
+Xo4TlyINL9V9UMKLf12w6CA3G41uZIBCN3G7aJEm++eGoMdrPZUXlbCpbSztO85/
+hbulVs+0hCIxWiI+mRmB5OoWlRYL4jA45oK/RtpEqSwZ95zlGNAChmH7rb0pTtNf
+N0/C2wKAEL4POLx9kscCAwEAAaNZMFcwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
+/wQFMAMBAf8wHQYDVR0OBBYEFHYActCjEWdtsA+Ju25gxJh/vaLQMBUGA1UdEQQO
+MAyCCmt1YmVybmV0ZXMwDQYJKoZIhvcNAQELBQADggEBAAfkYHecXUwyqvMSXmqr
+ETqEzDCBini14s89VDhaDHOXBID9TKMVyeePdEYcPAJz3wo8fbx/+TL37K6hEuo+
+7bUaamaumznsjg9L0Hth19GvuRKMXJlEpndRmE5K9hnaDLr94MLg9n1qGcEOt6tw
+O6X5qqHf9AuuL39vt1+kSw6PeZZFZNMDZ8BdiTssw4btjQ2bsWu0wSiOSz/F8iRf
+2vN5An5dheroDsFs4dZ9gnJ69TmqV1YqQxfRWqCxzfNJbgVm6AoBPwhL1JRuAU4N
+3nCNoM9n2tLFDojT4un1778UVU91PtcBVdM9Nq+RC2jhXIyLBqsEK0ofOqFYqj3F
+0EQ=
+-----END CERTIFICATE-----
+EOF
+
+curl -fsSL \
+    -H 'Accept: application/json' \
+    -H "Authorization: Bearer $(cat /etc/fetchcert/token)" \
+    --cacert "${tmpdir}"/ca.crt \
+    https://kubernetes.pyrocufflink.blue:6443/api/v1/namespaces/${namespace}/secrets/${secret} \
+    -o "${tmpdir}"/secret.json \
+    || exit
+
+jq -r '.data["tls.key"]' "${tmpdir}"/secret.json \
+    | base64 -d > "${tmpdir}"/server.key
+jq -r '.data["tls.crt"]' "${tmpdir}"/secret.json | \
+    base64 -d > "${tmpdir}"/server.crt
+
+if [ "$(b2sum < "${tmpdir}"/server.crt)" != "$(b2sum < "${crtout}")" ]; then
+    install -m u=rw,go= -o 101 -g 101 "${tmpdir}"/server.key "${keyout}"
+    install -m u=rw,go=r -o root -g root "${tmpdir}"/server.crt "${crtout}"
+    chcon -t container_file_t "${keyout}" "${crtout}"
+    echo 'Certificate updated, reloading nginx ...' >&2
+    podman exec -it systemd-nginx nginx -s reload
+fi

fetchcert.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Periodically fetch certificate from Kubernetes
+
+[Timer]
+OnCalendar=*-*-* 0:0:0
+RandomizedDelaySec=8h
+
+[Install]
+WantedBy=timers.target

fetchcert.yaml

@@ -0,0 +1,22 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  files:
+  - path: /etc/fetchcert/fetchcert.sh
+    mode: 0755
+    contents:
+      local: fetchcert.sh
+  - path: /etc/systemd/system/fetchcert.service
+    mode: 0644
+    contents:
+      local: fetchcert.service
+  - path: /etc/systemd/system/fetchcert.timer
+    mode: 0644
+    contents:
+      local: fetchcert.timer
+
+systemd:
+  units:
+  - name: fetchcert.timer
+    enabled: true

flash.sh

@@ -1,35 +0,0 @@
-#!/bin/sh
-# vim: set sw=4 ts=4 sts=4 et :
-
-set -eu
-
-ignition=${1}
-dev=${2:-/dev/disk/by-id/usb-Generic_STORAGE_DEVICE_000000001206-0:0}
-
-if [ -z "${ignition}" ]; then
-    printf 'usage: %s ignition\n' "${0##*/}" >&2
-    exit 2
-fi
-
-if [ $(id -u) -ne 0 ]; then
-    sudo=sudo
-fi
-
-${sudo} coreos-installer install \
-    -a aarch64 \
-    -s stable \
-    -i "${ignition}" \
-    --console ttyS0,115200n8 \
-    "${dev}"
-sync; sync; sync
-
-until [ -n "${efi_part}" ]; do
-    efi_part=$(
-        lsblk -o LABEL,PATH \
-        | awk '$1=="EFI-SYSTEM"{print $2}'
-    )
-done
-pmount "${efi_part}" coreos-efi
-rsync -ah --ignore-existing RPi4boot/boot/efi/ /media/coreos-efi/
-sync; sync; sync
-pumount coreos-efi

flash.zsh

@@ -0,0 +1,139 @@
+#!/bin/zsh
+# vim: set ft=sh sw=4 ts=4 sts=4 et :
+
+set -eu
+
+function copy_firmware() {
+    local dev="$1"
+    local efi_part
+    local mountpoint=/run/media/coreos-efi
+
+    if [ ! -d RPi4boot ]; then
+        download_firmware
+    fi
+
+    printf 'Waiting for device to settle ...' >&2
+    until [ -n "${efi_part-}" ]; do
+        printf '.'
+        efi_part=$(
+            lsblk -o LABEL,PATH "${dev}" \
+            | awk '$1=="EFI-SYSTEM"{print $2}'
+        )
+        sleep 0.25
+    done
+    printf '\n'
+
+    mkdir -p "${mountpoint}"
+    mount "${efi_part}" "${mountpoint}"
+    rsync -rtiOh --ignore-existing RPi4boot/boot/efi/ "${mountpoint}"
+    sync; sync; sync
+    umount "${mountpoint}"
+    rmdir "${mountpoint}"
+}
+
+function download_firmware() {
+    local rpm
+
+    echo 'Downloading Raspberry Pi Firmware blobs ...'
+    mkdir RPi4boot
+    podman run --rm -it \
+        -v $(PWD)/RPi4boot:/RPi4boot:rw,z \
+        registry.fedoraproject.org/fedora:38 \
+        dnf install -y \
+            --downloadonly \
+            --forcearch=aarch64 \
+            --destdir=/RPi4boot \
+            uboot-images-armv8 \
+            bcm283x-firmware \
+            bcm283x-overlays
+            --
+    for rpm in RPi4boot/*.rpm; do
+        rpm2cpio "${rpm}" | cpio -idv -D RPi4boot/
+    done
+
+    cp RPi4boot/usr/share/uboot/rpi_3/u-boot.bin RPi4boot/boot/efi/rpi3-u-boot.bin
+    cp RPi4boot/usr/share/uboot/rpi_4/u-boot.bin RPi4boot/boot/efi/rpi4-u-boot.bin
+    cp RPi4boot/usr/share/uboot/rpi_arm64/u-boot.bin RPi4boot/boot/efi/rpi-u-boot.bin
+}
+
+function hybridize_gpt() {
+    local dev="$1"
+
+    echo 'Creating Hybrid MBR partition table ...'
+    sgdisk -h 2:EE "${dev}"
+    echo type=0c,bootable | sfdisk -Y mbr -N 1 "${dev}"
+}
+
+function install_coreos() {
+    local ignition="$1"
+    local dev="$2"
+
+    coreos-installer install \
+        -a aarch64 \
+        -s stable \
+        -i "${ignition}" \
+        --console ttyS0,115200n8 \
+        "${dev}"
+    sync; sync; sync
+}
+
+function parse_args() {
+    pi=4
+    while [ $# -gt 0 ]; do
+        case "$1" in
+        --pi)
+            shift
+            pi=${1}
+            ;;
+        --pi=*)
+            pi=${1#--pi=}
+            ;;
+        *)
+            if [ -z "${ignition-}" ]; then
+                ignition="${1}"
+            elif [ -z "${dev-}" ]; then
+                dev="${1}"
+            else
+                usage >&2
+                exit 2
+            fi
+            ;;
+        esac
+        shift
+    done
+
+    if [ -z "${ignition-}" ]; then
+        usage >&2
+        exit 2
+    fi
+    if [ -z "${dev-}" ]; then
+        dev=/dev/disk/by-id/usb-Generic_STORAGE_DEVICE_000000001206-0:0
+    fi
+}
+
+function usage() {
+    printf 'usage: %s [--pi PI] ignition [device]\n' "${0##*/}"
+}
+
+parse_args "$@"
+
+case "${pi}" in
+0*|1)
+    printf 'Raspberry Pi model %s is not supported\n' "${pi}" >&2
+    exit 1
+    ;;
+esac
+
+if [ $(id -u) -ne 0 ]; then
+    exec sudo "$0" "$@"
+fi
+
+install_coreos "${ignition}" "${dev}"
+
+case "${pi}" in
+2|3)
+    hybridize_gpt "${dev}"
+    ;;
+esac
+
+copy_firmware "${dev}"

frigate.container

@@ -0,0 +1,28 @@
+[Unit]
+Description=Frigate NVR
+Wants=network-online.target
+After=network-online.target
+Requires=dev-apex_0.device
+After=dev-apex_0.device
+
+[Container]
+Image=ghcr.io/blakeblackshear/frigate:0.12.1
+PodmanArgs=--uidmap 0:209:1
+PodmanArgs=--gidmap 0:209:1
+PodmanArgs=--uidmap 1:6000001:65536
+PodmanArgs=--gidmap 1:6000001:65536
+PodmanArgs=--shm-size 256m
+EnvironmentFile=/etc/sysconfig/frigate
+Volume=/var/lib/frigate/media:/media/frigate:rw,z
+Volume=/var/lib/frigate/tmp:/tmp:rw,z
+Volume=/var/lib/frigate/config:/config:rw,z
+AddDevice=/dev/apex_0
+AddDevice=/dev/dri/renderD128
+Network=host
+
+[Service]
+UMask=0077
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

frigate.nginx

@@ -0,0 +1,77 @@
+# vim: set sw=4 ts=4 sts=4 et :
+
+resolver 127.0.0.53;
+
+set $upstream_authelia https://auth.pyrocufflink.blue/api/verify;
+
+## Virtual endpoint created by nginx to forward auth requests.
+location /authelia {
+    ## Essential Proxy Configuration
+    internal;
+    proxy_pass $upstream_authelia;
+
+    ## Headers
+    ## The headers starting with X-* are required.
+    proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
+    proxy_set_header X-Original-Method $request_method;
+    proxy_set_header X-Forwarded-Method $request_method;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Host $http_host;
+    proxy_set_header X-Forwarded-Uri $request_uri;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header Content-Length "";
+    proxy_set_header Connection "";
+
+    ## Basic Proxy Configuration
+    proxy_pass_request_body off;
+    proxy_next_upstream error timeout invalid_header http_500 http_502 http_503; # Timeout if the real server is dead
+    proxy_redirect http:// $scheme://;
+    proxy_http_version 1.1;
+    proxy_cache_bypass $cookie_session;
+    proxy_no_cache $cookie_session;
+    proxy_buffers 4 32k;
+    client_body_buffer_size 128k;
+
+    ## Advanced Proxy Configuration
+    send_timeout 5m;
+    proxy_read_timeout 240;
+    proxy_send_timeout 240;
+    proxy_connect_timeout 240;
+}
+
+location / {
+    ## Send a subrequest to Authelia to verify if the user is authenticated and has permission to access the resource.
+    auth_request /authelia;
+
+    ## Set the $target_url variable based on the original request.
+
+    ## Comment this line if you're using nginx without the http_set_misc module.
+    set_escape_uri $target_url $scheme://$http_host$request_uri;
+
+    ## Uncomment this line if you're using NGINX without the http_set_misc module.
+    # set $target_url $scheme://$http_host$request_uri;
+
+    ## Save the upstream response headers from Authelia to variables.
+    auth_request_set $user $upstream_http_remote_user;
+    auth_request_set $groups $upstream_http_remote_groups;
+    auth_request_set $name $upstream_http_remote_name;
+    auth_request_set $email $upstream_http_remote_email;
+
+    ## Inject the response headers from the variables into the request made to the backend.
+    proxy_set_header Remote-User $user;
+    proxy_set_header Remote-Groups $groups;
+    proxy_set_header Remote-Name $name;
+    proxy_set_header Remote-Email $email;
+
+    ## If the subreqest returns 200 pass to the backend, if the subrequest returns 401 redirect to the portal.
+    error_page 401 =302 https://auth.pyrocufflink.blue/?rd=$target_url;
+
+    proxy_pass http://127.0.0.1:5000/;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $http_connection;
+    proxy_http_version 1.1;
+}

frigate.sysusers

@@ -0,0 +1,2 @@
+g frigate 209
+u frigate 209:209 "Frigate" /var/lib/frigate /sbin/nologin

frigate.tmpfiles

@@ -0,0 +1,4 @@
+d /var/lib/frigate 0755 frigate frigate
+d /var/lib/frigate/config 0755 frigate frigate
+d /var/lib/frigate/media 0755 frigate frigate
+d /var/lib/frigate/tmp 0755 frigate frigate

frigate.yaml

@@ -0,0 +1,34 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  files:
+  - path: /etc/containers/systemd/frigate.container
+    mode: 0644
+    contents:
+      local: frigate.container
+  - path: /etc/sysusers.d/frigate.conf
+    mode: 0644
+    contents:
+      local: frigate.sysusers
+  - path: /etc/tmpfiles.d/frigate.conf
+    mode: 0644
+    contents:
+      local: frigate.tmpfiles
+  - path: /etc/sysconfig/frigate
+    mode: 0640
+    user:
+      id: 0
+    group:
+      id: 209
+    contents:
+      local: frigate.env
+  - path: /etc/nginx/default.d/frigate.conf
+    mode: 0644
+    contents:
+      local: frigate.nginx
+
+systemd:
+  units:
+  - name: frigate.service
+    enabled: true

gasket-driver.container

@@ -0,0 +1,17 @@
+[Unit]
+Description=Install the gasket/apex kernel modules
+Wants=network-online.target
+After=network-online.target
+
+[Container]
+Image=git.pyrocufflink.net/containerimages/gasket-driver:%v
+PodmanArgs=--privileged
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStop=/usr/sbin/rmmod apex
+ExecStop=/usr/sbin/rmmod gasket
+
+[Install]
+WantedBy=multi-user.target

gasket-driver.yaml

@@ -0,0 +1,19 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  files:
+  - path: /etc/containers/systemd/gasket-driver.container
+    mode: 0644
+    contents:
+      local: gasket-driver.container
+  - path: /etc/udev/rules.d/65-apex.rules
+    mode: 0644
+    contents:
+      local: 65-apex.rules
+
+systemd:
+  units:
+  - name: gasket-driver.service
+    enabled: true
+

install-packages.service

@@ -0,0 +1,17 @@
+# vim: set ft=systemd :
+[Unit]
+Description=Install collectd
+After=network-online.target
+Wants=network-online.target
+Before=zincati.service
+ConditionPathExists=/etc/ignition/packages.d
+ConditionPathExists=/etc/ignition/packages.installed
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh /etc/ignition/install-packages.sh
+ExecStartPost=/bin/touch /etc/ignition/packages.installed
+
+[Install]
+WantedBy=multi-user.target

install-packages.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+if [ ! -d /etc/ignition/packages.d ]; then
+    exit 0
+fi
+
+cat /etc/ignition/packages.d/* | xargs rpm-ostree install --apply-live -y

k8s-aarch64-n0.yaml

@@ -3,8 +3,9 @@ version: 1.4.0
 ignition:
   config:
     merge:
-    - local: sshkeys.ign
+    - local: common.ign
     - local: kubelet.ign
+
 storage:
   files:
   - path: /etc/hostname

local_exporter.config

@@ -0,0 +1,9 @@
+# vim: set ft=toml :
+[service]
+address = "0.0.0.0"
+port = 9598
+tls = false
+
+[bridge.selectors.zincati]
+kind = "uds"
+path = "/run/zincati/public/metrics.promsock"

local_exporter.container

@@ -0,0 +1,23 @@
+[Unit]
+Description=Bridge for local Prometheus metrics
+After=network.target
+
+[Container]
+Image=git.pyrocufflink.net/containerimages/local_exporter:latest
+Exec=serve
+Volume=/run:/run:rw
+Volume=/etc/local_exporter:/etc/local_exporter:ro
+PublishPort=9598:9598
+# Must run as user "zincati"
+User=981
+ReadOnly=yes
+VolatileTmp=yes
+# container_t does not have permission to write to var_run_t
+SecurityLabelDisable=yes
+NoNewPrivileges=yes
+
+[Service]
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

local_exporter.yaml

@@ -0,0 +1,21 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  directories:
+  - path: /etc/local_exporter
+
+  files:
+  - path: /etc/containers/systemd/local_exporter.container
+    mode: 0644
+    contents:
+      local: local_exporter.container
+  - path: /etc/local_exporter/config.toml
+    mode: 0644
+    contents:
+      local: local_exporter.config
+
+systemd:
+  units:
+  - name: local_exporter.service
+    enabled: true

nginx.conf

@@ -0,0 +1,76 @@
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile            on;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    keepalive_timeout   65;
+    types_hash_max_size 2048;
+
+    include             /etc/nginx/mime.types;
+    default_type        application/octet-stream;
+
+    # Load modular configuration files from the /etc/nginx/conf.d directory.
+    # See http://nginx.org/en/docs/ngx_core_module.html#include
+    # for more information.
+    include /etc/nginx/conf.d/*.conf;
+
+    server {
+        listen       80 default_server;
+        listen       [::]:80 default_server;
+        server_name  _;
+        root         /usr/share/nginx/html;
+
+        return 301 https://$host$request_uri;
+
+        error_page 404 /404.html;
+            location = /40x.html {
+        }
+
+        error_page 500 502 503 504 /50x.html;
+            location = /50x.html {
+        }
+    }
+
+# Settings for a TLS enabled server.
+
+    server {
+        listen       443 ssl http2 default_server;
+        listen       [::]:443 ssl http2 default_server;
+        server_name  _;
+        root         /usr/share/nginx/html;
+
+        ssl_certificate "/etc/pki/nginx/server.crt";
+        ssl_certificate_key "/etc/pki/nginx/private/server.key";
+        ssl_session_cache shared:SSL:1m;
+        ssl_session_timeout  10m;
+        ssl_ciphers TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256:TLS_AES_128_CCM_SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:AES256-GCM-SHA384:AES256-CCM:AES128-GCM-SHA256:AES128-CCM:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES256-CCM:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-CCM:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:PSK-AES256-GCM-SHA384:PSK-CHACHA20-POLY1305:PSK-AES256-CCM:PSK-AES128-GCM-SHA256:PSK-AES128-CCM:PSK-AES256-CBC-SHA:PSK-AES128-CBC-SHA256:PSK-AES128-CBC-SHA:DHE-PSK-AES256-GCM-SHA384:DHE-PSK-CHACHA20-POLY1305:DHE-PSK-AES256-CCM:DHE-PSK-AES128-GCM-SHA256:DHE-PSK-AES128-CCM:DHE-PSK-AES256-CBC-SHA:DHE-PSK-AES128-CBC-SHA256:DHE-PSK-AES128-CBC-SHA:ECDHE-PSK-CHACHA20-POLY1305:ECDHE-PSK-AES256-CBC-SHA:ECDHE-PSK-AES128-CBC-SHA256:ECDHE-PSK-AES128-CBC-SHA:RSA-PSK-AES256-GCM-SHA384:RSA-PSK-CHACHA20-POLY1305:RSA-PSK-AES128-GCM-SHA256:RSA-PSK-AES256-CBC-SHA:RSA-PSK-AES128-CBC-SHA256:RSA-PSK-AES128-CBC-SHA;
+        ssl_prefer_server_ciphers on;
+
+        # Load configuration files for the default server block.
+        include /etc/nginx/default.d/*.conf;
+
+        error_page 404 /404.html;
+        location = /40x.html {
+        }
+
+        error_page 500 502 503 504 /50x.html;
+        location = /50x.html {
+        }
+    }
+
+}

nginx.container

@@ -0,0 +1,23 @@
+[Unit]
+Description=nginx
+Wants=network.target
+After=network.target
+
+[Container]
+Image=docker.io/library/nginx:1.25
+User=101
+Group=101
+Volume=%E/nginx:/etc/nginx:ro
+Volume=%E/pki/nginx:/etc/pki/nginx:ro
+Tmpfs=/var/cache/nginx
+Tmpfs=/var/run/nginx
+ReadOnly=true
+AddCapability=CAP_NET_BIND_SERVICE
+Network=host
+
+[Service]
+Restart=always
+ExecReload=/usr/bin/podman exec -i systemd-%N nginx -s reload
+
+[Install]
+WantedBy=multi-user.target

nginx.yaml

@@ -0,0 +1,22 @@
+variant: fcos
+version: 1.4.0
+
+ignition:
+  config:
+    merge:
+    - local: fetchcert.ign
+
+storage:
+  files:
+  - path: /etc/containers/systemd/nginx.container
+    mode: 0644
+    contents:
+      local: nginx.container
+  - path: /etc/nginx/nginx.conf
+    mode: 0644
+    contents:
+      local: nginx.conf
+  directories:
+  - path: /etc/nginx/conf.d
+  - path: /etc/nginx/default.d
+  - path: /etc/pki/nginx

notify-shutdown.service

@@ -0,0 +1,38 @@
+[Unit]
+Description=Send notification on machine shutdown
+RefuseManualStop=yes
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStop=/usr/bin/curl -d '%H is going down' https://ntfy.pyrocufflink.blue/alerts
+DynamicUser=yes
+CapabilityBoundingSet=
+DeviceAllow=
+DevicePolicy=closed
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+NoNewPrivileges=yes
+PrivateDevices=yes
+PrivateUsers=yes
+PrivateTmp=yes
+ProcSubset=pid
+ProtectClock=yes
+ProtectControlGroups=yes
+ProtectHome=yes
+ProtectHostname=yes
+ProtectKernelLogs=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+ProtectProc=invisible
+ProtectSystem=strict
+RestrictAddressFamilies=AF_INET AF_INET6
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallFilter=~@privileged @resources
+
+[Install]
+WantedBy=multi-user.target

notify-shutdown.yaml

@@ -0,0 +1,14 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  files:
+  - path: /etc/systemd/system/notify-shutdown.service
+    mode: 0644
+    contents:
+      local: notify-shutdown.service
+
+systemd:
+  units:
+  - name: notify-shutdown.service
+    enabled: true

nvr1.yaml

@@ -0,0 +1,38 @@
+variant: fcos
+version: 1.4.0
+
+ignition:
+  config:
+    merge:
+    - local: common.ign
+    - local: zram.ign
+    - local: gasket-driver.ign
+    - local: frigate.ign
+    - local: nginx.ign
+
+kernel_arguments:
+  should_exist:
+  - console=ttyS0,115200
+
+storage:
+  files:
+  - path: /etc/hostname
+    mode: 0644
+    contents:
+      inline: nvr1.pyrocufflink.blue
+  - path: /etc/systemd/system/var-lib-frigate.mount
+    mode: 0644
+    contents:
+      inline: |
+        [Mount]
+        What=/dev/disk/by-label/frigate
+        Where=/var/lib/frigate
+  - path: /etc/collectd.d/md.conf
+    mode: 0644
+    contents:
+      inline: |
+        LoadPlugin md
+  - path: /etc/fetchcert/token
+    mode: 0600
+    contents:
+      local: frigate.token

packages.yaml

@@ -0,0 +1,16 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  directories:
+  - path: /etc/ignition/packages.d
+    mode: 0755
+  files:
+  - path: /etc/ignition/install-packages.sh
+    mode: 0755
+    contents:
+      local: install-packages.sh
+  - path: /etc/systemd/system/install-packages.service
+    mode: 0644
+    contents:
+      local: install-packages.service

ssh-bootstrap.service

@@ -0,0 +1,13 @@
+# vim: set ft=systemd :
+[Service]
+Description=Bootstrap SSH host certificates
+ConditionPathExistsGlob=!/etc/ssh/ssh_host_*_key-cert.pub
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/bin/sh /etc/ssh/bootstrap.sh
+
+[Install]
+WantedBy=multi-user.target

ssh-bootstrap.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+gen_sshd_config() {
+    {
+    for x in ssh_host_*_key-cert.pub; do
+        printf 'HostCertificate /etc/ssh/%s\n' "${x}"
+    done
+    } > sshd_config.d/10-hostcertificate.conf
+}
+
+parse_response() {
+    jq -r '.certificates | to_entries | .[] | .key + " " + .value' \
+        | while read filename contents; do
+            [ -n "${filename}" ] || continue
+            echo "${contents}" > "${filename}" || return
+    done
+}
+
+request_sign() {
+    set -- \
+        https://bootstrap.pyrocufflink.blue/sshkeys/sign \
+        -H 'Accept: application/json' \
+        -F hostname=$(hostname -f)
+    for f in /etc/ssh/ssh_host_*_key.pub; do
+        set -- "$@" -F keys=@"${f}"
+    done
+    curl -fsSL "$@"
+}
+
+cd /etc/ssh || exit
+request_sign | parse_response
+gen_sshd_config
+
+systemctl reload sshd

sshkeys.yaml

@@ -4,4 +4,4 @@ passwd:
   users:
   - name: core
     ssh_authorized_keys:
-    - sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAINZCN2cxMDwedJ1Ke23Z3CZRcOYjqW8fFqsooRus7RK0AAAABHNzaDo= dustin@rosalina.pyrocufflink.blue
+    - cert-authority ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBImIoTTmhynCVy/vJ/Q2bWydzqVsvwhGvDgBbklw0eDt8UEbbP9HHPhxiMDtiAhbvRTg5BhYVAlR1MgdooT5dwQ=

step-ssh-renew.env

@@ -0,0 +1,3 @@
+STEP_CA_URL=https://ca.pyrocufflink.blue:32599
+STEP_ROOT=/etc/pki/ca-trust/source/anchors/dch-root-ca.crt
+STEP_PROVISIONER=sshpop

step-ssh-renew.target

@@ -0,0 +1,6 @@
+[Unit]
+Description=Renew SSH host certificates
+StopWhenUnneeded=yes
+Wants=step-ssh-renew@ed25519.service
+Wants=step-ssh-renew@ecdsa.service
+Wants=step-ssh-renew@rsa.service

step-ssh-renew.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Periodically renew SSH host certificates
+
+[Timer]
+Unit=%N.target
+OnCalendar=Tue *-*-* 00:00:00
+RandomizedDelaySec=48h
+Persistent=yes
+
+[Install]
+WantedBy=timers.target

step-ssh-renew@.container

@@ -0,0 +1,20 @@
+[Unit]
+Description=Renew SSH host %I certificate
+After=network-online.target
+Wants=network-online.target
+ConditionPathExists=/etc/ssh/ssh_host_%I_key-cert.pub
+
+[Container]
+ContainerName=step-ssh-renew-%I
+Image=docker.io/smallstep/step-cli:0.25.0
+EnvironmentFile=/etc/sysconfig/step-ssh-renew
+Exec=step ssh renew -f /etc/ssh/ssh_host_%I_key-cert.pub /etc/ssh/ssh_host_%I_key
+Volume=/etc/ssh:/etc/ssh:rw
+Volume=/etc/pki:/etc/pki:ro
+# Required in order to be able to write to /etc/ssh
+SecurityLabelDisable=true
+User=0
+Group=0
+
+[Service]
+Type=oneshot

step-ssh.yaml

@@ -0,0 +1,29 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  files:
+  - path: /etc/ssh/bootstrap.sh
+    mode: 0755
+    contents:
+      local: ssh-bootstrap.sh
+  - path: /etc/containers/systemd/step-ssh-renew@.container
+    mode: 0644
+    contents:
+      local: step-ssh-renew@.container
+  - path: /etc/sysconfig/step-ssh-renew
+    mode: 0600
+    contents:
+      local: step-ssh-renew.env
+  - path: /etc/systemd/system/ssh-bootstrap.service
+    mode: 0644
+    contents:
+      local: ssh-bootstrap.service
+  - path: /etc/systemd/system/step-ssh-renew.target
+    mode: 0644
+    contents:
+      local: step-ssh-renew.target
+  - path: /etc/systemd/system/step-ssh-renew.timer
+    mode: 0644
+    contents:
+      local: step-ssh-renew.timer

zram.yaml

@@ -0,0 +1,10 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  files:
+  - path: /etc/systemd/zram-generator.conf
+    mode: 0644
+    contents:
+      inline: |
+        [zram0]