From e0c633c21ed045b4cb62a44b5a29696d264eb9d8 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sun, 17 Nov 2024 18:43:00 +0000 Subject: [PATCH 1/7] v-m: scrape: Fix Nextcloud URL Nextcloud uses a _client-side_ (Javascript) redirect to navigate the browser to its `index.php`. The page it serves with this redirect is static and will often load successfully, even if there is a problem with the application. This causes the Blackbox exporter to record the site as "up," even when it it definitely is not. To avoid this, we can scrape the `index.php` page explicitly, ensuring that the application is loaded. --- 20125/config.yml | 2 +- victoria-metrics/scrape.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/20125/config.yml b/20125/config.yml index 7a06670..64b9dc3 100644 --- a/20125/config.yml +++ b/20125/config.yml @@ -31,7 +31,7 @@ applications: - instance: homeassistant.pyrocufflink.blue - name: Nextcloud - url: &url https://nextcloud.pyrocufflink.net/ + url: &url https://nextcloud.pyrocufflink.net/index.php icon: url: icons/nextcloud.png alerts: diff --git a/victoria-metrics/scrape.yml b/victoria-metrics/scrape.yml index 2591f42..4c8b8ff 100644 --- a/victoria-metrics/scrape.yml +++ b/victoria-metrics/scrape.yml @@ -57,7 +57,7 @@ scrape_configs: - http://pyrocufflink.net/ - http://ebonfire.com/ - http://chmod777.sh/ - - https://nextcloud.pyrocufflink.net/ + - https://nextcloud.pyrocufflink.net/index.php - https://bitwarden.pyrocufflink.blue/ - https://git.pyrocufflink.blue/ - https://tabitha.biz/ From 4243823ba59cc5ffe51dd37abf7fcb3dfdbd1a1c Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 7 Dec 2024 09:28:44 -0600 Subject: [PATCH 2/7] invoice-ninja: Fix network policy for ingress Since the IP address assigned to the ingress controller is now managed by keepalived and known to Kubernetes, the network policy needs to allow access to it by pod namespace rather than IP address. It seems that the former takes precedence over the latter, so even though the IP address was explicitly allowed, traffic was not permitted because it was destined for a Kubernetes service that was not. --- invoice-ninja/network-policy.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/invoice-ninja/network-policy.yaml b/invoice-ninja/network-policy.yaml index ccc6f3a..111e826 100644 --- a/invoice-ninja/network-policy.yaml +++ b/invoice-ninja/network-policy.yaml @@ -29,8 +29,9 @@ spec: ports: - port: 25 - to: - - ipBlock: - cidr: 172.30.0.147/32 + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx ports: - port: 80 - port: 443 From 1c4b5e19a47525933081f10917df97e5fdacdcb8 Mon Sep 17 00:00:00 2001 From: bot Date: Sat, 21 Dec 2024 12:32:08 +0000 Subject: [PATCH 3/7] firefly-iii: Update to 6.1.25 --- firefly-iii/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firefly-iii/kustomization.yaml b/firefly-iii/kustomization.yaml index 694db58..49798d8 100644 --- a/firefly-iii/kustomization.yaml +++ b/firefly-iii/kustomization.yaml @@ -55,4 +55,4 @@ patches: defaultMode: 0640 images: - name: docker.io/fireflyiii/core - newTag: version-6.1.22 + newTag: version-6.1.25 From a79668dcf1f482273d2a9dadbaabfa3acf30bff6 Mon Sep 17 00:00:00 2001 From: bot Date: Sat, 21 Dec 2024 12:32:10 +0000 Subject: [PATCH 4/7] gotenberg: Update to 8.14.1 --- paperless-ngx/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paperless-ngx/kustomization.yaml b/paperless-ngx/kustomization.yaml index 6006e6b..5a233ab 100644 --- a/paperless-ngx/kustomization.yaml +++ b/paperless-ngx/kustomization.yaml @@ -47,6 +47,6 @@ images: - name: ghcr.io/paperless-ngx/paperless-ngx newTag: 2.13.5 - name: docker.io/gotenberg/gotenberg - newTag: 8.13.0 + newTag: 8.14.1 - name: docker.io/apache/tika newTag: 3.0.0.0 From d04c18cfcda2a7e4be8707bb749d229486d3e576 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 21 Dec 2024 12:14:11 -0600 Subject: [PATCH 5/7] v-m/alerts: Remove 'no file changes' alert It turns out this alert is not very useful, and indeed quite annoying. Many servers can go for days or even weeks with no changes, which is completely normal. --- victoria-metrics/alerts.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/victoria-metrics/alerts.yml b/victoria-metrics/alerts.yml index fb7a334..7205472 100644 --- a/victoria-metrics/alerts.yml +++ b/victoria-metrics/alerts.yml @@ -232,25 +232,6 @@ groups: Clients are scheduled to back up every day, but at least one has not been backed up in at least 7 days. Check the Restic configuration on that system to ensure backups are running properly. - - alert: No File Changes - expr: >- - max_over_time( - abs( - delta( - sum(restic_backup_size_total{ - client_hostname!="pxe0.pyrocufflink.blue", - client_hostname!="web0.pyrocufflink.blue", - }) - by (client_hostname, client_username) - ) - )[7d] - ) == 0 - annotations: - summary: The size of the Restic backup has not changed - description: >- - The size of the Restic backup for a particular client has not changed - in at least 7 days. This may indicate that the backup configuration - is incorrect. - name: Paperless-ngx rules: From a03d63841dcf250d20d40ebd5a63dd388e1ebf71 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 21 Dec 2024 12:17:45 -0600 Subject: [PATCH 6/7] v-m/alerts: Fire paperless email alert after 12h We don't need a notification about paperless not scheduling email tasks every time there is a gap in the metric. This can happen in some innocuous situations like when the pod restarts or if there is a brief disruption of service. Using the `absent_over_time` function with a range vector, we can have the alert fire only if there have been no email tasks scheduled within the last 12 hours. --- victoria-metrics/alerts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/victoria-metrics/alerts.yml b/victoria-metrics/alerts.yml index 7205472..d8c37ab 100644 --- a/victoria-metrics/alerts.yml +++ b/victoria-metrics/alerts.yml @@ -254,11 +254,11 @@ groups: for details about the task failures. - alert: Paperless email task not running expr: >- - absent( + absent_over_time( flower_events_total{ type="task-started", task="paperless_mail.tasks.process_mail_accounts" - } + }[12h] ) annotations: summary: Paperless task to process mail accounts has not run recently From b9d69ec0a36ec73a6ea4b89c1fa128574065c6da Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 21 Dec 2024 12:23:19 -0600 Subject: [PATCH 7/7] v-m/alerts: Ignore missing backups from Toad, Luma Toad and Luma can go offline for several days at a time if I don't use them. I don't need an alert telling me this. --- victoria-metrics/alerts.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/victoria-metrics/alerts.yml b/victoria-metrics/alerts.yml index d8c37ab..94c4dda 100644 --- a/victoria-metrics/alerts.yml +++ b/victoria-metrics/alerts.yml @@ -225,7 +225,10 @@ groups: the issue as soon as possible. - alert: Last Backup Age expr: >- - time() - restic_backup_timestamp > 604800 + time() - restic_backup_timestamp{ + client_hostname!="luma.pyrocufflink.blue", + client_hostname!="toad.pyrocufflink.blue", + }> 604800 annotations: summary: A Restic client has not backed up recently description: >-