From e0b2b3f5ae8845d365522bd0fb9053fa7ca54cdf Mon Sep 17 00:00:00 2001
From: "Dustin C. Hatch" <dustin@hatch.name>
Date: Sat, 24 Feb 2024 08:33:52 -0600
Subject: [PATCH] v-m: Scrape metrics from Patroni

Patroni, a component of the *postgres poerator*, exports metrics about
the PostgreSQL database servers it manages.  Notably, it provides
information about the current transaction log location for each server.
This allows us to monitor and alert on the health of database replicas.
---
 victoria-metrics/alerts.yml |  9 +++++++++
 victoria-metrics/scrape.yml | 21 +++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/victoria-metrics/alerts.yml b/victoria-metrics/alerts.yml
index d8cd087..e8e14ef 100644
--- a/victoria-metrics/alerts.yml
+++ b/victoria-metrics/alerts.yml
@@ -126,3 +126,12 @@ groups:
   - alert: Battery Low
     expr:
       homeassistant_sensor_battery_percent{entity!~"sensor\\.(pixel_|sm_p610).*"} < 10
+
+- name: PostgreSQL
+  rules:
+  - alert: Replica lag too high
+    expr:
+      (patroni_xlog_location != 0)
+      - ignoring (instance) group_right (scope) (patroni_xlog_replayed_location != 0)
+      > 10240
+    for: 10m
diff --git a/victoria-metrics/scrape.yml b/victoria-metrics/scrape.yml
index 91c8a79..426306f 100644
--- a/victoria-metrics/scrape.yml
+++ b/victoria-metrics/scrape.yml
@@ -338,3 +338,24 @@ scrape_configs:
   static_configs:
   - targets:
     - argocd-metrics.argocd:8082
+
+- job_name: patroni
+  kubernetes_sd_configs:
+  - role: pod
+    namespaces:
+      names:
+      - postgresql
+    selectors:
+    - role: pod
+      label: application=spilo
+  relabel_configs:
+  - action: keep
+    source_labels:
+    - __meta_kubernetes_pod_container_name
+    regex: postgres
+  - action: keep
+    source_labels:
+    - __meta_kubernetes_pod_container_port_number
+    regex: '8008'
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: instance