From d74e26d527fecd7957627df7f7d655c6cbcf0c53 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Fri, 10 May 2024 10:27:58 -0500 Subject: [PATCH] victoria-metrics: Send alerts via ntfy I don't like having alerts sent by e-mail. Since I don't get e-mail notifications on my watch, I often do not see alerts for quite some time. They are also much harder to read in an e-mail client (Fastmail web an K-9 Mail both display them poorly). I would much rather have them delivered via _ntfy_, just like all the rest of the ephemeral notifications I receive. Fortunately, it is easy enough to integrate Alertmanager and _ntfy_ using the webhook notifier in Alertmanager. Since _ntfy_ does not natively support the Alertmanager webhook API, though, a bridge is necessary to translate from one data format to the other. There are a few options for this bridge, but I chose [alexbakker/alertmanager-ntfy][0] because it looked the most complete while also having the simplest configuration format. Sadly, it does not expose any Prometheus metrics itself, and since it's deployed in the _victoria-metrics_ namespace, it needs to be explicitly excluded from the VMAgent scrape configuration. [0]: https://github.com/alexbakker/alertmanager-ntfy --- victoria-metrics/alertmanager-ntfy.config.yml | 24 +++++++ victoria-metrics/alertmanager-ntfy.yaml | 69 +++++++++++++++++++ victoria-metrics/alertmanager.config.yml | 13 ++-- victoria-metrics/kustomization.yaml | 5 ++ victoria-metrics/scrape.yml | 4 ++ 5 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 victoria-metrics/alertmanager-ntfy.config.yml create mode 100644 victoria-metrics/alertmanager-ntfy.yaml diff --git a/victoria-metrics/alertmanager-ntfy.config.yml b/victoria-metrics/alertmanager-ntfy.config.yml new file mode 100644 index 0000000..c1333bd --- /dev/null +++ b/victoria-metrics/alertmanager-ntfy.config.yml @@ -0,0 +1,24 @@ +http: + addr: :8000 +ntfy: + baseurl: http://ntfy.ntfy:2586 + notification: + topic: alerts + priority: | + status == "firing" ? "urgent" : "default" + tags: + - tag: heavy_check_mark + condition: status == "resolved" + - tag: rotating_light + condition: status == "firing" + templates: + title: >- + {{ if eq .Status "resolved" }}Resolved: {{ end -}} + {{ if index .Annotations "summary" -}} + {{ index .Annotations "summary" }} + {{ else -}} + {{ index .Labels "alertname" }} + {{ index .Labels "alertgroup" }} + {{ end -}} + description: | + {{ index .Annotations "description" }} diff --git a/victoria-metrics/alertmanager-ntfy.yaml b/victoria-metrics/alertmanager-ntfy.yaml new file mode 100644 index 0000000..084ce7f --- /dev/null +++ b/victoria-metrics/alertmanager-ntfy.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: alertmanager-ntfy + labels: + app.kubernetes.io/name: alertmanager-ntfy + app.kubernetes.io/component: alertmanager-ntfy +spec: + ports: + - port: 8000 + name: alertmanager-ntfy + selector: + app.kubernetes.io/name: alertmanager-ntfy + app.kubernetes.io/component: alertmanager-ntfy + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alertmanager-ntfy + labels: + app.kubernetes.io/name: alertmanager-ntfy + app.kubernetes.io/component: alertmanager-ntfy +spec: + selector: + matchLabels: + app.kubernetes.io/name: alertmanager-ntfy + app.kubernetes.io/component: alertmanager-ntfy + template: + metadata: + labels: + app.kubernetes.io/name: alertmanager-ntfy + app.kubernetes.io/component: alertmanager-ntfy + spec: + containers: + - name: alertmanager-ntfy + image: git.pyrocufflink.net/containerimages/alertmanager-ntfy + args: + - --configs=/config/config.yml + ports: + - containerPort: 8000 + name: http + readinessProbe: &probe + tcpSocket: + port: http + periodSeconds: 60 + startupProbe: + <<: *probe + periodSeconds: 1 + successThreshold: 1 + failureThreshold: 30 + timeoutSeconds: 1 + securityContext: + runAsNonRoot: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /config + name: config + readOnly: true + securityContext: + fsGroup: 21447 + runAsGroup: 21447 + runAsNonRoot: true + runAsUser: 21447 + volumes: + - name: config + configMap: + name: alertmanager-ntfy diff --git a/victoria-metrics/alertmanager.config.yml b/victoria-metrics/alertmanager.config.yml index 5a1e64e..34dc4de 100644 --- a/victoria-metrics/alertmanager.config.yml +++ b/victoria-metrics/alertmanager.config.yml @@ -8,16 +8,11 @@ receivers: - send_resolved: true to: gyrfalcon@ebonfire.com name: default-email +- name: ntfy + webhook_configs: + - url: http://alertmanager-ntfy:8000/hook route: group_by: - '...' - receiver: default-email - routes: - - group_by: - - alertname - group_wait: 1m - match: - job: homeassistant - receiver: default-email - repeat_interval: 120h + receiver: ntfy diff --git a/victoria-metrics/kustomization.yaml b/victoria-metrics/kustomization.yaml index fc10f5e..f25beda 100644 --- a/victoria-metrics/kustomization.yaml +++ b/victoria-metrics/kustomization.yaml @@ -20,6 +20,7 @@ resources: - vmagent.yaml - vmalert.yaml - alertmanager.yaml +- alertmanager-ntfy.yaml - blackbox-exporter.yaml - ingress.yaml - ../dch-root-ca @@ -43,6 +44,10 @@ configMapGenerator: options: disableNameSuffixHash: true +- name: alertmanager-ntfy + files: + - config.yml=alertmanager-ntfy.config.yml + - name: blackbox files: - blackbox.yml diff --git a/victoria-metrics/scrape.yml b/victoria-metrics/scrape.yml index 282e3ea..52d0daa 100644 --- a/victoria-metrics/scrape.yml +++ b/victoria-metrics/scrape.yml @@ -14,6 +14,10 @@ scrape_configs: separator: ':' regex: 'vmstorage:(vminsert|vmselect)' action: drop + - source_labels: + - __meta_kubernetes_pod_label_app_kubernetes_io_component + regex: alertmanager-ntfy + action: drop - source_labels: - __meta_kubernetes_pod_container_name regex: vmalert