From 29afcae52ee52a671ed28a8069a22a7d394a1685 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 17 Feb 2024 19:37:12 -0600 Subject: [PATCH 1/9] fetchcert: Deploy tool to get cert from k8s Secret The `fetchcert` tool is a short shell script that fetches an X.509 certificate and corresponding private key from a Kubernetes Secret, using the Kubernetes API. I originally wrote it for the Frigate server so it could fetch the _pyrocufflink.blue_ wildcard certificate, which is managed by _cert-manager_. Since then, I have adapted it to be more generic, so it will be useful to fetch the _loki.pyrocufflink.blue_ certificate for Grafana Loki. Although the script is rather simple, it does have several required configuration parameters. It needs to know the URL of the Kubernetes API server and have the certificate for the CA that signs the server certificate, as well as an authorization token. It also needs to know the namespace and name of the Secret from which it will fetch the certificate and private key. Finally, needs to know the paths to the files where the fetched data will be written. Generally, after certificates are updated, some action needs to be performed in order to make use of them. This typically involves restarting or reloading a daemon. Since the `fetchcert` tool runs in a container, it can't directly perform those actions, so it simply indicates via a special exit code that the certificate has been updated and some further action may be needed. The `/etc/fetchcert/postupdate.sh` script is executed by _systemd_ after `fetchcert` finishes. If the `EXIT_STATUS` environment variable (which is set by _systemd_ to the return code of the main service process) matches the expected code, the configured post-update actions will be executed. --- app/fetchcert/fetchcert.cue | 12 ++++++++ app/fetchcert/templates.cue | 39 +++++++++++++++++++++++++ env/prod/fetchcert.cue | 29 ++++++++++++++++++ templates/fetchcert/ca.crt | 1 + templates/fetchcert/fetchcert.container | 22 ++++++++++++++ templates/fetchcert/fetchcert.timer | 9 ++++++ templates/fetchcert/postupdate.sh | 8 +++++ templates/fetchcert/token | 1 + 8 files changed, 121 insertions(+) create mode 100644 app/fetchcert/fetchcert.cue create mode 100644 app/fetchcert/templates.cue create mode 100644 env/prod/fetchcert.cue create mode 100644 templates/fetchcert/ca.crt create mode 100644 templates/fetchcert/fetchcert.container create mode 100644 templates/fetchcert/fetchcert.timer create mode 100644 templates/fetchcert/postupdate.sh create mode 100644 templates/fetchcert/token diff --git a/app/fetchcert/fetchcert.cue b/app/fetchcert/fetchcert.cue new file mode 100644 index 0000000..4e29703 --- /dev/null +++ b/app/fetchcert/fetchcert.cue @@ -0,0 +1,12 @@ +package fetchcert + +#Fetchcert: { + ca_cert: string + kubernetes_url: string + token: string + namespace: string + secret: string + cert: string + key: string + postupdate?: string +} diff --git a/app/fetchcert/templates.cue b/app/fetchcert/templates.cue new file mode 100644 index 0000000..d14d236 --- /dev/null +++ b/app/fetchcert/templates.cue @@ -0,0 +1,39 @@ +package fetchcert + +import "du5t1n.me/cfg/base/schema/instructions" + +templates: [...instructions.#RenderInstruction] & [ + { + template: "fetchcert/ca.crt" + dest: "/etc/fetchcert/ca.crt" + }, + { + template: "fetchcert/token" + dest: "/etc/fetchcert/token" + }, + { + template: "fetchcert/postupdate.sh" + dest: "/etc/fetchcert/postupdate.sh" + mode: "u=rwx,go=rx" + }, + { + template: "fetchcert/fetchcert.container" + dest: "/etc/containers/systemd/fetchcert.container" + hooks: { + changed: [ + {run: "systemctl daemon-reload", immediate: true}, + ] + } + }, + { + template: "fetchcert/fetchcert.timer" + dest: "/etc/systemd/system/fetchcert.timer" + hooks: { + changed: [ + {run: "systemctl daemon-reload", immediate: true}, + {run: "systemctl enable fetchcert.timer"}, + {run: "systemctl start fetchcert.timer"}, + ] + } + }, +] diff --git a/env/prod/fetchcert.cue b/env/prod/fetchcert.cue new file mode 100644 index 0000000..4a7dee1 --- /dev/null +++ b/env/prod/fetchcert.cue @@ -0,0 +1,29 @@ +package prod + +import f "du5t1n.me/cfg/app/fetchcert" + +fetchcert: base: f.#Fetchcert & { + ca_cert: """ + -----BEGIN CERTIFICATE----- + MIIC/jCCAeagAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl + cm5ldGVzMB4XDTIyMDgwMTAyNTUzM1oXDTMyMDcyOTAyNTUzM1owFTETMBEGA1UE + AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMs6 + 2PUOzIClsAgPv1Mn9CTwzSFMntAn7OppwK5BQ4E5Vd1yMjz3p0uA1ZINv1ORorG0 + mLl95C7y+CWUGPx+stHKQr/40sLGyypbX+AfjoPzHiDbIcbZEff8X5RwKqzmT9V7 + Yt29KewADod0z+fqNYa62MJDaUunfwaV8kKFU/WJM8IKv2eJxAtWzvK3iHAFhx0j + Xo4TlyINL9V9UMKLf12w6CA3G41uZIBCN3G7aJEm++eGoMdrPZUXlbCpbSztO85/ + hbulVs+0hCIxWiI+mRmB5OoWlRYL4jA45oK/RtpEqSwZ95zlGNAChmH7rb0pTtNf + N0/C2wKAEL4POLx9kscCAwEAAaNZMFcwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB + /wQFMAMBAf8wHQYDVR0OBBYEFHYActCjEWdtsA+Ju25gxJh/vaLQMBUGA1UdEQQO + MAyCCmt1YmVybmV0ZXMwDQYJKoZIhvcNAQELBQADggEBAAfkYHecXUwyqvMSXmqr + ETqEzDCBini14s89VDhaDHOXBID9TKMVyeePdEYcPAJz3wo8fbx/+TL37K6hEuo+ + 7bUaamaumznsjg9L0Hth19GvuRKMXJlEpndRmE5K9hnaDLr94MLg9n1qGcEOt6tw + O6X5qqHf9AuuL39vt1+kSw6PeZZFZNMDZ8BdiTssw4btjQ2bsWu0wSiOSz/F8iRf + 2vN5An5dheroDsFs4dZ9gnJ69TmqV1YqQxfRWqCxzfNJbgVm6AoBPwhL1JRuAU4N + 3nCNoM9n2tLFDojT4un1778UVU91PtcBVdM9Nq+RC2jhXIyLBqsEK0ofOqFYqj3F + 0EQ= + -----END CERTIFICATE----- + """ + kubernetes_url: "https://kubernetes.pyrocufflink.blue:6443" + namespace: "dch-ca" +} diff --git a/templates/fetchcert/ca.crt b/templates/fetchcert/ca.crt new file mode 100644 index 0000000..168b239 --- /dev/null +++ b/templates/fetchcert/ca.crt @@ -0,0 +1 @@ +{{ fetchcert.ca_cert }} diff --git a/templates/fetchcert/fetchcert.container b/templates/fetchcert/fetchcert.container new file mode 100644 index 0000000..c9e541a --- /dev/null +++ b/templates/fetchcert/fetchcert.container @@ -0,0 +1,22 @@ +[Unit] +Description=Fetch HTTPS certificate from Kubernetes Secret API +Wants=network-online.target +After=network-online.target + +[Container] +Image=git.pyrocufflink.net/containerimages/fetchcert +Exec={{ fetchcert.namespace }} {{ fetchcert.secret }} /etc/fetchcert/certs/{{ fetchcert.key }} /etc/fetchcert/certs/{{ fetchcert.cert }} +ReadOnly=yes +ReadOnlyTmpfs=yes +Volume=/etc/fetchcert:/etc/fetchcert:ro +Volume=/etc/fetchcert/certs:/etc/fetchcert/certs:rw,z +Environment=KUBERNETES_URL={{ fetchcert.kubernetes_url }} +AddCapability=CAP_CHOWN +DropCapability=all +NoNewPrivileges=yes + +[Service] +Type=oneshot +SuccessExitStatus=20 +ExecStartPre=/bin/mkdir -p /etc/fetchcert/certs +ExecStopPost=-/etc/fetchcert/postupdate.sh diff --git a/templates/fetchcert/fetchcert.timer b/templates/fetchcert/fetchcert.timer new file mode 100644 index 0000000..489c582 --- /dev/null +++ b/templates/fetchcert/fetchcert.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Periodically fetch certificate from Kubernetes + +[Timer] +OnCalendar=*-*-* 0:0:0 +RandomizedDelaySec=8h + +[Install] +WantedBy=timers.target diff --git a/templates/fetchcert/postupdate.sh b/templates/fetchcert/postupdate.sh new file mode 100644 index 0000000..42b0e29 --- /dev/null +++ b/templates/fetchcert/postupdate.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +if [ ${EXIT_STATUS:-1} -ne 20 ]; then + exit 0 +fi +{% if fetchcert.postupdate %} +{{ fetchcert.postupdate }} +{% endif -%} diff --git a/templates/fetchcert/token b/templates/fetchcert/token new file mode 100644 index 0000000..fb5e66a --- /dev/null +++ b/templates/fetchcert/token @@ -0,0 +1 @@ +{{ fetchcert.token | decrypt }} From 011058aec3d3400b682ba90fcfea8457c58e36cd Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 17 Feb 2024 19:37:29 -0600 Subject: [PATCH 2/9] loki: Use fetchcert to manage server certificate Before going into production with Grafana Loki, I want to set it up to use TLS. To that end, I have configured _cert-manager_ to issue it a certificate, signed by _DCH CA_. In order to use said certificate, we need to configure `fetchcert` to run on the Loki server. --- env/prod/fetchcert.cue | 11 +++++++++ host/loki0.pyrocufflink.blue.cue | 30 ++++++++++++++++++++++++ host/loki0.pyrocufflink.blue.post.sh | 1 + instructions/loki0.pyrocufflink.blue.cue | 6 +++-- scripts/loki-cert.sh | 3 +++ templates/loki/config.yml | 3 +++ 6 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 scripts/loki-cert.sh diff --git a/env/prod/fetchcert.cue b/env/prod/fetchcert.cue index 4a7dee1..dec597c 100644 --- a/env/prod/fetchcert.cue +++ b/env/prod/fetchcert.cue @@ -27,3 +27,14 @@ fetchcert: base: f.#Fetchcert & { kubernetes_url: "https://kubernetes.pyrocufflink.blue:6443" namespace: "dch-ca" } + +fetchcert: loki: fetchcert.base & { + secret: "loki" + cert: "loki.cer" + key: "loki.key" + postupdate: """ + install -v -o root -g 10001 -m ugo=r /etc/fetchcert/certs/loki.cer /etc/loki/server.cer + install -v -o root -g 10001 -m ug=r,o= /etc/fetchcert/certs/loki.key /etc/loki/server.key + systemctl reload loki + """ +} diff --git a/host/loki0.pyrocufflink.blue.cue b/host/loki0.pyrocufflink.blue.cue index f1d4f03..81409b1 100644 --- a/host/loki0.pyrocufflink.blue.cue +++ b/host/loki0.pyrocufflink.blue.cue @@ -4,3 +4,33 @@ import ( ssh: prod.ssh sudo: prod.sudo + +fetchcert: prod.fetchcert.loki & { + token: """ + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtNTZzeW9XeWoycVdQa092 + N0VYL2grR0lLY1c4QXl2VHI3NmsxM253UlNVCmZLbFZWakJGVG9WakkyYmpJL1VR + YmVQQXRCTlhrQk9UYUE5UkRFZUlwNlkKLS0tIGg4R25ZaVhUU1BFVjdac2NqMVpQ + QmZRTndBalZndVF0VFpxdHBRemhNS1EKrNZG179fh2aS/3FOaM1xCHRG4uOt5jyx + 1m5h3Q9y2u7EbcbZHLIZR3wkQfsfscK1PS0+H0NiYAgh9u2L2kdhcLcesb3fhmSy + svHzW2q1ZkJ8DSwH3xCRBuKmH4Q172NcVUPzI39CgsI5SkqZdKjWnK9JJAs43Ihr + cM90hUN+5t50byUSzwTCmNY4xVW3N/pWMfrethCYk9E8cXts/L3A3EpgpIi3qrKn + gj2VfrvpHAWVcggX1rZVFlQwBg4LnPWMNztl5VRYIvwfJghykEjMlzkysLm3Q2is + /w+kthpBzYAvI4c1Tfx3/uMRVcWnmUgz15viKlqohVaAl9PHQ2y/te9w9D5ZtcYs + D33hfA7Aux9t18WJ/ru09rEJl649Al7ZxQd73upf9QrWGzkX4luHO85n8CBmcsuh + +ZcM1HMLiuxGCW6xyq66Eg6t/1pfPWGZtLCsFh4SRgJ6Uuq14FyU32Pkulq+yEMg + Sq2ZRUXU+e3M6/HcUhb+QQUTQF1wPHyEukUlecLGDd3i+xpjOrL5Eg7LjKVAv8Yj + 8U1yiYjgRHfdkvT27RJC/rxuf674vU8H8na3jGXrPARMqq4L4B0XkUzclJZMzSPC + cSTaEIgb5OpfWmMb4uC0p76vHYhr4XX3iIVpivfxaDLAgyx06D4/oXALcgjcCHWY + /7m5t8MbIGqluqcJLYRhSQ+G/aWiyZG3zlgRfpOIyVzQHwQwGf2CLh6ygv9n5cWP + Gr0ZfcyVps734gVsDNqZ3vTy4nxjTueUiUpNqRaznzxT/z7Mq9/i0s1aoWBef0PV + MZL0jxyMeQUfRf0DdP/iPqkTU5hxw8/yqwuu2i3TJImVQ8ga8O3InyvN577mPihE + EqFjRl1jZr+Uip0+SPz+CSLIgBJ8rpAo/HTpue6Oe88rYtC0437YQtcWpB3rnARD + uggtP70SfvS7FWFCbYy7nxZrUcDMloD5gcIYNobkWQZhGdGvXDGVxB/FT8Rg6tAU + EOpaSSc3wOmHpnB6qCyCJ45mb6HwRCGoZmxaG/5uWreys0R8AJsMIq8vFVAS3sDo + EONNYMWtlAZg8XOZcSgSnKpUF5VWlt+3HLkpwQkTBq3SvjvMd6shybPVGVNxMwbU + a2gey9Kv4lq8Suvvrn31DeYErGwUYy0qMwTL1a4Q8I08kMg6lqqaPotIC63RSlUu + SEoarQ== + -----END AGE ENCRYPTED FILE----- + """ +} diff --git a/host/loki0.pyrocufflink.blue.post.sh b/host/loki0.pyrocufflink.blue.post.sh index 17c32eb..b7087d9 100644 --- a/host/loki0.pyrocufflink.blue.post.sh +++ b/host/loki0.pyrocufflink.blue.post.sh @@ -1 +1,2 @@ . scripts/no-coreos-default-sudo.sh +. scripts/loki-cert.sh diff --git a/instructions/loki0.pyrocufflink.blue.cue b/instructions/loki0.pyrocufflink.blue.cue index 165c035..0c5cbd0 100644 --- a/instructions/loki0.pyrocufflink.blue.cue +++ b/instructions/loki0.pyrocufflink.blue.cue @@ -2,12 +2,14 @@ import ( "list" "du5t1n.me/cfg/app/collectd" + "du5t1n.me/cfg/app/fetchcert" "du5t1n.me/cfg/app/loki" "du5t1n.me/cfg/env/prod" ) render: list.Concat([ - collectd.templates, - loki.templates, prod.templates, + collectd.templates, + fetchcert.templates, + loki.templates, ]) diff --git a/scripts/loki-cert.sh b/scripts/loki-cert.sh new file mode 100644 index 0000000..8fe20f0 --- /dev/null +++ b/scripts/loki-cert.sh @@ -0,0 +1,3 @@ +if [ ! -f /host/etc/loki/server.cer ] || [ ! -f /host/etc/loki/server.key ]; then + systemctl start fetchcert +fi diff --git a/templates/loki/config.yml b/templates/loki/config.yml index b11f2f0..0ddb22d 100644 --- a/templates/loki/config.yml +++ b/templates/loki/config.yml @@ -2,6 +2,9 @@ auth_enabled: false server: http_listen_port: 3100 + http_tls_config: + cert_file: /etc/loki/server.cer + key_file: /etc/loki/server.key grpc_listen_port: 9096 common: From 4608f19724d6cbcf49b548b66c899216711f5278 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 17 Feb 2024 21:19:38 -0600 Subject: [PATCH 3/9] loki: Add ExecReload to systemd service unit According to the [Grafana Loki documentation][0], sending SIGHUP to the Loki process will instruct it to reload its configuration. This is necessary in order for it to re-read its server certificate after it has been renewed. [0]: https://grafana.com/docs/loki/latest/configure/#reload-at-runtime --- templates/loki/loki.container | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/loki/loki.container b/templates/loki/loki.container index e3f31e9..98956bc 100644 --- a/templates/loki/loki.container +++ b/templates/loki/loki.container @@ -6,6 +6,7 @@ Wants=network-online.target [Service] StateDirectory=%P +ExecReload=/usr/bin/podman kill --cidfile=%t/%N.cid --signal HUP [Container] Image=docker.io/grafana/loki:2.9.4 From 45c35c065a1b95b89e330be569cb80c11db8290c Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Tue, 13 Feb 2024 20:13:21 -0600 Subject: [PATCH 4/9] promtail: Deploy Loki Promtail Agent [Promtail][0] is the log collection agent for Grafana Loki. It reads logs from various locations, including local files and the _systemd_ journal and sends them to Loki via HTTP. Loki configuration is a highly-structured YAML document. Thus, instead of using Tera template syntax for loops, conditionals, etc., we can use the full power of CUE to construct the configuration. Using the `Marshal` function from the built-in `encoding/yaml` package, we serialize the final configuration structure as a string and write it verbatim to the configuration file. I have modeled most of the Promtail configuration schema in the `du5t1n.me/cfg/app/promtail/schema` package. Having the schema modeled will ensure the generated configuration is valid during development (i.e. `cue export` will fail if it is not), which will save time pushing changes to machines and having Loki complain. The `#promtail` "function" in `du5t1n.me/cfg/env/prod` makes it easy to build our desired configuration. It accepts an optional `#scrape` field, which can be used to provide specific log scraping definitions. If it is unspecified, the default configuration is to scrape the systemd journal. Hosts with additional needs can supply their own list, probably including the `promtail.scrape.journal` object in it to get the default journal scrape job. [0]: https://grafana.com/docs/loki/latest/send-data/promtail/ --- app/promtail/schema/metrics.cue | 38 ++++++++ app/promtail/schema/pipeline.cue | 106 +++++++++++++++++++++++ app/promtail/schema/schema.cue | 62 +++++++++++++ app/promtail/templates.cue | 33 +++++++ env/prod/promtail.cue | 86 ++++++++++++++++++ host/loki0.pyrocufflink.blue.cue | 2 + instructions/loki0.pyrocufflink.blue.cue | 2 + templates/promtail/ca.crt | 1 + templates/promtail/config.yml | 1 + templates/promtail/promtail.container | 23 +++++ 10 files changed, 354 insertions(+) create mode 100644 app/promtail/schema/metrics.cue create mode 100644 app/promtail/schema/pipeline.cue create mode 100644 app/promtail/schema/schema.cue create mode 100644 app/promtail/templates.cue create mode 100644 env/prod/promtail.cue create mode 100644 templates/promtail/ca.crt create mode 100644 templates/promtail/config.yml create mode 100644 templates/promtail/promtail.container diff --git a/app/promtail/schema/metrics.cue b/app/promtail/schema/metrics.cue new file mode 100644 index 0000000..6e3ca84 --- /dev/null +++ b/app/promtail/schema/metrics.cue @@ -0,0 +1,38 @@ +package schema + +#BaseMetric: { + type: string + description?: string + prefix?: string + source?: string + max_idle_duration?: string +} + +#CounterMetric: { + #BaseMetric + type: "Counter" + config: { + match_all?: bool + count_entry_bytes?: bool + value?: string + action: "inc" | "add" + } +} + +#GaugeMetric: { + #BaseMetric + type: "Gauge" + config: { + value?: string + action: "set" | "inc" | "dec" | "add" | "sub" + } +} + +#HistogramMetric: { + #BaseMetric + type: "Histogram" + config: { + value?: string + buckets: [int] + } +} diff --git a/app/promtail/schema/pipeline.cue b/app/promtail/schema/pipeline.cue new file mode 100644 index 0000000..d707907 --- /dev/null +++ b/app/promtail/schema/pipeline.cue @@ -0,0 +1,106 @@ +package schema + +#PipelineStage: { + cri?: { + max_partial_lines?: int + max_partial_line_size?: int + max_partial_line_size_truncate?: bool + } + + decolorize?: null + + drop?: { + source?: [...string] | string + separator?: string + expression?: string + value?: string + older_than?: string + longer_than?: string | int + drop_counter_reason?: string + } + + json?: { + expressions?: [string]: string + source?: string + drop_malformed?: bool + } + + labelallow?: [...string] + + labeldrop?: [...string] + + labels?: [string]: string + + limit?: { + rate?: int + burst?: int + by_label_name?: string + max_distinct_labels?: int + drop?: bool + } + + logfmt?: { + mapping?: [string]: string + source?: string + } + + match?: { + selector: string + pipeline_name?: string + action?: "keep" | "drop" + drop_counter_reason?: string + stages?: [#PipelineStage] + } + + metrics?: [string]: #CounterMetric | #GaugeMetric | #HistogramMetric + + multiline?: { + firstline: string + max_wait_time?: string + max_lines?: int + } + + output?: { + source: string + } + + pack?: { + labels: [string] + ingest_timestamp?: bool + } + + regex?: { + expression: string + source?: string + } + + replace?: { + expression: string + source?: string + replace?: string + } + + sampling?: { + rate?: float + } + + static_labels?: [string]: string + + template?: { + source: string + template: string + } + + tenant?: { + label?: string + source?: string + value?: string + } + + timestamp?: { + source: string + format: string + fallback_formats?: [...string] + location?: string + } +} diff --git a/app/promtail/schema/schema.cue b/app/promtail/schema/schema.cue new file mode 100644 index 0000000..41f72fa --- /dev/null +++ b/app/promtail/schema/schema.cue @@ -0,0 +1,62 @@ +package schema + +#Client: { + url: string + tls_config?: { + ca_file?: string + cert_file?: string + key_file?: string + server_name?: string + } +} + +#CommonScrapeConfig: { + labels?: [string]: string +} + +#JournalScrapeConfig: { + #CommonScrapeConfig + json: bool | *false +} + +#KubernetesScrapeConfig: { + #CommonScrapeConfig + api_server?: string + role: "node" | "service" | "pod" | "endpoints" | "ingress" +} + +#RelabelConfig: { + source_labels?: [...string] + separator?: string + target_label?: string + regex?: string + modulus?: int + replacement?: string + action?: "replace" | "keep" | "drop" | "hashmod" | "labelmap" | + "labeldrop" | "labelkeep" +} + +#ScrapeConfig: { + job_name: string + journal?: #JournalScrapeConfig + static_configs?: [...#CommonScrapeConfig] + kubernetes_sd_configs?: [...#KubernetesScrapeConfig] + pipeline_stages?: [...#PipelineStage] + relabel_configs?: [...#RelabelConfig] +} + +#PromtailConfig: { + server: { + http_listen_port: int | *9080 + grpc_listen_port: int | *0 + enable_runtime_reload: bool | *true + } + + clients: [...#Client] + + positions: { + filename: string | *"/var/lib/promtail/positions" + } + + scrape_configs: [...#ScrapeConfig] +} diff --git a/app/promtail/templates.cue b/app/promtail/templates.cue new file mode 100644 index 0000000..4930503 --- /dev/null +++ b/app/promtail/templates.cue @@ -0,0 +1,33 @@ +package promtail + +import "du5t1n.me/cfg/base/schema/instructions" + +templates: [...instructions.#RenderInstruction] & [ + { + template: "promtail/ca.crt" + dest: "/etc/promtail/ca.crt" + hooks: { + changed: [{run: "systemctl try-restart promtail"}] + } + }, + { + template: "promtail/config.yml" + dest: "/etc/promtail/config.yml" + hooks: { + changed: [{run: "systemctl try-restart promtail"}] + } + }, + { + template: "promtail/promtail.container" + dest: "/etc/containers/systemd/promtail.container" + hooks: { + changed: [ + { + run: "systemctl daemon-reload" + immediate: true + }, + {run: "systemctl restart promtail"}, + ] + } + }, +] diff --git a/env/prod/promtail.cue b/env/prod/promtail.cue new file mode 100644 index 0000000..13c2f3e --- /dev/null +++ b/env/prod/promtail.cue @@ -0,0 +1,86 @@ +package prod + +import "encoding/yaml" + +import "du5t1n.me/cfg/app/promtail/schema" + +promtail: { + scrape: { + journal: schema.#ScrapeConfig & { + job_name: "journal" + journal: { + labels: { + job: "systemd-journal" + } + } + relabel_configs: [ + { + source_labels: ["__journal__hostname"] + target_label: "hostname" + }, + { + source_labels: ["__journal__systemd_unit"] + target_label: "unit" + }, + { + source_labels: ["__journal_syslog_identifier"] + target_label: "syslog_identifier" + }, + { + source_labels: ["__journal_priority"] + target_label: "priority" + }, + { + source_labels: ["__journal_message_id"] + target_label: "message_id" + }, + { + source_labels: ["__journal__comm"] + target_label: "command" + }, + { + source_labels: ["__journal__transport"] + target_label: "transport" + }, + ] + } + } + + ca: """ + -----BEGIN CERTIFICATE----- + MIIBgTCCATOgAwIBAgIUTf/ZBSJEi8IQb8Ndoxp4/tHB/lcwBQYDK2VwMEAxCzAJ + BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxFzAVBgNVBAMMDkRD + SCBSb290IENBIFIzMB4XDTI0MDIxNzIwMjkzNloXDTM0MDIxNzIwMjkzNlowQDEL + MAkGA1UEBhMCVVMxGDAWBgNVBAoMD0R1c3RpbiBDLiBIYXRjaDEXMBUGA1UEAwwO + RENIIFJvb3QgQ0EgUjMwKjAFBgMrZXADIQDORylVcWcxwGDJvsJIc2NctfNfDaIU + T6mLebahKdshaKM/MD0wHQYDVR0OBBYEFLZoxAHBvWqbLWMga/DAAlG9ido5MA8G + A1UdEwEB/wQFMAMBAf8wCwYDVR0PBAQDAgEGMAUGAytlcANBANLV79joVd9s9bmL + 0a91HqvOotOnN/416Ek4UTl95jIqy/TvTfRjXX56wSALXqP1iYQM5i3zk3gVEhh4 + DaY+6wQ= + -----END CERTIFICATE----- + """ + + config: schema.#PromtailConfig & { + clients: [ + { + url: "https://loki.pyrocufflink.blue:3100/loki/api/v1/push" + tls_config: { + ca_file: "/etc/promtail/ca.crt" + } + }, + ] + } +} + +let Marshal = yaml.Marshal + +#promtail: { + #scrape: [...schema.#ScrapeConfig] | *[promtail.scrape.journal] + ca: string | *promtail.ca + + config: promtail.config & { + scrape_configs: #scrape + } + + yaml: Marshal(config) +} diff --git a/host/loki0.pyrocufflink.blue.cue b/host/loki0.pyrocufflink.blue.cue index 81409b1..ebc0c6e 100644 --- a/host/loki0.pyrocufflink.blue.cue +++ b/host/loki0.pyrocufflink.blue.cue @@ -5,6 +5,8 @@ import ( ssh: prod.ssh sudo: prod.sudo +promtail: prod.#promtail + fetchcert: prod.fetchcert.loki & { token: """ -----BEGIN AGE ENCRYPTED FILE----- diff --git a/instructions/loki0.pyrocufflink.blue.cue b/instructions/loki0.pyrocufflink.blue.cue index 0c5cbd0..7c7e958 100644 --- a/instructions/loki0.pyrocufflink.blue.cue +++ b/instructions/loki0.pyrocufflink.blue.cue @@ -3,6 +3,7 @@ import ( "du5t1n.me/cfg/app/collectd" "du5t1n.me/cfg/app/fetchcert" + "du5t1n.me/cfg/app/promtail" "du5t1n.me/cfg/app/loki" "du5t1n.me/cfg/env/prod" ) @@ -12,4 +13,5 @@ render: list.Concat([ collectd.templates, fetchcert.templates, loki.templates, + promtail.templates, ]) diff --git a/templates/promtail/ca.crt b/templates/promtail/ca.crt new file mode 100644 index 0000000..33645b2 --- /dev/null +++ b/templates/promtail/ca.crt @@ -0,0 +1 @@ +{{ promtail.ca }} diff --git a/templates/promtail/config.yml b/templates/promtail/config.yml new file mode 100644 index 0000000..3b2e7bb --- /dev/null +++ b/templates/promtail/config.yml @@ -0,0 +1 @@ +{{ promtail.yaml -}} diff --git a/templates/promtail/promtail.container b/templates/promtail/promtail.container new file mode 100644 index 0000000..a80334f --- /dev/null +++ b/templates/promtail/promtail.container @@ -0,0 +1,23 @@ +# vim: set ft=systemd : +[Unit] +Description=Grafana Loki Promtail Agent +After=network-online.target +Wants=network-online.target + +[Service] +StateDirectory=%P + +[Container] +Image=docker.io/grafana/promtail:2.9.4 +Exec=-config.file=/etc/promtail/config.yml +Volume=%S/%P:/var/lib/promtail:rw,Z,U +Volume=/etc/machine-id:/etc/machine-id:ro +Volume=/etc/promtail:/etc/promtail:ro +Volume=/run/log:/run/log:ro +Volume=/var/log:/var/log:ro +PublishPort=9080:9080 +# container_t is not allowed to read var_log_t / syslogd_var_run_t +SecurityLabelDisable=true + +[Install] +WantedBy=multi-user.target From ae948489e3e7d0bc2c04696ae7d500d83f2143e3 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sun, 18 Feb 2024 10:40:21 -0600 Subject: [PATCH 5/9] Deploy Promtail to all non-Kubernetes nodes All the stand-alone FCOS hosts now have Promtail running, forwarding _systemd_ journal messages to Grafana Loki. The Kubernetes nodes will have Promtail deployed as a Kubernetes pod. I would really like to come up with a way to define variables for groups of hosts, so that I do not have to add `promtail: prod.#promtail` to every host's values file individually... --- host/nut0.pyrocufflink.blue.cue | 13 +++++++------ host/nvr1.pyrocufflink.blue.cue | 9 +++++---- host/serial1.pyrocufflink.blue.cue | 2 ++ instructions/nut0.pyrocufflink.blue.cue | 4 +++- instructions/nvr1.pyrocufflink.blue.cue | 4 +++- instructions/serial1.pyrocufflink.blue.cue | 4 +++- 6 files changed, 23 insertions(+), 13 deletions(-) diff --git a/host/nut0.pyrocufflink.blue.cue b/host/nut0.pyrocufflink.blue.cue index 6af1c05..434b0e1 100644 --- a/host/nut0.pyrocufflink.blue.cue +++ b/host/nut0.pyrocufflink.blue.cue @@ -4,15 +4,16 @@ import ( "du5t1n.me/cfg/env/prod" ) +ssh: prod.ssh +sudo: prod.sudo + +promtail: prod.#promtail + +collectd: prod.collectd + nut: prod.nut nut: monitor: prod.#nut_monitor & { #server: "localhost" #username: "upsmon" } - -collectd: prod.collectd - -ssh: prod.ssh - -sudo: prod.sudo diff --git a/host/nvr1.pyrocufflink.blue.cue b/host/nvr1.pyrocufflink.blue.cue index c3a68b8..284310b 100644 --- a/host/nvr1.pyrocufflink.blue.cue +++ b/host/nvr1.pyrocufflink.blue.cue @@ -5,12 +5,13 @@ import ( "du5t1n.me/cfg/app/nut/schema" ) +ssh: prod.ssh +sudo: prod.sudo + +promtail: prod.#promtail + nut: monitor: schema.#NutMonitor nut: monitor: prod.#nut_monitor & { #username: "nvr1" } - -ssh: prod.ssh - -sudo: prod.sudo diff --git a/host/serial1.pyrocufflink.blue.cue b/host/serial1.pyrocufflink.blue.cue index 7c3864b..9801955 100644 --- a/host/serial1.pyrocufflink.blue.cue +++ b/host/serial1.pyrocufflink.blue.cue @@ -6,3 +6,5 @@ import ( ssh: prod.ssh sudo: prod.sudo + +promtail: prod.#promtail diff --git a/instructions/nut0.pyrocufflink.blue.cue b/instructions/nut0.pyrocufflink.blue.cue index cedb29c..88d43e8 100644 --- a/instructions/nut0.pyrocufflink.blue.cue +++ b/instructions/nut0.pyrocufflink.blue.cue @@ -5,13 +5,15 @@ import ( "du5t1n.me/cfg/app/collectd" "du5t1n.me/cfg/app/nut" + "du5t1n.me/cfg/app/promtail" "du5t1n.me/cfg/env/prod" ) render: list.Concat([ + prod.templates, collectd.templates, + promtail.templates, nut.templates, nut.monitor.templates, nut.collectd.templates, - prod.templates, ]) diff --git a/instructions/nvr1.pyrocufflink.blue.cue b/instructions/nvr1.pyrocufflink.blue.cue index d220b9c..05d6782 100644 --- a/instructions/nvr1.pyrocufflink.blue.cue +++ b/instructions/nvr1.pyrocufflink.blue.cue @@ -4,11 +4,13 @@ import ( "list" "du5t1n.me/cfg/app/nut" + "du5t1n.me/cfg/app/promtail" "du5t1n.me/cfg/env/prod" ) render: list.Concat([ + prod.templates, + promtail.templates, nut.sysusers.templates, nut.monitor.templates, - prod.templates, ]) diff --git a/instructions/serial1.pyrocufflink.blue.cue b/instructions/serial1.pyrocufflink.blue.cue index 98c008c..5bbad0b 100644 --- a/instructions/serial1.pyrocufflink.blue.cue +++ b/instructions/serial1.pyrocufflink.blue.cue @@ -4,10 +4,12 @@ import ( "list" "du5t1n.me/cfg/app/collectd" + "du5t1n.me/cfg/app/promtail" "du5t1n.me/cfg/env/prod" ) render: list.Concat([ - collectd.templates, prod.templates, + collectd.templates, + promtail.templates, ]) From 5e10f2c1e7f731e22ef8ae1682da458103a62f0d Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sun, 18 Feb 2024 12:01:27 -0600 Subject: [PATCH 6/9] promtail: Increase start timeout The Promtail container image is pretty big, so it takes quite some time to pull on a slow machine like a Raspberry Pi. Let's increase the startup timeout so the service is less likely to fail while the image is still being pulled. --- templates/promtail/promtail.container | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/promtail/promtail.container b/templates/promtail/promtail.container index a80334f..0e35623 100644 --- a/templates/promtail/promtail.container +++ b/templates/promtail/promtail.container @@ -6,6 +6,7 @@ Wants=network-online.target [Service] StateDirectory=%P +TimeoutStartSec=7m [Container] Image=docker.io/grafana/promtail:2.9.4 From 878ff7acb5b28af64aaeb0ef8841c85ddc0a3cd8 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Tue, 20 Feb 2024 07:25:30 -0600 Subject: [PATCH 7/9] loki: Deploy Caddy in front of Loki Grafana Loki explicitly eschews built-in authentication. In fact, its [documentation][0] states: > Operators are expected to run an authenticating reverse proxy in front > of your services. While I don't really want to require authentication for agents sending logs, I definitely want to restrict querying and viewing logs to trusted users. There are _many_ reverse proxy servers available, and normally I would choose _nginx_. In this case, though, I decided to try Caddy, mostly because of its built-in ACME support. I wasn't really happy with how the `fetchcert` system turned out, particularly using the Kubernetes API token for authentication. Since the token will eventually expire, it will require manual intervention to renew, thus mostly defeating the purpose of having an auto-renewing certificate. So instead of using _cert-manager_ to issue the certificate and store it in Kubernetes, and then having `fetchcert` download it via the Kubernetes API, I set up _step-ca_ to handle issuing the certificate directly to the server. When Caddy starts up, it contacts _step-ca_ via ACME and handles the challenge verification automatically. Further, it will automatically renew the certificate as necessary, again using ACME. I didn't spend a lot of time optimizing the Caddy configuration, so there's some duplication there (i.e. the multiple `reverse_proxy` statements), but the configuration works as desired. Clients may provide a certificate, which will be verified against the trusted issuer CA. If the certificate is valid, the client may access any Loki resource. Clients that do not provide a certificate can only access the ingestion path, as well as the "ready" and "metrics" resources. [0]: https://grafana.com/docs/loki/latest/operations/authentication/ --- app/loki/templates.cue | 34 ++++++++++++++++++++++++ env/prod/loki.cue | 34 ++++++++++++++++++++++++ host/loki0.pyrocufflink.blue.cue | 30 +-------------------- instructions/loki0.pyrocufflink.blue.cue | 2 -- templates/loki/Caddyfile | 30 +++++++++++++++++++++ templates/loki/caddy-acme-ca.crt | 1 + templates/loki/caddy-client-ca.crt | 1 + templates/loki/caddy.container | 22 +++++++++++++++ templates/loki/config.yml | 4 +-- templates/loki/loki.container | 2 +- 10 files changed, 125 insertions(+), 35 deletions(-) create mode 100644 env/prod/loki.cue create mode 100644 templates/loki/Caddyfile create mode 100644 templates/loki/caddy-acme-ca.crt create mode 100644 templates/loki/caddy-client-ca.crt create mode 100644 templates/loki/caddy.container diff --git a/app/loki/templates.cue b/app/loki/templates.cue index a202eaf..401d5f9 100644 --- a/app/loki/templates.cue +++ b/app/loki/templates.cue @@ -23,4 +23,38 @@ templates: [...instructions.#RenderInstruction] & [ ] } }, + { + template: "loki/caddy-client-ca.crt" + dest: "/etc/caddy/client-ca.crt" + hooks: { + changed: [{run: "systemctl try-reload-or-restart caddy"}] + } + }, + { + template: "loki/caddy-acme-ca.crt" + dest: "/etc/caddy/acme-ca.crt" + hooks: { + changed: [{run: "systemctl try-reload-or-restart caddy"}] + } + }, + { + template: "loki/Caddyfile" + dest: "/etc/caddy/Caddyfile" + hooks: { + changed: [{run: "systemctl try-reload-or-restart caddy"}] + } + }, + { + template: "loki/caddy.container" + dest: "/etc/containers/systemd/caddy.container" + hooks: { + changed: [ + { + run: "systemctl daemon-reload" + immediate: true + }, + {run: "systemctl restart caddy"}, + ] + } + }, ] diff --git a/env/prod/loki.cue b/env/prod/loki.cue new file mode 100644 index 0000000..e700645 --- /dev/null +++ b/env/prod/loki.cue @@ -0,0 +1,34 @@ +package prod + +loki: caddy: { + acme_ca: """ + -----BEGIN CERTIFICATE----- + MIICTzCCAgGgAwIBAgIUDNTFsSYYl8xsEcg9kTatxvOSkmUwBQYDK2VwMEAxCzAJ + BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxFzAVBgNVBAMMDkRD + SCBSb290IENBIFIzMB4XDTI0MDIxNzIwMjk0M1oXDTI1MDIxNzIwMjk0M1owOzEL + MAkGA1UEBhMCVVMxGDAWBgNVBAoMD0R1c3RpbiBDLiBIYXRjaDESMBAGA1UEAwwJ + RENIIENBIFIzMCowBQYDK2VwAyEA50stJ8iW6/f+uECPxAJwpSfQDRQg4/AgKJY2 + lpd3uNijggEQMIIBDDAdBgNVHQ4EFgQUtiqtFaZZ/c4IfWXV5SjJIOPbmoowHwYD + VR0jBBgwFoAUtmjEAcG9apstYyBr8MACUb2J2jkwEgYDVR0TAQH/BAgwBgEB/wIB + ADALBgNVHQ8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMEwG + CCsGAQUFBwEBBEAwPjA8BggrBgEFBQcwAoYwaHR0cHM6Ly9kdXN0aW4uaGF0Y2gu + bmFtZS9kY2gtY2EvZGNoLXJvb3QtY2EuY3J0MDwGA1UdHwQ1MDMwMaAvoC2GK2h0 + dHBzOi8vZHVzdGluLmhhdGNoLm5hbWUvZGNoLWNhL2RjaC1jYS5jcmwwBQYDK2Vw + A0EAACaKAJAKejpFXQV+mgPdDXaylvakc4rCEs1pFhPXbbMMGflNOeiiy+c+aMwt + yfObaZ8/YiXxCSjL6/KzRSSjAQ== + -----END CERTIFICATE----- + """ + client_ca: """ + -----BEGIN CERTIFICATE----- + MIIBlDCCAUagAwIBAgIUGNZ/ASP8F2ytev3YplTk4jA5a2EwBQYDK2VwMEgxCzAJ + BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxDTALBgNVBAsMBExv + a2kxEDAOBgNVBAMMB0xva2kgQ0EwHhcNMjQwMjIwMTUwMTQxWhcNMzQwMjIwMTUw + MTQxWjBIMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPRHVzdGluIEMuIEhhdGNoMQ0w + CwYDVQQLDARMb2tpMRAwDgYDVQQDDAdMb2tpIENBMCowBQYDK2VwAyEAnmMawEIo + WfzFaLgpSiaPD+DHg28NHknMFcs7XpyTM9CjQjBAMB0GA1UdDgQWBBTFth3c4S/f + y0BphQy9SucnKN2pLzASBgNVHRMBAf8ECDAGAQH/AgEAMAsGA1UdDwQEAwIBBjAF + BgMrZXADQQCn0JWERsXdJA4kMM45ZXhVgAciwLNQ8ikoucsJcbWBp7bSMjcMVi51 + I+slotQvQES/vfqp/zZFNl7KKyeeQ0sD + -----END CERTIFICATE----- + """ +} diff --git a/host/loki0.pyrocufflink.blue.cue b/host/loki0.pyrocufflink.blue.cue index ebc0c6e..40c2430 100644 --- a/host/loki0.pyrocufflink.blue.cue +++ b/host/loki0.pyrocufflink.blue.cue @@ -7,32 +7,4 @@ sudo: prod.sudo promtail: prod.#promtail -fetchcert: prod.fetchcert.loki & { - token: """ - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtNTZzeW9XeWoycVdQa092 - N0VYL2grR0lLY1c4QXl2VHI3NmsxM253UlNVCmZLbFZWakJGVG9WakkyYmpJL1VR - YmVQQXRCTlhrQk9UYUE5UkRFZUlwNlkKLS0tIGg4R25ZaVhUU1BFVjdac2NqMVpQ - QmZRTndBalZndVF0VFpxdHBRemhNS1EKrNZG179fh2aS/3FOaM1xCHRG4uOt5jyx - 1m5h3Q9y2u7EbcbZHLIZR3wkQfsfscK1PS0+H0NiYAgh9u2L2kdhcLcesb3fhmSy - svHzW2q1ZkJ8DSwH3xCRBuKmH4Q172NcVUPzI39CgsI5SkqZdKjWnK9JJAs43Ihr - cM90hUN+5t50byUSzwTCmNY4xVW3N/pWMfrethCYk9E8cXts/L3A3EpgpIi3qrKn - gj2VfrvpHAWVcggX1rZVFlQwBg4LnPWMNztl5VRYIvwfJghykEjMlzkysLm3Q2is - /w+kthpBzYAvI4c1Tfx3/uMRVcWnmUgz15viKlqohVaAl9PHQ2y/te9w9D5ZtcYs - D33hfA7Aux9t18WJ/ru09rEJl649Al7ZxQd73upf9QrWGzkX4luHO85n8CBmcsuh - +ZcM1HMLiuxGCW6xyq66Eg6t/1pfPWGZtLCsFh4SRgJ6Uuq14FyU32Pkulq+yEMg - Sq2ZRUXU+e3M6/HcUhb+QQUTQF1wPHyEukUlecLGDd3i+xpjOrL5Eg7LjKVAv8Yj - 8U1yiYjgRHfdkvT27RJC/rxuf674vU8H8na3jGXrPARMqq4L4B0XkUzclJZMzSPC - cSTaEIgb5OpfWmMb4uC0p76vHYhr4XX3iIVpivfxaDLAgyx06D4/oXALcgjcCHWY - /7m5t8MbIGqluqcJLYRhSQ+G/aWiyZG3zlgRfpOIyVzQHwQwGf2CLh6ygv9n5cWP - Gr0ZfcyVps734gVsDNqZ3vTy4nxjTueUiUpNqRaznzxT/z7Mq9/i0s1aoWBef0PV - MZL0jxyMeQUfRf0DdP/iPqkTU5hxw8/yqwuu2i3TJImVQ8ga8O3InyvN577mPihE - EqFjRl1jZr+Uip0+SPz+CSLIgBJ8rpAo/HTpue6Oe88rYtC0437YQtcWpB3rnARD - uggtP70SfvS7FWFCbYy7nxZrUcDMloD5gcIYNobkWQZhGdGvXDGVxB/FT8Rg6tAU - EOpaSSc3wOmHpnB6qCyCJ45mb6HwRCGoZmxaG/5uWreys0R8AJsMIq8vFVAS3sDo - EONNYMWtlAZg8XOZcSgSnKpUF5VWlt+3HLkpwQkTBq3SvjvMd6shybPVGVNxMwbU - a2gey9Kv4lq8Suvvrn31DeYErGwUYy0qMwTL1a4Q8I08kMg6lqqaPotIC63RSlUu - SEoarQ== - -----END AGE ENCRYPTED FILE----- - """ -} +loki: prod.loki diff --git a/instructions/loki0.pyrocufflink.blue.cue b/instructions/loki0.pyrocufflink.blue.cue index 7c7e958..8ab3f40 100644 --- a/instructions/loki0.pyrocufflink.blue.cue +++ b/instructions/loki0.pyrocufflink.blue.cue @@ -2,7 +2,6 @@ import ( "list" "du5t1n.me/cfg/app/collectd" - "du5t1n.me/cfg/app/fetchcert" "du5t1n.me/cfg/app/promtail" "du5t1n.me/cfg/app/loki" "du5t1n.me/cfg/env/prod" @@ -11,7 +10,6 @@ import ( render: list.Concat([ prod.templates, collectd.templates, - fetchcert.templates, loki.templates, promtail.templates, ]) diff --git a/templates/loki/Caddyfile b/templates/loki/Caddyfile new file mode 100644 index 0000000..66c8836 --- /dev/null +++ b/templates/loki/Caddyfile @@ -0,0 +1,30 @@ +loki.pyrocufflink.blue { + tls { + client_auth { + mode verify_if_given + trusted_ca_cert_file /etc/caddy/client-ca.crt + } + } + @anonymous { + expression {tls_client_subject} == null + } + handle @anonymous { + route /loki/api/v1/push { + reverse_proxy 127.0.0.1:3100 + } + route /metrics { + reverse_proxy 127.0.0.1:3100 + } + route /ready { + reverse_proxy 127.0.0.1:3100 + } + respond 403 + } + handle { + reverse_proxy 127.0.0.1:3100 + } + tls loki@pyrocufflink.blue { + ca https://ca.pyrocufflink.blue:32599/acme/acme/directory + ca_root /etc/caddy/acme-ca.crt + } +} diff --git a/templates/loki/caddy-acme-ca.crt b/templates/loki/caddy-acme-ca.crt new file mode 100644 index 0000000..e5c3239 --- /dev/null +++ b/templates/loki/caddy-acme-ca.crt @@ -0,0 +1 @@ +{{ loki.caddy.acme_ca }} diff --git a/templates/loki/caddy-client-ca.crt b/templates/loki/caddy-client-ca.crt new file mode 100644 index 0000000..2ad0b49 --- /dev/null +++ b/templates/loki/caddy-client-ca.crt @@ -0,0 +1 @@ +{{ loki.caddy.client_ca }} diff --git a/templates/loki/caddy.container b/templates/loki/caddy.container new file mode 100644 index 0000000..d695785 --- /dev/null +++ b/templates/loki/caddy.container @@ -0,0 +1,22 @@ +[Unit] +Description=Caddy web server +After=network-online.target +Wants=network-online.target + +[Container] +Image=docker.io/library/caddy:2 +Volume=/etc/caddy:/etc/caddy:ro +Volume=/var/lib/caddy/config:/config/caddy:rw,z +Volume=/var/lib/caddy/data:/data/caddy:rw,z +ReadOnly=yes +ReadOnlyTmpfs=yes +Network=host +AddCapability=CAP_NET_BIND_SERVICE +DropCapability=all + +[Service] +StateDirectory=%N/data %N/config +ExecReload=/usr/bin/podman exec systemd-%N caddy reload -c /etc/caddy/Caddyfile + +[Install] +WantedBy=multi-user.target diff --git a/templates/loki/config.yml b/templates/loki/config.yml index 0ddb22d..6359871 100644 --- a/templates/loki/config.yml +++ b/templates/loki/config.yml @@ -2,9 +2,7 @@ auth_enabled: false server: http_listen_port: 3100 - http_tls_config: - cert_file: /etc/loki/server.cer - key_file: /etc/loki/server.key + http_listen_address: 127.0.0.1 grpc_listen_port: 9096 common: diff --git a/templates/loki/loki.container b/templates/loki/loki.container index 98956bc..da8985a 100644 --- a/templates/loki/loki.container +++ b/templates/loki/loki.container @@ -13,7 +13,7 @@ Image=docker.io/grafana/loki:2.9.4 Exec=-config.file=/etc/loki/config.yml Volume=%S/%P:/var/lib/loki:rw,Z,U Volume=/etc/loki:/etc/loki:ro -PublishPort=3100:3100 +Network=host [Install] WantedBy=multi-user.target From cdd6a62b5decf1eaa75faaf9a4e6a22bc227f28f Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Tue, 20 Feb 2024 07:25:58 -0600 Subject: [PATCH 8/9] promtail: Update loki port With Loki behind a reverse proxy now, clients access it using the default HTTPS port (443). --- env/prod/promtail.cue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/prod/promtail.cue b/env/prod/promtail.cue index 13c2f3e..9dae255 100644 --- a/env/prod/promtail.cue +++ b/env/prod/promtail.cue @@ -63,7 +63,7 @@ promtail: { config: schema.#PromtailConfig & { clients: [ { - url: "https://loki.pyrocufflink.blue:3100/loki/api/v1/push" + url: "https://loki.pyrocufflink.blue/loki/api/v1/push" tls_config: { ca_file: "/etc/promtail/ca.crt" } From 01d8f7043be6dc123d049fd4eb172e00d1d81bdb Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Tue, 20 Feb 2024 20:11:53 -0600 Subject: [PATCH 9/9] loki: Require X-Grafana-User HTTP header I discovered today that if anonymous Grafana users have Viewer permission, they can use the Datasource API to make arbitrary queries to any backend, even if they cannot access the Explore page directly. This is documented ([issue #48313][0]) as expected behavior. I don't really mind giving anonymous access to the Victoria Metrics datasource, but I definitely don't want anonymous users to be able to make Loki queries and view log data. Since Grafana Datasource Permissions is limited to Grafana Enterprise and not available in the open source version of Grafana, the official recommendation from upstream is to use a separate Organization for the Loki datasource. Unfortunately, this would preclude having dashboards that have graphs from both data sources. Although I don't have any of those right now, I like the idea and may build some eventually. Fortunately, I discovered the `send_user_header` Grafana configuration option. With this enabled, Grafana will send an `X-Grafana-User` header with the username of the user on whose behalf it is making a request to the backend. If the user is not logged in, it does not send the header. Thus, we can detect the presence of this header on the backend and refuse to serve query requests if it is missing. [0]: https://github.com/grafana/grafana/issues/48313 --- templates/loki/Caddyfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/loki/Caddyfile b/templates/loki/Caddyfile index 66c8836..4356522 100644 --- a/templates/loki/Caddyfile +++ b/templates/loki/Caddyfile @@ -8,6 +8,9 @@ loki.pyrocufflink.blue { @anonymous { expression {tls_client_subject} == null } + @grafana { + header X-Grafana-User * + } handle @anonymous { route /loki/api/v1/push { reverse_proxy 127.0.0.1:3100 @@ -20,7 +23,7 @@ loki.pyrocufflink.blue { } respond 403 } - handle { + handle @grafana { reverse_proxy 127.0.0.1:3100 } tls loki@pyrocufflink.blue {