From 347cda74fd0d4e3a5129490427ed9fee63d3cb76 Mon Sep 17 00:00:00 2001
From: "Dustin C. Hatch" <dustin@hatch.name>
Date: Mon, 22 May 2023 21:21:08 -0500
Subject: [PATCH] metrics: Scrape metrics from Kubernetes API server

Kubernetes exports a *lot* of metrics in Prometheus format.  I am not
sure what all is there, yet, but apparently several thousand time series
were added.

To allow anonymous access to the metrics, I added this RoleBinding:

```yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups:
  - ""
  resources:
  - nodes/metrics
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get
```
---
 group_vars/metricspi/scrape.yml               |  8 ++++++++
 kube-root-ca.crt                              | 19 +++++++++++++++++++
 roles/vmagent/files/ca-certs/kube-root-ca.crt |  1 +
 roles/vmagent/tasks/deploy.yml                | 16 ++++++++++++++++
 4 files changed, 44 insertions(+)
 create mode 100644 kube-root-ca.crt
 create mode 120000 roles/vmagent/files/ca-certs/kube-root-ca.crt

diff --git a/group_vars/metricspi/scrape.yml b/group_vars/metricspi/scrape.yml
index b4c9c6d..098baf1 100644
--- a/group_vars/metricspi/scrape.yml
+++ b/group_vars/metricspi/scrape.yml
@@ -225,3 +225,11 @@ vmagent_scrape_configs:
   static_configs:
   - targets:
     - burp.pyrocufflink.blue:9000
+
+- job_name: kubernetes
+  scheme: https
+  tls_config:
+    ca_file: /etc/victoria-metrics/kube-root-ca.crt
+  static_configs:
+  - targets:
+    - kubernetes.pyrocufflink.blue:6443
diff --git a/kube-root-ca.crt b/kube-root-ca.crt
new file mode 100644
index 0000000..1952067
--- /dev/null
+++ b/kube-root-ca.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIC/jCCAeagAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
+cm5ldGVzMB4XDTIyMDgwMTAyNTUzM1oXDTMyMDcyOTAyNTUzM1owFTETMBEGA1UE
+AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMs6
+2PUOzIClsAgPv1Mn9CTwzSFMntAn7OppwK5BQ4E5Vd1yMjz3p0uA1ZINv1ORorG0
+mLl95C7y+CWUGPx+stHKQr/40sLGyypbX+AfjoPzHiDbIcbZEff8X5RwKqzmT9V7
+Yt29KewADod0z+fqNYa62MJDaUunfwaV8kKFU/WJM8IKv2eJxAtWzvK3iHAFhx0j
+Xo4TlyINL9V9UMKLf12w6CA3G41uZIBCN3G7aJEm++eGoMdrPZUXlbCpbSztO85/
+hbulVs+0hCIxWiI+mRmB5OoWlRYL4jA45oK/RtpEqSwZ95zlGNAChmH7rb0pTtNf
+N0/C2wKAEL4POLx9kscCAwEAAaNZMFcwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
+/wQFMAMBAf8wHQYDVR0OBBYEFHYActCjEWdtsA+Ju25gxJh/vaLQMBUGA1UdEQQO
+MAyCCmt1YmVybmV0ZXMwDQYJKoZIhvcNAQELBQADggEBAAfkYHecXUwyqvMSXmqr
+ETqEzDCBini14s89VDhaDHOXBID9TKMVyeePdEYcPAJz3wo8fbx/+TL37K6hEuo+
+7bUaamaumznsjg9L0Hth19GvuRKMXJlEpndRmE5K9hnaDLr94MLg9n1qGcEOt6tw
+O6X5qqHf9AuuL39vt1+kSw6PeZZFZNMDZ8BdiTssw4btjQ2bsWu0wSiOSz/F8iRf
+2vN5An5dheroDsFs4dZ9gnJ69TmqV1YqQxfRWqCxzfNJbgVm6AoBPwhL1JRuAU4N
+3nCNoM9n2tLFDojT4un1778UVU91PtcBVdM9Nq+RC2jhXIyLBqsEK0ofOqFYqj3F
+0EQ=
+-----END CERTIFICATE-----
diff --git a/roles/vmagent/files/ca-certs/kube-root-ca.crt b/roles/vmagent/files/ca-certs/kube-root-ca.crt
new file mode 120000
index 0000000..70b6c8f
--- /dev/null
+++ b/roles/vmagent/files/ca-certs/kube-root-ca.crt
@@ -0,0 +1 @@
+../../../../kube-root-ca.crt
\ No newline at end of file
diff --git a/roles/vmagent/tasks/deploy.yml b/roles/vmagent/tasks/deploy.yml
index 22ff28b..f9c8aa9 100644
--- a/roles/vmagent/tasks/deploy.yml
+++ b/roles/vmagent/tasks/deploy.yml
@@ -46,6 +46,22 @@
   - config
   - scrape-config
 
+- name: ensure additional ca certificates are installed
+  copy:
+    src: '{{ item }}'
+    dest: /etc/victoria-metrics/{{ item | basename }}
+    mode: u=rw,go=r
+    owner: root
+    group: root
+  with_fileglob:
+  - ca-certs/{{ inventory_hostname }}/*.crt
+  - ca-certs/*.crt
+  notify:
+  - reload vmagent
+  tags:
+  - cert
+  - scrape-config
+
 - name: ensure vmagent starts at boot
   service:
     name: vmagent