diff --git a/group_vars/grafana/main.yml b/group_vars/grafana/main.yml deleted file mode 100644 index 4792041..0000000 --- a/group_vars/grafana/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -nginx_redirect_http_https: true -grafana_domain: grafana.pyrocufflink.blue -grafana_anonymous_enabled: true -grafana_ldap_enabled: true -grafana_http_addr: '[::1]' -grafana_ldap_host: pyrocufflink.blue -grafana_ldap_ssl: true -grafana_ldap_start_tls: true -grafana_ldap_bind_dn: CN=svc.grafana,CN=Users,DC=pyrocufflink,DC=blue -grafana_ldap_search_filter: (sAMAccountName=%s) -grafana_ldap_search_base_dns: -- DC=pyrocufflink,DC=blue -grafana_ldap_attr_username: sAMAccountName -grafana_ldap_attr_email: mail -grafana_ldap_group_mappings: -- group_dn: CN=Grafana Admins,CN=Users,DC=pyrocufflink,DC=blue - org_role: Admin - grafana_admin: true -- group_dn: '*' - org_role: Viewer diff --git a/group_vars/grafana/secrets b/group_vars/grafana/secrets deleted file mode 100644 index dd9f6a2..0000000 --- a/group_vars/grafana/secrets +++ /dev/null @@ -1,9 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -35333639333036633432663463313536316163366130626436623962363466616234306462333239 -3338353961306664326137343262373565643234666238340a316163616236373636323836366363 -38653732643539666465323537613634376238343833313063623964363862633939376164313961 -3837366130386631370a323131333561353638353738393835346533393563393132323763316663 -37353735346438346435336465333565353866323434346131316434366362343964613933316530 -31633933346263323262323631623138326337343132383035613634383233313963663530333636 -33376232383937336463353837346264316537396431376636336264613439613538613038633637 -63316336313661386135 diff --git a/group_vars/metricspi/alertmanager.yml b/group_vars/metricspi/alertmanager.yml deleted file mode 100644 index 52ad2b7..0000000 --- a/group_vars/metricspi/alertmanager.yml +++ /dev/null @@ -1,24 +0,0 @@ -alertmanager_config: - global: - smtp_from: prometheus@pyrocufflink.blue - smtp_smarthost: mail.pyrocufflink.blue:25 - smtp_require_tls: false - - route: - receiver: default-email - group_by: - - ... - routes: - - receiver: default-email - group_wait: 1m - group_by: - - alertname - repeat_interval: 120h - match: - job: homeassistant - - receivers: - - name: default-email - email_configs: - - to: gyrfalcon@ebonfire.com - send_resolved: true diff --git a/group_vars/metricspi/alerts.yml b/group_vars/metricspi/alerts.yml deleted file mode 100644 index aa63e32..0000000 --- a/group_vars/metricspi/alerts.yml +++ /dev/null @@ -1,124 +0,0 @@ -vmalert_rules: - groups: - - name: default alert - rules: - - alert: DiskUsage - expr: >- - sum(collectd_df_df_complex{type!="free"}) by (instance, df) / sum(collectd_df_df_complex{df!="var-log", df!="var-lib-frigate"}) by (instance, df) > .75 - or sum(collectd_df_df_complex{type!="free"}) by (instance, df) / sum(collectd_df_df_complex{df="var-log"}) by (instance, df) > .95 - or sum(collectd_df_df_complex{type!="free"}) by (instance, df) / sum(collectd_df_df_complex{df="var-lib-frigate"}) by (instance, df) > .90 - for: 2h - - alert: TheWebsiteIsDown - expr: >- - probe_success{job="websites"} == 0 - for: 10m - - alert: Missing Metrics - expr: >- - up{instance!~"vmhost.*"} == 0 - for: 10m - - alert: NUT is offline - expr: >- - absent(collectd_nut_percent) - - - name: Bitwarden - rules: - - alert: vaultwarden is not running - expr: >- - collectd_processes_ps_count_processes{processes="vaultwarden"} < 1 - for: 5m - - - name: Active Directory - rules: - - alert: samba is not running - expr: >- - collectd_processes_ps_count_processes{processes=~"samba|smbd|winbindd|krb5kdc"} < 1 - for: 5m - - - name: Graylog - rules: - - alert: unprocessed messages - expr: >- - org_graylog2_journal_entries_uncommitted > 100 - for: 1h - - - name: mdraid - rules: - - alert: mdraid missing disk - expr: collectd_md_md_disks{type="missing", instance!~"burp.*"} != 0 - - alert: mdraid failed disk - expr: collectd_md_md_disks{type="failed"} != 0 - - - name: BURP - rules: - - alert: no recent backups - expr: absent(burp_client_last_backup_timestamp) - for: 8h - annotations: - summary: No clients have been backed up recently - description: >- - This alert indicates that NO clients have been backed up within the - last day. There is likely a problem with the BURP server. - - alert: missed client backup - expr: - time() - (burp_client_last_backup_timestamp > now() - 86400 * 90) > 86400 * 2 - for: 3h - annotations: - summary: A client has not backed up today - description: >- - A client has not been backed up for more than a day. This may be - because the client is offline, or because the backup process has - failed. Clients that have not been backed up for more than 90 days - will not trigger this alert. - - alert: disks need swapped - expr: - time() - tlast_change_over_time( - ( - collectd_md_md_disks{instance="burp1.pyrocufflink.blue", type="active"} - or last_over_time(collectd_md_md_disks{instance="burp1.pyrocufflink.blue", type="active"})[1d] - )[90d] - ) > 86400 * 30 - annotations: - summary: The disks in the BURP array need swapped - description: >- - The disks in the BURP RAID-1 (mirror) array should be swapped - periodically. One disk should be online and mounted while the other - is stored in the fireproof safe. Switching them ensures that even if - something happens to the active disk, such as hardware failure, power - surge, fire, or accidental `rm -rf`, the offline disk is only out of - date by a few weeks. - - alert: disk needs archived - expr: - sum( - collectd_md_md_disks{instance="burp1.pyrocufflink.blue", type=~"missing|spare"} - ) < 1 - annotations: - summary: One of the disks in the BURP array should be archived - description: >- - The disks in the BURP RAID-1 (mirror) array should be swapped - periodically. One disk should be online and mounted while the other - is stored in the fireproof safe. All of the disks are currently - online; one needs to be disconnected and moved to the safe as soon as - possible. - - - name: certificates - rules: - - alert: certificate will expire soon - expr: - probe_ssl_last_chain_expiry_timestamp_seconds - time() < 29 * 86400 - annotations: - summary: A certificate will expire in less than 29 days - description: >- - Generally, certificates are renewed automatically, approximately 30 - days before their expiration (NotAfter) date. There may be a problem - with the certificate renewal process that prevented this certificate - from being renewed. - - alert: certificate will expire very soon - expr: - probe_ssl_last_chain_expiry_timestamp_seconds - time() < 14 * 86400 - annotations: - summary: A certificate will expire in less than 14 days - description: >- - Generally, certificates are renewed automatically, approximately 30 - days before their expiration (NotAfter) date. There is most likely a - problem with the certificate renewal process that prevented this - certificate from being renewed. diff --git a/group_vars/metricspi/blackbox.yml b/group_vars/metricspi/blackbox.yml deleted file mode 100644 index 7da1480..0000000 --- a/group_vars/metricspi/blackbox.yml +++ /dev/null @@ -1,45 +0,0 @@ -blackbox_modules: - icmp: - prober: icmp - timeout: 5s - - http: - prober: http - timeout: 5s - http: - method: GET - headers: - Accept-Language: en-US - Accept-Charset: utf-8 - Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8 - - tcp: - prober: tcp - timeout: 5s - - dns_recursive: - prober: dns - timeout: 5s - dns: - query_name: news.ycombinator.com - query_type: A - - dns_pyrocufflink: - prober: dns - timeout: 2s - dns: - query_name: pyrocufflink.blue - query_type: SOA - validate_answer_rrs: - fail_if_not_matches_regexp: - - 'pyrocufflink\.blue\.\t\d+\tIN\tSOA\tdc.+\.pyrocufflink\.blue.*' - - smtp: - prober: tcp - timeout: 5s - tcp: - query_response: - - expect: "^220 ([^ ]+) ESMTP (.+)$" - - send: 'EHLO prober\r' - - expect: '^250[ -]SMTPUTF8' - - send: 'QUIT\r' diff --git a/group_vars/metricspi/grafana.yml b/group_vars/metricspi/grafana.yml deleted file mode 100644 index d587be4..0000000 --- a/group_vars/metricspi/grafana.yml +++ /dev/null @@ -1,18 +0,0 @@ -grafana_allow_anonymous: true -grafana_allow_sign_up: false -grafana_allow_org_create: false -grafana_ldap: true -grafana_ldap_allow_sign_up: true -grafana_ldap_host: pyrocufflink.blue -grafana_ldap_use_ssl: true -#grafana_ldap_root_ca_cert: /etc/ssl/certs/dch-root-ca.crt -grafana_ldap_bind_dn: CN=svc.grafana,CN=Users,DC=pyrocufflink,DC=blue -grafana_ldap_bind_password: '{{ vault_grafana_ldap_bind_password }}' -grafana_ldap_search_filter: '(sAMAccountName=%s)' -grafana_ldap_base_dn: DC=pyrocufflink,DC=blue -grafana_ldap_group_mapping: -- group_dn: CN=Grafana Admins,CN=Users,DC=pyrocufflinke,DC=blue - role: Admin - grafana_admin: true -- group_dn: '*' - role: Viewer diff --git a/group_vars/metricspi/main.yml b/group_vars/metricspi/main.yml deleted file mode 100644 index 73b9dc2..0000000 --- a/group_vars/metricspi/main.yml +++ /dev/null @@ -1,6 +0,0 @@ -vm_config: null -victoria_metrics_server_name: metrics.pyrocufflink.blue -scrape_collectd_extra_targets: -- nvr1.pyrocufflink.blue -- k8s-aarch64-n0.pyrocufflink.blue -- k8s-aarch64-n1.pyrocufflink.blue diff --git a/group_vars/metricspi/scrape.yml b/group_vars/metricspi/scrape.yml deleted file mode 100644 index 041142f..0000000 --- a/group_vars/metricspi/scrape.yml +++ /dev/null @@ -1,251 +0,0 @@ -vmagent_scrape_configs: - -- job_name: vmagent - static_configs: - - targets: - - '[::1]:8429' - -- job_name: blackbox - metrics_path: /probe - params: - module: - - icmp - static_configs: - - targets: - - 1.1.1.1 - - 8.8.8.8 - - 9.9.9.9 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: '[::1]:9115' - -- job_name: brandon - scrape_interval: 5s - metrics_path: /probe - params: - module: - - icmp - static_configs: - - targets: - - 173.172.96.1 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: '[::1]:9115' - -- job_name: websites - scrape_interval: 5m - metrics_path: /probe - params: - module: - - http - static_configs: - - targets: - - http://dustin.hatch.name/ - - https://darkchestofwonders.us/ - - http://nratonpass.com/ - - http://pyrocufflink.net/ - - http://ebonfire.com/ - - http://chmod777.sh/ - - https://hatch.chat/_matrix/client/versions - - https://nextcloud.pyrocufflink.net/ - - https://bitwarden.pyrocufflink.blue/ - - https://git.pyrocufflink.blue/ - - https://jenkins.pyrocufflink.blue/login - - https://tabitha.biz/ - - https://dustinandtabitha.com/ - - https://hatchlearningcenter.org/ - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: '[::1]:9115' - -- job_name: graylog - scrape_interval: 1m - scheme: https - metrics_path: /api/plugins/org.graylog.plugins.metrics.prometheus/metrics - basic_auth: - username: >- - {{ vault_graylog_scrape_token }} - password: token - static_configs: - - targets: - - graylog.pyrocufflink.blue:443 - -- job_name: collectd - scrape_interval: 10s - honor_labels: true - static_configs: - - targets: - - gw1.pyrocufflink.blue - - vmhost0.pyrocufflink.blue - - vmhost1.pyrocufflink.blue - file_sd_configs: - - files: - - /etc/prometheus/scrape-collectd.yml - relabel_configs: - - source_labels: [__address__] - target_label: __address__ - replacement: '$1:9103' - -- job_name: homeassistant - scrape_interval: 1m - scheme: https - metrics_path: /api/prometheus - bearer_token: >- - {{ vault_homeassistant_scrape_token }} - static_configs: - - targets: - - homeassistant.pyrocufflink.blue - -- job_name: sambadc - scrape_interval: 1m - metrics_path: /probe - params: - module: - - tcp - dns_sd_configs: - - names: - - _ldap._tcp.pyrocufflink.blue - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: '[::1]:9115' - -- job_name: gitea - scrape_interval: 1m - scheme: https - static_configs: - - targets: - - git.pyrocufflink.blue - -- job_name: synapse - scrape_interval: 1m - metrics_path: /_synapse/metrics - static_configs: - - targets: - - matrix0.pyrocufflink.blue - relabel_configs: - - source_labels: [__address__] - target_label: instance - - source_labels: [__address__] - target_label: __address__ - replacement: '$1:9000' - -- job_name: dns_recursive - scrape_interval: 1m - metrics_path: /probe - params: - module: - - dns_recursive - static_configs: - - targets: - - 172.30.0.1 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: '[::1]:9115' - -- job_name: dns_pyrocufflink - scrape_interval: 1m - metrics_path: /probe - params: - module: - - dns_pyrocufflink - static_configs: - - targets: - - 172.30.0.10 - - 172.30.0.9 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: '[::1]:9115' - -- job_name: smtp - scrape_interval: 1m - metrics_path: /probe - params: - module: - - smtp - dns_sd_configs: - - names: - - mail.pyrocufflink.blue - type: A - port: 25 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: '[::1]:9115' - -- job_name: unifi - scrape_interval: 1m - static_configs: - - targets: - - unifi.pyrocufflink.blue:9130 - -- job_name: jenkins - scrape_interval: 1m - metrics_path: /prometheus/ - scheme: https - static_configs: - - targets: - - jenkins.pyrocufflink.blue - -- job_name: burp - scrape_interval: 270s - scrape_timeout: 30s - static_configs: - - targets: - - burp.pyrocufflink.blue:9645 - -- job_name: minio-backups - metrics_path: /minio/v2/metrics/cluster - scheme: https - static_configs: - - targets: - - burp.pyrocufflink.blue:9000 - -- job_name: kubernetes - scheme: https - tls_config: - ca_file: /etc/victoria-metrics/kube-root-ca.crt - static_configs: - - targets: - - kubernetes.pyrocufflink.blue:6443 - -- job_name: zincati - scrape_interval: 1m - metrics_path: /bridge?selector=zincati - static_configs: - - targets: - - k8s-aarch64-n0.pyrocufflink.blue - - k8s-aarch64-n1.pyrocufflink.blue - - nvr1.pyrocufflink.blue - relabel_configs: - - source_labels: [__address__] - target_label: instance - - source_labels: [__address__] - target_label: __address__ - replacement: '$1:9598' diff --git a/group_vars/metricspi/secrets b/group_vars/metricspi/secrets deleted file mode 100644 index 709dfcd..0000000 --- a/group_vars/metricspi/secrets +++ /dev/null @@ -1,24 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -33663232353935666561613765313939376662396331326332373133343437343663323761633066 -3361353962646561633066353632373139666562323863330a346339323831376664636366336332 -63333662303862633938643734366662623434633562383033663637336330306161613336346135 -3566386161363438660a393436353461366433643738313661656434303462376436626162623061 -30626361623737636337383336626331633332393233656263303731313239663838313635356432 -34333733636139633766316162333266613533636234303830326632303765616330663034636430 -63373866373662383438636563643833323765653666663837663162356464326332396138656233 -33326465313437343232316132613538626537333332613531613332343566626337613835313639 -61353464633239353863313561646235316466393762306238306538376531353831643930336231 -34643366633664633937656232316166393835333461363564303938623633653432316561323032 -65666330316534306564653731366632613934343966653034393661363035356639656662613339 -36373131346561343731666331623833633932373765356263363434666566616131663833656364 -32663136303838376537346636633363666630386339633564633662343035653737616439626163 -63363234613237656639333035396539626337323932626632666663393864613063306138323465 -34343761613637656631323938323465376535613461313231323737636235396535363234646437 -34633230333236326331333738323161376230373766393336643636376339396530316632363638 -65373530613565663236666561326539343033633230616561633538313632343036346261323134 -65663166353632656635653365366664313139396562336133656336396334363063653332393136 -32663838363237626562366662383035333762366432323734616633346334646365323733303763 -38663461333431613630303330343764633963646432363537616466626133346136623535656539 -39353536633364653563366466363338643730663866626531653164663232323663653366363266 -62363535363835336262646236346637633033353731666335373663663537356362656362626265 -643234623230343334656464633134326136 diff --git a/group_vars/metricspi/vmalert.yml b/group_vars/metricspi/vmalert.yml deleted file mode 100644 index bebc5c8..0000000 --- a/group_vars/metricspi/vmalert.yml +++ /dev/null @@ -1,9 +0,0 @@ -vmalert_datasource_url: http://[::1]:8428 -vmalert_notifier_url: http://[::1]:9093 -vmalert_remote_read_url: http://[::1]:8428 -vmalert_remote_write_url: http://[::1]:8428 -vmalert_external_url: https://grafana.pyrocufflink.blue -vmalert_external_alert_source: >- - {% raw -%} - explore?orgId=1&left=%7B"queries":%5B%7B"expr":"{{$expr|queryEscape}}"%7D%5D%7D - {%- endraw %} diff --git a/hosts b/hosts index 4111c53..3106279 100644 --- a/hosts +++ b/hosts @@ -1,18 +1,12 @@ [all:vars] ansible_python_interpreter=/usr/bin/python3 -[alertmanager:children] -metricspi - [aria2] file0.pyrocufflink.blue [bitwarden_rs] bw0.pyrocufflink.blue -[blackbox-exporter:children] -metricspi - [burp-client] bw0.pyrocufflink.blue cloud0.pyrocufflink.blue @@ -58,8 +52,6 @@ file0.pyrocufflink.blue [gitea] git0.pyrocufflink.blue -[grafana] - [graylog] [hassdb] @@ -87,8 +79,6 @@ k8s-amd64-n2.pyrocufflink.blue k8s-controller k8s-node -[metricspi] - [minio:children] burp-server @@ -167,17 +157,6 @@ matrix0.pyrocufflink.blue [unifi] unifi1.pyrocufflink.blue -[victoria-metrics] - -[victoria-metrics:children] -metricspi - -[vmagent:children] -victoria-metrics - -[vmalert:children] -metricspi - [vm-hosts] [wheelhost]