Monitoring server: add blackbox exporter

This is a prometheus exporter that allows checking IPv4 and IPv6
responses, among other things. This sets it up to make sure that the
site is responding over both IPv4 and IPv6, so that I can monitor and
set up an alert if either stops working.

salt/salt/prometheus/exporters/blackbox.yml

@@ -0,0 +1,14 @@
+modules:
+  site_ipv4:
+    prober: http
+    timeout: 5s
+    http:
+      preferred_ip_protocol: "ip4"
+      ip_protocol_fallback: false
+
+  site_ipv6:
+    prober: http
+    timeout: 5s
+    http:
+      preferred_ip_protocol: "ip6"
+      ip_protocol_fallback: false

salt/salt/prometheus/exporters/blackbox_exporter.sls

@@ -0,0 +1,35 @@
+# Download/extract and set up the blackbox exporter
+include:
+  - prometheus.user
+
+unpack-blackbox-exporter:
+  archive.extracted:
+    - name: /opt/prometheus_blackbox_exporter
+    - source:
+      - salt://prometheus/exporters/blackbox_exporter-0.16.0.linux-amd64.tar.gz
+      - https://github.com/prometheus/blackbox_exporter/releases/download/v0.16.0/blackbox_exporter-0.16.0.linux-amd64.tar.gz
+    - source_hash: sha256=52d3444a518ea01f220e08eaa53eb717ef54da6724760c925ab41285d0d5a7bd
+    - if_missing: /opt/prometheus_blackbox_exporter
+    - user: prometheus
+    - group: prometheus
+    - options: --strip-components=1
+    - enforce_toplevel: False
+
+/opt/prometheus_blackbox_exporter/blackbox.yml:
+  file.managed:
+    - source: salt://prometheus/exporters/blackbox.yml
+    - user: prometheus
+    - group: prometheus
+    - mode: 644
+
+/etc/systemd/system/prometheus_blackbox_exporter.service:
+  file.managed:
+    - source: salt://prometheus/exporters/prometheus_blackbox_exporter.service
+    - user: root
+    - group: root
+    - mode: 644
+
+prometheus-blackbox-exporter-service:
+  service.running:
+    - name: prometheus_blackbox_exporter
+    - enable: True

salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Prometheus Blackbox Exporter
+After=syslog.target network.target
+
+[Service]
+Type=simple
+RemainAfterExit=no
+WorkingDirectory=/opt/prometheus_blackbox_exporter
+User=prometheus
+Group=prometheus
+ExecStart=/opt/prometheus_blackbox_exporter/blackbox_exporter --config.file=/opt/prometheus_blackbox_exporter/blackbox.yml
+
+[Install]
+WantedBy=multi-user.target

salt/salt/prometheus/prometheus.yml.jinja2

@@ -23,3 +23,35 @@ scrape_configs:
     tls_config:
       insecure_skip_verify: true
     {% endif %}
+
+  # use blackbox exporter to check site is responding on IPv4
+  - job_name: "site_ipv4"
+    metrics_path: /probe
+    params:
+      module: [site_ipv4]
+    static_configs:
+      - targets:
+        - https://{{ pillar['site_hostname'] }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port
+
+  # use blackbox exporter to check site is responding on IPv6
+  - job_name: "site_ipv6"
+    metrics_path: /probe
+    params:
+      module: [site_ipv6]
+    static_configs:
+      - targets:
+        - https://{{ pillar['site_hostname'] }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port

salt/salt/top.sls

@@ -42,3 +42,4 @@ base:
     - sentry
     - grafana
     - prometheus
+    - prometheus.exporters.blackbox_exporter