Add a blackbox monitor of our mailserver

To avoid potential alterting noise: I'll wait until this is deployed and
succeeding before declaring an additional alert.

refs: NixOS#485

Author: jfly

build/pluto/prometheus/exporters/blackbox.nix

@@ -1,29 +1,4 @@
-{ pkgs, ... }:
-
-let
-  mkProbe = module: targets: {
-    job_name = "blackbox-${module}";
-    metrics_path = "/probe";
-    params = {
-      module = [ module ];
-    };
-    static_configs = [ { inherit targets; } ];
-    relabel_configs = [
-      {
-        source_labels = [ "__address__" ];
-        target_label = "__param_target";
-      }
-      {
-        source_labels = [ "__param_target" ];
-        target_label = "instance";
-      }
-      {
-        target_label = "__address__";
-        replacement = "localhost:9115";
-      }
-    ];
-  };
-in
+{ config, pkgs, ... }:
 
 {
   services.prometheus = {
@@ -37,30 +12,105 @@ in
             tcp.tls = true;
             http.headers.User-Agent = "blackbox-exporter";
           };
+
+          # From https://github.com/prometheus/blackbox_exporter/blob/53e78c2b3535ecedfd072327885eeba2e9e51ea2/example.yml#L120-L133
+          modules.smtp_starttls = {
+            prober = "tcp";
+            timeout = "5s";
+            tcp = {
+              query_response = [
+                { expect = "^220 ([^ ]+) ESMTP (.+)$"; }
+                { send = "EHLO prober\r"; }
+                { expect = "^250-STARTTLS"; }
+                { send = "STARTTLS\r"; }
+                { expect = "^220"; }
+                { starttls = true; }
+                { send = "EHLO prober\r"; }
+                { expect = "^250-AUTH"; }
+                { send = "QUIT\r"; }
+              ];
+            };
+          };
         }
       );
     };
 
     scrapeConfigs = [
-      (mkProbe "https_success" [
-        "https://cache.nixos.org"
-        "https://channels.nixos.org"
-        "https://common-styles.nixos.org"
-        "https://discourse.nixos.org"
-        "https://hydra.nixos.org"
-        "https://mobile.nixos.org"
-        "https://monitoring.nixos.org"
-        "https://nixos.org"
-        "https://planet.nixos.org"
-        "https://releases.nixos.org"
-        "https://status.nixos.org"
-        "https://survey.nixos.org"
-        "https://tarballs.nixos.org"
-        "https://weekly.nixos.org"
-        "https://wiki.nixos.org"
-        "https://www.nixos.org"
-        "https://tracker.security.nixos.org"
-      ])
+      {
+        job_name = "blackbox-https_success";
+        metrics_path = "/probe";
+        params = {
+          module = [ "https_success" ];
+        };
+        static_configs = [
+          {
+            targets = [
+              "https://cache.nixos.org"
+              "https://channels.nixos.org"
+              "https://common-styles.nixos.org"
+              "https://discourse.nixos.org"
+              "https://hydra.nixos.org"
+              "https://mobile.nixos.org"
+              "https://monitoring.nixos.org"
+              "https://nixos.org"
+              "https://planet.nixos.org"
+              "https://releases.nixos.org"
+              "https://status.nixos.org"
+              "https://survey.nixos.org"
+              "https://tarballs.nixos.org"
+              "https://weekly.nixos.org"
+              "https://wiki.nixos.org"
+              "https://www.nixos.org"
+              "https://tracker.security.nixos.org"
+            ];
+          }
+        ];
+        relabel_configs = [
+          {
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
+          }
+        ];
+      }
+      {
+        job_name = "blackbox-smtp_starttls";
+        metrics_path = "/probe";
+        params = {
+          module = [ "smtp_starttls" ];
+        };
+        dns_sd_configs = [
+          {
+            names = [
+              "mail-test.nixos.org"
+            ];
+            # TODO: this is blocked on https://github.com/NixOS/nixpkgs/pull/382617
+            # <<< type = "MX";
+            port = 25;
+          }
+        ];
+        relabel_configs = [
+          {
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = [ "__meta_dns_name" ];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
+          }
+        ];
+      }
     ];
 
     ruleFiles = [