Add a blackbox monitor of our mailserver

I re-locked in order to pull in
<NixOS/nixpkgs#383081>

To avoid potential alerting noise: I'll wait until this is deployed and
succeeding before declaring an additional alert.

refs: NixOS#485

Author: jfly

build/flake.lock

@@ -1,30 +1,68 @@
-{ pkgs, ... }:
+{ config, pkgs, ... }:
 
 let
-  mkProbe = module: targets: {
-    job_name = "blackbox-${module}";
-    metrics_path = "/probe";
-    params = {
-      module = [ module ];
+  mkHttpsProbe =
+    targets:
+    let
+      module = "https_success";
+    in
+    {
+      job_name = "blackbox-${module}";
+      metrics_path = "/probe";
+      params = {
+        module = [ module ];
+      };
+      static_configs = [ { inherit targets; } ];
+      relabel_configs = [
+        {
+          source_labels = [ "__address__" ];
+          target_label = "__param_target";
+        }
+        {
+          source_labels = [ "__param_target" ];
+          target_label = "instance";
+        }
+        {
+          target_label = "__address__";
+          replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
+        }
+      ];
     };
-    static_configs = [ { inherit targets; } ];
-    relabel_configs = [
-      {
-        source_labels = [ "__address__" ];
-        target_label = "__param_target";
-      }
-      {
-        source_labels = [ "__param_target" ];
-        target_label = "instance";
-      }
-      {
-        target_label = "__address__";
-        replacement = "localhost:9115";
-      }
-    ];
-  };
-in
 
+  mkSmtpStartTlsProbe =
+    names:
+    let
+      module = "smtp_starttls";
+    in
+    {
+      job_name = "blackbox-${module}";
+      metrics_path = "/probe";
+      params = {
+        module = [ module ];
+      };
+      dns_sd_configs = [
+        {
+          inherit names;
+          type = "MX";
+          port = 25;
+        }
+      ];
+      relabel_configs = [
+        {
+          source_labels = [ "__address__" ];
+          target_label = "__param_target";
+        }
+        {
+          source_labels = [ "__meta_dns_name" ];
+          target_label = "instance";
+        }
+        {
+          target_label = "__address__";
+          replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
+        }
+      ];
+    };
+in
 {
   services.prometheus = {
     exporters.blackbox = {
@@ -37,12 +75,31 @@ in
             tcp.tls = true;
             http.headers.User-Agent = "blackbox-exporter";
           };
+
+          # From https://github.com/prometheus/blackbox_exporter/blob/53e78c2b3535ecedfd072327885eeba2e9e51ea2/example.yml#L120-L133
+          modules.smtp_starttls = {
+            prober = "tcp";
+            timeout = "5s";
+            tcp = {
+              query_response = [
+                { expect = "^220 ([^ ]+) ESMTP (.+)$"; }
+                { send = "EHLO prober\r"; }
+                { expect = "^250-STARTTLS"; }
+                { send = "STARTTLS\r"; }
+                { expect = "^220"; }
+                { starttls = true; }
+                { send = "EHLO prober\r"; }
+                { expect = "^250-AUTH"; }
+                { send = "QUIT\r"; }
+              ];
+            };
+          };
         }
       );
     };
 
     scrapeConfigs = [
-      (mkProbe "https_success" [
+      (mkHttpsProbe [
         "https://cache.nixos.org"
         "https://channels.nixos.org"
         "https://common-styles.nixos.org"
@@ -61,6 +118,9 @@ in
         "https://www.nixos.org"
         "https://tracker.security.nixos.org"
       ])
+      (mkSmtpStartTlsProbe [
+        "mail-test.nixos.org"
+      ])
     ];
 
     ruleFiles = [