Alert if scraping jobs fail

In addition to alerting on failing scraping jobs, we need to address our
existing failures, which I researched in
#551. Copy pasting analysis from
there:

- `r13y`: Last successful scrape: 2024-09-21. Website is down. @grahamc
  didn't respond to my query.
- `rfc39`: IMO, this is mis-architected. This scrapes a job that only
  runs periodically, which means we regularly get scrape failures. I've
  filed NixOS/rfc39#14 with the upstream
  project seeking advice.
  - Since we don't actually alert on these metrics, I propose doing the
    simplest thing and just disabling this scrape job for now.
- `hydra_notify`: Last successful scrape: 2024-08-02. We disabled this
  service [in
  2024](66da5cf),
  there's no reason to keep scraping it.

Author: jfly

build/pluto/prometheus/default.nix

@@ -1,4 +1,4 @@
-{ ... }:
+{ pkgs, ... }:
 
 {
   imports = [
@@ -16,8 +16,6 @@
     ./exporters/owncast.nix
     ./exporters/postgresql.nix
     ./exporters/rasdaemon.nix
-    ./exporters/r13y.nix
-    ./exporters/rfc39.nix
     ./exporters/zfs.nix
   ];
 
@@ -40,5 +38,28 @@
       "--web.external-url=https://prometheus.nixos.org/"
     ];
     globalConfig.scrape_interval = "15s";
+
+    ruleFiles = [
+      (pkgs.writeText "up.rules" (
+        builtins.toJSON {
+          groups = [
+            {
+              name = "up";
+              rules = [
+                {
+                  alert = "NotUp";
+                  expr = ''
+                    up == 0
+                  '';
+                  for = "10m";
+                  labels.severity = "warning";
+                  annotations.summary = "scrape job {{ $labels.job }} is failing on {{ $labels.instance }}";
+                }
+              ];
+            }
+          ];
+        }
+      ))
+    ];
   };
 }

build/pluto/prometheus/exporters/hydra.nix

@@ -36,12 +36,6 @@
         scheme = "https";
         static_configs = [ { targets = [ "hydra.nixos.org:443" ]; } ];
       }
-      {
-        job_name = "hydra_notify";
-        metrics_path = "/metrics";
-        scheme = "http";
-        static_configs = [ { targets = [ "hydra.nixos.org:9199" ]; } ];
-      }
       {
         job_name = "hydra_queue_runner";
         metrics_path = "/metrics";