Merge pull request #590 from jfly/issue-551

Alert if scraping jobs fail

Author: Mic92

build/pluto/prometheus/default.nix

@@ -1,4 +1,4 @@
-{ ... }:
+{ pkgs, ... }:
 
 {
   imports = [
@@ -16,8 +16,6 @@
     ./exporters/owncast.nix
     ./exporters/postgresql.nix
     ./exporters/rasdaemon.nix
-    ./exporters/r13y.nix
-    ./exporters/rfc39.nix
     ./exporters/zfs.nix
   ];
 
@@ -40,5 +38,28 @@
       "--web.external-url=https://prometheus.nixos.org/"
     ];
     globalConfig.scrape_interval = "15s";
+
+    ruleFiles = [
+      (pkgs.writeText "up.rules" (
+        builtins.toJSON {
+          groups = [
+            {
+              name = "up";
+              rules = [
+                {
+                  alert = "NotUp";
+                  expr = ''
+                    up == 0
+                  '';
+                  for = "10m";
+                  labels.severity = "warning";
+                  annotations.summary = "scrape job {{ $labels.job }} is failing on {{ $labels.instance }}";
+                }
+              ];
+            }
+          ];
+        }
+      ))
+    ];
   };
 }

build/pluto/prometheus/exporters/hydra.nix

@@ -36,12 +36,6 @@
         scheme = "https";
         static_configs = [ { targets = [ "hydra.nixos.org:443" ]; } ];
       }
-      {
-        job_name = "hydra_notify";
-        metrics_path = "/metrics";
-        scheme = "http";
-        static_configs = [ { targets = [ "hydra.nixos.org:9199" ]; } ];
-      }
       {
         job_name = "hydra_queue_runner";
         metrics_path = "/metrics";