Skip to content

Commit f2e6901

Browse files
mweineltmweinelt
authored
Merge pull request #535 from NixOS/nginx-abuse-502-etc
hydra: abuse handling, branded 502 error; prometheus: alert on full zfs pool
2 parents d23760f + 3e0eb3a commit f2e6901

File tree

4 files changed

+140
-5
lines changed

4 files changed

+140
-5
lines changed

‎build/haumea/postgresql.nix

+3-3
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
log_statement = "none";
3535

3636
# pgbadger-compatible logging
37-
log_transaction_sample_rate = 1.0e-2;
37+
log_transaction_sample_rate = 0.01;
3838
log_min_duration_statement = 5000;
3939
log_checkpoints = "on";
4040
log_connections = "on";
@@ -80,8 +80,8 @@
8080
# benefit from frequent vacuums, so this should
8181
# help. In particular, I'm thinking the jobsets
8282
# pages.
83-
autovacuum_vacuum_scale_factor = 2.0e-2;
84-
autovacuum_analyze_scale_factor = 1.0e-2;
83+
autovacuum_vacuum_scale_factor = 0.02;
84+
autovacuum_analyze_scale_factor = 0.01;
8585

8686
shared_preload_libraries = "pg_stat_statements";
8787
compute_query_id = "on";

‎build/hydra-proxy.nix

+47-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,27 @@
1-
{ config, ... }:
1+
{
2+
config,
3+
lib,
4+
pkgs,
5+
...
6+
}:
27

8+
let
9+
bannedUserAgentPatterns = [
10+
"Trident/"
11+
"Android\\s[123456789]\\."
12+
"iPod"
13+
"iPad\\sOS\\s"
14+
"iPhone\\sOS\\s[23456789]"
15+
"Opera/[89]"
16+
"(Chrome|CriOS)/(\\d\\d?\\.|1[01]|12[4])"
17+
"(Firefox|FxiOS)/(\\d\\d?\\.|1[01]|12[012345679]\\.)"
18+
"PPC\\sMac\\sOS"
19+
"Windows\\sCE"
20+
"Windows\\s95"
21+
"Windows\\s98"
22+
"Windows\\sNT\\s[12345]\\."
23+
];
24+
in
325
{
426
networking.firewall.allowedTCPPorts = [
527
80
@@ -27,20 +49,43 @@
2749
worker_connections 1024;
2850
'';
2951

52+
appendHttpConfig = ''
53+
map $http_user_agent $badagent {
54+
default 0;
55+
${lib.concatMapStringsSep "\n" (pattern: ''
56+
~${pattern} 1;
57+
'') bannedUserAgentPatterns}
58+
}
59+
'';
60+
3061
virtualHosts."hydra.nixos.org" = {
3162
forceSSL = true;
3263
enableACME = true;
3364

3465
extraConfig = ''
66+
error_page 502 /502.html;
3567
error_page 503 /503.html;
36-
location = /503.html {
68+
location ~ /(502|503).html {
3769
root ${./nginx-error-pages};
3870
internal;
3971
}
4072
'';
4173

74+
# Ask robots not to scrape hydra, it has various expensive endpoints
75+
locations."=/robots.txt".alias = pkgs.writeText "hydra.nixos.org-robots.txt" ''
76+
User-agent: *
77+
Disallow: /
78+
Allow: /$
79+
'';
80+
4281
locations."/" = {
4382
proxyPass = "http://127.0.0.1:3000";
83+
extraConfig = ''
84+
if ($badagent) {
85+
access_log /var/log/nginx/abuse.log;
86+
return 403;
87+
}
88+
'';
4489
};
4590

4691
locations."/static/" = {

‎build/nginx-error-pages/502.html

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<title>Error 502 - hydra.nixos.org</title>
5+
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
6+
<meta http-equiv="X-UA-Compatible" content="IE=Edge" />
7+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
8+
<link
9+
rel="stylesheet"
10+
href="https://nixos.org/bootstrap/css/bootstrap.min.css"
11+
/>
12+
<link
13+
rel="stylesheet"
14+
href="https://nixos.org/bootstrap/css/bootstrap-responsive.min.css"
15+
/>
16+
<style>
17+
body {
18+
padding-top: 0;
19+
margin-top: 4em;
20+
margin-bottom: 4em;
21+
}
22+
body > div {
23+
max-width: 800px;
24+
}
25+
h1 {
26+
margin: 0 auto;
27+
text-align: center;
28+
}
29+
p {
30+
text-align: center;
31+
}
32+
</style>
33+
</head>
34+
<body>
35+
<div class="container jumbotron">
36+
<div class="jumbotron">
37+
<p class="lead">
38+
<a href="https://nixos.org/nixos">
39+
<img
40+
src="https://nixos.org/logo/nixos-hires.png"
41+
width="500px"
42+
alt="logo"
43+
/>
44+
</a>
45+
</p>
46+
47+
<h1>HTTP Error 502</h1>
48+
49+
<p class="lead">This service is currently unavailable!</p>
50+
</div>
51+
<hr>
52+
<div class="help">
53+
<p>
54+
You can check the following resources for further informations:<br>
55+
<a href="https://prometheus.nixos.org/alerts">Alerts</a> |
56+
<a href="https://grafana.nixos.org/">Dashboards</a> |
57+
<a href="https://github.com/NixOS/infra/issues">Issues</a> |
58+
<a href="https://matrix.to/#/#infra:nixos.org">Chatroom</a>
59+
</p>
60+
</div>
61+
</div>
62+
</body>
63+
</html>

‎build/pluto/prometheus/exporters/zfs.nix

+27
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
{
2+
pkgs,
3+
...
4+
}:
15
{
26
services.prometheus = {
37
scrapeConfigs = [
@@ -14,5 +18,28 @@
1418
];
1519
}
1620
];
21+
ruleFiles = [
22+
(pkgs.writeText "node-exporter.rules" (
23+
builtins.toJSON {
24+
groups = [
25+
{
26+
name = "zfs";
27+
rules = [
28+
{
29+
alert = "ZfsPoolFull";
30+
expr = ''
31+
(zfs_pool_free_bytes / zfs_pool_size_bytes) * 100 < 15
32+
'';
33+
for = "30m";
34+
labels.severity = "warning";
35+
annotations.summary = "ZFS pool {{ $labels.pool }} on {{ $labels.instance }} has only {{ $value }}% free space.";
36+
annotations.grafana = "https://grafana.nixos.org/d/rYdddlPWk/node-exporter-full?orgId=1&var-job=node&var-node={{ $labels.instance }}";
37+
}
38+
];
39+
}
40+
];
41+
}
42+
))
43+
];
1744
};
1845
}

0 commit comments

Comments
 (0)