From 58fe4e3c0b7e52967af13d67b40669065effd370 Mon Sep 17 00:00:00 2001 From: tigattack <10629864+tigattack@users.noreply.github.com> Date: Sat, 20 Jul 2024 11:08:51 +0100 Subject: [PATCH] Initial commit --- .github/workflows/ansible-lint.yml | 17 + .gitignore | 4 + .vscode/settings.json | 8 + README.md | 49 + ansible.cfg | 10 + host_vars/labbot_host.example.yml | 26 + hosts.yml | 5 + playbook.yml | 13 + requirements.txt | 2 + requirements.yml | 5 + roles/labbot/defaults/main.yml | 62 + roles/labbot/files/home_dashboard.json | 126 + roles/labbot/files/labbot-fav32.png | Bin 0 -> 2150 bytes roles/labbot/handlers/main.yml | 36 + roles/labbot/tasks/app_configs.yml | 83 + roles/labbot/tasks/backup_setup.yml | 41 + roles/labbot/tasks/main.yml | 63 + roles/labbot/tasks/proxy_setup.yml | 114 + roles/labbot/templates/backup.sh.j2 | 159 + .../templates/grafana_dashboards/bot.json.j2 | 2829 +++++++++++++++++ .../templates/grafana_dashboards/host.json.j2 | 1142 +++++++ .../dashboards/dashboards.yml.j2 | 15 + .../datasources/monitoring.yml.j2 | 21 + roles/labbot/templates/nginx_grafana.conf.j2 | 63 + roles/labbot/templates/prometheus-web.yml.j2 | 4 + roles/labbot/templates/prometheus.yml.j2 | 12 + roles/labbot/vars/main.yml | 151 + roles/labbot/vars/rclone.yml | 24 + 28 files changed, 5084 insertions(+) create mode 100644 .github/workflows/ansible-lint.yml create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 README.md create mode 100644 ansible.cfg create mode 100644 host_vars/labbot_host.example.yml create mode 100644 hosts.yml create mode 100644 playbook.yml create mode 100644 requirements.txt create mode 100644 requirements.yml create mode 100644 roles/labbot/defaults/main.yml create mode 100644 roles/labbot/files/home_dashboard.json create mode 100755 roles/labbot/files/labbot-fav32.png create mode 100644 roles/labbot/handlers/main.yml create mode 100644 roles/labbot/tasks/app_configs.yml create mode 100644 roles/labbot/tasks/backup_setup.yml create mode 100644 roles/labbot/tasks/main.yml create mode 100644 roles/labbot/tasks/proxy_setup.yml create mode 100644 roles/labbot/templates/backup.sh.j2 create mode 100644 roles/labbot/templates/grafana_dashboards/bot.json.j2 create mode 100644 roles/labbot/templates/grafana_dashboards/host.json.j2 create mode 100644 roles/labbot/templates/grafana_provisioning/dashboards/dashboards.yml.j2 create mode 100644 roles/labbot/templates/grafana_provisioning/datasources/monitoring.yml.j2 create mode 100644 roles/labbot/templates/nginx_grafana.conf.j2 create mode 100644 roles/labbot/templates/prometheus-web.yml.j2 create mode 100644 roles/labbot/templates/prometheus.yml.j2 create mode 100644 roles/labbot/vars/main.yml create mode 100644 roles/labbot/vars/rclone.yml diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml new file mode 100644 index 0000000..ed7f557 --- /dev/null +++ b/.github/workflows/ansible-lint.yml @@ -0,0 +1,17 @@ +--- +name: Ansible Lint + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run ansible-lint + uses: ansible/ansible-lint@v24.7.0 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a59521e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.dependencies +venv +plugins/ +host_vars/labbot_host.yml diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a166d01 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "files.associations": { + "*.yml": "ansible", + "*.yaml": "ansible", + "*.j2": "ansible-jinja", + }, + "ansible.python.activationScript": "venv/bin/activate", +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..37e0bad --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# labbot-deploy + +This repository containers an Ansible playbook & role to deploy LabBot and related monitoring infrastructure (Grafana, Prometheus, cAdvisor, and node_exporter). + +While it may work with others, this role is designed to be run against a Debian-based Linux distribution. + +## Usage + +1. Enter the [`host_vars`](host_vars/) directory and create a copy of [`labbot_host.example.yml`](host_vars/labbot_host.example.yml) named `labbot_host.yml`. +2. Set all relevant values in `labbot_host.yml`. +3. Run the following commands: + +```sh +# Create a Python virtual environment (venv) +python3 -m venv venv +# Activate the venv +source venv/bin/activate +# Install Ansible (in the venv) +pip install ansible +# Install required Ansible roles/collections +ansible-galaxy install -r requirements.yml +# Run the playbook to initiate the deployment +ansible-playbook playbook.yml +``` + +> [!TIP] +> The `labbot` Ansible role contains several optional components that are not included in the `labbot_host.yml` example file for the sake of simplicity; see [`roles/labbot/defaults/main.yml`](roles/labbot/defaults/main.yml) for advanced values such as backup options and more. + +## Bot monitoring setup + +If this is a fresh deployment, LabBot will not automatically serve Prometheus metrics. This is because the `prometheus_exporter` cog will not yet be installed. + +If this is the case, you will see a task named _"Set LabBot prometheus_exporter cog scrape interval"_ fail during the Ansible deployment. + +Follow the steps below and then re-run the playbook. It should no longer fail. + +Enter a chat with the bot and run the following (where `[p]` is the configured prefix): + +``` +[p]repo add homelab https://github.com/rHomelab/LabBot-Cogs +[p]cog install homelab prometheus_exporter +[p]load prometheus_exporter +``` + +--- + +## Disclaimer + +Most common settings are variablised, however I only prepared this for shared use/more generic environments after extracting it from my own infrastructure definitions, so there may be some things that are still 'hardcoded', otherwise inoptimal for a shared configuration, or not fully tested. diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..8ea1765 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,10 @@ +[defaults] +inventory = ./hosts.yml +stdout_callback = yaml +roles_path = .dependencies:roles +lookup_plugins = ./plugins/lookup +nocows = true +ansible_managed = Ansible Managed. Last edited on %Y-%m-%d at %H:%M:%S. + +[diff] +always = true diff --git a/host_vars/labbot_host.example.yml b/host_vars/labbot_host.example.yml new file mode 100644 index 0000000..3e79b56 --- /dev/null +++ b/host_vars/labbot_host.example.yml @@ -0,0 +1,26 @@ +#################################### NOTE #################################### +# The settings in this file are the minimum required to run the playbook. +# For more advanced configurations, please see roles/labbot/defaults/main.yml +############################################################################## + +# Connection settings +ansible_host: # ip or host.name +ansible_user: # username +ansible_password: # password +# OR +ansible_ssh_private_key_file: # path to private key + +# Bot and monitoring settings +labbot_discord_token: '!CHANGE_ME!' + +labbot_grafana_domain: labbot.tiga.tech +labbot_grafana_password: '!CHANGE_ME!' + +labbot_prometheus_users: + - username: admin + password: '!CHANGE_ME!' + # Hashed version of the password + # htpasswd -nBC 10 '' | tr -d ':\n' + password_bcrypt: '' + +labbot_certbot_letsencrypt_email: "admin@{{ labbot_grafana_domain }}" diff --git a/hosts.yml b/hosts.yml new file mode 100644 index 0000000..618a62f --- /dev/null +++ b/hosts.yml @@ -0,0 +1,5 @@ +--- + +all: + hosts: + labbot_host: diff --git a/playbook.yml b/playbook.yml new file mode 100644 index 0000000..df1272b --- /dev/null +++ b/playbook.yml @@ -0,0 +1,13 @@ +--- + +- name: LabBot deployment + hosts: all + become: true + module_defaults: + community.docker.docker_container: + restart_policy: always + state: started + comparisons: + '*': strict + env: allow_more_present + roles: [labbot] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9460f6b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ansible==10.2.0 +ansible-lint==24.7.0 diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 0000000..ffc0716 --- /dev/null +++ b/requirements.yml @@ -0,0 +1,5 @@ +--- + +roles: + - name: stefangweichinger.ansible_rclone + version: 0.1.6 diff --git a/roles/labbot/defaults/main.yml b/roles/labbot/defaults/main.yml new file mode 100644 index 0000000..258c08c --- /dev/null +++ b/roles/labbot/defaults/main.yml @@ -0,0 +1,62 @@ +--- + +###################### NOTE ###################### +# All commented settings must be defined in host_vars! +################################################## + +# Parent directory for container config paths +labbot_app_base_dir: /opt + +# Discord bot token for LabBot +# labbot_discord_token: + +# labbot_grafana_domain: +labbot_grafana_username: labbot_admin +# labbot_grafana_password: + +labbot_grafana_container_user: "{{ ansible_user_uid }}" +labbot_prometheus_container_user: "{{ ansible_user_uid }}" + +# List of dictionaries with keys `username`, `password`, and `password_bcrypt` +labbot_prometheus_users: [] + +labbot_prometheus_scrape_interval: 10 # seconds + +# Whether to bind the prometheus web UI port on the host +# Necessary only for testing/debugging +labbot_prometheus_open_port: false + +# Whether to bind the cadvisor web UI port on the host +# Necessary only for testing/debugging +labbot_cadvisor_open_port: false + +# Settings for bot data backup +labbot_enable_bot_backup: false + +# NOTE: All backup variables below must be defined ONLY if labbot_enable_bot_backup is true + +# labbot_backup_webdav_url: +# labbot_backup_webdav_username: +# labbot_backup_webdav_password: + +# Discord webhook URL for backup report +# labbot_backup_report_webhook: '' + +# User ID to mention upon backup failure +# labbot_backup_report_mention_user_id: '' + +# Certbot settings for Grafana SSL certificate +# Whether to run certbot in dry-run mode +labbot_certbot_dry_run: false + +# Email address to register with LetsEncrypt +# labbot_certbot_letsencrypt_email: + +# Misc settings + +# [OPTIONAL] SSH keys to install on the bot host +# EXAMPLE: +# labbot_ssh_keys: +# - key: +# comment: +labbot_ssh_keys: [] diff --git a/roles/labbot/files/home_dashboard.json b/roles/labbot/files/home_dashboard.json new file mode 100644 index 0000000..fcf4ba8 --- /dev/null +++ b/roles/labbot/files/home_dashboard.json @@ -0,0 +1,126 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "#
LabBot Monitoring
\n####
Welcome to the home of LabBot & Homelab Discord monitoring.
\n\n## Homelab Community Links\n\n* [Homelab on Discord](https://discord.gg/homelab)\n* [Homelab on Reddit](https://reddit.com/r/homelab)\n* [Homelab on Github](https://github.com/rHomelab)\n\nSee the dashboards below for bot and Discord server metrics:", + "mode": "markdown" + }, + "pluginVersion": "9.3.6", + "transparent": true, + "type": "text" + }, + { + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "maxItems": 10, + "query": "LabBot", + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "transparent": true, + "type": "dashlist" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Home", + "version": 2, + "weekStart": "" +} diff --git a/roles/labbot/files/labbot-fav32.png b/roles/labbot/files/labbot-fav32.png new file mode 100755 index 0000000000000000000000000000000000000000..5642e02476fb4f8441f2b0da8d4c3c406cbddd6d GIT binary patch literal 2150 zcmV-s2$}bZP)BE1ZQLxAY_vg;$?V;oX(lt{{G@3qJ`_O^!KmPq;Qw2}Cn0L9?Tam?khm;cmfj@-EE=vY<#k5o0f^m&*sM6pBI0EL{uRZtU4R5Z z`^X8`V)zNfJ8YaF;u6EP09P2E5Ad?jnUOMxUqvh^s*L``+lU9}BlcPZ2XpY-jyyG} zttleQMaed+noc9ry@UIIwng1*#M^^UkFzj+6yF25_LnMhmf?pGpH<}CbcnAmX|u%; z|7E#p3GpKUUAw%6atiT2#A8)^yd3Cssj4^8z$>TJnP(8+gLrkpJj%xIMEtceUz`ec zeo~wa<_{hK;NmJRuzL3+9o0iVC08s$)vACv7|KExH6jmbd4gn8^0FkllzkRT6fd|P0n zrwSVAXrbt(XrcJk;OyVRDf#{Yw^VBW$m{6QFhZvPBk<6w!tZERBfJcLd1$M3&)TehHS09YRtkN(Z(GDi^F2;vKBX;HY+Q*UEnTqUvgh@A9J5aYSek}G;8j! zqp=F+X_rylpF&#LEj%b>g+0Q~5^H*5N-?MwSLAe5T{?GXqX;9WfX;*4pXw6&V*io?m|<6;W0GAb4f<-njQ>$>Q>9MPr@ zwU=n+0I+*9IKpp9{{0u*d8K7@p9VPmCp-6AwmA>rtOL;dXv;Q$XXCx^0IWWyc=l3w zC&Xgk0yx!!!@*Zi8qcdA>-AfBZYACT*nGWS|NH%Vee(m<-vGFv{{#Oecnig2*2@3@ z061k>NoGw=04e|g00;m9hiL!=000010000Q0000000N)_00aO40096103e_P00aO4 z0096103ZMW0056pK*<0A1J6lBK~z`?rIve0Q&AkpzuTPYTp2Se%+%5{N_-bo>tQP^ za72n$^q?0h7!?#zL)h7u z+}(5SjP3r;@BDth@8|q>;9TH8hIVhAmwf-D1T=$5mJ{_Hn5nv+>I6A)x)HSy+yE@$ zya*@GY+3+a;X|q~K6ybD0a(IB#l3%qC`Jqx1o10@$O0${PmE6|^~KTFcvrC1f-nJC z;&jJxiI(SHw`Yo|EHV6nu)6UV085zEr{dCGOSHu)g&+=AM|?p<0G7B~VUggsK|%aM z1W1#}TuImj2a>0zV3{==r6*2c<&Hdbc6Olf^sjS3YtoLn{hSXi4VFG>vechHSC6hv z2dZlxU|x1EF4Z?6E^P*CYi>$I#kpK95+Dm8W=o8-_xce%A`pSYVzDPK1aB*L!kn=Y zH7&uCNWQ6R08-X5BtfX|`R;`!Yd@coOce7B$ixcnupg~hzBnCu+cOauxCsYC!qL^% zf@Yf$4YlPcs{QCM{hu{e9e8pz7u_FvarpdI1V^pHqSVc>q%6nMIs1?{!Hv}6L;n#_ za&UE^V8H}rtzC_z>_TkKNJjN*7iR`fV~ zasO5WHdHntFX9_Y%Wj~%ssiSmLzrVd3B&LZw7>fRuh#=>Jl}cXIs*jN<#-Q`PKW&J zHv)NXRXeo0S8#eVQQ!OqYpfZ#QC^8Yw-*=qMA~ZLhE{JP4+UNZ2~;{&dAP?hXWuNy6xXH1d^>p$0ul zO0Hwq5(|!BxCXP!hyXp`-ejZbO(qp%&7asD`0V_KGiA4-PmaU2M~^Ww)+~nltHta9 zy*}wJ=I=5gDlt=Li()UGq6sQY>-|~zV~Otr#Kn3iAbzFmnZ)=GDolJVapnt&Zvhqv zg=p`v+ZS2avX8e$V{3bMCx%Cu>iyLL>LgDV1o2Z4>EA+k&Dt@^^W8O>(;*8$1C&5$ zoLXU;rwBj;Ss(?Gy*RaTzT*E(Yp|Z&05q8eLf6PsuGPLVzP4%rk}L- + {{ lookup('filetree', 'templates/grafana_provisioning/') }} + loop_control: + label: "{{ item.path }}" + when: item.state == 'file' + notify: Restart Grafana + +# No need to notify the 'Restart Grafana' handler here since Grafana scans for updated +# files per the `updateIntervalSeconds` setting in the `dashboards.yml` provisioning config. +- name: Template Grafana dashboards + ansible.builtin.template: + src: "{{ item.root + item.path }}" + dest: "{{ grafana_app_dir }}/dashboards/{{ item.path | replace('.j2', '') }}" + owner: "{{ grafana_user }}" + group: "{{ grafana_user }}" + mode: '0440' + loop: >- + {{ lookup('filetree', 'templates/grafana_dashboards/') }} + loop_control: + label: "{{ item.path }}" + when: item.state == 'file' + +- name: Copy other Grafana files + ansible.builtin.copy: + src: "{{ item }}" + dest: "{{ grafana_app_dir + '/' + item }}" + mode: '0644' + loop: + - labbot-fav32.png + - home_dashboard.json + notify: Restart Grafana diff --git a/roles/labbot/tasks/backup_setup.yml b/roles/labbot/tasks/backup_setup.yml new file mode 100644 index 0000000..5fef8d9 --- /dev/null +++ b/roles/labbot/tasks/backup_setup.yml @@ -0,0 +1,41 @@ +--- + +- name: Include rclone vars + ansible.builtin.include_vars: + file: rclone.yml + +# Install Rclone and configure remote(s) +- name: Install & configure Rclone + ansible.builtin.include_role: + name: stefangweichinger.ansible_rclone + apply: + tags: rclone + tags: rclone + +- name: Install jq + ansible.builtin.apt: + name: jq + state: present + +- name: Template bot backup script + ansible.builtin.template: + src: backup.sh.j2 + dest: "{{ labbot_app_base_dir }}/backup-labbot.sh" + mode: '0755' + vars: + bot_dir: "{{ (bot_container.volumes[0] | split(':'))[0] }}" + container_name: "{{ bot_container.name }}" + webhook: "{{ labbot_backup_report_webhook }}" + +- name: Install cron + ansible.builtin.apt: + name: cron + state: present + +- name: Install cron backup schedules + ansible.builtin.cron: + name: labbot backup schedule + cron_file: labbot_backup + user: root + special_time: daily + job: "{{ labbot_app_base_dir }}/backup-labbot.sh" diff --git a/roles/labbot/tasks/main.yml b/roles/labbot/tasks/main.yml new file mode 100644 index 0000000..68ebac5 --- /dev/null +++ b/roles/labbot/tasks/main.yml @@ -0,0 +1,63 @@ +--- + +- name: Set facts + ansible.builtin.set_fact: + grafana_app_dir: "{{ labbot_app_base_dir }}/grafana" + grafana_user: >- + {{ (labbot_containers | selectattr('image', 'search', 'grafana/grafana'))[0]['user'] }} + bot_container: >- + {{ labbot_containers | selectattr('image', 'search', 'red-discordbot') | first }} + tags: always + +- name: Provision configs + ansible.builtin.include_tasks: + file: app_configs.yml + apply: + tags: app_configs + tags: app_configs + +- name: Create Docker network + community.docker.docker_network: + name: "{{ item.name }}" + state: present + register: docker_network + loop: "{{ labbot_containers | selectattr('networks', 'defined') | map(attribute='networks') | default([]) | flatten | unique }}" + +- name: Deploy containers # noqa args[module] + community.docker.docker_container: "{{ item }}" + loop: "{{ labbot_containers }}" + loop_control: + label: "{{ item.name }}" + notify: Prune Docker images + +- name: Deploy reverse proxy for Grafana + ansible.builtin.include_tasks: + file: proxy_setup.yml + apply: + tags: proxy + tags: proxy + +- name: Configure backups + ansible.builtin.include_tasks: + file: backup_setup.yml + apply: + tags: backup + tags: backup + when: labbot_enable_bot_backup + +- name: Install SSH keys + ansible.posix.authorized_key: + user: "{{ ansible_user }}" + key: "{{ item.key }}" + exclusive: false + comment: "{{ item.comment if item.comment is defined else omit }}" + loop: "{{ labbot_ssh_keys }}" + +# Hacky workaround to occasional memory leaks in Red-DiscordBot +- name: Create bot restart cron entries + ansible.builtin.cron: + name: "{{ bot_container.name }} restart schedule" + cron_file: redbot_restart + user: root + special_time: monthly + job: docker restart {{ bot_container.name }} diff --git a/roles/labbot/tasks/proxy_setup.yml b/roles/labbot/tasks/proxy_setup.yml new file mode 100644 index 0000000..9cfcac1 --- /dev/null +++ b/roles/labbot/tasks/proxy_setup.yml @@ -0,0 +1,114 @@ +--- + +- name: Get Cloudflare proxy IPs + ansible.builtin.uri: + url: "https://api.cloudflare.com/client/v4/ips" + method: GET + return_content: true + headers: + accept: application/json + check_mode: false + register: cloudflare_ips + +- name: Create NGINX dir + ansible.builtin.file: + path: "{{ item }}" + mode: '0755' + owner: root + group: root + state: directory + loop: + - "{{ labbot_nginx_conf_path_host }}" + - "{{ labbot_nginx_dhparam_path_host | dirname }}" + +- name: Generate Diffie-Hellman parameters (dhparam) + community.crypto.openssl_dhparam: + path: "{{ labbot_nginx_dhparam_path_host }}" + mode: '0644' + owner: root + group: root + state: present + +- name: Check for certificate + ansible.builtin.stat: + path: "{{ labbot_certbot_cert_bind | split(':') | first }}/live/{{ labbot_grafana_domain }}/fullchain.pem" + register: stat_cert + +- name: Deploy NGINX container + community.docker.docker_container: + name: nginx + image: nginx:1.27-alpine + ports: [80:80, 443:443] + volumes: + - "{{ labbot_nginx_conf_path_host }}:/etc/nginx/conf.d:ro" + - "{{ labbot_certbot_www_bind }}:ro" + - "{{ labbot_certbot_cert_bind }}:ro" + networks: [name: web] + notify: Prune Docker images + +- name: Certbot block + when: not stat_cert.stat.exists + block: + - name: Template NGINX config for ACME challenge + ansible.builtin.template: + src: nginx_grafana.conf.j2 + dest: "{{ labbot_nginx_conf_path_host }}/labbot_grafana.conf" + mode: '0644' + owner: root + group: root + vars: + acme_only: true + + - name: Reload nginx + community.docker.docker_container_exec: + container: nginx + command: nginx -s reload + + - name: Run certbot + community.docker.docker_container: + name: certbot + image: certbot/certbot:latest + restart_policy: "no" + command: >- + certonly --non-interactive --agree-tos + --webroot --webroot-path {{ labbot_certbot_webroot_path }}/ + -m {{ labbot_certbot_letsencrypt_email }} + -d {{ labbot_grafana_domain }} + {{ '--dry-run' if labbot_certbot_dry_run else '' }} + volumes: + - "{{ labbot_certbot_www_bind }}" + - "{{ labbot_certbot_cert_bind }}" + - "{{ labbot_app_base_dir }}/nginx/certbot_logs:/var/log/letsencrypt" + detach: false + output_logs: true + cleanup: true + state: started + +- name: Template NGINX config + ansible.builtin.template: + src: nginx_grafana.conf.j2 + dest: "{{ labbot_nginx_conf_path_host }}/labbot_grafana.conf" + mode: '0644' + owner: root + group: root + notify: Reload nginx + +- name: Install certificate renewal cron schedule + ansible.builtin.cron: + name: "Certbot renewal" + cron_file: certbot_renew + user: root + special_time: weekly + # 2024-07-20: Test is bugged lmao + # noqa jinja[spacing] + job: >- + {%- set certbot_container = labbot_nginx_containers | selectattr('image', 'search', 'certbot') | first -%} + {%- set nginx_container = labbot_nginx_containers | selectattr('image', 'search', 'nginx') | first -%} + docker run --rm -it + --name {{ certbot_container.name }} + {%- for volume in certbot_container.volumes %} + -v {{ volume }} + {%- endfor %} + {{ certbot_container.image }} + {{ certbot_container.command }} + && docker exec {{ nginx_container.name }} nginx -s reload diff --git a/roles/labbot/templates/backup.sh.j2 b/roles/labbot/templates/backup.sh.j2 new file mode 100644 index 0000000..67f48e6 --- /dev/null +++ b/roles/labbot/templates/backup.sh.j2 @@ -0,0 +1,159 @@ +#!/bin/bash + +# Set start time seconds +start=$SECONDS + +DISCORD_USER_ID="{{ labbot_backup_report_mention_user_id }}" + +log="/var/log/backup-{{ container_name }}.log" + +exit_on_error() { + exit_code=$1 + last_command=${@:2} + docker unpause {{ container_name }} + >&2 echo "\"${last_command}\" command failed with exit code ${exit_code}." + backup_notify "failed" + exit $exit_code +} + +format_data_size() { + declare -i i=${1:-$(&1 4>&2 +trap 'exec 2>&4 1>&3' 0 1 2 3 +# Print START and current date to the log. +date>$log +printf "START \n">>$log +exec 1>>$log 2>&1 + +archive="/tmp/{{ container_name }}-$(date +%y-%m-%d_%H-%M).tar.gz" + +cd "{{ bot_dir }}" + +# Pause container +docker pause {{ container_name }} || exit_on_error + +# Archive appdata +(tar --exclude='__pycache__' -cvf - . | pigz -c -5 -p $(nproc) > "$archive") || exit_on_error + +# Get file count +file_count=$(gzip -cd "$archive" | tar -tvv | grep -c ^-) + +# Unpause container +docker unpause {{ container_name }} + +# Upload to cloud +cd $(dirname "$archive") +rclone mkdir {{ labbot_rclone_configs[0].name }}:{{ container_name }} + +file=$(basename "$archive") +rclone moveto -v "$archive" {{ labbot_rclone_configs[0].name }}:{{ container_name }}/"$file" + +# Remove old backups +# M=month, m=min +rclone delete -v --min-age 1M {{ labbot_rclone_configs[0].name }}:{{ container_name }} + +# Print END and two empty lines to the log. +printf "END\n" + +backup_notify "success" diff --git a/roles/labbot/templates/grafana_dashboards/bot.json.j2 b/roles/labbot/templates/grafana_dashboards/bot.json.j2 new file mode 100644 index 0000000..3db52e5 --- /dev/null +++ b/roles/labbot/templates/grafana_dashboards/bot.json.j2 @@ -0,0 +1,2829 @@ +{% set db_name = 'Prometheus' %} +{% set container_name = 'labbot' %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [ + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "LabBot Host", + "tooltip": "Open dashboard", + "type": "link", + "url": "https://labbot.tiga.tech/d/host" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "LabBot Cogs", + "tooltip": "Open GitHub repository", + "type": "link", + "url": "https://github.com/rHomelab/LabBot-Cogs" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Homelab Discord", + "tooltip": "Join the Homelab Discord", + "type": "link", + "url": "https://discord.gg/homelab" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 32, + "panels": [], + "title": "Container Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdhms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "Container Uptime", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "time() - container_start_time_seconds{name=\"$container_name\"}", + "fullMetaSearch": false, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "measurement": "docker_container_status", + "orderByTime": "ASC", + "policy": "default", + "range": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "uptime_ns" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "/1000000" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ], + "useBackend": false + } + ], + "title": "LabBot Container Uptime", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 5, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "alias": "Container CPU", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(container_cpu_usage_seconds_total{name=\"$container_name\"}[$__rate_interval]) * 100", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "Container CPU", + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=", + "value": "{{ container_name }}" + } + ] + } + ], + "title": "Container CPU", + "transformations": [ + { + "id": "configFromData", + "options": { + "applyTo": { + "id": "byFrameRefID", + "options": "A" + }, + "configRefId": "A", + "mappings": [ + { + "fieldName": "Container CPU", + "handlerKey": "max", + "reducerId": "max" + } + ] + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "max": 2500000000, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 85 + }, + { + "color": "orange", + "value": 90 + }, + { + "color": "dark-red", + "value": 95 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 11, + "y": 1 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "alias": "Container Memory", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "container_memory_usage_bytes{name=\"$container_name\"}", + "fullMetaSearch": false, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Container Memory", + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ], + "useBackend": false + } + ], + "title": "Container Memory", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^(Container Net [RT]x (Dropped|Errors))$" + }, + "properties": [ + { + "id": "unit", + "value": "none" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 1 + }, + "id": 30, + "interval": "10s", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "alias": "Container Net RX", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(container_network_receive_bytes_total{name=\"$container_name\"}[$__rate_interval])", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "Container Net Rx", + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + }, + { + "params": [ + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ] + }, + { + "alias": "Container Net TX", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(container_network_transmit_bytes_total{name=\"$container_name\"}[$__rate_interval])", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "Container Net Tx", + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + }, + { + "params": [ + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ] + }, + { + "alias": "Container Net RX", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(container_network_receive_errors_total{name=\"$container_name\"}[$__rate_interval])", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "Container Net Rx Errors", + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + }, + { + "params": [ + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ] + }, + { + "alias": "Container Net TX", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(container_network_transmit_errors_total{name=\"$container_name\"}[$__rate_interval])", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "Container Net Tx Errors", + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + }, + { + "params": [ + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ] + }, + { + "alias": "Container Net RX", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(container_network_receive_packets_dropped_total{name=\"$container_name\"}[$__rate_interval])", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "Container Net Rx Dropped", + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + }, + { + "params": [ + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ] + }, + { + "alias": "Container Net TX", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(container_network_transmit_packets_dropped_total{name=\"$container_name\"}[$__rate_interval])", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "Container Net Tx Dropped", + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + }, + { + "params": [ + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ] + } + ], + "title": "Container Network", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 4 + }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "alias": "Read total", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "container_blkio_device_usage_total{name=\"$container_name\", operation=\"Read\"}", + "fullMetaSearch": false, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Read total", + "measurement": "docker_container_blkio", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "io_service_bytes_recursive_read" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ], + "useBackend": false + }, + { + "alias": "Write total", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "container_blkio_device_usage_total{name=\"$container_name\", operation=\"Write\"}", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "Write total", + "measurement": "docker_container_blkio", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "io_service_bytes_recursive_write" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=~", + "value": "/^$container_name$/" + } + ] + } + ], + "title": "Container Disk", + "transparent": true, + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "panels": [], + "title": "Discord Server Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "LabBot Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "dtdurationms" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-red", + "value": 3240000 + }, + { + "color": "dark-orange", + "value": 21240000 + }, + { + "color": "yellow", + "value": 42840000 + }, + { + "color": "green", + "value": 86040000 + } + ] + } + }, + { + "id": "decimals", + "value": 1 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 10, + "interval": "$interval", + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"text_channels\"}) + sum(discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"voice_channels\"}) + sum(discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"stage_channels\"})", + "hide": false, + "instant": true, + "legendFormat": "channels", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"categories\"}", + "hide": false, + "instant": true, + "legendFormat": "{% raw %}{{ stat_type }}{% endraw %}", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"text_channels\"}", + "hide": false, + "instant": true, + "legendFormat": "{% raw %}{{ stat_type }}{% endraw %}", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"voice_channels\"}", + "hide": false, + "instant": true, + "legendFormat": "{% raw %}{{ stat_type }}{% endraw %}", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"roles\"}", + "hide": false, + "instant": true, + "legendFormat": "{% raw %}{{ stat_type }}{% endraw %}", + "range": false, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"emojis\"}", + "hide": false, + "instant": true, + "legendFormat": "{% raw %}{{ stat_type }}{% endraw %}", + "range": false, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"animated_emojis\"}", + "hide": false, + "instant": true, + "legendFormat": "{% raw %}{{ stat_type }}{% endraw %}", + "range": false, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"static_emojis\"}", + "hide": false, + "instant": true, + "legendFormat": "{% raw %}{{ stat_type }}{% endraw %}", + "range": false, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "discord_metrics_guild_stats_count{server_id=\"$discord_guild\", stat_type=\"members\"}", + "hide": false, + "instant": true, + "legendFormat": "{% raw %}{{ stat_type }}{% endraw %}", + "range": false, + "refId": "I" + } + ], + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "LabBot Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "dtdurationms" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-red", + "value": 3240000 + }, + { + "color": "dark-orange", + "value": 21240000 + }, + { + "color": "yellow", + "value": 42840000 + }, + { + "color": "green", + "value": 86040000 + } + ] + } + }, + { + "id": "decimals", + "value": 1 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 33, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "alias": "Members", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "discord_metrics_guild_stats_count{server_id=\"184315303323238400\", stat_type=\"members\"}", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "{% raw %}{{ stat_type }}{% endraw %}", + "measurement": "Servers", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Members" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [] + } + ], + "title": "Members", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 6, + "title": "Discord Server Users", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Offline users excluded.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "offline" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 10 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 35, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(discord_metrics_guild_user_status_count{server_id=\"$discord_guild\", client_type=\"total\", status=\"online\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": false, + "instant": false, + "legendFormat": "online", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [] + }, + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(discord_metrics_guild_user_status_count{server_id=\"$discord_guild\", client_type=\"total\", status=\"idle\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": false, + "instant": false, + "legendFormat": "idle", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [] + }, + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(discord_metrics_guild_user_status_count{server_id=\"$discord_guild\", client_type=\"total\", status=\"dnd\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": false, + "instant": false, + "legendFormat": "dnd", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [] + }, + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(discord_metrics_guild_user_status_count{server_id=\"$discord_guild\", client_type=\"total\", status=\"invisible\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": true, + "instant": false, + "legendFormat": "invisible", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [] + }, + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(discord_metrics_guild_user_status_count{server_id=\"$discord_guild\", client_type=\"total\", status=\"offline\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": true, + "instant": false, + "legendFormat": "offline", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [] + } + ], + "title": "Users by Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "offline" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 10 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 14, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "discord_metrics_guild_user_activity_count{server_id=\"$discord_guild\"}", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": false, + "instant": false, + "legendFormat": "{% raw %}{{ activity }}{% endraw %}", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [] + } + ], + "title": "Users by Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 2, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(discord_metrics_guild_user_status_count{server_id=\"$discord_guild\", client_type=\"desktop\", status!=\"offline\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "desktop", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(discord_metrics_guild_user_status_count{server_id=\"$discord_guild\", client_type=\"mobile\", status!=\"offline\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "mobile", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(discord_metrics_guild_user_status_count{server_id=\"$discord_guild\", client_type=\"web\", status!=\"offline\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "web", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "Online", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(discord_metrics_guild_user_status_count{server_id=\"$discord_guild\", client_type=\"total\", status!=\"offline\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "hide": true, + "legendFormat": "total", + "measurement": "-", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "range": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users Online" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Users by Client", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of users currently in a voice channel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 23, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "alias": "Users in Voice", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(discord_metrics_guild_voice_stats_count{server_id=\"$discord_guild\"})", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "legendFormat": "count", + "measurement": "Servers", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "Users in a VC" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [] + } + ], + "title": "Users in Voice", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 8, + "panels": [], + "title": "Discord API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Latency of all shards. [More info](https://discordpy.readthedocs.io/en/stable/api.html#discord.AutoShardedClient.latency).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 12, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "alias": "Shard 1", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "discord_metrics_bot_latency_seconds", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "legendFormat": "{% raw %}{{ __name__ }}{% endraw %}", + "limit": "", + "measurement": "Shard", + "orderByTime": "ASC", + "policy": "default", + "range": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "1" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "tz": "" + } + ], + "title": "Discord API Latency", + "type": "timeseries" + } + ], + "refresh": "auto", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "current": { + "selected": false, + "text": "1m", + "value": "1m" + }, + "hide": 2, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "{{ db_name }}", + "value": "prometheus" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "/^{{ db_name }}$/", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "{{ container_name }}", + "value": "{{ container_name }}" + }, + "hide": 2, + "name": "container_name", + "options": [ + { + "selected": true, + "text": "{{ container_name }}", + "value": "{{ container_name }}" + } + ], + "query": "{{ container_name }}", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "184315303323238400", + "value": "184315303323238400" + }, + "hide": 2, + "name": "discord_guild", + "options": [ + { + "selected": true, + "text": "184315303323238400", + "value": "184315303323238400" + } + ], + "query": "184315303323238400", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "LabBot", + "uid": "bot", + "version": 11, + "weekStart": "" +} diff --git a/roles/labbot/templates/grafana_dashboards/host.json.j2 b/roles/labbot/templates/grafana_dashboards/host.json.j2 new file mode 100644 index 0000000..4374190 --- /dev/null +++ b/roles/labbot/templates/grafana_dashboards/host.json.j2 @@ -0,0 +1,1142 @@ +{% set db_name = 'Prometheus' %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 3, + "links": [ + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "LabBot", + "tooltip": "Open dashboard", + "type": "link", + "url": "https://labbot.tiga.tech/d/bot" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "LabBot Cogs", + "tooltip": "Open GitHub repository", + "type": "link", + "url": "https://github.com/rHomelab/LabBot-Cogs" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Homelab Discord", + "tooltip": "Join the Homelab Discord", + "type": "link", + "url": "https://discord.gg/homelab" + } + ], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "dtdhms" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "time() - node_boot_time_seconds", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "#32ac2df7", + "value": null + }, + { + "color": "#eab839", + "value": 70 + }, + { + "color": "#ff780a", + "value": 80 + }, + { + "color": "#f53636e6", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 5, + "y": 0 + }, + "id": 3, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "CPU", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "#32ac2df7", + "value": null + }, + { + "color": "#eab839", + "value": 70 + }, + { + "color": "#ff780a", + "value": 80 + }, + { + "color": "#f53636e6", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 8, + "y": 0 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 * (1 - ((avg_over_time(node_memory_MemFree_bytes[$__rate_interval]) + avg_over_time(node_memory_Cached_bytes[$__rate_interval]) + avg_over_time(node_memory_Buffers_bytes[$__rate_interval])) / avg_over_time(node_memory_MemTotal_bytes[$__rate_interval])))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "#32ac2df7", + "value": null + }, + { + "color": "#eab839", + "value": 70 + }, + { + "color": "#ff780a", + "value": 80 + }, + { + "color": "#f53636e6", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 11, + "y": 0 + }, + "id": 6, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - ((node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"rootfs\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Disk", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 10, + "x": 14, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_load1", + "instant": false, + "legendFormat": "{% raw %}{{ __name__ }}{% endraw %}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_load5", + "hide": false, + "instant": false, + "legendFormat": "{% raw %}{{ __name__ }}{% endraw %}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_load15", + "hide": false, + "instant": false, + "legendFormat": "{% raw %}{{ __name__ }}{% endraw %}", + "range": true, + "refId": "C" + } + ], + "title": "Load Average", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "#32ac2df7", + "value": null + }, + { + "color": "#eab839", + "value": 70 + }, + { + "color": "#ff780a", + "value": 80 + }, + { + "color": "#f53636e6", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)", + "instant": false, + "legendFormat": "node_cpu_seconds_total", + "range": true, + "refId": "A" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "#32ac2df7", + "value": null + }, + { + "color": "#eab839", + "value": 70 + }, + { + "color": "#ff780a", + "value": 80 + }, + { + "color": "#f53636e6", + "value": 90 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", + "instant": false, + "legendFormat": "node_memory_used_bytes", + "range": true, + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_filesystem_size_bytes{mountpoint=\"/\"} - node_filesystem_free_bytes{mountpoint=\"/\"}", + "instant": false, + "legendFormat": "node_filesystem_used_bytes", + "range": true, + "refId": "A" + } + ], + "title": "Disk Used (/)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 13 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_read_bytes_total[$__rate_interval])", + "instant": false, + "legendFormat": "node_disk_read_bytes_total", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_written_bytes_total[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "node_disk_written_bytes_total", + "range": true, + "refId": "B" + } + ], + "title": "Disk R/W (sda)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 13 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_read_time_seconds_total[$__rate_interval]) / rate(node_disk_reads_completed_total[$__rate_interval])", + "instant": false, + "legendFormat": "node_disk_read_time", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_write_time_seconds_total[$__rate_interval]) / rate(node_disk_writes_completed_total[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "node_disk_write_time", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_disk_io_time_weighted_seconds_total[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "node_disk_io_time_weighted_seconds", + "range": true, + "refId": "C" + } + ], + "title": "Disk I/O (sda)", + "type": "timeseries" + } + ], + "refresh": "auto", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 2, + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "10s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "{{ db_name }}", + "value": "prometheus" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "/^{{ db_name }}$/", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "LabBot Host", + "uid": "host", + "version": 2, + "weekStart": "" +} diff --git a/roles/labbot/templates/grafana_provisioning/dashboards/dashboards.yml.j2 b/roles/labbot/templates/grafana_provisioning/dashboards/dashboards.yml.j2 new file mode 100644 index 0000000..ba70fd2 --- /dev/null +++ b/roles/labbot/templates/grafana_provisioning/dashboards/dashboards.yml.j2 @@ -0,0 +1,15 @@ +{% set dashboards_path = (labbot_containers | selectattr('image', 'search', 'grafana/grafana') | map(attribute='volumes') | + flatten | select('match', '^(?!.*provisioning).*dashboards.*') | map('split', ':') | flatten)[1] %} +--- +apiVersion: 1 + +providers: + - name: dashboards + orgId: 1 + type: file + disableDeletion: true + updateIntervalSeconds: 10 + allowUiUpdates: false + options: + path: {{ dashboards_path }} + foldersFromFilesStructure: false diff --git a/roles/labbot/templates/grafana_provisioning/datasources/monitoring.yml.j2 b/roles/labbot/templates/grafana_provisioning/datasources/monitoring.yml.j2 new file mode 100644 index 0000000..df8ccf1 --- /dev/null +++ b/roles/labbot/templates/grafana_provisioning/datasources/monitoring.yml.j2 @@ -0,0 +1,21 @@ +--- +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + uid: prometheus + url: http://prometheus:9090 + isDefault: true + basicAuth: true + basicAuthUser: admin + jsonData: + httpMethod: POST + prometheusType: Prometheus + prometheusVersion: 2.50.1 + manageAlerts: true + timeInterval: {{ labbot_prometheus_scrape_interval }}s + secureJsonData: + basicAuthPassword: "{{ labbot_prometheus_users[0].password }}" diff --git a/roles/labbot/templates/nginx_grafana.conf.j2 b/roles/labbot/templates/nginx_grafana.conf.j2 new file mode 100644 index 0000000..6c3016a --- /dev/null +++ b/roles/labbot/templates/nginx_grafana.conf.j2 @@ -0,0 +1,63 @@ +{{ ansible_managed | comment }} + +# Handle ACME challenge, redirect all other traffic to https +server { + listen 80; + server_name {{ labbot_grafana_domain }}; + server_tokens off; + + location /.well-known/acme-challenge/ { + root {{ labbot_certbot_webroot_path }}; + } +{% if not acme_only | default(false) %} + + location / { + return 301 https://$host$request_uri; + } +} +server { + listen 443 ssl; + http2 on; + server_name {{ labbot_grafana_domain }}; + server_tokens off; + +{% if cloudflare_ips.json is defined %} + # Cloudflare proxy real IP +{% for ip in cloudflare_ips.json.result.ipv4_cidrs %} + set_real_ip_from {{ ip }}; +{% endfor %} + real_ip_header CF-Connecting-IP; +{% endif %} + # SSL settings + ssl_certificate {{ labbot_certbot_cert_path }}/live/{{ labbot_grafana_domain }}/fullchain.pem; + ssl_certificate_key {{ labbot_certbot_cert_path }}/live/{{ labbot_grafana_domain }}/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers off; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; + ssl_dhparam {{ labbot_certbot_cert_path }}/dhparam.pem; + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + resolver 8.8.8.8 8.8.4.4 valid=300s; + # HSTS + add_header Strict-Transport-Security "max-age=63072000" always; + + # Reverse proxy settings + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 443; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Generic settings + add_header X-Frame-Options SAMEORIGIN; + add_header X-Content-Type-Options nosniff; + client_max_body_size 50m; + + location / { + proxy_pass http://grafana:3000; + } +{% endif %} +} diff --git a/roles/labbot/templates/prometheus-web.yml.j2 b/roles/labbot/templates/prometheus-web.yml.j2 new file mode 100644 index 0000000..8eb1b5e --- /dev/null +++ b/roles/labbot/templates/prometheus-web.yml.j2 @@ -0,0 +1,4 @@ +basic_auth_users: +{% for user in labbot_prometheus_users %} + {{ user.username }}: {{ user.password_bcrypt }} +{% endfor %} diff --git a/roles/labbot/templates/prometheus.yml.j2 b/roles/labbot/templates/prometheus.yml.j2 new file mode 100644 index 0000000..842ac95 --- /dev/null +++ b/roles/labbot/templates/prometheus.yml.j2 @@ -0,0 +1,12 @@ +global: + scrape_interval: {{ labbot_prometheus_scrape_interval }}s # How frequently to scrape targets. + evaluation_interval: 15s # How frequently to evaluate rules. + scrape_timeout: {{ labbot_prometheus_scrape_interval }}s # How long until a scrape request times out. + +scrape_configs: + - job_name: labbot + static_configs: [targets: [labbot:9900]] + - job_name: cadvisor + static_configs: [targets: [cadvisor:8080]] + - job_name: node_exporter + static_configs: [targets: [node-exporter:9100]] diff --git a/roles/labbot/vars/main.yml b/roles/labbot/vars/main.yml new file mode 100644 index 0000000..f2075ca --- /dev/null +++ b/roles/labbot/vars/main.yml @@ -0,0 +1,151 @@ +--- + +labbot_containers: + - name: labbot + image: ghcr.io/rhomelab/red-discordbot:3.5.9 + env: + INSTANCE_NAME: LabBot + PREFIX: '^' + TOKEN: "{{ labbot_discord_token }}" + TEAM_MEMBERS_ARE_OWNERS: 'true' + # Useful if a cog is fatally breaking the bot. + # See other params here: https://github.com/rHomelab/Red-DiscordBot-Docker/blob/main/.github/redbot-arguments.txt + # command: '--no-cogs --load-cogs downloader' + command: '--message-cache-size 2000' # Default 1000 + volumes: ["{{ labbot_app_base_dir }}/labbot:/redbot/data"] + networks: [name: bots] + memory: 2G + memory_swap: 2500M + + - name: prometheus + image: prom/prometheus:v2.53.0 + user: "{{ labbot_prometheus_container_user }}" + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.config.file=/etc/prometheus/web.yml + - --storage.tsdb.retention.time=5y + - --storage.tsdb.wal-compression + volumes: + - "{{ labbot_app_base_dir }}/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro" + - "{{ labbot_app_base_dir }}/prometheus/web.yml:/etc/prometheus/web.yml:ro" + - "{{ labbot_app_base_dir }}/prometheus/data:/prometheus/data" + networks: + - name: web + - name: bots + - name: monitoring + ports: "{{ ['9090:9090'] if labbot_prometheus_open_port else [] }}" + + - name: cadvisor + image: gcr.io/cadvisor/cadvisor:v0.49.1 + privileged: true + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + devices: + - /dev/kmsg + networks: [name: monitoring] + ports: "{{ ['8080:8080'] if labbot_cadvisor_open_port else [] }}" + + - name: node-exporter + image: quay.io/prometheus/node-exporter:v1.8.1 + command: + - --path.rootfs=/host + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --no-collector.arp + - --no-collector.bcache + - --no-collector.bonding + - --no-collector.btrfs + - --no-collector.conntrack + - --no-collector.dmi + - --no-collector.edac + - --no-collector.entropy + - --no-collector.fibrechannel + - --no-collector.filefd + - --no-collector.hwmon + - --no-collector.infiniband + - --no-collector.ipvs + - --no-collector.mdadm + - --no-collector.nfs + - --no-collector.nfsd + - --no-collector.powersupplyclass + - --no-collector.pressure + - --no-collector.rapl + - --no-collector.schedstat + - --no-collector.sockstat + - --no-collector.softnet + - --no-collector.tapestats + - --no-collector.thermal_zone + - --no-collector.time + - --no-collector.timex + - --no-collector.udp_queues + - --no-collector.xfs + - --no-collector.zfs + pid_mode: host + volumes: + - /:/host:ro,rslave + networks: [name: monitoring] + + - name: grafana + image: grafana/grafana:11.1.0 + user: "{{ labbot_grafana_container_user }}" + ports: [3000:3000] + env: + TZ: Europe/London + GF_ANALYTICS_REPORTING_ENABLED: 'false' + GF_AUTH_ANONYMOUS_ENABLED: 'true' + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_DEFAULT_INSTANCE_NAME: LabBot + GF_LOG_MODE: console + GF_SECURITY_ADMIN_USER: "{{ labbot_grafana_username }}" + GF_SECURITY_ADMIN_PASSWORD: "{{ labbot_grafana_password }}" + GF_SECURITY_DISABLE_GRAVATAR: 'true' + GF_SERVER_ENABLE_GZIP: 'true' + volumes: + - "{{ labbot_app_base_dir }}/grafana/data:/var/lib/grafana" + - "{{ labbot_app_base_dir }}/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro" + - "{{ labbot_app_base_dir }}/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro" + - "{{ labbot_app_base_dir }}/grafana/provisioning/alerting:/etc/grafana/provisioning/alerting:ro" + - "{{ labbot_app_base_dir }}/grafana/dashboards:/mnt/dashboards" + - "{{ labbot_app_base_dir }}/grafana/home_dashboard.json:/usr/share/grafana/public/dashboards/home.json:ro" + - "{{ labbot_app_base_dir }}/grafana/labbot-fav32.png:/usr/share/grafana/public/img/fav32.png:ro" + - "{{ labbot_app_base_dir }}/grafana/labbot-fav32.png:/usr/share/grafana/public/img/apple-touch-icon.png:ro" + networks: [name: web] + + +labbot_certbot_webroot_path: /var/www/certbot +labbot_certbot_cert_path: /etc/letsencrypt + +labbot_certbot_www_bind: "{{ labbot_app_base_dir }}/nginx/certbot_www:{{ labbot_certbot_webroot_path }}" +labbot_certbot_cert_bind: "{{ labbot_app_base_dir }}/nginx/certbot_certs:{{ labbot_certbot_cert_path }}" + +labbot_nginx_dhparam_path_host: "{{ labbot_certbot_cert_bind | split(':') | first }}/dhparam.pem" + +labbot_nginx_conf_path_host: "{{ labbot_app_base_dir }}/nginx/conf" + +labbot_nginx_containers: + - name: nginx + image: nginx:1.27-alpine + ports: [80:80, 443:443] + volumes: + - "{{ labbot_nginx_conf_path_host }}:/etc/nginx/conf.d:ro" + - "{{ labbot_certbot_www_bind }}:ro" + - "{{ labbot_certbot_cert_bind }}:ro" + networks: [name: web] + + - name: certbot + image: certbot/certbot:latest + restart_policy: "no" + command: >- + certonly --non-interactive --agree-tos + --webroot --webroot-path {{ labbot_certbot_webroot_path }}/ + -m {{ labbot_certbot_letsencrypt_email }} + -d {{ labbot_grafana_domain }}{{ + ' --dry-run' if labbot_certbot_dry_run else '' }} + volumes: + - "{{ labbot_certbot_www_bind }}" + - "{{ labbot_certbot_cert_bind }}" + - "{{ labbot_app_base_dir }}/nginx/certbot_logs:/var/log/letsencrypt" + state: present diff --git a/roles/labbot/vars/rclone.yml b/roles/labbot/vars/rclone.yml new file mode 100644 index 0000000..be6df34 --- /dev/null +++ b/roles/labbot/vars/rclone.yml @@ -0,0 +1,24 @@ +--- + +labbot_rclone_release: stable + +labbot_install_manpages: true + +labbot_rclone_arch: >- + {{ 'amd64' if ansible_architecture == 'x86_64' + else 'arm64' if ansible_architecture == 'aarch64' + else 'arm-v7' if ansible_architecture == 'armv7l' + else ansible_architecture }} + +labbot_rclone_configs: + - name: backups + properties: + type: webdav + vendor: other + url: "{{ labbot_backup_webdav_url }}" + user: "{{ labbot_backup_webdav_username }}" + pass: "{{ labbot_backup_webdav_password }}" + +labbot_rclone_config_owner: + OWNER: root + GROUP: root