From d53c7af467945892d3583ed2b45557e2700db0be Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sun, 12 Jan 2025 20:25:23 -0500 Subject: [PATCH 01/15] stub out loki role and deprecate new-do-droplet --- new-server-vars.yml => deprecated/new-server-vars.yml | 0 {playbooks => deprecated/playbooks}/new-do-droplet.yml | 0 {roles => deprecated/roles}/new-do-droplet/README.md | 0 .../roles}/new-do-droplet/defaults/main.yml | 0 .../roles}/new-do-droplet/tasks/main.yml | 0 .../new-do-droplet/templates/cloud-config.yml.tpl | 0 playbooks/loki.yml | 10 ++++++++++ 7 files changed, 10 insertions(+) rename new-server-vars.yml => deprecated/new-server-vars.yml (100%) rename {playbooks => deprecated/playbooks}/new-do-droplet.yml (100%) rename {roles => deprecated/roles}/new-do-droplet/README.md (100%) rename {roles => deprecated/roles}/new-do-droplet/defaults/main.yml (100%) rename {roles => deprecated/roles}/new-do-droplet/tasks/main.yml (100%) rename {roles => deprecated/roles}/new-do-droplet/templates/cloud-config.yml.tpl (100%) create mode 100644 playbooks/loki.yml diff --git a/new-server-vars.yml b/deprecated/new-server-vars.yml similarity index 100% rename from new-server-vars.yml rename to deprecated/new-server-vars.yml diff --git a/playbooks/new-do-droplet.yml b/deprecated/playbooks/new-do-droplet.yml similarity index 100% rename from playbooks/new-do-droplet.yml rename to deprecated/playbooks/new-do-droplet.yml diff --git a/roles/new-do-droplet/README.md b/deprecated/roles/new-do-droplet/README.md similarity index 100% rename from roles/new-do-droplet/README.md rename to deprecated/roles/new-do-droplet/README.md diff --git a/roles/new-do-droplet/defaults/main.yml b/deprecated/roles/new-do-droplet/defaults/main.yml similarity index 100% rename from roles/new-do-droplet/defaults/main.yml rename to deprecated/roles/new-do-droplet/defaults/main.yml diff --git a/roles/new-do-droplet/tasks/main.yml b/deprecated/roles/new-do-droplet/tasks/main.yml similarity index 100% rename from roles/new-do-droplet/tasks/main.yml rename to deprecated/roles/new-do-droplet/tasks/main.yml diff --git a/roles/new-do-droplet/templates/cloud-config.yml.tpl b/deprecated/roles/new-do-droplet/templates/cloud-config.yml.tpl similarity index 100% rename from roles/new-do-droplet/templates/cloud-config.yml.tpl rename to deprecated/roles/new-do-droplet/templates/cloud-config.yml.tpl diff --git a/playbooks/loki.yml b/playbooks/loki.yml new file mode 100644 index 0000000..9b54f86 --- /dev/null +++ b/playbooks/loki.yml @@ -0,0 +1,10 @@ +- name: Set up a host as a Loki log aggregator + hosts: all + vars: + ansible_user: admin + roles: + - common + - harden + - docker + - traefik + - loki \ No newline at end of file From 6df2f9c5bcca6e09468cf8c74b084451c39d2528 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sun, 12 Jan 2025 20:44:37 -0500 Subject: [PATCH 02/15] draft new ssh config with hardening --- playbooks/loki.yml | 2 +- requirements.yml | 3 +- roles/ssh-config-and-harden/tasks/main.yml | 68 ++++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 roles/ssh-config-and-harden/tasks/main.yml diff --git a/playbooks/loki.yml b/playbooks/loki.yml index 9b54f86..3179e31 100644 --- a/playbooks/loki.yml +++ b/playbooks/loki.yml @@ -4,7 +4,7 @@ ansible_user: admin roles: - common - - harden + - ssh-config-and-harden - docker - traefik - loki \ No newline at end of file diff --git a/requirements.yml b/requirements.yml index baade1e..0e8ad08 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,4 +1,5 @@ --- collections: - name: community.docker - - name: community.general \ No newline at end of file + - name: community.general + - name: ansible.posix \ No newline at end of file diff --git a/roles/ssh-config-and-harden/tasks/main.yml b/roles/ssh-config-and-harden/tasks/main.yml new file mode 100644 index 0000000..ae84b67 --- /dev/null +++ b/roles/ssh-config-and-harden/tasks/main.yml @@ -0,0 +1,68 @@ +- name: Add all users listed to the host. + ansible.builtin.user: + name: {{ username }} + state: present + groups: sudo + append: true + shell: "/bin/bash" + create_home: true + loop: "{{ gh_user_keys_to_add }}" + loop_control: + loop_var: username + +- name: Create the .ssh folder for each user. + ansible.builtin.file: + path: "/home/{{ username }}/.ssh/" + state: directory + mode: "0700" + owner: "{{ username }}" + group: users + +- name: Set ssh keys from Github for all listed users. + ansible.posix.authorized_key: + user: {{ username }} + state: present + key: https://github.com/{{ username }}.keys + loop: "{{ gh_user_keys_to_add }}" + loop_control: + loop_var: username + +- name: Setup passwordless sudo + ansible.builtin.lineinfile: + path: /etc/sudoers + state: present + regex: '^%sudo' + line: '%sudo ALL=(ALL) NOPASSWD: ALL' + validate: '/usr/sbin/visudo -cf %s' + +- name: Disable password login for everyone + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + state: present + regexp: '^#?PasswordAuthentication' + line: 'PasswordAuthentication no' + validate: "/usr/sbin/sshd -t -f %s" + +- name: Disable login for root + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + state: present + regexp: '^#?PermitRootLogin' + line: 'PermitRootLogin no' + validate: "/usr/sbin/sshd -t -f %s" + +- name: Restart sshd + ansible.builtin.systemd: + name: ssh + daemon_reload: true + state: restarted + +- name: UFW - Allow SSH connections + community.general.ufw: + rule: allow + name: OpenSSH + +- name: UFW - Enable and deny by default + community.general.ufw: + state: enabled + default: deny \ No newline at end of file From 8b80ae70baa2d4928aee04016f0fb1bbca8f43c6 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sun, 12 Jan 2025 20:50:48 -0500 Subject: [PATCH 03/15] fill in some more loki config --- playbooks/loki.yml | 8 ++---- roles/loki/meta/main.yml | 6 +++++ roles/loki/tasks/main.yml | 0 roles/loki/templates/loki-config.yaml | 36 +++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 roles/loki/meta/main.yml create mode 100644 roles/loki/tasks/main.yml create mode 100644 roles/loki/templates/loki-config.yaml diff --git a/playbooks/loki.yml b/playbooks/loki.yml index 3179e31..fc6b584 100644 --- a/playbooks/loki.yml +++ b/playbooks/loki.yml @@ -1,10 +1,6 @@ -- name: Set up a host as a Loki log aggregator +- name: Deploy and Configure Loki hosts: all vars: - ansible_user: admin + placeholder: placeholder roles: - - common - - ssh-config-and-harden - - docker - - traefik - loki \ No newline at end of file diff --git a/roles/loki/meta/main.yml b/roles/loki/meta/main.yml new file mode 100644 index 0000000..145504a --- /dev/null +++ b/roles/loki/meta/main.yml @@ -0,0 +1,6 @@ +--- +dependencies: + - role: common + - role: ssh-config-and-harden + - role: docker + - role: traefik diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml new file mode 100644 index 0000000..e69de29 diff --git a/roles/loki/templates/loki-config.yaml b/roles/loki/templates/loki-config.yaml new file mode 100644 index 0000000..8cfa1a4 --- /dev/null +++ b/roles/loki/templates/loki-config.yaml @@ -0,0 +1,36 @@ +auth_enabled: false # TODO: We'll want auth of some sort here. + +server: + http_listen_port: 3100 + +common: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + replication_factor: 1 + path_prefix: /loki + +schema_config: + configs: + - from: '2020-10-24' + store: boltdb-shipper + object_store: aws + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: /data/loki/boltdb-shipper-active + cache_location: /data/loki/boltdb-shipper-cache + cache_ttl: 24h + shared_store: aws + aws: # TODO: This bucket needs to be created in DO. + bucketnames: # TODO: These need to be properly templated, and defined as vars in the inventory/vault files. + endpoint: + region: + access_key_id: + secret_access_key: + s3forcepathstyle: true \ No newline at end of file From e24634921e8367f04b125095d6bc4da19f5ae672 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Mon, 13 Jan 2025 20:28:45 -0500 Subject: [PATCH 04/15] draft loki configuration --- roles/loki/files/docker-compose.yaml | 24 ++++++++ roles/loki/tasks/main.yml | 55 +++++++++++++++++++ .../{loki-config.yaml => loki-config.tpl} | 12 ++-- 3 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 roles/loki/files/docker-compose.yaml rename roles/loki/templates/{loki-config.yaml => loki-config.tpl} (57%) diff --git a/roles/loki/files/docker-compose.yaml b/roles/loki/files/docker-compose.yaml new file mode 100644 index 0000000..6dbc6b7 --- /dev/null +++ b/roles/loki/files/docker-compose.yaml @@ -0,0 +1,24 @@ +version: '3' +services: + loki: + image: grafana/loki # TODO: Pin rather than latest + container_name: loki + ports: + - "0.0.0.0:3100:3100" + volumes: + - "./loki-config.yaml:/etc/loki/local-config.yaml" + networks: + - proxy + restart: always + labels: + - "traefik.enable=true" + - "traefik.http.routers.myapp.rule=Host(`loki.planetary.tools`)" + - "traefik.http.routers.myapp.entrypoints=websecure" + - "traefik.http.routers.myapp.tls.certresolver=nosresolver" + resources: + limits: + cpus: '2' + memory: 6G +networks: + proxy: + external: true \ No newline at end of file diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml index e69de29..4e3c4eb 100644 --- a/roles/loki/tasks/main.yml +++ b/roles/loki/tasks/main.yml @@ -0,0 +1,55 @@ +- name: Create a user for Loki + become: true + ansible.builtin.user: + name: loki + home: /home/loki + create_home: yes + group: admin + +- name: Create directory for Loki + become: true + ansible.builtin.file: + path: "/home/loki/loki" + state: directory + mode: '0755' + +- name: Clone the Loki repo + become_user: loki + ansible.builtin.git: + repo: https://github.com/grafana/loki.git + dest: "/home/loki/loki" + version: v3.0.0 + +- name: Interpolate Loki Configuration File + become_user: loki + ansible.builtin.template: + src: "loki-config.tpl" + dest: '/home/loki/loki/loki-config.yaml' + mode: '0600' + +- name: Copy the docker-compose.yaml + become_user: loki + ansible.builtin.copy: + src: /files/docker-compose.yaml + dest: /home/loki/loki + mode: '0600' + +- name: Pull down old Loki + community.docker.docker_compose_v2: + project_src: /home/loki/loki + state: absent + +- name: Start new Loki + community.docker.docker_compose_v2: + project_src: /home/loki/loki + wait: true + wait_timeout: 180 + register: output + +- name: Check that Loki is running + ansible.builtin.assert: + that: + - loki_container.State == 'running' + vars: + web_container: >- + {{ output.containers | selectattr("Service", "equalto", "loki") | first }} \ No newline at end of file diff --git a/roles/loki/templates/loki-config.yaml b/roles/loki/templates/loki-config.tpl similarity index 57% rename from roles/loki/templates/loki-config.yaml rename to roles/loki/templates/loki-config.tpl index 8cfa1a4..79fec58 100644 --- a/roles/loki/templates/loki-config.yaml +++ b/roles/loki/templates/loki-config.tpl @@ -27,10 +27,10 @@ storage_config: cache_location: /data/loki/boltdb-shipper-cache cache_ttl: 24h shared_store: aws - aws: # TODO: This bucket needs to be created in DO. - bucketnames: # TODO: These need to be properly templated, and defined as vars in the inventory/vault files. - endpoint: - region: - access_key_id: - secret_access_key: + aws: + bucketnames: {{ do_spaces_bucket_name }} + endpoint: {{ do_spaces_bucket_endpoint }} + region: {{ do_spaces_bucket_region }} + access_key_id: {{ do_spaces_access_key }} + secret_access_key: {{ do_spaces_secret_key }} s3forcepathstyle: true \ No newline at end of file From 45a2295861305d4037bcb627847b6ac79d9b432b Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Mon, 13 Jan 2025 21:10:32 -0500 Subject: [PATCH 05/15] draft loki inventory --- inventories/loki/group_vars/all/vault.yml | 12 ++++++ inventories/loki/inventory.yml | 10 +++++ inventories/metrics/group_vars/all/vault.yml | 44 ++++++++++---------- playbooks/loki.yml | 8 ++-- roles/loki/README.md | 2 + roles/loki/files/docker-compose.yaml | 8 ++-- roles/loki/tasks/main.yml | 6 +++ roles/loki/templates/loki-config.tpl | 3 ++ 8 files changed, 65 insertions(+), 28 deletions(-) create mode 100644 inventories/loki/group_vars/all/vault.yml create mode 100644 inventories/loki/inventory.yml create mode 100644 roles/loki/README.md diff --git a/inventories/loki/group_vars/all/vault.yml b/inventories/loki/group_vars/all/vault.yml new file mode 100644 index 0000000..42c5e4e --- /dev/null +++ b/inventories/loki/group_vars/all/vault.yml @@ -0,0 +1,12 @@ +$ANSIBLE_VAULT;1.1;AES256 +30353938303062346433333531343536373635623830666330666565386534386335323062346333 +3665323566333430323837616236616463353331353932660a613166663834666633613966643631 +63346238653835323861356266363263383563616533326535313233623436366335663435636264 +3330353962663939380a376130616531363235336538663362313163623837366664386465386566 +38643062396335343862613464343739303236323338616138646331386639306663353532343038 +64383736303064396333393534363261356633363035643931633862316537376138303864306363 +36613433333735383537383439633863643737623333306632313635393334363637656337613962 +34353638633533396230333536623635663434333066373966613964323031333965326364386435 +33313232636666356336643134323961663934646533633662653164373064306632613162333136 +30633036633333313033386431326532386162303263633362373762306237373835306635396634 +346663613430636338366662343934383162 diff --git a/inventories/loki/inventory.yml b/inventories/loki/inventory.yml new file mode 100644 index 0000000..47fa7b9 --- /dev/null +++ b/inventories/loki/inventory.yml @@ -0,0 +1,10 @@ +all: + vars: + do_spaces_bucket_name: loki-storage + do_spaces_bucket_endpoint: {{ do_spaces_bucket_endpoint }} + do_spaces_bucket_region: nyc-1 + do_spaces_access_key: {{ do_spaces_access_key }} + do_spaces_secret_key: {{ do_spaces_secret_key }} + loki_password: {{ loki_password }} + hosts: + loki.planetary.tools: \ No newline at end of file diff --git a/inventories/metrics/group_vars/all/vault.yml b/inventories/metrics/group_vars/all/vault.yml index 127d4da..e1bbc91 100644 --- a/inventories/metrics/group_vars/all/vault.yml +++ b/inventories/metrics/group_vars/all/vault.yml @@ -1,23 +1,23 @@ $ANSIBLE_VAULT;1.1;AES256 -32666631616333303464343061653764316464326566663438303437623062383832363232313031 -3335633661643061393063656163616331613230663063350a373938346336323930653030316663 -66313935303035383465353634356466316562663333613361663463646138373361643064636236 -6330303662396337630a633435663430626139396530373262646233613236343562353934383263 -61376565643839306232316362303335336162633733333733363936303637353338656635373737 -63306663663030316462316635383731393161666232333364316261663262383365366363353337 -64383432333337333031633237393737376431656536653232666363633538633330316436396163 -38353237356165393039386261343564623433366666386632633764366535396261306135663836 -32613237393439363066633435396631303938353632613534343837613164303230323632393665 -31313638313937613663646232623335393961626634393030623733363062646137346637383431 -33616336633639643864393539303262303536346665333338306638623037643164656533363538 -32396636333665633262383730346265343135633531666361333165653863346330353934663963 -64613738613364323864313630356530653130376435306332343432633436343338666264336635 -33383065636564633938313130326332316631306466323538353134333030323631626464653961 -64626166623066616436633062356531383033396161383032616133386237633832383337653931 -32626130306434613963393137303563336534373163313661343636613663353832336465386136 -64613831353965663863333165303335303038313163346335343432323266333461353337343932 -33663861316233613062393338343039336538376534393932353939396338613136643466386562 -39616334383633653233323839643334383931353239313036323932353032623563663233383562 -33633531306630343132626432656563383732303766626166326634343165626235363836316662 -64373266316338386463666637326334616333383330333532643339336366363334663262323562 -3730663130376165626438633839626439633933623131613037 +65656565356431383338383962376330303338336532626330383430613936653064666166666339 +6364363432346638653834613735386537633565393837320a623562303034386632616365383161 +33306463343964366438323238383765646538316165383330383937653131343631656362346633 +6464353666316134390a656532333535356633323132343165356232623164666432303437666433 +62393030323636313632616430373931396537373662353434613334353235313336666130376562 +33643334316439303763613132366637366261323432623338653539323066343535313933386534 +34353933656237646566303133626431313865303064326235626538623864336563373139306334 +34363730633533333037623161656466386139666361356261613261643439656564393031633237 +37323838623462393631373064636238343664646239343165353232633736376335643737363733 +31393030383033373161396538386565653531303066333163343530643165613732323633353165 +65306638333239323837306237366530633935313933636562366531373634323963323262633732 +38626138336566623133613239343262643163666134383832313265363133356434366666343462 +32363135336237366439396163383761613935663736626461303937383562343066343436343235 +39316630626361633866303864383633353539376665343730663833623833663134626237353965 +30636366643962306530363538376336373730336632626661366663373864613166643463393636 +61373965633436613164343938626137653636646465613438383661646237636139306263393163 +38373838356632383931373066386234386162326339363962626633303736313132656365316630 +30386362336638343332363830386532616530356434383535613862633235333462346366363936 +61366532666161623563316337663463353931326431363533363239323266633631366336616633 +30383163666337386531326530396434353139353162353730333736313735666561363130626662 +61336130666635336565353531386332383436663739376662643138383363616265656630383934 +3736306462316362373861356239656236343934656238646262 diff --git a/playbooks/loki.yml b/playbooks/loki.yml index fc6b584..66bb5ff 100644 --- a/playbooks/loki.yml +++ b/playbooks/loki.yml @@ -1,6 +1,8 @@ - name: Deploy and Configure Loki hosts: all - vars: - placeholder: placeholder roles: - - loki \ No newline at end of file + - loki + +# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml + + diff --git a/roles/loki/README.md b/roles/loki/README.md new file mode 100644 index 0000000..c038110 --- /dev/null +++ b/roles/loki/README.md @@ -0,0 +1,2 @@ +# Loki +Verse uses Loki for log aggregation, configured with grafana. This role will take a newly-created Droplet (created using the terraform repo) and configure it to host a running Loki server, that uses DigitalOcean Spaces as the block storage location. \ No newline at end of file diff --git a/roles/loki/files/docker-compose.yaml b/roles/loki/files/docker-compose.yaml index 6dbc6b7..58f2fb5 100644 --- a/roles/loki/files/docker-compose.yaml +++ b/roles/loki/files/docker-compose.yaml @@ -12,9 +12,11 @@ services: restart: always labels: - "traefik.enable=true" - - "traefik.http.routers.myapp.rule=Host(`loki.planetary.tools`)" - - "traefik.http.routers.myapp.entrypoints=websecure" - - "traefik.http.routers.myapp.tls.certresolver=nosresolver" + - "traefik.http.routers.loki.rule=Host(`loki.planetary.tools`)" + - "traefik.http.routers.loki.entrypoints=websecure" + - "traefik.http.routers.loki.tls.certresolver=nosresolver" + - "traefik.http.middlewares.webapp-auth.basicauth.users=verse:temp" + - "traefik.http.routers.loki.middlewares=webapp-auth" resources: limits: cpus: '2' diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml index 4e3c4eb..77aee3c 100644 --- a/roles/loki/tasks/main.yml +++ b/roles/loki/tasks/main.yml @@ -34,6 +34,12 @@ dest: /home/loki/loki mode: '0600' +- name: Replace 'temp' with 'loki_password' in docker-compose.yaml + replace: + path: /home/loki/loki/docker-compose.yaml + regexp: 'traefik.http.middlewares.webapp-auth.basicauth.users=verse:temp' + replace: 'traefik.http.middlewares.webapp-auth.basicauth.users=verse:{{ loki_password }}' + - name: Pull down old Loki community.docker.docker_compose_v2: project_src: /home/loki/loki diff --git a/roles/loki/templates/loki-config.tpl b/roles/loki/templates/loki-config.tpl index 79fec58..2241152 100644 --- a/roles/loki/templates/loki-config.tpl +++ b/roles/loki/templates/loki-config.tpl @@ -1,3 +1,6 @@ +# Recommended config pulled from Digital Ocean: +# https://www.digitalocean.com/community/developer-center/how-to-install-loki-stack-in-doks-cluster#step-5-setting-persistent-storage-for-loki + auth_enabled: false # TODO: We'll want auth of some sort here. server: From 69ab69e44f1c35d0b971cf9a3bba3390d17432b0 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Mon, 13 Jan 2025 21:15:23 -0500 Subject: [PATCH 06/15] missed a placeholder in there --- roles/loki/files/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/loki/files/docker-compose.yaml b/roles/loki/files/docker-compose.yaml index 58f2fb5..8bc3201 100644 --- a/roles/loki/files/docker-compose.yaml +++ b/roles/loki/files/docker-compose.yaml @@ -15,8 +15,8 @@ services: - "traefik.http.routers.loki.rule=Host(`loki.planetary.tools`)" - "traefik.http.routers.loki.entrypoints=websecure" - "traefik.http.routers.loki.tls.certresolver=nosresolver" - - "traefik.http.middlewares.webapp-auth.basicauth.users=verse:temp" - - "traefik.http.routers.loki.middlewares=webapp-auth" + - "traefik.http.middlewares.loki-auth.basicauth.users=verse:temp" + - "traefik.http.routers.loki.middlewares=loki-auth" resources: limits: cpus: '2' From fe302cae3535729fcf4ce85e88e24ed48c449ed6 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Tue, 14 Jan 2025 21:20:41 -0500 Subject: [PATCH 07/15] WIP: Stopped short of starting up the loki server --- inventories/loki/group_vars/all/vault.yml | 27 +++++++++++++--------- inventories/loki/inventory.yml | 21 +++++++++++------ playbooks/loki.yml | 2 +- requirements.yml | 3 ++- roles/docker/tasks/main.yml | 7 ++++-- roles/loki/tasks/main.yml | 13 ++++------- roles/ssh-config-and-harden/tasks/main.yml | 15 ++++-------- roles/traefik/tasks/main.yml | 1 - 8 files changed, 48 insertions(+), 41 deletions(-) diff --git a/inventories/loki/group_vars/all/vault.yml b/inventories/loki/group_vars/all/vault.yml index 42c5e4e..017015a 100644 --- a/inventories/loki/group_vars/all/vault.yml +++ b/inventories/loki/group_vars/all/vault.yml @@ -1,12 +1,17 @@ $ANSIBLE_VAULT;1.1;AES256 -30353938303062346433333531343536373635623830666330666565386534386335323062346333 -3665323566333430323837616236616463353331353932660a613166663834666633613966643631 -63346238653835323861356266363263383563616533326535313233623436366335663435636264 -3330353962663939380a376130616531363235336538663362313163623837366664386465386566 -38643062396335343862613464343739303236323338616138646331386639306663353532343038 -64383736303064396333393534363261356633363035643931633862316537376138303864306363 -36613433333735383537383439633863643737623333306632313635393334363637656337613962 -34353638633533396230333536623635663434333066373966613964323031333965326364386435 -33313232636666356336643134323961663934646533633662653164373064306632613162333136 -30633036633333313033386431326532386162303263633362373762306237373835306635396634 -346663613430636338366662343934383162 +33363133356235643965376632653035653963633337363833373236373336623138616463626435 +3239616232343963636330333031633735333263383230390a663966653233643861346532376463 +31373761353763373261303562336438386436316637333232363834653135343133656234306139 +3039643133376339360a343239666566346430363262636638386566653863323330363738343438 +31646337323830393562316336613631383164366563626263626435653365623530383036336232 +34316264663639386638306434376237313362626634323561363931386334633232316439623930 +37353536363435623565363961376538316631666534333930333832306662313862323064363636 +39653235366630373564373834646136303433656230666634383062333635643733346338653238 +62653038313835666663363236623665653738653263663036386431383835616464326435656361 +63323434633738623739356165326233643338396633616562353638623363373139333363346333 +66663135363365343763613730356638623833353763643337353330303566663331633938643364 +64373431623262356463313339393633353931326137333433653330346362313066343236383064 +38343438383733353863646235613831633466396434373732343763653662376661316137313639 +65366135346633636236323934623936656438616562666432613430303636663833626336393633 +66386236396437306466623437613864663564656236396438636465653738343933313861353962 +62366431373464623435 diff --git a/inventories/loki/inventory.yml b/inventories/loki/inventory.yml index 47fa7b9..53e0e69 100644 --- a/inventories/loki/inventory.yml +++ b/inventories/loki/inventory.yml @@ -1,10 +1,17 @@ all: vars: - do_spaces_bucket_name: loki-storage - do_spaces_bucket_endpoint: {{ do_spaces_bucket_endpoint }} - do_spaces_bucket_region: nyc-1 - do_spaces_access_key: {{ do_spaces_access_key }} - do_spaces_secret_key: {{ do_spaces_secret_key }} - loki_password: {{ loki_password }} + do_spaces_bucket_name: verse-loki-storage + do_spaces_bucket_endpoint: "{{ do_spaces_bucket_endpoint }}" + do_spaces_bucket_region: nyc-3 + do_spaces_access_key: "{{ do_spaces_access_key }}" + do_spaces_secret_key: "{{ do_spaces_secret_key }}" + loki_password: "{{ loki_password }}" + homedir: loki + domain: loki.planetary.tools + gh_user_keys_to_add: + - nbenmoody + - mplorentz + - dcadenas hosts: - loki.planetary.tools: \ No newline at end of file + loki.planetary.tools: + ansible_user: root \ No newline at end of file diff --git a/playbooks/loki.yml b/playbooks/loki.yml index 66bb5ff..8843080 100644 --- a/playbooks/loki.yml +++ b/playbooks/loki.yml @@ -3,6 +3,6 @@ roles: - loki -# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml +# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml --private-key /path/to/default-root-ssh-key diff --git a/requirements.yml b/requirements.yml index 0e8ad08..f69437b 100644 --- a/requirements.yml +++ b/requirements.yml @@ -2,4 +2,5 @@ collections: - name: community.docker - name: community.general - - name: ansible.posix \ No newline at end of file + - name: ansible.posix + diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index dfdc28d..ddc9111 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -68,12 +68,15 @@ state: present -- name: Add admin user to Docker +- name: Add all users to Docker become: true ansible.builtin.user: - name: "{{ admin_username }}" + name: "{{ username }}" groups: docker append: true + loop: "{{ gh_user_keys_to_add }}" + loop_control: + loop_var: username - name: Download compose plugin diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml index 77aee3c..578e7b4 100644 --- a/roles/loki/tasks/main.yml +++ b/roles/loki/tasks/main.yml @@ -13,13 +13,6 @@ state: directory mode: '0755' -- name: Clone the Loki repo - become_user: loki - ansible.builtin.git: - repo: https://github.com/grafana/loki.git - dest: "/home/loki/loki" - version: v3.0.0 - - name: Interpolate Loki Configuration File become_user: loki ansible.builtin.template: @@ -30,7 +23,7 @@ - name: Copy the docker-compose.yaml become_user: loki ansible.builtin.copy: - src: /files/docker-compose.yaml + src: "{{ role_path }}/files/docker-compose.yaml" dest: /home/loki/loki mode: '0600' @@ -40,6 +33,10 @@ regexp: 'traefik.http.middlewares.webapp-auth.basicauth.users=verse:temp' replace: 'traefik.http.middlewares.webapp-auth.basicauth.users=verse:{{ loki_password }}' + +#FIXME: Stopped here for tonight +# FAILED! => {"changed": false, "cmd": "/usr/bin/docker --host unix:///var/run/docker.sock compose --ansi never --progress plain --project-directory /home/loki/loki ps --format json --all --no-trunc", "msg": "validating /home/loki/loki/docker-compose.yaml: services.loki Additional property resources is not allowed", "rc": 15, "stderr": "validating /home/loki/loki/docker-compose.yaml: services.loki Additional property resources is not allowed\n", "stderr_lines": ["validating /home/loki/loki/docker-compose.yaml: services.loki Additional property resources is not allowed"], "stdout": "", "stdout_lines": []} + - name: Pull down old Loki community.docker.docker_compose_v2: project_src: /home/loki/loki diff --git a/roles/ssh-config-and-harden/tasks/main.yml b/roles/ssh-config-and-harden/tasks/main.yml index ae84b67..17a1ee7 100644 --- a/roles/ssh-config-and-harden/tasks/main.yml +++ b/roles/ssh-config-and-harden/tasks/main.yml @@ -1,6 +1,6 @@ - name: Add all users listed to the host. ansible.builtin.user: - name: {{ username }} + name: "{{ username }}" state: present groups: sudo append: true @@ -17,10 +17,13 @@ mode: "0700" owner: "{{ username }}" group: users + loop: "{{ gh_user_keys_to_add }}" + loop_control: + loop_var: username - name: Set ssh keys from Github for all listed users. ansible.posix.authorized_key: - user: {{ username }} + user: "{{ username }}" state: present key: https://github.com/{{ username }}.keys loop: "{{ gh_user_keys_to_add }}" @@ -43,14 +46,6 @@ line: 'PasswordAuthentication no' validate: "/usr/sbin/sshd -t -f %s" -- name: Disable login for root - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - state: present - regexp: '^#?PermitRootLogin' - line: 'PermitRootLogin no' - validate: "/usr/sbin/sshd -t -f %s" - - name: Restart sshd ansible.builtin.systemd: name: ssh diff --git a/roles/traefik/tasks/main.yml b/roles/traefik/tasks/main.yml index d00b6db..d7f870c 100644 --- a/roles/traefik/tasks/main.yml +++ b/roles/traefik/tasks/main.yml @@ -30,7 +30,6 @@ force: false mode: 0600 - - name: Copy necessary template files to traefik dir ansible.builtin.template: src: "{{ item.src }}" From 1dcccce58f89cde0878223019653c6ae45e2a032 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sat, 18 Jan 2025 10:48:12 -0500 Subject: [PATCH 08/15] wrap up loki configuration --- inventories/loki/group_vars/all/vault.yml | 65 ++++++++++++++----- roles/loki/tasks/main.yml | 55 ++++++++-------- .../docker-compose.tpl} | 8 +-- roles/loki/templates/loki-config.tpl | 18 +++-- 4 files changed, 89 insertions(+), 57 deletions(-) rename roles/loki/{files/docker-compose.yaml => templates/docker-compose.tpl} (89%) diff --git a/inventories/loki/group_vars/all/vault.yml b/inventories/loki/group_vars/all/vault.yml index 017015a..88c1d8d 100644 --- a/inventories/loki/group_vars/all/vault.yml +++ b/inventories/loki/group_vars/all/vault.yml @@ -1,17 +1,50 @@ $ANSIBLE_VAULT;1.1;AES256 -33363133356235643965376632653035653963633337363833373236373336623138616463626435 -3239616232343963636330333031633735333263383230390a663966653233643861346532376463 -31373761353763373261303562336438386436316637333232363834653135343133656234306139 -3039643133376339360a343239666566346430363262636638386566653863323330363738343438 -31646337323830393562316336613631383164366563626263626435653365623530383036336232 -34316264663639386638306434376237313362626634323561363931386334633232316439623930 -37353536363435623565363961376538316631666534333930333832306662313862323064363636 -39653235366630373564373834646136303433656230666634383062333635643733346338653238 -62653038313835666663363236623665653738653263663036386431383835616464326435656361 -63323434633738623739356165326233643338396633616562353638623363373139333363346333 -66663135363365343763613730356638623833353763643337353330303566663331633938643364 -64373431623262356463313339393633353931326137333433653330346362313066343236383064 -38343438383733353863646235613831633466396434373732343763653662376661316137313639 -65366135346633636236323934623936656438616562666432613430303636663833626336393633 -66386236396437306466623437613864663564656236396438636465653738343933313861353962 -62366431373464623435 +36313434323630633731303130323733333634633237396565346563316566303533363834396438 +6531373666393763353238303930333762343536663861380a306138346364393466363632653137 +36326464393837386365623863646464633165333537623932613830626239363534643965343639 +3435633062323932330a653832646630616633653866656563633839316433336439366135666534 +30393337346631626133653737343664356639623939653566383066346238623830393566343165 +61396333373463643032373231373361376637316362663639386634653430623737323964303138 +61643362303330346635663161623037653237323666333436363234656436636366383735643937 +61363936643930313431613933346336363463613735363636393538306265313733663839323465 +35353434303635393734316231356632633531616363313764323633393433383463653835383137 +62396661653966656237613162373032383537356265316561646435643734346634346134386136 +36303735393663656233386665336639313662316338643562613338646337663665656265653134 +63396265663834303962353035623239656234663537303330383138303964323062616664373332 +33383361633030646530643365313234336266646434613165323738303031303432646633623531 +65396432343436383661653263353634616131656235616362306566373135303364383934363438 +33303163393630393632613033636233396166626534323632353333333133613133333464613463 +33353337353831623666366365646137326433313638636163663839313733633133316135313034 +63366263326133333163323733366366383231396136383066353662633031623662353033363161 +33366536356333646438663364633061303439396232326665353464303632653330626265636232 +36616233306135326164353663313435323532343935636465376133393662363766616330656332 +66306331646264343930663538373766346662636662623438646235646633346166396565656164 +39616631373862613534393763623162653636396636333137333365313637623230303331333438 +38363530346464646663393165613065313130383736316436353530303134386438613431643936 +33373431303130313263646633363033646166353533323466626365666333613962643966646461 +63386361636135363463363533326162333136336364316465343038636636396230316634643466 +38613064326239323664663364366534376633396364643066633861393434316536646563383839 +64623736643339353763303135363764346233633062376332653835383432663432666163633934 +35616239663638316462643462333763356338376461393335363437316265336637656434346664 +39363437326139313233323139313261396262373830356630643862383537303066313263316366 +31366162343239653939393538386531343365613534383265613632623964636664643835373665 +61636634663664613965353964383064326139363964643861653532653939366366336461343563 +35626638363932346564323033396132666631653439616565353433373938383730363565393365 +32306561326365393762303663313765356431306530663836396637666364343361633535646165 +35313866323233376637663839666535323062366235323561666538616633613933643733666535 +30343132363734353763356533623833366261623039333562643337366165353139363035313664 +36633865346235303635323737353033633331643766623738353735643035643531643734363030 +38323837386230646439303237666239386531626633663338663562666332376537336137653964 +66663230636563383234323538636664653034633962313432663630346133396165373632643930 +38333963623639326431616533653431386439313735353433393334643830363636333362646266 +64623962636336343337363161316632333365623337653962336636666439316130336564323133 +34373164396431386433356136356631616161323731336265346165363266383762393239326630 +33363636636563336665336664633236373733633136643137386362386264333361376233336139 +32613739393238653466623666306535323566396263333633336563313835633237363538343763 +34303536306330663433353266656531646337366661363739333163623739633338323437363366 +61633766376239386466653739666266386531623961613634326130636464303632643334343732 +66373732323964613739633737353263393938333735663862613762663337376134363630393833 +66653439323662666263373131333838396166376130363830633230343536306563396164303935 +61376363656565616165663938623432393062653532663865616633623736363334313132653834 +35353762633938333666653761613131333463626535613637326637656361373266613132306633 +3937 diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml index 578e7b4..8fea485 100644 --- a/roles/loki/tasks/main.yml +++ b/roles/loki/tasks/main.yml @@ -13,46 +13,49 @@ state: directory mode: '0755' -- name: Interpolate Loki Configuration File - become_user: loki +- name: Interpolate Loki configuration file + become: true ansible.builtin.template: src: "loki-config.tpl" dest: '/home/loki/loki/loki-config.yaml' + owner: loki + mode: '0777' + +- name: Interpolate docker-compose manifest + become: true + ansible.builtin.template: + src: "docker-compose.tpl" + dest: '/home/loki/loki/docker-compose.yaml' + owner: loki mode: '0600' -- name: Copy the docker-compose.yaml - become_user: loki - ansible.builtin.copy: - src: "{{ role_path }}/files/docker-compose.yaml" - dest: /home/loki/loki - mode: '0600' - -- name: Replace 'temp' with 'loki_password' in docker-compose.yaml - replace: - path: /home/loki/loki/docker-compose.yaml - regexp: 'traefik.http.middlewares.webapp-auth.basicauth.users=verse:temp' - replace: 'traefik.http.middlewares.webapp-auth.basicauth.users=verse:{{ loki_password }}' - - -#FIXME: Stopped here for tonight -# FAILED! => {"changed": false, "cmd": "/usr/bin/docker --host unix:///var/run/docker.sock compose --ansi never --progress plain --project-directory /home/loki/loki ps --format json --all --no-trunc", "msg": "validating /home/loki/loki/docker-compose.yaml: services.loki Additional property resources is not allowed", "rc": 15, "stderr": "validating /home/loki/loki/docker-compose.yaml: services.loki Additional property resources is not allowed\n", "stderr_lines": ["validating /home/loki/loki/docker-compose.yaml: services.loki Additional property resources is not allowed"], "stdout": "", "stdout_lines": []} - - name: Pull down old Loki + become: true community.docker.docker_compose_v2: project_src: /home/loki/loki state: absent - name: Start new Loki + become_user: loki community.docker.docker_compose_v2: project_src: /home/loki/loki wait: true wait_timeout: 180 register: output -- name: Check that Loki is running - ansible.builtin.assert: - that: - - loki_container.State == 'running' - vars: - web_container: >- - {{ output.containers | selectattr("Service", "equalto", "loki") | first }} \ No newline at end of file +# BUG: This isn't currently working. Encounters some sort of error with the selectattr +# - name: Check if Loki container is running +# become: true +# ansible.builtin.assert: +# that: +# - (output.containers | selectattr("Name", "equalto", "loki") | first).State == 'running' +# fail_msg: "Loki container is not running" + + +# - name: Check that Loki is running +# ansible.builtin.assert: +# that: +# - loki_container.State == 'running' +# vars: +# loki_container: >- +# {{ output.containers | selectattr("Service", "equalto", "loki") | first }} \ No newline at end of file diff --git a/roles/loki/files/docker-compose.yaml b/roles/loki/templates/docker-compose.tpl similarity index 89% rename from roles/loki/files/docker-compose.yaml rename to roles/loki/templates/docker-compose.tpl index 8bc3201..2f41487 100644 --- a/roles/loki/files/docker-compose.yaml +++ b/roles/loki/templates/docker-compose.tpl @@ -3,6 +3,8 @@ services: loki: image: grafana/loki # TODO: Pin rather than latest container_name: loki + mem_limit: 6g + cpus: 2.0 ports: - "0.0.0.0:3100:3100" volumes: @@ -15,12 +17,8 @@ services: - "traefik.http.routers.loki.rule=Host(`loki.planetary.tools`)" - "traefik.http.routers.loki.entrypoints=websecure" - "traefik.http.routers.loki.tls.certresolver=nosresolver" - - "traefik.http.middlewares.loki-auth.basicauth.users=verse:temp" + - "traefik.http.middlewares.loki-auth.basicauth.users=verse:{{ loki_password_hashed_escaped }}" - "traefik.http.routers.loki.middlewares=loki-auth" - resources: - limits: - cpus: '2' - memory: 6G networks: proxy: external: true \ No newline at end of file diff --git a/roles/loki/templates/loki-config.tpl b/roles/loki/templates/loki-config.tpl index 2241152..9ad93e4 100644 --- a/roles/loki/templates/loki-config.tpl +++ b/roles/loki/templates/loki-config.tpl @@ -1,7 +1,7 @@ # Recommended config pulled from Digital Ocean: # https://www.digitalocean.com/community/developer-center/how-to-install-loki-stack-in-doks-cluster#step-5-setting-persistent-storage-for-loki -auth_enabled: false # TODO: We'll want auth of some sort here. +auth_enabled: false server: http_listen_port: 3100 @@ -16,20 +16,18 @@ common: schema_config: configs: - - from: '2020-10-24' - store: boltdb-shipper - object_store: aws - schema: v11 + - from: 2020-05-15 + store: tsdb + object_store: s3 + schema: v13 index: prefix: index_ period: 24h storage_config: - boltdb_shipper: - active_index_directory: /data/loki/boltdb-shipper-active - cache_location: /data/loki/boltdb-shipper-cache - cache_ttl: 24h - shared_store: aws + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache aws: bucketnames: {{ do_spaces_bucket_name }} endpoint: {{ do_spaces_bucket_endpoint }} From cc742d65da8c9f8d521277d82fe8c7c192352e3c Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sat, 18 Jan 2025 10:57:08 -0500 Subject: [PATCH 09/15] update to the container healthcheck task for loki --- roles/loki/tasks/main.yml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml index 8fea485..88b185e 100644 --- a/roles/loki/tasks/main.yml +++ b/roles/loki/tasks/main.yml @@ -43,19 +43,10 @@ wait_timeout: 180 register: output -# BUG: This isn't currently working. Encounters some sort of error with the selectattr -# - name: Check if Loki container is running -# become: true -# ansible.builtin.assert: -# that: -# - (output.containers | selectattr("Name", "equalto", "loki") | first).State == 'running' -# fail_msg: "Loki container is not running" - - -# - name: Check that Loki is running -# ansible.builtin.assert: -# that: -# - loki_container.State == 'running' -# vars: -# loki_container: >- -# {{ output.containers | selectattr("Service", "equalto", "loki") | first }} \ No newline at end of file +- name: Check if Loki container is running + community.docker.docker_container_info: + name: "loki" + until: "container_info.container.State.Status == 'running'" + register: container_info + retries: 15 + delay: 10 From 28ff1b68d83a2019ffdc8841ac45d2d3985f9b72 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sat, 18 Jan 2025 11:03:33 -0500 Subject: [PATCH 10/15] add ansible_ssh_key for others to use --- README.md | 5 +++++ ansible_ssh_key | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 ansible_ssh_key diff --git a/README.md b/README.md index fddbd8b..abccfae 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ published here in case they are useful for other scuttlebutt/nostr users. 1. Install Ansible and Ansible Galaxy 2. Run `ansible-galaxy install -r requirements.yml` +# Running ansible against Droplets created under the `terraform` repository +All new droplets created using the [terraform repo](https://github.com/verse-pbc/terraform) will require the use of the [ansible_ssh_key](./ansible_ssh_key). +This key is encrypted with ansible-vault, and can be decrypted with a call to `ansible-vault decrypt ./ansible_ssh_key`, so you can use it locally while executing +playbooks. Just remember to always re-encrypt it before pushing to Github. + # Repo structure Ansible is structured around running playbooks against an inventory of servers. Our repo is mainly structured, around our diff --git a/ansible_ssh_key b/ansible_ssh_key new file mode 100644 index 0000000..5100715 --- /dev/null +++ b/ansible_ssh_key @@ -0,0 +1,44 @@ +$ANSIBLE_VAULT;1.1;AES256 +36373066653564633361396164636162346562346334323832323339616465373539363333646637 +3531363662343866313138663838323064353234353832380a303030313634613233393334623761 +35363735333832326666313639626539336462396335306231356665636536623339663934393739 +6638653663326435350a363634343935393834363230633933633430646464613838656139663865 +66366562306638623465343162373264363733653334363231613338633131393536383562303034 +34643165346531616633633539623231343530613664623131333035663434393666396137316533 +38663532363763356339613265643264663738383464306165613932643236373637633731373632 +37633533336363313864313136363036303535633733623234613133383563353938343663633464 +39363139376237656665383131333933633034303264636239396262623436663031363534373637 +63306161386534373165663065353330386431626539316561636132326537386432623163363530 +37616365653161323138636264383862363137366465313532633931626334666263353235323236 +39653338323834633465353731316536646332663030353237353662343064396361303264623530 +37633736653462363165373631353838623933643634343766666462643236373734313564393437 +35623762353237353864346665653934383936373462636438303064396161386334626135386236 +31643733376263333039346138646564656262333666376362666162316466386561373164613532 +33333763616232326431313730643234666133666532313035343535313232616331616362656235 +35363466663361323735323034316236386164393963326137616634326333656337656135626164 +64636462323433363537313866633837326238393739356561623531636339303565666235393234 +30663737363939306638343035346435383265356666666339363165626465313134343862356536 +64396234626537633932323531646638626564316535363962323932336330303232353265646539 +65666531623765313934383066653838636336613666323833343139353338323830333734383363 +37343061393338653461343033623638383336383163366137663965336666643961663833373333 +37663162373961383965666238626566356333393661363563363334393662393965663466613863 +61633433333964346164626136613663366538313162336334356230303637653938663330653936 +36373334636563656266316638303661313865663737633761373961396461633738376263663832 +37323231353262343235346334303134323534663436303232333434643566393238346465386362 +62316435333639353330343165313835353162363266383032336630663661383839303564653564 +35626436303961656635343762326466326230303466323932323738643637623434386435636162 +36653137363331666665326233353637316336626633393363633338646138656636613764393938 +37626239343261393239303265633239343363306566316636656131336564373830356232383533 +36613564323764623332333962343336303362613362666531306132333365666134303864353662 +32323231613965353436633133613933333861656633306330393234343834386462383735373766 +37353732373033396234366633643936656536656136346661623535383832616235373430656263 +63613437333761656265353966306132353961316134393862396136316138623538313038393262 +39623263326563306539616162616462613735633235626631313235363531366338353031306137 +62623139663966383934656637363432366431386439353737306232373331626231616436306563 +66616466643538653634646435643734373836373562323665313462613233653665303939366432 +36313635386238393064643930373332613533313263623961326636383962323731333163316430 +33616137383631623465316634613361306433336663636436623334653262373464303532643734 +30323034646338343031343462373039363064646630376633363132653961393438373432346466 +35323238333363363833353962303937653939656532313630363661653634306566303732313236 +66326639333939633934613962633161313233313437353032363061306637656335346631316234 +353763646534623261363332663464393539 From e6fa7111dc445a469c0155f04c67e974c25804e2 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sun, 19 Jan 2025 16:28:25 -0500 Subject: [PATCH 11/15] draft alloy scripts --- .../loki-alloy/group_vars/all/vault.yml | 10 ++ inventories/loki-alloy/inventory.yml | 8 ++ inventories/loki/group_vars/all/vault.yml | 98 +++++++++---------- inventories/loki/inventory.yml | 1 + playbooks/loki-alloy.yml | 6 ++ playbooks/loki.yml | 4 +- roles/loki-alloy/README.md | 2 + roles/loki-alloy/tasks/main.yml | 70 +++++++++++++ roles/loki-alloy/templates/config.alloy.tpl | 23 +++++ 9 files changed, 170 insertions(+), 52 deletions(-) create mode 100644 inventories/loki-alloy/group_vars/all/vault.yml create mode 100644 inventories/loki-alloy/inventory.yml create mode 100644 playbooks/loki-alloy.yml create mode 100644 roles/loki-alloy/README.md create mode 100644 roles/loki-alloy/tasks/main.yml create mode 100644 roles/loki-alloy/templates/config.alloy.tpl diff --git a/inventories/loki-alloy/group_vars/all/vault.yml b/inventories/loki-alloy/group_vars/all/vault.yml new file mode 100644 index 0000000..ea8212a --- /dev/null +++ b/inventories/loki-alloy/group_vars/all/vault.yml @@ -0,0 +1,10 @@ +$ANSIBLE_VAULT;1.1;AES256 +33306266316437353030353330303034376433613635393637333131663663316265373961366564 +3636316236333562613463353263343464666331306138650a633831396436323439663362633935 +33363032613562626238343536373938326238383833323366323561643834643036363065343432 +3639643235376263370a356364346632383939306538323933626233636133623635383366643162 +63346163636666326264656462623463663939336164666361353739393330303365656563653263 +34323037613864373933336335333534323037623637333065343266343161316263663638656435 +63663665346638303063363762643464343462366562363237626265313437623162623234653366 +39616165636536643538373737633030376464623735393036653537346532343438363331346265 +3637 diff --git a/inventories/loki-alloy/inventory.yml b/inventories/loki-alloy/inventory.yml new file mode 100644 index 0000000..af95d51 --- /dev/null +++ b/inventories/loki-alloy/inventory.yml @@ -0,0 +1,8 @@ +all: + vars: + loki_password_hashed_escaped: "{{ loki_password_hashed_escaped }}" + hosts: + relay.nos.social: + ansible_user: admin + loki.planetary.tools: + ansible_user: root \ No newline at end of file diff --git a/inventories/loki/group_vars/all/vault.yml b/inventories/loki/group_vars/all/vault.yml index 88c1d8d..f242ea6 100644 --- a/inventories/loki/group_vars/all/vault.yml +++ b/inventories/loki/group_vars/all/vault.yml @@ -1,50 +1,50 @@ $ANSIBLE_VAULT;1.1;AES256 -36313434323630633731303130323733333634633237396565346563316566303533363834396438 -6531373666393763353238303930333762343536663861380a306138346364393466363632653137 -36326464393837386365623863646464633165333537623932613830626239363534643965343639 -3435633062323932330a653832646630616633653866656563633839316433336439366135666534 -30393337346631626133653737343664356639623939653566383066346238623830393566343165 -61396333373463643032373231373361376637316362663639386634653430623737323964303138 -61643362303330346635663161623037653237323666333436363234656436636366383735643937 -61363936643930313431613933346336363463613735363636393538306265313733663839323465 -35353434303635393734316231356632633531616363313764323633393433383463653835383137 -62396661653966656237613162373032383537356265316561646435643734346634346134386136 -36303735393663656233386665336639313662316338643562613338646337663665656265653134 -63396265663834303962353035623239656234663537303330383138303964323062616664373332 -33383361633030646530643365313234336266646434613165323738303031303432646633623531 -65396432343436383661653263353634616131656235616362306566373135303364383934363438 -33303163393630393632613033636233396166626534323632353333333133613133333464613463 -33353337353831623666366365646137326433313638636163663839313733633133316135313034 -63366263326133333163323733366366383231396136383066353662633031623662353033363161 -33366536356333646438663364633061303439396232326665353464303632653330626265636232 -36616233306135326164353663313435323532343935636465376133393662363766616330656332 -66306331646264343930663538373766346662636662623438646235646633346166396565656164 -39616631373862613534393763623162653636396636333137333365313637623230303331333438 -38363530346464646663393165613065313130383736316436353530303134386438613431643936 -33373431303130313263646633363033646166353533323466626365666333613962643966646461 -63386361636135363463363533326162333136336364316465343038636636396230316634643466 -38613064326239323664663364366534376633396364643066633861393434316536646563383839 -64623736643339353763303135363764346233633062376332653835383432663432666163633934 -35616239663638316462643462333763356338376461393335363437316265336637656434346664 -39363437326139313233323139313261396262373830356630643862383537303066313263316366 -31366162343239653939393538386531343365613534383265613632623964636664643835373665 -61636634663664613965353964383064326139363964643861653532653939366366336461343563 -35626638363932346564323033396132666631653439616565353433373938383730363565393365 -32306561326365393762303663313765356431306530663836396637666364343361633535646165 -35313866323233376637663839666535323062366235323561666538616633613933643733666535 -30343132363734353763356533623833366261623039333562643337366165353139363035313664 -36633865346235303635323737353033633331643766623738353735643035643531643734363030 -38323837386230646439303237666239386531626633663338663562666332376537336137653964 -66663230636563383234323538636664653034633962313432663630346133396165373632643930 -38333963623639326431616533653431386439313735353433393334643830363636333362646266 -64623962636336343337363161316632333365623337653962336636666439316130336564323133 -34373164396431386433356136356631616161323731336265346165363266383762393239326630 -33363636636563336665336664633236373733633136643137386362386264333361376233336139 -32613739393238653466623666306535323566396263333633336563313835633237363538343763 -34303536306330663433353266656531646337366661363739333163623739633338323437363366 -61633766376239386466653739666266386531623961613634326130636464303632643334343732 -66373732323964613739633737353263393938333735663862613762663337376134363630393833 -66653439323662666263373131333838396166376130363830633230343536306563396164303935 -61376363656565616165663938623432393062653532663865616633623736363334313132653834 -35353762633938333666653761613131333463626535613637326637656361373266613132306633 -3937 +64393530333533613263316561666462376631666337366665383332333939616164663563383734 +6365616430616138613361333033306434376230306231650a656130373739393131663064613336 +31383462653035663961353863363738333630383230303661336139333933306364636161333137 +6661306465633332300a333838396430383535376332343433656636323137363533326362613236 +62626635663536373834363832633661366366393738393866383563353835643465316230396333 +66633863613738616432376634393762633039336434633230386332663034326535643166636661 +61616235666465663666303437613734393238356231663039333466376632663036623165383435 +65613233613762356661613838393731313031383764666638613462383836353463383130393439 +38336235373336366335623665386131653938356235656539333232353938656638323064353533 +38336430663436666236616234393663306637373863306137656465636538386336663362386231 +30636637326535336132636633643538623430383338343861366333353937326339616530663034 +35396363396637643330323730316461316462343433316630386536333634323833363538373936 +35353835306236633732626132653639636165333539313338393066373063393330323934353130 +66323365376264653930386133666138623530316634396538313133613465656137623533323534 +66373737623964643334376463666562633862333163663561626666326535313961373063333633 +66393130393863313266306163326439636666643332386564613031636531616437323332316138 +38666530343731316439333931353564323563656463326630623035333234323538636137666562 +66643661396534393863343532636565343032353338636137373335376562663538336539383435 +31663933336465636466376163306336373639613162656661343866656135353532666439323034 +33646366363264623361616336363032316166383961616430646236666139323261636139643732 +65616539643465343331333232313564383534653666373731343932653936623462626537626661 +34306566616331623466303936393330363235336266626138336235303930326533396435373937 +37623331396162653335653366386136336461626634343135386630363630336362386665616439 +37383431316533623964613233656363356534633462376336363130373631336436383333356163 +64366636313735316165363261396364346132653762313664306531306165373434326636643466 +30353263343564373539346433623464383336393431346564393136363635623135346132393534 +38636663636165666239373535373032376133363635636635363139303963616366653263333735 +63613662633530313264646337383865303833376134666530303635626333356339623938663032 +64313765623936353034383133333562643936663662643135303539383536313839333566333333 +61383234636262656135323762383233636466313635646639393434646638373630376264396639 +37383364653039376663366533646664626361396435343162336165616262313066633833633366 +66636238386333323030383665666537343736316439303333393834393161363764343434646530 +32363165656131333661393730353932393633643965316262653431373138353664626562373764 +66663936313835653366623530313133653964633136353539653736613666616135323863663739 +37623333303836613731303130386562316133393836633763343535306235326536386235613634 +34643061353138343936636633323336373233616563343263363161343537633139383836343365 +37316465663965323062343632393866333030316665646435393438373039313565626661656535 +37323030363363653361343535666431323866633966363264616331653334636566353865636139 +34613234343766663263333362343837366463306635663038383039343832346134386235643633 +39303131363535313739346637313133633631303366316237643432313133626365303739316238 +34376533333231323962343264623361636463313338393631333331303664373230633564323564 +63323537356630663332663661373463663762343862656566613134383438353335643333383239 +36323033666534623164336332363439623833356234346137363733383561336234303038323835 +36313436356638666232373334303630626561363238343831373236663539346230323565313461 +64633330363434643737656466666534326464663331303262613939623738363830356136643863 +63373736616565313436623465373832653835383466316339346664383637666137316261386231 +35666237346337306331633063623165623032626531343030333936656332363664353862356130 +36666166353433353037626238353466316466383132613733336164383862343639363939393765 +6565 diff --git a/inventories/loki/inventory.yml b/inventories/loki/inventory.yml index 53e0e69..d34a7ed 100644 --- a/inventories/loki/inventory.yml +++ b/inventories/loki/inventory.yml @@ -6,6 +6,7 @@ all: do_spaces_access_key: "{{ do_spaces_access_key }}" do_spaces_secret_key: "{{ do_spaces_secret_key }}" loki_password: "{{ loki_password }}" + loki_password_hashed_escaped: "{{ loki_password_hashed_escaped }}" homedir: loki domain: loki.planetary.tools gh_user_keys_to_add: diff --git a/playbooks/loki-alloy.yml b/playbooks/loki-alloy.yml new file mode 100644 index 0000000..b7972a9 --- /dev/null +++ b/playbooks/loki-alloy.yml @@ -0,0 +1,6 @@ +- name: Deploy and Configure the Grafana Alloy Agent + hosts: all + roles: + - loki-alloy + +# Deployment: ansible-playbook -i inventories/loki-alloy playbooks/loki-alloy.yml \ No newline at end of file diff --git a/playbooks/loki.yml b/playbooks/loki.yml index 8843080..af0d1f5 100644 --- a/playbooks/loki.yml +++ b/playbooks/loki.yml @@ -3,6 +3,4 @@ roles: - loki -# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml --private-key /path/to/default-root-ssh-key - - +# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml --private-key /path/to/default-root-ssh-key \ No newline at end of file diff --git a/roles/loki-alloy/README.md b/roles/loki-alloy/README.md new file mode 100644 index 0000000..a985ee9 --- /dev/null +++ b/roles/loki-alloy/README.md @@ -0,0 +1,2 @@ +# Loki Alloy +Verse uses the Grafana Alloy agent (Grafana Labs's distribution of the Open Telemetry Collector), to collect logs from target hosts and ship them to our Loki instance, which itself is configured as a datasource in our Grafana instance, for use there. This role installs the Alloy agent on the target host and configures it to push logs to our Loki instance for docker containers. \ No newline at end of file diff --git a/roles/loki-alloy/tasks/main.yml b/roles/loki-alloy/tasks/main.yml new file mode 100644 index 0000000..9e6ebc6 --- /dev/null +++ b/roles/loki-alloy/tasks/main.yml @@ -0,0 +1,70 @@ +- name: Create a user for Alloy + become: true + ansible.builtin.user: + name: alloy + home: /home/alloy + create_home: yes + group: admin + +- name: Create directory for Alloy + become: true + ansible.builtin.file: + path: "/home/alloy/grafana_alloy" + state: directory + mode: '0744' + +- name: Ensure gpg is installed + become: true + ansible.builtin.apt: + pkg: + - gpg + state: present + +- name: Import Alloy GPG Key + become: true + ansible.builtin.command: + cmd: | + mkdir -p /etc/apt/keyrings/ && \ + wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null && \ + echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee /etc/apt/sources.list.d/grafana.list + +- name: Update apt cache + become: true + ansible.builtin.apt: + update_cache: true + +- name: Install Alloy + become: true + ansible.builtin.apt: + pkg: + - alloy + state: present + autoremove: true + +- name: Interpolate Alloy configuration file + become: true + ansible.builtin.template: + src: "config.alloy.tpl" + dest: '/etc/alloy/config.alloy' + owner: alloy + mode: '0600' + +- name: Enable and start Alloy + become: true + ansible.builtin.systemd_service: + name: alloy + state: started + enabled: true + +- name: Wait for Alloy to start + become: true + ansible.builtin.command: + cmd: systemctl status alloy + register: result + until: + - "'Active: active (running)' in result.stdout" + retries: 10 + delay: 5 + failed_when: + - "'Active: active (running)' not in result.stdout" + changed_when: false \ No newline at end of file diff --git a/roles/loki-alloy/templates/config.alloy.tpl b/roles/loki-alloy/templates/config.alloy.tpl new file mode 100644 index 0000000..3226f47 --- /dev/null +++ b/roles/loki-alloy/templates/config.alloy.tpl @@ -0,0 +1,23 @@ +discovery.docker "linux_host" { + host = "unix:///var/run/docker.sock" +} + +loki.source.docker "all_containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.docker.linux_host.targets + labels = { + "source" = "docker" + "host" = "{{ inventory_hostname }}" + } + forward_to = [loki.write.verse_loki_endpoint.receiver] +} + +loki.write "verse_loki_endpoint" { + endpoint { + url = "loki.planetary.tools:3100/loki/api/v1/push" + basic_auth { + username = "verse" + password = "{{ loki_password_hashed_escaped }}" + } + } +} \ No newline at end of file From 2ffd594a1ef9be1eea80ff1fe38a407fb15d6b29 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Mon, 20 Jan 2025 15:14:09 -0500 Subject: [PATCH 12/15] remove the alloy user creation, which conflicted with apt --- roles/loki-alloy/tasks/main.yml | 37 +++++++++------------ roles/loki-alloy/templates/config.alloy.tpl | 4 +-- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/roles/loki-alloy/tasks/main.yml b/roles/loki-alloy/tasks/main.yml index 9e6ebc6..cb1cb1b 100644 --- a/roles/loki-alloy/tasks/main.yml +++ b/roles/loki-alloy/tasks/main.yml @@ -1,18 +1,3 @@ -- name: Create a user for Alloy - become: true - ansible.builtin.user: - name: alloy - home: /home/alloy - create_home: yes - group: admin - -- name: Create directory for Alloy - become: true - ansible.builtin.file: - path: "/home/alloy/grafana_alloy" - state: directory - mode: '0744' - - name: Ensure gpg is installed become: true ansible.builtin.apt: @@ -22,11 +7,13 @@ - name: Import Alloy GPG Key become: true - ansible.builtin.command: - cmd: | - mkdir -p /etc/apt/keyrings/ && \ - wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null && \ - echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee /etc/apt/sources.list.d/grafana.list + ansible.builtin.shell: | + mkdir -p /etc/apt/keyrings/ && + wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | tee /etc/apt/keyrings/grafana.gpg > /dev/null && + echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | tee /etc/apt/sources.list.d/grafana.list + register: result + changed_when: result.rc != 0 + failed_when: result.rc != 0 - name: Update apt cache become: true @@ -46,8 +33,9 @@ ansible.builtin.template: src: "config.alloy.tpl" dest: '/etc/alloy/config.alloy' + mode: '0640' owner: alloy - mode: '0600' + group: alloy - name: Enable and start Alloy become: true @@ -55,6 +43,11 @@ name: alloy state: started enabled: true + daemon_reload: true + +- name: Wait for 10 seconds before checking Alloy status + pause: + seconds: 10 - name: Wait for Alloy to start become: true @@ -63,7 +56,7 @@ register: result until: - "'Active: active (running)' in result.stdout" - retries: 10 + retries: 3 delay: 5 failed_when: - "'Active: active (running)' not in result.stdout" diff --git a/roles/loki-alloy/templates/config.alloy.tpl b/roles/loki-alloy/templates/config.alloy.tpl index 3226f47..45865b3 100644 --- a/roles/loki-alloy/templates/config.alloy.tpl +++ b/roles/loki-alloy/templates/config.alloy.tpl @@ -6,8 +6,8 @@ loki.source.docker "all_containers" { host = "unix:///var/run/docker.sock" targets = discovery.docker.linux_host.targets labels = { - "source" = "docker" - "host" = "{{ inventory_hostname }}" + "source" = "docker", + "host" = "{{ inventory_hostname }}", } forward_to = [loki.write.verse_loki_endpoint.receiver] } From 5c570f21272422161a4c53f6dacb7b9f035bff08 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Tue, 21 Jan 2025 20:33:55 -0500 Subject: [PATCH 13/15] another pass --- .../followers_server/group_vars/all/vault.yml | 26 ++++---- .../loki-alloy/group_vars/all/vault.yml | 17 +++-- inventories/loki/group_vars/all/vault.yml | 66 +++++-------------- roles/loki-alloy/tasks/main.yml | 7 ++ roles/loki-alloy/templates/config.alloy.tpl | 45 ++++++++++--- roles/loki/templates/docker-compose.tpl | 8 +-- roles/traefik/vars/main.yml | 20 +++--- 7 files changed, 95 insertions(+), 94 deletions(-) diff --git a/inventories/followers_server/group_vars/all/vault.yml b/inventories/followers_server/group_vars/all/vault.yml index c1cf6b5..ad6bb5c 100644 --- a/inventories/followers_server/group_vars/all/vault.yml +++ b/inventories/followers_server/group_vars/all/vault.yml @@ -1,14 +1,14 @@ $ANSIBLE_VAULT;1.1;AES256 -34343963653734633631653433363330613161643164623038663463303464326461663261623732 -3632366436363931623736663337343063633739303036640a383938633231636463316533383633 -62636632333463353132336262396438343438393364636339313633346466653964636430313530 -3832303366636665620a323762663830346464393734303066363038303336663563393431333664 -34653066666533366130336462316535366434346135666431316561333037313833656532663533 -31616466373962313564323563623763316333633164623661313734393334643435323066653464 -65313165343031623763626465386163613838383133386233376563363064656363393764393164 -33346564363864653933313934633239636266333938316463353064386330613038386362613830 -30383631363932623237333832636137353634366363626562343964623666356537383762633535 -35313231343934313036373461653230656361663931653935356561323631653833373261333163 -62343665373861333337316232643366386331323339626430316466353237386538303438656464 -64303030343636356630633034396538623435656435353765373836623738356262643264343031 -3134 +31363030313061623966386534373563343763376338613033373434336636343236386663366566 +6462636239303565383739376439366331636432396233350a613737366235626337663132316264 +65663739343133323033303464313066323635383062303138663934353738626563363539656339 +6531333337346237320a333239616534616430383166646539333062393832666639393533623139 +37383934316362663436376632323832346563393939353835323031363464316137396163376430 +37663166333665393364353330303035633937383832303135366631303464356663636438656436 +34616537376362306135326564623265323663663431346263356633613064396464663965633164 +64656439396330383434376266333765303461623965356431333338333465396337333630383235 +33313565636332306630663733343565363565366137613362356639386236626433373330303536 +64383661623135613838316164616430313365613932316338343936336630613431366530373338 +30313133326330326337373662323133303238386264383439613335386531303631343561373134 +31383966343832386231383263356632393633646164373736656230623434393864643138363932 +3635 diff --git a/inventories/loki-alloy/group_vars/all/vault.yml b/inventories/loki-alloy/group_vars/all/vault.yml index ea8212a..430b03b 100644 --- a/inventories/loki-alloy/group_vars/all/vault.yml +++ b/inventories/loki-alloy/group_vars/all/vault.yml @@ -1,10 +1,9 @@ $ANSIBLE_VAULT;1.1;AES256 -33306266316437353030353330303034376433613635393637333131663663316265373961366564 -3636316236333562613463353263343464666331306138650a633831396436323439663362633935 -33363032613562626238343536373938326238383833323366323561643834643036363065343432 -3639643235376263370a356364346632383939306538323933626233636133623635383366643162 -63346163636666326264656462623463663939336164666361353739393330303365656563653263 -34323037613864373933336335333534323037623637333065343266343161316263663638656435 -63663665346638303063363762643464343462366562363237626265313437623162623234653366 -39616165636536643538373737633030376464623735393036653537346532343438363331346265 -3637 +31313536656632373238623439393462623131326438396462636634653166666333313139313065 +6336653834636637373166376339653131343262313431350a643936663464613862343134373530 +31326237356132386363386265366636653431303061326466633833353833383662393062313031 +6363663162643165320a353264646664363665323334306361356564393665373837646630613035 +61646537653638303736623831326164663831623361616632373137383539653961303435353465 +30613137356163633230383665323535343763666338393030396366323463366261633863643663 +37343965366562396263303166386334343830623065333339323565363036373661383630633036 +38313966353033343364 diff --git a/inventories/loki/group_vars/all/vault.yml b/inventories/loki/group_vars/all/vault.yml index f242ea6..81062b3 100644 --- a/inventories/loki/group_vars/all/vault.yml +++ b/inventories/loki/group_vars/all/vault.yml @@ -1,50 +1,18 @@ $ANSIBLE_VAULT;1.1;AES256 -64393530333533613263316561666462376631666337366665383332333939616164663563383734 -6365616430616138613361333033306434376230306231650a656130373739393131663064613336 -31383462653035663961353863363738333630383230303661336139333933306364636161333137 -6661306465633332300a333838396430383535376332343433656636323137363533326362613236 -62626635663536373834363832633661366366393738393866383563353835643465316230396333 -66633863613738616432376634393762633039336434633230386332663034326535643166636661 -61616235666465663666303437613734393238356231663039333466376632663036623165383435 -65613233613762356661613838393731313031383764666638613462383836353463383130393439 -38336235373336366335623665386131653938356235656539333232353938656638323064353533 -38336430663436666236616234393663306637373863306137656465636538386336663362386231 -30636637326535336132636633643538623430383338343861366333353937326339616530663034 -35396363396637643330323730316461316462343433316630386536333634323833363538373936 -35353835306236633732626132653639636165333539313338393066373063393330323934353130 -66323365376264653930386133666138623530316634396538313133613465656137623533323534 -66373737623964643334376463666562633862333163663561626666326535313961373063333633 -66393130393863313266306163326439636666643332386564613031636531616437323332316138 -38666530343731316439333931353564323563656463326630623035333234323538636137666562 -66643661396534393863343532636565343032353338636137373335376562663538336539383435 -31663933336465636466376163306336373639613162656661343866656135353532666439323034 -33646366363264623361616336363032316166383961616430646236666139323261636139643732 -65616539643465343331333232313564383534653666373731343932653936623462626537626661 -34306566616331623466303936393330363235336266626138336235303930326533396435373937 -37623331396162653335653366386136336461626634343135386630363630336362386665616439 -37383431316533623964613233656363356534633462376336363130373631336436383333356163 -64366636313735316165363261396364346132653762313664306531306165373434326636643466 -30353263343564373539346433623464383336393431346564393136363635623135346132393534 -38636663636165666239373535373032376133363635636635363139303963616366653263333735 -63613662633530313264646337383865303833376134666530303635626333356339623938663032 -64313765623936353034383133333562643936663662643135303539383536313839333566333333 -61383234636262656135323762383233636466313635646639393434646638373630376264396639 -37383364653039376663366533646664626361396435343162336165616262313066633833633366 -66636238386333323030383665666537343736316439303333393834393161363764343434646530 -32363165656131333661393730353932393633643965316262653431373138353664626562373764 -66663936313835653366623530313133653964633136353539653736613666616135323863663739 -37623333303836613731303130386562316133393836633763343535306235326536386235613634 -34643061353138343936636633323336373233616563343263363161343537633139383836343365 -37316465663965323062343632393866333030316665646435393438373039313565626661656535 -37323030363363653361343535666431323866633966363264616331653334636566353865636139 -34613234343766663263333362343837366463306635663038383039343832346134386235643633 -39303131363535313739346637313133633631303366316237643432313133626365303739316238 -34376533333231323962343264623361636463313338393631333331303664373230633564323564 -63323537356630663332663661373463663762343862656566613134383438353335643333383239 -36323033666534623164336332363439623833356234346137363733383561336234303038323835 -36313436356638666232373334303630626561363238343831373236663539346230323565313461 -64633330363434643737656466666534326464663331303262613939623738363830356136643863 -63373736616565313436623465373832653835383466316339346664383637666137316261386231 -35666237346337306331633063623165623032626531343030333936656332363664353862356130 -36666166353433353037626238353466316466383132613733336164383862343639363939393765 -6565 +37303137636532643332643231386664666635313839623931643137626464616234663135343836 +6565326331353739616434633062396339386534363836380a636134343234333339386230313838 +31373231343938313833303330376463396536303039323864623831323130653262306234393632 +3064303563336532660a383336396333646436613236333239343062303930643166323034366534 +36343435363335663362626534623933626266303762393730616536393364633836663933333438 +38653566376434646338666632396666353531613630326538306636666362323866616364613931 +66343137653039613062326530633133323931653038636339336239333933333032343566343434 +31623735363933386336353966663634653336313433393533636438323664636437653834653335 +37346662373638626433363763303635663864363963626138643234383963326439316664336361 +37646662393035366236633163353665656266326261303966383366336332323531326266353264 +35386661303666663439313361336538356437653565333863636133613532653266626361343939 +30353331653332363564383836373834306562383432333237653032626366333630656366653033 +34643130343635623963643937636633663534306433353539386363643933653532653662333962 +39653066303733393139663931333937623234653261393936326366653832653737333236383734 +61383162303963353337326430343763343961303063666438656661623634623533336132343832 +30386566623165333638313865393565333234663130323736363238633434336235643430336366 +63333266343430623439376533316366663238653832303537636130353832653931 diff --git a/roles/loki-alloy/tasks/main.yml b/roles/loki-alloy/tasks/main.yml index cb1cb1b..db9c318 100644 --- a/roles/loki-alloy/tasks/main.yml +++ b/roles/loki-alloy/tasks/main.yml @@ -37,6 +37,13 @@ owner: alloy group: alloy +- name: Add the alloy user to the docker group + become: true + ansible.builtin.user: + name: alloy + groups: docker + append: true + - name: Enable and start Alloy become: true ansible.builtin.systemd_service: diff --git a/roles/loki-alloy/templates/config.alloy.tpl b/roles/loki-alloy/templates/config.alloy.tpl index 45865b3..34b4a18 100644 --- a/roles/loki-alloy/templates/config.alloy.tpl +++ b/roles/loki-alloy/templates/config.alloy.tpl @@ -2,22 +2,49 @@ discovery.docker "linux_host" { host = "unix:///var/run/docker.sock" } +discovery.relabel "logs_integrations_docker" { + targets = [] + + + rule { + target_label = "job" + replacement = "integrations/docker" + } + + + rule { + target_label = "instance" + replacement = constants.hostname + } + + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + + rule { + source_labels = ["__meta_docker_container_log_stream"] + target_label = "stream" + } +} + loki.source.docker "all_containers" { - host = "unix:///var/run/docker.sock" - targets = discovery.docker.linux_host.targets - labels = { - "source" = "docker", - "host" = "{{ inventory_hostname }}", - } + host = "unix:///var/run/docker.sock" + targets = discovery.docker.linux_host.targets + relabel_rules = discovery.relabel.logs_integrations_docker.rules + refresh_interval = "5s" forward_to = [loki.write.verse_loki_endpoint.receiver] } loki.write "verse_loki_endpoint" { endpoint { - url = "loki.planetary.tools:3100/loki/api/v1/push" + url = "https://loki.planetary.tools/loki/api/v1/push" basic_auth { - username = "verse" - password = "{{ loki_password_hashed_escaped }}" + username = "{{ vault_traefik_user }}" + password = "{{ vault_traefik_password | password_hash(hashtype='md5') }}" } } } \ No newline at end of file diff --git a/roles/loki/templates/docker-compose.tpl b/roles/loki/templates/docker-compose.tpl index 2f41487..1374e34 100644 --- a/roles/loki/templates/docker-compose.tpl +++ b/roles/loki/templates/docker-compose.tpl @@ -6,7 +6,7 @@ services: mem_limit: 6g cpus: 2.0 ports: - - "0.0.0.0:3100:3100" + - "3100:3100" volumes: - "./loki-config.yaml:/etc/loki/local-config.yaml" networks: @@ -16,9 +16,9 @@ services: - "traefik.enable=true" - "traefik.http.routers.loki.rule=Host(`loki.planetary.tools`)" - "traefik.http.routers.loki.entrypoints=websecure" - - "traefik.http.routers.loki.tls.certresolver=nosresolver" - - "traefik.http.middlewares.loki-auth.basicauth.users=verse:{{ loki_password_hashed_escaped }}" - - "traefik.http.routers.loki.middlewares=loki-auth" + - "traefik.http.routers.loki.tls.certresolver=letsencrypt" + - "traefik.http.middlewares.user-auth.basicauth.users={{ vault_traefik_user }}:{{ vault_traefik_password | password_hash(hashtype='md5') }}" + - "traefik.http.services.loki.loadbalancer.server.port=3100" networks: proxy: external: true \ No newline at end of file diff --git a/roles/traefik/vars/main.yml b/roles/traefik/vars/main.yml index 63da391..0381029 100644 --- a/roles/traefik/vars/main.yml +++ b/roles/traefik/vars/main.yml @@ -1,11 +1,11 @@ $ANSIBLE_VAULT;1.1;AES256 -32613239626262383064663234323364663832333963653564383938306238386131383633383436 -3038303136616233363033663331346539666266383633310a366164653031633265363561366537 -62363730373233623133386332343933636136303436383035623461343930396533633633353364 -6138396337666466330a636639303033313362623661303235643433653662306330393432386563 -33386534666138663332623338613532373238306130326664366433383730386235303635663063 -65326238613231663333663565353066336136633236326338383962663062363766666265303932 -36616134323832343730663539646366643832333639666136373665613533333738353635663138 -39303033656135366166383134303263366637643836316332346138363435323233333961343764 -30643466323539386462316136396539363661366630373438306334626565353236613364653733 -3532363535323534306337333539643666393066326232383434 +30646134346163323134333765373033313738323665393038663364623538663465373466653939 +6533386430663235666635366639623162306538326163650a366166363266656330353137653966 +37366666373839366438623935336532663930326465326137306231366634363637383831643437 +6338363163613932300a356130373639356166393864396437383262316538316533373231343261 +65363030356632366330353961643534643934326438393932643062303466653838373931316130 +30306165333835353662613362333634633930376339646331323231336636313538663466393262 +61653561323730373261653136616434643863303663303833383836396462383863616662636164 +63323763663362303630363161343330306566316264323963626434346133376361356138363763 +38643664356439663735633065646565396463383162373661383961626165336230306536656132 +6430633431613734633432633066383938353630626362343232 From ad236eeeb2abce522b04bce68c31417151219a8f Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sat, 25 Jan 2025 20:55:08 -0500 Subject: [PATCH 14/15] Allow for updates to alloy config on extant services --- playbooks/loki.yml | 2 +- roles/loki-alloy/tasks/main.yml | 9 ++++ roles/loki-alloy/templates/config.alloy.tpl | 54 ++++++++------------- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/playbooks/loki.yml b/playbooks/loki.yml index af0d1f5..2ae5161 100644 --- a/playbooks/loki.yml +++ b/playbooks/loki.yml @@ -3,4 +3,4 @@ roles: - loki -# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml --private-key /path/to/default-root-ssh-key \ No newline at end of file +# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml \ No newline at end of file diff --git a/roles/loki-alloy/tasks/main.yml b/roles/loki-alloy/tasks/main.yml index db9c318..173a720 100644 --- a/roles/loki-alloy/tasks/main.yml +++ b/roles/loki-alloy/tasks/main.yml @@ -44,6 +44,15 @@ groups: docker append: true +- name: Stop the alloy systemd service if it exists + become: true + ansible.builtin.systemd_service: + name: alloy + state: stopped + enabled: false + daemon_reload: true + ignore_errors: true + - name: Enable and start Alloy become: true ansible.builtin.systemd_service: diff --git a/roles/loki-alloy/templates/config.alloy.tpl b/roles/loki-alloy/templates/config.alloy.tpl index 34b4a18..72dabfa 100644 --- a/roles/loki-alloy/templates/config.alloy.tpl +++ b/roles/loki-alloy/templates/config.alloy.tpl @@ -1,42 +1,30 @@ -discovery.docker "linux_host" { - host = "unix:///var/run/docker.sock" +discovery.docker "docker_containers" { + host = "unix:///var/run/docker.sock" } -discovery.relabel "logs_integrations_docker" { - targets = [] - - - rule { - target_label = "job" - replacement = "integrations/docker" - } - - - rule { - target_label = "instance" - replacement = constants.hostname - } - - - rule { - source_labels = ["__meta_docker_container_name"] - regex = "/(.*)" - target_label = "container" - } +discovery.relabel "docker_containers" { + targets = discovery.docker.docker_containers.targets + rule { + source_labels = ["__meta_docker_container_name"] + target_label = "container" + } +} - rule { - source_labels = ["__meta_docker_container_log_stream"] - target_label = "stream" - } +loki.source.docker "docker_logs" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.docker_containers.output + forward_to = [loki.process.process_logs.receiver] } -loki.source.docker "all_containers" { - host = "unix:///var/run/docker.sock" - targets = discovery.docker.linux_host.targets - relabel_rules = discovery.relabel.logs_integrations_docker.rules - refresh_interval = "5s" - forward_to = [loki.write.verse_loki_endpoint.receiver] +loki.process "process_logs" { + stage.docker { } + stage.static_labels { + values = { + hostname = "{{ inventory_hostname }}", + } + } + forward_to = [loki.write.verse_loki_endpoint.receiver] } loki.write "verse_loki_endpoint" { From cd5b237c5e2ef041617ce5b9892aa70dc89f1aa9 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Tue, 28 Jan 2025 20:53:32 -0500 Subject: [PATCH 15/15] Update readme for the short term --- README.md | 9 ++++++--- ansible_ssh_key | 44 -------------------------------------------- 2 files changed, 6 insertions(+), 47 deletions(-) delete mode 100644 ansible_ssh_key diff --git a/README.md b/README.md index abccfae..bde993c 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,12 @@ published here in case they are useful for other scuttlebutt/nostr users. 2. Run `ansible-galaxy install -r requirements.yml` # Running ansible against Droplets created under the `terraform` repository -All new droplets created using the [terraform repo](https://github.com/verse-pbc/terraform) will require the use of the [ansible_ssh_key](./ansible_ssh_key). -This key is encrypted with ansible-vault, and can be decrypted with a call to `ansible-vault decrypt ./ansible_ssh_key`, so you can use it locally while executing -playbooks. Just remember to always re-encrypt it before pushing to Github. +All new droplets created using the [terraform repo](https://github.com/verse-pbc/terraform) will begin with only `DEFAULT_DROPLET_ROOT_SSH_KEY`, which is +securely stored in DigitalOcean. Individual admin users are created on a Droplet, and have their SSH keys added, through +the [ssh-config-and-harden](./roles/ssh-config-and-harden) role. In the next few week, this playbook will be automatically +executed against new Droplets upon creation, so user SSH access for running Ansible scripts against new Droplets will +be available for those configured during Droplet-creation. In the short-term, though, you can message Ben, and he can +execute the `ssh-config-and-harden` role against the new Droplet to grant access. # Repo structure diff --git a/ansible_ssh_key b/ansible_ssh_key deleted file mode 100644 index 5100715..0000000 --- a/ansible_ssh_key +++ /dev/null @@ -1,44 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -36373066653564633361396164636162346562346334323832323339616465373539363333646637 -3531363662343866313138663838323064353234353832380a303030313634613233393334623761 -35363735333832326666313639626539336462396335306231356665636536623339663934393739 -6638653663326435350a363634343935393834363230633933633430646464613838656139663865 -66366562306638623465343162373264363733653334363231613338633131393536383562303034 -34643165346531616633633539623231343530613664623131333035663434393666396137316533 -38663532363763356339613265643264663738383464306165613932643236373637633731373632 -37633533336363313864313136363036303535633733623234613133383563353938343663633464 -39363139376237656665383131333933633034303264636239396262623436663031363534373637 -63306161386534373165663065353330386431626539316561636132326537386432623163363530 -37616365653161323138636264383862363137366465313532633931626334666263353235323236 -39653338323834633465353731316536646332663030353237353662343064396361303264623530 -37633736653462363165373631353838623933643634343766666462643236373734313564393437 -35623762353237353864346665653934383936373462636438303064396161386334626135386236 -31643733376263333039346138646564656262333666376362666162316466386561373164613532 -33333763616232326431313730643234666133666532313035343535313232616331616362656235 -35363466663361323735323034316236386164393963326137616634326333656337656135626164 -64636462323433363537313866633837326238393739356561623531636339303565666235393234 -30663737363939306638343035346435383265356666666339363165626465313134343862356536 -64396234626537633932323531646638626564316535363962323932336330303232353265646539 -65666531623765313934383066653838636336613666323833343139353338323830333734383363 -37343061393338653461343033623638383336383163366137663965336666643961663833373333 -37663162373961383965666238626566356333393661363563363334393662393965663466613863 -61633433333964346164626136613663366538313162336334356230303637653938663330653936 -36373334636563656266316638303661313865663737633761373961396461633738376263663832 -37323231353262343235346334303134323534663436303232333434643566393238346465386362 -62316435333639353330343165313835353162363266383032336630663661383839303564653564 -35626436303961656635343762326466326230303466323932323738643637623434386435636162 -36653137363331666665326233353637316336626633393363633338646138656636613764393938 -37626239343261393239303265633239343363306566316636656131336564373830356232383533 -36613564323764623332333962343336303362613362666531306132333365666134303864353662 -32323231613965353436633133613933333861656633306330393234343834386462383735373766 -37353732373033396234366633643936656536656136346661623535383832616235373430656263 -63613437333761656265353966306132353961316134393862396136316138623538313038393262 -39623263326563306539616162616462613735633235626631313235363531366338353031306137 -62623139663966383934656637363432366431386439353737306232373331626231616436306563 -66616466643538653634646435643734373836373562323665313462613233653665303939366432 -36313635386238393064643930373332613533313263623961326636383962323731333163316430 -33616137383631623465316634613361306433336663636436623334653262373464303532643734 -30323034646338343031343462373039363064646630376633363132653961393438373432346466 -35323238333363363833353962303937653939656532313630363661653634306566303732313236 -66326639333939633934613962633161313233313437353032363061306637656335346631316234 -353763646534623261363332663464393539