Configure loki

README.md

@@ -10,6 +10,14 @@ published here in case they are useful for other scuttlebutt/nostr users.
 1. Install Ansible and Ansible Galaxy
 2. Run `ansible-galaxy install -r requirements.yml`
 
+# Running ansible against Droplets created under the `terraform` repository
+All new droplets created using the [terraform repo](https://github.com/verse-pbc/terraform) will begin with only `DEFAULT_DROPLET_ROOT_SSH_KEY`, which is 
+securely stored in DigitalOcean. Individual admin users are created on a Droplet, and have their SSH keys added, through
+the [ssh-config-and-harden](./roles/ssh-config-and-harden) role. In the next few week, this playbook will be automatically
+executed against new Droplets upon creation, so user SSH access for running Ansible scripts against new Droplets will
+be available for those configured during Droplet-creation. In the short-term, though, you can message Ben, and he can
+execute the `ssh-config-and-harden` role against the new Droplet to grant access.
+
 # Repo structure
 
 Ansible is structured around running playbooks against an inventory of servers.  Our repo is mainly structured, around our

inventories/followers_server/group_vars/all/vault.yml

@@ -1,14 +1,14 @@
 $ANSIBLE_VAULT;1.1;AES256
-34343963653734633631653433363330613161643164623038663463303464326461663261623732
-3632366436363931623736663337343063633739303036640a383938633231636463316533383633
-62636632333463353132336262396438343438393364636339313633346466653964636430313530
-3832303366636665620a323762663830346464393734303066363038303336663563393431333664
-34653066666533366130336462316535366434346135666431316561333037313833656532663533
-31616466373962313564323563623763316333633164623661313734393334643435323066653464
-65313165343031623763626465386163613838383133386233376563363064656363393764393164
-33346564363864653933313934633239636266333938316463353064386330613038386362613830
-30383631363932623237333832636137353634366363626562343964623666356537383762633535
-35313231343934313036373461653230656361663931653935356561323631653833373261333163
-62343665373861333337316232643366386331323339626430316466353237386538303438656464
-64303030343636356630633034396538623435656435353765373836623738356262643264343031
-3134
+31363030313061623966386534373563343763376338613033373434336636343236386663366566
+6462636239303565383739376439366331636432396233350a613737366235626337663132316264
+65663739343133323033303464313066323635383062303138663934353738626563363539656339
+6531333337346237320a333239616534616430383166646539333062393832666639393533623139
+37383934316362663436376632323832346563393939353835323031363464316137396163376430
+37663166333665393364353330303035633937383832303135366631303464356663636438656436
+34616537376362306135326564623265323663663431346263356633613064396464663965633164
+64656439396330383434376266333765303461623965356431333338333465396337333630383235
+33313565636332306630663733343565363565366137613362356639386236626433373330303536
+64383661623135613838316164616430313365613932316338343936336630613431366530373338
+30313133326330326337373662323133303238386264383439613335386531303631343561373134
+31383966343832386231383263356632393633646164373736656230623434393864643138363932
+3635

inventories/loki-alloy/group_vars/all/vault.yml

@@ -0,0 +1,9 @@
+$ANSIBLE_VAULT;1.1;AES256
+31313536656632373238623439393462623131326438396462636634653166666333313139313065
+6336653834636637373166376339653131343262313431350a643936663464613862343134373530
+31326237356132386363386265366636653431303061326466633833353833383662393062313031
+6363663162643165320a353264646664363665323334306361356564393665373837646630613035
+61646537653638303736623831326164663831623361616632373137383539653961303435353465
+30613137356163633230383665323535343763666338393030396366323463366261633863643663
+37343965366562396263303166386334343830623065333339323565363036373661383630633036
+38313966353033343364

inventories/loki-alloy/inventory.yml

@@ -0,0 +1,8 @@
+all:
+  vars:
+    loki_password_hashed_escaped: "{{ loki_password_hashed_escaped }}"
+  hosts:
+    relay.nos.social:
+      ansible_user: admin
+    loki.planetary.tools:
+      ansible_user: root

inventories/loki/group_vars/all/vault.yml

@@ -0,0 +1,18 @@
+$ANSIBLE_VAULT;1.1;AES256
+37303137636532643332643231386664666635313839623931643137626464616234663135343836
+6565326331353739616434633062396339386534363836380a636134343234333339386230313838
+31373231343938313833303330376463396536303039323864623831323130653262306234393632
+3064303563336532660a383336396333646436613236333239343062303930643166323034366534
+36343435363335663362626534623933626266303762393730616536393364633836663933333438
+38653566376434646338666632396666353531613630326538306636666362323866616364613931
+66343137653039613062326530633133323931653038636339336239333933333032343566343434
+31623735363933386336353966663634653336313433393533636438323664636437653834653335
+37346662373638626433363763303635663864363963626138643234383963326439316664336361
+37646662393035366236633163353665656266326261303966383366336332323531326266353264
+35386661303666663439313361336538356437653565333863636133613532653266626361343939
+30353331653332363564383836373834306562383432333237653032626366333630656366653033
+34643130343635623963643937636633663534306433353539386363643933653532653662333962
+39653066303733393139663931333937623234653261393936326366653832653737333236383734
+61383162303963353337326430343763343961303063666438656661623634623533336132343832
+30386566623165333638313865393565333234663130323736363238633434336235643430336366
+63333266343430623439376533316366663238653832303537636130353832653931

inventories/loki/inventory.yml

@@ -0,0 +1,18 @@
+all:
+  vars:
+    do_spaces_bucket_name: verse-loki-storage
+    do_spaces_bucket_endpoint: "{{ do_spaces_bucket_endpoint }}"
+    do_spaces_bucket_region: nyc-3
+    do_spaces_access_key: "{{ do_spaces_access_key }}"
+    do_spaces_secret_key: "{{ do_spaces_secret_key }}"
+    loki_password: "{{ loki_password }}"
+    loki_password_hashed_escaped: "{{ loki_password_hashed_escaped }}"
+    homedir: loki
+    domain: loki.planetary.tools
+    gh_user_keys_to_add:
+      - nbenmoody
+      - mplorentz
+      - dcadenas
+  hosts:
+    loki.planetary.tools:
+      ansible_user: root

inventories/metrics/group_vars/all/vault.yml

@@ -1,23 +1,23 @@
 $ANSIBLE_VAULT;1.1;AES256
-32666631616333303464343061653764316464326566663438303437623062383832363232313031
-3335633661643061393063656163616331613230663063350a373938346336323930653030316663
-66313935303035383465353634356466316562663333613361663463646138373361643064636236
-6330303662396337630a633435663430626139396530373262646233613236343562353934383263
-61376565643839306232316362303335336162633733333733363936303637353338656635373737
-63306663663030316462316635383731393161666232333364316261663262383365366363353337
-64383432333337333031633237393737376431656536653232666363633538633330316436396163
-38353237356165393039386261343564623433366666386632633764366535396261306135663836
-32613237393439363066633435396631303938353632613534343837613164303230323632393665
-31313638313937613663646232623335393961626634393030623733363062646137346637383431
-33616336633639643864393539303262303536346665333338306638623037643164656533363538
-32396636333665633262383730346265343135633531666361333165653863346330353934663963
-64613738613364323864313630356530653130376435306332343432633436343338666264336635
-33383065636564633938313130326332316631306466323538353134333030323631626464653961
-64626166623066616436633062356531383033396161383032616133386237633832383337653931
-32626130306434613963393137303563336534373163313661343636613663353832336465386136
-64613831353965663863333165303335303038313163346335343432323266333461353337343932
-33663861316233613062393338343039336538376534393932353939396338613136643466386562
-39616334383633653233323839643334383931353239313036323932353032623563663233383562
-33633531306630343132626432656563383732303766626166326634343165626235363836316662
-64373266316338386463666637326334616333383330333532643339336366363334663262323562
-3730663130376165626438633839626439633933623131613037
+65656565356431383338383962376330303338336532626330383430613936653064666166666339
+6364363432346638653834613735386537633565393837320a623562303034386632616365383161
+33306463343964366438323238383765646538316165383330383937653131343631656362346633
+6464353666316134390a656532333535356633323132343165356232623164666432303437666433
+62393030323636313632616430373931396537373662353434613334353235313336666130376562
+33643334316439303763613132366637366261323432623338653539323066343535313933386534
+34353933656237646566303133626431313865303064326235626538623864336563373139306334
+34363730633533333037623161656466386139666361356261613261643439656564393031633237
+37323838623462393631373064636238343664646239343165353232633736376335643737363733
+31393030383033373161396538386565653531303066333163343530643165613732323633353165
+65306638333239323837306237366530633935313933636562366531373634323963323262633732
+38626138336566623133613239343262643163666134383832313265363133356434366666343462
+32363135336237366439396163383761613935663736626461303937383562343066343436343235
+39316630626361633866303864383633353539376665343730663833623833663134626237353965
+30636366643962306530363538376336373730336632626661366663373864613166643463393636
+61373965633436613164343938626137653636646465613438383661646237636139306263393163
+38373838356632383931373066386234386162326339363962626633303736313132656365316630
+30386362336638343332363830386532616530356434383535613862633235333462346366363936
+61366532666161623563316337663463353931326431363533363239323266633631366336616633
+30383163666337386531326530396434353139353162353730333736313735666561363130626662
+61336130666635336565353531386332383436663739376662643138383363616265656630383934
+3736306462316362373861356239656236343934656238646262

playbooks/loki-alloy.yml

@@ -0,0 +1,6 @@
+- name: Deploy and Configure the Grafana Alloy Agent
+  hosts: all
+  roles:
+    - loki-alloy
+
+# Deployment: ansible-playbook -i inventories/loki-alloy playbooks/loki-alloy.yml

playbooks/loki.yml

@@ -0,0 +1,6 @@
+- name: Deploy and Configure Loki
+  hosts: all
+  roles:
+    - loki
+
+# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml

requirements.yml

@@ -1,4 +1,6 @@
 ---
 collections:
   - name: community.docker
-  - name: community.general
+  - name: community.general
+  - name: ansible.posix
+

roles/docker/tasks/main.yml

@@ -68,12 +68,15 @@
     state: present
 
 
-- name: Add admin user to Docker
+- name: Add all users to Docker
   become: true
   ansible.builtin.user:
-    name: "{{ admin_username }}"
+    name: "{{ username }}"
     groups: docker
     append: true
+  loop: "{{ gh_user_keys_to_add }}"
+  loop_control:
+    loop_var: username
 
 
 - name: Download compose plugin

roles/loki-alloy/README.md

@@ -0,0 +1,2 @@
+# Loki Alloy
+Verse uses the Grafana Alloy agent (Grafana Labs's distribution of the Open Telemetry Collector), to collect logs from target hosts and ship them to our Loki instance, which itself is configured as a datasource in our Grafana instance, for use there. This role installs the Alloy agent on the target host and configures it to push logs to our Loki instance for docker containers.

roles/loki-alloy/tasks/main.yml

@@ -0,0 +1,79 @@
+- name: Ensure gpg is installed
+  become: true
+  ansible.builtin.apt:
+    pkg:
+      - gpg
+    state: present
+
+- name: Import Alloy GPG Key
+  become: true
+  ansible.builtin.shell: |
+    mkdir -p /etc/apt/keyrings/ && 
+    wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | tee /etc/apt/keyrings/grafana.gpg > /dev/null && 
+    echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | tee /etc/apt/sources.list.d/grafana.list
+  register: result
+  changed_when: result.rc != 0
+  failed_when: result.rc != 0
+
+- name: Update apt cache
+  become: true
+  ansible.builtin.apt:
+    update_cache: true
+
+- name: Install Alloy
+  become: true
+  ansible.builtin.apt:
+    pkg:
+      - alloy
+    state: present
+    autoremove: true
+
+- name: Interpolate Alloy configuration file
+  become: true
+  ansible.builtin.template:
+    src: "config.alloy.tpl"
+    dest: '/etc/alloy/config.alloy'
+    mode: '0640'
+    owner: alloy
+    group: alloy
+
+- name: Add the alloy user to the docker group
+  become: true
+  ansible.builtin.user:
+    name: alloy
+    groups: docker
+    append: true
+
+- name: Stop the alloy systemd service if it exists
+  become: true
+  ansible.builtin.systemd_service:
+    name: alloy
+    state: stopped
+    enabled: false
+    daemon_reload: true
+  ignore_errors: true
+
+- name: Enable and start Alloy
+  become: true
+  ansible.builtin.systemd_service:
+    name: alloy
+    state: started
+    enabled: true
+    daemon_reload: true
+
+- name: Wait for 10 seconds before checking Alloy status
+  pause:
+    seconds: 10
+
+- name: Wait for Alloy to start
+  become: true
+  ansible.builtin.command:
+    cmd: systemctl status alloy
+  register: result
+  until:
+    - "'Active: active (running)' in result.stdout"
+  retries: 3
+  delay: 5
+  failed_when:
+    - "'Active: active (running)' not in result.stdout"
+  changed_when: false

roles/loki-alloy/templates/config.alloy.tpl

@@ -0,0 +1,38 @@
+discovery.docker "docker_containers" {
+    host = "unix:///var/run/docker.sock"
+}
+
+discovery.relabel "docker_containers" {
+    targets = discovery.docker.docker_containers.targets
+
+    rule {
+        source_labels = ["__meta_docker_container_name"]
+        target_label  = "container"
+    }
+}
+
+loki.source.docker "docker_logs" {
+    host    = "unix:///var/run/docker.sock"
+    targets = discovery.relabel.docker_containers.output
+    forward_to = [loki.process.process_logs.receiver]
+}
+
+loki.process "process_logs" {
+    stage.docker { }
+    stage.static_labels {
+      values = {
+        hostname = "{{ inventory_hostname }}",
+      }
+    }
+    forward_to = [loki.write.verse_loki_endpoint.receiver]
+}
+
+loki.write "verse_loki_endpoint" {
+  endpoint {
+    url = "https://loki.planetary.tools/loki/api/v1/push"
+    basic_auth {
+        username = "{{ vault_traefik_user }}"
+        password = "{{ vault_traefik_password | password_hash(hashtype='md5') }}"
+    }
+  }
+}

roles/loki/README.md

@@ -0,0 +1,2 @@
+# Loki
+Verse uses Loki for log aggregation, configured with grafana. This role will take a newly-created Droplet (created using the terraform repo) and configure it to host a running Loki server, that uses DigitalOcean Spaces as the block storage location.

roles/loki/meta/main.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: common
+  - role: ssh-config-and-harden
+  - role: docker
+  - role: traefik

roles/loki/tasks/main.yml

@@ -0,0 +1,52 @@
+- name: Create a user for Loki
+  become: true
+  ansible.builtin.user:
+    name: loki
+    home: /home/loki
+    create_home: yes
+    group: admin
+
+- name: Create directory for Loki
+  become: true
+  ansible.builtin.file:
+    path: "/home/loki/loki"
+    state: directory
+    mode: '0755'
+
+- name: Interpolate Loki configuration file
+  become: true
+  ansible.builtin.template:
+    src: "loki-config.tpl"
+    dest: '/home/loki/loki/loki-config.yaml'
+    owner: loki
+    mode: '0777'
+
+- name: Interpolate docker-compose manifest
+  become: true
+  ansible.builtin.template:
+    src: "docker-compose.tpl"
+    dest: '/home/loki/loki/docker-compose.yaml'
+    owner: loki
+    mode: '0600'
+
+- name: Pull down old Loki
+  become: true
+  community.docker.docker_compose_v2:
+    project_src: /home/loki/loki
+    state: absent
+
+- name: Start new Loki
+  become_user: loki
+  community.docker.docker_compose_v2:
+    project_src: /home/loki/loki
+    wait: true
+    wait_timeout: 180
+  register: output
+
+- name: Check if Loki container is running
+  community.docker.docker_container_info:
+    name: "loki"
+  until: "container_info.container.State.Status == 'running'"
+  register: container_info
+  retries: 15
+  delay: 10