Configure loki

inventories/loki/group_vars/all/vault.yml

@@ -0,0 +1,17 @@
+$ANSIBLE_VAULT;1.1;AES256
+33363133356235643965376632653035653963633337363833373236373336623138616463626435
+3239616232343963636330333031633735333263383230390a663966653233643861346532376463
+31373761353763373261303562336438386436316637333232363834653135343133656234306139
+3039643133376339360a343239666566346430363262636638386566653863323330363738343438
+31646337323830393562316336613631383164366563626263626435653365623530383036336232
+34316264663639386638306434376237313362626634323561363931386334633232316439623930
+37353536363435623565363961376538316631666534333930333832306662313862323064363636
+39653235366630373564373834646136303433656230666634383062333635643733346338653238
+62653038313835666663363236623665653738653263663036386431383835616464326435656361
+63323434633738623739356165326233643338396633616562353638623363373139333363346333
+66663135363365343763613730356638623833353763643337353330303566663331633938643364
+64373431623262356463313339393633353931326137333433653330346362313066343236383064
+38343438383733353863646235613831633466396434373732343763653662376661316137313639
+65366135346633636236323934623936656438616562666432613430303636663833626336393633
+66386236396437306466623437613864663564656236396438636465653738343933313861353962
+62366431373464623435

inventories/loki/inventory.yml

@@ -0,0 +1,17 @@
+all:
+  vars:
+    do_spaces_bucket_name: verse-loki-storage
+    do_spaces_bucket_endpoint: "{{ do_spaces_bucket_endpoint }}"
+    do_spaces_bucket_region: nyc-3
+    do_spaces_access_key: "{{ do_spaces_access_key }}"
+    do_spaces_secret_key: "{{ do_spaces_secret_key }}"
+    loki_password: "{{ loki_password }}"
+    homedir: loki
+    domain: loki.planetary.tools
+    gh_user_keys_to_add:
+      - nbenmoody
+      - mplorentz
+      - dcadenas
+  hosts:
+    loki.planetary.tools:
+      ansible_user: root

inventories/metrics/group_vars/all/vault.yml

@@ -1,23 +1,23 @@
 $ANSIBLE_VAULT;1.1;AES256
-32666631616333303464343061653764316464326566663438303437623062383832363232313031
-3335633661643061393063656163616331613230663063350a373938346336323930653030316663
-66313935303035383465353634356466316562663333613361663463646138373361643064636236
-6330303662396337630a633435663430626139396530373262646233613236343562353934383263
-61376565643839306232316362303335336162633733333733363936303637353338656635373737
-63306663663030316462316635383731393161666232333364316261663262383365366363353337
-64383432333337333031633237393737376431656536653232666363633538633330316436396163
-38353237356165393039386261343564623433366666386632633764366535396261306135663836
-32613237393439363066633435396631303938353632613534343837613164303230323632393665
-31313638313937613663646232623335393961626634393030623733363062646137346637383431
-33616336633639643864393539303262303536346665333338306638623037643164656533363538
-32396636333665633262383730346265343135633531666361333165653863346330353934663963
-64613738613364323864313630356530653130376435306332343432633436343338666264336635
-33383065636564633938313130326332316631306466323538353134333030323631626464653961
-64626166623066616436633062356531383033396161383032616133386237633832383337653931
-32626130306434613963393137303563336534373163313661343636613663353832336465386136
-64613831353965663863333165303335303038313163346335343432323266333461353337343932
-33663861316233613062393338343039336538376534393932353939396338613136643466386562
-39616334383633653233323839643334383931353239313036323932353032623563663233383562
-33633531306630343132626432656563383732303766626166326634343165626235363836316662
-64373266316338386463666637326334616333383330333532643339336366363334663262323562
-3730663130376165626438633839626439633933623131613037
+65656565356431383338383962376330303338336532626330383430613936653064666166666339
+6364363432346638653834613735386537633565393837320a623562303034386632616365383161
+33306463343964366438323238383765646538316165383330383937653131343631656362346633
+6464353666316134390a656532333535356633323132343165356232623164666432303437666433
+62393030323636313632616430373931396537373662353434613334353235313336666130376562
+33643334316439303763613132366637366261323432623338653539323066343535313933386534
+34353933656237646566303133626431313865303064326235626538623864336563373139306334
+34363730633533333037623161656466386139666361356261613261643439656564393031633237
+37323838623462393631373064636238343664646239343165353232633736376335643737363733
+31393030383033373161396538386565653531303066333163343530643165613732323633353165
+65306638333239323837306237366530633935313933636562366531373634323963323262633732
+38626138336566623133613239343262643163666134383832313265363133356434366666343462
+32363135336237366439396163383761613935663736626461303937383562343066343436343235
+39316630626361633866303864383633353539376665343730663833623833663134626237353965
+30636366643962306530363538376336373730336632626661366663373864613166643463393636
+61373965633436613164343938626137653636646465613438383661646237636139306263393163
+38373838356632383931373066386234386162326339363962626633303736313132656365316630
+30386362336638343332363830386532616530356434383535613862633235333462346366363936
+61366532666161623563316337663463353931326431363533363239323266633631366336616633
+30383163666337386531326530396434353139353162353730333736313735666561363130626662
+61336130666635336565353531386332383436663739376662643138383363616265656630383934
+3736306462316362373861356239656236343934656238646262

playbooks/loki.yml

@@ -0,0 +1,8 @@
+- name: Deploy and Configure Loki
+  hosts: all
+  roles:
+    - loki
+
+# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml --private-key /path/to/default-root-ssh-key
+
+

requirements.yml

@@ -1,4 +1,6 @@
 ---
 collections:
   - name: community.docker
-  - name: community.general
+  - name: community.general
+  - name: ansible.posix
+

roles/docker/tasks/main.yml

@@ -68,12 +68,15 @@
     state: present
 
 
-- name: Add admin user to Docker
+- name: Add all users to Docker
   become: true
   ansible.builtin.user:
-    name: "{{ admin_username }}"
+    name: "{{ username }}"
     groups: docker
     append: true
+  loop: "{{ gh_user_keys_to_add }}"
+  loop_control:
+    loop_var: username
 
 
 - name: Download compose plugin

roles/loki/README.md

@@ -0,0 +1,2 @@
+# Loki
+Verse uses Loki for log aggregation, configured with grafana. This role will take a newly-created Droplet (created using the terraform repo) and configure it to host a running Loki server, that uses DigitalOcean Spaces as the block storage location.

roles/loki/files/docker-compose.yaml

@@ -0,0 +1,26 @@
+version: '3'
+services:
+  loki:
+    image: grafana/loki  # TODO: Pin rather than latest
+    container_name: loki
+    ports:
+      - "0.0.0.0:3100:3100"
+    volumes:
+      - "./loki-config.yaml:/etc/loki/local-config.yaml"
+    networks:
+     - proxy
+    restart: always
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.loki.rule=Host(`loki.planetary.tools`)"
+      - "traefik.http.routers.loki.entrypoints=websecure"
+      - "traefik.http.routers.loki.tls.certresolver=nosresolver"
+      - "traefik.http.middlewares.loki-auth.basicauth.users=verse:temp"
+      - "traefik.http.routers.loki.middlewares=loki-auth"
+    resources:
+      limits:
+        cpus: '2'
+        memory: 6G
+networks:
+  proxy:
+    external: true

roles/loki/meta/main.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: common
+  - role: ssh-config-and-harden
+  - role: docker
+  - role: traefik

roles/loki/tasks/main.yml

@@ -0,0 +1,58 @@
+- name: Create a user for Loki
+  become: true
+  ansible.builtin.user:
+    name: loki
+    home: /home/loki
+    create_home: yes
+    group: admin
+
+- name: Create directory for Loki
+  become: true
+  ansible.builtin.file:
+    path: "/home/loki/loki"
+    state: directory
+    mode: '0755'
+
+- name: Interpolate Loki Configuration File
+  become_user: loki
+  ansible.builtin.template:
+    src: "loki-config.tpl"
+    dest: '/home/loki/loki/loki-config.yaml'
+    mode: '0600'
+
+- name: Copy the docker-compose.yaml
+  become_user: loki
+  ansible.builtin.copy:
+    src: "{{ role_path }}/files/docker-compose.yaml"
+    dest: /home/loki/loki
+    mode: '0600'
+
+- name: Replace 'temp' with 'loki_password' in docker-compose.yaml
+  replace:
+    path: /home/loki/loki/docker-compose.yaml
+    regexp: 'traefik.http.middlewares.webapp-auth.basicauth.users=verse:temp'
+    replace: 'traefik.http.middlewares.webapp-auth.basicauth.users=verse:{{ loki_password }}'
+
+
+#FIXME: Stopped here for tonight
+# FAILED! => {"changed": false, "cmd": "/usr/bin/docker --host unix:///var/run/docker.sock compose --ansi never --progress plain --project-directory /home/loki/loki ps --format json --all --no-trunc", "msg": "validating /home/loki/loki/docker-compose.yaml: services.loki Additional property resources is not allowed", "rc": 15, "stderr": "validating /home/loki/loki/docker-compose.yaml: services.loki Additional property resources is not allowed\n", "stderr_lines": ["validating /home/loki/loki/docker-compose.yaml: services.loki Additional property resources is not allowed"], "stdout": "", "stdout_lines": []}
+
+- name: Pull down old Loki
+  community.docker.docker_compose_v2:
+    project_src: /home/loki/loki
+    state: absent
+
+- name: Start new Loki
+  community.docker.docker_compose_v2:
+    project_src: /home/loki/loki
+    wait: true
+    wait_timeout: 180
+  register: output
+
+- name: Check that Loki is running
+  ansible.builtin.assert:
+    that:
+      - loki_container.State == 'running'
+  vars:
+    web_container: >-
+      {{ output.containers | selectattr("Service", "equalto", "loki") | first }}

roles/loki/templates/loki-config.tpl

@@ -0,0 +1,39 @@
+# Recommended config pulled from Digital Ocean: 
+# https://www.digitalocean.com/community/developer-center/how-to-install-loki-stack-in-doks-cluster#step-5-setting-persistent-storage-for-loki
+
+auth_enabled: false  # TODO: We'll want auth of some sort here.
+
+server:
+  http_listen_port: 3100
+
+common:
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+  replication_factor: 1
+  path_prefix: /loki
+
+schema_config:
+  configs:
+  - from: '2020-10-24'
+    store: boltdb-shipper
+    object_store: aws
+    schema: v11
+    index:
+      prefix: index_
+      period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /data/loki/boltdb-shipper-active
+    cache_location: /data/loki/boltdb-shipper-cache
+    cache_ttl: 24h
+    shared_store: aws
+  aws:
+    bucketnames: {{ do_spaces_bucket_name }}
+    endpoint: {{ do_spaces_bucket_endpoint }}
+    region: {{ do_spaces_bucket_region }}
+    access_key_id: {{ do_spaces_access_key }}
+    secret_access_key: {{ do_spaces_secret_key }}
+    s3forcepathstyle: true

roles/ssh-config-and-harden/tasks/main.yml

@@ -0,0 +1,63 @@
+- name: Add all users listed to the host.
+  ansible.builtin.user:
+    name: "{{ username }}"
+    state: present
+    groups: sudo 
+    append: true
+    shell: "/bin/bash"
+    create_home: true
+  loop: "{{ gh_user_keys_to_add }}"
+  loop_control:
+    loop_var: username
+
+- name: Create the .ssh folder for each user.
+  ansible.builtin.file:
+    path: "/home/{{ username }}/.ssh/"
+    state: directory
+    mode: "0700"
+    owner: "{{ username }}"
+    group: users
+  loop: "{{ gh_user_keys_to_add }}"
+  loop_control:
+    loop_var: username
+
+- name: Set ssh keys from Github for all listed users.
+  ansible.posix.authorized_key:
+    user: "{{ username }}"
+    state: present
+    key: https://github.com/{{ username }}.keys
+  loop: "{{ gh_user_keys_to_add }}"
+  loop_control:
+    loop_var: username
+
+- name: Setup passwordless sudo
+  ansible.builtin.lineinfile:
+    path: /etc/sudoers
+    state: present
+    regex: '^%sudo'
+    line: '%sudo ALL=(ALL) NOPASSWD: ALL'
+    validate: '/usr/sbin/visudo -cf %s'
+
+- name: Disable password login for everyone
+  ansible.builtin.lineinfile:
+    path: /etc/ssh/sshd_config
+    state: present
+    regexp: '^#?PasswordAuthentication'
+    line: 'PasswordAuthentication no'
+    validate: "/usr/sbin/sshd -t -f %s"
+
+- name: Restart sshd
+  ansible.builtin.systemd:
+    name: ssh
+    daemon_reload: true
+    state: restarted
+
+- name: UFW - Allow SSH connections
+  community.general.ufw:
+    rule: allow
+    name: OpenSSH
+
+- name: UFW - Enable and deny by default
+  community.general.ufw:
+    state: enabled
+    default: deny

roles/traefik/tasks/main.yml

@@ -30,7 +30,6 @@
     force: false
     mode: 0600
 
-
 - name: Copy necessary template files to traefik dir
   ansible.builtin.template:
     src: "{{ item.src }}"