Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Configure loki #144

Merged
merged 17 commits into from
Feb 10, 2025
Merged
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions inventories/followers_server/group_vars/all/vault.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
$ANSIBLE_VAULT;1.1;AES256
34343963653734633631653433363330613161643164623038663463303464326461663261623732
3632366436363931623736663337343063633739303036640a383938633231636463316533383633
62636632333463353132336262396438343438393364636339313633346466653964636430313530
3832303366636665620a323762663830346464393734303066363038303336663563393431333664
34653066666533366130336462316535366434346135666431316561333037313833656532663533
31616466373962313564323563623763316333633164623661313734393334643435323066653464
65313165343031623763626465386163613838383133386233376563363064656363393764393164
33346564363864653933313934633239636266333938316463353064386330613038386362613830
30383631363932623237333832636137353634366363626562343964623666356537383762633535
35313231343934313036373461653230656361663931653935356561323631653833373261333163
62343665373861333337316232643366386331323339626430316466353237386538303438656464
64303030343636356630633034396538623435656435353765373836623738356262643264343031
3134
31363030313061623966386534373563343763376338613033373434336636343236386663366566
6462636239303565383739376439366331636432396233350a613737366235626337663132316264
65663739343133323033303464313066323635383062303138663934353738626563363539656339
6531333337346237320a333239616534616430383166646539333062393832666639393533623139
37383934316362663436376632323832346563393939353835323031363464316137396163376430
37663166333665393364353330303035633937383832303135366631303464356663636438656436
34616537376362306135326564623265323663663431346263356633613064396464663965633164
64656439396330383434376266333765303461623965356431333338333465396337333630383235
33313565636332306630663733343565363565366137613362356639386236626433373330303536
64383661623135613838316164616430313365613932316338343936336630613431366530373338
30313133326330326337373662323133303238386264383439613335386531303631343561373134
31383966343832386231383263356632393633646164373736656230623434393864643138363932
3635
9 changes: 9 additions & 0 deletions inventories/loki-alloy/group_vars/all/vault.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
$ANSIBLE_VAULT;1.1;AES256
31313536656632373238623439393462623131326438396462636634653166666333313139313065
6336653834636637373166376339653131343262313431350a643936663464613862343134373530
31326237356132386363386265366636653431303061326466633833353833383662393062313031
6363663162643165320a353264646664363665323334306361356564393665373837646630613035
61646537653638303736623831326164663831623361616632373137383539653961303435353465
30613137356163633230383665323535343763666338393030396366323463366261633863643663
37343965366562396263303166386334343830623065333339323565363036373661383630633036
38313966353033343364
8 changes: 8 additions & 0 deletions inventories/loki-alloy/inventory.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
all:
vars:
loki_password_hashed_escaped: "{{ loki_password_hashed_escaped }}"
hosts:
relay.nos.social:
ansible_user: admin
loki.planetary.tools:
ansible_user: root
66 changes: 17 additions & 49 deletions inventories/loki/group_vars/all/vault.yml
Original file line number Diff line number Diff line change
@@ -1,50 +1,18 @@
$ANSIBLE_VAULT;1.1;AES256
36313434323630633731303130323733333634633237396565346563316566303533363834396438
6531373666393763353238303930333762343536663861380a306138346364393466363632653137
36326464393837386365623863646464633165333537623932613830626239363534643965343639
3435633062323932330a653832646630616633653866656563633839316433336439366135666534
30393337346631626133653737343664356639623939653566383066346238623830393566343165
61396333373463643032373231373361376637316362663639386634653430623737323964303138
61643362303330346635663161623037653237323666333436363234656436636366383735643937
61363936643930313431613933346336363463613735363636393538306265313733663839323465
35353434303635393734316231356632633531616363313764323633393433383463653835383137
62396661653966656237613162373032383537356265316561646435643734346634346134386136
36303735393663656233386665336639313662316338643562613338646337663665656265653134
63396265663834303962353035623239656234663537303330383138303964323062616664373332
33383361633030646530643365313234336266646434613165323738303031303432646633623531
65396432343436383661653263353634616131656235616362306566373135303364383934363438
33303163393630393632613033636233396166626534323632353333333133613133333464613463
33353337353831623666366365646137326433313638636163663839313733633133316135313034
63366263326133333163323733366366383231396136383066353662633031623662353033363161
33366536356333646438663364633061303439396232326665353464303632653330626265636232
36616233306135326164353663313435323532343935636465376133393662363766616330656332
66306331646264343930663538373766346662636662623438646235646633346166396565656164
39616631373862613534393763623162653636396636333137333365313637623230303331333438
38363530346464646663393165613065313130383736316436353530303134386438613431643936
33373431303130313263646633363033646166353533323466626365666333613962643966646461
63386361636135363463363533326162333136336364316465343038636636396230316634643466
38613064326239323664663364366534376633396364643066633861393434316536646563383839
64623736643339353763303135363764346233633062376332653835383432663432666163633934
35616239663638316462643462333763356338376461393335363437316265336637656434346664
39363437326139313233323139313261396262373830356630643862383537303066313263316366
31366162343239653939393538386531343365613534383265613632623964636664643835373665
61636634663664613965353964383064326139363964643861653532653939366366336461343563
35626638363932346564323033396132666631653439616565353433373938383730363565393365
32306561326365393762303663313765356431306530663836396637666364343361633535646165
35313866323233376637663839666535323062366235323561666538616633613933643733666535
30343132363734353763356533623833366261623039333562643337366165353139363035313664
36633865346235303635323737353033633331643766623738353735643035643531643734363030
38323837386230646439303237666239386531626633663338663562666332376537336137653964
66663230636563383234323538636664653034633962313432663630346133396165373632643930
38333963623639326431616533653431386439313735353433393334643830363636333362646266
64623962636336343337363161316632333365623337653962336636666439316130336564323133
34373164396431386433356136356631616161323731336265346165363266383762393239326630
33363636636563336665336664633236373733633136643137386362386264333361376233336139
32613739393238653466623666306535323566396263333633336563313835633237363538343763
34303536306330663433353266656531646337366661363739333163623739633338323437363366
61633766376239386466653739666266386531623961613634326130636464303632643334343732
66373732323964613739633737353263393938333735663862613762663337376134363630393833
66653439323662666263373131333838396166376130363830633230343536306563396164303935
61376363656565616165663938623432393062653532663865616633623736363334313132653834
35353762633938333666653761613131333463626535613637326637656361373266613132306633
3937
37303137636532643332643231386664666635313839623931643137626464616234663135343836
6565326331353739616434633062396339386534363836380a636134343234333339386230313838
31373231343938313833303330376463396536303039323864623831323130653262306234393632
3064303563336532660a383336396333646436613236333239343062303930643166323034366534
36343435363335663362626534623933626266303762393730616536393364633836663933333438
38653566376434646338666632396666353531613630326538306636666362323866616364613931
66343137653039613062326530633133323931653038636339336239333933333032343566343434
31623735363933386336353966663634653336313433393533636438323664636437653834653335
37346662373638626433363763303635663864363963626138643234383963326439316664336361
37646662393035366236633163353665656266326261303966383366336332323531326266353264
35386661303666663439313361336538356437653565333863636133613532653266626361343939
30353331653332363564383836373834306562383432333237653032626366333630656366653033
34643130343635623963643937636633663534306433353539386363643933653532653662333962
39653066303733393139663931333937623234653261393936326366653832653737333236383734
61383162303963353337326430343763343961303063666438656661623634623533336132343832
30386566623165333638313865393565333234663130323736363238633434336235643430336366
63333266343430623439376533316366663238653832303537636130353832653931
1 change: 1 addition & 0 deletions inventories/loki/inventory.yml
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ all:
do_spaces_access_key: "{{ do_spaces_access_key }}"
do_spaces_secret_key: "{{ do_spaces_secret_key }}"
loki_password: "{{ loki_password }}"
loki_password_hashed_escaped: "{{ loki_password_hashed_escaped }}"
homedir: loki
domain: loki.planetary.tools
gh_user_keys_to_add:
6 changes: 6 additions & 0 deletions playbooks/loki-alloy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
- name: Deploy and Configure the Grafana Alloy Agent
hosts: all
roles:
- loki-alloy

# Deployment: ansible-playbook -i inventories/loki-alloy playbooks/loki-alloy.yml
4 changes: 1 addition & 3 deletions playbooks/loki.yml
Original file line number Diff line number Diff line change
@@ -3,6 +3,4 @@
roles:
- loki

# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml --private-key /path/to/default-root-ssh-key


# Deployment: ansible-playbook -i inventories/loki playbooks/loki.yml
2 changes: 2 additions & 0 deletions roles/loki-alloy/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Loki Alloy
Verse uses the Grafana Alloy agent (Grafana Labs's distribution of the Open Telemetry Collector), to collect logs from target hosts and ship them to our Loki instance, which itself is configured as a datasource in our Grafana instance, for use there. This role installs the Alloy agent on the target host and configures it to push logs to our Loki instance for docker containers.
79 changes: 79 additions & 0 deletions roles/loki-alloy/tasks/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
- name: Ensure gpg is installed
become: true
ansible.builtin.apt:
pkg:
- gpg
state: present

- name: Import Alloy GPG Key
become: true
ansible.builtin.shell: |
mkdir -p /etc/apt/keyrings/ &&
wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | tee /etc/apt/keyrings/grafana.gpg > /dev/null &&
echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | tee /etc/apt/sources.list.d/grafana.list
register: result
changed_when: result.rc != 0
failed_when: result.rc != 0

- name: Update apt cache
become: true
ansible.builtin.apt:
update_cache: true

- name: Install Alloy
become: true
ansible.builtin.apt:
pkg:
- alloy
state: present
autoremove: true

- name: Interpolate Alloy configuration file
become: true
ansible.builtin.template:
src: "config.alloy.tpl"
dest: '/etc/alloy/config.alloy'
mode: '0640'
owner: alloy
group: alloy

- name: Add the alloy user to the docker group
become: true
ansible.builtin.user:
name: alloy
groups: docker
append: true

- name: Stop the alloy systemd service if it exists
become: true
ansible.builtin.systemd_service:
name: alloy
state: stopped
enabled: false
daemon_reload: true
ignore_errors: true

- name: Enable and start Alloy
become: true
ansible.builtin.systemd_service:
name: alloy
state: started
enabled: true
daemon_reload: true

- name: Wait for 10 seconds before checking Alloy status
pause:
seconds: 10

- name: Wait for Alloy to start
become: true
ansible.builtin.command:
cmd: systemctl status alloy
register: result
until:
- "'Active: active (running)' in result.stdout"
retries: 3
delay: 5
failed_when:
- "'Active: active (running)' not in result.stdout"
changed_when: false
38 changes: 38 additions & 0 deletions roles/loki-alloy/templates/config.alloy.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
discovery.docker "docker_containers" {
host = "unix:///var/run/docker.sock"
}

discovery.relabel "docker_containers" {
targets = discovery.docker.docker_containers.targets

rule {
source_labels = ["__meta_docker_container_name"]
target_label = "container"
}
}

loki.source.docker "docker_logs" {
host = "unix:///var/run/docker.sock"
targets = discovery.relabel.docker_containers.output
forward_to = [loki.process.process_logs.receiver]
}

loki.process "process_logs" {
stage.docker { }
stage.static_labels {
values = {
hostname = "{{ inventory_hostname }}",
}
}
forward_to = [loki.write.verse_loki_endpoint.receiver]
}

loki.write "verse_loki_endpoint" {
endpoint {
url = "https://loki.planetary.tools/loki/api/v1/push"
basic_auth {
username = "{{ vault_traefik_user }}"
password = "{{ vault_traefik_password | password_hash(hashtype='md5') }}"
}
}
}
8 changes: 4 additions & 4 deletions roles/loki/templates/docker-compose.tpl
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@ services:
mem_limit: 6g
cpus: 2.0
ports:
- "0.0.0.0:3100:3100"
- "3100:3100"
volumes:
- "./loki-config.yaml:/etc/loki/local-config.yaml"
networks:
@@ -16,9 +16,9 @@ services:
- "traefik.enable=true"
- "traefik.http.routers.loki.rule=Host(`loki.planetary.tools`)"
- "traefik.http.routers.loki.entrypoints=websecure"
- "traefik.http.routers.loki.tls.certresolver=nosresolver"
- "traefik.http.middlewares.loki-auth.basicauth.users=verse:{{ loki_password_hashed_escaped }}"
- "traefik.http.routers.loki.middlewares=loki-auth"
- "traefik.http.routers.loki.tls.certresolver=letsencrypt"
- "traefik.http.middlewares.user-auth.basicauth.users={{ vault_traefik_user }}:{{ vault_traefik_password | password_hash(hashtype='md5') }}"
- "traefik.http.services.loki.loadbalancer.server.port=3100"
networks:
proxy:
external: true
20 changes: 10 additions & 10 deletions roles/traefik/vars/main.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
$ANSIBLE_VAULT;1.1;AES256
32613239626262383064663234323364663832333963653564383938306238386131383633383436
3038303136616233363033663331346539666266383633310a366164653031633265363561366537
62363730373233623133386332343933636136303436383035623461343930396533633633353364
6138396337666466330a636639303033313362623661303235643433653662306330393432386563
33386534666138663332623338613532373238306130326664366433383730386235303635663063
65326238613231663333663565353066336136633236326338383962663062363766666265303932
36616134323832343730663539646366643832333639666136373665613533333738353635663138
39303033656135366166383134303263366637643836316332346138363435323233333961343764
30643466323539386462316136396539363661366630373438306334626565353236613364653733
3532363535323534306337333539643666393066326232383434
30646134346163323134333765373033313738323665393038663364623538663465373466653939
6533386430663235666635366639623162306538326163650a366166363266656330353137653966
37366666373839366438623935336532663930326465326137306231366634363637383831643437
6338363163613932300a356130373639356166393864396437383262316538316533373231343261
65363030356632366330353961643534643934326438393932643062303466653838373931316130
30306165333835353662613362333634633930376339646331323231336636313538663466393262
61653561323730373261653136616434643863303663303833383836396462383863616662636164
63323763663362303630363161343330306566316264323963626434346133376361356138363763
38643664356439663735633065646565396463383162373661383961626165336230306536656132
6430633431613734633432633066383938353630626362343232