From 51521cdc43782373c2c3b94a1e2d0fe556c324de Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Mon, 26 Dec 2022 19:05:26 +0000 Subject: [PATCH 1/2] Configure logrotate to prevent worker crashes --- .../tasks/celery-service.yml | 5 +++++ .../templates/celery-logrotate.j2 | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 deployment/ansible/roles/model-my-watershed.celery-worker/templates/celery-logrotate.j2 diff --git a/deployment/ansible/roles/model-my-watershed.celery-worker/tasks/celery-service.yml b/deployment/ansible/roles/model-my-watershed.celery-worker/tasks/celery-service.yml index 60ca68666..510b65f12 100644 --- a/deployment/ansible/roles/model-my-watershed.celery-worker/tasks/celery-service.yml +++ b/deployment/ansible/roles/model-my-watershed.celery-worker/tasks/celery-service.yml @@ -15,3 +15,8 @@ special_time: daily job: /usr/sbin/service celeryd restart state: present + +- name: Configure Celery log rotation + template: src=celery-logrotate.j2 + dest=/etc/logrotate.d/celery + mode=0755 diff --git a/deployment/ansible/roles/model-my-watershed.celery-worker/templates/celery-logrotate.j2 b/deployment/ansible/roles/model-my-watershed.celery-worker/templates/celery-logrotate.j2 new file mode 100644 index 000000000..0ae479667 --- /dev/null +++ b/deployment/ansible/roles/model-my-watershed.celery-worker/templates/celery-logrotate.j2 @@ -0,0 +1,5 @@ +{{ celery_log_dir }}/* { + rotate 3 + daily + compress +} From 6e786fdcf207caf6893bcedbad4cb648a4b39414 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Tue, 27 Dec 2022 17:39:19 +0000 Subject: [PATCH 2/2] Increase worker EBS volume to 12GB These instances start up 99% full as soon as they are built. The usage breakdown is as follows: ``` $ df -h / Filesystem Size Used Avail Use% Mounted on /dev/root 7.6G 7.5G 141M 99% / ``` ``` $ sudo du -hs /* ... 3.8G usr 5.3G var ``` ``` $ sudo du -sh /usr/* | grep M 406M /usr/bin 215M /usr/include 99M /usr/libexec 670M /usr/local 34M /usr/sbin 332M /usr/share 285M /usr/src ``` ``` $ sudo du -sh /var/lib/* | grep G 4.3G /var/lib/docker ``` It is likely that the size of dependencies has grown over time, particularly Python libraries and Docker executables. This leaves little room for operating the instances, which fill up sooner and sooner, necessitating manual termination and restarts. This increased space will give us more breathing room for the next few years. --- deployment/packer/mmw.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deployment/packer/mmw.json b/deployment/packer/mmw.json index faf82aa0d..ba668433b 100644 --- a/deployment/packer/mmw.json +++ b/deployment/packer/mmw.json @@ -62,6 +62,12 @@ "ssh_username": "ubuntu", "ami_name": "mmw-worker-{{timestamp}}-{{user `version`}}", "ami_block_device_mappings": [ + { + "device_name": "/dev/sda1", + "volume_type": "gp2", + "volume_size": 12, + "delete_on_termination": true + }, { "device_name": "/dev/sdf", "snapshot_id": "snap-0211cbbff8a81266f",