From 5ac51617687379ff532d7cd942f145e1677e4006 Mon Sep 17 00:00:00 2001
From: David Baggerman <david.baggerman@iress.com>
Date: Tue, 4 May 2021 10:34:09 +1000
Subject: [PATCH] Instance lifetime timeout

This adds a timeout based on instance lifetime, intended to be a
supplement to idle-timeout in the agent itself. It prevents busy agents
from living forever if they never get enough idle time to turn
themselves off.
---
 packer/linux/conf/bin/bk-install-elastic-stack.sh      |  6 ++++--
 .../systemd/buildkite-agent-hard-limit.service         |  6 ++++++
 .../systemd/buildkite-agent-hard-limit.timer           | 10 ++++++++++
 .../systemd/buildkite-agent-soft-limit.service         |  6 ++++++
 .../systemd/buildkite-agent-soft-limit.timer           |  9 +++++++++
 packer/linux/scripts/install-buildkite-agent.sh        |  3 ++-
 6 files changed, 37 insertions(+), 3 deletions(-)
 create mode 100644 packer/linux/conf/buildkite-agent/systemd/buildkite-agent-hard-limit.service
 create mode 100644 packer/linux/conf/buildkite-agent/systemd/buildkite-agent-hard-limit.timer
 create mode 100644 packer/linux/conf/buildkite-agent/systemd/buildkite-agent-soft-limit.service
 create mode 100644 packer/linux/conf/buildkite-agent/systemd/buildkite-agent-soft-limit.timer

diff --git a/packer/linux/conf/bin/bk-install-elastic-stack.sh b/packer/linux/conf/bin/bk-install-elastic-stack.sh
index be98ee641..04c9c8da7 100755
--- a/packer/linux/conf/bin/bk-install-elastic-stack.sh
+++ b/packer/linux/conf/bin/bk-install-elastic-stack.sh
@@ -158,8 +158,10 @@ if ! docker ps ; then
   exit 1
 fi
 
-systemctl enable "buildkite-agent"
-systemctl start "buildkite-agent"
+systemctl enable --now "buildkite-agent"
+
+systemctl enable --now "buildkite-agent-soft-limit.timer"
+systemctl enable --now "buildkite-agent-hard-limit.timer"
 
 # let the stack know that this host has been initialized successfully
 /opt/aws/bin/cfn-signal \
diff --git a/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-hard-limit.service b/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-hard-limit.service
new file mode 100644
index 000000000..2f7c02cc1
--- /dev/null
+++ b/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-hard-limit.service
@@ -0,0 +1,6 @@
+[Unit]
+Description=Hard stop buildkite agent
+
+[Service]
+ExecStart=/bin/systemctl stop buildkite-agent
+RemainAfterExit=true
diff --git a/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-hard-limit.timer b/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-hard-limit.timer
new file mode 100644
index 000000000..399996df3
--- /dev/null
+++ b/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-hard-limit.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Initiate hard stop of agent
+After=buildkite-agent.service
+
+[Timer]
+# 3hr soft timer + 21hr grace period
+OnBootSec=24hr
+
+[Install]
+WantedBy=timers.target
diff --git a/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-soft-limit.service b/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-soft-limit.service
new file mode 100644
index 000000000..90f2b503c
--- /dev/null
+++ b/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-soft-limit.service
@@ -0,0 +1,6 @@
+[Unit]
+Description=Soft stop buildkite agent
+
+[Service]
+ExecStart=/bin/systemctl kill --signal=TERM --kill-who=main buildkite-agent
+RemainAfterExit=true
diff --git a/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-soft-limit.timer b/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-soft-limit.timer
new file mode 100644
index 000000000..c0539f692
--- /dev/null
+++ b/packer/linux/conf/buildkite-agent/systemd/buildkite-agent-soft-limit.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Initiate soft stop of agent
+After=buildkite-agent.service
+
+[Timer]
+OnBootSec=3hr
+
+[Install]
+WantedBy=timers.target
diff --git a/packer/linux/scripts/install-buildkite-agent.sh b/packer/linux/scripts/install-buildkite-agent.sh
index 88817f376..7301405c6 100755
--- a/packer/linux/scripts/install-buildkite-agent.sh
+++ b/packer/linux/scripts/install-buildkite-agent.sh
@@ -60,7 +60,8 @@ sudo mkdir -p /var/lib/buildkite-agent/plugins
 sudo chown -R buildkite-agent: /var/lib/buildkite-agent/plugins
 
 echo "Adding systemd service template..."
-sudo cp /tmp/conf/buildkite-agent/systemd/buildkite-agent.service /etc/systemd/system/buildkite-agent.service
+sudo cp /tmp/conf/buildkite-agent/systemd/*.service /etc/systemd/system/
+sudo cp /tmp/conf/buildkite-agent/systemd/*.timer /etc/systemd/system/
 
 echo "Adding cloud-init failure safety check..."
 sudo mkdir -p /etc/systemd/system/cloud-final.service.d/