diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 89162d9..92be89c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -18,13 +18,13 @@ jobs: uses: actions/checkout@v1 - name: Start containers - run: docker-compose -f "local-dev/docker-compose.yml" up -d --build + run: docker compose -f "local-dev/docker-compose.yml" up -d --build - name: Build dev environment run: docker exec cicada_dev make dev - # - name: Run tests - # run: docker exec cicada_dev make pytest + - name: Run tests + run: docker exec cicada_dev make pytest - name: Build coverage file run: docker exec cicada_dev /bin/bash -c "source /opt/cicada/venv/bin/activate; pytest --cache-clear --cov=cicada --cov-fail-under=70" > pytest-coverage.txt @@ -34,4 +34,4 @@ jobs: - name: Stop containers if: always() - run: docker-compose -f "local-dev/docker-compose.yml" down + run: docker compose -f "local-dev/docker-compose.yml" down diff --git a/README.md b/README.md index 1e706d2..aaf847a 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,7 @@ - Non-running schedules can easily be moved from one node to another - Running schedules can easily be terminated ### Execution flow diagram - -![process_flow](https://github.com/transferwise/cicada/blob/master/docs/execution_flow.png?raw=True) +![Execution flow diagram](https://github.com/transferwise/cicada/blob/main/docs/execution_flow.png) ## Setup Central Database @@ -39,8 +38,7 @@ Verified on **PostgreSQL** versions *9.6* to *12.9* 3. Execute as **cicada** user [setup/schema.sql](setup/schema.sql) ### Database ERD - -![erd](https://github.com/transferwise/cicada/blob/master/docs/erd.png?raw=True) +![Database ERD](https://github.com/transferwise/cicada/blob/main/docs/erd.png) ## Setup Node diff --git a/cicada/commands/exec_schedule.py b/cicada/commands/exec_schedule.py index c626029..ae3357d 100644 --- a/cicada/commands/exec_schedule.py +++ b/cicada/commands/exec_schedule.py @@ -2,7 +2,6 @@ import datetime import subprocess -import psutil import os import signal import time @@ -176,8 +175,6 @@ def main(schedule_id, dbname=None): signal.signal(signal.SIGTERM, catch_sigterm) signal.signal(signal.SIGQUIT, catch_sigquit) - cicada_pid = os.getpid() - error_detail = None returncode = None @@ -205,6 +202,8 @@ def main(schedule_id, dbname=None): returncode = -15 error_detail = "Cicada abort_running" unset_abort_running(db_cur, schedule_id) + child_process.terminate() + db_cur.close() db_conn.close() except Exception as error: @@ -242,11 +241,6 @@ def main(schedule_id, dbname=None): returncode = 999 error_detail = "Crazy Unknown Error" finally: - - # Terminate any zombie processes - for zombie in psutil.Process(cicada_pid).children(recursive=True): - zombie.send_signal(signal.SIGTERM) - # Repeatedly attempt to finalize schedule, even if db is unavailable while True: try: diff --git a/docs/execution_flow.graphml b/docs/execution_flow.graphml index eb34390..1fd11d9 100644 --- a/docs/execution_flow.graphml +++ b/docs/execution_flow.graphml @@ -1,6 +1,6 @@ - + @@ -256,7 +256,7 @@ - executable + executable @@ -444,7 +444,7 @@ - Syncronous + Syncronous @@ -464,7 +464,7 @@ - + @@ -474,7 +474,7 @@ - + @@ -484,7 +484,7 @@ - + @@ -494,7 +494,7 @@ - + @@ -504,7 +504,7 @@ - + @@ -513,9 +513,8 @@ - - - + + @@ -526,9 +525,9 @@ - + + - @@ -539,9 +538,8 @@ - + - @@ -552,9 +550,8 @@ - + - diff --git a/docs/execution_flow.png b/docs/execution_flow.png index 3092dda..f74d6c9 100644 Binary files a/docs/execution_flow.png and b/docs/execution_flow.png differ diff --git a/local-dev/README.md b/local-dev/README.md index 6aa03d6..4c4cb86 100644 --- a/local-dev/README.md +++ b/local-dev/README.md @@ -12,7 +12,7 @@ Go to the main folder of the repository (the parent of this one) and To create l ```sh $ cd local-dev/ -$ docker-compose up --build +$ docker compose up --build ``` Wait until `Cicada Dev environment is ready in Docker container(s).` message. At the first run this can @@ -51,5 +51,5 @@ To refresh the containers with new local code changes stop the running instances ```sh $ ./refresh-local-dev.sh -$ docker-compose up +$ docker compose up ``` diff --git a/local-dev/entrypoint.sh b/local-dev/entrypoint.sh index d3a6f54..0c125af 100755 --- a/local-dev/entrypoint.sh +++ b/local-dev/entrypoint.sh @@ -16,6 +16,9 @@ fi alias ll='ls -alF' alias la='ls -A' alias l='ls -CF' + +source /opt/cicada/venv/bin/activate + EOL diff --git a/local-dev/refresh-local-dev.sh b/local-dev/refresh-local-dev.sh index 0b3c3cd..8dbdd8c 100755 --- a/local-dev/refresh-local-dev.sh +++ b/local-dev/refresh-local-dev.sh @@ -1,6 +1,6 @@ #!/usr/bin/bash -docker-compose down +docker compose down docker system prune --volumes --force sudo rm ../venv/ -Rf diff --git a/setup.py b/setup.py index c4e6d3e..24ead34 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name="cicada", - version="0.3.4", + version="0.4.0", description="Lightweight, agent-based, distributed scheduler", long_description=long_description, long_description_content_type="text/markdown", @@ -18,18 +18,17 @@ "Programming Language :: Python :: 3 :: Only", ], install_requires=[ - "psycopg2-binary==2.9.3", + "psycopg2-binary==2.9.5", "pyyaml==6.0", "croniter==1.3.7", - "tabulate==0.8.10", + "tabulate==0.9.0", "slack-sdk==3.19.1", - "backoff==2.1.2", - "psutil==5.9.2", + "backoff==2.2.1", ], extras_require={ "dev": [ "pytest==7.1.3", - "pytest-cov==3.0.0", + "pytest-cov==4.0.0", "black==22.10.0", "flake8==5.0.4", "freezegun==1.2.2",