From bb75d03509329a9097df82be3fa7e5a398a2ddef Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Fri, 6 Jan 2023 11:27:21 +0000 Subject: [PATCH] Add --only_enabled_servers option to spread_schedules (#113) --- README.md | 3 ++ cicada/cli.py | 6 ++++ cicada/commands/spread_schedules.py | 8 ++--- local-dev/README.md | 2 +- setup.py | 12 +++---- tests/test_functional_cli_entrypoint.py | 8 +++-- tests/test_functional_spread_schedules.py | 42 +++++++++++++++++++++-- 7 files changed, 66 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index aaf847a..2319103 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Cicada scheduler +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + *Centralized Distributed Scheduler* - [Overview](#overview) diff --git a/cicada/cli.py b/cicada/cli.py index 205e530..029b297 100644 --- a/cicada/cli.py +++ b/cicada/cli.py @@ -183,6 +183,12 @@ def spread_schedules(): required=True, help="List of target server_ids to spread schedules to", ) + parser.add_argument( + "--exclude_disabled_servers", + default=False, + action="store_true", + help="Exclude disabled servers from target server_ids", + ) parser.add_argument( "--commit", default=False, diff --git a/cicada/commands/spread_schedules.py b/cicada/commands/spread_schedules.py index 75b6b84..4126eba 100644 --- a/cicada/commands/spread_schedules.py +++ b/cicada/commands/spread_schedules.py @@ -47,13 +47,13 @@ def get_last_week_schedules_by_load(db_cur, server_ids: [int] = None): return last_week_schedules_by_load -def get_enabled_servers(db_cur, enabled_only: bool = True, server_ids: [int] = None): +def get_target_servers(db_cur, exclude_disabled, server_ids: [int] = None): """Get valid servers""" - sql_enabled_filter = " and is_enabled = 1" if enabled_only else "" + sql_enabled_filter = " AND is_enabled = 1" if exclude_disabled else "" sql_server_id_filter = "" if server_ids: sql_server_ids = ",".join(str(server_id) for server_id in server_ids) - sql_server_id_filter = f" and server_id in ({sql_server_ids})" + sql_server_id_filter = f" AND server_id in ({sql_server_ids})" sqlquery = f""" SELECT server_id FROM servers @@ -81,7 +81,7 @@ def main(spread_details, dbname=None): from_server_ids = csv_to_list(spread_details["from_server_ids"]) to_server_ids = csv_to_list(spread_details["to_server_ids"]) - valid_target_servers = get_enabled_servers(db_cur, server_ids=to_server_ids) + valid_target_servers = get_target_servers(db_cur, spread_details["exclude_disabled_servers"], to_server_ids) valid_server_count = len(valid_target_servers) if valid_server_count == 0: diff --git a/local-dev/README.md b/local-dev/README.md index 4c4cb86..ffc6079 100644 --- a/local-dev/README.md +++ b/local-dev/README.md @@ -36,7 +36,7 @@ check the [local-dev/.env](../local-dev/.env) file for the credentials. ### Running tests ``` sh -$ docker exec -it cicada_dev make pytest +$ docker exec -it cicada_dev make dev pytest ``` ### Running linters diff --git a/setup.py b/setup.py index 24ead34..2a56375 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name="cicada", - version="0.4.0", + version="0.4.1", description="Lightweight, agent-based, distributed scheduler", long_description=long_description, long_description_content_type="text/markdown", @@ -20,17 +20,17 @@ install_requires=[ "psycopg2-binary==2.9.5", "pyyaml==6.0", - "croniter==1.3.7", + "croniter==1.3.8", "tabulate==0.9.0", - "slack-sdk==3.19.1", + "slack-sdk==3.19.5", "backoff==2.2.1", ], extras_require={ "dev": [ - "pytest==7.1.3", + "pytest==7.2.0", "pytest-cov==4.0.0", - "black==22.10.0", - "flake8==5.0.4", + "black==22.12.0", + "flake8==6.0.0", "freezegun==1.2.2", ] }, diff --git a/tests/test_functional_cli_entrypoint.py b/tests/test_functional_cli_entrypoint.py index ea51ef3..25abb76 100644 --- a/tests/test_functional_cli_entrypoint.py +++ b/tests/test_functional_cli_entrypoint.py @@ -194,7 +194,8 @@ def test_spread_schedules(): actual = subprocess.run(["cicada", "spread_schedules"], check=False, stderr=subprocess.PIPE).stderr.decode("utf-8") expected = """usage: spread_schedules [-h] --from_server_ids FROM_SERVER_IDS --to_server_ids - TO_SERVER_IDS [--commit] [--force] + TO_SERVER_IDS [--exclude_disabled_servers] [--commit] + [--force] spread_schedules: error: the following arguments are required: --from_server_ids, --to_server_ids """ @@ -208,7 +209,8 @@ def test_spread_schedules_help(): ) expected = """usage: spread_schedules [-h] --from_server_ids FROM_SERVER_IDS --to_server_ids - TO_SERVER_IDS [--commit] [--force] + TO_SERVER_IDS [--exclude_disabled_servers] [--commit] + [--force] Spread schedules accross servers @@ -218,6 +220,8 @@ def test_spread_schedules_help(): List of source server_ids to collect schedules from --to_server_ids TO_SERVER_IDS List of target server_ids to spread schedules to + --exclude_disabled_servers + Exclude disabled servers from target server_ids --commit Commits changes to backend DB, otherwise only print output --force If schedule is moving servers and also currently diff --git a/tests/test_functional_spread_schedules.py b/tests/test_functional_spread_schedules.py index 0ddc87e..037c931 100644 --- a/tests/test_functional_spread_schedules.py +++ b/tests/test_functional_spread_schedules.py @@ -89,13 +89,13 @@ def test_create_dummy_servers(): VALUES (1, '1', '1', '192.168.0.1', 1), (2, '2', '2', '192.168.0.2', 1), - (3, '3', '3', '192.168.0.3', 1), + (3, '3', '3', '192.168.0.3', 0), (4, '4', '4', '192.168.0.4', 1) ; """ ) - results = query_test_db(f"SELECT count(*) FROM servers WHERE is_enabled=1")[0][0] + results = query_test_db(f"SELECT count(*) FROM servers")[0][0] assert results == 4 @@ -172,6 +172,7 @@ def test_spread_schedules(): spread_details["to_server_ids"] = "1,2,3,4" spread_details["commit"] = False spread_details["force"] = False + spread_details["exclude_disabled_servers"] = False spread_schedules.main(spread_details, pytest.db_test) @@ -204,6 +205,7 @@ def test_spread_schedules_commit(): spread_details["to_server_ids"] = "1,2,3,4" spread_details["commit"] = True spread_details["force"] = False + spread_details["exclude_disabled_servers"] = False spread_schedules.main(spread_details, pytest.db_test) @@ -236,6 +238,7 @@ def test_spread_schedules_force(): spread_details["to_server_ids"] = "1,2,3,4" spread_details["commit"] = True spread_details["force"] = True + spread_details["exclude_disabled_servers"] = False spread_schedules.main(spread_details, pytest.db_test) @@ -259,6 +262,41 @@ def test_spread_schedules_force(): ] +@freeze_time("2022-01-02 02:00:00") +def test_spread_schedules_exclude_disabled_servers(): + """test_spread_schedules_force""" + spread_details = {} + + spread_details["from_server_ids"] = "1,2,3,4" + spread_details["to_server_ids"] = "1,2,3,4" + spread_details["commit"] = True + spread_details["force"] = True + spread_details["exclude_disabled_servers"] = True + + spread_schedules.main(spread_details, pytest.db_test) + + result = query_test_db( + f"SELECT server_id, schedule_id, is_enabled, abort_running, adhoc_execute FROM schedules ORDER BY schedule_id" + ) + + print(result) + + assert result == [ + (4, "1-1", 1, 0, 0), + (1, "1-2", 1, 1, 1), + (2, "1-3", 1, 0, 0), + (4, "1-4", 1, 0, 0), + (2, "2-1", 1, 0, 0), + (4, "2-2", 1, 0, 0), + (1, "2-3", 1, 0, 0), + (2, "2-4", 1, 0, 0), + (1, "3-1", 1, 0, 0), + (2, "3-2", 1, 0, 0), + (4, "3-3", 1, 0, 0), + (1, "3-4", 1, 0, 0), + ] + + def test_db_teardown(): """test_db_teardown""" pg_conn = psycopg2.connect(