From 82a6392ebbe0749e8421f2a51b8ff56a0c228fec Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 10:26:09 -0500 Subject: [PATCH 01/37] PYTHON-5413 Handle flaky tests --- .evergreen/scripts/setup_tests.py | 4 --- test/asynchronous/test_client_bulk_write.py | 4 +-- test/asynchronous/test_csot.py | 7 ++-- test/asynchronous/test_cursor.py | 7 ++-- test/asynchronous/unified_format.py | 4 --- test/asynchronous/utils.py | 38 +++++++++++++++++++++ test/test_client_bulk_write.py | 4 +-- test/test_csot.py | 7 ++-- test/test_cursor.py | 7 ++-- test/unified_format.py | 4 --- test/unified_format_shared.py | 2 -- test/utils.py | 38 +++++++++++++++++++++ 12 files changed, 92 insertions(+), 34 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 13444fe9ca..a4ff77ab71 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -162,10 +162,6 @@ def handle_test_env() -> None: write_env("PIP_PREFER_BINARY") # Prefer binary dists by default. write_env("UV_FROZEN") # Do not modify lock files. - # Skip CSOT tests on non-linux platforms. - if PLATFORM != "linux": - write_env("SKIP_CSOT_TESTS") - # Set an environment variable for the test name and sub test name. write_env(f"TEST_{test_name.upper()}") write_env("TEST_NAME", test_name) diff --git a/test/asynchronous/test_client_bulk_write.py b/test/asynchronous/test_client_bulk_write.py index 2f48466af8..0402d75e0f 100644 --- a/test/asynchronous/test_client_bulk_write.py +++ b/test/asynchronous/test_client_bulk_write.py @@ -25,6 +25,7 @@ async_client_context, unittest, ) +from test.asynchronous.utils import flaky from test.utils_shared import ( OvertCommandListener, ) @@ -619,8 +620,6 @@ async def test_15_unacknowledged_write_across_batches(self): # https://github.com/mongodb/specifications/blob/master/source/client-side-operations-timeout/tests/README.md#11-multi-batch-bulkwrites class TestClientBulkWriteCSOT(AsyncIntegrationTest): async def asyncSetUp(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") await super().asyncSetUp() self.max_write_batch_size = await async_client_context.max_write_batch_size self.max_bson_object_size = await async_client_context.max_bson_size @@ -628,6 +627,7 @@ async def asyncSetUp(self): @async_client_context.require_version_min(8, 0, 0, -24) @async_client_context.require_failCommand_fail_point + @flaky async def test_timeout_in_multi_batch_bulk_write(self): _OVERHEAD = 500 diff --git a/test/asynchronous/test_csot.py b/test/asynchronous/test_csot.py index 46c97ce6d3..6e4a82f5b2 100644 --- a/test/asynchronous/test_csot.py +++ b/test/asynchronous/test_csot.py @@ -23,6 +23,7 @@ from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest from test.asynchronous.unified_format import generate_test_classes +from test.asynchronous.utils import flaky import pymongo from pymongo import _csot @@ -43,9 +44,8 @@ class TestCSOT(AsyncIntegrationTest): RUN_ON_LOAD_BALANCER = True + @flaky async def test_timeout_nested(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") coll = self.db.coll self.assertEqual(_csot.get_timeout(), None) self.assertEqual(_csot.get_deadline(), float("inf")) @@ -82,9 +82,8 @@ async def test_timeout_nested(self): self.assertEqual(_csot.get_rtt(), 0.0) @async_client_context.require_change_streams + @flaky async def test_change_stream_can_resume_after_timeouts(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") coll = self.db.test await coll.insert_one({}) async with await coll.watch() as stream: diff --git a/test/asynchronous/test_cursor.py b/test/asynchronous/test_cursor.py index 861345cb08..bb387d0b21 100644 --- a/test/asynchronous/test_cursor.py +++ b/test/asynchronous/test_cursor.py @@ -31,6 +31,7 @@ sys.path[0:0] = [""] from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest +from test.asynchronous.utils import flaky from test.utils_shared import ( AllowListEventListener, EventListener, @@ -1415,9 +1416,8 @@ async def test_to_list_length(self): docs = await c.to_list(3) self.assertEqual(len(docs), 2) + @flaky async def test_to_list_csot_applied(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") client = await self.async_single_client(timeoutMS=500, w=1) coll = client.pymongo.test # Initialize the client with a larger timeout to help make test less flakey @@ -1458,9 +1458,8 @@ async def test_command_cursor_to_list_length(self): self.assertEqual(len(await result.to_list(1)), 1) @async_client_context.require_failCommand_blockConnection + @flaky async def test_command_cursor_to_list_csot_applied(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") client = await self.async_single_client(timeoutMS=500, w=1) coll = client.pymongo.test # Initialize the client with a larger timeout to help make test less flakey diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index fbd1f87755..05c9e07591 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -40,7 +40,6 @@ from test.unified_format_shared import ( KMS_TLS_OPTS, PLACEHOLDER_MAP, - SKIP_CSOT_TESTS, EventListenerUtil, MatchEvaluatorUtil, coerce_result, @@ -1374,9 +1373,6 @@ async def verify_outcome(self, spec): self.assertListEqual(sorted_expected_documents, actual_documents) async def run_scenario(self, spec, uri=None): - if "csot" in self.id().lower() and SKIP_CSOT_TESTS: - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") - # Kill all sessions before and after each test to prevent an open # transaction (from a test failure) from blocking collection/database # operations during test set up and tear down. diff --git a/test/asynchronous/utils.py b/test/asynchronous/utils.py index ca80d1f6dd..cadf48f22e 100644 --- a/test/asynchronous/utils.py +++ b/test/asynchronous/utils.py @@ -18,9 +18,11 @@ import asyncio import contextlib import random +import sys import threading # Used in the synchronized version of this file import time from asyncio import iscoroutinefunction +from functools import wraps from bson.son import SON from pymongo import AsyncMongoClient @@ -154,6 +156,42 @@ async def async_joinall(tasks): await asyncio.wait([t.task for t in tasks if t is not None], timeout=300) +def flaky(func=None, *, max_runs=2, min_passes=1, delay=1, affects_cpython_linux=False): + is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" + if is_cpython_linux and not affects_cpython_linux: + max_runs = 1 + min_passes = 1 + + def decorator(target_func): + @wraps(target_func) + async def wrapper(*args, **kwargs): + passes = 0 + failure = None + for i in range(max_runs): + try: + result = await target_func(*args, **kwargs) + passes += 1 + if passes == min_passes: + return result + except Exception as e: + failure = e + await asyncio.sleep(delay) + if failure is not None and i < max_runs - 1: + print("Flaky failure:", failure) + if failure: + raise failure + raise RuntimeError(f"Only passed {passes} of {min_passes} times") + + return wrapper + + # If `func` is callable, the decorator was used without arguments (`@flaky`) + if callable(func): + return decorator(func) + + # Otherwise, return the decorator function, allowing arguments (`@flaky(max_runs=...)`) + return decorator + + class AsyncMockConnection: def __init__(self): self.cancel_context = _CancellationContext() diff --git a/test/test_client_bulk_write.py b/test/test_client_bulk_write.py index 84313c5be0..ed04dd1554 100644 --- a/test/test_client_bulk_write.py +++ b/test/test_client_bulk_write.py @@ -25,6 +25,7 @@ client_context, unittest, ) +from test.utils import flaky from test.utils_shared import ( OvertCommandListener, ) @@ -615,8 +616,6 @@ def test_15_unacknowledged_write_across_batches(self): # https://github.com/mongodb/specifications/blob/master/source/client-side-operations-timeout/tests/README.md#11-multi-batch-bulkwrites class TestClientBulkWriteCSOT(IntegrationTest): def setUp(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") super().setUp() self.max_write_batch_size = client_context.max_write_batch_size self.max_bson_object_size = client_context.max_bson_size @@ -624,6 +623,7 @@ def setUp(self): @client_context.require_version_min(8, 0, 0, -24) @client_context.require_failCommand_fail_point + @flaky def test_timeout_in_multi_batch_bulk_write(self): _OVERHEAD = 500 diff --git a/test/test_csot.py b/test/test_csot.py index ff907cc9c5..b47f113a35 100644 --- a/test/test_csot.py +++ b/test/test_csot.py @@ -23,6 +23,7 @@ from test import IntegrationTest, client_context, unittest from test.unified_format import generate_test_classes +from test.utils import flaky import pymongo from pymongo import _csot @@ -43,9 +44,8 @@ class TestCSOT(IntegrationTest): RUN_ON_LOAD_BALANCER = True + @flaky def test_timeout_nested(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") coll = self.db.coll self.assertEqual(_csot.get_timeout(), None) self.assertEqual(_csot.get_deadline(), float("inf")) @@ -82,9 +82,8 @@ def test_timeout_nested(self): self.assertEqual(_csot.get_rtt(), 0.0) @client_context.require_change_streams + @flaky def test_change_stream_can_resume_after_timeouts(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") coll = self.db.test coll.insert_one({}) with coll.watch() as stream: diff --git a/test/test_cursor.py b/test/test_cursor.py index c33f509565..0de348e96c 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -31,6 +31,7 @@ sys.path[0:0] = [""] from test import IntegrationTest, client_context, unittest +from test.utils import flaky from test.utils_shared import ( AllowListEventListener, EventListener, @@ -1406,9 +1407,8 @@ def test_to_list_length(self): docs = c.to_list(3) self.assertEqual(len(docs), 2) + @flaky def test_to_list_csot_applied(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") client = self.single_client(timeoutMS=500, w=1) coll = client.pymongo.test # Initialize the client with a larger timeout to help make test less flakey @@ -1449,9 +1449,8 @@ def test_command_cursor_to_list_length(self): self.assertEqual(len(result.to_list(1)), 1) @client_context.require_failCommand_blockConnection + @flaky def test_command_cursor_to_list_csot_applied(self): - if os.environ.get("SKIP_CSOT_TESTS", ""): - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") client = self.single_client(timeoutMS=500, w=1) coll = client.pymongo.test # Initialize the client with a larger timeout to help make test less flakey diff --git a/test/unified_format.py b/test/unified_format.py index 0db037c654..685bc028a6 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -38,7 +38,6 @@ from test.unified_format_shared import ( KMS_TLS_OPTS, PLACEHOLDER_MAP, - SKIP_CSOT_TESTS, EventListenerUtil, MatchEvaluatorUtil, coerce_result, @@ -1361,9 +1360,6 @@ def verify_outcome(self, spec): self.assertListEqual(sorted_expected_documents, actual_documents) def run_scenario(self, spec, uri=None): - if "csot" in self.id().lower() and SKIP_CSOT_TESTS: - raise unittest.SkipTest("SKIP_CSOT_TESTS is set, skipping...") - # Kill all sessions before and after each test to prevent an open # transaction (from a test failure) from blocking collection/database # operations during test set up and tear down. diff --git a/test/unified_format_shared.py b/test/unified_format_shared.py index ea0f2f233e..17dd73ec8c 100644 --- a/test/unified_format_shared.py +++ b/test/unified_format_shared.py @@ -91,8 +91,6 @@ from pymongo.server_description import ServerDescription from pymongo.topology_description import TopologyDescription -SKIP_CSOT_TESTS = os.getenv("SKIP_CSOT_TESTS") - JSON_OPTS = json_util.JSONOptions(tz_aware=False) IS_INTERRUPTED = False diff --git a/test/utils.py b/test/utils.py index 25d95d1d3c..3e17811e6d 100644 --- a/test/utils.py +++ b/test/utils.py @@ -18,9 +18,11 @@ import asyncio import contextlib import random +import sys import threading # Used in the synchronized version of this file import time from asyncio import iscoroutinefunction +from functools import wraps from bson.son import SON from pymongo import MongoClient @@ -152,6 +154,42 @@ def joinall(tasks): asyncio.wait([t.task for t in tasks if t is not None], timeout=300) +def flaky(func=None, *, max_runs=2, min_passes=1, delay=1, affects_cpython_linux=False): + is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" + if is_cpython_linux and not affects_cpython_linux: + max_runs = 1 + min_passes = 1 + + def decorator(target_func): + @wraps(target_func) + def wrapper(*args, **kwargs): + passes = 0 + failure = None + for i in range(max_runs): + try: + result = target_func(*args, **kwargs) + passes += 1 + if passes == min_passes: + return result + except Exception as e: + failure = e + time.sleep(delay) + if failure is not None and i < max_runs - 1: + print("Flaky failure:", failure) + if failure: + raise failure + raise RuntimeError(f"Only passed {passes} of {min_passes} times") + + return wrapper + + # If `func` is callable, the decorator was used without arguments (`@flaky`) + if callable(func): + return decorator(func) + + # Otherwise, return the decorator function, allowing arguments (`@flaky(max_runs=...)`) + return decorator + + class MockConnection: def __init__(self): self.cancel_context = _CancellationContext() From c30c898938e07139f36d20c24bc859204098d3b3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 11:13:10 -0500 Subject: [PATCH 02/37] clean up flaky test handling --- test/asynchronous/unified_format.py | 51 +++++++++-------------------- test/asynchronous/utils.py | 20 +++++++++-- test/unified_format.py | 51 +++++++++-------------------- test/utils.py | 20 +++++++++-- 4 files changed, 68 insertions(+), 74 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 05c9e07591..d2d5981d01 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -35,7 +35,7 @@ client_knobs, unittest, ) -from test.asynchronous.utils import async_get_pool +from test.asynchronous.utils import async_get_pool, flaky from test.asynchronous.utils_spec_runner import SpecRunnerTask from test.unified_format_shared import ( KMS_TLS_OPTS, @@ -529,20 +529,10 @@ def maybe_skip_test(self, spec): self.skipTest("Implement PYTHON-1894") if "timeoutMS applied to entire download" in spec["description"]: self.skipTest("PyMongo's open_download_stream does not cap the stream's lifetime") - if ( - "Error returned from connection pool clear with interruptInUseConnections=true is retryable" - in spec["description"] - and not _IS_SYNC - ): - self.skipTest("PYTHON-5170 tests are flakey") - if "Driver extends timeout while streaming" in spec["description"] and not _IS_SYNC: - self.skipTest("PYTHON-5174 tests are flakey") class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: - if "gridfs" in class_name and sys.platform == "win32": - self.skipTest("PYTHON-3522 CSOT GridFS tests are flaky on Windows") if async_client_context.storage_engine == "mmapv1": self.skipTest( "MMAPv1 does not support retryable writes which is required for CSOT tests" @@ -1378,30 +1368,21 @@ async def run_scenario(self, spec, uri=None): # operations during test set up and tear down. await self.kill_all_sessions() - if "csot" in self.id().lower(): - # Retry CSOT tests up to 2 times to deal with flakey tests. - attempts = 3 - for i in range(attempts): - try: - return await self._run_scenario(spec, uri) - except (AssertionError, OperationFailure) as exc: - if isinstance(exc, OperationFailure) and ( - _IS_SYNC or "failpoint" not in exc._message - ): - raise - if i < attempts - 1: - print( - f"Retrying after attempt {i+1} of {self.id()} failed with:\n" - f"{traceback.format_exc()}", - file=sys.stderr, - ) - await self.asyncSetUp() - continue - raise - return None - else: - await self._run_scenario(spec, uri) - return None + # Handle flaky tests. + func = functools.partial(self._run_scenario, spec, uri) + flaky_tests = [ + # PYTHON-5170 + ".*test_discovery_and_monitoring.TestUnifiedInterruptInUsePoolClear.*", + # PYTHON-5174 + ".*Driver_extends_timeout_while_streaming", + ".*csot.*", + ] + for flaky_test in flaky_tests: + if re.match(flaky_test, self.id()) is not None: + decorator = flaky(reset_func=self.asyncSetUp, func_name=self.id()) + func = decorator(func) + break + await func() async def _run_scenario(self, spec, uri=None): # maybe skip test manually diff --git a/test/asynchronous/utils.py b/test/asynchronous/utils.py index cadf48f22e..2c59be81ba 100644 --- a/test/asynchronous/utils.py +++ b/test/asynchronous/utils.py @@ -21,6 +21,7 @@ import sys import threading # Used in the synchronized version of this file import time +import traceback from asyncio import iscoroutinefunction from functools import wraps @@ -156,7 +157,16 @@ async def async_joinall(tasks): await asyncio.wait([t.task for t in tasks if t is not None], timeout=300) -def flaky(func=None, *, max_runs=2, min_passes=1, delay=1, affects_cpython_linux=False): +def flaky( + func=None, + *, + max_runs=2, + min_passes=1, + delay=1, + affects_cpython_linux=False, + func_name=None, + reset_func=None, +): is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" if is_cpython_linux and not affects_cpython_linux: max_runs = 1 @@ -177,7 +187,13 @@ async def wrapper(*args, **kwargs): failure = e await asyncio.sleep(delay) if failure is not None and i < max_runs - 1: - print("Flaky failure:", failure) + print( + f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with:\n" + f"{traceback.format_exc()}", + file=sys.stderr, + ) + if reset_func: + await reset_func() if failure: raise failure raise RuntimeError(f"Only passed {passes} of {min_passes} times") diff --git a/test/unified_format.py b/test/unified_format.py index 685bc028a6..b0a35b4793 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -47,7 +47,7 @@ parse_collection_or_database_options, with_metaclass, ) -from test.utils import get_pool +from test.utils import flaky, get_pool from test.utils_shared import ( camel_to_snake, camel_to_snake_args, @@ -528,20 +528,10 @@ def maybe_skip_test(self, spec): self.skipTest("Implement PYTHON-1894") if "timeoutMS applied to entire download" in spec["description"]: self.skipTest("PyMongo's open_download_stream does not cap the stream's lifetime") - if ( - "Error returned from connection pool clear with interruptInUseConnections=true is retryable" - in spec["description"] - and not _IS_SYNC - ): - self.skipTest("PYTHON-5170 tests are flakey") - if "Driver extends timeout while streaming" in spec["description"] and not _IS_SYNC: - self.skipTest("PYTHON-5174 tests are flakey") class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: - if "gridfs" in class_name and sys.platform == "win32": - self.skipTest("PYTHON-3522 CSOT GridFS tests are flaky on Windows") if client_context.storage_engine == "mmapv1": self.skipTest( "MMAPv1 does not support retryable writes which is required for CSOT tests" @@ -1365,30 +1355,21 @@ def run_scenario(self, spec, uri=None): # operations during test set up and tear down. self.kill_all_sessions() - if "csot" in self.id().lower(): - # Retry CSOT tests up to 2 times to deal with flakey tests. - attempts = 3 - for i in range(attempts): - try: - return self._run_scenario(spec, uri) - except (AssertionError, OperationFailure) as exc: - if isinstance(exc, OperationFailure) and ( - _IS_SYNC or "failpoint" not in exc._message - ): - raise - if i < attempts - 1: - print( - f"Retrying after attempt {i+1} of {self.id()} failed with:\n" - f"{traceback.format_exc()}", - file=sys.stderr, - ) - self.setUp() - continue - raise - return None - else: - self._run_scenario(spec, uri) - return None + # Handle flaky tests. + func = functools.partial(self._run_scenario, spec, uri) + flaky_tests = [ + # PYTHON-5170 + ".*test_discovery_and_monitoring.TestUnifiedInterruptInUsePoolClear.*", + # PYTHON-5174 + ".*Driver_extends_timeout_while_streaming", + ".*csot.*", + ] + for flaky_test in flaky_tests: + if re.match(flaky_test, self.id()) is not None: + decorator = flaky(reset_func=self.setUp, func_name=self.id()) + func = decorator(func) + break + func() def _run_scenario(self, spec, uri=None): # maybe skip test manually diff --git a/test/utils.py b/test/utils.py index 3e17811e6d..9f49bb24ba 100644 --- a/test/utils.py +++ b/test/utils.py @@ -21,6 +21,7 @@ import sys import threading # Used in the synchronized version of this file import time +import traceback from asyncio import iscoroutinefunction from functools import wraps @@ -154,7 +155,16 @@ def joinall(tasks): asyncio.wait([t.task for t in tasks if t is not None], timeout=300) -def flaky(func=None, *, max_runs=2, min_passes=1, delay=1, affects_cpython_linux=False): +def flaky( + func=None, + *, + max_runs=2, + min_passes=1, + delay=1, + affects_cpython_linux=False, + func_name=None, + reset_func=None, +): is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" if is_cpython_linux and not affects_cpython_linux: max_runs = 1 @@ -175,7 +185,13 @@ def wrapper(*args, **kwargs): failure = e time.sleep(delay) if failure is not None and i < max_runs - 1: - print("Flaky failure:", failure) + print( + f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with:\n" + f"{traceback.format_exc()}", + file=sys.stderr, + ) + if reset_func: + reset_func() if failure: raise failure raise RuntimeError(f"Only passed {passes} of {min_passes} times") From a8a1dd263df61dc942ebc798eb908e03938a8776 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 11:54:15 -0500 Subject: [PATCH 03/37] debug --- test/asynchronous/test_srv_polling.py | 2 ++ test/asynchronous/unified_format.py | 4 ++-- test/test_srv_polling.py | 2 ++ test/unified_format.py | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/test_srv_polling.py b/test/asynchronous/test_srv_polling.py index 3ba50e77a8..35a8863297 100644 --- a/test/asynchronous/test_srv_polling.py +++ b/test/asynchronous/test_srv_polling.py @@ -18,6 +18,7 @@ import asyncio import sys import time +from test.asynchronous.utils import flaky from test.utils_shared import FunctionCallRecorder from typing import Any @@ -254,6 +255,7 @@ def final_callback(): # Nodelist should reflect new valid DNS resolver response. await self.assert_nodelist_change(response_final, client) + @flaky # PYTHON-5315 async def test_recover_from_initially_empty_seedlist(self): def empty_seedlist(): return [] diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index d2d5981d01..1efc23baa2 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -1380,8 +1380,8 @@ async def run_scenario(self, spec, uri=None): for flaky_test in flaky_tests: if re.match(flaky_test, self.id()) is not None: decorator = flaky(reset_func=self.asyncSetUp, func_name=self.id()) - func = decorator(func) - break + await decorator(func) + return await func() async def _run_scenario(self, spec, uri=None): diff --git a/test/test_srv_polling.py b/test/test_srv_polling.py index 971c3bad50..fd98cf7b41 100644 --- a/test/test_srv_polling.py +++ b/test/test_srv_polling.py @@ -18,6 +18,7 @@ import asyncio import sys import time +from test.utils import flaky from test.utils_shared import FunctionCallRecorder from typing import Any @@ -254,6 +255,7 @@ def final_callback(): # Nodelist should reflect new valid DNS resolver response. self.assert_nodelist_change(response_final, client) + @flaky # PYTHON-5315 def test_recover_from_initially_empty_seedlist(self): def empty_seedlist(): return [] diff --git a/test/unified_format.py b/test/unified_format.py index b0a35b4793..33953b72f6 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -1367,8 +1367,8 @@ def run_scenario(self, spec, uri=None): for flaky_test in flaky_tests: if re.match(flaky_test, self.id()) is not None: decorator = flaky(reset_func=self.setUp, func_name=self.id()) - func = decorator(func) - break + decorator(func) + return func() def _run_scenario(self, spec, uri=None): From c6ce564f941f50f907ff1ed4042ab377bbaf759e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 13:03:10 -0500 Subject: [PATCH 04/37] fix usage --- test/asynchronous/unified_format.py | 2 +- test/unified_format.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 1efc23baa2..737f83193b 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -1380,7 +1380,7 @@ async def run_scenario(self, spec, uri=None): for flaky_test in flaky_tests: if re.match(flaky_test, self.id()) is not None: decorator = flaky(reset_func=self.asyncSetUp, func_name=self.id()) - await decorator(func) + await decorator(func)() return await func() diff --git a/test/unified_format.py b/test/unified_format.py index 33953b72f6..d9fcbfaa15 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -1367,7 +1367,7 @@ def run_scenario(self, spec, uri=None): for flaky_test in flaky_tests: if re.match(flaky_test, self.id()) is not None: decorator = flaky(reset_func=self.setUp, func_name=self.id()) - decorator(func) + decorator(func)() return func() From 45efac918aae55ebbb5162adf989febd16339806 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 13:41:01 -0500 Subject: [PATCH 05/37] increase retries for csot --- test/asynchronous/unified_format.py | 11 +++++++---- test/unified_format.py | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 737f83193b..b3902951c2 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -1369,7 +1369,6 @@ async def run_scenario(self, spec, uri=None): await self.kill_all_sessions() # Handle flaky tests. - func = functools.partial(self._run_scenario, spec, uri) flaky_tests = [ # PYTHON-5170 ".*test_discovery_and_monitoring.TestUnifiedInterruptInUsePoolClear.*", @@ -1379,10 +1378,14 @@ async def run_scenario(self, spec, uri=None): ] for flaky_test in flaky_tests: if re.match(flaky_test, self.id()) is not None: - decorator = flaky(reset_func=self.asyncSetUp, func_name=self.id()) - await decorator(func)() + func_name = self.id() + options = dict(reset_func=self.asyncSetUp, func_name=func_name) + if "csot" in func_name: + options["max_runs"] = 3 + decorator = flaky(**options) + await decorator(self._run_scenario)(spec, uri) return - await func() + await self._run_scenario(spec, uri) async def _run_scenario(self, spec, uri=None): # maybe skip test manually diff --git a/test/unified_format.py b/test/unified_format.py index d9fcbfaa15..7e0808d93d 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -1356,7 +1356,6 @@ def run_scenario(self, spec, uri=None): self.kill_all_sessions() # Handle flaky tests. - func = functools.partial(self._run_scenario, spec, uri) flaky_tests = [ # PYTHON-5170 ".*test_discovery_and_monitoring.TestUnifiedInterruptInUsePoolClear.*", @@ -1366,10 +1365,14 @@ def run_scenario(self, spec, uri=None): ] for flaky_test in flaky_tests: if re.match(flaky_test, self.id()) is not None: - decorator = flaky(reset_func=self.setUp, func_name=self.id()) - decorator(func)() + func_name = self.id() + options = dict(reset_func=self.setUp, func_name=func_name) + if "csot" in func_name: + options["max_runs"] = 3 + decorator = flaky(**options) + decorator(self._run_scenario)(spec, uri) return - func() + self._run_scenario(spec, uri) def _run_scenario(self, spec, uri=None): # maybe skip test manually From b83f9bacdf1e7ef83146ecd3b907f2f17648b5f5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 15:59:11 -0500 Subject: [PATCH 06/37] try again --- test/asynchronous/unified_format.py | 10 ++++++++-- test/asynchronous/utils.py | 18 ++++++++---------- test/unified_format.py | 10 ++++++++-- test/utils.py | 18 ++++++++---------- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index b3902951c2..94e0495d2e 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -533,6 +533,14 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: + if "gridfs" in class_name and sys.platform == "win32": + self.skipTest("PYTHON-3522 CSOT GridFS tests are flaky on Windows") + if ( + "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset" + in description + and sys.platform != "linux" + ): + self.skipTest("PYTHON-3522 CSOT test is flaky on Windows and MacOS") if async_client_context.storage_engine == "mmapv1": self.skipTest( "MMAPv1 does not support retryable writes which is required for CSOT tests" @@ -1380,8 +1388,6 @@ async def run_scenario(self, spec, uri=None): if re.match(flaky_test, self.id()) is not None: func_name = self.id() options = dict(reset_func=self.asyncSetUp, func_name=func_name) - if "csot" in func_name: - options["max_runs"] = 3 decorator = flaky(**options) await decorator(self._run_scenario)(spec, uri) return diff --git a/test/asynchronous/utils.py b/test/asynchronous/utils.py index 2c59be81ba..39cf3a6bb6 100644 --- a/test/asynchronous/utils.py +++ b/test/asynchronous/utils.py @@ -17,6 +17,7 @@ import asyncio import contextlib +import os import random import sys import threading # Used in the synchronized version of this file @@ -168,7 +169,8 @@ def flaky( reset_func=None, ): is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" - if is_cpython_linux and not affects_cpython_linux: + disable_flaky = "DISABLE_FLAKY" in os.environ + if disable_flaky or (is_cpython_linux and not affects_cpython_linux): max_runs = 1 min_passes = 1 @@ -176,7 +178,6 @@ def decorator(target_func): @wraps(target_func) async def wrapper(*args, **kwargs): passes = 0 - failure = None for i in range(max_runs): try: result = await target_func(*args, **kwargs) @@ -184,27 +185,24 @@ async def wrapper(*args, **kwargs): if passes == min_passes: return result except Exception as e: - failure = e - await asyncio.sleep(delay) - if failure is not None and i < max_runs - 1: + if i == max_runs - 1: + raise e print( f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with:\n" f"{traceback.format_exc()}", file=sys.stderr, ) + await asyncio.sleep(delay) if reset_func: await reset_func() - if failure: - raise failure - raise RuntimeError(f"Only passed {passes} of {min_passes} times") return wrapper - # If `func` is callable, the decorator was used without arguments (`@flaky`) + # If `func` is callable, the decorator was used without arguments (`@flaky`). if callable(func): return decorator(func) - # Otherwise, return the decorator function, allowing arguments (`@flaky(max_runs=...)`) + # Otherwise, return the decorator function, allowing arguments (`@flaky(max_runs=...)`). return decorator diff --git a/test/unified_format.py b/test/unified_format.py index 7e0808d93d..262abafc1d 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -532,6 +532,14 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: + if "gridfs" in class_name and sys.platform == "win32": + self.skipTest("PYTHON-3522 CSOT GridFS tests are flaky on Windows") + if ( + "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset" + in description + and sys.platform != "linux" + ): + self.skipTest("PYTHON-3522 CSOT test is flaky on Windows and MacOS") if client_context.storage_engine == "mmapv1": self.skipTest( "MMAPv1 does not support retryable writes which is required for CSOT tests" @@ -1367,8 +1375,6 @@ def run_scenario(self, spec, uri=None): if re.match(flaky_test, self.id()) is not None: func_name = self.id() options = dict(reset_func=self.setUp, func_name=func_name) - if "csot" in func_name: - options["max_runs"] = 3 decorator = flaky(**options) decorator(self._run_scenario)(spec, uri) return diff --git a/test/utils.py b/test/utils.py index 9f49bb24ba..b75f854f7b 100644 --- a/test/utils.py +++ b/test/utils.py @@ -17,6 +17,7 @@ import asyncio import contextlib +import os import random import sys import threading # Used in the synchronized version of this file @@ -166,7 +167,8 @@ def flaky( reset_func=None, ): is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" - if is_cpython_linux and not affects_cpython_linux: + disable_flaky = "DISABLE_FLAKY" in os.environ + if disable_flaky or (is_cpython_linux and not affects_cpython_linux): max_runs = 1 min_passes = 1 @@ -174,7 +176,6 @@ def decorator(target_func): @wraps(target_func) def wrapper(*args, **kwargs): passes = 0 - failure = None for i in range(max_runs): try: result = target_func(*args, **kwargs) @@ -182,27 +183,24 @@ def wrapper(*args, **kwargs): if passes == min_passes: return result except Exception as e: - failure = e - time.sleep(delay) - if failure is not None and i < max_runs - 1: + if i == max_runs - 1: + raise e print( f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with:\n" f"{traceback.format_exc()}", file=sys.stderr, ) + time.sleep(delay) if reset_func: reset_func() - if failure: - raise failure - raise RuntimeError(f"Only passed {passes} of {min_passes} times") return wrapper - # If `func` is callable, the decorator was used without arguments (`@flaky`) + # If `func` is callable, the decorator was used without arguments (`@flaky`). if callable(func): return decorator(func) - # Otherwise, return the decorator function, allowing arguments (`@flaky(max_runs=...)`) + # Otherwise, return the decorator function, allowing arguments (`@flaky(max_runs=...)`). return decorator From 7a316f7d6b2b1339dfa1cc204801ca15273d94c3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 16:15:48 -0500 Subject: [PATCH 07/37] try 3 times for csot --- test/asynchronous/unified_format.py | 2 ++ test/unified_format.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 94e0495d2e..487de1f174 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -1388,6 +1388,8 @@ async def run_scenario(self, spec, uri=None): if re.match(flaky_test, self.id()) is not None: func_name = self.id() options = dict(reset_func=self.asyncSetUp, func_name=func_name) + if "csot" in func_name: + options["max_runs"] = 3 decorator = flaky(**options) await decorator(self._run_scenario)(spec, uri) return diff --git a/test/unified_format.py b/test/unified_format.py index 262abafc1d..47c8998b0e 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -1375,6 +1375,8 @@ def run_scenario(self, spec, uri=None): if re.match(flaky_test, self.id()) is not None: func_name = self.id() options = dict(reset_func=self.setUp, func_name=func_name) + if "csot" in func_name: + options["max_runs"] = 3 decorator = flaky(**options) decorator(self._run_scenario)(spec, uri) return From 618447591748dd94cb1686b3e07b5c93b47700a8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 20:48:41 -0500 Subject: [PATCH 08/37] use tailored list of skips --- test/asynchronous/unified_format.py | 21 ++++++++++++++------- test/unified_format.py | 21 ++++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 487de1f174..88c5d594b0 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -533,14 +533,21 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: - if "gridfs" in class_name and sys.platform == "win32": - self.skipTest("PYTHON-3522 CSOT GridFS tests are flaky on Windows") - if ( + slow_win32 = [ + "maxTimeMS value in the command is less than timeoutMS", + "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset", + "timeoutMS applied to update during a rename", + "timeoutMS applied to find to get chunks", + "test timeoutMS applied to find to get files document", + "test timeoutMS applied to find command", + ] + slow_macos = [ "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset" - in description - and sys.platform != "linux" - ): - self.skipTest("PYTHON-3522 CSOT test is flaky on Windows and MacOS") + ] + if sys.platform == "win32" and description in slow_win32: + self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") + if sys.platform == "darwin" and description in slow_macos: + self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if async_client_context.storage_engine == "mmapv1": self.skipTest( "MMAPv1 does not support retryable writes which is required for CSOT tests" diff --git a/test/unified_format.py b/test/unified_format.py index 47c8998b0e..52ff68117a 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -532,14 +532,21 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: - if "gridfs" in class_name and sys.platform == "win32": - self.skipTest("PYTHON-3522 CSOT GridFS tests are flaky on Windows") - if ( + slow_win32 = [ + "maxTimeMS value in the command is less than timeoutMS", + "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset", + "timeoutMS applied to update during a rename", + "timeoutMS applied to find to get chunks", + "test timeoutMS applied to find to get files document", + "test timeoutMS applied to find command", + ] + slow_macos = [ "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset" - in description - and sys.platform != "linux" - ): - self.skipTest("PYTHON-3522 CSOT test is flaky on Windows and MacOS") + ] + if sys.platform == "win32" and description in slow_win32: + self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") + if sys.platform == "darwin" and description in slow_macos: + self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if client_context.storage_engine == "mmapv1": self.skipTest( "MMAPv1 does not support retryable writes which is required for CSOT tests" From c36dd267fbe71ab2bf6f318f9cd5ad6d99f477d4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 21:08:52 -0500 Subject: [PATCH 09/37] fix skip handling --- test/asynchronous/unified_format.py | 15 ++++++++------- test/unified_format.py | 15 ++++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 88c5d594b0..e84d34a36d 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -533,16 +533,17 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: + # Skip tests that are too slow to run on a given platform. slow_win32 = [ - "maxTimeMS value in the command is less than timeoutMS", - "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset", - "timeoutMS applied to update during a rename", - "timeoutMS applied to find to get chunks", - "test timeoutMS applied to find to get files document", - "test timeoutMS applied to find command", + "maxtimems value in the command is less than maxtimems", + "non-tailable cursor lifetime remaining maxtimems applied to getmore if timeoutmode is unset", + "maxtimems applied to update during a rename", + "maxtimems applied to find to get chunks", + "maxtimems applied to find to get files document", + "maxtimems applied to find command", ] slow_macos = [ - "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset" + "non-tailable cursor lifetime remaining maxtimems applied to getmore if timeoutmode is unset" ] if sys.platform == "win32" and description in slow_win32: self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") diff --git a/test/unified_format.py b/test/unified_format.py index 52ff68117a..24155dd829 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -532,16 +532,17 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: + # Skip tests that are too slow to run on a given platform. slow_win32 = [ - "maxTimeMS value in the command is less than timeoutMS", - "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset", - "timeoutMS applied to update during a rename", - "timeoutMS applied to find to get chunks", - "test timeoutMS applied to find to get files document", - "test timeoutMS applied to find command", + "maxtimems value in the command is less than maxtimems", + "non-tailable cursor lifetime remaining maxtimems applied to getmore if timeoutmode is unset", + "maxtimems applied to update during a rename", + "maxtimems applied to find to get chunks", + "maxtimems applied to find to get files document", + "maxtimems applied to find command", ] slow_macos = [ - "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset" + "non-tailable cursor lifetime remaining maxtimems applied to getmore if timeoutmode is unset" ] if sys.platform == "win32" and description in slow_win32: self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") From 85bc62d9e2e3cfcb93629e26c45da65083d02d5d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 21:30:02 -0500 Subject: [PATCH 10/37] fix skip --- test/asynchronous/unified_format.py | 4 ++-- test/unified_format.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index e84d34a36d..d45c66dd6a 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -536,14 +536,14 @@ def maybe_skip_test(self, spec): # Skip tests that are too slow to run on a given platform. slow_win32 = [ "maxtimems value in the command is less than maxtimems", - "non-tailable cursor lifetime remaining maxtimems applied to getmore if timeoutmode is unset", + "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset", "maxtimems applied to update during a rename", "maxtimems applied to find to get chunks", "maxtimems applied to find to get files document", "maxtimems applied to find command", ] slow_macos = [ - "non-tailable cursor lifetime remaining maxtimems applied to getmore if timeoutmode is unset" + "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutode is unset" ] if sys.platform == "win32" and description in slow_win32: self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") diff --git a/test/unified_format.py b/test/unified_format.py index 24155dd829..bc84658db1 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -535,14 +535,14 @@ def maybe_skip_test(self, spec): # Skip tests that are too slow to run on a given platform. slow_win32 = [ "maxtimems value in the command is less than maxtimems", - "non-tailable cursor lifetime remaining maxtimems applied to getmore if timeoutmode is unset", + "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset", "maxtimems applied to update during a rename", "maxtimems applied to find to get chunks", "maxtimems applied to find to get files document", "maxtimems applied to find command", ] slow_macos = [ - "non-tailable cursor lifetime remaining maxtimems applied to getmore if timeoutmode is unset" + "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutode is unset" ] if sys.platform == "win32" and description in slow_win32: self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") From 35a33005eabb8d8cbbc50baa8fa4766e8550decd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 21:31:28 -0500 Subject: [PATCH 11/37] fix skips --- test/asynchronous/unified_format.py | 4 ++-- test/unified_format.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index d45c66dd6a..c93b170f5d 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -535,7 +535,7 @@ def maybe_skip_test(self, spec): if "csot" in class_name: # Skip tests that are too slow to run on a given platform. slow_win32 = [ - "maxtimems value in the command is less than maxtimems", + "maxtimems value in the command is less than timeoutms", "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset", "maxtimems applied to update during a rename", "maxtimems applied to find to get chunks", @@ -543,7 +543,7 @@ def maybe_skip_test(self, spec): "maxtimems applied to find command", ] slow_macos = [ - "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutode is unset" + "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset" ] if sys.platform == "win32" and description in slow_win32: self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") diff --git a/test/unified_format.py b/test/unified_format.py index bc84658db1..74f8183121 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -534,7 +534,7 @@ def maybe_skip_test(self, spec): if "csot" in class_name: # Skip tests that are too slow to run on a given platform. slow_win32 = [ - "maxtimems value in the command is less than maxtimems", + "maxtimems value in the command is less than timeoutms", "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset", "maxtimems applied to update during a rename", "maxtimems applied to find to get chunks", @@ -542,7 +542,7 @@ def maybe_skip_test(self, spec): "maxtimems applied to find command", ] slow_macos = [ - "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutode is unset" + "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset" ] if sys.platform == "win32" and description in slow_win32: self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") From 3ba37205ca0edc2a3a01b5aeb83875fc26135280 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Jun 2025 21:52:10 -0500 Subject: [PATCH 12/37] fix skips --- test/asynchronous/unified_format.py | 4 ++-- test/unified_format.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index c93b170f5d..79a9ea26f1 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -536,14 +536,14 @@ def maybe_skip_test(self, spec): # Skip tests that are too slow to run on a given platform. slow_win32 = [ "maxtimems value in the command is less than timeoutms", - "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset", + "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", "maxtimems applied to update during a rename", "maxtimems applied to find to get chunks", "maxtimems applied to find to get files document", "maxtimems applied to find command", ] slow_macos = [ - "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset" + "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" ] if sys.platform == "win32" and description in slow_win32: self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") diff --git a/test/unified_format.py b/test/unified_format.py index 74f8183121..162bbc3f09 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -535,14 +535,14 @@ def maybe_skip_test(self, spec): # Skip tests that are too slow to run on a given platform. slow_win32 = [ "maxtimems value in the command is less than timeoutms", - "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset", + "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", "maxtimems applied to update during a rename", "maxtimems applied to find to get chunks", "maxtimems applied to find to get files document", "maxtimems applied to find command", ] slow_macos = [ - "non-tailable cursor lifetime remaining timeoutms applied to getMore if timeoutmode is unset" + "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" ] if sys.platform == "win32" and description in slow_win32: self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") From 08e7324e128bb2d734334802c351b7c0d93ca34a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 08:02:32 -0500 Subject: [PATCH 13/37] handle more flakiness --- test/asynchronous/test_client_bulk_write.py | 2 +- test/asynchronous/test_encryption.py | 5 +++++ test/asynchronous/unified_format.py | 16 ++++++++++------ test/test_client_bulk_write.py | 2 +- test/test_encryption.py | 5 +++++ test/test_topology.py | 4 +++- test/unified_format.py | 16 ++++++++++------ 7 files changed, 35 insertions(+), 15 deletions(-) diff --git a/test/asynchronous/test_client_bulk_write.py b/test/asynchronous/test_client_bulk_write.py index 0402d75e0f..4ca0f8fd3a 100644 --- a/test/asynchronous/test_client_bulk_write.py +++ b/test/asynchronous/test_client_bulk_write.py @@ -627,7 +627,7 @@ async def asyncSetUp(self): @async_client_context.require_version_min(8, 0, 0, -24) @async_client_context.require_failCommand_fail_point - @flaky + @flaky(max_runs=3, affects_cpython_linux=True) async def test_timeout_in_multi_batch_bulk_write(self): _OVERHEAD = 500 diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 9093b97ab4..6ebba8c19f 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -32,6 +32,7 @@ import warnings from test.asynchronous import AsyncIntegrationTest, AsyncPyMongoTestCase, async_client_context from test.asynchronous.test_bulk import AsyncBulkTestBase +from test.asynchronous.utils import flaky from test.asynchronous.utils_spec_runner import AsyncSpecRunner, AsyncSpecTestCreator from threading import Thread from typing import Any, Dict, Mapping, Optional @@ -3275,6 +3276,8 @@ async def asyncSetUp(self): OPTS, ) + # PYTHON-4982 + @flaky async def test_01_simple_create(self): coll, _ = await self.client_encryption.create_encrypted_collection( database=self.db, @@ -3491,6 +3494,8 @@ async def asyncSetUp(self) -> None: hello = await self.mongocryptd_client.db.command("hello") self.assertNotIn("logicalSessionTimeoutMinutes", hello) + # PYTHON-4982 + @flaky async def test_implicit_session_ignored_when_unsupported(self): self.listener.reset() with self.assertRaises(OperationFailure): diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 79a9ea26f1..68477a2692 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -537,16 +537,12 @@ def maybe_skip_test(self, spec): slow_win32 = [ "maxtimems value in the command is less than timeoutms", "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", - "maxtimems applied to update during a rename", - "maxtimems applied to find to get chunks", - "maxtimems applied to find to get files document", - "maxtimems applied to find command", ] slow_macos = [ "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" ] - if sys.platform == "win32" and description in slow_win32: - self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") + if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name: + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") if sys.platform == "darwin" and description in slow_macos: self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if async_client_context.storage_engine == "mmapv1": @@ -1390,6 +1386,13 @@ async def run_scenario(self, spec, uri=None): ".*test_discovery_and_monitoring.TestUnifiedInterruptInUsePoolClear.*", # PYTHON-5174 ".*Driver_extends_timeout_while_streaming", + # PYTHON-5315 + ".*TestSrvPolling.test_recover_from_initially_.*", + # PYTHON-4987 + ".*UnknownTransactionCommitResult_labels_to_connection_errors", + # PYTHON-3689 + ".*TestProse.test_load_balancing", + # PYTHON-3522 ".*csot.*", ] for flaky_test in flaky_tests: @@ -1398,6 +1401,7 @@ async def run_scenario(self, spec, uri=None): options = dict(reset_func=self.asyncSetUp, func_name=func_name) if "csot" in func_name: options["max_runs"] = 3 + options["affects_cpython_linux"] = True decorator = flaky(**options) await decorator(self._run_scenario)(spec, uri) return diff --git a/test/test_client_bulk_write.py b/test/test_client_bulk_write.py index ed04dd1554..b7071e6da4 100644 --- a/test/test_client_bulk_write.py +++ b/test/test_client_bulk_write.py @@ -623,7 +623,7 @@ def setUp(self): @client_context.require_version_min(8, 0, 0, -24) @client_context.require_failCommand_fail_point - @flaky + @flaky(max_runs=3, affects_cpython_linux=True) def test_timeout_in_multi_batch_bulk_write(self): _OVERHEAD = 500 diff --git a/test/test_encryption.py b/test/test_encryption.py index 3a86838af3..bd38c554a9 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -32,6 +32,7 @@ import warnings from test import IntegrationTest, PyMongoTestCase, client_context from test.test_bulk import BulkTestBase +from test.utils import flaky from test.utils_spec_runner import SpecRunner, SpecTestCreator from threading import Thread from typing import Any, Dict, Mapping, Optional @@ -3257,6 +3258,8 @@ def setUp(self): OPTS, ) + # PYTHON-4982 + @flaky def test_01_simple_create(self): coll, _ = self.client_encryption.create_encrypted_collection( database=self.db, @@ -3473,6 +3476,8 @@ def setUp(self) -> None: hello = self.mongocryptd_client.db.command("hello") self.assertNotIn("logicalSessionTimeoutMinutes", hello) + # PYTHON-4982 + @flaky def test_implicit_session_ignored_when_unsupported(self): self.listener.reset() with self.assertRaises(OperationFailure): diff --git a/test/test_topology.py b/test/test_topology.py index 141b2d7f21..2cbe95eb45 100644 --- a/test/test_topology.py +++ b/test/test_topology.py @@ -23,7 +23,7 @@ from test import client_knobs, unittest from test.pymongo_mocks import DummyMonitor -from test.utils import MockPool +from test.utils import MockPool, flaky from test.utils_shared import wait_until from bson.objectid import ObjectId @@ -750,6 +750,8 @@ def get_primary(): class TestTopologyErrors(TopologyTest): # Errors when calling hello. + # PYTHON-5366 + @flaky def test_pool_reset(self): # hello succeeds at first, then always raises socket error. hello_count = [0] diff --git a/test/unified_format.py b/test/unified_format.py index 162bbc3f09..a785a86e1c 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -536,16 +536,12 @@ def maybe_skip_test(self, spec): slow_win32 = [ "maxtimems value in the command is less than timeoutms", "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", - "maxtimems applied to update during a rename", - "maxtimems applied to find to get chunks", - "maxtimems applied to find to get files document", - "maxtimems applied to find command", ] slow_macos = [ "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" ] - if sys.platform == "win32" and description in slow_win32: - self.skipTest("PYTHON-3522 CSOT test run too slow on Windows") + if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name: + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") if sys.platform == "darwin" and description in slow_macos: self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if client_context.storage_engine == "mmapv1": @@ -1377,6 +1373,13 @@ def run_scenario(self, spec, uri=None): ".*test_discovery_and_monitoring.TestUnifiedInterruptInUsePoolClear.*", # PYTHON-5174 ".*Driver_extends_timeout_while_streaming", + # PYTHON-5315 + ".*TestSrvPolling.test_recover_from_initially_.*", + # PYTHON-4987 + ".*UnknownTransactionCommitResult_labels_to_connection_errors", + # PYTHON-3689 + ".*TestProse.test_load_balancing", + # PYTHON-3522 ".*csot.*", ] for flaky_test in flaky_tests: @@ -1385,6 +1388,7 @@ def run_scenario(self, spec, uri=None): options = dict(reset_func=self.setUp, func_name=func_name) if "csot" in func_name: options["max_runs"] = 3 + options["affects_cpython_linux"] = True decorator = flaky(**options) decorator(self._run_scenario)(spec, uri) return From 3cace1a5c20d21321a6c13d28038882a1074f16d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 08:35:01 -0500 Subject: [PATCH 14/37] fix flaky test setup --- test/asynchronous/test_encryption.py | 3 +-- test/test_encryption.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 6ebba8c19f..6179a01b04 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3262,6 +3262,7 @@ async def test_kms_retry(self): class TestAutomaticDecryptionKeys(AsyncEncryptionIntegrationTest): @async_client_context.require_no_standalone @async_client_context.require_version_min(7, 0, -1) + @flaky # PYTHON-4982 async def asyncSetUp(self): await super().asyncSetUp() self.key1_document = json_data("etc", "data", "keys", "key1-document.json") @@ -3276,8 +3277,6 @@ async def asyncSetUp(self): OPTS, ) - # PYTHON-4982 - @flaky async def test_01_simple_create(self): coll, _ = await self.client_encryption.create_encrypted_collection( database=self.db, diff --git a/test/test_encryption.py b/test/test_encryption.py index bd38c554a9..1811206a02 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3244,6 +3244,7 @@ def test_kms_retry(self): class TestAutomaticDecryptionKeys(EncryptionIntegrationTest): @client_context.require_no_standalone @client_context.require_version_min(7, 0, -1) + @flaky # PYTHON-4982 def setUp(self): super().setUp() self.key1_document = json_data("etc", "data", "keys", "key1-document.json") @@ -3258,8 +3259,6 @@ def setUp(self): OPTS, ) - # PYTHON-4982 - @flaky def test_01_simple_create(self): coll, _ = self.client_encryption.create_encrypted_collection( database=self.db, From b2ddcc0d284ab8d33a398c60d1546d4c96ee6550 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 08:53:42 -0500 Subject: [PATCH 15/37] fix flaky test setup --- test/asynchronous/test_encryption.py | 3 +-- test/test_encryption.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 6179a01b04..3b3e12c649 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3481,6 +3481,7 @@ class TestNoSessionsSupport(AsyncEncryptionIntegrationTest): mongocryptd_client: AsyncMongoClient MONGOCRYPTD_PORT = 27020 + @flaky # PYTHON-4982 async def asyncSetUp(self) -> None: await super().asyncSetUp() start_mongocryptd(self.MONGOCRYPTD_PORT) @@ -3493,8 +3494,6 @@ async def asyncSetUp(self) -> None: hello = await self.mongocryptd_client.db.command("hello") self.assertNotIn("logicalSessionTimeoutMinutes", hello) - # PYTHON-4982 - @flaky async def test_implicit_session_ignored_when_unsupported(self): self.listener.reset() with self.assertRaises(OperationFailure): diff --git a/test/test_encryption.py b/test/test_encryption.py index 1811206a02..ad0f00f835 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3463,6 +3463,7 @@ class TestNoSessionsSupport(EncryptionIntegrationTest): mongocryptd_client: MongoClient MONGOCRYPTD_PORT = 27020 + @flaky # PYTHON-4982 def setUp(self) -> None: super().setUp() start_mongocryptd(self.MONGOCRYPTD_PORT) @@ -3475,8 +3476,6 @@ def setUp(self) -> None: hello = self.mongocryptd_client.db.command("hello") self.assertNotIn("logicalSessionTimeoutMinutes", hello) - # PYTHON-4982 - @flaky def test_implicit_session_ignored_when_unsupported(self): self.listener.reset() with self.assertRaises(OperationFailure): From ae4e571c4227674fe99487e91cd254b20c1f4f9a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 09:56:36 -0500 Subject: [PATCH 16/37] fix flaky test --- test/asynchronous/test_encryption.py | 1 + test/test_encryption.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 3b3e12c649..32b3d682ae 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3494,6 +3494,7 @@ async def asyncSetUp(self) -> None: hello = await self.mongocryptd_client.db.command("hello") self.assertNotIn("logicalSessionTimeoutMinutes", hello) + @flaky # PYTHON-4982 async def test_implicit_session_ignored_when_unsupported(self): self.listener.reset() with self.assertRaises(OperationFailure): diff --git a/test/test_encryption.py b/test/test_encryption.py index ad0f00f835..21d2e22f58 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3476,6 +3476,7 @@ def setUp(self) -> None: hello = self.mongocryptd_client.db.command("hello") self.assertNotIn("logicalSessionTimeoutMinutes", hello) + @flaky # PYTHON-4982 def test_implicit_session_ignored_when_unsupported(self): self.listener.reset() with self.assertRaises(OperationFailure): From 54fe7c24be8db554049707f8e0c527a4cc9850ee Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 10:13:46 -0500 Subject: [PATCH 17/37] fix flaky test --- test/asynchronous/test_encryption.py | 6 ++++-- test/test_encryption.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 32b3d682ae..41ec2ac772 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3481,7 +3481,6 @@ class TestNoSessionsSupport(AsyncEncryptionIntegrationTest): mongocryptd_client: AsyncMongoClient MONGOCRYPTD_PORT = 27020 - @flaky # PYTHON-4982 async def asyncSetUp(self) -> None: await super().asyncSetUp() start_mongocryptd(self.MONGOCRYPTD_PORT) @@ -3494,7 +3493,6 @@ async def asyncSetUp(self) -> None: hello = await self.mongocryptd_client.db.command("hello") self.assertNotIn("logicalSessionTimeoutMinutes", hello) - @flaky # PYTHON-4982 async def test_implicit_session_ignored_when_unsupported(self): self.listener.reset() with self.assertRaises(OperationFailure): @@ -3507,6 +3505,8 @@ async def test_implicit_session_ignored_when_unsupported(self): self.assertNotIn("lsid", self.listener.started_events[1].command) + await self.mongocryptd_client.close() + async def test_explicit_session_errors_when_unsupported(self): self.listener.reset() async with self.mongocryptd_client.start_session() as s: @@ -3519,6 +3519,8 @@ async def test_explicit_session_errors_when_unsupported(self): ): await self.mongocryptd_client.db.test.insert_one({"x": 1}, session=s) + await self.mongocryptd_client.close() + if __name__ == "__main__": unittest.main() diff --git a/test/test_encryption.py b/test/test_encryption.py index 21d2e22f58..b4daf641f8 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3463,7 +3463,6 @@ class TestNoSessionsSupport(EncryptionIntegrationTest): mongocryptd_client: MongoClient MONGOCRYPTD_PORT = 27020 - @flaky # PYTHON-4982 def setUp(self) -> None: super().setUp() start_mongocryptd(self.MONGOCRYPTD_PORT) @@ -3476,7 +3475,6 @@ def setUp(self) -> None: hello = self.mongocryptd_client.db.command("hello") self.assertNotIn("logicalSessionTimeoutMinutes", hello) - @flaky # PYTHON-4982 def test_implicit_session_ignored_when_unsupported(self): self.listener.reset() with self.assertRaises(OperationFailure): @@ -3489,6 +3487,8 @@ def test_implicit_session_ignored_when_unsupported(self): self.assertNotIn("lsid", self.listener.started_events[1].command) + self.mongocryptd_client.close() + def test_explicit_session_errors_when_unsupported(self): self.listener.reset() with self.mongocryptd_client.start_session() as s: @@ -3501,6 +3501,8 @@ def test_explicit_session_errors_when_unsupported(self): ): self.mongocryptd_client.db.test.insert_one({"x": 1}, session=s) + self.mongocryptd_client.close() + if __name__ == "__main__": unittest.main() From ad91418bb72e270fb5f87e903ddc1c055f44d07f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 10:25:15 -0500 Subject: [PATCH 18/37] skip test on pypy --- test/asynchronous/test_encryption.py | 2 ++ test/test_encryption.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 41ec2ac772..6dea9e5b82 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3482,6 +3482,8 @@ class TestNoSessionsSupport(AsyncEncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 async def asyncSetUp(self) -> None: + if sys.implementation.lower() == "pypy": + raise self.skipTest("PYTHON-4982 Skipping test on pypy") await super().asyncSetUp() start_mongocryptd(self.MONGOCRYPTD_PORT) diff --git a/test/test_encryption.py b/test/test_encryption.py index b4daf641f8..00c8197b1a 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3464,6 +3464,8 @@ class TestNoSessionsSupport(EncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 def setUp(self) -> None: + if sys.implementation.lower() == "pypy": + raise self.skipTest("PYTHON-4982 Skipping test on pypy") super().setUp() start_mongocryptd(self.MONGOCRYPTD_PORT) From da6c68a46e65f547240d4eeefe0fe72415774be3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 10:47:49 -0500 Subject: [PATCH 19/37] fix skip --- test/asynchronous/test_encryption.py | 2 +- test/test_encryption.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 6dea9e5b82..4446280ff6 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3482,7 +3482,7 @@ class TestNoSessionsSupport(AsyncEncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 async def asyncSetUp(self) -> None: - if sys.implementation.lower() == "pypy": + if sys.implementation.name.lower() == "pypy": raise self.skipTest("PYTHON-4982 Skipping test on pypy") await super().asyncSetUp() start_mongocryptd(self.MONGOCRYPTD_PORT) diff --git a/test/test_encryption.py b/test/test_encryption.py index 00c8197b1a..7e3f04c7b4 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3464,7 +3464,7 @@ class TestNoSessionsSupport(EncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 def setUp(self) -> None: - if sys.implementation.lower() == "pypy": + if sys.implementation.name.lower() == "pypy": raise self.skipTest("PYTHON-4982 Skipping test on pypy") super().setUp() start_mongocryptd(self.MONGOCRYPTD_PORT) From 89de6b2b2cee8c255e893a73289f6d416029eb8f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 11:22:37 -0500 Subject: [PATCH 20/37] fix skip --- test/asynchronous/test_encryption.py | 6 ++++-- test/test_encryption.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 4446280ff6..754ffb86b1 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3482,8 +3482,6 @@ class TestNoSessionsSupport(AsyncEncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 async def asyncSetUp(self) -> None: - if sys.implementation.name.lower() == "pypy": - raise self.skipTest("PYTHON-4982 Skipping test on pypy") await super().asyncSetUp() start_mongocryptd(self.MONGOCRYPTD_PORT) @@ -3496,6 +3494,8 @@ async def asyncSetUp(self) -> None: self.assertNotIn("logicalSessionTimeoutMinutes", hello) async def test_implicit_session_ignored_when_unsupported(self): + if sys.implementation.name.lower() == "pypy": + raise self.skipTest("PYTHON-4982 Skipping test on pypy") self.listener.reset() with self.assertRaises(OperationFailure): await self.mongocryptd_client.db.test.find_one() @@ -3510,6 +3510,8 @@ async def test_implicit_session_ignored_when_unsupported(self): await self.mongocryptd_client.close() async def test_explicit_session_errors_when_unsupported(self): + if sys.implementation.name.lower() == "pypy": + raise self.skipTest("PYTHON-4982 Skipping test on pypy") self.listener.reset() async with self.mongocryptd_client.start_session() as s: with self.assertRaisesRegex( diff --git a/test/test_encryption.py b/test/test_encryption.py index 7e3f04c7b4..4c256dc9d1 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3464,8 +3464,6 @@ class TestNoSessionsSupport(EncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 def setUp(self) -> None: - if sys.implementation.name.lower() == "pypy": - raise self.skipTest("PYTHON-4982 Skipping test on pypy") super().setUp() start_mongocryptd(self.MONGOCRYPTD_PORT) @@ -3478,6 +3476,8 @@ def setUp(self) -> None: self.assertNotIn("logicalSessionTimeoutMinutes", hello) def test_implicit_session_ignored_when_unsupported(self): + if sys.implementation.name.lower() == "pypy": + raise self.skipTest("PYTHON-4982 Skipping test on pypy") self.listener.reset() with self.assertRaises(OperationFailure): self.mongocryptd_client.db.test.find_one() @@ -3492,6 +3492,8 @@ def test_implicit_session_ignored_when_unsupported(self): self.mongocryptd_client.close() def test_explicit_session_errors_when_unsupported(self): + if sys.implementation.name.lower() == "pypy": + raise self.skipTest("PYTHON-4982 Skipping test on pypy") self.listener.reset() with self.mongocryptd_client.start_session() as s: with self.assertRaisesRegex( From 5f1bbc639773055e62b39df5d6ee96c9adb4bb13 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 11:30:49 -0500 Subject: [PATCH 21/37] add another skip --- test/asynchronous/test_encryption.py | 2 ++ test/test_encryption.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 754ffb86b1..cd41cf47f1 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3482,6 +3482,8 @@ class TestNoSessionsSupport(AsyncEncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 async def asyncSetUp(self) -> None: + if sys.implementation.name.lower() == "pypy": + return await super().asyncSetUp() start_mongocryptd(self.MONGOCRYPTD_PORT) diff --git a/test/test_encryption.py b/test/test_encryption.py index 4c256dc9d1..bd09a00eb9 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3464,6 +3464,8 @@ class TestNoSessionsSupport(EncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 def setUp(self) -> None: + if sys.implementation.name.lower() == "pypy": + return super().setUp() start_mongocryptd(self.MONGOCRYPTD_PORT) From 22b837394cf2fa4026a16575eab79cf773fe26d2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 12:27:58 -0500 Subject: [PATCH 22/37] fix teardown --- test/__init__.py | 18 ++++++++++++------ test/asynchronous/__init__.py | 18 ++++++++++++------ test/asynchronous/test_encryption.py | 6 ------ test/test_encryption.py | 6 ------ 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/test/__init__.py b/test/__init__.py index 0e6046b527..8eed8cbf2b 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -32,6 +32,7 @@ import warnings from asyncio import iscoroutinefunction +from pymongo.errors import AutoReconnect from pymongo.synchronous.uri_parser import parse_uri try: @@ -1237,12 +1238,17 @@ def teardown(): c = client_context.client if c: if not client_context.is_data_lake: - c.drop_database("pymongo-pooling-tests") - c.drop_database("pymongo_test") - c.drop_database("pymongo_test1") - c.drop_database("pymongo_test2") - c.drop_database("pymongo_test_mike") - c.drop_database("pymongo_test_bernie") + try: + c.drop_database("pymongo-pooling-tests") + c.drop_database("pymongo_test") + c.drop_database("pymongo_test1") + c.drop_database("pymongo_test2") + c.drop_database("pymongo_test_mike") + c.drop_database("pymongo_test_bernie") + except AutoReconnect: + # PYTHON-4982 + if sys.implementation.name.lower() != "pypy": + raise c.close() print_running_clients() diff --git a/test/asynchronous/__init__.py b/test/asynchronous/__init__.py index 52583d30ef..1cf05ff516 100644 --- a/test/asynchronous/__init__.py +++ b/test/asynchronous/__init__.py @@ -33,6 +33,7 @@ from asyncio import iscoroutinefunction from pymongo.asynchronous.uri_parser import parse_uri +from pymongo.errors import AutoReconnect try: import ipaddress @@ -1253,12 +1254,17 @@ async def async_teardown(): c = async_client_context.client if c: if not async_client_context.is_data_lake: - await c.drop_database("pymongo-pooling-tests") - await c.drop_database("pymongo_test") - await c.drop_database("pymongo_test1") - await c.drop_database("pymongo_test2") - await c.drop_database("pymongo_test_mike") - await c.drop_database("pymongo_test_bernie") + try: + await c.drop_database("pymongo-pooling-tests") + await c.drop_database("pymongo_test") + await c.drop_database("pymongo_test1") + await c.drop_database("pymongo_test2") + await c.drop_database("pymongo_test_mike") + await c.drop_database("pymongo_test_bernie") + except AutoReconnect: + # PYTHON-4982 + if sys.implementation.name.lower() != "pypy": + raise await c.close() print_running_clients() diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index cd41cf47f1..41ec2ac772 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3482,8 +3482,6 @@ class TestNoSessionsSupport(AsyncEncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 async def asyncSetUp(self) -> None: - if sys.implementation.name.lower() == "pypy": - return await super().asyncSetUp() start_mongocryptd(self.MONGOCRYPTD_PORT) @@ -3496,8 +3494,6 @@ async def asyncSetUp(self) -> None: self.assertNotIn("logicalSessionTimeoutMinutes", hello) async def test_implicit_session_ignored_when_unsupported(self): - if sys.implementation.name.lower() == "pypy": - raise self.skipTest("PYTHON-4982 Skipping test on pypy") self.listener.reset() with self.assertRaises(OperationFailure): await self.mongocryptd_client.db.test.find_one() @@ -3512,8 +3508,6 @@ async def test_implicit_session_ignored_when_unsupported(self): await self.mongocryptd_client.close() async def test_explicit_session_errors_when_unsupported(self): - if sys.implementation.name.lower() == "pypy": - raise self.skipTest("PYTHON-4982 Skipping test on pypy") self.listener.reset() async with self.mongocryptd_client.start_session() as s: with self.assertRaisesRegex( diff --git a/test/test_encryption.py b/test/test_encryption.py index bd09a00eb9..b4daf641f8 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3464,8 +3464,6 @@ class TestNoSessionsSupport(EncryptionIntegrationTest): MONGOCRYPTD_PORT = 27020 def setUp(self) -> None: - if sys.implementation.name.lower() == "pypy": - return super().setUp() start_mongocryptd(self.MONGOCRYPTD_PORT) @@ -3478,8 +3476,6 @@ def setUp(self) -> None: self.assertNotIn("logicalSessionTimeoutMinutes", hello) def test_implicit_session_ignored_when_unsupported(self): - if sys.implementation.name.lower() == "pypy": - raise self.skipTest("PYTHON-4982 Skipping test on pypy") self.listener.reset() with self.assertRaises(OperationFailure): self.mongocryptd_client.db.test.find_one() @@ -3494,8 +3490,6 @@ def test_implicit_session_ignored_when_unsupported(self): self.mongocryptd_client.close() def test_explicit_session_errors_when_unsupported(self): - if sys.implementation.name.lower() == "pypy": - raise self.skipTest("PYTHON-4982 Skipping test on pypy") self.listener.reset() with self.mongocryptd_client.start_session() as s: with self.assertRaisesRegex( From 78bcd996eec0f8941dddfc7dc187c31f814f6bc3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 13:36:33 -0500 Subject: [PATCH 23/37] add more retries --- test/asynchronous/test_client_bulk_write.py | 2 +- test/test_client_bulk_write.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/test_client_bulk_write.py b/test/asynchronous/test_client_bulk_write.py index 4ca0f8fd3a..e7efaad43d 100644 --- a/test/asynchronous/test_client_bulk_write.py +++ b/test/asynchronous/test_client_bulk_write.py @@ -627,7 +627,7 @@ async def asyncSetUp(self): @async_client_context.require_version_min(8, 0, 0, -24) @async_client_context.require_failCommand_fail_point - @flaky(max_runs=3, affects_cpython_linux=True) + @flaky(max_runs=3 if sys.platform == "linux" else 5, affects_cpython_linux=True) async def test_timeout_in_multi_batch_bulk_write(self): _OVERHEAD = 500 diff --git a/test/test_client_bulk_write.py b/test/test_client_bulk_write.py index b7071e6da4..e697f3da66 100644 --- a/test/test_client_bulk_write.py +++ b/test/test_client_bulk_write.py @@ -623,7 +623,7 @@ def setUp(self): @client_context.require_version_min(8, 0, 0, -24) @client_context.require_failCommand_fail_point - @flaky(max_runs=3, affects_cpython_linux=True) + @flaky(max_runs=3 if sys.platform == "linux" else 5, affects_cpython_linux=True) def test_timeout_in_multi_batch_bulk_write(self): _OVERHEAD = 500 From ebcf1748ceb7be7ca0697525dedb5bd588d64458 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 13:37:04 -0500 Subject: [PATCH 24/37] add comment --- test/asynchronous/test_client_bulk_write.py | 1 + test/test_client_bulk_write.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/asynchronous/test_client_bulk_write.py b/test/asynchronous/test_client_bulk_write.py index e7efaad43d..43de47e0a1 100644 --- a/test/asynchronous/test_client_bulk_write.py +++ b/test/asynchronous/test_client_bulk_write.py @@ -627,6 +627,7 @@ async def asyncSetUp(self): @async_client_context.require_version_min(8, 0, 0, -24) @async_client_context.require_failCommand_fail_point + # PYTHON-5290 @flaky(max_runs=3 if sys.platform == "linux" else 5, affects_cpython_linux=True) async def test_timeout_in_multi_batch_bulk_write(self): _OVERHEAD = 500 diff --git a/test/test_client_bulk_write.py b/test/test_client_bulk_write.py index e697f3da66..8bc6028464 100644 --- a/test/test_client_bulk_write.py +++ b/test/test_client_bulk_write.py @@ -623,6 +623,7 @@ def setUp(self): @client_context.require_version_min(8, 0, 0, -24) @client_context.require_failCommand_fail_point + # PYTHON-5290 @flaky(max_runs=3 if sys.platform == "linux" else 5, affects_cpython_linux=True) def test_timeout_in_multi_batch_bulk_write(self): _OVERHEAD = 500 From 11dfd7304775dfef172cc1d7b005a3a560a0c891 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 18 Jun 2025 17:32:16 -0500 Subject: [PATCH 25/37] Skip test on Windows --- test/asynchronous/test_client_bulk_write.py | 4 +++- test/test_client_bulk_write.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/test_client_bulk_write.py b/test/asynchronous/test_client_bulk_write.py index 43de47e0a1..7e153ca3c9 100644 --- a/test/asynchronous/test_client_bulk_write.py +++ b/test/asynchronous/test_client_bulk_write.py @@ -628,8 +628,10 @@ async def asyncSetUp(self): @async_client_context.require_version_min(8, 0, 0, -24) @async_client_context.require_failCommand_fail_point # PYTHON-5290 - @flaky(max_runs=3 if sys.platform == "linux" else 5, affects_cpython_linux=True) + @flaky(max_runs=3, affects_cpython_linux=True) async def test_timeout_in_multi_batch_bulk_write(self): + if sys.platform == "win32": + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") _OVERHEAD = 500 internal_client = await self.async_rs_or_single_client(timeoutMS=None) diff --git a/test/test_client_bulk_write.py b/test/test_client_bulk_write.py index 8bc6028464..32c41f6312 100644 --- a/test/test_client_bulk_write.py +++ b/test/test_client_bulk_write.py @@ -624,8 +624,10 @@ def setUp(self): @client_context.require_version_min(8, 0, 0, -24) @client_context.require_failCommand_fail_point # PYTHON-5290 - @flaky(max_runs=3 if sys.platform == "linux" else 5, affects_cpython_linux=True) + @flaky(max_runs=3, affects_cpython_linux=True) def test_timeout_in_multi_batch_bulk_write(self): + if sys.platform == "win32": + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") _OVERHEAD = 500 internal_client = self.rs_or_single_client(timeoutMS=None) From 29e55c58fc86a573fb39309765fad033c8cef1fb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 19 Jun 2025 05:33:56 -0500 Subject: [PATCH 26/37] handle PYTHON-3689 --- test/asynchronous/test_server_selection_in_window.py | 2 ++ test/test_server_selection_in_window.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/asynchronous/test_server_selection_in_window.py b/test/asynchronous/test_server_selection_in_window.py index 3fe448d4dd..66aa8de4b8 100644 --- a/test/asynchronous/test_server_selection_in_window.py +++ b/test/asynchronous/test_server_selection_in_window.py @@ -23,6 +23,7 @@ from test.asynchronous.helpers import ConcurrentRunner from test.asynchronous.utils_selection_tests import create_topology from test.asynchronous.utils_spec_runner import AsyncSpecTestCreator +from test.utils import flaky from test.utils_shared import ( CMAPListener, OvertCommandListener, @@ -137,6 +138,7 @@ async def frequencies(self, client, listener, n_finds=10): @async_client_context.require_failCommand_appName @async_client_context.require_multiple_mongoses + @flaky # PYTHON-3689 async def test_load_balancing(self): listener = OvertCommandListener() cmap_listener = CMAPListener() diff --git a/test/test_server_selection_in_window.py b/test/test_server_selection_in_window.py index 4aad34050c..4a9ce36f90 100644 --- a/test/test_server_selection_in_window.py +++ b/test/test_server_selection_in_window.py @@ -21,6 +21,7 @@ from pathlib import Path from test import IntegrationTest, client_context, unittest from test.helpers import ConcurrentRunner +from test.utils import flaky from test.utils_selection_tests import create_topology from test.utils_shared import ( CMAPListener, @@ -137,6 +138,7 @@ def frequencies(self, client, listener, n_finds=10): @client_context.require_failCommand_appName @client_context.require_multiple_mongoses + @flaky # PYTHON-3689 def test_load_balancing(self): listener = OvertCommandListener() cmap_listener = CMAPListener() From 15ae7130c3451b8d4b59a295835287a400f8303e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 19 Jun 2025 06:05:09 -0500 Subject: [PATCH 27/37] fix import --- test/asynchronous/test_server_selection_in_window.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/asynchronous/test_server_selection_in_window.py b/test/asynchronous/test_server_selection_in_window.py index 66aa8de4b8..843de4bbd8 100644 --- a/test/asynchronous/test_server_selection_in_window.py +++ b/test/asynchronous/test_server_selection_in_window.py @@ -21,9 +21,9 @@ from pathlib import Path from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest from test.asynchronous.helpers import ConcurrentRunner +from test.asynchronous.utils import flaky from test.asynchronous.utils_selection_tests import create_topology from test.asynchronous.utils_spec_runner import AsyncSpecTestCreator -from test.utils import flaky from test.utils_shared import ( CMAPListener, OvertCommandListener, From 9d8759ec964fc7f5ef951aa85f42b84dc1567760 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 19 Jun 2025 21:52:30 -0500 Subject: [PATCH 28/37] skip more tests --- test/asynchronous/test_client_bulk_write.py | 4 ++-- test/asynchronous/unified_format.py | 9 ++------- test/test_client_bulk_write.py | 4 ++-- test/unified_format.py | 9 ++------- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/test/asynchronous/test_client_bulk_write.py b/test/asynchronous/test_client_bulk_write.py index 7e153ca3c9..01eb14e475 100644 --- a/test/asynchronous/test_client_bulk_write.py +++ b/test/asynchronous/test_client_bulk_write.py @@ -630,8 +630,8 @@ async def asyncSetUp(self): # PYTHON-5290 @flaky(max_runs=3, affects_cpython_linux=True) async def test_timeout_in_multi_batch_bulk_write(self): - if sys.platform == "win32": - self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") + if sys.platform != "linux": + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows and MacOS") _OVERHEAD = 500 internal_client = await self.async_rs_or_single_client(timeoutMS=None) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 68477a2692..75c67bf1fd 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -533,16 +533,11 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: - # Skip tests that are too slow to run on a given platform. - slow_win32 = [ - "maxtimems value in the command is less than timeoutms", - "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", - ] + if sys.platform == "win32": + self.skipTest("PYTHON-3522 CSOT tests run too slow on Windows") slow_macos = [ "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" ] - if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name: - self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") if sys.platform == "darwin" and description in slow_macos: self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if async_client_context.storage_engine == "mmapv1": diff --git a/test/test_client_bulk_write.py b/test/test_client_bulk_write.py index 32c41f6312..32d8e89f29 100644 --- a/test/test_client_bulk_write.py +++ b/test/test_client_bulk_write.py @@ -626,8 +626,8 @@ def setUp(self): # PYTHON-5290 @flaky(max_runs=3, affects_cpython_linux=True) def test_timeout_in_multi_batch_bulk_write(self): - if sys.platform == "win32": - self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") + if sys.platform != "linux": + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows and MacOS") _OVERHEAD = 500 internal_client = self.rs_or_single_client(timeoutMS=None) diff --git a/test/unified_format.py b/test/unified_format.py index a785a86e1c..aae6b7083e 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -532,16 +532,11 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: - # Skip tests that are too slow to run on a given platform. - slow_win32 = [ - "maxtimems value in the command is less than timeoutms", - "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", - ] + if sys.platform == "win32": + self.skipTest("PYTHON-3522 CSOT tests run too slow on Windows") slow_macos = [ "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" ] - if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name: - self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") if sys.platform == "darwin" and description in slow_macos: self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if client_context.storage_engine == "mmapv1": From 90656396613f2fdbd5d994254f684afec7b0ee83 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 20 Jun 2025 06:04:07 -0500 Subject: [PATCH 29/37] more cleanup --- test/asynchronous/test_retryable_writes.py | 3 ++- test/asynchronous/unified_format.py | 13 +++++++++---- test/test_retryable_writes.py | 3 ++- test/unified_format.py | 13 +++++++++---- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/test/asynchronous/test_retryable_writes.py b/test/asynchronous/test_retryable_writes.py index b399fa50e4..a0e409f1db 100644 --- a/test/asynchronous/test_retryable_writes.py +++ b/test/asynchronous/test_retryable_writes.py @@ -20,7 +20,7 @@ import pprint import sys import threading -from test.asynchronous.utils import async_set_fail_point +from test.asynchronous.utils import async_set_fail_point, flaky sys.path[0:0] = [""] @@ -497,6 +497,7 @@ class TestPoolPausedError(AsyncIntegrationTest): @async_client_context.require_failCommand_blockConnection @async_client_context.require_retryable_writes @client_knobs(heartbeat_frequency=0.05, min_heartbeat_interval=0.05) + @flaky # PYTHON-5291 async def test_pool_paused_error_is_retryable(self): cmap_listener = CMAPListener() cmd_listener = OvertCommandListener() diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 75c67bf1fd..2ea6bb3aa4 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -533,11 +533,16 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: - if sys.platform == "win32": - self.skipTest("PYTHON-3522 CSOT tests run too slow on Windows") + # Skip tests that are too slow to run on a given platform. + slow_win32 = [ + "maxtimems value in the command is less than timeoutms", + "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", + ] slow_macos = [ "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" ] + if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name: + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") if sys.platform == "darwin" and description in slow_macos: self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if async_client_context.storage_engine == "mmapv1": @@ -1378,7 +1383,7 @@ async def run_scenario(self, spec, uri=None): # Handle flaky tests. flaky_tests = [ # PYTHON-5170 - ".*test_discovery_and_monitoring.TestUnifiedInterruptInUsePoolClear.*", + ".*test_discovery_and_monitoring.*", # PYTHON-5174 ".*Driver_extends_timeout_while_streaming", # PYTHON-5315 @@ -1394,7 +1399,7 @@ async def run_scenario(self, spec, uri=None): if re.match(flaky_test, self.id()) is not None: func_name = self.id() options = dict(reset_func=self.asyncSetUp, func_name=func_name) - if "csot" in func_name: + if "csot" in func_name.lower(): options["max_runs"] = 3 options["affects_cpython_linux"] = True decorator = flaky(**options) diff --git a/test/test_retryable_writes.py b/test/test_retryable_writes.py index ad5b0671e7..eb2b5d0ecd 100644 --- a/test/test_retryable_writes.py +++ b/test/test_retryable_writes.py @@ -20,7 +20,7 @@ import pprint import sys import threading -from test.utils import set_fail_point +from test.utils import flaky, set_fail_point sys.path[0:0] = [""] @@ -495,6 +495,7 @@ class TestPoolPausedError(IntegrationTest): @client_context.require_failCommand_blockConnection @client_context.require_retryable_writes @client_knobs(heartbeat_frequency=0.05, min_heartbeat_interval=0.05) + @flaky # PYTHON-5291 def test_pool_paused_error_is_retryable(self): cmap_listener = CMAPListener() cmd_listener = OvertCommandListener() diff --git a/test/unified_format.py b/test/unified_format.py index aae6b7083e..ab860c33e6 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -532,11 +532,16 @@ def maybe_skip_test(self, spec): class_name = self.__class__.__name__.lower() description = spec["description"].lower() if "csot" in class_name: - if sys.platform == "win32": - self.skipTest("PYTHON-3522 CSOT tests run too slow on Windows") + # Skip tests that are too slow to run on a given platform. + slow_win32 = [ + "maxtimems value in the command is less than timeoutms", + "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", + ] slow_macos = [ "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" ] + if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name: + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") if sys.platform == "darwin" and description in slow_macos: self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if client_context.storage_engine == "mmapv1": @@ -1365,7 +1370,7 @@ def run_scenario(self, spec, uri=None): # Handle flaky tests. flaky_tests = [ # PYTHON-5170 - ".*test_discovery_and_monitoring.TestUnifiedInterruptInUsePoolClear.*", + ".*test_discovery_and_monitoring.*", # PYTHON-5174 ".*Driver_extends_timeout_while_streaming", # PYTHON-5315 @@ -1381,7 +1386,7 @@ def run_scenario(self, spec, uri=None): if re.match(flaky_test, self.id()) is not None: func_name = self.id() options = dict(reset_func=self.setUp, func_name=func_name) - if "csot" in func_name: + if "csot" in func_name.lower(): options["max_runs"] = 3 options["affects_cpython_linux"] = True decorator = flaky(**options) From b40dc2cb2eafdcadfd7a36b8136752a3842084b4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 20 Jun 2025 08:40:20 -0500 Subject: [PATCH 30/37] skip more tests --- CONTRIBUTING.md | 7 +++++++ test/asynchronous/unified_format.py | 26 +++++++++++++++++--------- test/asynchronous/utils.py | 10 ++++++++++ test/unified_format.py | 26 +++++++++++++++++--------- test/utils.py | 10 ++++++++++ 5 files changed, 61 insertions(+), 18 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5a2bf4d913..55e94d810a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -404,6 +404,13 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. +## Handling flaky tests + +We have a custom ``flaky`` decorator in [test/asynchronous/utils.py](test/asynchronous/utils.py) that can be used for +tests that are ``flaky``. By default the decorator only applies when not running on CPython on Linux, since other +runtimes tend to have more variation. When using the ``flaky`` decorator, open a corresponding ticket and +a comment with the ticket number. + ## Specification Tests The MongoDB [specifications repository](https://github.com/mongodb/specifications) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 2ea6bb3aa4..fc41d7edf1 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -534,17 +534,25 @@ def maybe_skip_test(self, spec): description = spec["description"].lower() if "csot" in class_name: # Skip tests that are too slow to run on a given platform. - slow_win32 = [ - "maxtimems value in the command is less than timeoutms", - "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", - ] slow_macos = [ - "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" + "operation fails after two consecutive socket timeouts.*", + "operation succeeds after one socket timeout.*", + "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset", + ] + slow_win32 = [ + *slow_macos, + "maxTimeMS value in the command is less than timeoutMS", ] - if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name: - self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") - if sys.platform == "darwin" and description in slow_macos: - self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") + if sys.platform == "win32" and "gridfs" in class_name: + self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows") + if sys.platform == "win32": + for pat in slow_win32: + if re.match(pat, description.lower()): + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") + if sys.platform == "darwin": + for pat in slow_macos: + if re.match(pat, description.lower()): + self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if async_client_context.storage_engine == "mmapv1": self.skipTest( "MMAPv1 does not support retryable writes which is required for CSOT tests" diff --git a/test/asynchronous/utils.py b/test/asynchronous/utils.py index 39cf3a6bb6..96eeeb9764 100644 --- a/test/asynchronous/utils.py +++ b/test/asynchronous/utils.py @@ -168,6 +168,16 @@ def flaky( func_name=None, reset_func=None, ): + """Decorate a test as flaky. + + :param max_runs: the maximum number of runs before raising an error + :param min_passes: the minimum number of passing runs + :param delay: the delay in seconds between retries + :param affects_cpython_links: whether the test is flaky on CPython on Linux + :param func_name: the name of the function, used for the rety message + :param reset_func: a function to call before retrying + + """ is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" disable_flaky = "DISABLE_FLAKY" in os.environ if disable_flaky or (is_cpython_linux and not affects_cpython_linux): diff --git a/test/unified_format.py b/test/unified_format.py index ab860c33e6..f8abb7b9d9 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -533,17 +533,25 @@ def maybe_skip_test(self, spec): description = spec["description"].lower() if "csot" in class_name: # Skip tests that are too slow to run on a given platform. - slow_win32 = [ - "maxtimems value in the command is less than timeoutms", - "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset", - ] slow_macos = [ - "non-tailable cursor lifetime remaining timeoutms applied to getmore if timeoutmode is unset" + "operation fails after two consecutive socket timeouts.*", + "operation succeeds after one socket timeout.*", + "Non-tailable cursor lifetime remaining timeoutMS applied to getMore if timeoutMode is unset", + ] + slow_win32 = [ + *slow_macos, + "maxTimeMS value in the command is less than timeoutMS", ] - if sys.platform == "win32" and description in slow_win32 or "gridfs" in class_name: - self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") - if sys.platform == "darwin" and description in slow_macos: - self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") + if sys.platform == "win32" and "gridfs" in class_name: + self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows") + if sys.platform == "win32": + for pat in slow_win32: + if re.match(pat, description.lower()): + self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") + if sys.platform == "darwin": + for pat in slow_macos: + if re.match(pat, description.lower()): + self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if client_context.storage_engine == "mmapv1": self.skipTest( "MMAPv1 does not support retryable writes which is required for CSOT tests" diff --git a/test/utils.py b/test/utils.py index b75f854f7b..39fd4ca18f 100644 --- a/test/utils.py +++ b/test/utils.py @@ -166,6 +166,16 @@ def flaky( func_name=None, reset_func=None, ): + """Decorate a test as flaky. + + :param max_runs: the maximum number of runs before raising an error + :param min_passes: the minimum number of passing runs + :param delay: the delay in seconds between retries + :param affects_cpython_links: whether the test is flaky on CPython on Linux + :param func_name: the name of the function, used for the rety message + :param reset_func: a function to call before retrying + + """ is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" disable_flaky = "DISABLE_FLAKY" in os.environ if disable_flaky or (is_cpython_linux and not affects_cpython_linux): From 7e29849623fcc6a17aeb176e37306e7f3dbaa447 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 20 Jun 2025 08:52:16 -0500 Subject: [PATCH 31/37] fix skip --- test/asynchronous/unified_format.py | 4 ++-- test/unified_format.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index fc41d7edf1..a76e480317 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -547,11 +547,11 @@ def maybe_skip_test(self, spec): self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows") if sys.platform == "win32": for pat in slow_win32: - if re.match(pat, description.lower()): + if re.match(pat.lower(), description): self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") if sys.platform == "darwin": for pat in slow_macos: - if re.match(pat, description.lower()): + if re.match(pat.lower(), description): self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if async_client_context.storage_engine == "mmapv1": self.skipTest( diff --git a/test/unified_format.py b/test/unified_format.py index f8abb7b9d9..f156cb023a 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -546,11 +546,11 @@ def maybe_skip_test(self, spec): self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows") if sys.platform == "win32": for pat in slow_win32: - if re.match(pat, description.lower()): + if re.match(pat.lower(), description): self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows") if sys.platform == "darwin": for pat in slow_macos: - if re.match(pat, description.lower()): + if re.match(pat.lower(), description): self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") if client_context.storage_engine == "mmapv1": self.skipTest( From e7da55be74a9a7ad2ff3f337209d834820f029ff Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 20 Jun 2025 11:02:46 -0500 Subject: [PATCH 32/37] cleanup --- test/asynchronous/test_csot.py | 4 ++-- test/asynchronous/test_cursor.py | 4 ++-- test/test_csot.py | 4 ++-- test/test_cursor.py | 4 ++-- test/test_topology.py | 3 +-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/test/asynchronous/test_csot.py b/test/asynchronous/test_csot.py index 6e4a82f5b2..3f32631a14 100644 --- a/test/asynchronous/test_csot.py +++ b/test/asynchronous/test_csot.py @@ -44,7 +44,7 @@ class TestCSOT(AsyncIntegrationTest): RUN_ON_LOAD_BALANCER = True - @flaky + @flaky # PYTHON-3522 async def test_timeout_nested(self): coll = self.db.coll self.assertEqual(_csot.get_timeout(), None) @@ -82,7 +82,7 @@ async def test_timeout_nested(self): self.assertEqual(_csot.get_rtt(), 0.0) @async_client_context.require_change_streams - @flaky + @flaky # PYTHON-3522 async def test_change_stream_can_resume_after_timeouts(self): coll = self.db.test await coll.insert_one({}) diff --git a/test/asynchronous/test_cursor.py b/test/asynchronous/test_cursor.py index bb387d0b21..dfb0fd99d5 100644 --- a/test/asynchronous/test_cursor.py +++ b/test/asynchronous/test_cursor.py @@ -1416,7 +1416,7 @@ async def test_to_list_length(self): docs = await c.to_list(3) self.assertEqual(len(docs), 2) - @flaky + @flaky # PYTHON-3522 async def test_to_list_csot_applied(self): client = await self.async_single_client(timeoutMS=500, w=1) coll = client.pymongo.test @@ -1458,7 +1458,7 @@ async def test_command_cursor_to_list_length(self): self.assertEqual(len(await result.to_list(1)), 1) @async_client_context.require_failCommand_blockConnection - @flaky + @flaky # PYTHON-3522 async def test_command_cursor_to_list_csot_applied(self): client = await self.async_single_client(timeoutMS=500, w=1) coll = client.pymongo.test diff --git a/test/test_csot.py b/test/test_csot.py index b47f113a35..fa87d58a07 100644 --- a/test/test_csot.py +++ b/test/test_csot.py @@ -44,7 +44,7 @@ class TestCSOT(IntegrationTest): RUN_ON_LOAD_BALANCER = True - @flaky + @flaky # PYTHON-3522 def test_timeout_nested(self): coll = self.db.coll self.assertEqual(_csot.get_timeout(), None) @@ -82,7 +82,7 @@ def test_timeout_nested(self): self.assertEqual(_csot.get_rtt(), 0.0) @client_context.require_change_streams - @flaky + @flaky # PYTHON-3522 def test_change_stream_can_resume_after_timeouts(self): coll = self.db.test coll.insert_one({}) diff --git a/test/test_cursor.py b/test/test_cursor.py index 0de348e96c..174c2238c3 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -1407,7 +1407,7 @@ def test_to_list_length(self): docs = c.to_list(3) self.assertEqual(len(docs), 2) - @flaky + @flaky # PYTHON-3522 def test_to_list_csot_applied(self): client = self.single_client(timeoutMS=500, w=1) coll = client.pymongo.test @@ -1449,7 +1449,7 @@ def test_command_cursor_to_list_length(self): self.assertEqual(len(result.to_list(1)), 1) @client_context.require_failCommand_blockConnection - @flaky + @flaky # PYTHON-3522 def test_command_cursor_to_list_csot_applied(self): client = self.single_client(timeoutMS=500, w=1) coll = client.pymongo.test diff --git a/test/test_topology.py b/test/test_topology.py index 2cbe95eb45..bd1c127c4b 100644 --- a/test/test_topology.py +++ b/test/test_topology.py @@ -750,8 +750,7 @@ def get_primary(): class TestTopologyErrors(TopologyTest): # Errors when calling hello. - # PYTHON-5366 - @flaky + @flaky # PYTHON-5366 def test_pool_reset(self): # hello succeeds at first, then always raises socket error. hello_count = [0] From 5430ee25694c4909da773db7c64f4f95d6d2407f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 20 Jun 2025 11:13:11 -0500 Subject: [PATCH 33/37] add a required reason parameter --- CONTRIBUTING.md | 8 +++---- test/asynchronous/test_client_bulk_write.py | 3 +-- test/asynchronous/test_csot.py | 4 ++-- test/asynchronous/test_cursor.py | 4 ++-- test/asynchronous/test_encryption.py | 2 +- test/asynchronous/test_retryable_writes.py | 2 +- .../test_server_selection_in_window.py | 2 +- test/asynchronous/test_srv_polling.py | 2 +- test/asynchronous/unified_format.py | 22 +++++++------------ test/asynchronous/utils.py | 12 +++++----- test/test_client_bulk_write.py | 3 +-- test/test_csot.py | 4 ++-- test/test_cursor.py | 4 ++-- test/test_encryption.py | 2 +- test/test_retryable_writes.py | 2 +- test/test_server_selection_in_window.py | 2 +- test/test_srv_polling.py | 2 +- test/unified_format.py | 22 +++++++------------ test/utils.py | 12 +++++----- 19 files changed, 48 insertions(+), 66 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 55e94d810a..34dd269690 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -406,10 +406,10 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. ## Handling flaky tests -We have a custom ``flaky`` decorator in [test/asynchronous/utils.py](test/asynchronous/utils.py) that can be used for -tests that are ``flaky``. By default the decorator only applies when not running on CPython on Linux, since other -runtimes tend to have more variation. When using the ``flaky`` decorator, open a corresponding ticket and -a comment with the ticket number. +We have a custom `flaky` decorator in [test/asynchronous/utils.py](test/asynchronous/utils.py) that can be used for +tests that are `flaky`. By default the decorator only applies when not running on CPython on Linux, since other +runtimes tend to have more variation. When using the `flaky` decorator, open a corresponding ticket and +a use the ticket number as the "reason" parameter to the decorator, e.g. `@flaky(reason="PYTHON-1234")`. ## Specification Tests diff --git a/test/asynchronous/test_client_bulk_write.py b/test/asynchronous/test_client_bulk_write.py index 01eb14e475..7f85df5467 100644 --- a/test/asynchronous/test_client_bulk_write.py +++ b/test/asynchronous/test_client_bulk_write.py @@ -627,8 +627,7 @@ async def asyncSetUp(self): @async_client_context.require_version_min(8, 0, 0, -24) @async_client_context.require_failCommand_fail_point - # PYTHON-5290 - @flaky(max_runs=3, affects_cpython_linux=True) + @flaky(reason="PYTHON-5290", max_runs=3, affects_cpython_linux=True) async def test_timeout_in_multi_batch_bulk_write(self): if sys.platform != "linux": self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows and MacOS") diff --git a/test/asynchronous/test_csot.py b/test/asynchronous/test_csot.py index 3f32631a14..a978d1ccc0 100644 --- a/test/asynchronous/test_csot.py +++ b/test/asynchronous/test_csot.py @@ -44,7 +44,7 @@ class TestCSOT(AsyncIntegrationTest): RUN_ON_LOAD_BALANCER = True - @flaky # PYTHON-3522 + @flaky(reason="PYTHON-3522") async def test_timeout_nested(self): coll = self.db.coll self.assertEqual(_csot.get_timeout(), None) @@ -82,7 +82,7 @@ async def test_timeout_nested(self): self.assertEqual(_csot.get_rtt(), 0.0) @async_client_context.require_change_streams - @flaky # PYTHON-3522 + @flaky(reason="PYTHON-3522") async def test_change_stream_can_resume_after_timeouts(self): coll = self.db.test await coll.insert_one({}) diff --git a/test/asynchronous/test_cursor.py b/test/asynchronous/test_cursor.py index dfb0fd99d5..fb174be1fc 100644 --- a/test/asynchronous/test_cursor.py +++ b/test/asynchronous/test_cursor.py @@ -1416,7 +1416,7 @@ async def test_to_list_length(self): docs = await c.to_list(3) self.assertEqual(len(docs), 2) - @flaky # PYTHON-3522 + @flaky(reason="PYTHON-3522") async def test_to_list_csot_applied(self): client = await self.async_single_client(timeoutMS=500, w=1) coll = client.pymongo.test @@ -1458,7 +1458,7 @@ async def test_command_cursor_to_list_length(self): self.assertEqual(len(await result.to_list(1)), 1) @async_client_context.require_failCommand_blockConnection - @flaky # PYTHON-3522 + @flaky(reason="PYTHON-3522") async def test_command_cursor_to_list_csot_applied(self): client = await self.async_single_client(timeoutMS=500, w=1) coll = client.pymongo.test diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 41ec2ac772..f36176318a 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -3262,7 +3262,7 @@ async def test_kms_retry(self): class TestAutomaticDecryptionKeys(AsyncEncryptionIntegrationTest): @async_client_context.require_no_standalone @async_client_context.require_version_min(7, 0, -1) - @flaky # PYTHON-4982 + @flaky(reason="PYTHON-4982") async def asyncSetUp(self): await super().asyncSetUp() self.key1_document = json_data("etc", "data", "keys", "key1-document.json") diff --git a/test/asynchronous/test_retryable_writes.py b/test/asynchronous/test_retryable_writes.py index a0e409f1db..3d5c6f0520 100644 --- a/test/asynchronous/test_retryable_writes.py +++ b/test/asynchronous/test_retryable_writes.py @@ -497,7 +497,7 @@ class TestPoolPausedError(AsyncIntegrationTest): @async_client_context.require_failCommand_blockConnection @async_client_context.require_retryable_writes @client_knobs(heartbeat_frequency=0.05, min_heartbeat_interval=0.05) - @flaky # PYTHON-5291 + @flaky(reason="PYTHON-5291") async def test_pool_paused_error_is_retryable(self): cmap_listener = CMAPListener() cmd_listener = OvertCommandListener() diff --git a/test/asynchronous/test_server_selection_in_window.py b/test/asynchronous/test_server_selection_in_window.py index 843de4bbd8..dd0ff734f7 100644 --- a/test/asynchronous/test_server_selection_in_window.py +++ b/test/asynchronous/test_server_selection_in_window.py @@ -138,7 +138,7 @@ async def frequencies(self, client, listener, n_finds=10): @async_client_context.require_failCommand_appName @async_client_context.require_multiple_mongoses - @flaky # PYTHON-3689 + @flaky(reason="PYTHON-3689") async def test_load_balancing(self): listener = OvertCommandListener() cmap_listener = CMAPListener() diff --git a/test/asynchronous/test_srv_polling.py b/test/asynchronous/test_srv_polling.py index 35a8863297..bc167432c3 100644 --- a/test/asynchronous/test_srv_polling.py +++ b/test/asynchronous/test_srv_polling.py @@ -255,7 +255,7 @@ def final_callback(): # Nodelist should reflect new valid DNS resolver response. await self.assert_nodelist_change(response_final, client) - @flaky # PYTHON-5315 + @flaky(reason="PYTHON-5315") async def test_recover_from_initially_empty_seedlist(self): def empty_seedlist(): return [] diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index a76e480317..600e6e0b73 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -1390,23 +1390,17 @@ async def run_scenario(self, spec, uri=None): # Handle flaky tests. flaky_tests = [ - # PYTHON-5170 - ".*test_discovery_and_monitoring.*", - # PYTHON-5174 - ".*Driver_extends_timeout_while_streaming", - # PYTHON-5315 - ".*TestSrvPolling.test_recover_from_initially_.*", - # PYTHON-4987 - ".*UnknownTransactionCommitResult_labels_to_connection_errors", - # PYTHON-3689 - ".*TestProse.test_load_balancing", - # PYTHON-3522 - ".*csot.*", + ("PYTHON-5170", ".*test_discovery_and_monitoring.*"), + ("PYTHON-5174", ".*Driver_extends_timeout_while_streaming"), + ("PYTHON-5315", ".*TestSrvPolling.test_recover_from_initially_.*"), + ("PYTHON-4987", ".*UnknownTransactionCommitResult_labels_to_connection_errors"), + ("PYTHON-3689", ".*TestProse.test_load_balancing"), + ("PYTHON-3522", ".*csot.*"), ] - for flaky_test in flaky_tests: + for reason, flaky_test in flaky_tests: if re.match(flaky_test, self.id()) is not None: func_name = self.id() - options = dict(reset_func=self.asyncSetUp, func_name=func_name) + options = dict(reason=reason, reset_func=self.asyncSetUp, func_name=func_name) if "csot" in func_name.lower(): options["max_runs"] = 3 options["affects_cpython_linux"] = True diff --git a/test/asynchronous/utils.py b/test/asynchronous/utils.py index 96eeeb9764..982564a0a5 100644 --- a/test/asynchronous/utils.py +++ b/test/asynchronous/utils.py @@ -159,8 +159,8 @@ async def async_joinall(tasks): def flaky( - func=None, *, + reason=None, max_runs=2, min_passes=1, delay=1, @@ -170,6 +170,7 @@ def flaky( ): """Decorate a test as flaky. + :param reason: the reason why the test is flaky :param max_runs: the maximum number of runs before raising an error :param min_passes: the minimum number of passing runs :param delay: the delay in seconds between retries @@ -178,6 +179,8 @@ def flaky( :param reset_func: a function to call before retrying """ + if reason is None: + raise ValueError("flaky requires a reason input") is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" disable_flaky = "DISABLE_FLAKY" in os.environ if disable_flaky or (is_cpython_linux and not affects_cpython_linux): @@ -198,7 +201,7 @@ async def wrapper(*args, **kwargs): if i == max_runs - 1: raise e print( - f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with:\n" + f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with ({reason})):\n" f"{traceback.format_exc()}", file=sys.stderr, ) @@ -208,11 +211,6 @@ async def wrapper(*args, **kwargs): return wrapper - # If `func` is callable, the decorator was used without arguments (`@flaky`). - if callable(func): - return decorator(func) - - # Otherwise, return the decorator function, allowing arguments (`@flaky(max_runs=...)`). return decorator diff --git a/test/test_client_bulk_write.py b/test/test_client_bulk_write.py index 32d8e89f29..4cc4ab4122 100644 --- a/test/test_client_bulk_write.py +++ b/test/test_client_bulk_write.py @@ -623,8 +623,7 @@ def setUp(self): @client_context.require_version_min(8, 0, 0, -24) @client_context.require_failCommand_fail_point - # PYTHON-5290 - @flaky(max_runs=3, affects_cpython_linux=True) + @flaky(reason="PYTHON-5290", max_runs=3, affects_cpython_linux=True) def test_timeout_in_multi_batch_bulk_write(self): if sys.platform != "linux": self.skipTest("PYTHON-3522 CSOT test runs too slow on Windows and MacOS") diff --git a/test/test_csot.py b/test/test_csot.py index fa87d58a07..981af1ed03 100644 --- a/test/test_csot.py +++ b/test/test_csot.py @@ -44,7 +44,7 @@ class TestCSOT(IntegrationTest): RUN_ON_LOAD_BALANCER = True - @flaky # PYTHON-3522 + @flaky(reason="PYTHON-3522") def test_timeout_nested(self): coll = self.db.coll self.assertEqual(_csot.get_timeout(), None) @@ -82,7 +82,7 @@ def test_timeout_nested(self): self.assertEqual(_csot.get_rtt(), 0.0) @client_context.require_change_streams - @flaky # PYTHON-3522 + @flaky(reason="PYTHON-3522") def test_change_stream_can_resume_after_timeouts(self): coll = self.db.test coll.insert_one({}) diff --git a/test/test_cursor.py b/test/test_cursor.py index 174c2238c3..8407429e57 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -1407,7 +1407,7 @@ def test_to_list_length(self): docs = c.to_list(3) self.assertEqual(len(docs), 2) - @flaky # PYTHON-3522 + @flaky(reason="PYTHON-3522") def test_to_list_csot_applied(self): client = self.single_client(timeoutMS=500, w=1) coll = client.pymongo.test @@ -1449,7 +1449,7 @@ def test_command_cursor_to_list_length(self): self.assertEqual(len(result.to_list(1)), 1) @client_context.require_failCommand_blockConnection - @flaky # PYTHON-3522 + @flaky(reason="PYTHON-3522") def test_command_cursor_to_list_csot_applied(self): client = self.single_client(timeoutMS=500, w=1) coll = client.pymongo.test diff --git a/test/test_encryption.py b/test/test_encryption.py index b4daf641f8..fe962cbcc6 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -3244,7 +3244,7 @@ def test_kms_retry(self): class TestAutomaticDecryptionKeys(EncryptionIntegrationTest): @client_context.require_no_standalone @client_context.require_version_min(7, 0, -1) - @flaky # PYTHON-4982 + @flaky(reason="PYTHON-4982") def setUp(self): super().setUp() self.key1_document = json_data("etc", "data", "keys", "key1-document.json") diff --git a/test/test_retryable_writes.py b/test/test_retryable_writes.py index eb2b5d0ecd..71f8f5e6e4 100644 --- a/test/test_retryable_writes.py +++ b/test/test_retryable_writes.py @@ -495,7 +495,7 @@ class TestPoolPausedError(IntegrationTest): @client_context.require_failCommand_blockConnection @client_context.require_retryable_writes @client_knobs(heartbeat_frequency=0.05, min_heartbeat_interval=0.05) - @flaky # PYTHON-5291 + @flaky(reason="PYTHON-5291") def test_pool_paused_error_is_retryable(self): cmap_listener = CMAPListener() cmd_listener = OvertCommandListener() diff --git a/test/test_server_selection_in_window.py b/test/test_server_selection_in_window.py index 4a9ce36f90..fcf2cce0e0 100644 --- a/test/test_server_selection_in_window.py +++ b/test/test_server_selection_in_window.py @@ -138,7 +138,7 @@ def frequencies(self, client, listener, n_finds=10): @client_context.require_failCommand_appName @client_context.require_multiple_mongoses - @flaky # PYTHON-3689 + @flaky(reason="PYTHON-3689") def test_load_balancing(self): listener = OvertCommandListener() cmap_listener = CMAPListener() diff --git a/test/test_srv_polling.py b/test/test_srv_polling.py index fd98cf7b41..3ab1a514c1 100644 --- a/test/test_srv_polling.py +++ b/test/test_srv_polling.py @@ -255,7 +255,7 @@ def final_callback(): # Nodelist should reflect new valid DNS resolver response. self.assert_nodelist_change(response_final, client) - @flaky # PYTHON-5315 + @flaky(reason="PYTHON-5315") def test_recover_from_initially_empty_seedlist(self): def empty_seedlist(): return [] diff --git a/test/unified_format.py b/test/unified_format.py index f156cb023a..e83a60bdd8 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -1377,23 +1377,17 @@ def run_scenario(self, spec, uri=None): # Handle flaky tests. flaky_tests = [ - # PYTHON-5170 - ".*test_discovery_and_monitoring.*", - # PYTHON-5174 - ".*Driver_extends_timeout_while_streaming", - # PYTHON-5315 - ".*TestSrvPolling.test_recover_from_initially_.*", - # PYTHON-4987 - ".*UnknownTransactionCommitResult_labels_to_connection_errors", - # PYTHON-3689 - ".*TestProse.test_load_balancing", - # PYTHON-3522 - ".*csot.*", + ("PYTHON-5170", ".*test_discovery_and_monitoring.*"), + ("PYTHON-5174", ".*Driver_extends_timeout_while_streaming"), + ("PYTHON-5315", ".*TestSrvPolling.test_recover_from_initially_.*"), + ("PYTHON-4987", ".*UnknownTransactionCommitResult_labels_to_connection_errors"), + ("PYTHON-3689", ".*TestProse.test_load_balancing"), + ("PYTHON-3522", ".*csot.*"), ] - for flaky_test in flaky_tests: + for reason, flaky_test in flaky_tests: if re.match(flaky_test, self.id()) is not None: func_name = self.id() - options = dict(reset_func=self.setUp, func_name=func_name) + options = dict(reason=reason, reset_func=self.setUp, func_name=func_name) if "csot" in func_name.lower(): options["max_runs"] = 3 options["affects_cpython_linux"] = True diff --git a/test/utils.py b/test/utils.py index 39fd4ca18f..855f453c8a 100644 --- a/test/utils.py +++ b/test/utils.py @@ -157,8 +157,8 @@ def joinall(tasks): def flaky( - func=None, *, + reason=None, max_runs=2, min_passes=1, delay=1, @@ -168,6 +168,7 @@ def flaky( ): """Decorate a test as flaky. + :param reason: the reason why the test is flaky :param max_runs: the maximum number of runs before raising an error :param min_passes: the minimum number of passing runs :param delay: the delay in seconds between retries @@ -176,6 +177,8 @@ def flaky( :param reset_func: a function to call before retrying """ + if reason is None: + raise ValueError("flaky requires a reason input") is_cpython_linux = sys.platform == "linux" and sys.implementation.name == "cpython" disable_flaky = "DISABLE_FLAKY" in os.environ if disable_flaky or (is_cpython_linux and not affects_cpython_linux): @@ -196,7 +199,7 @@ def wrapper(*args, **kwargs): if i == max_runs - 1: raise e print( - f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with:\n" + f"Retrying after attempt {i+1} of {func_name or target_func.__name__} failed with ({reason})):\n" f"{traceback.format_exc()}", file=sys.stderr, ) @@ -206,11 +209,6 @@ def wrapper(*args, **kwargs): return wrapper - # If `func` is callable, the decorator was used without arguments (`@flaky`). - if callable(func): - return decorator(func) - - # Otherwise, return the decorator function, allowing arguments (`@flaky(max_runs=...)`). return decorator From a1965e516b95a968bef0fb1742273181384fef99 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 20 Jun 2025 13:13:41 -0500 Subject: [PATCH 34/37] fix test --- test/test_topology.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_topology.py b/test/test_topology.py index bd1c127c4b..37555cb986 100644 --- a/test/test_topology.py +++ b/test/test_topology.py @@ -750,7 +750,7 @@ def get_primary(): class TestTopologyErrors(TopologyTest): # Errors when calling hello. - @flaky # PYTHON-5366 + @flaky(reason="PYTHON-5366") def test_pool_reset(self): # hello succeeds at first, then always raises socket error. hello_count = [0] From 36d1aa759d496aaff0f95ee429997a0dac0d6d20 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 20 Jun 2025 20:32:32 -0500 Subject: [PATCH 35/37] fix skip condition --- test/asynchronous/unified_format.py | 2 +- test/unified_format.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 207b060bb4..61a294dece 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -1371,7 +1371,7 @@ async def run_scenario(self, spec, uri=None): ("PYTHON-3522", ".*csot.*"), ] for reason, flaky_test in flaky_tests: - if re.match(flaky_test, self.id()) is not None: + if re.match(flaky_test.lower(), self.id().lower()) is not None: func_name = self.id() options = dict(reason=reason, reset_func=self.asyncSetUp, func_name=func_name) if "csot" in func_name.lower(): diff --git a/test/unified_format.py b/test/unified_format.py index b8988ac382..1b2275c5d2 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -1358,7 +1358,7 @@ def run_scenario(self, spec, uri=None): ("PYTHON-3522", ".*csot.*"), ] for reason, flaky_test in flaky_tests: - if re.match(flaky_test, self.id()) is not None: + if re.match(flaky_test.lower(), self.id().lower()) is not None: func_name = self.id() options = dict(reason=reason, reset_func=self.setUp, func_name=func_name) if "csot" in func_name.lower(): From 3fa57c2e5399dcfff359f48fcdc5c3bf58a04ea2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 30 Jun 2025 17:37:45 -0500 Subject: [PATCH 36/37] add another win skip --- test/asynchronous/unified_format.py | 1 + test/unified_format.py | 1 + 2 files changed, 2 insertions(+) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 66b005c311..5c4affdb0e 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -531,6 +531,7 @@ def maybe_skip_test(self, spec): slow_win32 = [ *slow_macos, "maxTimeMS value in the command is less than timeoutMS", + "timeoutMS applies to whole operation.*", ] if sys.platform == "win32" and "gridfs" in class_name: self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows") diff --git a/test/unified_format.py b/test/unified_format.py index 8251d195f5..24339d7ee8 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -530,6 +530,7 @@ def maybe_skip_test(self, spec): slow_win32 = [ *slow_macos, "maxTimeMS value in the command is less than timeoutMS", + "timeoutMS applies to whole operation.*", ] if sys.platform == "win32" and "gridfs" in class_name: self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows") From 771bde9242ce86cfe73abd4aabd292c7a7a49fa5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 30 Jun 2025 18:44:46 -0500 Subject: [PATCH 37/37] add pypy skips --- test/asynchronous/unified_format.py | 7 +++++++ test/unified_format.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 5c4affdb0e..0a1f142c7d 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -533,6 +533,9 @@ def maybe_skip_test(self, spec): "maxTimeMS value in the command is less than timeoutMS", "timeoutMS applies to whole operation.*", ] + slow_pypy = [ + "timeoutMS applies to whole operation.*", + ] if sys.platform == "win32" and "gridfs" in class_name: self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows") if sys.platform == "win32": @@ -543,6 +546,10 @@ def maybe_skip_test(self, spec): for pat in slow_macos: if re.match(pat.lower(), description): self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") + if sys.implementation.name.lower() == "pypy": + for pat in slow_pypy: + if re.match(pat.lower(), description): + self.skipTest("PYTHON-3522 CSOT test runs too slow on PyPy") if "change" in description or "change" in class_name: self.skipTest("CSOT not implemented for watch()") if "cursors" in class_name: diff --git a/test/unified_format.py b/test/unified_format.py index 24339d7ee8..477842ad90 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -532,6 +532,9 @@ def maybe_skip_test(self, spec): "maxTimeMS value in the command is less than timeoutMS", "timeoutMS applies to whole operation.*", ] + slow_pypy = [ + "timeoutMS applies to whole operation.*", + ] if sys.platform == "win32" and "gridfs" in class_name: self.skipTest("PYTHON-3522 CSOT GridFS test runs too slow on Windows") if sys.platform == "win32": @@ -542,6 +545,10 @@ def maybe_skip_test(self, spec): for pat in slow_macos: if re.match(pat.lower(), description): self.skipTest("PYTHON-3522 CSOT test runs too slow on MacOS") + if sys.implementation.name.lower() == "pypy": + for pat in slow_pypy: + if re.match(pat.lower(), description): + self.skipTest("PYTHON-3522 CSOT test runs too slow on PyPy") if "change" in description or "change" in class_name: self.skipTest("CSOT not implemented for watch()") if "cursors" in class_name: