From 5d6b600fbc4086dcad8bdb14e41de3b43453f4ae Mon Sep 17 00:00:00 2001 From: Tom White Date: Mon, 28 Oct 2024 15:55:58 +0000 Subject: [PATCH] Change `use_backups` default to False for local executors --- cubed/runtime/executors/local.py | 2 +- docs/configuration.md | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cubed/runtime/executors/local.py b/cubed/runtime/executors/local.py index 97d1a1abf..38cf53bd4 100644 --- a/cubed/runtime/executors/local.py +++ b/cubed/runtime/executors/local.py @@ -80,7 +80,7 @@ async def map_unordered( function: Callable[..., Any], input: Iterable[Any], retries: int = 2, - use_backups: bool = True, + use_backups: bool = False, batch_size: Optional[int] = None, return_stats: bool = False, name: Optional[str] = None, diff --git a/docs/configuration.md b/docs/configuration.md index e88831e6e..d965e4ed0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -118,23 +118,26 @@ since it is deliberately designed not to have anything except the most basic fea | Property | Default | Description | |------------------------------|---------|----------------------------------------------------------------------------------------------------| | `retries` | 2 | The number of times to retry a task if it fails. | -| `use_backups` | `True` | Whether to use backup tasks for mitigating stragglers. | +| `use_backups` | `False` | Whether to use backup tasks for mitigating stragglers. | | `batch_size` | `None` | Number of input tasks to submit to be run in parallel. The default is not to batch. | | `compute_arrays_in_parallel` | `False` | Whether arrays are computed one at a time or in parallel. | | `max_workers` | `None` | The maximum number of workers to use in the `ThreadPoolExecutor`. Defaults to number of CPU cores. | +Note that `use_backups` is `False` by default since stragglers are not generally a problem on a local machine. #### `processes` | Property | Default | Description | |------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------| -| `use_backups` | `True` | Whether to use backup tasks for mitigating stragglers. | +| `use_backups` | `False` | Whether to use backup tasks for mitigating stragglers. | | `batch_size` | `None` | Number of input tasks to submit to be run in parallel. `None` means don't batch. | | `compute_arrays_in_parallel` | `False` | Whether arrays are computed one at a time or in parallel. | | `max_workers` | `None` | The maximum number of workers to use in the `ProcessPoolExecutor`. Defaults to number of CPU cores. | | `max_tasks_per_child` | `None` | The number of tasks to run in each child process. See the Python documentation for `concurrent.futures.ProcessPoolExecutor`. (Python 3.11) | -Note that `retries` is not currently supported for the `processes` executor. +Note that `use_backups` is `False` by default since stragglers are not generally a problem on a local machine. + +Also, `retries` is not currently supported for the `processes` executor. #### `beam`