Moving linting to ruff (#121)

* feat: Add ruff configuration

* Deactivate flake8 in precommit

* chore: Fixing ruff warnings

* ci: Minimal ruff workflow

* ci: Replace isort and black by ruff

* ci: Disable checks for python 3.7 and 3.8 which are end-of-life

* doc: Fix dead link

* ci: fix codecov upload

Author: chr1st1ank

.github/workflows/ruff.yml

@@ -0,0 +1,18 @@
+name: Ruff
+on: push
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff
+      # Update output format to enable automatic inline annotations.
+      - name: Run Ruff
+        run: ruff check --output-format=github .

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10"]
         exclude:
           # No numpy wheels yet:
           - os: windows-latest
@@ -62,7 +62,7 @@ jobs:
           pip install poetry tox tox-gh-actions
           poetry install --with test,dev,doc
 
-      - name: Lint and test with tox
+      - name: Test with tox
         if: ${{ matrix.tox-full-run }}
         run:
           poetry run tox
@@ -77,10 +77,11 @@ jobs:
         with:
           junit_files: pytest.xml
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         if: ${{ matrix.publish-results && always() }}
         with:
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.xml
 
   publish_dev_build:

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ ci:
   submodules: false
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v5.0.0
   hooks:
   - id: check-merge-conflict
   - id: check-json
@@ -14,17 +14,9 @@ repos:
     args: [--unsafe]
   - id: debug-statements
   - id: end-of-file-fixer
-- repo: https://github.com/pre-commit/mirrors-isort
-  rev: v5.10.1
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.6.9
   hooks:
-  - id: isort
-- repo: https://github.com/ambv/black
-  rev: 22.10.0
-  hooks:
-  - id: black
-    language_version: python3.9
-- repo: https://github.com/pycqa/flake8
-  rev: 6.0.0
-  hooks:
-  - id: flake8
-    additional_dependencies: [flake8-typing-imports==1.14.0]
+    - id: ruff
+      args: [ --fix ]
+    - id: ruff-format

dike/__init__.py

@@ -1,4 +1,5 @@
-"""Decorator library"""
+"""Decorator library."""
+
 import functools
 import inspect
 from typing import Callable

dike/_batch.py

@@ -1,4 +1,5 @@
-"""Implementation of the @dike.batch decorator"""
+"""Implementation of the @dike.batch decorator."""
+
 import asyncio
 import functools
 from contextlib import contextmanager
@@ -11,18 +12,18 @@
 
 
 # Deactivate mccabe's complexity warnings which doesn't like closures
-# flake8: noqa: C901
-def batch(
+def batch(  # noqa: PLR0915, C901
     *,
     target_batch_size: int,
     max_waiting_time: float,
     max_processing_time: float = 10.0,
     argument_type: str = "list",
 ) -> Callable[[Callable[..., Coroutine[Any, Any, Any]]], Callable[..., Coroutine[Any, Any, Any]]]:
     """@batch is a decorator to cumulate function calls and process them in batches.
-        Not thread-safe.
 
-    The function to wrap must have arguments of type list or numpy.array which can be aggregated.
+    Not thread-safe.
+
+    The function to wrap must have arguments of type list or `numpy.array` which can be aggregated.
     It must return just a single value of the same type. The type has to be specified with the
     `argument_type` parameter of the decorator.
 
@@ -102,7 +103,7 @@ def batch(
     if argument_type == "numpy" and np is None:
         raise ValueError('Unable to use "numpy" as argument_type because numpy is not available')
 
-    def decorator(func):
+    def decorator(func):  # noqa: C901, PLR0915
         next_free_batch: int = 0
         call_args_queue: List[Tuple[List, Dict]] = []
         n_rows_in_queue: int = 0
@@ -112,16 +113,21 @@ def decorator(func):
 
         @functools.wraps(func)
         async def batching_call(*args, **kwargs):
-            """This is the actual wrapper function which controls the process"""
-            nonlocal results, num_results_ready, result_ready_events, call_args_queue, n_rows_in_queue
+            """This is the actual wrapper function which controls the process."""
+            nonlocal \
+                results, \
+                num_results_ready, \
+                result_ready_events, \
+                call_args_queue, \
+                n_rows_in_queue
 
             with enqueue(args, kwargs) as (my_batch_no, start_index, stop_index):
                 await wait_for_calculation(my_batch_no)
                 return get_results(start_index, stop_index, my_batch_no)
 
         @contextmanager
         def enqueue(args, kwargs) -> (int, int, int):
-            """Add call arguments to queue and get the batch number and result indices"""
+            """Add call arguments to queue and get the batch number and result indices."""
             batch_no = next_free_batch
             if batch_no not in result_ready_events:
                 result_ready_events[batch_no] = asyncio.Event()
@@ -132,7 +138,7 @@ def enqueue(args, kwargs) -> (int, int, int):
                 remove_result(batch_no)
 
         def add_args_to_queue(args, kwargs):
-            """Add a new argument vector to the queue and return result indices"""
+            """Add a new argument vector to the queue and return result indices."""
             nonlocal call_args_queue, n_rows_in_queue
 
             if call_args_queue and (
@@ -155,7 +161,7 @@ def add_args_to_queue(args, kwargs):
             return offset, n_rows_in_queue
 
         async def wait_for_calculation(batch_no_to_calculate):
-            """Pause until the result becomes available or trigger the calculation on timeout"""
+            """Pause until the result becomes available or trigger the calculation on timeout."""
             if n_rows_in_queue >= target_batch_size:
                 await calculate(batch_no_to_calculate)
             else:
@@ -173,20 +179,20 @@ async def wait_for_calculation(batch_no_to_calculate):
                         )
 
         async def calculate(batch_no_to_calculate):
-            """Call the decorated coroutine with batched arguments"""
+            """Call the decorated coroutine with batched arguments."""
             nonlocal results, call_args_queue, num_results_ready
             if next_free_batch == batch_no_to_calculate:
                 n_results = len(call_args_queue)
                 args, kwargs = pop_args_from_queue()
                 try:
                     results[batch_no_to_calculate] = await func(*args, **kwargs)
-                except Exception as e:  # pylint: disable=broad-except
+                except Exception as e:  # pylint: disable=broad-except  # noqa: BLE001
                     results[batch_no_to_calculate] = e
                 num_results_ready[batch_no_to_calculate] = n_results
                 result_ready_events[batch_no_to_calculate].set()
 
         def pop_args_from_queue():
-            """Get all collected arguments from the queue as batch"""
+            """Get all collected arguments from the queue as batch."""
             nonlocal next_free_batch, call_args_queue, n_rows_in_queue
 
             n_args = len(call_args_queue[0][0])
@@ -215,7 +221,7 @@ def pop_args_from_queue():
             return args, kwargs
 
         def get_results(start_index: int, stop_index: int, batch_no):
-            """Pop the results for a certain index range from the output buffer"""
+            """Pop the results for a certain index range from the output buffer."""
             nonlocal results
 
             if isinstance(results[batch_no], Exception):
@@ -225,7 +231,7 @@ def get_results(start_index: int, stop_index: int, batch_no):
             return results_to_return
 
         def remove_result(batch_no):
-            """Reduce reference count to output buffer and eventually delete it"""
+            """Reduce reference count to output buffer and eventually delete it."""
             nonlocal num_results_ready, result_ready_events, results
 
             if num_results_ready[batch_no] == 1:

dike/_limit_jobs.py

@@ -1,11 +1,12 @@
-"""Implementation of the @dike.limit_jobs decorator"""
+"""Implementation of the @dike.limit_jobs decorator."""
+
 import functools
 import inspect
 from typing import Any, Callable, Coroutine
 
 
-class TooManyCalls(Exception):
-    """Error raised by @limit_jobs when a call exceeds the preset limit"""
+class TooManyCalls(Exception):  # noqa: N818
+    """Error raised by @limit_jobs when a call exceeds the preset limit."""
 
 
 def limit_jobs(*, limit: int) -> Callable[..., Coroutine[Any, Any, Any]]:
@@ -63,7 +64,7 @@ def limit_jobs(*, limit: int) -> Callable[..., Coroutine[Any, Any, Any]]:
 
     def decorator(func):
         if not inspect.iscoroutinefunction(func):
-            raise ValueError(f"Error when wrapping {str(func)}. Only coroutines can be wrapped!")
+            raise ValueError(f"Error when wrapping {func!s}. Only coroutines can be wrapped!")
 
         @functools.wraps(func)
         async def limited_call(*args, **kwargs):

dike/_retry.py

@@ -1,19 +1,20 @@
-"""Implementation of the @dike.retry decorator"""
+"""Implementation of the @dike.retry decorator."""
+
 import asyncio
 import datetime
 import functools
 import inspect
 import logging
-from typing import Awaitable, Callable, Tuple, Type, Union
+from typing import Awaitable, Callable, Optional, Tuple, Type, Union
 
 logger = logging.getLogger("dike")
 
 
 def retry(
     *,
-    attempts: int = None,
+    attempts: Optional[int] = None,
     exception_types: Union[Type[BaseException], Tuple[Type[BaseException]]] = Exception,
-    delay: datetime.timedelta = None,
+    delay: Optional[datetime.timedelta] = None,
     backoff: int = 1,
     log_exception_info: bool = True,
 ) -> Callable[[Callable[..., Awaitable]], Callable[..., Awaitable]]:
@@ -70,7 +71,7 @@ def retry(
 
     def decorator(func: Callable[..., Awaitable]) -> Callable[..., Awaitable]:
         if not inspect.iscoroutinefunction(func):
-            raise ValueError(f"Error when wrapping {str(func)}. Only coroutines can be wrapped!")
+            raise ValueError(f"Error when wrapping {func!s}. Only coroutines can be wrapped!")
 
         @functools.wraps(func)
         async def guarded_call(*args, **kwargs):
@@ -86,7 +87,7 @@ async def guarded_call(*args, **kwargs):
                     if not counter:
                         raise
                     logger.warning(
-                        f"Caught exception {repr(e)}. Retrying in {next_delay:.3g}s ...",
+                        f"Caught exception {e!r}. Retrying in {next_delay:.3g}s ...",
                         exc_info=log_exception_info,
                     )
                     await asyncio.sleep(next_delay)

docs/contributing.md

@@ -1,7 +1,3 @@
 {%
   include-markdown "../CONTRIBUTING.md"
 %}
-
-{%
-  include-markdown "../AUTHORS.md"
-%}

examples/ml_prediction_service/api.py

@@ -1,9 +1,13 @@
-# Dependencies: fastapi, uvicorn
-# Run with: python api.py
-#
-# Simple load test with github.com/codesenberg/bombardier:
-# bombardier -c 25 -r 300 -d 10s -l 'localhost:8000/predict?number=5'
-#
+"""Minimal example API using dike for microbatching.
+
+Dependencies: fastapi, uvicorn
+Run with: python api.py
+
+Simple load test with github.com/codesenberg/bombardier:
+
+    bombardier -c 25 -r 300 -d 10s -l 'localhost:8000/predict?number=5'
+"""
+
 import asyncio
 import concurrent
 import random
@@ -18,7 +22,7 @@
 
 
 def predict(numbers: List[float]):
-    """Dummy machine learning operation"""
+    """Dummy machine learning operation."""
     arr = np.array(numbers)
     for _ in range(10000):
         arr = np.sqrt(arr + random.random() * 2)
@@ -28,17 +32,19 @@ def predict(numbers: List[float]):
 @dike.limit_jobs(limit=20)
 @dike.batch(target_batch_size=10, max_waiting_time=0.1)
 async def predict_in_pool(numbers: List[float]):
+    """Wrapper function for the predictions to allow batching."""
     loop = asyncio.get_running_loop()
     return await loop.run_in_executor(app.state.pool, predict, numbers)
 
 
 @app.on_event("startup")
-async def on_startup():
+async def on_startup():  # noqa: D103
     app.state.pool = concurrent.futures.ProcessPoolExecutor(max_workers=2)
 
 
 @app.get("/predict")
 async def get_predict(number: float, response: Response):
+    """API endpoint for machine learning inference."""
     try:
         x = await predict_in_pool([number])
         return {"result": x}

ruff.toml

@@ -0,0 +1,47 @@
+line-length = 100
+##extend-exclude = ["*.ipynb", "**/*_pb2.py"]
+
+[lint]
+ignore = [
+    "PLR0913", # Too many arguments
+    "S311", # suspicious-non-cryptographic-random-usage
+    "G004" # Logging statement uses f-string
+]
+select = [
+    "A",  # flake8-builtins
+    "C4",  # flake8-comprehensions
+    "DTZ",  # flake8-datetimez
+    "B",
+    "B9",
+    "C",
+    "G",  # flake8-logging-format
+    "PYI",  # flake8-pyi
+    "PT",  # flake8-pytest-style
+    "TCH",  #  flake8-type-checking
+    "C90",  # MCCabe
+    "D",  # Pydocstyle
+    "E",  # Pycodestyle error
+    "F",  # Pyflakes
+    "I",  # Isort
+    "N",  # pep8-naming
+    "W",  # Pycodestyle warning
+#    "ANN",  # flake8-annotations
+    "S",  # flake8-bandit
+    "BLE",  # flake8-blind-except
+    "PD",  # pandas-vet
+    "PL",  # Pylint
+    "RUF",  # Ruff
+]
+
+[lint.per-file-ignores]
+"__init__.py" = ["F401"]
+"tests/*" = ["ANN", "D", "S101", "PT", "PLR", "PD901"]
+"integration_tests/*" = ["ANN", "D", "S101", "PT", "PLR", "PD901"]
+
+[lint.pydocstyle]
+convention = "google"
+
+#[mccabe]
+#max-complexity = 18
+#
+#

tasks.py

@@ -1,8 +1,8 @@
-"""
-Tasks for maintaining the project.
+"""Tasks for maintaining the project.
 
 Execute 'invoke --list' for guidance on using Invoke
 """
+
 import platform
 import webbrowser
 from pathlib import Path