From bfcca270a7ad0ec08ae7544fb3ce7a06f8f680f9 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 17 Jan 2023 10:24:37 -0600 Subject: [PATCH] chore: Run mypy on the EDK (#49) * chore: Run mypy on the EDK * Add quotes to `3.10` --- .github/workflows/test.yml | 24 ++++++++++++++++ meltano/edk/extension.py | 2 +- meltano/edk/logging.py | 56 ++++++++++++++++++++++++++++++-------- meltano/edk/process.py | 26 ++++++++++++------ meltano/py.typed | 0 poetry.lock | 14 +++++++++- pyproject.toml | 1 + 7 files changed, 101 insertions(+), 22 deletions(-) create mode 100644 meltano/py.typed diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index edc3f9b..d93fbf3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,3 +36,27 @@ jobs: - name: Test with pytest run: | poetry run pytest + + mypy: + name: Static type checking with mypy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Poetry + run: | + pipx install poetry + poetry --version + + - name: Install dependencies + run: | + poetry env use 3.10 + poetry install + + - name: Run mypy + run: | + poetry run mypy --namespace-packages -p meltano.edk diff --git a/meltano/edk/extension.py b/meltano/edk/extension.py index f15e4f8..fb5e024 100644 --- a/meltano/edk/extension.py +++ b/meltano/edk/extension.py @@ -76,7 +76,7 @@ def describe(self) -> models.Describe: """ pass - def describe_formatted( + def describe_formatted( # type: ignore[return] self, output_format: DescribeFormat = DescribeFormat.text ) -> str: """Return a formatted description of the extensions commands and capabilities. diff --git a/meltano/edk/logging.py b/meltano/edk/logging.py index 68d0d36..fefca88 100644 --- a/meltano/edk/logging.py +++ b/meltano/edk/logging.py @@ -4,6 +4,7 @@ import logging import os import sys +from typing import Callable import structlog @@ -18,7 +19,36 @@ DEFAULT_LEVEL = "info" -def parse_log_level(log_level: dict[str, int]) -> int: +def strtobool(val: str) -> bool: + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + + Case is ignored in string comparisons. + + Re-implemented from distutils.util.strtobool to avoid importing distutils. + + Args: + val: The string to convert to a boolean. + + Returns: + True if the string represents a truthy value, False otherwise. + + Raises: + ValueError: If the string is not a valid representation of a boolean. + """ + val = val.lower() + if val in {"y", "yes", "t", "true", "on", "1"}: + return True + elif val in {"n", "no", "f", "false", "off", "0"}: + return False + + raise ValueError(f"invalid truth value {val!r}") + + +def parse_log_level(log_level: str) -> int: """Parse a level descriptor into an logging level. Args: @@ -44,12 +74,18 @@ def default_logging_config( levels: include levels in the log. json_format: if True, use JSON format, otherwise use human-readable format. """ - processors = [] + processors: list[Callable] = [] if timestamps: processors.append(structlog.processors.TimeStamper(fmt="iso")) if levels: processors.append(structlog.processors.add_log_level) + renderer: structlog.processors.JSONRenderer | structlog.dev.ConsoleRenderer = ( + structlog.processors.JSONRenderer() + if json_format + else structlog.dev.ConsoleRenderer(colors=False) + ) + processors.extend( [ # If log level is too low, abort pipeline and throw away log entry. @@ -65,9 +101,7 @@ def default_logging_config( structlog.processors.UnicodeDecoder(), structlog.processors.ExceptionPrettyPrinter(), # Render the final event dict as JSON. - structlog.processors.JSONRenderer() - if json_format - else structlog.dev.ConsoleRenderer(colors=False), + renderer, ] ) @@ -100,13 +134,13 @@ def pass_through_logging_config() -> None: and MELTANO_LOG_JSON env vars. """ log_level = os.environ.get("LOG_LEVEL", "INFO") - log_timestamps = os.environ.get("LOG_TIMESTAMPS", False) - log_levels = os.environ.get("LOG_LEVELS", False) - meltano_log_json = os.environ.get("MELTANO_LOG_JSON", False) + log_timestamps = os.environ.get("LOG_TIMESTAMPS", "False") + log_levels = os.environ.get("LOG_LEVELS", "False") + meltano_log_json = os.environ.get("MELTANO_LOG_JSON", "False") default_logging_config( level=parse_log_level(log_level), - timestamps=log_timestamps, - levels=log_levels, - json_format=meltano_log_json, + timestamps=strtobool(log_timestamps), + levels=strtobool(log_levels), + json_format=strtobool(meltano_log_json), ) diff --git a/meltano/edk/process.py b/meltano/edk/process.py index 5f0451f..179dc87 100644 --- a/meltano/edk/process.py +++ b/meltano/edk/process.py @@ -4,11 +4,12 @@ import asyncio import os import subprocess -from typing import IO, Any +from typing import IO, Any, Union import structlog log = structlog.get_logger() +_ExecArg = Union[str, bytes] def log_subprocess_error( @@ -37,8 +38,8 @@ class Invoker: def __init__( self, bin: str, - cwd: str = None, - env: dict[str, any] | None = None, + cwd: str | None = None, + env: dict[str, Any] | None = None, ) -> None: """Minimal invoker for running subprocesses. @@ -53,7 +54,7 @@ def __init__( def run( self, - *args: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *args: _ExecArg, stdout: None | int | IO = subprocess.PIPE, stderr: None | int | IO = subprocess.PIPE, text: bool = True, @@ -113,9 +114,9 @@ async def _log_stdio(reader: asyncio.streams.StreamReader) -> None: async def _exec( self, sub_command: str | None = None, - *args: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *args: _ExecArg, ) -> asyncio.subprocess.Process: - popen_args = [] + popen_args: list[_ExecArg] = [] if sub_command: popen_args.append(sub_command) if args: @@ -130,9 +131,16 @@ async def _exec( env=self.popen_env, ) + streams: list[asyncio.streams.StreamReader] = [] + + if p.stderr: + streams.append(p.stderr) + + if p.stdout: + streams.append(p.stdout) + results = await asyncio.gather( - asyncio.create_task(self._log_stdio(p.stderr)), - asyncio.create_task(self._log_stdio(p.stdout)), + *[asyncio.create_task(self._log_stdio(stream)) for stream in streams], return_exceptions=True, ) @@ -146,7 +154,7 @@ async def _exec( def run_and_log( self, sub_command: str | None = None, - *args: str | bytes | os.PathLike[str] | os.PathLike[bytes], + *args: _ExecArg, ) -> None: """Run a subprocess and stream the output to the logger. diff --git a/meltano/py.typed b/meltano/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/poetry.lock b/poetry.lock index 2ccabe3..a06b7bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1480,6 +1480,18 @@ files = [ {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.2" +description = "Typing stubs for PyYAML" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.2.tar.gz", hash = "sha256:6840819871c92deebe6a2067fb800c11b8a063632eb4e3e755914e7ab3604e83"}, + {file = "types_PyYAML-6.0.12.2-py3-none-any.whl", hash = "sha256:1e94e80aafee07a7e798addb2a320e32956a373f376655128ae20637adb2655b"}, +] + [[package]] name = "typing-extensions" version = "4.4.0" @@ -1543,4 +1555,4 @@ docs = ["sphinx", "sphinx-rtd-theme", "sphinx-copybutton", "myst-parser", "sphin [metadata] lock-version = "2.0" python-versions = "<3.12,>=3.7" -content-hash = "fa2be45221d49f80763dcbff473d69ef1d615e157071223688de7c4b951903e4" +content-hash = "5e1d133d83793e029d3709285b654c2cc15adcab73bcd682006a08526db65917" diff --git a/pyproject.toml b/pyproject.toml index 29f2f73..83ab143 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ flake8-docstrings = "^1.6.0" [tool.poetry.group.dev.dependencies] copier = "^6.2.0" +types-pyyaml = "^6.0.12.2" [tool.isort] profile = "black"