Merge branch 'main' into edgarrmondragon/feat/python3.13

meltano · Aug 28, 2024 · 6c3ab3b · 6c3ab3b
2 parents 8f8a5be + 0e2df68
commit 6c3ab3b
Show file tree

Hide file tree

Showing 13 changed files with 285 additions and 102 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -29,13 +29,12 @@ body:
       label: Python Version
       description: Version of Python you are using
       options:
-        - "3.6 (EOL)"
-        - "3.7 (EOL)"
-        - "3.8"
-        - "3.9"
-        - "3.10"
-        - "3.11"
         - "3.12"
+        - "3.11"
+        - "3.10"
+        - "3.9"
+        - "3.8"
+        - "3.7 or earlier"
         - "NA"
     validations:
       required: true
@@ -78,3 +77,9 @@ body:
       render: Python
     validations:
       required: false
+  - type: input
+    id: slack_or_linen
+    attributes:
+      label: Link to Slack/Linen
+      description: Provide a link to the Slack or Linen conversation, if applicable
+      placeholder: "https://..."
diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,4 +1,4 @@
-griffe==1.1.0
+griffe==1.2.0
 pip==24.2
 poetry==1.8.3
 poetry-plugin-export==1.8.0

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -43,14 +43,14 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.2
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.7
+  rev: v0.6.2
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml
@@ -3,7 +3,26 @@
 
 name: Test {{cookiecutter.mapper_id}}
 
-on: [push]
+on:
+  push:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  pull_request:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  workflow_dispatch:
 
 jobs:
   pytest:
@@ -13,7 +32,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version:
+        - "3.8"
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python {{ '${{ matrix.python-version }}' }}

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml
@@ -3,7 +3,26 @@
 
 name: Test {{cookiecutter.tap_id}}
 
-on: [push]
+on:
+  push:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  pull_request:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  workflow_dispatch:
 
 jobs:
   pytest:
@@ -13,7 +32,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version:
+        - "3.8"
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python {{ '${{ matrix.python-version }}' }}

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml b/cookiecutter/target-template/{{cookiecutter.target_id}}/.github/workflows/test.yml
@@ -3,7 +3,26 @@
 
 name: Test {{cookiecutter.target_id}}
 
-on: [push]
+on:
+  push:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  pull_request:
+    branches: [main]
+    paths:
+    - .github/workflows/test.yml
+    - {{ cookiecutter.library_name }}/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - tox.ini
+  workflow_dispatch:
 
 jobs:
   pytest:
@@ -13,7 +32,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version:
+        - "3.8"
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python {{ '${{ matrix.python-version }}' }}

diff --git a/docs/incremental_replication.md b/docs/incremental_replication.md
@@ -38,6 +38,7 @@ class CommentsStream(RESTStream):
 
 3. Last, we have to adapt the query to the remote system, in this example by adding a query parameter with the ISO timestamp.
 
+   The [`get_starting_timestamp`](singer_sdk.Stream.get_starting_timestamp) method and the related [`get_starting_replication_key_value`](singer_sdk.Stream.get_starting_replication_key_value) method, are provided by the SDK and return the last replication key value seen in the previous run. If the tap is run for the first time and the value for the `start_date` setting is null, the method will return `None`.
 
 ```{note}
 - The SDK will throw an error if records come out of order when `is_sorted` is true.

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -75,17 +75,22 @@ sphinx-reredirects = {version = ">=0.1.5", python = ">=3.9", optional = true}
 fs-s3fs = {version = ">=1.1.1", optional = true}
 
 # Parquet file dependencies installed as optional 'parquet' extras
+# We add Python constraints to force Poetry to add the latest supported Numpy version
+# for all Python versions to 'poetry.lock'. If we don't do this, Poetry will add only
+# the version of Numpy that is compatible with the earliest Python version supported
+# by this project, but that may not be compatible with the latest Python version.
 numpy = [
-    { version = ">=1.22,<1.25", python = ">=3.8,<3.9", optional = true },
-    { version = ">=1.22", python = ">=3.9", optional = true },
+    { version = ">=1.22,<1.25", python = "==3.8", optional = true },
+    { version = ">=1.22,<2.1", python = "==3.9", optional = true },
+    { version = ">=1.22", python = ">=3.10", optional = true },
 ]
 pyarrow = { version = ">=13", optional = true }
 
 # Testing dependencies installed as optional 'testing' extras
 pytest = {version=">=7.2.1", optional = true}
 
 # installed as optional 'faker' extra
-faker = {version = ">=22.5,<27.0", optional = true}
+faker = {version = ">=22.5,<29.0", optional = true}
 
 # Crypto extras
 cryptography = { version = ">=3.4.6", optional = true }

diff --git a/singer_sdk/sinks/core.py b/singer_sdk/sinks/core.py
@@ -611,7 +611,7 @@ def _after_process_record(self, context: dict) -> None:
 
     # SDK developer overrides:
 
-    def preprocess_record(self, record: dict, context: dict) -> dict:  # noqa: ARG002, PLR6301
+    def preprocess_record(self, record: dict, context: dict) -> dict:  # noqa: PLR6301, ARG002
         """Process incoming record and return a modified result.
 
         Args:
@@ -743,12 +743,15 @@ def process_batch_files(
                     tail,
                     mode="rb",
                 ) as file:
-                    context_file = (
-                        gzip_open(file) if encoding.compression == "gzip" else file
-                    )
-                    context = {
-                        "records": [deserialize_json(line) for line in context_file]  # type: ignore[attr-defined]
-                    }
+                    if encoding.compression == "gzip":
+                        with gzip_open(file) as context_file:
+                            context = {
+                                "records": [
+                                    deserialize_json(line) for line in context_file
+                                ]
+                            }
+                    else:
+                        context = {"records": [deserialize_json(line) for line in file]}
                     self.process_batch(context)
             elif (
                 importlib.util.find_spec("pyarrow")
@@ -760,8 +763,7 @@ def process_batch_files(
                     tail,
                     mode="rb",
                 ) as file:
-                    context_file = file
-                    table = pq.read_table(context_file)
+                    table = pq.read_table(file)
                     context = {"records": table.to_pylist()}
                     self.process_batch(context)
             else:

diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py
@@ -365,6 +365,23 @@ def _write_replication_key_signpost(
         state = self.get_context_state(context)
         write_replication_key_signpost(state, value)
 
+    def _parse_datetime(self, value: str) -> datetime.datetime:  # noqa: PLR6301
+        """Parse a datetime string.
+
+        Args:
+            value: The datetime string.
+
+        Returns:
+            The parsed datetime, timezone-aware preferred.
+        """
+        result = datetime_fromisoformat(value)
+
+        # Ensure datetime is timezone-aware
+        if not result.tzinfo:
+            result = result.replace(tzinfo=datetime.timezone.utc)
+
+        return result
+
     def compare_start_date(self, value: str, start_date_value: str) -> str:
         """Compare a bookmark value to a start date and return the most recent value.
 
@@ -384,7 +401,7 @@ def compare_start_date(self, value: str, start_date_value: str) -> str:
             The most recent value between the bookmark and start date.
         """
         if self.is_timestamp_replication_key:
-            return max(value, start_date_value, key=datetime_fromisoformat)
+            return max(value, start_date_value, key=self._parse_datetime)
 
         return value
 

diff --git a/singer_sdk/testing/factory.py b/singer_sdk/testing/factory.py
@@ -140,7 +140,7 @@ def config(self) -> SuiteConfig:  # noqa: PLR6301
                 return suite_config or SuiteConfig()
 
             @pytest.fixture
-            def resource(self) -> t.Any:  # noqa: ANN401, PLR6301, PT004
+            def resource(self) -> t.Any:  # noqa: ANN401, PLR6301
                 yield  # noqa: PT022
 
             @pytest.fixture(scope="class")
@@ -345,7 +345,7 @@ def config(self) -> SuiteConfig:  # noqa: PLR6301
                 return suite_config or SuiteConfig()
 
             @pytest.fixture
-            def resource(self) -> t.Any:  # noqa: ANN401, PLR6301, PT004
+            def resource(self) -> t.Any:  # noqa: ANN401, PLR6301
                 yield  # noqa: PT022
 
             @pytest.fixture

diff --git a/tests/core/test_streams.py b/tests/core/test_streams.py
@@ -141,6 +141,18 @@ def test_stream_apply_catalog(stream: Stream):
             parse(CONFIG_START_DATE).replace(tzinfo=datetime.timezone.utc),
             id="datetime-repl-key-old-bookmark",
         ),
+        pytest.param(
+            "test",
+            None,
+            "2021-01-02T00:00:00-08:00",
+            datetime.datetime(
+                2021,
+                1,
+                2,
+                tzinfo=datetime.timezone(datetime.timedelta(hours=-8)),
+            ),
+            id="datetime-repl-key-recent-bookmark-tz-aware",
+        ),
         pytest.param(
             "unix_ts",
             None,