Merge branch 'main' into http-connector

.github/ISSUE_TEMPLATE/bug.yml

@@ -13,7 +13,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.37.0"
+      placeholder: "0.39.1"
     validations:
       required: true
   - type: checkboxes

.github/dependabot.yml

@@ -3,23 +3,35 @@ updates:
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
-      interval: daily
+      interval: weekly
       time: "12:00"
     reviewers: [meltano/engineering]
-    labels: [deps]
+    labels: [Dependencies]
+    groups:
+      development-dependencies:
+        dependency-type: development
+      runtime-dependencies:
+        dependency-type: production
+        update-types:
+          - "minor"
+          - "patch"
   - package-ecosystem: pip
     directory: "/.github/workflows"
     schedule:
-      interval: daily
+      interval: weekly
       time: "12:00"
     reviewers: [meltano/engineering]
-    labels: [deps]
+    labels: [Dependencies]
+    groups:
+      ci:
+        patterns:
+          - "*"
   - package-ecosystem: github-actions
     directory: "/"
     schedule:
       interval: weekly
     reviewers: [meltano/engineering]
-    labels: [deps]
+    labels: [Dependencies]
     groups:
       actions:
         patterns:

.github/semantic.yml

@@ -15,6 +15,7 @@ types:
   - docs
   - feat
   - fix
+  - packaging
   - perf
   - refactor
   - revert

.github/workflows/api-changes.yml

@@ -0,0 +1,53 @@
+name: API Changes
+
+on:
+  pull_request:
+    paths:
+    - singer_sdk/**
+    - .github/workflows/api-changes.yml
+    - CHANGELOG.md
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
+jobs:
+  check-api-changes:
+    name: Check API Changes
+    runs-on: ubuntu-latest
+    env:
+      NOXSESSION: api
+    steps:
+    - name: Check out the repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+
+    - name: Install tools
+      env:
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
+      run: |
+        python -Im pip install -U pip
+        pipx install griffe nox
+        pipx inject nox nox-poetry
+        pipx list
+
+    - name: Set REF
+      id: set-ref
+      if: always() && !startsWith(github.head_ref, 'release/')
+      run: |
+        echo "ref=${{ github.event.pull_request.base.sha }}" >> $GITHUB_OUTPUT
+
+    # Check API against the latest commit on the base branch
+    - name: Run Nox
+      run: |
+        nox -- ${{ steps.set-ref.outputs.ref }}

.github/workflows/codspeed.yml

@@ -48,7 +48,7 @@ jobs:
         --with benchmark
         --all-extras
 
-    - uses: CodSpeedHQ/action@v2
+    - uses: CodSpeedHQ/action@v3
       with:
         token: ${{ secrets.CODSPEED_TOKEN }}
         run: pytest tests/ --codspeed

.github/workflows/constraints.txt

@@ -1,7 +1,8 @@
-pip==24.0
+griffe==0.48.0
+pip==24.2
 poetry==1.8.3
 poetry-plugin-export==1.8.0
-poetry-dynamic-versioning==1.3.0
-pre-commit==3.7.1
+poetry-dynamic-versioning==1.4.0
+pre-commit==3.8.0
 nox==2024.4.15
 nox-poetry==1.0.3

.github/workflows/release.yml

@@ -28,7 +28,7 @@ jobs:
         name: Packages
         path: dist
     - name: Publish
-      uses: pypa/gh-action-pypi-publish@v1.8.14
+      uses: pypa/gh-action-pypi-publish@v1.9.0
 
   upload-to-release:
     name: Upload files to release

.github/workflows/test.yml

@@ -113,9 +113,6 @@ jobs:
       SAMPLE_TAP_GITLAB_GROUP_IDS: ${{ secrets.SAMPLE_TAP_GITLAB_GROUP_IDS }}
       SAMPLE_TAP_GITLAB_PROJECT_IDS: ${{ secrets.SAMPLE_TAP_GITLAB_PROJECT_IDS }}
       SAMPLE_TAP_GITLAB_START_DATE: "2022-01-01T00:00:00Z"
-      SAMPLE_TAP_GOOGLE_ANALYTICS_CLIENT_EMAIL: ${{ secrets.SAMPLE_TAP_GOOGLE_ANALYTICS_CLIENT_EMAIL }}
-      SAMPLE_TAP_GOOGLE_ANALYTICS_PRIVATE_KEY: ${{ secrets.SAMPLE_TAP_GOOGLE_ANALYTICS_PRIVATE_KEY }}
-      SAMPLE_TAP_GOOGLE_ANALYTICS_VIEW_ID: ${{ secrets.SAMPLE_TAP_GOOGLE_ANALYTICS_VIEW_ID }}
 
     steps:
     - uses: actions/checkout@v4

.pre-commit-config.yaml

@@ -43,14 +43,14 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.28.4
+  rev: 0.29.1
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.5
+  rev: v0.5.7
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
@@ -64,18 +64,6 @@ repos:
         cookiecutter/.*
       )$
 
-- repo: https://github.com/pycqa/flake8
-  rev: 7.0.0
-  hooks:
-  - id: flake8
-    additional_dependencies:
-    - darglint==1.8.1
-    files: |
-        (?x)^(
-          singer_sdk/.*|
-          samples/.*
-        )$
-
 - repo: https://github.com/python-poetry/poetry
   rev: 1.8.0
   hooks:

.readthedocs.yml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.12"
 
 sphinx:
   builder: html

CHANGELOG.md

@@ -5,6 +5,81 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.39.1 (2024-08-07)
+
+### 🐛 Fixes
+
+- [#2589](https://github.com/meltano/sdk/issues/2589) Make sink assertion compatible with stream maps -- _**Thanks @JCotton1123!**_
+- [#2592](https://github.com/meltano/sdk/issues/2592) Fixed typos in `--about` plain text output
+- [#2580](https://github.com/meltano/sdk/issues/2580) Date fields are now properly serialized as ISO dates, i.e. "YYYY-MM-DD"
+- [#2583](https://github.com/meltano/sdk/issues/2583) Quote add-column-ddl with column starting with `_` (underscore) based on engine -- _**Thanks @haleemur!**_
+- [#2582](https://github.com/meltano/sdk/issues/2582) DDL for adding a column now uses a SQLAlchemy-compiled clause to apply proper quoting -- _**Thanks @haleemur!**_
+- [#2418](https://github.com/meltano/sdk/issues/2418) Check replication method instead of key to determine if a SQL stream is sorted
+
+### ⚙️ Under the Hood
+
+- [#2520](https://github.com/meltano/sdk/issues/2520) Remove unused dependencies `pendulum` and `python-dateutil`
+
+### 📚 Documentation Improvements
+
+- [#2579](https://github.com/meltano/sdk/issues/2579) Documented support for `packaging` semantic type in contributing guide
+
+## v0.39.0 (2024-07-27)
+
+### ✨ New
+
+- [#2432](https://github.com/meltano/sdk/issues/2432) Developers can now customize the default logging configuration for their taps/targets by adding `default_logging.yml` to their package
+- [#2531](https://github.com/meltano/sdk/issues/2531) The `json` module is now avaiable to stream maps -- _**Thanks @grigi!**_
+- [#2529](https://github.com/meltano/sdk/issues/2529) Stream sync context is now available to all instances methods as a `Stream.context` attribute
+
+### 🐛 Fixes
+
+- [#2554](https://github.com/meltano/sdk/issues/2554) Use mapped stream aliases when handling `ACTIVATE_VERSION` messages in the base target class
+- [#2526](https://github.com/meltano/sdk/issues/2526) Moved up the supported Python versions in the Markdown output of `--about`
+
+### ⚙️ Under the Hood
+
+- [#2564](https://github.com/meltano/sdk/issues/2564) Make `SQLSink` a generic with a `SQLConnector` type parameter
+- [#2540](https://github.com/meltano/sdk/issues/2540) Implement abstract `serialize_message` for Singer writers
+- [#2259](https://github.com/meltano/sdk/issues/2259) Centralize JSON SerDe into helper functions -- _**Thanks @BuzzCutNorman!**_
+- [#2525](https://github.com/meltano/sdk/issues/2525) Make `PyJWT` and `cryptography` dependencies optional
+- [#2528](https://github.com/meltano/sdk/issues/2528) Moved class-level attributes to the top in REST tap template
+- [#2132](https://github.com/meltano/sdk/issues/2132) Limit internal usage of pendulum
+
+### 📚 Documentation Improvements
+
+- [#2557](https://github.com/meltano/sdk/issues/2557) Document that `get_starting_timestamp` requires setting a non-null replication_key
+- [#2556](https://github.com/meltano/sdk/issues/2556) Reference state partitioning in stream partitioning page
+- [#2536](https://github.com/meltano/sdk/issues/2536) Prepare for RTD addons migration
+- [#2535](https://github.com/meltano/sdk/issues/2535) Added more intersphinx links to Python and Faker docs
+- [#2530](https://github.com/meltano/sdk/issues/2530) Explained how the request URL is generated from `url_base`, `path` and the sync context
+- [#2527](https://github.com/meltano/sdk/issues/2527) Updated the footer
+- [#2506](https://github.com/meltano/sdk/issues/2506) Fixed a typo in the stream maps docs
+
+## v0.38.0 (2024-06-17)
+
+### ✨ New
+
+- [#2433](https://github.com/meltano/sdk/issues/2433) Tap developers can now disable HTTP redirects
+- [#2426](https://github.com/meltano/sdk/issues/2426) Added an optional GitHub workflow to publish to PyPI with trusted publishers
+
+### 🐛 Fixes
+
+- [#2438](https://github.com/meltano/sdk/issues/2438) Null replication values are now handled when incrementing bookmarks
+- [#2431](https://github.com/meltano/sdk/issues/2431) Updated cookiecutter VSCode `launch.json` to use `debugpy`
+- [#2421](https://github.com/meltano/sdk/issues/2421) An error message is now logged every time schema validation fails for any record
+
+### ⚙️ Under the Hood
+
+- [#2455](https://github.com/meltano/sdk/issues/2455) Use parent `datetime.datetime` class in type conforming checks
+- [#2453](https://github.com/meltano/sdk/issues/2453) Change to return type of `utc_now` from `pendulum.DateTime` to `datetime.datetime`
+
+### 📚 Documentation Improvements
+
+- [#2449](https://github.com/meltano/sdk/issues/2449) Add a short guide on defining a configuration schema
+- [#2436](https://github.com/meltano/sdk/issues/2436) Documented how context fields are passed to a child stream
+- [#2435](https://github.com/meltano/sdk/issues/2435) Using an empty list for `__key_properties__` to disable a stream primary keys is now recommended as an alternative to `null`
+
 ## v0.37.0 (2024-04-29)
 
 ### ✨ New

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/dependabot.yml

@@ -8,19 +8,34 @@ updates:
   - package-ecosystem: pip
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: weekly
     commit-message:
       prefix: "chore(deps): "
       prefix-development: "chore(deps-dev): "
+    groups:
+      development-dependencies:
+        dependency-type: development
+      runtime-dependencies:
+        dependency-type: production
+        update-types:
+          - "patch"
   - package-ecosystem: pip
     directory: "/.github/workflows"
     schedule:
-      interval: daily
+      interval: weekly
     commit-message:
       prefix: "ci: "
+    groups:
+      ci:
+        patterns:
+          - "*"
   - package-ecosystem: github-actions
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: weekly
     commit-message:
       prefix: "ci: "
+    groups:
+      actions:
+        patterns:
+          - "*"

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml

@@ -42,4 +42,4 @@ jobs:
     - name: Publish
       ## TODO: create a trusted publisher on PyPI
       ## https://docs.pypi.org/trusted-publishers/
-      uses: pypa/gh-action-pypi-publish@v1.8.14
+      uses: pypa/gh-action-pypi-publish@v1.9.0

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml

@@ -18,19 +18,19 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.28.3
+  rev: 0.29.1
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.4
+  rev: v0.5.6
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.10.0
+  rev: v1.11.1
   hooks:
   - id: mypy

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/meltano.yml

@@ -13,18 +13,26 @@ plugins:
       streams:
       - stream_name: animals
         input_filename: https://raw.githubusercontent.com/meltano/tap-smoke-test/main/demo-data/animals-data.jsonl
+
   loaders:
   - name: target-jsonl
     variant: andyh1203
     pip_url: target-jsonl
+
   mappers:
   - name: "{{cookiecutter.mapper_id}}"
-    pip_url: -e .
     namespace: "{{cookiecutter.library_name}}"
-    # TODO: replace these with the actual settings
+    pip_url: -e .
+
+    # TODO: Declare settings and their types here:
     settings:
     - name: example_config
       kind: string
+      label: Example Config
+      description: An example configuration setting
+
+    # TODO: Declare mapping instances here:
+    # https://docs.meltano.com/guide/mappers/#example-1
     mappings:
     - name: example
       config: