Merge branch 'main' into 1626-configurable-batch-size-and-max-wait-li…

…mit-for-targets

.github/workflows/test.yml

@@ -39,6 +39,7 @@ jobs:
   tests:
     name: "Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }} / SQLAlchemy: ${{ matrix.sqlalchemy }}"
     runs-on: ${{ matrix.os }}
+    continue-on-error: true
     env:
       NOXPYTHON: ${{ matrix.python-version }}
       NOXSESSION: ${{ matrix.session }}

.pre-commit-config.yaml

@@ -8,6 +8,10 @@ repos:
   rev: v4.5.0
   hooks:
   - id: check-json
+    exclude: |
+        (?x)^(
+            .*/launch.json
+        )$
   - id: check-toml
     exclude: |
         (?x)^(

codecov.yml

@@ -0,0 +1,15 @@
+coverage:
+  precision: 2
+  range: "70...100"
+  round: down
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%
+        base: auto
+    patch:
+      default:
+        target: 100%
+        threshold: 1%
+        base: auto

cookiecutter/.gitignore

@@ -0,0 +1,2 @@
+# Include template VSCode directory
+!*/*/.vscode/

cookiecutter/tap-template/hooks/post_gen_project.py

@@ -28,4 +28,4 @@
         shutil.rmtree(".github")
 
     if "{{ cookiecutter.ide }}" != "VSCode":
-        shutil.rmtree(".vscode")
+        shutil.rmtree(".vscode", ignore_errors=True)

cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml

@@ -8,6 +8,10 @@ repos:
   rev: v4.5.0
   hooks:
   - id: check-json
+    exclude: |
+      (?x)^(
+        \.vscode/.*\.json
+      )$
   - id: check-toml
   - id: check-yaml
   - id: end-of-file-fixer
@@ -20,7 +24,7 @@ repos:
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.11
+  rev: v0.1.14
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "{{ cookiecutter.tap_id }}",
+            "type": "python",
+            "request": "launch",
+            "cwd": "${workspaceFolder}",
+            "program": "{{ cookiecutter.library_name }}",
+            "justMyCode": false,
+            "args": [
+                "--config",
+                ".secrets/config.json",
+            ],
+        },
+    ]
+}

docs/conf.py

@@ -117,7 +117,12 @@
     "css/custom.css",
 ]
 
+# -- Options for MyST --------------------------------------------------------
+# https://myst-parser.readthedocs.io/en/latest/configuration.html
 myst_heading_anchors = 3
+myst_enable_extensions = {
+    "colon_fence",
+}
 
 redirects = {
     "porting.html": "guides/porting.html",

docs/dev_guide.md

@@ -55,6 +55,10 @@ pipx install poetry
 pipx install tox
 ```
 
+:::{tip}
+The minimum recommended version of cookiecutter is `2.2.0` (released 2023-07-06).
+:::
+
 Now you can initialize your new project with the Cookiecutter template for taps:
 
 ```bash

docs/faq.md

@@ -26,3 +26,12 @@ class MyStream(Stream):
         super().__init__(tap)
         self.conn...
 ```
+
+## I'm seeing `Note: Progress is not resumable if interrupted.` in my state files
+
+If the stream attribute [`is_sorted`](singer_sdk.Stream.is_sorted) (default: `False`) is not set to `True`, the records are assumed not to be monotonically increasing, i.e. there's no guarantee that newer records always come later and the tap has to run to completion so the state can safely reflect the largest replication value seen. If you know that the records are monotonically increasing, you can set `is_sorted` to `True` and the sync will be resumable if it's interrupted at any point and the state file will reflect this.
+
+```python
+class MyStream(Stream):
+    is_sorted = True
+```

pyproject.toml

@@ -44,19 +44,19 @@ backports-datetime-fromisoformat = { version = ">=2.0.1", python = "<3.11" }
 click = "~=8.0"
 cryptography = ">=3.4.6"
 fs = ">=2.4.16"
-importlib-metadata = {version = "<7.0.0", python = "<3.12"}
+importlib-metadata = {version = "<8.0.0", python = "<3.12"}
 importlib-resources = {version = ">=5.12.0", python = "<3.9"}
 inflection = ">=0.5.1"
 joblib = ">=1.0.1"
 jsonpath-ng = ">=1.5.3"
 jsonschema = ">=4.16.0"
-memoization = { version = ">=0.3.2,<0.5.0", python = "<4" }
 packaging = ">=23.1"
 pendulum = ">=2.1.0,<4"
 PyJWT = "~=2.4"
 python-dateutil = ">=2.8.2"
 python-dotenv = ">=0.20"
 PyYAML = ">=6.0"
+referencing = ">=0.30.0"
 requests = ">=2.25.1"
 simpleeval = ">=0.9.13"
 simplejson = ">=3.17.6"
@@ -187,6 +187,7 @@ exclude_also = [
     '''if (t\.)?TYPE_CHECKING:''',
 ]
 fail_under = 82
+show_missing = true
 
 [tool.mypy]
 exclude = "tests"

singer_sdk/_singerlib/messages.py

@@ -226,7 +226,12 @@ def format_message(message: Message) -> str:
     Returns:
         The formatted message.
     """
-    return json.dumps(message.to_dict(), use_decimal=True, default=_default_encoding)
+    return json.dumps(
+        message.to_dict(),
+        use_decimal=True,
+        default=_default_encoding,
+        separators=(",", ":"),
+    )
 
 
 def write_message(message: Message) -> None:

singer_sdk/_singerlib/schema.py

@@ -5,7 +5,11 @@
 import typing as t
 from dataclasses import dataclass
 
-from jsonschema import RefResolver
+from referencing import Registry
+from referencing.jsonschema import DRAFT202012
+
+if t.TYPE_CHECKING:
+    from referencing._core import Resolver
 
 # These are keys defined in the JSON Schema spec that do not themselves contain
 # schemas (or lists of schemas)
@@ -148,17 +152,25 @@ def resolve_schema_references(
         A schema dict with all $refs replaced with the appropriate dict.
     """
     refs = refs or {}
-    return _resolve_schema_references(schema, RefResolver("", schema, store=refs))
+    registry: Registry = Registry()
+    schema_resource = DRAFT202012.create_resource(schema)
+    registry = registry.with_resource("", schema_resource)
+    registry = registry.with_resources(
+        [(k, DRAFT202012.create_resource(v)) for k, v in refs.items()]
+    )
+
+    resolver = registry.resolver()
+    return _resolve_schema_references(schema, resolver)
 
 
 def _resolve_schema_references(
     schema: dict[str, t.Any],
-    resolver: RefResolver,
+    resolver: Resolver,
 ) -> dict[str, t.Any]:
     if _SchemaKey.ref in schema:
         reference_path = schema.pop(_SchemaKey.ref, None)
-        resolved = resolver.resolve(reference_path)[1]
-        schema.update(resolved)
+        resolved = resolver.lookup(reference_path)
+        schema.update(resolved.contents)
         return _resolve_schema_references(schema, resolver)
 
     if _SchemaKey.properties in schema: