From 445439467878c79c6161d8c2c23c69487a9ab6a9 Mon Sep 17 00:00:00 2001 From: Josh lloyd Date: Mon, 4 Mar 2024 13:25:55 -0700 Subject: [PATCH] initial code dump --- .github/dependabot.yml | 26 +++++ .github/workflows/test.yml | 32 ++++++ .gitignore | 45 ++------ .pre-commit-config.yaml | 38 +++++++ .secrets/.gitignore | 10 ++ LICENSE | 202 ++++++++++++++++++++++++++++++++++ README.md | 131 +++++++++++++++++++++- meltano.yml | 30 +++++ output/.gitignore | 4 + pyproject.toml | 71 ++++++++++++ tap_clari/__init__.py | 1 + tap_clari/__main__.py | 7 ++ tap_clari/client.py | 150 +++++++++++++++++++++++++ tap_clari/schemas/__init__.py | 1 + tap_clari/streams.py | 72 ++++++++++++ tap_clari/tap.py | 58 ++++++++++ tests/__init__.py | 1 + tests/test_core.py | 22 ++++ tox.ini | 19 ++++ 19 files changed, 886 insertions(+), 34 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/test.yml create mode 100644 .pre-commit-config.yaml create mode 100644 .secrets/.gitignore create mode 100644 LICENSE create mode 100644 meltano.yml create mode 100644 output/.gitignore create mode 100644 pyproject.toml create mode 100644 tap_clari/__init__.py create mode 100644 tap_clari/__main__.py create mode 100644 tap_clari/client.py create mode 100644 tap_clari/schemas/__init__.py create mode 100644 tap_clari/streams.py create mode 100644 tap_clari/tap.py create mode 100644 tests/__init__.py create mode 100644 tests/test_core.py create mode 100644 tox.ini diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..933e6b1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,26 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "chore(deps): " + prefix-development: "chore(deps-dev): " + - package-ecosystem: pip + directory: "/.github/workflows" + schedule: + interval: daily + commit-message: + prefix: "ci: " + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "ci: " diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..ad7a3e2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,32 @@ +### A CI workflow template that runs linting and python testing +### TODO: Modify as needed or as desired. + +name: Test tap-clari + +on: [push] + +jobs: + pytest: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Poetry + run: | + pip install poetry + - name: Install dependencies + run: | + poetry env use ${{ matrix.python-version }} + poetry install + - name: Test with pytest + run: | + poetry run pytest diff --git a/.gitignore b/.gitignore index 68bc17f..1b214d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# Secrets and internal config files +**/.secrets/* + +# Ignore meltano internal cache and sqlite systemdb + +.meltano/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -20,6 +27,7 @@ parts/ sdist/ var/ wheels/ +pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg @@ -49,7 +57,6 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ -cover/ # Translations *.mo @@ -72,7 +79,6 @@ instance/ docs/_build/ # PyBuilder -.pybuilder/ target/ # Jupyter Notebook @@ -83,9 +89,7 @@ profile_default/ ipython_config.py # pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version +.python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. @@ -94,22 +98,7 @@ ipython_config.py # install all needed dependencies. #Pipfile.lock -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +# PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff @@ -146,15 +135,5 @@ dmypy.json # Pyre type checker .pyre/ -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +# IDEs +.idea \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d2661e3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +ci: + autofix_prs: true + autoupdate_schedule: weekly + autoupdate_commit_msg: 'chore: pre-commit autoupdate' + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-json + exclude: | + (?x)^( + \.vscode/.*\.json + )$ + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + +- repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.27.3 + hooks: + - id: check-dependabot + - id: check-github-workflows + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.14 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix, --show-fixes] + - id: ruff-format + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.8.0 + hooks: + - id: mypy + additional_dependencies: + - types-requests diff --git a/.secrets/.gitignore b/.secrets/.gitignore new file mode 100644 index 0000000..33c6acd --- /dev/null +++ b/.secrets/.gitignore @@ -0,0 +1,10 @@ +# IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, +# make sure those are never staged for commit into your git repo. You can store them here or another +# secure location. +# +# Note: This may be redundant with the global .gitignore for, and is provided +# for redundancy. If the `.secrets` folder is not needed, you may delete it +# from the project. + +* +!.gitignore diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..34cc0c3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + + Copyright 2024 Josh Lloyd + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index a612eb7..0549dc1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,131 @@ # tap-clari -Singer Tap for Clari + +`tap-clari` is a Singer tap for Clari. + +Built with the [Meltano Tap SDK](https://sdk.meltano.com) for Singer Taps. + + + +## Configuration + +### Accepted Config Options + + + +A full list of supported settings and capabilities for this +tap is available by running: + +```bash +tap-clari --about +``` + +### Configure using environment variables + +This Singer tap will automatically import any environment variables within the working directory's +`.env` if the `--config=ENV` is provided, such that config values will be considered if a matching +environment variable is set either in the terminal context or in the `.env` file. + +### Source Authentication and Authorization + + + +## Usage + +You can easily run `tap-clari` by itself or in a pipeline using [Meltano](https://meltano.com/). + +### Executing the Tap Directly + +```bash +tap-clari --version +tap-clari --help +tap-clari --config CONFIG --discover > ./catalog.json +``` + +## Developer Resources + +Follow these instructions to contribute to this project. + +### Initialize your Development Environment + +```bash +pipx install poetry +poetry install +``` + +### Create and Run Tests + +Create tests within the `tests` subfolder and + then run: + +```bash +poetry run pytest +``` + +You can also test the `tap-clari` CLI interface directly using `poetry run`: + +```bash +poetry run tap-clari --help +``` + +### Testing with [Meltano](https://www.meltano.com) + +_**Note:** This tap will work in any Singer environment and does not require Meltano. +Examples here are for convenience and to streamline end-to-end orchestration scenarios._ + + + +Next, install Meltano (if you haven't already) and any needed plugins: + +```bash +# Install meltano +pipx install meltano +# Initialize meltano within this directory +cd tap-clari +meltano install +``` + +Now you can test and orchestrate using Meltano: + +```bash +# Test invocation: +meltano invoke tap-clari --version +# OR run a test `elt` pipeline: +meltano elt tap-clari target-jsonl +``` + +### SDK Dev Guide + +See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html) for more instructions on how to use the SDK to +develop your own taps and targets. diff --git a/meltano.yml b/meltano.yml new file mode 100644 index 0000000..be9acc5 --- /dev/null +++ b/meltano.yml @@ -0,0 +1,30 @@ +version: 1 +send_anonymous_usage_stats: true +project_id: "tap-clari" +default_environment: test +environments: +- name: test +plugins: + extractors: + - name: "tap-clari" + namespace: "tap_clari" + pip_url: -e . + capabilities: + - state + - catalog + - discover + - about + - stream-maps + config: + start_date: '2010-01-01T00:00:00Z' + settings: + # TODO: To configure using Meltano, declare settings and their types here: + - name: username + - name: password + kind: password + - name: start_date + value: '2010-01-01T00:00:00Z' + loaders: + - name: target-jsonl + variant: andyh1203 + pip_url: target-jsonl diff --git a/output/.gitignore b/output/.gitignore new file mode 100644 index 0000000..80ff9d2 --- /dev/null +++ b/output/.gitignore @@ -0,0 +1,4 @@ +# This directory is used as a target by target-jsonl, so ignore all files + +* +!.gitignore diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a211400 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +[tool.poetry] +name = "acquia-tap-clari" +version = "0.0.1" +description = "`tap-clari` is a Singer tap for Clari, built with the Meltano Singer SDK." +readme = "README.md" +authors = ["Josh Lloyd "] +keywords = [ + "ELT", + "Clari", +] +classifiers = [ + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +license = "Apache-2.0" +packages = [ + { include = "tap_clari" }, +] + +[tool.poetry.dependencies] +python = ">=3.8" +importlib-resources = { version = "==6.1.*", python = "<3.9" } +singer-sdk = { version="~=0.36.0" } +fs-s3fs = { version = "~=1.1.1", optional = true } +requests = "~=2.31.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=7.4.0" +singer-sdk = { version="~=0.36.0", extras = ["testing"] } + +[tool.poetry.extras] +s3 = ["fs-s3fs"] + +[tool.mypy] +python_version = "3.11" +warn_unused_configs = true + +[tool.ruff] +src = ["tap_clari"] +target-version = "py38" + +[tool.ruff.lint] +ignore = [ + "ANN101", # missing-type-self + "ANN102", # missing-type-cls + "COM812", # missing-trailing-comma + "ISC001", # single-line-implicit-string-concatenation +] +select = ["ALL"] + +[tool.ruff.lint.flake8-annotations] +allow-star-arg-any = true + +[tool.ruff.lint.isort] +known-first-party = ["tap_clari"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[build-system] +requires = ["poetry-core==1.8.1"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +# CLI declaration +tap-clari = 'tap_clari.tap:TapClari.cli' diff --git a/tap_clari/__init__.py b/tap_clari/__init__.py new file mode 100644 index 0000000..7efa7b8 --- /dev/null +++ b/tap_clari/__init__.py @@ -0,0 +1 @@ +"""Tap for Clari.""" diff --git a/tap_clari/__main__.py b/tap_clari/__main__.py new file mode 100644 index 0000000..4f36e16 --- /dev/null +++ b/tap_clari/__main__.py @@ -0,0 +1,7 @@ +"""Clari entry point.""" + +from __future__ import annotations + +from tap_clari.tap import TapClari + +TapClari.cli() diff --git a/tap_clari/client.py b/tap_clari/client.py new file mode 100644 index 0000000..083d54f --- /dev/null +++ b/tap_clari/client.py @@ -0,0 +1,150 @@ +"""REST client handling, including ClariStream base class.""" + +from __future__ import annotations + +import sys +from typing import Any, Callable, Iterable + +import requests +from singer_sdk.authenticators import APIKeyAuthenticator +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 +from singer_sdk.streams import RESTStream + +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + +_Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest] + +# TODO: Delete this is if not using json files for schema definition +SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas" + + +class ClariStream(RESTStream): + """Clari stream class.""" + + @property + def url_base(self) -> str: + """Return the API URL root, configurable via tap settings.""" + # TODO: hardcode a value here, or retrieve it from self.config + return "https://api.mysample.com" + + records_jsonpath = "$[*]" # Or override `parse_response`. + + # Set this value or override `get_new_paginator`. + next_page_token_jsonpath = "$.next_page" # noqa: S105 + + @property + def authenticator(self) -> APIKeyAuthenticator: + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ + return APIKeyAuthenticator.create_for_stream( + self, + key="x-api-key", + value=self.config.get("auth_token", ""), + location="header", + ) + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Returns: + A dictionary of HTTP headers. + """ + headers = {} + if "user_agent" in self.config: + headers["User-Agent"] = self.config.get("user_agent") + # If not using an authenticator, you may also provide inline auth headers: + # headers["Private-Token"] = self.config.get("auth_token") # noqa: ERA001 + return headers + + def get_new_paginator(self) -> BaseAPIPaginator: + """Create a new pagination helper instance. + + If the source API can make use of the `next_page_token_jsonpath` + attribute, or it contains a `X-Next-Page` header in the response + then you can remove this method. + + If you need custom pagination that uses page numbers, "next" links, or + other approaches, please read the guide: https://sdk.meltano.com/en/v0.25.0/guides/pagination-classes.html. + + Returns: + A pagination helper instance. + """ + return super().get_new_paginator() + + def get_url_params( + self, + context: dict | None, # noqa: ARG002 + next_page_token: Any | None, # noqa: ANN401 + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization. + + Args: + context: The stream context. + next_page_token: The next page index or value. + + Returns: + A dictionary of URL query parameters. + """ + params: dict = {} + if next_page_token: + params["page"] = next_page_token + if self.replication_key: + params["sort"] = "asc" + params["order_by"] = self.replication_key + return params + + def prepare_request_payload( + self, + context: dict | None, # noqa: ARG002 + next_page_token: Any | None, # noqa: ARG002, ANN401 + ) -> dict | None: + """Prepare the data payload for the REST API request. + + By default, no payload will be sent (return None). + + Args: + context: The stream context. + next_page_token: The next page index or value. + + Returns: + A dictionary with the JSON body for a POST requests. + """ + # TODO: Delete this method if no payload is required. (Most REST APIs.) + return None + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result records. + + Args: + response: The HTTP ``requests.Response`` object. + + Yields: + Each record from the source. + """ + # TODO: Parse response body and return a set of records. + yield from extract_jsonpath(self.records_jsonpath, input=response.json()) + + def post_process( + self, + row: dict, + context: dict | None = None, # noqa: ARG002 + ) -> dict | None: + """As needed, append or transform raw data to match expected structure. + + Args: + row: An individual record from the stream. + context: The stream context. + + Returns: + The updated record dictionary, or ``None`` to skip the record. + """ + # TODO: Delete this method if not needed. + return row diff --git a/tap_clari/schemas/__init__.py b/tap_clari/schemas/__init__.py new file mode 100644 index 0000000..06c0a19 --- /dev/null +++ b/tap_clari/schemas/__init__.py @@ -0,0 +1 @@ +"""JSON schema files for the REST API.""" diff --git a/tap_clari/streams.py b/tap_clari/streams.py new file mode 100644 index 0000000..7980f6a --- /dev/null +++ b/tap_clari/streams.py @@ -0,0 +1,72 @@ +"""Stream type classes for tap-clari.""" + +from __future__ import annotations + +import sys +import typing as t + +from singer_sdk import typing as th # JSON Schema typing helpers + +from tap_clari.client import ClariStream + +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + + +# TODO: Delete this is if not using json files for schema definition +SCHEMAS_DIR = importlib_resources.files(__package__) / "schemas" +# TODO: - Override `UsersStream` and `GroupsStream` with your own stream definition. +# - Copy-paste as many times as needed to create multiple stream types. + + +class UsersStream(ClariStream): + """Define custom stream.""" + + name = "users" + path = "/users" + primary_keys: t.ClassVar[list[str]] = ["id"] + replication_key = None + # Optionally, you may also use `schema_filepath` in place of `schema`: + # schema_filepath = SCHEMAS_DIR / "users.json" # noqa: ERA001 + schema = th.PropertiesList( + th.Property("name", th.StringType), + th.Property( + "id", + th.StringType, + description="The user's system ID", + ), + th.Property( + "age", + th.IntegerType, + description="The user's age in years", + ), + th.Property( + "email", + th.StringType, + description="The user's email address", + ), + th.Property("street", th.StringType), + th.Property("city", th.StringType), + th.Property( + "state", + th.StringType, + description="State name in ISO 3166-2 format", + ), + th.Property("zip", th.StringType), + ).to_dict() + + +class GroupsStream(ClariStream): + """Define custom stream.""" + + name = "groups" + path = "/groups" + primary_keys: t.ClassVar[list[str]] = ["id"] + replication_key = "modified" + schema = th.PropertiesList( + th.Property("name", th.StringType), + th.Property("id", th.StringType), + th.Property("modified", th.DateTimeType), + ).to_dict() diff --git a/tap_clari/tap.py b/tap_clari/tap.py new file mode 100644 index 0000000..31fe6e6 --- /dev/null +++ b/tap_clari/tap.py @@ -0,0 +1,58 @@ +"""Clari tap class.""" + +from __future__ import annotations + +from singer_sdk import Tap +from singer_sdk import typing as th # JSON schema typing helpers + +# TODO: Import your custom stream types here: +from tap_clari import streams + + +class TapClari(Tap): + """Clari tap class.""" + + name = "tap-clari" + + # TODO: Update this section with the actual config values you expect: + config_jsonschema = th.PropertiesList( + th.Property( + "auth_token", + th.StringType, + required=True, + secret=True, # Flag config as protected. + description="The token to authenticate against the API service", + ), + th.Property( + "project_ids", + th.ArrayType(th.StringType), + required=True, + description="Project IDs to replicate", + ), + th.Property( + "start_date", + th.DateTimeType, + description="The earliest record date to sync", + ), + th.Property( + "api_url", + th.StringType, + default="https://api.mysample.com", + description="The url for the API service", + ), + ).to_dict() + + def discover_streams(self) -> list[streams.ClariStream]: + """Return a list of discovered streams. + + Returns: + A list of discovered streams. + """ + return [ + streams.GroupsStream(self), + streams.UsersStream(self), + ] + + +if __name__ == "__main__": + TapClari.cli() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..a1b65ed --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for tap-clari.""" diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..494881d --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,22 @@ +"""Tests standard tap features using the built-in SDK tests library.""" + +import datetime + +from singer_sdk.testing import get_tap_test_class + +from tap_clari.tap import TapClari + +SAMPLE_CONFIG = { + "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + # TODO: Initialize minimal tap config +} + + +# Run standard built-in tap tests from the SDK: +TestTapClari = get_tap_test_class( + tap_class=TapClari, + config=SAMPLE_CONFIG, +) + + +# TODO: Create additional tests as appropriate for your tap. diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..6be1c11 --- /dev/null +++ b/tox.ini @@ -0,0 +1,19 @@ +# This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy + +[tox] +envlist = py{38,39,310,311,312} +isolated_build = true + +[testenv] +allowlist_externals = poetry +commands = + poetry install -v + poetry run pytest + +[testenv:pytest] +# Run the python tests. +# To execute, run `tox -e pytest` +envlist = py{38,39,310,311,312} +commands = + poetry install -v + poetry run pytest