From 8cb48854443165211c7ba18adf37ae932b177a36 Mon Sep 17 00:00:00 2001 From: Hunter Kuffel Date: Thu, 22 Jun 2023 13:13:12 -0700 Subject: [PATCH] initial commit --- .gitignore | 136 +++++++++++++++++++++ .pre-commit-config.yaml | 38 ++++++ .secrets/.gitignore | 10 ++ LICENSE | 201 +++++++++++++++++++++++++++++++ README.md | 131 ++++++++++++++++++++ output/.gitignore | 4 + pyproject.toml | 55 +++++++++ schemas/commissions.schema.json | 99 ++++++++++++++++ tap_cj/__init__.py | 1 + tap_cj/client.py | 204 ++++++++++++++++++++++++++++++++ tap_cj/streams.py | 67 +++++++++++ tap_cj/tap.py | 48 ++++++++ tests/__init__.py | 1 + tests/conftest.py | 3 + tests/test_core.py | 22 ++++ tox.ini | 19 +++ 16 files changed, 1039 insertions(+) create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .secrets/.gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 output/.gitignore create mode 100644 pyproject.toml create mode 100644 schemas/commissions.schema.json create mode 100644 tap_cj/__init__.py create mode 100644 tap_cj/client.py create mode 100644 tap_cj/streams.py create mode 100644 tap_cj/tap.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_core.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..475019c --- /dev/null +++ b/.gitignore @@ -0,0 +1,136 @@ +# Secrets and internal config files +**/.secrets/* + +# Ignore meltano internal cache and sqlite systemdb + +.meltano/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..24251f4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +ci: + autofix_prs: true + autoupdate_schedule: weekly + autoupdate_commit_msg: 'chore: pre-commit autoupdate' + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-json + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + +- repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.23.0 + hooks: + - id: check-dependabot + - id: check-github-workflows + +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.269 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + +- repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.3.0 + hooks: + - id: mypy + additional_dependencies: + - types-requests diff --git a/.secrets/.gitignore b/.secrets/.gitignore new file mode 100644 index 0000000..33c6acd --- /dev/null +++ b/.secrets/.gitignore @@ -0,0 +1,10 @@ +# IMPORTANT! This folder is hidden from git - if you need to store config files or other secrets, +# make sure those are never staged for commit into your git repo. You can store them here or another +# secure location. +# +# Note: This may be redundant with the global .gitignore for, and is provided +# for redundancy. If the `.secrets` folder is not needed, you may delete it +# from the project. + +* +!.gitignore diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5642739 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 Meltano + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7864b97 --- /dev/null +++ b/README.md @@ -0,0 +1,131 @@ +# tap-cj + +`tap-cj` is a Singer tap for cj. + +Built with the [Meltano Tap SDK](https://sdk.meltano.com) for Singer Taps. + + + +## Configuration + +### Accepted Config Options + + + +A full list of supported settings and capabilities for this +tap is available by running: + +```bash +tap-cj --about +``` + +### Configure using environment variables + +This Singer tap will automatically import any environment variables within the working directory's +`.env` if the `--config=ENV` is provided, such that config values will be considered if a matching +environment variable is set either in the terminal context or in the `.env` file. + +### Source Authentication and Authorization + + + +## Usage + +You can easily run `tap-cj` by itself or in a pipeline using [Meltano](https://meltano.com/). + +### Executing the Tap Directly + +```bash +tap-cj --version +tap-cj --help +tap-cj --config CONFIG --discover > ./catalog.json +``` + +## Developer Resources + +Follow these instructions to contribute to this project. + +### Initialize your Development Environment + +```bash +pipx install poetry +poetry install +``` + +### Create and Run Tests + +Create tests within the `tests` subfolder and + then run: + +```bash +poetry run pytest +``` + +You can also test the `tap-cj` CLI interface directly using `poetry run`: + +```bash +poetry run tap-cj --help +``` + +### Testing with [Meltano](https://www.meltano.com) + +_**Note:** This tap will work in any Singer environment and does not require Meltano. +Examples here are for convenience and to streamline end-to-end orchestration scenarios._ + + + +Next, install Meltano (if you haven't already) and any needed plugins: + +```bash +# Install meltano +pipx install meltano +# Initialize meltano within this directory +cd tap-cj +meltano install +``` + +Now you can test and orchestrate using Meltano: + +```bash +# Test invocation: +meltano invoke tap-cj --version +# OR run a test `elt` pipeline: +meltano elt tap-cj target-jsonl +``` + +### SDK Dev Guide + +See the [dev guide](https://sdk.meltano.com/en/latest/dev_guide.html) for more instructions on how to use the SDK to +develop your own taps and targets. diff --git a/output/.gitignore b/output/.gitignore new file mode 100644 index 0000000..80ff9d2 --- /dev/null +++ b/output/.gitignore @@ -0,0 +1,4 @@ +# This directory is used as a target by target-jsonl, so ignore all files + +* +!.gitignore diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c6a2989 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,55 @@ +[tool.poetry] +name = "tap-cj" +version = "0.0.1" +description = "`tap-cj` is a Singer tap for cj, built with the Meltano Singer SDK." +readme = "README.md" +authors = ["Hunter Kuffel"] +keywords = [ + "ELT", + "cj", +] +license = "Apache-2.0" + +[tool.poetry.dependencies] +python = "<3.12,>=3.7.1" +singer-sdk = { version="^0.28.0" } +fs-s3fs = { version = "^1.1.1", optional = true } +requests = "^2.31.0" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.2.1" +singer-sdk = { version="^0.28.0", extras = ["testing"] } + +[tool.poetry.extras] +s3 = ["fs-s3fs"] + +[tool.mypy] +python_version = "3.9" +warn_unused_configs = true + +[tool.ruff] +ignore = [ + "ANN101", # missing-type-self + "ANN102", # missing-type-cls +] +select = ["ALL"] +src = ["tap_cj"] +target-version = "py37" + + +[tool.ruff.flake8-annotations] +allow-star-arg-any = true + +[tool.ruff.isort] +known-first-party = ["tap_cj"] + +[tool.ruff.pydocstyle] +convention = "google" + +[build-system] +requires = ["poetry-core>=1.0.8"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +# CLI declaration +tap-cj = 'tap_cj.tap:Tapcj.cli' diff --git a/schemas/commissions.schema.json b/schemas/commissions.schema.json new file mode 100644 index 0000000..587ece8 --- /dev/null +++ b/schemas/commissions.schema.json @@ -0,0 +1,99 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "actionTrackerId": { + "type": ["null", "string"] + }, + "websiteName": { + "type": ["null", "string"] + }, + "advertiserName": { + "type": ["null", "string"] + }, + "commissionId": { + "type": ["null", "string"] + }, + "actionTrackerName": { + "type": ["null", "string"] + }, + "postingDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "eventDate": { + "type": ["null", "string"] + }, + "clickDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "actionStatus": { + "type": ["null", "string"] + }, + "actionType": { + "type": ["null", "string"] + }, + "shopperId": { + "type": ["null", "string"] + }, + "publisherId": { + "type": ["null", "string"] + }, + "websiteId": { + "type": ["null", "string"] + }, + "advertiserId": { + "type": ["null", "string"] + }, + "orderDiscountUsd": { + "type": ["null", "integer", "number"] + }, + "clickReferringURL": { + "type": ["null", "string"] + }, + "pubCommissionAmountUsd": { + "type": ["null", "number"] + }, + "saleAmountUsd": { + "type": ["null", "number"] + }, + "orderId": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "items": { + "type": ["null", "array"], + "items": { + "properties": { + "totalCommissionPubCurrency": { + "type": ["null", "number"] + }, + "perItemSaleAmountPubCurrency": { + "type": ["null", "number"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "sku": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "verticalAttributes": { + "properties": { + "brand": { + "type": ["null", "string"] + }, + "itemName": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + } +} diff --git a/tap_cj/__init__.py b/tap_cj/__init__.py new file mode 100644 index 0000000..c0110fa --- /dev/null +++ b/tap_cj/__init__.py @@ -0,0 +1 @@ +"""Tap for cj.""" diff --git a/tap_cj/client.py b/tap_cj/client.py new file mode 100644 index 0000000..8e8602d --- /dev/null +++ b/tap_cj/client.py @@ -0,0 +1,204 @@ +"""GraphQL client handling, including cjStream base class.""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, Any, Iterable + +from singer_sdk.pagination import BaseAPIPaginator +from singer_sdk.streams import GraphQLStream + +if TYPE_CHECKING: + import requests + from requests import Response + + +class DayChunkPaginator(BaseAPIPaginator): + """A paginator that increments days in a date range.""" + + def __init__( + self, + start_date: str, + increment: int = 1, + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__(start_date) + self._value = datetime.strptime(start_date, "%Y-%m-%d") + self._end = datetime.today() + self._increment = increment + + @property + def end_date(self): + """Get the end pagination value. + + Returns: + End date. + """ + return self._end + + @property + def increment(self): + """Get the paginator increment. + + Returns: + Increment. + """ + return self._increment + + def get_next(self, response: Response): + return ( + self.current_value + timedelta(days=self.increment) + if self.has_more(response) + else None + ) + + def has_more(self, response: Response) -> bool: + """Checks if there are more days to process. + + Args: + response: API response object. + + Returns: + Boolean flag used to indicate if the endpoint has more pages. + """ + return self.current_value < self.end_date + + +def set_none_or_cast(value, expected_type): + if value == "" or value is None: + return None + elif not isinstance(value, expected_type): + return expected_type(value) + else: + return value + + +class CJStream(GraphQLStream): + """cj stream class.""" + + @property + def url_base(self) -> str: + """Return the API URL root, configurable via tap settings.""" + return "https://commissions.api.cj.com/query" + + @property + def http_headers(self) -> dict: + """Return the http headers needed. + + Returns: + A dictionary of HTTP headers. + """ + headers = {} + if "user_agent" in self.config: + headers["User-Agent"] = self.config.get("user_agent") + # If not using an authenticator, you may also provide inline auth headers: + headers["Authorization"] = "Bearer " + self.config.get("auth_token") + return headers + + def get_new_paginator(self) -> DayChunkPaginator: + return DayChunkPaginator(start_date=self.config.get("start_date"), increment=28) + + def get_url_params( + self, + context: dict | None, # noqa: ARG002 + next_page_token: Any | None, + ) -> dict[str, Any] | str: + params = { + "PUB_ID": self.config.get("publisher_id"), + } + date_format_str = "%Y-%m-%d" + next_page_date = datetime.strftime(next_page_token, date_format_str) + if next_page_date: + params["FROM_DATE"] = next_page_date + end_datetime = datetime.strptime( + next_page_date, + date_format_str, + ) + timedelta(days=27) + params["TO_DATE"] = datetime.strftime(end_datetime, date_format_str) + return params + + def prepare_request_payload( + self, + context: dict | None, + next_page_token: Any | None, + ) -> dict | None: + """Prepare the data payload for the GraphQL API request. + + Developers generally should generally not need to override this method. + Instead, developers set the payload by properly configuring the `query` + attribute. + + Args: + context: Stream partition or context dictionary. + next_page_token: Token, page number or any request argument to request the + next page of data. + + Returns: + Dictionary with the body to use for the request. + + Raises: + ValueError: If the `query` property is not set in the request body. + """ + params = self.get_url_params(context, next_page_token) + query = self.query + + if query is None: + msg = "Graphql `query` property not set." + raise ValueError(msg) + + if not query.lstrip().startswith("query"): + # Wrap text in "query { }" if not already wrapped + query = "query { " + query + " }" + + query = query.lstrip() + query = ( + query.replace("$PUB_ID", params["PUB_ID"]) + .replace("$FROM_DATE", params["FROM_DATE"]) + .replace("$TO_DATE", params["TO_DATE"]) + ) + request_data = { + "query": (" ".join([line.strip() for line in query.splitlines()])), + } + self.logger.debug("Attempting query:\n%s", query) + return request_data + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result records. + + Args: + response: The HTTP ``requests.Response`` object. + + Yields: + Each record from the source. + """ + resp_json = response.json() + yield from resp_json.get("data", {}).get("publisherCommissions", {}).get( + "records", + [], + ) + + def post_process( + self, + row: dict, + context: dict | None = None, # noqa: ARG002 + ) -> dict | None: + """As needed, append or transform raw data to match expected structure. + + Args: + row: An individual record from the stream. + context: The stream context. + + Returns: + The updated record dictionary, or ``None`` to skip the record. + """ + for field_tuple in [ + ("orderDiscountUsd", float), + ("pubCommissionAmountUsd", float), + ("saleAmountUsd", float), + ("totalCommissionPubCurrency", float), + ]: + field_name = field_tuple[0] + field_type = field_tuple[1] + row[field_name] = set_none_or_cast(row[field_name], field_type) + return row diff --git a/tap_cj/streams.py b/tap_cj/streams.py new file mode 100644 index 0000000..4d6b6e6 --- /dev/null +++ b/tap_cj/streams.py @@ -0,0 +1,67 @@ +"""Stream type classes for tap-cj.""" + +from __future__ import annotations + +from tap_cj.client import CJStream + + +class CommissionsStream(CJStream): + """Define custom stream.""" + + name = "commissions" + # Optionally, you may also use `schema_filepath` in place of `schema`: + schema_filepath = "schemas/commissions.schema.json" + + @property + def next_page_token(self) -> str: + """Return the API URL root, configurable via tap settings.""" + return self.config.get("start_date", "") + + @property + def query(self) -> str: + return """ + publisherCommissions( + forPublishers: ["$PUB_ID"], + sincePostingDate:"$FROM_DATET00:00:00Z", + beforePostingDate:"$TO_DATET00:00:00Z" + ){ + count + payloadComplete + records + {actionTrackerName + actionTrackerId + websiteName + advertiserName + postingDate + eventDate + commissionId + clickDate + actionStatus + actionType + shopperId + publisherId + websiteId + advertiserId + orderDiscountUsd + clickReferringURL + pubCommissionAmountUsd + saleAmountUsd + orderId + source + items + { + quantity + perItemSaleAmountPubCurrency + totalCommissionPubCurrency + sku + } + verticalAttributes + { + itemName + brand + } + + } + + } + """ diff --git a/tap_cj/tap.py b/tap_cj/tap.py new file mode 100644 index 0000000..9ed44eb --- /dev/null +++ b/tap_cj/tap.py @@ -0,0 +1,48 @@ +"""cj tap class.""" + +from __future__ import annotations + +from singer_sdk import Tap +from singer_sdk import typing as th # JSON schema typing helpers + +from tap_cj import streams + + +class Tapcj(Tap): + """cj tap class.""" + + name = "tap-cj" + + config_jsonschema = th.PropertiesList( + th.Property( + "auth_token", + th.StringType, + required=True, + secret=True, # Flag config as protected. + description="The token to authenticate against the API service", + ), + th.Property( + "start_date", + th.DateTimeType, + description="The earliest record date to sync", + ), + th.Property( + "publisher_id", + th.StringType, + description="The publisher ID to sync", + ), + ).to_dict() + + def discover_streams(self) -> list[streams.cjStream]: + """Return a list of discovered streams. + + Returns: + A list of discovered streams. + """ + return [ + streams.CommissionsStream(self), + ] + + +if __name__ == "__main__": + Tapcj.cli() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..618edb3 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for tap-cj.""" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6bb3ec2 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,3 @@ +"""Test Configuration.""" + +pytest_plugins = ("singer_sdk.testing.pytest_plugin",) diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..4c9b978 --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,22 @@ +"""Tests standard tap features using the built-in SDK tests library.""" + +import datetime + +from singer_sdk.testing import get_tap_test_class + +from tap_cj.tap import Tapcj + +SAMPLE_CONFIG = { + "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + # TODO: Initialize minimal tap config +} + + +# Run standard built-in tap tests from the SDK: +TestTapcj = get_tap_test_class( + tap_class=Tapcj, + config=SAMPLE_CONFIG, +) + + +# TODO: Create additional tests as appropriate for your tap. diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..70b9e4a --- /dev/null +++ b/tox.ini @@ -0,0 +1,19 @@ +# This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy + +[tox] +envlist = py37, py38, py39, py310, py311 +isolated_build = true + +[testenv] +allowlist_externals = poetry +commands = + poetry install -v + poetry run pytest + +[testenv:pytest] +# Run the python tests. +# To execute, run `tox -e pytest` +envlist = py37, py38, py39, py310, py311 +commands = + poetry install -v + poetry run pytest