diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0b4d752..b63374b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,5 +34,6 @@ repos: rev: v1.4.1 hooks: - id: mypy + exclude: "dev/.*\\.py" additional_dependencies: - types-requests diff --git a/dev/infer-schema.py b/dev/infer-schema.py new file mode 100644 index 0000000..8e52403 --- /dev/null +++ b/dev/infer-schema.py @@ -0,0 +1,114 @@ +#! /usr/bin/env python +""" Natively infer the top level schema from of an Inventio stream + + Inventio records are commonly flat objects with string values. + Any item which appears to only be null is assumed to possibly + be a string +""" + +from __future__ import annotations + +import sys +import json +import argparse +from collections import defaultdict + + +def log(*args, **kwargs): + print(*args, **kwargs, file=sys.stderr) + + +def serialize(obj): + if isinstance(obj, set): + return serialize(list(obj)) + + if obj == ["null"]: + # null by itself is not useful, add str by default + return ["null", "string"] + + return obj + + +def get_record(line: str, *, is_singer_format=False) -> dict | None: + try: + record = json.loads(line) + + if not isinstance(record, dict): + log(f"row doesn't look like a record: {line!r}") + return None + + if not is_singer_format: + return record + + if record.get("type") == "RECORD": + return record.get("record") + + except json.JSONDecodeError as err: + log(f"failed to pass {line!r}, error: {err}") + + return None + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "-p", "--pretty", action="store_true", help="pretty print schema" + ) + parser.add_argument("-r", "--required", nargs="+", help="required keys") + parser.add_argument( + "-s", + "--singer-style", + action="store_true", + help="are records coming straight from the tap? extract only record content", + ) + + args = parser.parse_args() + + properties = defaultdict(lambda: {"type": {"null"}}) + + schema = {"type": "object", "properties": properties} + + for line in sys.stdin.read().split("\n"): + if line: + record = get_record(line, is_singer_format=args.singer_style) + + if not record: + continue + + else: + for key, value in record.items(): + if isinstance(value, dict): + _type = "object" + + elif isinstance(value, list): + _type = "array" + + elif isinstance(value, (int, float)): + _type = "number" + + elif isinstance(value, bool): + _type = "bool" + + elif value is None: + _type = "null" + + else: # string is the default type + _type = "string" + + schema["properties"][key]["type"].add(_type) + + if args.required: + schema["required"] = {"company_name"} | set(args.required) + for key in schema["required"]: + if key not in schema["properties"]: + raise ValueError( + f"required property {key!r} was not found in object keys {list(schema['properties'])}" + ) + + print(json.dumps(schema, default=serialize, indent=(2 if args.pretty else None))) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dev/inventio_get.py b/dev/inventio_get.py new file mode 100644 index 0000000..7ca7a86 --- /dev/null +++ b/dev/inventio_get.py @@ -0,0 +1,80 @@ +#! /usr/bin/env python +"""Get response from Inventio API and format as json.""" + +from __future__ import annotations + +import argparse +import json +import sys + +import requests +import xmltodict + + +def log(*args, **kwargs): + print(*args, **kwargs, file=sys.stderr) + + +def get(url) -> dict: + return xmltodict.parse(requests.get(url).content) + + +def main(argv: str | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--url", help="complete url query") + parser.add_argument("-c", "--company", help="company") + parser.add_argument("-t", "--type", help="the 'type' of endpoint") + parser.add_argument("-k", "--token", help="endpoint token") + parser.add_argument( + "-p", + "--pretty", + action="store_true", + help="pretty format as indented json", + ) + parser.add_argument( + "-l", + "--limit", + type=int, + help="limit the response from the API", + ) + + args = parser.parse_args(argv) + exit_code = 0 + + if not bool(args.url) ^ bool( + args.company and args.type and args.token, + ): # xor (^) means choose one + msg = "Too many arguments. Supply only --url, or all of --company, --type, and --token" + raise ValueError(msg) + + limit_str = f"&limit={args.limit}" if args.limit else "" + + url = ( + args.url + or f"https://app.cloud.inventio.it/{args.company}/smartapi/?type={args.type}&token={args.token}{limit_str}" + ) + log(f"getting from {url!r}") + + content_json = get(url) + + if "error" in content_json: + log(f"error: {content_json}") + exit_code = 1 + + try: + print( + json.dumps( + content_json, + **({"indent": 2, "default": str} if args.pretty else {}), + ), + ) + + except json.JSONDecodeError as e: + log(f"failed to parse result json: {e}") + exit_code = 1 + + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 03e4c89..3056b6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ ignore = [ select = ["ALL"] src = ["tap_inventio"] target-version = "py37" +exclude = ["dev"] [tool.ruff.flake8-annotations] diff --git a/tap_inventio/client.py b/tap_inventio/client.py index 9e21f80..c92e111 100644 --- a/tap_inventio/client.py +++ b/tap_inventio/client.py @@ -92,7 +92,6 @@ class InventioStream(RESTStream): # path is required by design of the RESTStream. it is not used path = None - records_jsonpath = "$.entries.entry[*]" # .entries.entry[*] _current_company_name: str | None = None @@ -249,6 +248,10 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: self.path = self.name # So the endpoint will be printed in the error raise FatalAPIError(self.response_error_message(response)) + if not self.records_jsonpath: + msg = "'records_jsonpath' must be specified" + raise NotImplementedError(msg) + yield from extract_jsonpath(self.records_jsonpath, input=json_response) def post_process( diff --git a/tap_inventio/schemas/Customer.schema.json b/tap_inventio/schemas/Customer.schema.json new file mode 100644 index 0000000..dccf90b --- /dev/null +++ b/tap_inventio/schemas/Customer.schema.json @@ -0,0 +1,183 @@ +{ + "type": "object", + "properties": { + "no": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "address": { + "type": [ + "null", + "string" + ] + }, + "address-2": { + "type": [ + "null", + "string" + ] + }, + "post-code": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "country-region-code": { + "type": [ + "null", + "string" + ] + }, + "contact": { + "type": [ + "null", + "string" + ] + }, + "e-mail": { + "type": [ + "null", + "string" + ] + }, + "phone-no": { + "type": [ + "null", + "string" + ] + }, + "currency-code": { + "type": [ + "null", + "string" + ] + }, + "business-posting-group": { + "type": [ + "null", + "string" + ] + }, + "vat-business-posting-group": { + "type": [ + "null", + "string" + ] + }, + "posting-group": { + "type": [ + "null", + "string" + ] + }, + "vat-registration-no": { + "type": [ + "null", + "string" + ] + }, + "blocked": { + "type": [ + "null", + "string" + ] + }, + "blocked-status": { + "type": [ + "null", + "string" + ] + }, + "payment-terms-code": { + "type": [ + "null", + "string" + ] + }, + "bill-to-customer-no": { + "type": [ + "null", + "string" + ] + }, + "customer-price-group": { + "type": [ + "null", + "string" + ] + }, + "language-code": { + "type": [ + "null", + "string" + ] + }, + "payment-method-code": { + "type": [ + "null", + "string" + ] + }, + "customer-disc-group": { + "type": [ + "null", + "string" + ] + }, + "salesperson-code": { + "type": [ + "null", + "string" + ] + }, + "gln": { + "type": [ + "null", + "string" + ] + }, + "reminder-terms-code": { + "type": [ + "null", + "string" + ] + }, + "fin-charge-terms-code": { + "type": [ + "null", + "string" + ] + }, + "oioubl-profile-code": { + "type": [ + "null", + "string" + ] + }, + "company_name": { + "type": [ + "null", + "string" + ] + } + }, + "required": [ + "company_name", + "no" + ] +} diff --git a/tap_inventio/schemas/DimensionSetEntry.schema.json b/tap_inventio/schemas/DimensionSetEntry.schema.json new file mode 100644 index 0000000..1a3d7ba --- /dev/null +++ b/tap_inventio/schemas/DimensionSetEntry.schema.json @@ -0,0 +1,34 @@ +{ + "type": "object", + "properties": { + "entry-no": { + "type": [ + "string", + "null" + ] + }, + "code": { + "type": [ + "string", + "null" + ] + }, + "value-code": { + "type": [ + "string", + "null" + ] + }, + "company_name": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "code", + "company_name", + "entry-no" + ] +} diff --git a/tap_inventio/streams.py b/tap_inventio/streams.py index be649b8..c1b2aa2 100644 --- a/tap_inventio/streams.py +++ b/tap_inventio/streams.py @@ -75,12 +75,31 @@ class GLEntryStream(InventioStream): - """ GLEntry-GET Stream """ + """GLEntry-GET Stream.""" name = "GLEntry" - primary_keys = ["company_name", "entry-no"] + records_jsonpath = "$.entries.entry[*]" + primary_keys = ("company_name", "entry-no") + + +class DimensionSetEntry(InventioStream): + """DimensionSetEntry-GET Stream.""" + + name = "DimensionSetEntry" + records_jsonpath = "$.dimension-entries.dimension-entry[*]" + primary_keys = ("company_name", "entry-no", "code") + + +class Customer(InventioStream): + """Customer-GET Stream.""" + + name = "Customer" + records_jsonpath = "$.customers.customer[*]" + primary_keys = ("company_name", "no") STREAMS = [ GLEntryStream, + DimensionSetEntry, + Customer, ] diff --git a/tests/test_core.py b/tests/test_core.py index adab695..7a6d53a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,14 +1,32 @@ """Tests standard tap features using the built-in SDK tests library.""" -import datetime from singer_sdk.testing import get_tap_test_class from tap_inventio.tap import TapInventio +# Example tokens from inventio documentation SAMPLE_CONFIG = { - "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), - # TODO: Initialize minimal tap config + "endpoints": { + "GLENTRY": { + "companies": { + "20220422122248574": "{5B3C070F-BD90-4293-84BB-DCBB1E521B54}", + }, + "limit": 10, + }, + "DIMENSIONSETENTRY": { + "companies": { + "20220422122248574": "{5B3C070F-BD90-4293-84BB-DCBB1E521B54}", + }, + "limit": 1, + }, + "CUSTOMER": { + "companies": { + "20220422122248574": "{5B3C070F-BD90-4293-84BB-DCBB1E521B54}", + }, + "limit": 10, + }, + }, } @@ -17,6 +35,3 @@ tap_class=TapInventio, config=SAMPLE_CONFIG, ) - - -# TODO: Create additional tests as appropriate for your tap.