From 2417f325c76ac6f5563e5fcadfa225c1275e95db Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Tue, 19 Mar 2024 15:45:31 +0530 Subject: [PATCH 01/18] Attempts to detect the payload path of a response --- openapi_python_client/parser/endpoints.py | 34 ++++++++++- openapi_python_client/parser/models.py | 21 +++++++ openapi_python_client/parser/paths.py | 54 ++++++++++++++++- openapi_python_client/parser/responses.py | 71 ++++++++++++++++++++++- tests/parser/test_parser.py | 39 ++++++++++++- 5 files changed, 211 insertions(+), 8 deletions(-) diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index e391f9e4a..7f4201380 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -2,7 +2,8 @@ from typing import Optional, Literal, cast, Union, List, Dict, Any, Iterable, Tuple, Set -from dataclasses import dataclass +from dataclasses import dataclass, field + import openapi_schema_pydantic as osp @@ -111,6 +112,7 @@ class Response: raw_schema: osp.Response content_schema: Optional[SchemaWrapper] list_property: Optional[DataPropertyPath] = None + payload: Optional[DataPropertyPath] = None @property def has_content(self) -> bool: @@ -147,8 +149,23 @@ def from_reference( if (content_type == "application/json" or content_type.endswith("+json")) and media_type.media_type_schema: content_schema = SchemaWrapper.from_reference(media_type.media_type_schema, context) + payload_schema: Optional[SchemaWrapper] = content_schema + payload: Optional[DataPropertyPath] = None + if payload_schema: + payload_path = [] + while len(payload_schema.properties) == 1 and payload_schema.properties[0].is_object: + # Schema contains only a single object property. The payload is inside + prop_obj = payload_schema.properties[0] + payload_path.append(prop_obj.name) + payload_schema = prop_obj.schema + payload = DataPropertyPath(tuple(payload_path), payload_schema) + return cls( - status_code=status_code, description=description, raw_schema=raw_schema, content_schema=content_schema + status_code=status_code, + description=description, + raw_schema=raw_schema, + content_schema=content_schema, + payload=payload, ) @@ -167,7 +184,8 @@ class Endpoint: python_name: PythonIdentifier credentials: Optional[CredentialsProperty] - parent: Optional["Endpoint"] = None + _parent: Optional["Endpoint"] = None + children: List["Endpoint"] = field(default_factory=list) summary: Optional[str] = None description: Optional[str] = None @@ -192,6 +210,16 @@ def to_docstring(self) -> str: lines = [self.path_summary, self.summary, self.path_description, self.description] return "\n".join(line for line in lines if line) + @property + def parent(self) -> Optional["Endpoint"]: + return self._parent + + @parent.setter + def parent(self, value: Optional["Endpoint"]) -> None: + self._parent = value + if value: + value.children.append(self) + @property def path_parameters(self) -> Dict[str, Parameter]: return {p.name: p for p in self.parameters.values() if p.location == "path"} diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index 2fe87d2d1..c457ce260 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -68,6 +68,12 @@ def __getitem__(self, item: str) -> "Property": except StopIteration: raise KeyError(f"No property with name {item} in {self.name}") + def __contains__(self, item: str) -> bool: + return any(prop.name == item for prop in self.properties) + + def __iter__(self) -> Iterable["str"]: + return (prop.name for prop in self.properties) + @property def has_properties(self) -> bool: return bool(self.properties or self.any_of or self.all_of) @@ -283,6 +289,21 @@ def __init__(self) -> None: self.all_properties: Dict[Tuple[str, ...], SchemaWrapper] = {} self.required_properties: List[Tuple[str, ...]] = [] # Paths of required properties + def __getitem__(self, item: Tuple[str, ...]) -> SchemaWrapper: + return self.all_properties[item] + + def __contains__(self, item: Tuple[str, ...]) -> bool: + return item in self.all_properties + + def __iter__(self) -> Iterable[Tuple[str, ...]]: + return iter(self.all_properties.keys()) + + def __len__(self) -> int: + return len(self.all_properties) + + def __bool__(self) -> bool: + return bool(self.all_properties) + def _is_optional(self, path: Tuple[str, ...]) -> bool: """Check whether the property itself or any of its parents is nullable""" check_path = list(path) diff --git a/openapi_python_client/parser/paths.py b/openapi_python_client/parser/paths.py index 54590024e..01464c375 100644 --- a/openapi_python_client/parser/paths.py +++ b/openapi_python_client/parser/paths.py @@ -1,4 +1,6 @@ -from typing import Iterable, Dict +from collections import defaultdict + +from typing import Iterable, Dict, Tuple, Sequence import os.path @@ -28,3 +30,53 @@ def table_names_from_paths(paths: Iterable[str]) -> Dict[str, str]: split_paths = [[p for p in path.split("/") if p and not p.startswith("{")] for path in norm_paths] return {key: "_".join(value) for key, value in zip(paths, split_paths)} + + +def find_common_prefix(paths: Iterable[Tuple[str, ...]]) -> Tuple[str, ...]: + paths = list(paths) + if not paths: + return () + + common_prefix = list(paths[0]) + + for path in paths[1:]: + # Compare the current common prefix with the next path + # Truncate the common prefix or keep it as is + common_prefix = [ + common_prefix[i] for i in range(min(len(common_prefix), len(path))) if common_prefix[i] == path[i] + ] + + return tuple(common_prefix) + + +def find_longest_common_prefix(paths: Iterable[Tuple[str, ...]]) -> Tuple[str, ...]: + """Given a list of path tuples, return the longest prefix + that is common to all of them. This may be the root path which is simply + an empty tuple. + + For example: + >>> find_longest_common_prefix([("a", "b", "c"), ("a", "b", "d"), ("a", "b"), ("a", "b", "c", "d")]) + ('a', 'b') + + >>> find_longest_common_prefix([("a", "b", "c"), ("k", "b"), ("a", "b"), ("a", "b", "c", "d")]) + () + + >>> find_longest_common_prefix(("a",), ("a", "b"), ("a", "b", "c"), ("a", "b", "d")) + ("a", "b") + """ + paths = set(paths) + + if not paths: + return () + + prefix = find_common_prefix(paths) + while True: + # Do multiple passes to find the most nested prefix + paths.discard(prefix) + longer_prefix = find_common_prefix(paths) + + if not longer_prefix or longer_prefix == prefix: + break + prefix = longer_prefix + + return prefix diff --git a/openapi_python_client/parser/responses.py b/openapi_python_client/parser/responses.py index 0b5d214f6..5eed27d2c 100644 --- a/openapi_python_client/parser/responses.py +++ b/openapi_python_client/parser/responses.py @@ -1,11 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING, Dict, Tuple if TYPE_CHECKING: from openapi_python_client.parser.endpoints import EndpointCollection, Endpoint, Response from openapi_python_client.parser.models import DataPropertyPath from openapi_python_client.utils import count_by_length +from openapi_python_client.parser.paths import find_longest_common_prefix def process_responses(endpoint_collection: "EndpointCollection") -> None: @@ -19,6 +20,74 @@ def process_responses(endpoint_collection: "EndpointCollection") -> None: table_ranks[endpoint.table_name] = max(table_ranks.get(endpoint.table_name, 0), len(unique_models)) for endpoint in all_endpoints: endpoint.rank = table_ranks[endpoint.table_name] + for endpoint in all_endpoints: + find_payload(endpoint.data_response, endpoint, endpoint_collection) + + +def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointCollection) -> None: + schema = response.content_schema + if not response.payload: + return None # TODO: response has no schema, is endpoint ignored? + schema = response.payload.prop + root_path = response.payload.path + + # response_props = set(response.content_schema.crawlepd_properties.all_properties) + response_props = set(schema.crawled_properties.all_properties) + payload_props = set(response_props) + + for other in endpoints.all_endpoints_to_render: + if other.path == endpoint.path: + continue + other_payload = other.data_response.payload + if not other_payload: + continue + other_schema = other_payload.prop + other_props = set(other_schema.crawled_properties) # type: ignore[call-overload] + + # Remove all common props from the payload, assume most of those are metadata + common_props = response_props & other_props + # remaining_props = response_props - common_props + # remaining props including their ancestry starting at top level parents + new_props = payload_props - common_props + # # Check if new props contain any object type props + # has_object_props = any( + # prop_path in schema.crawled_properties.object_properties + # or prop_path in schema.crawled_properties.list_properties + # for prop_path in new_props + # ) + # if len(new_props) == 1: + # prop_obj = schema.crawled_properties[list(new_props)[0]] + # if not prop_obj.is_object and not prop_obj.is_list: + # new_props.add(()) + + if not new_props: + # Don't remove all props + continue + payload_props = new_props + + payload_path: Tuple[str, ...] = () + + if len(payload_props) == 1: + # When there's only one remaining prop it can mean the payload is just + # very similar to another endpoint. + prop_path = list(payload_props)[0] + if len(prop_path) > 0: + payload_schema = schema.crawled_properties[prop_path] + if not payload_schema.is_object and not payload_schema.is_list: + prop_path = tuple(list(prop_path)[:-1]) + payload_path = prop_path # type: ignore[assignment] + + if not payload_path: + # Payload path is the deepest nested parent of all remaining props + payload_path = find_longest_common_prefix(list(payload_props)) + + payload_schema = schema.crawled_properties[payload_path] + ret = DataPropertyPath(root_path + payload_path, payload_schema) + print(endpoint.path) + print(ret.path) + print(ret.prop.name) + print("---") + response.payload = ret def _process_response_list( diff --git a/tests/parser/test_parser.py b/tests/parser/test_parser.py index 78d772204..f6fe815ae 100644 --- a/tests/parser/test_parser.py +++ b/tests/parser/test_parser.py @@ -1,15 +1,21 @@ +import pytest + from openapi_python_client.parser.openapi_parser import OpenapiParser from openapi_schema_pydantic import Reference from tests.cases import case_path -def test_openapi_parser() -> None: +@pytest.fixture(scope="module") +def spotify_parser() -> OpenapiParser: + """Re-use parsed spec to save time""" parser = OpenapiParser(case_path("spotify.json")) - parser.parse() + return parser + - endpoints = parser.endpoints.endpoints_by_path +def test_new_releases_list_property(spotify_parser: OpenapiParser) -> None: + endpoints = spotify_parser.endpoints.endpoints_by_path endpoint = endpoints["/browse/new-releases"] @@ -26,3 +32,30 @@ def test_openapi_parser() -> None: assert "id" in prop_names assert "name" in prop_names assert "release_date" in prop_names + + +def test_spotify_single_item_endpoints(spotify_parser: OpenapiParser) -> None: + endpoint = spotify_parser.endpoints.endpoints_by_path["/albums/{id}"] + + assert endpoint.is_transformer + schema = endpoint.data_response.content_schema + + +def test_extract_payload(spotify_parser: OpenapiParser) -> None: + endpoints = spotify_parser.endpoints + pl_tr_endpoint = endpoints.endpoints_by_path["/playlists/{playlist_id}/tracks"] + new_releases_endpoint = endpoints.endpoints_by_path["/browse/new-releases"] + saved_tracks_endpoint = endpoints.endpoints_by_path["/me/tracks"] + related_artists_endpoint = endpoints.endpoints_by_path["/artists/{id}/related-artists"] + + assert new_releases_endpoint.data_response.payload.path == ("albums", "items", "[*]") + assert new_releases_endpoint.data_response.payload.prop.name == "SimplifiedAlbumObject" + + assert pl_tr_endpoint.data_response.payload.path == ("items", "[*]") + assert pl_tr_endpoint.data_response.payload.prop.name == "PlaylistTrackObject" + + assert saved_tracks_endpoint.data_response.payload.path == ("items", "[*]") + assert saved_tracks_endpoint.data_response.payload.prop.name == "SavedTrackObject" + + assert related_artists_endpoint.data_response.payload.path == ("artists", "[*]") + assert related_artists_endpoint.data_response.payload.prop.name == "ArtistObject" From 0c3767232dc64e16517d0585e59bdaccf7ede18e Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Wed, 20 Mar 2024 17:38:39 +0530 Subject: [PATCH 02/18] Payload detect hacks --- openapi_python_client/parser/endpoints.py | 46 +++++++------ openapi_python_client/parser/models.py | 32 ++++++++- openapi_python_client/parser/paths.py | 24 +++++-- openapi_python_client/parser/responses.py | 84 +++++++++++++---------- tests/parser/test_paths.py | 32 +++++++-- 5 files changed, 149 insertions(+), 69 deletions(-) diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index 7f4201380..df06bfb6f 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -210,6 +210,17 @@ def to_docstring(self) -> str: lines = [self.path_summary, self.summary, self.path_description, self.description] return "\n".join(line for line in lines if line) + @property + def payload(self) -> Optional[DataPropertyPath]: + if not self.data_response: + return None + return self.data_response.payload + + @property + def is_list(self) -> bool: + payload = self.payload + return payload.is_list if payload else False + @property def parent(self) -> Optional["Endpoint"]: return self._parent @@ -285,30 +296,23 @@ def has_content(self) -> bool: @property def table_name(self) -> str: - # TODO: - # 1. Media schema ref name - # 2. Media schema title property - # 3. Endpoint title or path component (e.g. first part of path that's not common with all other endpoints) name: Optional[str] = None - if self.data_response: - if self.list_property: - name = self.list_property.prop.name - else: - name = self.data_response.content_schema.name + if self.payload: + name = self.payload.name if name: return name return self.path_table_name - @property - def list_property(self) -> Optional[DataPropertyPath]: - if not self.data_response: - return None - return self.data_response.list_property + # @property + # def list_property(self) -> Optional[DataPropertyPath]: + # if not self.data_response: + # return None + # return self.data_response.list_property @property def data_json_path(self) -> str: - list_prop = self.list_property - return list_prop.json_path if list_prop else "" + payload = self.payload + return payload.json_path if payload else "" @property def is_transformer(self) -> bool: @@ -318,7 +322,7 @@ def is_transformer(self) -> bool: def transformer(self) -> Optional[TransformerSetting]: if not self.parent: return None - if not self.parent.list_property: + if not self.parent.is_list: return None if not self.path_parameters: return None @@ -326,7 +330,9 @@ def transformer(self) -> Optional[TransformerSetting]: # TODO: Can't handle endpoints with more than 1 path param for now return None path_param = list(self.path_parameters.values())[-1] - list_object = self.parent.list_property.prop + payload = self.parent.payload + assert payload + list_object = payload.prop transformer_arg = list_object.crawled_properties.find_property_by_name(path_param.name, fallback="id") if not transformer_arg: return None @@ -411,7 +417,7 @@ def endpoints_by_path(self) -> Dict[str, Endpoint]: @property def root_endpoints(self) -> List[Endpoint]: - return [e for e in self.all_endpoints_to_render if e.list_property and not e.path_parameters] + return [e for e in self.all_endpoints_to_render if e.is_list and not e.path_parameters] @property def transformer_endpoints(self) -> List[Endpoint]: @@ -476,7 +482,7 @@ def find_nearest_list_parent(self, path: str) -> Optional[Endpoint]: for part in parts: current_node = current_node[part] # type: ignore[assignment] if parent_endpoint := current_node.get(""): - if cast(Endpoint, parent_endpoint).list_property: + if cast(Endpoint, parent_endpoint).is_list: return cast(Endpoint, parent_endpoint) return None diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index c457ce260..88aa960c8 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -1,5 +1,19 @@ from __future__ import annotations -from typing import Literal, TYPE_CHECKING, Optional, Union, List, TypeVar, Any, Iterable, Sequence, cast, Tuple, Dict +from typing import ( + Literal, + TYPE_CHECKING, + Optional, + Union, + List, + TypeVar, + Any, + Iterable, + Sequence, + cast, + Tuple, + Dict, + Iterator, +) from itertools import chain from dataclasses import dataclass, field @@ -26,10 +40,18 @@ class DataPropertyPath: path: Tuple[str, ...] prop: "SchemaWrapper" # TODO: Why is this not pointing to Property? + @property + def name(self) -> str: + return self.prop.name + @property def json_path(self) -> str: return ".".join(self.path) + @property + def is_list(self) -> bool: + return self.prop.is_list + def __str__(self) -> str: return f"DataPropertyPath {self.path}: {self.prop.name}" @@ -304,6 +326,14 @@ def __len__(self) -> int: def __bool__(self) -> bool: return bool(self.all_properties) + def paths_with_types(self) -> Iterator[tuple[tuple[str, ...], tuple[TSchemaType, ...]]]: + for path, schema in self.all_properties.items(): + # if schema.is_list and schema.array_item: + # # Include the array item type for full comparison + # yield path, tuple(schema.types + schema.array_item.types]) + # else: + yield path, tuple(schema.types) + def _is_optional(self, path: Tuple[str, ...]) -> bool: """Check whether the property itself or any of its parents is nullable""" check_path = list(path) diff --git a/openapi_python_client/parser/paths.py b/openapi_python_client/parser/paths.py index 01464c375..6bfdc108b 100644 --- a/openapi_python_client/parser/paths.py +++ b/openapi_python_client/parser/paths.py @@ -40,11 +40,22 @@ def find_common_prefix(paths: Iterable[Tuple[str, ...]]) -> Tuple[str, ...]: common_prefix = list(paths[0]) for path in paths[1:]: - # Compare the current common prefix with the next path - # Truncate the common prefix or keep it as is - common_prefix = [ - common_prefix[i] for i in range(min(len(common_prefix), len(path))) if common_prefix[i] == path[i] - ] + # Initialize a new prefix list for comparison results + new_prefix = [] + for i in range(min(len(common_prefix), len(path))): + if common_prefix[i] == path[i]: + new_prefix.append(common_prefix[i]) + else: + # As soon as a mismatch is found, break the loop + break + common_prefix = new_prefix + + # for path in paths[1:]: + # # Compare the current common prefix with the next path + # # Truncate the common prefix or keep it as is + # common_prefix = [ + # common_prefix[i] for i in range(min(len(common_prefix), len(path))) if common_prefix[i] == path[i] + # ] return tuple(common_prefix) @@ -63,6 +74,9 @@ def find_longest_common_prefix(paths: Iterable[Tuple[str, ...]]) -> Tuple[str, . >>> find_longest_common_prefix(("a",), ("a", "b"), ("a", "b", "c"), ("a", "b", "d")) ("a", "b") + + >>> find_longest_common_prefix({('data', '[*]', 'email', '[*]'), ('data', '[*]', 'phone', '[*]')}) + ("data", "[*]") """ paths = set(paths) diff --git a/openapi_python_client/parser/responses.py b/openapi_python_client/parser/responses.py index 5eed27d2c..4f627872e 100644 --- a/openapi_python_client/parser/responses.py +++ b/openapi_python_client/parser/responses.py @@ -14,14 +14,19 @@ def process_responses(endpoint_collection: "EndpointCollection") -> None: all_endpoints = endpoint_collection.all_endpoints_to_render table_ranks: Dict[str, int] = {} for endpoint in all_endpoints: - _process_response_list(endpoint.data_response, endpoint, endpoint_collection) - response = endpoint.data_response - unique_models = set(t.name for t in response.content_schema.crawled_properties.object_properties.values()) + find_payload(endpoint.data_response, endpoint, endpoint_collection) + for endpoint in all_endpoints: + # _process_response_list(endpoint.data_response, endpoint, endpoint_collection) + payload = endpoint.payload + if not payload: + continue + unique_models = set(t.name for t in payload.prop.crawled_properties.object_properties.values()) + # unique_models = set(t.name for t in response.content_schema.crawled_properties.object_properties.values()) table_ranks[endpoint.table_name] = max(table_ranks.get(endpoint.table_name, 0), len(unique_models)) for endpoint in all_endpoints: endpoint.rank = table_ranks[endpoint.table_name] - for endpoint in all_endpoints: - find_payload(endpoint.data_response, endpoint, endpoint_collection) + # for endpoint in all_endpoints: + # find_payload(endpoint.data_response, endpoint, endpoint_collection) def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointCollection) -> None: @@ -32,17 +37,20 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl root_path = response.payload.path # response_props = set(response.content_schema.crawlepd_properties.all_properties) - response_props = set(schema.crawled_properties.all_properties) + # response_props = set(schema.crawled_properties.all_properties) + # payload_props = set(response_props) + response_props = set(schema.crawled_properties.paths_with_types()) payload_props = set(response_props) - for other in endpoints.all_endpoints_to_render: + for other in sorted(endpoints.all_endpoints_to_render, key=lambda ep: ep.path): if other.path == endpoint.path: continue other_payload = other.data_response.payload if not other_payload: continue other_schema = other_payload.prop - other_props = set(other_schema.crawled_properties) # type: ignore[call-overload] + # other_props = set(other_schema.crawled_properties) # type: ignore[call-overload] + other_props = set(other_schema.crawled_properties.paths_with_types()) # Remove all common props from the payload, assume most of those are metadata common_props = response_props & other_props @@ -70,7 +78,7 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl if len(payload_props) == 1: # When there's only one remaining prop it can mean the payload is just # very similar to another endpoint. - prop_path = list(payload_props)[0] + prop_path = list(payload_props)[0][0] if len(prop_path) > 0: payload_schema = schema.crawled_properties[prop_path] if not payload_schema.is_object and not payload_schema.is_list: @@ -79,7 +87,7 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl if not payload_path: # Payload path is the deepest nested parent of all remaining props - payload_path = find_longest_common_prefix(list(payload_props)) + payload_path = find_longest_common_prefix([path for path, _ in payload_props]) payload_schema = schema.crawled_properties[payload_path] ret = DataPropertyPath(root_path + payload_path, payload_schema) @@ -90,31 +98,31 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl response.payload = ret -def _process_response_list( - response: Response, - endpoint: Endpoint, - endpoints: EndpointCollection, -) -> None: - if not response.list_properties: - return - if () in response.list_properties: # Response is a top level list - response.list_property = DataPropertyPath((), response.list_properties[()]) - return - - level_counts = count_by_length(response.list_properties.keys()) - - # Get list properties max 2 levels down - props_first_levels = [ - (path, prop) for path, prop in sorted(response.list_properties.items(), key=lambda k: len(k)) if len(path) <= 2 - ] - - # If there is only one list property 1 or 2 levels down, this is the list - for path, prop in props_first_levels: - if not prop.is_object: # Only looking for object lists - continue - levels = len(path) - if level_counts[levels] == 1: - response.list_property = DataPropertyPath(path, prop) - parent = endpoints.find_immediate_parent(endpoint.path) - if parent and not parent.required_parameters: - response.list_property = None +# def _process_response_list( +# response: Response, +# endpoint: Endpoint, +# endpoints: EndpointCollection, +# ) -> None: +# if not response.list_properties: +# return +# if () in response.list_properties: # Response is a top level list +# response.list_property = DataPropertyPath((), response.list_properties[()]) +# return + +# level_counts = count_by_length(response.list_properties.keys()) + +# # Get list properties max 2 levels down +# props_first_levels = [ +# (path, prop) for path, prop in sorted(response.list_properties.items(), key=lambda k: len(k)) if len(path) <= 2 +# ] + +# # If there is only one list property 1 or 2 levels down, this is the list +# for path, prop in props_first_levels: +# if not prop.is_object: # Only looking for object lists +# continue +# levels = len(path) +# if level_counts[levels] == 1: +# response.list_property = DataPropertyPath(path, prop) +# parent = endpoints.find_immediate_parent(endpoint.path) +# if parent and not parent.required_parameters: +# response.list_property = None diff --git a/tests/parser/test_paths.py b/tests/parser/test_paths.py index 0a851d58b..572471129 100644 --- a/tests/parser/test_paths.py +++ b/tests/parser/test_paths.py @@ -1,4 +1,6 @@ -from openapi_python_client.parser.paths import table_names_from_paths +import pytest + +from openapi_python_client.parser.paths import table_names_from_paths, find_longest_common_prefix def test_table_names_from_paths_prefixed() -> None: @@ -45,7 +47,27 @@ def test_table_names_from_paths_no_prefix() -> None: } -if __name__ == "__main__": - import pytest - - pytest.main(["tests/parser", "-k", "table_names_from_paths", "--pdb"]) +@pytest.mark.parametrize( + "paths, expected", + [ + ( + [("data", "[*]", "email", "[*]"), ("data", "[*]", "phone", "[*]")], + ("data", "[*]"), + ), + ( + [("a", "b", "c"), ("a", "b", "d"), ("a", "b"), ("a", "b", "c", "d")], + ("a", "b"), + ), + ( + [("a", "b", "c"), ("k", "b"), ("a", "b"), ("a", "b", "c", "d")], + (), + ), + ( + [("a",), ("a", "b"), ("a", "b", "c"), ("a", "b", "d")], + ("a", "b"), + ), + ], +) +def test_find_longest_common_prefix(paths: list[tuple[str, ...]], expected: tuple[str, ...]) -> None: + result = find_longest_common_prefix(paths) + assert result == expected From 67ca2ff6ff81e4732b9a50d18e18f107db28f505 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Wed, 20 Mar 2024 18:47:05 +0530 Subject: [PATCH 03/18] Fix ranking, fix find transformer param, pokeapi test --- openapi_python_client/parser/endpoints.py | 11 +- openapi_python_client/parser/models.py | 26 +- openapi_python_client/parser/responses.py | 7 +- tests/cases/pokeapi.yml | 5056 +++++++++++++++++++++ tests/parser/test_parser.py | 28 +- 5 files changed, 5115 insertions(+), 13 deletions(-) create mode 100644 tests/cases/pokeapi.yml diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index df06bfb6f..1b67db795 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -3,6 +3,7 @@ from typing import Optional, Literal, cast, Union, List, Dict, Any, Iterable, Tuple, Set from dataclasses import dataclass, field +from itertools import groupby import openapi_schema_pydantic as osp @@ -395,15 +396,19 @@ class EndpointCollection: @property def all_endpoints_to_render(self) -> List[Endpoint]: # return [e for e in self.endpoints if e.has_content] - return sorted( + # Sum of endpoint ranks by table name + to_render = sorted( [ e for e in self.endpoints if (not self.names_to_render or e.python_name in self.names_to_render) and e.has_content ], - key=lambda e: e.rank, - reverse=True, + key=lambda e: e.table_name, ) + groups = groupby(to_render, key=lambda e: e.table_name) + groups = [(name, list(group)) for name, group in groups] + groups = sorted(groups, key=lambda g: max(e.rank for e in g[1]), reverse=True) + return [e for _, group in groups for e in group] @property def endpoints_by_id(self) -> Dict[str, Endpoint]: diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index 88aa960c8..bdb5903ed 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -42,6 +42,8 @@ class DataPropertyPath: @property def name(self) -> str: + if self.is_list and self.prop.array_item: + return self.prop.array_item.name return self.prop.name @property @@ -359,18 +361,34 @@ def find_property_by_name(self, name: str, fallback: Optional[str] = None) -> Op """ named = [] fallbacks = [] + named_optional = [] + fallbacks_optional = [] for path, prop in self.all_properties.items(): - if name in path and not self._is_optional(path): - named.append((path, prop)) - if fallback and fallback in path and not self._is_optional(path): - fallbacks.append((path, prop)) + if name in path: + if self._is_optional(path): + named_optional.append((path, prop)) + else: + named.append((path, prop)) + if fallback and fallback in path: + if self._is_optional(path): + fallbacks_optional.append((path, prop)) + else: + fallbacks.append((path, prop)) # Prefer the least nested path named.sort(key=lambda item: len(item[0])) fallbacks.sort(key=lambda item: len(item[0])) + named_optional.sort(key=lambda item: len(item[0])) + fallbacks_optional.sort(key=lambda item: len(item[0])) + # Prefer required property and required fallback over optional properties + # If not required props found, assume the spec is wrong and optional properties are required in practice if named: return DataPropertyPath(*named[0]) elif fallbacks: return DataPropertyPath(*fallbacks[0]) + elif named_optional: + return DataPropertyPath(*named_optional[0]) + elif fallbacks_optional: + return DataPropertyPath(*fallbacks_optional[0]) return None def crawl(self, schema: SchemaWrapper, path: Tuple[str, ...] = ()) -> None: diff --git a/openapi_python_client/parser/responses.py b/openapi_python_client/parser/responses.py index 4f627872e..99ad98ea7 100644 --- a/openapi_python_client/parser/responses.py +++ b/openapi_python_client/parser/responses.py @@ -89,8 +89,13 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl # Payload path is the deepest nested parent of all remaining props payload_path = find_longest_common_prefix([path for path, _ in payload_props]) + payload_path = root_path + payload_path + while payload_path and payload_path[-1] == "[*]": + # We want the path to point to the list property, not the list item + # so that payload is correctly detected as list + payload_path = payload_path[:-1] payload_schema = schema.crawled_properties[payload_path] - ret = DataPropertyPath(root_path + payload_path, payload_schema) + ret = DataPropertyPath(payload_path, payload_schema) print(endpoint.path) print(ret.path) print(ret.prop.name) diff --git a/tests/cases/pokeapi.yml b/tests/cases/pokeapi.yml new file mode 100644 index 000000000..9bf8f770f --- /dev/null +++ b/tests/cases/pokeapi.yml @@ -0,0 +1,5056 @@ +# Source: https://gist.github.com/NiccoMlt/073b18934a6001fc5a2414c590e3b8ba +# Credit goes to Niccolò Maltoni for building the baseline version of this specification +openapi: 3.0.0 +info: + description: '' + title: 'pokeapi' + version: '20220523' +servers: +- url: 'https://pokeapi.co/' +paths: + /api/v2/ability/: + get: + operationId: ability_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: A list of abilities + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of abilities available + next: + type: string + description: The URL for the next page of abilities (null if none) + previous: + type: string + description: The URL for the previous page of abilities (null if none) + results: + type: array + items: + $ref: '#/components/schemas/Ability' + tags: + - ability + /api/v2/ability/{id}/: + get: + operationId: ability_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this ability. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Ability' + tags: + - ability + /api/v2/berry-firmness/: + get: + operationId: berry-firmness_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + "200": + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of berry firmnesses available from this API. + example: 5 + next: + type: string + nullable: true + description: The URL for the next page of results, or null if there are no more results. + previous: + type: string + nullable: true + description: The URL for the previous page of results, or null if this is the first page. + results: + type: array + items: + $ref: '#/components/schemas/BerryFirmness' + tags: + - berry-firmness + /api/v2/berry-firmness/{id}/: + get: + operationId: berry-firmness_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this berry firmness. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/BerryFirmness' + tags: + - berry-firmness + /api/v2/berry-flavor/: + get: + operationId: berry-flavor_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of berry flavors + example: 5 + next: + type: string + description: The URL to the next page of berry flavors (if any) + example: "https://pokeapi.co/api/v2/berry-flavor?offset=5&limit=5" + previous: + type: string + description: The URL to the previous page of berry flavors (if any) + example: null + results: + type: array + items: + $ref: '#/components/schemas/BerryFlavor' + tags: + - berry-flavor + /api/v2/berry-flavor/{id}/: + get: + operationId: berry-flavor_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this berry flavor. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/BerryFlavor' + tags: + - berry-flavor + /api/v2/berry/: + get: + operationId: berry_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Berry' + tags: + - berry + /api/v2/berry/{id}/: + get: + operationId: berry_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this berry. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Berry' + tags: + - berry + /api/v2/characteristic/: + get: + operationId: characteristic_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: Successful response with a list of characteristics + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Characteristic' + tags: + - characteristic + /api/v2/characteristic/{id}/: + get: + operationId: characteristic_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this characteristic. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Characteristic' + tags: + - characteristic + /api/v2/contest-effect/: + get: + operationId: contest-effect_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + "200": + description: "A list of contest effects" + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: "The total number of contest effects" + next: + type: string + nullable: true + description: "The URL to get the next page of contest effects (if it exists)" + previous: + type: string + nullable: true + description: "The URL to get the previous page of contest effects (if it exists)" + results: + type: array + items: + $ref: "#/components/schemas/ContestEffect" + tags: + - contest-effect + /api/v2/contest-effect/{id}/: + get: + operationId: contest-effect_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this contest effect. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ContestEffect' + tags: + - contest-effect + /api/v2/contest-type/: + get: + operationId: contest-type_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: Successful response containing a list of contest types + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of contest types returned + next: + type: string + nullable: true + description: URL to the next page of contest types, if any + previous: + type: string + nullable: true + description: URL to the previous page of contest types, if any + results: + type: array + items: + $ref: '#/components/schemas/ContestType' + tags: + - contest-type + /api/v2/contest-type/{id}/: + get: + operationId: contest-type_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this contest type. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ContestType' + tags: + - contest-type + /api/v2/egg-group/: + get: + operationId: egg-group_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/EggGroup' + tags: + - egg-group + /api/v2/egg-group/{id}/: + get: + operationId: egg-group_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this egg group. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/EggGroup' + tags: + - egg-group + /api/v2/encounter-condition-value/: + get: + operationId: encounter-condition-value_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: Successful response containing a list of encounter condition values + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/EncounterConditionValue' + tags: + - encounter-condition-value + /api/v2/encounter-condition-value/{id}/: + get: + operationId: encounter-condition-value_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this encounter condition + value. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/EncounterConditionValue' + tags: + - encounter-condition-value + /api/v2/encounter-condition/: + get: + operationId: encounter-condition_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of encounter conditions. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of encounter conditions. + example: https://pokeapi.co/api/v2/encounter-condition/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of encounter conditions. + results: + type: array + items: + $ref: '#/components/schemas/EncounterCondition' + tags: + - encounter-condition + /api/v2/encounter-condition/{id}/: + get: + operationId: encounter-condition_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this encounter condition. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/EncounterCondition' + tags: + - encounter-condition + /api/v2/encounter-method/: + get: + operationId: encounter-method_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of encounter methods. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of encounter methods. + example: https://pokeapi.co/api/v2/encounter-method/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of encounter methods. + results: + type: array + items: + $ref: '#/components/schemas/EncounterMethod' + tags: + - encounter-method + /api/v2/encounter-method/{id}/: + get: + operationId: encounter-method_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this encounter method. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/EncounterMethod' + tags: + - encounter-method + /api/v2/evolution-chain/: + get: + operationId: evolution-chain_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of evolution chains. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of evolution chains. + example: https://pokeapi.co/api/v2/evolution-chain/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of evolution chains. + results: + type: array + items: + $ref: '#/components/schemas/EvolutionChain' + tags: + - evolution-chain + /api/v2/evolution-chain/{id}/: + get: + operationId: evolution-chain_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this evolution chain. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/EvolutionChain' + tags: + - evolution-chain + /api/v2/evolution-trigger/: + get: + operationId: evolution-trigger_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of evolution triggers. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of evolution triggers. + example: https://pokeapi.co/api/v2/evolution-trigger/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of evolution triggers. + results: + type: array + items: + $ref: '#/components/schemas/EvolutionTrigger' + tags: + - evolution-trigger + /api/v2/evolution-trigger/{id}/: + get: + operationId: evolution-trigger_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this evolution trigger. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/EvolutionTrigger' + tags: + - evolution-trigger + /api/v2/gender/: + get: + operationId: gender_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of genders. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of genders. + example: https://pokeapi.co/api/v2/gender/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of genders. + results: + type: array + items: + $ref: '#/components/schemas/Gender' + tags: + - gender + /api/v2/gender/{id}/: + get: + operationId: gender_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this gender. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Gender' + tags: + - gender + /api/v2/generation/: + get: + operationId: generation_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of generations. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of generations. + example: https://pokeapi.co/api/v2/generation/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of generations. + results: + type: array + items: + $ref: '#/components/schemas/Generation' + tags: + - generation + /api/v2/generation/{id}/: + get: + operationId: generation_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this generation. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Generation' + tags: + - generation + /api/v2/growth-rate/: + get: + operationId: growth-rate_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of growth rates. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of growth rates. + example: https://pokeapi.co/api/v2/growth-rate/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of growth rates. + results: + type: array + items: + $ref: '#/components/schemas/GrowthRate' + tags: + - growth-rate + /api/v2/growth-rate/{id}/: + get: + operationId: growth-rate_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this growth rate. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/GrowthRate' + tags: + - growth-rate + /api/v2/item-attribute/: + get: + operationId: item-attribute_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of item attributes. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of item attributes. + example: https://pokeapi.co/api/v2/item-attribute/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of item attributes. + results: + type: array + items: + $ref: '#/components/schemas/ItemAttribute' + tags: + - item-attribute + /api/v2/item-attribute/{id}/: + get: + operationId: item-attribute_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this item attribute. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ItemAttribute' + tags: + - item-attribute + /api/v2/item-category/: + get: + operationId: item-category_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of item categories. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of item categories. + example: https://pokeapi.co/api/v2/item-category/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of item categories. + results: + type: array + items: + $ref: '#/components/schemas/ItemCategory' + tags: + - item-category + /api/v2/item-category/{id}/: + get: + operationId: item-category_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this item category. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ItemCategory' + tags: + - item-category + /api/v2/item-fling-effect/: + get: + operationId: item-fling-effect_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of item fling effects. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of item fling effects. + example: https://pokeapi.co/api/v2/item-fling-effect/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of item fling effects. + results: + type: array + items: + $ref: '#/components/schemas/ItemFlingEffect' + tags: + - item-fling-effect + /api/v2/item-fling-effect/{id}/: + get: + operationId: item-fling-effect_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this item fling effect. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ItemFlingEffect' + tags: + - item-fling-effect + /api/v2/item-pocket/: + get: + operationId: item-pocket_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of item pockets. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of item pockets. + example: https://pokeapi.co/api/v2/item-pocket/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of item pockets. + results: + type: array + items: + $ref: '#/components/schemas/ItemPocket' + tags: + - item-pocket + /api/v2/item-pocket/{id}/: + get: + operationId: item-pocket_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this item pocket. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/ItemPocket' + tags: + - item-pocket + /api/v2/item/: + get: + operationId: item_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of items. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of items. + example: https://pokeapi.co/api/v2/item/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of items. + results: + type: array + items: + $ref: '#/components/schemas/Item' + tags: + - item + /api/v2/item/{id}/: + get: + operationId: item_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this item. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Item' + tags: + - item + /api/v2/language/: + get: + operationId: language_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of languages. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of languages. + example: https://pokeapi.co/api/v2/language/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of languages. + results: + type: array + items: + $ref: '#/components/schemas/Language' + tags: + - language + /api/v2/language/{id}/: + get: + operationId: language_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this language. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Language' + tags: + - language + /api/v2/location-area/: + get: + operationId: location-area_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of location areas. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of location areas. + example: https://pokeapi.co/api/v2/location-area/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of location areas. + results: + type: array + items: + $ref: '#/components/schemas/LocationArea' + tags: + - location-area + /api/v2/location-area/{id}/: + get: + operationId: location-area_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this location area. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/LocationArea' + tags: + - location-area + /api/v2/location/: + get: + operationId: location_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of locations. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of locations. + example: https://pokeapi.co/api/v2/location/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of locations. + results: + type: array + items: + $ref: '#/components/schemas/Location' + tags: + - location + /api/v2/location/{id}/: + get: + operationId: location_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this location. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Location' + tags: + - location + /api/v2/machine/: + get: + operationId: machine_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of machines. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of machines. + example: https://pokeapi.co/api/v2/machine/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of machines. + results: + type: array + items: + $ref: '#/components/schemas/Machine' + tags: + - machine + /api/v2/machine/{id}/: + get: + operationId: machine_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this machine. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Machine' + tags: + - machine + /api/v2/move-ailment/: + get: + operationId: move-ailment_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of move ailments. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of move ailments. + example: https://pokeapi.co/api/v2/move-ailment/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of move ailments. + results: + type: array + items: + $ref: '#/components/schemas/MoveAilment' + tags: + - move-ailment + /api/v2/move-ailment/{id}/: + get: + operationId: move-ailment_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this move meta ailment. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/MoveAilment' + tags: + - move-ailment + /api/v2/move-battle-style/: + get: + operationId: move-battle-style_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of move battle styles. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of move battle styles. + example: https://pokeapi.co/api/v2/move-battle-style/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of move battle styles. + results: + type: array + items: + $ref: '#/components/schemas/MoveBattleStyle' + tags: + - move-battle-style + /api/v2/move-battle-style/{id}/: + get: + operationId: move-battle-style_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this move battle style. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/MoveBattleStyle' + tags: + - move-battle-style + /api/v2/move-category/: + get: + operationId: move-category_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of move categories. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of move categories. + example: https://pokeapi.co/api/v2/move-category/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of move categories. + results: + type: array + items: + $ref: '#/components/schemas/MoveCategory' + tags: + - move-category + /api/v2/move-category/{id}/: + get: + operationId: move-category_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this move meta category. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/MoveCategory' + tags: + - move-category + /api/v2/move-damage-class/: + get: + operationId: move-damage-class_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of move damage classes. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of move damage classes. + example: https://pokeapi.co/api/v2/move-damage-class/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of move damage classes. + results: + type: array + items: + $ref: '#/components/schemas/MoveDamageClass' + tags: + - move-damage-class + /api/v2/move-damage-class/{id}/: + get: + operationId: move-damage-class_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this move damage class. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/MoveDamageClass' + tags: + - move-damage-class + /api/v2/move-learn-method/: + get: + operationId: move-learn-method_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of move learn methods. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of move learn methods. + example: https://pokeapi.co/api/v2/move-learn-method/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of move learn methods. + results: + type: array + items: + $ref: '#/components/schemas/MoveLearnMethod' + tags: + - move-learn-method + /api/v2/move-learn-method/{id}/: + get: + operationId: move-learn-method_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this move learn method. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/MoveLearnMethod' + tags: + - move-learn-method + /api/v2/move-target/: + get: + operationId: move-target_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of move targets. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of move targets. + example: https://pokeapi.co/api/v2/move-target/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of move targets. + results: + type: array + items: + $ref: '#/components/schemas/MoveTarget' + tags: + - move-target + /api/v2/move-target/{id}/: + get: + operationId: move-target_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this move target. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/MoveTarget' + tags: + - move-target + /api/v2/move/: + get: + operationId: move_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of moves. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of moves. + example: https://pokeapi.co/api/v2/move/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of moves. + results: + type: array + items: + $ref: '#/components/schemas/Move' + tags: + - move + /api/v2/move/{id}/: + get: + operationId: move_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this move. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Move' + tags: + - move + /api/v2/nature/: + get: + operationId: nature_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of natures. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of natures. + example: https://pokeapi.co/api/v2/nature/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of natures. + results: + type: array + items: + $ref: '#/components/schemas/Nature' + tags: + - nature + /api/v2/nature/{id}/: + get: + operationId: nature_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this nature. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Nature' + tags: + - nature + /api/v2/pal-park-area/: + get: + operationId: pal-park-area_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of pal park areas. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of pal park areas. + example: https://pokeapi.co/api/v2/pal-park-area/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of pal park areas. + results: + type: array + items: + $ref: '#/components/schemas/PalParkArea' + tags: + - pal-park-area + /api/v2/pal-park-area/{id}/: + get: + operationId: pal-park-area_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this pal park area. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/PalParkArea' + tags: + - pal-park-area + /api/v2/pokeathlon-stat/: + get: + operationId: pokeathlon-stat_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of pokeathlon stats. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of pokeathlon stats. + example: https://pokeapi.co/api/v2/pokeathlon-stat/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of pokeathlon stat. + results: + type: array + items: + $ref: '#/components/schemas/PokeathlonStat' + tags: + - pokeathlon-stat + /api/v2/pokeathlon-stat/{id}/: + get: + operationId: pokeathlon-stat_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this pokeathlon stat. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/PokeathlonStat' + tags: + - pokeathlon-stat + /api/v2/pokedex/: + get: + operationId: pokedex_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of pokedexes. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of pokedexes. + example: https://pokeapi.co/api/v2/pokedex/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of pokedexes. + results: + type: array + items: + $ref: '#/components/schemas/Pokedex' + tags: + - pokedex + /api/v2/pokedex/{id}/: + get: + operationId: pokedex_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this pokedex. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Pokedex' + tags: + - pokedex + /api/v2/pokemon-color/: + get: + operationId: pokemon-color_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of pokemon colors. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of pokemon colors. + example: https://pokeapi.co/api/v2/pokemon-color/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of pokemon colors. + results: + type: array + items: + $ref: '#/components/schemas/PokemonColor' + tags: + - pokemon-color + /api/v2/pokemon-color/{id}/: + get: + operationId: pokemon-color_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this pokemon color. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/PokemonColor' + tags: + - pokemon-color + /api/v2/pokemon-form/: + get: + operationId: pokemon-form_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of pokemon forms. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of pokemon forms. + example: https://pokeapi.co/api/v2/pokemon-form/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of pokemon forms. + results: + type: array + items: + $ref: '#/components/schemas/PokemonForm' + tags: + - pokemon-form + /api/v2/pokemon-form/{id}/: + get: + operationId: pokemon-form_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this pokemon form. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/PokemonForm' + tags: + - pokemon-form + /api/v2/pokemon-habitat/: + get: + operationId: pokemon-habitat_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of pokemon habitats. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of pokemon habitats. + example: https://pokeapi.co/api/v2/language/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of pokemon habitats. + results: + type: array + items: + $ref: '#/components/schemas/PokemonHabitat' + tags: + - pokemon-habitat + /api/v2/pokemon-habitat/{id}/: + get: + operationId: pokemon-habitat_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this pokemon habitat. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/PokemonHabitat' + tags: + - pokemon-habitat + /api/v2/pokemon-shape/: + get: + operationId: pokemon-shape_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of pokemon shapes. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of pokemon shapes. + example: https://pokeapi.co/api/v2/pokemon-shape/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of pokemon shapes. + results: + type: array + items: + $ref: '#/components/schemas/PokemonShape' + tags: + - pokemon-shape + /api/v2/pokemon-shape/{id}/: + get: + operationId: pokemon-shape_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this pokemon shape. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/PokemonShape' + tags: + - pokemon-shape + /api/v2/pokemon-species/: + get: + operationId: pokemon-species_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of pokemon species list. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of pokemon species list. + example: https://pokeapi.co/api/v2/pokemon-species/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of pokemon species list. + results: + type: array + items: + $ref: '#/components/schemas/PokemonSpecies' + tags: + - pokemon-species + /api/v2/pokemon-species/{id}/: + get: + operationId: pokemon-species_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this pokemon species. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/PokemonSpecies' + tags: + - pokemon-species + /api/v2/pokemon/: + get: + operationId: pokemon_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of pokemons. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of pokemons. + example: https://pokeapi.co/api/v2/pokemon/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of pokemons. + results: + type: array + items: + $ref: '#/components/schemas/Pokemon' + tags: + - pokemon + /api/v2/pokemon/{id}/: + get: + operationId: pokemon_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this pokemon. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Pokemon' + tags: + - pokemon + /api/v2/region/: + get: + operationId: region_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of regions. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of regions. + example: https://pokeapi.co/api/v2/region/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of regions. + results: + type: array + items: + $ref: '#/components/schemas/Region' + tags: + - region + /api/v2/region/{id}/: + get: + operationId: region_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this region. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Region' + tags: + - region + /api/v2/stat/: + get: + operationId: stat_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of stats. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of stats. + example: https://pokeapi.co/api/v2/stat/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of stats. + results: + type: array + items: + $ref: '#/components/schemas/Stat' + tags: + - stat + /api/v2/stat/{id}/: + get: + operationId: stat_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this stat. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Stat' + tags: + - stat + /api/v2/super-contest-effect/: + get: + operationId: super-contest-effect_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of super contest effects. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of super contest effects. + example: https://pokeapi.co/api/v2/super-contest-effect/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of super contest effects. + results: + type: array + items: + $ref: '#/components/schemas/SuperContestEffect' + tags: + - super-contest-effect + /api/v2/super-contest-effect/{id}/: + get: + operationId: super-contest-effect_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this super contest effect. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/SuperContestEffect' + tags: + - super-contest-effect + /api/v2/type/: + get: + operationId: type_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of types. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of types. + example: https://pokeapi.co/api/v2/type/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of types. + results: + type: array + items: + $ref: '#/components/schemas/Type' + tags: + - type + /api/v2/type/{id}/: + get: + operationId: type_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this type. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Type' + tags: + - type + /api/v2/version-group/: + get: + operationId: version-group_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of version groups. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of version groups. + example: https://pokeapi.co/api/v2/version-group/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of version groups. + results: + type: array + items: + $ref: '#/components/schemas/VersionGroup' + tags: + - version-group + /api/v2/version-group/{id}/: + get: + operationId: version-group_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this version group. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/VersionGroup' + tags: + - version-group + /api/v2/version/: + get: + operationId: version_list + parameters: + - in: query + name: limit + schema: + description: Number of results to return per page. + title: Limit + type: integer + - in: query + name: offset + schema: + description: The initial index from which to return the results. + title: Offset + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + count: + type: integer + description: The total number of versions. + example: 3 + next: + type: string + nullable: true + description: URL to retrieve the next page of versions. + example: https://pokeapi.co/api/v2/version/?offset=20&limit=20 + previous: + type: string + nullable: true + description: URL to retrieve the previous page of versions. + results: + type: array + items: + $ref: '#/components/schemas/Version' + tags: + - version + /api/v2/version/{id}/: + get: + operationId: version_read + parameters: + - in: path + name: id + required: true + schema: + description: A unique integer value identifying this version. + title: ID + type: integer + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '#/components/schemas/Version' + tags: + - version +components: + schemas: + Ability: + type: object + properties: + id: + type: integer + format: int32 + description: The identifier for this ability resource. + name: + type: string + description: The name for this ability resource. + is_main_series: + type: boolean + description: Whether or not this ability originated in the main series of the video games. + generation: + type: object + properties: + name: + type: string + description: The generation this ability originated in. + url: + type: string + description: The URL of the API endpoint for this generation. + names: + type: array + items: + type: object + properties: + name: + type: string + description: The localized name for an API resource in a specific language. + language: + type: object + properties: + name: + type: string + description: The name of the language. + url: + type: string + description: The URL of the API endpoint for the language. + AbilityEffectChange: + type: object + properties: + effect_entries: + type: array + items: + $ref: '#/components/schemas/Effect' + version_group: + description: The version group in which the ability effect change occurred. + $ref: '#/components/schemas/NamedAPIResource' + required: + - effect_entries + - version_group + APIResource: + type: object + properties: + url: + description: The URL of the referenced resource. + type: string + name: + description: The name of the referenced resource. + type: string + id: + description: The ID of the referenced resource. + type: integer + format: int32 + xml: + name: resource + additionalProperties: false + Berry: + type: object + properties: + id: + type: integer + description: The identifier for this berry resource + name: + type: string + description: The name for this berry resource + growth_time: + type: integer + description: Time it takes the tree to grow one stage, in hours. Berry trees go through four of these growth stages before they can be picked. + max_harvest: + type: integer + description: The maximum number of these berries that can grow on one tree in Generation IV + natural_gift_power: + type: integer + description: The power of the move "Natural Gift" when used with this Berry + size: + type: integer + description: The size of this Berry, in millimeters + smoothness: + type: integer + description: The smoothness of this Berry, used in making Pokéblocks or Poffins + soil_dryness: + type: integer + description: The speed at which this Berry dries out the soil as it grows. A higher rate means the soil dries more quickly. + firmness: + $ref: '#/components/schemas/BerryFirmness' + flavors: + type: array + description: A list of references to each flavor a berry can have and the potency of each of those flavors in regard to this berry. + items: + $ref: '#/components/schemas/BerryFlavorMap' + required: + - id + - name + - growth_time + - max_harvest + - natural_gift_power + - size + - smoothness + - soil_dryness + - firmness + - flavors + BerryFirmness: + type: object + properties: + id: + type: integer + description: The identifier for this berry firmness resource + name: + type: string + description: The name for this berry firmness resource + required: + - id + - name + BerryFlavor: + type: object + properties: + id: + type: integer + description: The identifier for this berry flavor resource + name: + type: string + description: The name for this berry flavor resource + berries: + type: array + description: A list of the berries with this flavor + items: + $ref: '#/components/schemas/BerryFlavorMap' + required: + - id + - name + BerryFlavorMap: + type: object + properties: + potency: + type: integer + description: How powerful the referenced flavor is for this berry + flavor: + $ref: '#/components/schemas/BerryFlavor' + required: + - potency + - flavor + Characteristic: + type: object + properties: + id: + type: integer + description: The identifier for this characteristic resource + gene_modulo: + type: integer + description: The remainder of the highest stat/IV divided by 5 + possible_values: + type: array + items: + type: integer + description: The possible values of the highest stat that would result in a Pokémon recieving this characteristic when divided by 5 + required: + - id + - gene_modulo + - possible_values + ContestComboDetail: + type: object + properties: + use_before: + type: array + items: + $ref: '#/components/schemas/Move' + description: A list of moves to use before this move. + use_after: + type: array + items: + $ref: '#/components/schemas/Move' + description: A list of moves to use after this move. + ContestComboSets: + type: object + properties: + normal: + $ref: '#/components/schemas/ContestComboDetail' + super: + $ref: '#/components/schemas/ContestComboDetail' + ContestName: + type: object + properties: + name: + type: string + description: The localized name for an API resource in a specific language. + example: Beauty + ContestType: + type: object + properties: + id: + type: integer + description: The identifier for this contest type resource + name: + type: string + description: The name for this contest type resource + berry_flavor: + $ref: '#/components/schemas/BerryFlavor' + names: + type: array + items: + $ref: '#/components/schemas/ContestName' + description: The name of this contest type listed in different languages + required: + - id + - name + - berry_flavor + - names + ContestEffect: + type: object + properties: + id: + type: integer + description: The identifier for this contest effect resource + appeal: + type: integer + description: The base number of hearts the user of this move gets + jam: + type: integer + description: The base number of hearts the user's opponent loses + effect_entries: + type: array + items: + $ref: '#/components/schemas/VerboseEffect' + description: The flavor text of this contest effect listed in different languages + flavor_text_entries: + type: array + items: + $ref: '#/components/schemas/FlavorText' + description: The flavor text of this contest effect listed in different languages + required: + - id + - appeal + - jam + - effect_entries + - flavor_text_entries + Description: + type: object + properties: + description: + type: string + language: + $ref: '#/components/schemas/Language' + Effect: + type: object + description: An effect that occurs in a game, e.g. causing a Pokémon to fall asleep. + properties: + id: + type: integer + description: The identifier for this effect resource + example: 1 + name: + type: string + description: The name for this effect resource + example: "no-type-damage" + effect_entries: + type: array + description: The list of effect text entries + items: + $ref: '#/components/schemas/EffectEffect' + pokemon_flavor_text_entries: + type: array + description: The flavor text entries that describe this effect + items: + $ref: '#/components/schemas/EffectEffect' + target_species: + description: The species that this effect is against + $ref: '#/components/schemas/PokemonSpecies' + effect_changes: + type: array + description: The list of effects that are changed by this ability + items: + $ref: '#/components/schemas/AbilityEffectChange' + flavor_text_entries: + type: array + description: The flavor text entries that describe this effect + items: + $ref: '#/components/schemas/FlavorText' + generation: + description: The generation this effect originated in + $ref: '#/components/schemas/Generation' + machines: + type: array + description: The machines that teach this move + items: + $ref: '#/components/schemas/MachineVersionDetail' + meta: + description: Meta data about this effect + $ref: '#/components/schemas/MoveMetaData' + short_effect: + type: string + description: The short description of this effect listed in different languages + example: "null" + effect_chance: + type: integer + description: The chance of this move having an additional effect listed in percentage + example: 30 + stat_changes: + type: array + description: The list of stat changes that are caused by this effect + items: + $ref: '#/components/schemas/MoveStatChange' + super_contest_effect: + description: The detail of how effective this move is to the super contest + $ref: '#/components/schemas/SuperContestEffect' + contest_combos: + description: A detail of combos this move can be used in + $ref: '#/components/schemas/ContestComboSets' + contest_type: + description: The type of appeal this move gives a Pokémon when used in a contest + $ref: '#/components/schemas/ContestType' + EffectEffect: + type: object + description: "The various effects of the move `effect_entries`" + properties: + effect: + description: "The localized effect text of this effect" + type: string + language: + $ref: '#/components/schemas/Language' + required: + - effect + - language + EggGroup: + type: object + properties: + id: + type: integer + description: The identifier for this egg group resource + name: + type: string + description: The name for this egg group resource + names: + type: array + items: + $ref: '#/components/schemas/Name' + description: The name of this egg group listed in different languages + pokemon_species: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + description: A list of all Pokemon species that are members of this egg group + required: + - id + - name + - names + - pokemon_species + Encounter: + type: object + properties: + min_level: + type: integer + description: The lowest level the Pokémon could be encountered at. + max_level: + type: integer + description: The highest level the Pokémon could be encountered at. + condition_values: + type: array + items: + $ref: '#/components/schemas/EncounterConditionValue' + description: The condition which triggers this encounter. + chance: + type: integer + description: Percent chance that this encounter will occur. + method: + $ref: '#/components/schemas/EncounterMethod' + description: The method by which this encounter happens. + required: + - min_level + - max_level + - method + EncounterConditionValue: + type: object + properties: + id: + type: integer + description: The identifier for this encounter condition value resource + name: + type: string + description: The name for this encounter condition value resource + condition: + $ref: '#/components/schemas/NamedAPIResource' + description: The condition this encounter condition value pertains to + names: + type: array + items: + $ref: '#/components/schemas/Name' + description: The name of this encounter condition value listed in different languages + required: + - id + - name + - condition + - names + EncounterCondition: + type: object + properties: + id: + type: integer + description: The identifier for this encounter condition resource + name: + type: string + description: The name for this encounter condition resource + values: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + description: A list of values that are used with this encounter condition + names: + type: array + items: + $ref: '#/components/schemas/Name' + description: The name of this encounter condition listed in different languages + required: + - id + - name + - values + - names + EncounterMethod: + type: object + properties: + id: + type: integer + description: The identifier for this encounter method resource + name: + type: string + description: The name for this encounter method resource + order: + type: integer + description: A good value for sorting + required: + - id + - name + - order + EncounterMethodRate: + type: object + properties: + encounter_method: + $ref: '#/components/schemas/EncounterMethod' + version_details: + type: array + items: + $ref: '#/components/schemas/EncounterVersionDetails' + EncounterVersionDetails: + type: object + properties: + version: + $ref: '#/components/schemas/NamedAPIResource' + description: The game version this encounter happens in. + max_chance: + type: integer + description: The total percentage of all encounter potential. + encounter_details: + type: array + items: + $ref: '#/components/schemas/Encounter' + description: A list of encounters and their specifics. + description: Version details of an encounter. + EvolutionChain: + type: object + properties: + id: + type: integer + description: The identifier for this evolution chain resource + baby_trigger_item: + nullable: true + oneOf: + - type: null + - $ref: "#/components/schemas/Item" + description: The item that a baby Pokémon would be holding when born during a forced evolution + chain: + type: object + properties: + is_baby: + type: boolean + description: Whether or not this is a baby Pokémon + species: + $ref: "#/components/schemas/PokemonSpecies" + description: The Pokémon species at this point in the evolution chain + evolution_details: + nullable: true + type: array + items: + type: object + properties: + item: + oneOf: + - type: null + - $ref: "#/components/schemas/Item" + trigger: + $ref: "#/components/schemas/EvolutionTrigger" + gender: + nullable: true + type: integer + description: The required female gender of the evolving Pokémon species. Must be either 1 or 2, or null if the Pokémon species has no gender or the gender is fixed. + held_item: + oneOf: + - type: null + - $ref: "#/components/schemas/Item" + known_move: + oneOf: + - type: null + - $ref: "#/components/schemas/Move" + known_move_type: + oneOf: + - type: null + - $ref: "#/components/schemas/Type" + location: + oneOf: + - type: null + - $ref: "#/components/schemas/Location" + min_level: + nullable: true + type: integer + description: The minimum required level of the evolving Pokémon species + min_happiness: + nullable: true + type: integer + description: The minimum required happiness of the evolving Pokémon species + min_beauty: + nullable: true + type: integer + description: The minimum required beauty of the evolving Pokémon species + min_affection: + nullable: true + type: integer + description: The minimum required affection of the evolving Pokémon species + needs_overworld_rain: + type: boolean + description: Whether or not it must be raining in the overworld to evolve into this Pokémon species + party_species: + oneOf: + - type: null + - $ref: "#/components/schemas/PokemonSpecies" + party_type: + oneOf: + - type: null + - $ref: "#/components/schemas/Type" + relative_physical_stats: + nullable: true + type: integer + description: The required relation between the Pokémon's Attack and Defense stats. 1 means Attack > Defense, 0 means Attack = Defense, and -1 means Attack < Defense. + time_of_day: + type: string + enum: + - day + - night + description: The required time of day. Day or night. + trade_species: + oneOf: + - type: null + - $ref: "#/components/schemas/PokemonSpecies" + turn_upside_down: + type: boolean + description: Whether or not the 3DS needs to be turned upside-down as this Pokémon levels up. + description: The chain of Pokémon species that forms part of this evolution chain + required: + - id + - chain + EvolutionTrigger: + type: object + properties: + id: + type: integer + description: The identifier for this evolution trigger resource + name: + type: string + description: The name for this evolution trigger resource + FlavorText: + type: object + properties: + flavor_text: + type: string + language: + $ref: '#/components/schemas/NamedAPIResource' + version: + $ref: '#/components/schemas/NamedAPIResource' + Gender: + type: object + properties: + id: + type: integer + description: The identifier for this gender resource + name: + type: string + description: The name for this gender resource + pokemon_species_details: + type: array + items: + type: object + properties: + rate: + type: integer + description: The chance of this Pokémon being female, in eighths; or -1 for genderless + Generation: + type: object + properties: + id: + type: integer + description: The identifier for this generation resource + name: + type: string + description: The name for this generation resource + abilities: + type: array + items: + type: object + properties: + name: + type: string + description: The name of this ability + is_hidden: + type: boolean + description: Whether or not this ability is a hidden one + slot: + type: integer + description: The slot this ability occupies in this Pokémon species + names: + type: array + items: + type: object + properties: + name: + type: string + description: The localized name for an API resource in a specific language + language: + $ref: '#/components/schemas/Language' + GenerationGameIndex: + type: object + properties: + game_index: + type: integer + format: int32 + description: The internal id of an API resource within game data. + generation: + $ref: '#/components/schemas/Generation' + Genus: + type: object + properties: + genus: + type: string + language: + $ref: '#/components/schemas/Language' + GrowthRate: + type: object + properties: + id: + type: integer + description: The identifier for this growth rate resource + name: + type: string + description: The name for this growth rate resource + formula: + type: string + description: The formula used to calculate the rate at which the Pokémon species gains level + descriptions: + type: array + description: The description of this growth rate listed in different languages + items: + $ref: '#/components/schemas/Description' + ItemAttribute: + type: object + properties: + id: + type: integer + description: The identifier for this item attribute resource + name: + type: string + description: The name for this item attribute resource + items: + type: array + description: A list of items that have this attribute + items: + $ref: '#/components/schemas/NamedAPIResource' + ItemCategory: + type: object + properties: + id: + type: integer + description: The identifier for this item category resource + name: + type: string + description: The name for this item category resource + items: + type: array + description: A list of items that are a part of this category + items: + $ref: '#/components/schemas/NamedAPIResource' + ItemFlingEffect: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + effect_entries: + type: array + items: + $ref: "#/components/schemas/VerboseEffect" + items: + type: array + items: + $ref: "#/components/schemas/Item" + ItemPocket: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + categories: + type: array + items: + $ref: "#/components/schemas/ItemCategory" + Item: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + cost: + type: integer + format: int32 + fling_power: + type: integer + format: int32 + effect_entries: + type: array + items: + $ref: "#/components/schemas/VerboseEffect" + flavor_text_entries: + type: array + items: + $ref: "#/components/schemas/FlavorText" + attributes: + type: array + items: + $ref: "#/components/schemas/ItemAttribute" + category: + $ref: "#/components/schemas/ItemCategory" + fling_effect: + $ref: "#/components/schemas/ItemFlingEffect" + Language: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + official: + type: boolean + iso639: + type: string + iso3166: + type: string + names: + type: array + items: + $ref: "#/components/schemas/Name" + LocationArea: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + game_index: + type: integer + format: int32 + encounter_method_rates: + type: array + items: + $ref: '#/components/schemas/EncounterMethodRate' + location: + $ref: '#/components/schemas/Location' + names: + type: array + items: + $ref: '#/components/schemas/Name' + pokemon_encounters: + type: array + items: + $ref: '#/components/schemas/PokemonEncounter' + Location: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + region: + $ref: '#/components/schemas/NamedAPIResource' + names: + type: array + items: + $ref: '#/components/schemas/Name' + game_indices: + type: array + items: + $ref: '#/components/schemas/GenerationGameIndex' + areas: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + Machine: + type: object + properties: + id: + type: integer + format: int32 + item: + $ref: '#/components/schemas/NamedAPIResource' + move: + $ref: '#/components/schemas/NamedAPIResource' + version_group: + $ref: '#/components/schemas/NamedAPIResource' + MachineVersionDetail: + type: object + properties: + machine: + description: The machine that teaches a move + type: object + $ref: '#/components/schemas/APIResource' + version_group: + description: The version group of this specific machine + type: object + $ref: '#/components/schemas/APIResource' + required: + - machine + - version_group + Move: + type: object + properties: + id: + type: integer + format: int32 + description: The identifier for this move resource + name: + type: string + description: The name for this move resource + accuracy: + type: integer + format: int32 + description: The percent value of how likely this move is to be successful + nullable: true + effect_chance: + type: integer + format: int32 + description: The percent value of the additional effects this move has occuring + nullable: true + pp: + type: integer + format: int32 + description: Power points. The number of times this move can be used + priority: + type: integer + format: int32 + description: A value of 0 means this move goes last in the turn, and 1 means it goes first + power: + type: integer + format: int32 + description: The base power of this move with a value of 0 if it does not have a base power + nullable: true + contest_combos: + $ref: '#/components/schemas/ContestComboSets' + contest_type: + type: object + properties: + name: + type: string + url: + type: string + required: + - name + - url + contest_effect: + type: object + properties: + url: + type: string + required: + - url + damage_class: + type: object + properties: + name: + type: string + url: + type: string + required: + - name + - url + effect_entries: + type: array + items: + $ref: '#/components/schemas/VerboseEffect' + effect_changes: + type: array + items: + $ref: '#/components/schemas/AbilityEffectChange' + generation: + type: object + properties: + name: + type: string + url: + type: string + required: + - name + - url + meta: + $ref: '#/components/schemas/MoveMetaData' + names: + type: array + items: + $ref: '#/components/schemas/Name' + past_values: + type: array + items: + $ref: '#/components/schemas/PastMoveStatValues' + stat_changes: + type: array + items: + $ref: '#/components/schemas/MoveStatChange' + super_contest_effect: + type: object + properties: + url: + type: string + required: + - url + target: + type: object + properties: + name: + type: string + url: + type: string + required: + - name + - url + type: + type: object + properties: + name: + type: string + url: + type: string + required: + - name + - url + required: + - id + - name + - pp + - priority + - generation + - target + - type + MoveAilment: + type: object + properties: + id: + type: integer + example: 1 + name: + type: string + example: "paralysis" + moves: + type: array + items: + $ref: "#/components/schemas/Move" + required: + - id + - name + - moves + MoveBattleStyle: + type: object + properties: + id: + type: integer + example: 1 + name: + type: string + example: "attack" + required: + - id + - name + MoveBattleStylePreference: + type: object + properties: + low_hp_preference: + type: integer + description: Chance of using the move, in percent, if HP is under one half of maximum HP. + minimum: 0 + maximum: 100 + high_hp_preference: + type: integer + description: Chance of using the move, in percent, if HP is over one half of maximum HP. + minimum: 0 + maximum: 100 + move_battle_style: + $ref: '#/components/schemas/NamedAPIResource' + required: + - low_hp_preference + - high_hp_preference + - move_battle_style + MoveCategory: + type: object + properties: + id: + type: integer + example: 1 + name: + type: string + example: "ailment" + required: + - id + - name + MoveDamageClass: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + descriptions: + type: array + items: + $ref: '#/components/schemas/Description' + required: + - id + - name + - descriptions + MoveLearnMethod: + type: object + description: Methods by which Pokémon can learn moves. + properties: + id: + type: integer + description: The identifier for this move learn method resource. + example: 1 + name: + type: string + description: The name for this move learn method resource. + example: level-up + descriptions: + type: array + items: + $ref: '#/components/schemas/Description' + description: The description of this move learn method listed in different languages. + names: + type: array + items: + $ref: '#/components/schemas/Name' + description: The name of this move learn method listed in different languages. + required: + - id + - name + MoveMetaData: + type: object + properties: + ailment: + description: The type of status effect caused by the move if any. + $ref: '#/components/schemas/NamedAPIResource' + category: + description: The category of move this move falls under, e.g. damage or ailment. + $ref: '#/components/schemas/NamedAPIResource' + min_hits: + description: The minimum number of times this move hits. Null if it always only hits once. + type: integer + nullable: true + max_hits: + description: The maximum number of times this move hits. Null if it always only hits once. + type: integer + nullable: true + min_turns: + description: The minimum number of turns this move continues to take effect. Null if it always only lasts one turn. + type: integer + nullable: true + max_turns: + description: The maximum number of turns this move continues to take effect. Null if it always only lasts one turn. + type: integer + nullable: true + drain: + description: The amount of hp gained by the attacking Pokemon when it uses this move. + type: integer + nullable: true + healing: + description: The amount of hp gained by the target Pokemon when it receives this move. + type: integer + nullable: true + crit_rate: + description: Critical hit rate bonus. + type: integer + nullable: true + ailment_chance: + description: The chance of the target being inflicted with the status condition ailment. + type: integer + nullable: true + flinch_chance: + description: The chance of the target flinching when hit by this move. + type: integer + nullable: true + stat_chance: + description: The chance of the attacking Pokemon lowering the target's stat. + type: integer + nullable: true + required: + - ailment + - category + MoveStatAffect: + type: object + properties: + change: + type: integer + description: The amount of change to the referenced stat. + example: -1 + move: + type: object + description: The move causing the change. + $ref: '#/components/schemas/NamedAPIResource' + required: + - change + - move + MoveStatAffectSets: + type: object + properties: + increase: + type: array + items: + $ref: '#/components/schemas/MoveStatAffect' + decrease: + type: array + items: + $ref: '#/components/schemas/MoveStatAffect' + required: + - increase + - decrease + MoveStatChange: + type: object + properties: + change: + type: integer + description: The amount of change to the referenced stat. + stat: + $ref: '#/components/schemas/NamedAPIResource' + required: + - change + - stat + MoveTarget: + type: object + description: "Targets moves can be directed at during battle. Targets can be Pokémon, adjacent positions, all opponents, etc." + properties: + id: + type: integer + description: "The identifier for this move target resource" + example: 1 + name: + type: string + description: "The name for this move target resource" + example: "specific-move" + descriptions: + type: array + items: + $ref: "#/components/schemas/Description" + description: "The description of this move target listed in different languages" + required: + - id + - name + - descriptions + NamedAPIResource: + type: object + properties: + name: + type: string + url: + type: string + format: uri + required: + - name + - url + Name: + type: object + properties: + language: + $ref: '#/components/schemas/NamedAPIResource' + name: + type: string + Nature: + type: object + properties: + id: + type: integer + example: 5 + name: + type: string + example: "Hardy" + decreased_stat: + type: string + example: "Attack" + increased_stat: + type: string + example: "Attack" + hates_flavor: + type: string + example: "Spicy" + likes_flavor: + type: string + example: "Spicy" + pokeathlon_stat_changes: + type: array + items: + $ref: "#/components/schemas/NatureStatChange" + move_battle_style_preferences: + type: array + items: + $ref: "#/components/schemas/MoveBattleStylePreference" + required: + - id + - name + - decreased_stat + - increased_stat + - hates_flavor + - likes_flavor + additionalProperties: false + NaturePokeathlonStatAffectSets: + type: object + properties: + increase: + type: array + items: + $ref: '#/components/schemas/PokeathlonStatAffect' + decrease: + type: array + items: + $ref: '#/components/schemas/PokeathlonStatAffect' + required: + - increase + - decrease + NatureStatAffect: + type: object + properties: + increase: + type: array + items: + $ref: '#/components/schemas/Nature' + decrease: + type: array + items: + $ref: '#/components/schemas/Nature' + NatureStatAffectSets: + type: object + properties: + increase: + type: array + items: + $ref: '#/components/schemas/NatureStatAffect' + decrease: + type: array + items: + $ref: '#/components/schemas/NatureStatAffect' + required: + - increase + - decrease + NatureStatChange: + type: object + properties: + max_change: + type: integer + pokeathlon_stat: + $ref: '#/components/schemas/PokeathlonStatName' + PokeAthlon: + type: object + properties: + id: + type: integer + example: 1 + name: + type: string + example: "Sprint" + names: + type: array + items: + $ref: "#/components/schemas/PokeathlonStatName" + required: + - id + - name + - names + additionalProperties: false + PokeathlonStatName: + type: object + properties: + name: + type: string + names: + type: array + items: + $ref: '#/components/schemas/Name' + Pokedex: + type: object + properties: + id: + type: integer + example: 2 + name: + type: string + example: "National" + is_main_series: + type: boolean + example: true + descriptions: + type: array + items: + $ref: "#/components/schemas/Description" + pokemon_entries: + type: array + items: + $ref: "#/components/schemas/PokemonEntry" + region: + $ref: "#/components/schemas/NamedAPIResource" + required: + - id + - name + - is_main_series + - descriptions + - pokemon_entries + - region + additionalProperties: false + PalParkArea: + type: object + properties: + id: + type: integer + description: The identifier for this pal park area resource + name: + type: string + description: The name for this pal park area resource + required: + - id + - name + PalParkEncounterArea: + type: object + properties: + base_score: + type: integer + description: The base score given to the player when the referenced Pokemon is caught during a pal park run. + rate: + type: integer + description: The base rate for encountering the referenced Pokemon in this pal park area. + area: + $ref: '#/components/schemas/NamedAPIResource' + description: Areas where the Pokémon species can be encountered in pal park. + PastMoveStatValues: + type: object + properties: + accuracy: + type: integer + description: The percent value of how likely this move is to be successful. + minimum: 0 + maximum: 100 + effect_chance: + type: integer + description: The percent value of effect occurring. + minimum: 0 + maximum: 100 + power: + type: integer + description: The base power of this move with a value of 0 if it does not have a base power. + minimum: 0 + pp: + type: integer + description: The power points this move has left. + minimum: 0 + effect_entries: + type: array + description: The list of previous effects this move has had across version groups. + items: + type: object + properties: + effect: + $ref: '#/components/schemas/VerboseEffect' + version_group: + $ref: '#/components/schemas/NamedAPIResource' + type: + $ref: '#/components/schemas/Type' + required: + - accuracy + - power + - pp + PokeathlonStat: + type: object + properties: + id: + type: integer + description: The identifier for this Pokéathlon stat resource + example: 1 + name: + type: string + description: The name for this Pokéathlon stat resource + example: speed + names: + type: array + items: + $ref: '#/components/schemas/Name' + description: The name of this Pokéathlon stat listed in different languages + affecting_natures: + type: object + description: A detail of natures which affect this Pokéathlon stat positively or negatively + properties: + increase: + type: array + items: + $ref: '#/components/schemas/NaturePokeathlonStatAffectSets' + description: A list of natures that positively affect this Pokéathlon stat + decrease: + type: array + items: + $ref: '#/components/schemas/NaturePokeathlonStatAffectSets' + description: A list of natures that negatively affect this Pokéathlon stat + required: + - id + - name + - names + - affecting_natures + PokeathlonStatAffect: + type: object + properties: + max_change: + type: integer + description: The maximum amount of change to the referenced Pokéathlon stat. + nature: + $ref: '#/components/schemas/Nature' + description: The nature causing the change. + required: + - max_change + - nature + PokemonAbility: + type: object + properties: + is_hidden: + type: boolean + slot: + type: integer + ability: + $ref: '#/components/schemas/Ability' + PokemonColor: + type: object + properties: + id: + type: integer + description: The identifier for this Pokemon color resource + example: 1 + name: + type: string + description: The name for this Pokemon color resource + example: red + PokemonEntry: + type: object + properties: + entry_number: + type: integer + description: The index number within the Pokédex. + example: 6 + pokemon_species: + $ref: "#/components/schemas/NamedAPIResource" + description: The Pokémon species being encountered. + PokemonForm: + type: object + properties: + id: + type: integer + description: The identifier for this Pokemon form resource + example: 1 + name: + type: string + description: The name for this Pokemon form resource + example: bulbasaur + PokemonHabitat: + type: object + properties: + id: + type: integer + description: The identifier for this Pokemon habitat resource + example: 1 + name: + type: string + description: The name for this Pokemon habitat resource + example: cave + PokemonHeldItem: + type: object + properties: + item: + $ref: '#/components/schemas/Item' + version_details: + type: array + items: + type: object + properties: + rarity: + type: integer + version: + $ref: '#/components/schemas/Version' + PokemonMove: + type: object + properties: + move: + $ref: '#/components/schemas/NamedAPIResource' + description: The move the Pokémon can learn + version_group_details: + type: array + items: + type: object + properties: + level_learned_at: + type: integer + description: The minimum level to learn the move + example: 0 + move_learn_method: + $ref: '#/components/schemas/NamedAPIResource' + description: The method by which the Pokémon learns the move + version_group: + $ref: '#/components/schemas/NamedAPIResource' + description: The version group in which the move can be learned + required: + - level_learned_at + - move_learn_method + - version_group + description: | + A list of details showing how the Pokémon can learn the move + PokemonShape: + type: object + properties: + id: + type: integer + description: The identifier for this Pokemon shape resource + example: 1 + name: + type: string + description: The name for this Pokemon shape resource + example: ball + PokemonSpecies: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + order: + type: integer + format: int32 + gender_rate: + type: integer + format: int32 + capture_rate: + type: integer + format: int32 + base_happiness: + type: integer + format: int32 + is_baby: + type: boolean + hatch_counter: + type: integer + format: int32 + has_gender_differences: + type: boolean + forms_switchable: + type: boolean + growth_rate: + $ref: '#/components/schemas/GrowthRate' + pokedex_numbers: + type: array + items: + $ref: '#/components/schemas/PokemonSpeciesDexEntry' + egg_groups: + type: array + items: + $ref: '#/components/schemas/EggGroup' + color: + $ref: '#/components/schemas/PokemonColor' + shape: + $ref: '#/components/schemas/PokemonShape' + evolves_from_species: + $ref: '#/components/schemas/PokemonSpecies' + evolution_chain: + $ref: '#/components/schemas/EvolutionChain' + habitat: + $ref: '#/components/schemas/PokemonHabitat' + generation: + $ref: '#/components/schemas/Generation' + names: + type: array + items: + $ref: '#/components/schemas/Name' + pal_park_encounters: + type: array + items: + $ref: '#/components/schemas/PalParkEncounterArea' + flavor_text_entries: + type: array + items: + $ref: '#/components/schemas/FlavorText' + form_descriptions: + type: array + items: + $ref: '#/components/schemas/Description' + genera: + type: array + items: + $ref: '#/components/schemas/Genus' + varieties: + type: array + items: + $ref: '#/components/schemas/PokemonSpeciesVariety' + PokemonSpeciesDexEntry: + type: object + properties: + entry_number: + type: integer + description: The index number of the Pokedex entry. + pokedex: + $ref: '#/components/schemas/NamedAPIResource' + description: An entry of a Pokemon species seen in the Pokedex. + PokemonSpeciesVariety: + type: object + properties: + is_default: + type: boolean + description: | + Whether this is the default "natural" variety of this species. Note that "default" is + subjective and that it may not match the Pokémon games' official status. + example: true + pokemon: + $ref: '#/components/schemas/Pokemon' + name: + type: string + description: The name of this Pokémon species variety + example: "Bulbasaur" + PokemonSprites: + type: object + properties: + front_default: + description: The default depiction of this Pokémon from the front in battle. + type: string + format: uri + front_female: + description: The shiny depiction of this Pokémon from the front in battle, for female gendered Pokémon. + type: string + format: uri + front_shiny: + description: The shiny depiction of this Pokémon from the front in battle. + type: string + format: uri + front_shiny_female: + description: The shiny depiction of this Pokémon from the front in battle, for female gendered Pokémon. + type: string + format: uri + back_default: + description: The default depiction of this Pokémon from the back in battle. + type: string + format: uri + back_female: + description: The shiny depiction of this Pokémon from the back in battle, for female gendered Pokémon. + type: string + format: uri + back_shiny: + description: The shiny depiction of this Pokémon from the back in battle. + type: string + format: uri + back_shiny_female: + description: The shiny depiction of this Pokémon from the back in battle, for female gendered Pokémon. + type: string + format: uri + PokemonStat: + type: object + properties: + stat: + description: The stat the Pokémon has. + $ref: '#/components/schemas/NamedAPIResource' + effort: + description: The effort points (EV) the Pokémon has in the stat. + type: integer + base_stat: + description: The base value of the stat. + type: integer + PokemonType: + type: object + properties: + slot: + description: The order the Pokémon's types are listed in. + type: integer + format: int32 + type: + description: The type the Pokémon has. + $ref: '#/components/schemas/NamedAPIResource' + PokemonEncounter: + type: object + properties: + pokemon: + $ref: '#/components/schemas/NamedAPIResource' + version_details: + type: array + items: + $ref: '#/components/schemas/EncounterVersionDetails' + description: "Encounters Pokemon that can be encountered in the game and the version groups in which they can be encountered." + Pokemon: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + base_experience: + type: integer + format: int32 + height: + type: integer + format: int32 + is_default: + type: boolean + order: + type: integer + format: int32 + weight: + type: integer + format: int32 + abilities: + type: array + items: + $ref: '#/components/schemas/PokemonAbility' + forms: + type: array + items: + $ref: '#/components/schemas/PokemonForm' + game_indices: + type: array + items: + $ref: '#/components/schemas/VersionGameIndex' + held_items: + type: array + items: + $ref: '#/components/schemas/PokemonHeldItem' + location_area_encounters: + type: string + moves: + type: array + items: + $ref: '#/components/schemas/PokemonMove' + sprites: + $ref: '#/components/schemas/PokemonSprites' + species: + $ref: '#/components/schemas/NamedAPIResource' + stats: + type: array + items: + $ref: '#/components/schemas/PokemonStat' + types: + type: array + items: + $ref: '#/components/schemas/PokemonType' + required: + - id + - name + - base_experience + - height + - is_default + - order + - weight + - abilities + - forms + - game_indices + - held_items + - location_area_encounters + - moves + - sprites + - species + - stats + - types + Region: + type: object + properties: + id: + type: integer + format: int32 + locations: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + name: + type: string + names: + type: array + items: + $ref: '#/components/schemas/Name' + main_generation: + $ref: '#/components/schemas/NamedAPIResource' + pokedexes: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + version_groups: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + required: + - id + - locations + - name + - names + - main_generation + - pokedexes + - version_groups + Stat: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + game_index: + type: integer + format: int32 + is_battle_only: + type: boolean + affecting_moves: + $ref: '#/components/schemas/MoveStatAffectSets' + affecting_natures: + $ref: '#/components/schemas/NatureStatAffectSets' + required: + - id + - name + - game_index + + SuperContestEffect: + type: object + properties: + id: + type: integer + format: int32 + appeal: + type: integer + format: int32 + flavor_text_entries: + type: array + items: + $ref: '#/components/schemas/FlavorText' + moves: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + required: + - id + - appeal + - flavor_text_entries + - moves + + VersionGroup: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + order: + type: integer + format: int32 + generation: + $ref: '#/components/schemas/NamedAPIResource' + move_learn_methods: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + pokedexes: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + regions: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + versions: + type: array + items: + $ref: '#/components/schemas/NamedAPIResource' + required: + - id + - name + - order + - generation + - move_learn_methods + - pokedexes + - regions + - versions + Version: + type: object + properties: + id: + type: integer + format: int32 + name: + type: string + names: + type: array + items: + $ref: '#/components/schemas/Name' + version_group: + $ref: '#/components/schemas/NamedAPIResource' + required: + - id + - name + - names + - version_group + Type: + type: object + properties: + id: + type: integer + format: int32 + description: The identifier for this type resource. + name: + type: string + description: The name for this type resource. + damage_relations: + type: object + properties: + double_damage_from: + type: array + items: + type: object + properties: + name: + type: string + description: The name of the related type. + url: + type: string + description: The URL of the API endpoint for the related type. + double_damage_to: + type: array + items: + type: object + properties: + name: + type: string + description: The name of the related type. + url: + type: string + description: The URL of the API endpoint for the related type. + half_damage_from: + type: array + items: + type: object + properties: + name: + type: string + description: The name of the related type. + url: + type: string + description: The URL of the API endpoint for the related type. + half_damage_to: + type: array + items: + type: object + properties: + name: + type: string + description: The name of the related type. + url: + type: string + description: The URL of the API endpoint for the related type. + no_damage_from: + type: array + items: + type: object + properties: + name: + type: string + description: The name of the related type. + url: + type: string + description: The URL of the API endpoint for the related type. + no_damage_to: + type: array + items: + type: object + properties: + name: + type: string + description: The name of the related type. + url: + type: string + description: The URL of the API endpoint for the related type. + game_indices: + type: array + items: + type: object + properties: + game_index: + type: integer + format: int32 + description: The internal id of an api resource within game data. + generation: + type: object + properties: + name: + type: string + description: The generation this game index is related to. + url: + type: string + description: The URL of the API endpoint for the generation. + generation: + type: object + properties: + name: + type: string + description: The generation this type originated in. + url: + type: string + description: The URL of the API endpoint for this generation. + move_damage_class: + type: object + properties: + name: + type: string + description: The name of this move damage class. + url: + type: string + description: The URL of the API endpoint for this move damage class. + names: + type: array + items: + type: object + properties: + name: + type: string + description: The localized name for an API resource in a specific language. + language: + type: object + properties: + name: + type: string + description: The name of the language. + url: + type: string + description: The URL of the API endpoint for the language. + pokemon: + type: array + items: + type: object + properties: + slot: + type: integer + VerboseEffect: + type: object + properties: + effect: + type: string + description: The localized effect text for an API resource in a specific language. + example: Raises the user's Defense by 1 stage. + VersionGameIndex: + type: object + properties: + game_index: + type: integer + description: | + The internal id of the game used in generation VI and VII to identify different + versions of the same Pokémon species. + example: 12 + version: + $ref: '#/components/schemas/NamedAPIResource' + description: The version relevent to this game index diff --git a/tests/parser/test_parser.py b/tests/parser/test_parser.py index f6fe815ae..fe5252373 100644 --- a/tests/parser/test_parser.py +++ b/tests/parser/test_parser.py @@ -14,12 +14,20 @@ def spotify_parser() -> OpenapiParser: return parser +@pytest.fixture(scope="module") +def pokemon_parser() -> OpenapiParser: + """Re-use parsed spec to save time""" + parser = OpenapiParser(case_path("pokeapi.yml")) + parser.parse() + return parser + + def test_new_releases_list_property(spotify_parser: OpenapiParser) -> None: endpoints = spotify_parser.endpoints.endpoints_by_path endpoint = endpoints["/browse/new-releases"] - list_prop = endpoint.list_property + list_prop = endpoint.payload assert list_prop is not None assert list_prop.path == ("albums", "items") @@ -34,11 +42,11 @@ def test_new_releases_list_property(spotify_parser: OpenapiParser) -> None: assert "release_date" in prop_names -def test_spotify_single_item_endpoints(spotify_parser: OpenapiParser) -> None: - endpoint = spotify_parser.endpoints.endpoints_by_path["/albums/{id}"] +# def test_spotify_single_item_endpoints(spotify_parser: OpenapiParser) -> None: +# endpoint = spotify_parser.endpoints.endpoints_by_path["/albums/{id}"] - assert endpoint.is_transformer - schema = endpoint.data_response.content_schema +# assert endpoint.is_transformer +# schema = endpoint.data_response.content_schema def test_extract_payload(spotify_parser: OpenapiParser) -> None: @@ -59,3 +67,13 @@ def test_extract_payload(spotify_parser: OpenapiParser) -> None: assert related_artists_endpoint.data_response.payload.path == ("artists", "[*]") assert related_artists_endpoint.data_response.payload.prop.name == "ArtistObject" + + +def test_find_path_param(pokemon_parser: OpenapiParser) -> None: + endpoints = pokemon_parser.endpoints + endpoint = endpoints.endpoints_by_path["/api/v2/pokemon-species/"] + + schema = endpoint.payload.prop + result = schema.crawled_properties.find_property_by_name("id", fallback="id") + + assert result.path == ("[*]", "id") From 1c63412298c2606f81a0a4924f4d034a512c9121 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Wed, 20 Mar 2024 19:00:47 +0530 Subject: [PATCH 04/18] Fix root path --- openapi_python_client/parser/models.py | 6 ++++++ openapi_python_client/parser/responses.py | 4 ++-- tests/parser/test_parser.py | 26 +++++++++++++---------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index bdb5903ed..b4d78ec81 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -54,6 +54,12 @@ def json_path(self) -> str: def is_list(self) -> bool: return self.prop.is_list + @property + def schema(self) -> "SchemaWrapper": + if self.is_list and self.prop.array_item: + return self.prop.array_item + return self.prop + def __str__(self) -> str: return f"DataPropertyPath {self.path}: {self.prop.name}" diff --git a/openapi_python_client/parser/responses.py b/openapi_python_client/parser/responses.py index 99ad98ea7..5a98e9419 100644 --- a/openapi_python_client/parser/responses.py +++ b/openapi_python_client/parser/responses.py @@ -89,13 +89,13 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl # Payload path is the deepest nested parent of all remaining props payload_path = find_longest_common_prefix([path for path, _ in payload_props]) - payload_path = root_path + payload_path + payload_path = payload_path while payload_path and payload_path[-1] == "[*]": # We want the path to point to the list property, not the list item # so that payload is correctly detected as list payload_path = payload_path[:-1] payload_schema = schema.crawled_properties[payload_path] - ret = DataPropertyPath(payload_path, payload_schema) + ret = DataPropertyPath(root_path + payload_path, payload_schema) print(endpoint.path) print(ret.path) print(ret.prop.name) diff --git a/tests/parser/test_parser.py b/tests/parser/test_parser.py index fe5252373..a3abd296c 100644 --- a/tests/parser/test_parser.py +++ b/tests/parser/test_parser.py @@ -32,10 +32,10 @@ def test_new_releases_list_property(spotify_parser: OpenapiParser) -> None: assert list_prop.path == ("albums", "items") - list_item_schema = list_prop.prop + list_item_schema = list_prop.schema - assert list_item_schema.name == "SimplifiedAlbumObject" - assert list_item_schema.types == ["object"] + assert list_prop.name == "SimplifiedAlbumObject" + assert list_prop.prop.array_item.types == ["object"] prop_names = [p.name for p in list_item_schema.properties] assert "id" in prop_names assert "name" in prop_names @@ -56,17 +56,21 @@ def test_extract_payload(spotify_parser: OpenapiParser) -> None: saved_tracks_endpoint = endpoints.endpoints_by_path["/me/tracks"] related_artists_endpoint = endpoints.endpoints_by_path["/artists/{id}/related-artists"] - assert new_releases_endpoint.data_response.payload.path == ("albums", "items", "[*]") - assert new_releases_endpoint.data_response.payload.prop.name == "SimplifiedAlbumObject" + assert new_releases_endpoint.data_response.payload.path == ( + "albums", + "items", + ) + assert new_releases_endpoint.data_response.payload.name == "SimplifiedAlbumObject" - assert pl_tr_endpoint.data_response.payload.path == ("items", "[*]") - assert pl_tr_endpoint.data_response.payload.prop.name == "PlaylistTrackObject" + # TODO: + # assert pl_tr_endpoint.data_response.payload.path == ("items",) + # assert pl_tr_endpoint.data_response.payload.name == "PlaylistTrackObject" - assert saved_tracks_endpoint.data_response.payload.path == ("items", "[*]") - assert saved_tracks_endpoint.data_response.payload.prop.name == "SavedTrackObject" + assert saved_tracks_endpoint.data_response.payload.path == ("items",) + assert saved_tracks_endpoint.data_response.payload.name == "SavedTrackObject" - assert related_artists_endpoint.data_response.payload.path == ("artists", "[*]") - assert related_artists_endpoint.data_response.payload.prop.name == "ArtistObject" + assert related_artists_endpoint.data_response.payload.path == ("artists",) + assert related_artists_endpoint.data_response.payload.name == "ArtistObject" def test_find_path_param(pokemon_parser: OpenapiParser) -> None: From b13024f95dfd1e2da5dc5d3c4ef9a4b57f0545e1 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Fri, 22 Mar 2024 16:40:53 +0530 Subject: [PATCH 05/18] Handle multiple path params in transformers --- openapi_python_client/parser/endpoints.py | 138 +++++--------- openapi_python_client/parser/models.py | 16 +- openapi_python_client/parser/parameters.py | 172 ++++++++++++++++++ .../templates/endpoint_macros.py.jinja | 4 +- .../templates/endpoint_module.py.jinja | 2 +- .../templates/utils.py.jinja | 34 ++-- 6 files changed, 253 insertions(+), 113 deletions(-) create mode 100644 openapi_python_client/parser/parameters.py diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index 1b67db795..2e2296011 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -16,94 +16,23 @@ from openapi_python_client.parser.credentials import CredentialsProperty from openapi_python_client.parser.pagination import Pagination from openapi_python_client.parser.types import DataType +from openapi_python_client.parser.parameters import Parameter TMethod = Literal["get", "post", "put", "patch"] -TParamIn = Literal["query", "header", "path", "cookie"] Tree = Dict[str, Union["Endpoint", "Tree"]] @dataclass class TransformerSetting: parent_endpoint: Endpoint - parent_property: DataPropertyPath - path_parameter: Parameter - - -@dataclass -class Parameter: - name: str - description: Optional[str] - schema: SchemaWrapper - raw_schema: osp.Parameter - required: bool - location: TParamIn - python_name: PythonIdentifier - explode: bool - style: Optional[str] = None - - def get_imports(self) -> List[str]: - imports = [] - if self.schema.is_union: - imports.append("from typing import Union") - return imports - - @property - def types(self) -> List[TSchemaType]: - return self.schema.types - - @property - def template(self) -> str: - return self.schema.property_template - - @property - def default(self) -> Optional[Any]: - return self.schema.default - - @property - def nullable(self) -> bool: - return self.schema.nullable + # parent_property: DataPropertyPath + # path_parameter: Parameter + path_parameters: List[Parameter] + parent_properties: List[DataPropertyPath] @property - def type_hint(self) -> str: - return DataType.from_schema(self.schema, required=self.required).type_hint - - def to_string(self) -> str: - type_hint = self.type_hint - default = self.default - if default is None and not self.required: - default = "UNSET" - - base_string = f"{self.python_name}: {type_hint}" - if default is not None: - base_string += f" = {default}" - return base_string - - def to_docstring(self) -> str: - doc = f"{self.python_name}: {self.description or ''}" - if self.default: - doc += f" Default: {self.default}." - # TODO: Example - return doc - - @classmethod - def from_reference(cls, param_ref: Union[osp.Reference, osp.Parameter], context: OpenapiContext) -> "Parameter": - osp_param = context.parameter_from_reference(param_ref) - schema = SchemaWrapper.from_reference(osp_param.param_schema, context) - description = param_ref.description or osp_param.description or schema.description - location = osp_param.param_in - required = osp_param.required - - return cls( - name=osp_param.name, - description=description, - raw_schema=osp_param, - schema=schema, - location=cast(TParamIn, location), - required=required, - python_name=PythonIdentifier(osp_param.name, prefix=context.config.field_prefix), - explode=osp_param.explode or False, - style=osp_param.style, - ) + def path_params_mapping(self) -> Dict[str, str]: + return {param.name: prop.json_path for param, prop in zip(self.path_parameters, self.parent_properties)} @dataclass @@ -323,24 +252,49 @@ def is_transformer(self) -> bool: def transformer(self) -> Optional[TransformerSetting]: if not self.parent: return None - if not self.parent.is_list: - return None - if not self.path_parameters: - return None - if len(self.path_parameters) > 1: - # TODO: Can't handle endpoints with more than 1 path param for now - return None - path_param = list(self.path_parameters.values())[-1] - payload = self.parent.payload - assert payload - list_object = payload.prop - transformer_arg = list_object.crawled_properties.find_property_by_name(path_param.name, fallback="id") - if not transformer_arg: + # if not self.parent.is_list: + # return None + path_parameters = self.path_parameters + if not path_parameters: return None + # # Are we nested more than 1 level? + # # Must ensure all path params are resolved in parent transformers + # if len(self.path_parameters) > 1: + # for i in range(len(self.path_parameters) - 1): + # parent = self.parent + # if not parent.is_transformer: + # return None + + # Must resolve all path parameters from the parent response + # TODO: Consider multiple levels of transformers. + # This would need to forward resolved ancestor params through meta arg + parent_payload = self.parent.payload + assert parent_payload + resolved_props = [] + for param in path_parameters.values(): + prop = param.find_input_property(parent_payload.schema, fallback="id") + if not prop: + return None + resolved_props.append(prop) + return TransformerSetting( - parent_endpoint=self.parent, parent_property=transformer_arg, path_parameter=path_param + parent_endpoint=self.parent, + parent_properties=resolved_props, + path_parameters=list(path_parameters.values()), ) + # path_param = list(self.path_parameters.values())[-1] + # payload = self.parent.payload + # assert payload + # list_object = payload.prop + # # transformer_arg = list_object.crawled_properties.find_property_by_name(path_param.name, fallback="id") + # transformer_arg = path_param.find_input_property(list_object, fallback="id") + # if not transformer_arg: + # return None + # return TransformerSetting( + # parent_endpoint=self.parent, parent_property=transformer_arg, path_parameter=path_param + # ) + @classmethod def from_operation( cls, diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index b4d78ec81..af6141174 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -87,6 +87,9 @@ class SchemaWrapper: crawled_properties: SchemaCrawler hash_key: str + type_format: Optional[str] = None + """Format (e.g. datetime, uuid) as an extension of the data type""" + array_item: Optional["SchemaWrapper"] = None all_of: List["SchemaWrapper"] = field(default_factory=list) any_of: List["SchemaWrapper"] = field(default_factory=list) @@ -258,6 +261,7 @@ def from_reference( default=default, crawled_properties=crawler, hash_key=digest128(schema.json(sort_keys=True)), + type_format=schema.schema_format, ) crawler.crawl(result) return result @@ -334,15 +338,21 @@ def __len__(self) -> int: def __bool__(self) -> bool: return bool(self.all_properties) - def paths_with_types(self) -> Iterator[tuple[tuple[str, ...], tuple[TSchemaType, ...]]]: + def items(self) -> Iterable[Tuple[Tuple[str, ...], SchemaWrapper]]: + return self.all_properties.items() + + def paths_with_types(self) -> Iterator[tuple[tuple[str, ...], tuple[tuple[TSchemaType, ...], Optional[str]]]]: + """ + yields a tuple of (path, ( (types, ...), "format")) for each property in the schema + """ for path, schema in self.all_properties.items(): # if schema.is_list and schema.array_item: # # Include the array item type for full comparison # yield path, tuple(schema.types + schema.array_item.types]) # else: - yield path, tuple(schema.types) + yield path, (tuple(schema.types), schema.type_format) - def _is_optional(self, path: Tuple[str, ...]) -> bool: + def is_optional(self, path: Tuple[str, ...]) -> bool: """Check whether the property itself or any of its parents is nullable""" check_path = list(path) while check_path: diff --git a/openapi_python_client/parser/parameters.py b/openapi_python_client/parser/parameters.py new file mode 100644 index 000000000..f318cab44 --- /dev/null +++ b/openapi_python_client/parser/parameters.py @@ -0,0 +1,172 @@ +from dataclasses import dataclass +from typing import Any, List, Literal, Optional, Union, cast + +import openapi_schema_pydantic as osp + +from openapi_python_client.utils import PythonIdentifier +from openapi_python_client.parser.models import SchemaWrapper, TSchemaType, DataPropertyPath +from openapi_python_client.parser.types import DataType +from openapi_python_client.parser.context import OpenapiContext + +TParamIn = Literal["query", "header", "path", "cookie"] + + +@dataclass +class Parameter: + name: str + description: Optional[str] + schema: SchemaWrapper + raw_schema: osp.Parameter + required: bool + location: TParamIn + python_name: PythonIdentifier + explode: bool + style: Optional[str] = None + + def get_imports(self) -> List[str]: + imports = [] + if self.schema.is_union: + imports.append("from typing import Union") + return imports + + @property + def types(self) -> List[TSchemaType]: + return self.schema.types + + @property + def type_format(self) -> Optional[str]: + return self.schema.type_format + + @property + def template(self) -> str: + return self.schema.property_template + + @property + def default(self) -> Optional[Any]: + return self.schema.default + + @property + def nullable(self) -> bool: + return self.schema.nullable + + @property + def type_hint(self) -> str: + return DataType.from_schema(self.schema, required=self.required).type_hint + + def to_string(self) -> str: + type_hint = self.type_hint + default = self.default + if default is None and not self.required: + default = "UNSET" + + base_string = f"{self.python_name}: {type_hint}" + if default is not None: + base_string += f" = {default}" + return base_string + + def to_docstring(self) -> str: + doc = f"{self.python_name}: {self.description or ''}" + if self.default: + doc += f" Default: {self.default}." + # TODO: Example + return doc + + def _matches_type(self, schema: SchemaWrapper) -> bool: + return schema.types == self.types and schema.type_format == self.type_format + + def find_input_property(self, schema: SchemaWrapper, fallback: Optional[str] = None) -> Optional[DataPropertyPath]: + """Find property in the given schema that's potentially an input to this parameter""" + name = self.name + named = [] + fallbacks = [] + named_optional = [] + fallbacks_optional = [] + partial_named = [] + partial_named_optional = [] + partial_fallbacks = [] + partial_fallbacks_optional = [] + + for path, prop_schema in schema.crawled_properties.items(): + if path and path[-1] == name: + if not self._matches_type(prop_schema): + continue + if schema.crawled_properties.is_optional(path): + named_optional.append((path, prop_schema)) + else: + named.append((path, prop_schema)) + if fallback and path and path[-1] == fallback: + if not self._matches_type(prop_schema): + continue + if schema.crawled_properties.is_optional(path): + fallbacks_optional.append((path, prop_schema)) + else: + fallbacks.append((path, prop_schema)) + # Check for partial name matches of the same type + if path and name.lower() in path[-1].lower(): + if not self._matches_type(prop_schema): + continue + if schema.crawled_properties.is_optional(path): + partial_named_optional.append((path, prop_schema)) + else: + partial_named.append((path, prop_schema)) + if fallback and path and fallback.lower() in path[-1].lower(): + if not self._matches_type(prop_schema): + continue + if schema.crawled_properties.is_optional(path): + partial_fallbacks_optional.append((path, prop_schema)) + else: + partial_fallbacks.append((path, prop_schema)) + + # Prefer the least nested path + for arr in [ + named, + fallbacks, + named_optional, + fallbacks_optional, + partial_named, + partial_fallbacks, + partial_named_optional, + partial_fallbacks_optional, + ]: + arr.sort(key=lambda item: len(item[0])) + + # Prefer required property and required fallback over optional properties + # If not required props found, assume the spec is wrong and optional properties are required in practice + # Partial name matches are least preferred + if named: + return DataPropertyPath(*named[0]) + elif fallbacks: + return DataPropertyPath(*fallbacks[0]) + elif named_optional: + return DataPropertyPath(*named_optional[0]) + elif fallbacks_optional: + return DataPropertyPath(*fallbacks_optional[0]) + elif partial_named: + return DataPropertyPath(*partial_named[0]) + elif partial_fallbacks: + return DataPropertyPath(*partial_fallbacks[0]) + elif partial_named_optional: + return DataPropertyPath(*partial_named_optional[0]) + elif partial_fallbacks_optional: + return DataPropertyPath(*partial_fallbacks_optional[0]) + return None + + @classmethod + def from_reference(cls, param_ref: Union[osp.Reference, osp.Parameter], context: OpenapiContext) -> "Parameter": + osp_param = context.parameter_from_reference(param_ref) + schema = SchemaWrapper.from_reference(osp_param.param_schema, context) + description = param_ref.description or osp_param.description or schema.description + location = osp_param.param_in + required = osp_param.required + + return cls( + name=osp_param.name, + description=description, + raw_schema=osp_param, + schema=schema, + location=cast(TParamIn, location), + required=required, + python_name=PythonIdentifier(osp_param.name, prefix=context.config.field_prefix), + explode=osp_param.explode or False, + style=osp_param.style, + ) diff --git a/openapi_python_client/templates/endpoint_macros.py.jinja b/openapi_python_client/templates/endpoint_macros.py.jinja index aeb03f595..d1f50c392 100644 --- a/openapi_python_client/templates/endpoint_macros.py.jinja +++ b/openapi_python_client/templates/endpoint_macros.py.jinja @@ -88,8 +88,8 @@ credentials: {{ endpoint.credentials.type_hint }} = dlt.secrets.value, {% endif %} data_json_path: Optional[str]="{{endpoint.data_json_path}}", {% if endpoint.transformer %} -parent_property_path: str="{{ endpoint.transformer.parent_property.json_path }}", -path_parameter_name: str="{{ endpoint.transformer.path_parameter.name }}", +{# Render the dict of path params #} +path_parameter_paths={{ endpoint.transformer.path_params_mapping }} {% endif %} {% endmacro %} diff --git a/openapi_python_client/templates/endpoint_module.py.jinja b/openapi_python_client/templates/endpoint_module.py.jinja index b71ca0c3c..3ea2bc24f 100644 --- a/openapi_python_client/templates/endpoint_module.py.jinja +++ b/openapi_python_client/templates/endpoint_module.py.jinja @@ -34,7 +34,7 @@ def {{ endpoint.python_name }}({{ resource_arguments(endpoint) | indent(4) }}) - params_dict: Dict[str, Any] = dict( {{ transformer_kwargs(endpoint) }} ) - for child_kwargs in extract_iterate_parent(data, parent_property_path, path_parameter_name, "{{ endpoint.path }}"): + for child_kwargs in extract_iterate_parent(data, path_parameter_paths, "{{ endpoint.path }}"): formatted_url = build_formatted_url( endpoint_url, dict(params_dict, **child_kwargs), diff --git a/openapi_python_client/templates/utils.py.jinja b/openapi_python_client/templates/utils.py.jinja index 3ce1663e3..e1b22a6f0 100644 --- a/openapi_python_client/templates/utils.py.jinja +++ b/openapi_python_client/templates/utils.py.jinja @@ -25,32 +25,36 @@ def extract_url_like_values(data: Dict[str, Any]) -> Iterator[str]: yield value -def extract_iterate_parent( - data: TDataItems, property_path: TJsonPath, path_param_name: str, endpoint_url: str -) -> Iterator[Any]: +def extract_iterate_parent(data: TDataItems, path_parameters: Dict[str, TJsonPath], endpoint_url: str) -> Iterator[Any]: """Extract kwargs to use in transformer requests""" if not isinstance(data, list): data = [data] for item in data: - property_values = find_values(property_path, item) - if property_values: - yield {path_param_name: property_values[0]} + result = {} + for path_param_name, property_path in path_parameters.items(): + property_values = find_values(property_path, item) + if property_values: + result[path_param_name] = property_values[0] + + if result: + yield result continue # Try looking for a URL property which matches the endpoint URL we're dealing with urls = extract_url_like_values(item) - property_value: Optional[str] = None + property_values = {} for url in urls: - property_value = pluck_param_from_result_url(url, endpoint_url) - if not property_value: - continue - else: - break + for path_param_name in path_parameters.keys(): + property_value = pluck_param_from_result_url(url, endpoint_url) + if property_value is not None: + property_values[path_param_name] = property_value + break - if property_value is not None: - yield {path_param_name: property_value} + # Has all keys + if len(property_values) == len(path_parameters): + yield property_values else: - raise ValueError(f"Could not find id parameter {path_param_name} (endpoint {endpoint_url}) in: {item}") + raise ValueError(f"Could not find id parameters {path_parameters} (endpoint {endpoint_url}) in: {item}") def pluck_param_from_result_url(result_url: str, endpoint_url: str) -> Optional[str]: From 20401bf1f35491f74b0a97dc175af7bef5f70746 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Tue, 26 Mar 2024 11:48:22 +0530 Subject: [PATCH 06/18] Remove bundled rest client --- openapi_python_client/__init__.py | 6 +- openapi_python_client/sources/.dlt/.sources | 36 -- .../sources/.dlt/config.toml | 6 - openapi_python_client/sources/.gitignore | 10 - .../sources/requirements.txt | 1 - .../sources/rest_api/__init__.py | 346 ------------------ .../sources/rest_api/auth.py | 228 ------------ .../sources/rest_api/client.py | 255 ------------- .../sources/rest_api/config_setup.py | 279 -------------- .../sources/rest_api/detector.py | 161 -------- .../sources/rest_api/exceptions.py | 5 - .../sources/rest_api/paginators.py | 179 --------- .../sources/rest_api/requirements.txt | 1 - .../sources/rest_api/typing.py | 110 ------ .../sources/rest_api/utils.py | 15 - .../sources/rest_api_pipeline.py | 131 ------- .../templates/pyproject.toml.jinja | 2 +- .../templates/source.py.jinja | 2 +- poetry.lock | 20 +- pyproject.toml | 2 +- 20 files changed, 18 insertions(+), 1777 deletions(-) delete mode 100644 openapi_python_client/sources/.dlt/.sources delete mode 100644 openapi_python_client/sources/.dlt/config.toml delete mode 100644 openapi_python_client/sources/.gitignore delete mode 100644 openapi_python_client/sources/requirements.txt delete mode 100644 openapi_python_client/sources/rest_api/__init__.py delete mode 100644 openapi_python_client/sources/rest_api/auth.py delete mode 100644 openapi_python_client/sources/rest_api/client.py delete mode 100644 openapi_python_client/sources/rest_api/config_setup.py delete mode 100644 openapi_python_client/sources/rest_api/detector.py delete mode 100644 openapi_python_client/sources/rest_api/exceptions.py delete mode 100644 openapi_python_client/sources/rest_api/paginators.py delete mode 100644 openapi_python_client/sources/rest_api/requirements.txt delete mode 100644 openapi_python_client/sources/rest_api/typing.py delete mode 100644 openapi_python_client/sources/rest_api/utils.py delete mode 100644 openapi_python_client/sources/rest_api_pipeline.py diff --git a/openapi_python_client/__init__.py b/openapi_python_client/__init__.py index 007afc1d8..2813bac3d 100644 --- a/openapi_python_client/__init__.py +++ b/openapi_python_client/__init__.py @@ -165,9 +165,9 @@ def _create_package(self) -> None: api_helpers_path = self.package_dir / "api_helpers.py" api_helpers_path.write_text(api_helpers_template.render(), encoding=self.file_encoding) - # For now just copy the rest_api source into the package - rest_api_dir = Path(__file__).parent / "sources" / "rest_api" - shutil.copytree(rest_api_dir, self.package_dir / "rest_api") + # # For now just copy the rest_api source into the package + # rest_api_dir = Path(__file__).parent / "sources" / "rest_api" + # shutil.copytree(rest_api_dir, self.package_dir / "rest_api") def _build_dlt_config(self) -> None: config_dir = self.project_dir / ".dlt" diff --git a/openapi_python_client/sources/.dlt/.sources b/openapi_python_client/sources/.dlt/.sources deleted file mode 100644 index 544d5a43c..000000000 --- a/openapi_python_client/sources/.dlt/.sources +++ /dev/null @@ -1,36 +0,0 @@ -engine_version: 1 -sources: - rest_api: - is_dirty: false - last_commit_sha: ac63f25e752a20eb84c1c354a2a5eac8c6d063b9 - last_commit_timestamp: '2024-03-08T21:11:34+03:00' - files: - rest_api/utils.py: - commit_sha: ac63f25e752a20eb84c1c354a2a5eac8c6d063b9 - git_sha: 61640ba3166021ae693a7ec11d4d8306575cea84 - sha3_256: 55e0e901269e9db79bfc7d979fc2724dc1feeb13c5cb86516b3e6901a5db330e - rest_api/detector.py: - commit_sha: ac63f25e752a20eb84c1c354a2a5eac8c6d063b9 - git_sha: 91d4a2e8d45484917409b903ff7ded8ca61b8167 - sha3_256: 0ed1728ea78def2e2a67e36d87120e9ed1e199c51fd4873b8a41ea02689574d3 - rest_api/client.py: - commit_sha: ac63f25e752a20eb84c1c354a2a5eac8c6d063b9 - git_sha: 4b881b18fecf455d195b956b6f6694d76aa44f41 - sha3_256: fa4d7e66efffea7ac618d8617b22e54b2cdf012859a111f822cc8e32b20903c8 - rest_api/auth.py: - commit_sha: ac63f25e752a20eb84c1c354a2a5eac8c6d063b9 - git_sha: ed30fa495d2d496e486d0e482ccd42e0f05cc2ff - sha3_256: 56f9ef5450a6894baaeb00d0d74877e9aaa5bd2644261e4bcb8b05c12a7e9f15 - rest_api/typing.py: - commit_sha: ac63f25e752a20eb84c1c354a2a5eac8c6d063b9 - git_sha: 85d373c63858bc953e150a4805abf2a27bd94446 - sha3_256: 373c37d0d29797d6bcf19f8f8db48456dda78a6279323332577cd19f3ed313c5 - rest_api/paginators.py: - commit_sha: ac63f25e752a20eb84c1c354a2a5eac8c6d063b9 - git_sha: 1c307290deebefde860e122b5aa0b1f00280f95e - sha3_256: 6bfcf1d96ae60ee3d77db162d805cb85474fcc96fd4ee27b9969fed3fefe7da9 - rest_api/__init__.py: - commit_sha: ac63f25e752a20eb84c1c354a2a5eac8c6d063b9 - git_sha: 2ce0e38cf1d4f6ff6e9915dfab5d89b0245b31f9 - sha3_256: 2b415d98f639226358270e44fa089c012ae4bdc7cdadde59acddd1c7729e592e - dlt_version_constraint: '>=0.4.4' diff --git a/openapi_python_client/sources/.dlt/config.toml b/openapi_python_client/sources/.dlt/config.toml deleted file mode 100644 index abbec8402..000000000 --- a/openapi_python_client/sources/.dlt/config.toml +++ /dev/null @@ -1,6 +0,0 @@ -# put your configuration values here - -[runtime] -log_level="WARNING" # the system log level of dlt -# use the dlthub_telemetry setting to enable/disable anonymous usage data reporting, see https://dlthub.com/docs/telemetry -dlthub_telemetry = false diff --git a/openapi_python_client/sources/.gitignore b/openapi_python_client/sources/.gitignore deleted file mode 100644 index 3b28aa3f6..000000000 --- a/openapi_python_client/sources/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -# ignore secrets, virtual environments and typical python compilation artifacts -secrets.toml -# ignore basic python artifacts -.env -**/__pycache__/ -**/*.py[cod] -**/*$py.class -# ignore duckdb -*.duckdb -*.wal \ No newline at end of file diff --git a/openapi_python_client/sources/requirements.txt b/openapi_python_client/sources/requirements.txt deleted file mode 100644 index d4e912680..000000000 --- a/openapi_python_client/sources/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -dlt[postgres]>=0.4.4 \ No newline at end of file diff --git a/openapi_python_client/sources/rest_api/__init__.py b/openapi_python_client/sources/rest_api/__init__.py deleted file mode 100644 index 3c12cec02..000000000 --- a/openapi_python_client/sources/rest_api/__init__.py +++ /dev/null @@ -1,346 +0,0 @@ -"""Generic API Source""" - -from typing import ( - Type, - Any, - Dict, - Tuple, - List, - Optional, - Generator, - Callable, - cast, -) -import graphlib # type: ignore[import,unused-ignore] - -import dlt -from dlt.common.validation import validate_dict -from dlt.extract.incremental import Incremental -from dlt.extract.source import DltResource, DltSource -from dlt.common import logger, jsonpath -from dlt.common.schema.schema import Schema -from dlt.common.schema.typing import TSchemaContract -from dlt.common.configuration.specs import BaseConfiguration - -from .client import RESTClient -from .detector import single_entity_path -from .paginators import BasePaginator -from .typing import ( - ClientConfig, - ResolvedParam, - Endpoint, - EndpointResource, - RESTAPIConfig, - HTTPMethodBasic, -) -from .config_setup import ( - create_auth, - create_paginator, - build_resource_dependency_graph, - make_parent_key_name, - setup_incremental_object, - create_response_hooks, -) - - -def rest_api_source( - config: RESTAPIConfig, - name: str = None, - section: str = None, - max_table_nesting: int = None, - root_key: bool = False, - schema: Schema = None, - schema_contract: TSchemaContract = None, - spec: Type[BaseConfiguration] = None, -) -> DltSource: - """ - Creates and configures a REST API source for data extraction. - - Example: - pokemon_source = rest_api_source({ - "client": { - "base_url": "https://pokeapi.co/api/v2/", - "paginator": "json_links", - }, - "endpoints": { - "pokemon": { - "params": { - "limit": 100, # Default page size is 20 - }, - "resource": { - "primary_key": "id", - } - }, - }, - }) - """ - decorated = dlt.source( - rest_api_resources, - name, - section, - max_table_nesting, - root_key, - schema, - schema_contract, - spec, - ) - - return decorated(config) - - -def rest_api_resources(config: RESTAPIConfig) -> List[DltResource]: - """ - Creates and configures a REST API source for data extraction. - - Example: - github_source = rest_api_resources_v3({ - "client": { - "base_url": "https://api.github.com/repos/dlt-hub/dlt/", - "auth": { - "token": dlt.secrets["token"], - }, - }, - "resource_defaults": { - "primary_key": "id", - "write_disposition": "merge", - "endpoint": { - "params": { - "per_page": 100, - }, - }, - }, - "resources": [ - { - "name": "issues", - "endpoint": { - "path": "issues", - "params": { - "sort": "updated", - "direction": "desc", - "state": "open", - "since": { - "type": "incremental", - "cursor_path": "updated_at", - "initial_value": "2024-01-25T11:21:28Z", - }, - }, - }, - }, - { - "name": "issue_comments", - "endpoint": { - "path": "issues/{issue_number}/comments", - "params": { - "issue_number": { - "type": "resolve", - "resource": "issues", - "field": "number", - } - }, - }, - }, - ], - }) - """ - - validate_dict(RESTAPIConfig, config, path=".") - - client_config = config["client"] - client = RESTClient( - base_url=client_config["base_url"], - auth=create_auth(client_config.get("auth")), - paginator=create_paginator(client_config.get("paginator")), - ) - - resource_defaults = config.get("resource_defaults", {}) - - resource_list = config.get("resources") - - if not resource_list: - raise ValueError("No resources defined") - - ( - dependency_graph, - endpoint_resource_map, - resolved_param_map, - ) = build_resource_dependency_graph( - resource_defaults, - resource_list, - ) - - resources = create_resources( - client, - dependency_graph, - endpoint_resource_map, - resolved_param_map, - ) - - return list(resources.values()) - - -def create_resources( - client: RESTClient, - dependency_graph: graphlib.TopologicalSorter, - endpoint_resource_map: Dict[str, EndpointResource], - resolved_param_map: Dict[str, Optional[ResolvedParam]], -) -> Dict[str, DltResource]: - resources = {} - - for resource_name in dependency_graph.static_order(): - resource_name = cast(str, resource_name) - endpoint_resource = endpoint_resource_map[resource_name] - endpoint_config = cast(Endpoint, endpoint_resource.pop("endpoint")) - request_params = endpoint_config.get("params", {}) - paginator = create_paginator(endpoint_config.get("paginator")) - - resolved_param: ResolvedParam = resolved_param_map[resource_name] - - include_from_parent: List[str] = endpoint_resource.pop( - "include_from_parent", [] - ) - if not resolved_param and include_from_parent: - raise ValueError( - f"Resource {resource_name} has include_from_parent but is not " - "dependent on another resource" - ) - - incremental_object, incremental_param = setup_incremental_object( - request_params, endpoint_resource.get("incremental") - ) - - hooks = create_response_hooks(endpoint_config.get("response_actions")) - - # try to guess if list of entities or just single entity is returned - if single_entity_path(endpoint_config["path"]): - data_selector = "$" - else: - data_selector = None - - if resolved_param is None: - - def paginate_resource( - method: HTTPMethodBasic, - path: str, - params: Dict[str, Any], - paginator: Optional[BasePaginator], - data_selector: Optional[jsonpath.TJsonPath], - hooks: Optional[Dict[str, Any]], - incremental_object: Optional[Incremental[Any]] = incremental_object, - incremental_param: str = incremental_param, - ) -> Generator[Any, None, None]: - if incremental_object: - params[incremental_param] = incremental_object.last_value - - yield from client.paginate( - method=method, - path=path, - params=params, - paginator=paginator, - data_selector=data_selector, - hooks=hooks, - ) - - resources[resource_name] = dlt.resource( # type: ignore[call-overload] - paginate_resource, - **endpoint_resource, # TODO: implement typing.Unpack - )( - method=endpoint_config.get("method", "get"), - path=endpoint_config.get("path"), - params=request_params, - paginator=paginator, - data_selector=endpoint_config.get("data_selector") or data_selector, - hooks=hooks, - ) - - else: - predecessor = resources[resolved_param.resolve_config.resource_name] - - request_params.pop(resolved_param.param_name, None) - - def paginate_dependent_resource( - items: List[Dict[str, Any]], - method: HTTPMethodBasic, - path: str, - params: Dict[str, Any], - paginator: Optional[BasePaginator], - data_selector: Optional[jsonpath.TJsonPath], - hooks: Optional[Dict[str, Any]], - resolved_param: ResolvedParam = resolved_param, - include_from_parent: List[str] = include_from_parent, - ) -> Generator[Any, None, None]: - field_path = resolved_param.resolve_config.field_path - - items = items or [] - for item in items: - formatted_path = path.format( - **{resolved_param.param_name: item[field_path]} - ) - parent_resource_name = resolved_param.resolve_config.resource_name - - parent_record = ( - { - make_parent_key_name(parent_resource_name, key): item[key] - for key in include_from_parent - } - if include_from_parent - else None - ) - - for child_page in client.paginate( - method=method, - path=formatted_path, - params=params, - paginator=paginator, - data_selector=data_selector, - hooks=hooks, - ): - if parent_record: - for child_record in child_page: - child_record.update(parent_record) - yield child_page - - resources[resource_name] = dlt.resource( # type: ignore[call-overload] - paginate_dependent_resource, - data_from=predecessor, - **endpoint_resource, # TODO: implement typing.Unpack - )( - method=endpoint_config.get("method", "get"), - path=endpoint_config.get("path"), - params=request_params, - paginator=paginator, - data_selector=endpoint_config.get("data_selector") or data_selector, - hooks=hooks, - ) - - return resources - - -def check_connection( - source: DltSource, - *resource_names: str, -) -> Tuple[bool, str]: - try: - list(source.with_resources(*resource_names).add_limit(1)) - return (True, "") - except Exception as e: - logger.error(f"Error checking connection: {e}") - return (False, str(e)) - - -# XXX: This is a workaround pass test_dlt_init.py -# since the source uses dlt.source as a function -def _register_source(source_func: Callable[..., DltSource]) -> None: - import inspect - from dlt.common.configuration import get_fun_spec - from dlt.common.source import _SOURCES, SourceInfo - - spec = get_fun_spec(source_func) - func_module = inspect.getmodule(source_func) - _SOURCES[source_func.__name__] = SourceInfo( - SPEC=spec, - f=source_func, - module=func_module, - ) - - -_register_source(rest_api_source) diff --git a/openapi_python_client/sources/rest_api/auth.py b/openapi_python_client/sources/rest_api/auth.py deleted file mode 100644 index 42ce40745..000000000 --- a/openapi_python_client/sources/rest_api/auth.py +++ /dev/null @@ -1,228 +0,0 @@ -from base64 import b64encode -import math -from typing import Dict, Final, Literal, Optional, Union, Any, cast, Iterable -from dlt.sources.helpers import requests -from requests.auth import AuthBase -from requests import PreparedRequest # noqa: I251 -import pendulum -import jwt -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes - - -from dlt.common import logger - -from dlt.common.configuration.specs.base_configuration import configspec -from dlt.common.configuration.specs import CredentialsConfiguration -from dlt.common.configuration.specs.exceptions import NativeValueError -from dlt.common.typing import TSecretStrValue - - -TApiKeyLocation = Literal[ - "header", "cookie", "query", "param" -] # Alias for scheme "in" field - - -class AuthConfigBase(AuthBase, CredentialsConfiguration): - """Authenticator base which is both `requests` friendly AuthBase and dlt SPEC - configurable via env variables or toml files - """ - - pass - - -@configspec -class BearerTokenAuth(AuthConfigBase): - type: Final[Literal["http"]] = "http" # noqa: A003 - scheme: Literal["bearer"] = "bearer" - token: TSecretStrValue - - # def __init__(self, token: TSecretStrValue) -> None: - # self.token = token - - def parse_native_representation(self, value: Any) -> None: - if isinstance(value, str): - self.token = cast(TSecretStrValue, value) - else: - raise NativeValueError( - type(self), - value, - f"BearerTokenAuth token must be a string, got {type(value)}", - ) - - def __call__(self, request: PreparedRequest) -> PreparedRequest: - request.headers["Authorization"] = f"Bearer {self.token}" - return request - - -@configspec -class APIKeyAuth(AuthConfigBase): - type: Final[Literal["apiKey"]] = "apiKey" # noqa: A003 - location: TApiKeyLocation = "header" - name: str = "Authorization" - api_key: TSecretStrValue - - # def __init__( - # self, name: str, api_key: TSecretStrValue, location: TApiKeyLocation = "header" - # ) -> None: - # self.name = name - # self.api_key = api_key - # self.location = location - - def parse_native_representation(self, value: Any) -> None: - if isinstance(value, str): - self.api_key = cast(TSecretStrValue, value) - else: - raise NativeValueError( - type(self), - value, - f"APIKeyAuth api_key must be a string, got {type(value)}", - ) - - def __call__(self, request: PreparedRequest) -> PreparedRequest: - if self.location == "header": - request.headers[self.name] = self.api_key - elif self.location in ["query", "param"]: - request.prepare_url(request.url, {self.name: self.api_key}) - elif self.location == "cookie": - raise NotImplementedError() - return request - - -@configspec -class HttpBasicAuth(AuthConfigBase): - type: Final[Literal["http"]] = "http" # noqa: A003 - scheme: Literal["basic"] = "basic" - username: str - password: TSecretStrValue - - # def __init__(self, username: str, password: TSecretStrValue) -> None: - # self.username = username - # self.password = password - - def parse_native_representation(self, value: Any) -> None: - if isinstance(value, Iterable) and not isinstance(value, str): - value = list(value) - if len(value) == 2: - self.username, self.password = value - return - raise NativeValueError( - type(self), - value, - f"HttpBasicAuth username and password must be a tuple of two strings, got {type(value)}", - ) - - def __call__(self, request: PreparedRequest) -> PreparedRequest: - encoded = b64encode(f"{self.username}:{self.password}".encode()).decode() - request.headers["Authorization"] = f"Basic {encoded}" - return request - - -@configspec -class OAuth2AuthBase(AuthConfigBase): - """Base class for oauth2 authenticators. requires access_token""" - - # TODO: Separate class for flows (implicit, authorization_code, client_credentials, etc) - type: Final[Literal["oauth2"]] = "oauth2" # noqa: A003 - access_token: TSecretStrValue - - # def __init__(self, access_token: TSecretStrValue) -> None: - # self.access_token = access_token - - def parse_native_representation(self, value: Any) -> None: - if isinstance(value, str): - self.access_token = cast(TSecretStrValue, value) - else: - raise NativeValueError( - type(self), - value, - f"OAuth2AuthBase access_token must be a string, got {type(value)}", - ) - - def __call__(self, request: PreparedRequest) -> PreparedRequest: - request.headers["Authorization"] = f"Bearer {self.access_token}" - return request - - -@configspec -class OAuthJWTAuth(BearerTokenAuth): - """This is a form of Bearer auth, actually there's not standard way to declare it in openAPI""" - - format: Final[Literal["JWT"]] = "JWT" # noqa: A003 - client_id: str - private_key: TSecretStrValue - auth_endpoint: str - scopes: Optional[str] = None - headers: Optional[Dict[str, str]] = None - private_key_passphrase: Optional[TSecretStrValue] = None - - def __init__( - self, - client_id: str, - private_key: TSecretStrValue, - auth_endpoint: str, - scopes: str, - headers: Optional[Dict[str, str]] = None, - private_key_passphrase: Optional[TSecretStrValue] = None, - default_token_expiration: int = 3600, - ): - self.client_id = client_id - self.private_key = private_key - self.private_key_passphrase = private_key_passphrase - self.auth_endpoint = auth_endpoint - self.scopes = scopes if isinstance(scopes, str) else " ".join(scopes) - self.headers = headers - self.token = None - self.token_expiry: Optional[pendulum.DateTime] = None - self.default_token_expiration = default_token_expiration - - def __call__(self, r: PreparedRequest) -> PreparedRequest: - if self.token is None or self.is_token_expired(): - self.obtain_token() - r.headers["Authorization"] = f"Bearer {self.token}" - return r - - def is_token_expired(self) -> bool: - return not self.token_expiry or pendulum.now() >= self.token_expiry - - def obtain_token(self) -> None: - payload = self.create_jwt_payload() - data = { - "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", - "assertion": jwt.encode( - payload, self.load_private_key(), algorithm="RS256" - ), - } - - logger.debug(f"Obtaining token from {self.auth_endpoint}") - - response = requests.post(self.auth_endpoint, headers=self.headers, data=data) - response.raise_for_status() - - token_response = response.json() - self.token = token_response["access_token"] - self.token_expiry = pendulum.now().add( - seconds=token_response.get("expires_in", self.default_token_expiration) - ) - - def create_jwt_payload(self) -> Dict[str, Union[str, int]]: - now = pendulum.now() - return { - "iss": self.client_id, - "sub": self.client_id, - "aud": self.auth_endpoint, - "exp": math.floor((now.add(hours=1)).timestamp()), - "iat": math.floor(now.timestamp()), - "scope": self.scopes, - } - - def load_private_key(self) -> PrivateKeyTypes: - private_key_bytes = self.private_key.encode("utf-8") - return serialization.load_pem_private_key( - private_key_bytes, - password=self.private_key_passphrase.encode("utf-8") - if self.private_key_passphrase - else None, - backend=default_backend(), - ) diff --git a/openapi_python_client/sources/rest_api/client.py b/openapi_python_client/sources/rest_api/client.py deleted file mode 100644 index 82bb8087c..000000000 --- a/openapi_python_client/sources/rest_api/client.py +++ /dev/null @@ -1,255 +0,0 @@ -from typing import ( - Iterator, - Optional, - List, - Dict, - Any, - TypeVar, - Iterable, - Callable, - cast, -) -import copy -from urllib.parse import urlparse - -from requests import Session as BaseSession # noqa: I251 - -from dlt.common import logger -from dlt.common import jsonpath -from dlt.sources.helpers.requests.retry import Client -from dlt.sources.helpers.requests import Response, Request - -from .typing import HTTPMethodBasic, HTTPMethod -from .paginators import BasePaginator -from .auth import AuthConfigBase -from .detector import PaginatorFactory, find_records -from .exceptions import IgnoreResponseException - -from .utils import join_url - - -_T = TypeVar("_T") - - -class PageData(List[_T]): - """A list of elements in a single page of results with attached request context. - - The context allows to inspect the response, paginator and authenticator, modify the request - """ - - def __init__( - self, - __iterable: Iterable[_T], - request: Request, - response: Response, - paginator: BasePaginator, - auth: AuthConfigBase, - ): - super().__init__(__iterable) - self.request = request - self.response = response - self.paginator = paginator - self.auth = auth - - -class RESTClient: - """A generic REST client for making requests to an API with support for - pagination and authentication. - - Args: - base_url (str): The base URL of the API to make requests to. - headers (Optional[Dict[str, str]]): Default headers to include in all requests. - auth (Optional[AuthConfigBase]): Authentication configuration for all requests. - paginator (Optional[BasePaginator]): Default paginator for handling paginated responses. - data_selector (Optional[jsonpath.TJsonPath]): JSONPath selector for extracting data from responses. - session (BaseSession): HTTP session for making requests. - paginator_factory (Optional[PaginatorFactory]): Factory for creating paginator instances, - used for detecting paginators. - """ - - def __init__( - self, - base_url: str, - headers: Optional[Dict[str, str]] = None, - auth: Optional[AuthConfigBase] = None, - paginator: Optional[BasePaginator] = None, - data_selector: Optional[jsonpath.TJsonPath] = None, - session: BaseSession = None, - paginator_factory: Optional[PaginatorFactory] = None, - ) -> None: - self.base_url = base_url - self.headers = headers - self.auth = auth - - if session: - self._validate_session_raise_for_status(session) - self.session = session - else: - self.session = Client(raise_for_status=False).session - - self.paginator = paginator - self.pagination_factory = paginator_factory or PaginatorFactory() - - self.data_selector = data_selector - - def _validate_session_raise_for_status(self, session: BaseSession) -> None: - # dlt.sources.helpers.requests.session.Session - # has raise_for_status=True by default - if getattr(self.session, "raise_for_status", False): - logger.warning( - "The session provided has raise_for_status enabled. " - "This may cause unexpected behavior." - ) - - def _create_request( - self, - path: str, - method: HTTPMethod, - params: Dict[str, Any], - json: Optional[Dict[str, Any]] = None, - auth: Optional[AuthConfigBase] = None, - hooks: Optional[Dict[str, Any]] = None, - ) -> Request: - parsed_url = urlparse(path) - if parsed_url.scheme in ("http", "https"): - url = path - else: - url = join_url(self.base_url, path) - - return Request( - method=method, - url=url, - headers=self.headers, - params=params, - json=json, - auth=auth or self.auth, - hooks=hooks, - ) - - def _send_request(self, request: Request) -> Response: - logger.info( - f"Making {request.method.upper()} request to {request.url}" - f" with params={request.params}, json={request.json}" - ) - - prepared_request = self.session.prepare_request(request) - - return self.session.send(prepared_request) - - def request( - self, path: str = "", method: HTTPMethod = "GET", **kwargs: Any - ) -> Response: - prepared_request = self._create_request( - path=path, - method=method, - **kwargs, - ) - return self._send_request(prepared_request) - - def get( - self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any - ) -> Response: - return self.request(path, method="GET", params=params, **kwargs) - - def post( - self, path: str, json: Optional[Dict[str, Any]] = None, **kwargs: Any - ) -> Response: - return self.request(path, method="POST", json=json, **kwargs) - - def paginate( - self, - path: str = "", - method: HTTPMethodBasic = "GET", - params: Optional[Dict[str, Any]] = None, - json: Optional[Dict[str, Any]] = None, - auth: Optional[AuthConfigBase] = None, - paginator: Optional[BasePaginator] = None, - data_selector: Optional[jsonpath.TJsonPath] = None, - response_actions: Optional[List[Dict[str, Any]]] = None, - hooks: Optional[Dict[str, Any]] = None, - ) -> Iterator[PageData[Any]]: - """Iterates over paginated API responses, yielding pages of data. - - Args: - path (str): Endpoint path for the request, relative to `base_url`. - method (HTTPMethodBasic): HTTP method for the request, defaults to 'get'. - params (Optional[Dict[str, Any]]): URL parameters for the request. - json (Optional[Dict[str, Any]]): JSON payload for the request. - auth (Optional[AuthConfigBase]): Authentication configuration for the request. - paginator (Optional[BasePaginator]): Paginator instance for handling - pagination logic. - data_selector (Optional[jsonpath.TJsonPath]): JSONPath selector for - extracting data from the response. - response_actions (Optional[List[Dict[str, Any]]]): Actions to take based on - response content or status codes. - hooks (Optional[Dict[str, Any]]): Hooks to modify request/response objects. - - Yields: - PageData[Any]: A page of data from the paginated API response, along with request and response context. - - Example: - >>> client = RESTClient(base_url="https://api.example.com") - >>> for page in client.paginate("/search", method="post", json={"query": "foo"}): - >>> print(page) - """ - paginator = paginator if paginator else copy.deepcopy(self.paginator) - auth = auth or self.auth - data_selector = data_selector or self.data_selector - hooks = hooks or {} - - request = self._create_request( - path=path, method=method, params=params, json=json, auth=auth, hooks=hooks - ) - - while True: - try: - response = self._send_request(request) - except IgnoreResponseException: - break - - if paginator is None: - paginator = self.detect_paginator(response) - - data = self.extract_response(response, data_selector) - paginator.update_state(response) - paginator.update_request(request) - - # yield data with context - yield PageData( - data, request=request, response=response, paginator=paginator, auth=auth - ) - - if not paginator.has_next_page: - break - - def extract_response( - self, response: Response, data_selector: jsonpath.TJsonPath - ) -> List[Any]: - if data_selector: - # we should compile data_selector - data: Any = jsonpath.find_values(data_selector, response.json()) - # extract if single item selected - data = data[0] if isinstance(data, list) and len(data) == 1 else data - else: - data = find_records(response.json()) - # wrap single pages into lists - if not isinstance(data, list): - data = [data] - return cast(List[Any], data) - - def detect_paginator(self, response: Response) -> BasePaginator: - """Detects a paginator for the response and returns it. - - Args: - response (Response): The response to detect the paginator for. - - Returns: - BasePaginator: The paginator instance that was detected. - """ - paginator = self.pagination_factory.create_paginator(response) - if paginator is None: - raise ValueError( - f"No suitable paginator found for the response at {response.url}" - ) - logger.info(f"Detected paginator: {paginator.__class__.__name__}") - return paginator diff --git a/openapi_python_client/sources/rest_api/config_setup.py b/openapi_python_client/sources/rest_api/config_setup.py deleted file mode 100644 index 635f9cddf..000000000 --- a/openapi_python_client/sources/rest_api/config_setup.py +++ /dev/null @@ -1,279 +0,0 @@ -import copy -from typing import ( - Type, - Any, - Dict, - Tuple, - List, - Optional, - Union, - Callable, - cast, -) -import graphlib # type: ignore[import,unused-ignore] - -import dlt -from dlt.extract.incremental import Incremental -from dlt.common import logger -from dlt.common.utils import update_dict_nested -from dlt.common.typing import TSecretStrValue -from dlt.sources.helpers.requests import Response - -from .auth import BearerTokenAuth, AuthConfigBase -from .paginators import ( - BasePaginator, - HeaderLinkPaginator, - JSONResponsePaginator, - SinglePagePaginator, - JSONResponseCursorPaginator, -) -from .typing import ( - AuthConfig, - IncrementalArgs, - IncrementalConfig, - PaginatorType, - ResolveConfig, - ResolvedParam, - ResponseAction, - Endpoint, - EndpointResource, - DefaultEndpointResource, -) -from .exceptions import IgnoreResponseException - - -PAGINATOR_MAP: Dict[str, Type[BasePaginator]] = { - "json_links": JSONResponsePaginator, - "header_links": HeaderLinkPaginator, - "auto": None, - "single_page": SinglePagePaginator, - "cursor": JSONResponseCursorPaginator, -} - - -def get_paginator_class(paginator_type: str) -> Type[BasePaginator]: - try: - return PAGINATOR_MAP[paginator_type] - except KeyError: - available_options = ", ".join(PAGINATOR_MAP.keys()) - raise ValueError( - f"Invalid paginator: {paginator_type}. " - f"Available options: {available_options}" - ) - - -def create_paginator(paginator_config: PaginatorType) -> Optional[BasePaginator]: - if isinstance(paginator_config, BasePaginator): - return paginator_config - - if isinstance(paginator_config, str): - paginator_class = get_paginator_class(paginator_config) - return paginator_class() - - if isinstance(paginator_config, dict): - paginator_type = paginator_config.pop("type", "auto") - paginator_class = get_paginator_class(paginator_type) - return paginator_class(**paginator_config) - - return None - - -def create_auth( - auth_config: Optional[Union[AuthConfig, AuthConfigBase]], -) -> Optional[AuthConfigBase]: - if isinstance(auth_config, AuthConfigBase): - return auth_config - return ( - BearerTokenAuth(cast(TSecretStrValue, auth_config.get("token"))) - if auth_config - else None - ) - - -def setup_incremental_object( - request_params: Dict[str, Any], - incremental_config: Optional[IncrementalConfig] = None, -) -> Tuple[Optional[Incremental[Any]], Optional[str]]: - for key, value in request_params.items(): - if isinstance(value, dlt.sources.incremental): - return value, key - if isinstance(value, dict): - param_type = value.pop("type") - if param_type == "incremental": - return ( - dlt.sources.incremental(**value), - key, - ) - if incremental_config: - param = incremental_config.pop("param") - return ( - dlt.sources.incremental(**cast(IncrementalArgs, incremental_config)), - param, - ) - - return None, None - - -def make_parent_key_name(resource_name: str, field_name: str) -> str: - return f"_{resource_name}_{field_name}" - - -def build_resource_dependency_graph( - resource_defaults: DefaultEndpointResource, - resource_list: List[Union[str, EndpointResource]], -) -> Tuple[Any, Dict[str, EndpointResource], Dict[str, Optional[ResolvedParam]]]: - dependency_graph = graphlib.TopologicalSorter() - endpoint_resource_map: Dict[str, EndpointResource] = {} - resolved_param_map: Dict[str, ResolvedParam] = {} - - for resource_kwargs in resource_list: - endpoint_resource = make_endpoint_resource(resource_kwargs, resource_defaults) - - resource_name = endpoint_resource["name"] - - if not isinstance(resource_name, str): - raise ValueError( - f"Resource name must be a string, got {type(resource_name)}" - ) - - if resource_name in endpoint_resource_map: - raise ValueError(f"Resource {resource_name} has already been defined") - - resolved_params = find_resolved_params( - cast(Endpoint, endpoint_resource["endpoint"]) - ) - - if len(resolved_params) > 1: - raise ValueError( - f"Multiple resolved params for resource {resource_name}: {resolved_params}" - ) - - predecessors = set(x.resolve_config.resource_name for x in resolved_params) - - dependency_graph.add(resource_name, *predecessors) - - endpoint_resource_map[resource_name] = endpoint_resource - resolved_param_map[resource_name] = ( - resolved_params[0] if resolved_params else None - ) - - return dependency_graph, endpoint_resource_map, resolved_param_map - - -def make_endpoint_resource( - resource: Union[str, EndpointResource], default_config: EndpointResource -) -> EndpointResource: - """ - Creates an EndpointResource object based on the provided resource - definition and merges it with the default configuration. - - This function supports defining a resource in multiple formats: - - As a string: The string is interpreted as both the resource name - and its endpoint path. - - As a dictionary: The dictionary must include `name` and `endpoint` - keys. The `endpoint` can be a string representing the path, - or a dictionary for more complex configurations. If the `endpoint` - is missing the `path` key, the resource name is used as the `path`. - """ - if isinstance(resource, str): - resource = {"name": resource, "endpoint": {"path": resource}} - return update_dict_nested(copy.deepcopy(default_config), resource) # type: ignore[type-var] - - if "endpoint" in resource and isinstance(resource["endpoint"], str): - resource["endpoint"] = {"path": resource["endpoint"]} - - if "name" not in resource: - raise ValueError("Resource must have a name") - - if "path" not in resource["endpoint"]: - resource["endpoint"]["path"] = resource["name"] # type: ignore - - return update_dict_nested(copy.deepcopy(default_config), resource) # type: ignore[type-var] - - -def make_resolved_param( - key: str, value: Union[ResolveConfig, Dict[str, Any]] -) -> Optional[ResolvedParam]: - if isinstance(value, ResolveConfig): - return ResolvedParam(key, value) - if isinstance(value, dict) and value.get("type") == "resolve": - return ResolvedParam( - key, - ResolveConfig(resource_name=value["resource"], field_path=value["field"]), - ) - return None - - -def find_resolved_params(endpoint_config: Endpoint) -> List[ResolvedParam]: - """ - Find all resolved params in the endpoint configuration and return - a list of ResolvedParam objects. - - Resolved params are either of type ResolveConfig or are dictionaries - with a key "type" set to "resolve". - """ - return [ - make_resolved_param(key, value) - for key, value in endpoint_config.get("params", {}).items() - if isinstance(value, ResolveConfig) - or (isinstance(value, dict) and value.get("type") == "resolve") - ] - - -def _handle_response_actions( - response: Response, actions: List[ResponseAction] -) -> Optional[str]: - """Handle response actions based on the response and the provided actions. - - Example: - response_actions = [ - {"status_code": 404, "action": "ignore"}, - {"content": "Not found", "action": "ignore"}, - {"status_code": 429, "action": "retry"}, - {"status_code": 200, "content": "some text", "action": "retry"}, - ] - action_type = client.handle_response_actions(response, response_actions) - """ - content = response.text - - for action in actions: - status_code = action.get("status_code") - content_substr: str = action.get("content") - action_type: str = action.get("action") - - if status_code is not None and content_substr is not None: - if response.status_code == status_code and content_substr in content: - return action_type - - elif status_code is not None: - if response.status_code == status_code: - return action_type - - elif content_substr is not None: - if content_substr in content: - return action_type - - return None - - -def _create_response_actions_hook( - response_actions: List[ResponseAction], -) -> Callable[[Response, Any, Any], None]: - def response_actions_hook(response: Response, *args: Any, **kwargs: Any) -> None: - action_type = _handle_response_actions(response, response_actions) - if action_type == "ignore": - logger.info( - f"Ignoring response with code {response.status_code} " - f"and content '{response.json()}'." - ) - raise IgnoreResponseException - - return response_actions_hook - - -def create_response_hooks( - response_actions: Optional[List[ResponseAction]], -) -> Optional[Dict[str, Any]]: - if response_actions: - return {"response": [_create_response_actions_hook(response_actions)]} - return None diff --git a/openapi_python_client/sources/rest_api/detector.py b/openapi_python_client/sources/rest_api/detector.py deleted file mode 100644 index 030949566..000000000 --- a/openapi_python_client/sources/rest_api/detector.py +++ /dev/null @@ -1,161 +0,0 @@ -import re -from typing import List, Dict, Any, Tuple, Union, Optional, Callable, Iterable - -from dlt.sources.helpers.requests import Response - -from .paginators import ( - BasePaginator, - HeaderLinkPaginator, - JSONResponsePaginator, - SinglePagePaginator, -) - -RECORD_KEY_PATTERNS = frozenset( - [ - "data", - "items", - "results", - "entries", - "records", - "rows", - "entities", - "payload", - "content", - "objects", - ] -) - -NON_RECORD_KEY_PATTERNS = frozenset( - [ - "meta", - "metadata", - "pagination", - "links", - "extras", - "headers", - ] -) - -NEXT_PAGE_KEY_PATTERNS = frozenset(["next", "nextpage", "nexturl"]) -NEXT_PAGE_DICT_KEY_PATTERNS = frozenset(["href", "url"]) - - -def single_entity_path(path: str) -> bool: - """Checks if path ends with path param indicating that single object is returned""" - return re.search(r"\{([a-zA-Z_][a-zA-Z0-9_]*)\}$", path) is not None - - -def find_all_lists( - dict_: Dict[str, Any], - result: List[Tuple[int, str, List[Any]]] = None, - level: int = 0, -) -> List[Tuple[int, str, List[Any]]]: - """Recursively looks for lists in dict_ and returns tuples - in format (nesting level, dictionary key, list) - """ - if level > 2: - return [] - - for key, value in dict_.items(): - if isinstance(value, list): - result.append((level, key, value)) - elif isinstance(value, dict): - find_all_lists(value, result, level + 1) - - return result - - -def find_records( - response: Union[Dict[str, Any], List[Any], Any], -) -> Union[Dict[str, Any], List[Any], Any]: - # when a list was returned (or in rare case a simple type or null) - if not isinstance(response, dict): - return response - lists = find_all_lists(response, result=[]) - if len(lists) == 0: - # could not detect anything - return response - # we are ordered by nesting level, find the most suitable list - try: - return next( - list_info[2] - for list_info in lists - if list_info[1] in RECORD_KEY_PATTERNS - and list_info[1] not in NON_RECORD_KEY_PATTERNS - ) - except StopIteration: - # return the least nested element - return lists[0][2] - - -def matches_any_pattern(key: str, patterns: Iterable[str]) -> bool: - normalized_key = key.lower() - return any(pattern in normalized_key for pattern in patterns) - - -def find_next_page_key( - dictionary: Dict[str, Any], path: Optional[List[str]] = None -) -> Optional[List[str]]: - if not isinstance(dictionary, dict): - return None - - if path is None: - path = [] - - for key, value in dictionary.items(): - if matches_any_pattern(key, NEXT_PAGE_KEY_PATTERNS): - if isinstance(value, dict): - for dict_key in value: - if matches_any_pattern(dict_key, NEXT_PAGE_DICT_KEY_PATTERNS): - return [*path, key, dict_key] - return [*path, key] - - if isinstance(value, dict): - result = find_next_page_key(value, [*path, key]) - if result: - return result - - return None - - -def header_links_detector(response: Response) -> Optional[HeaderLinkPaginator]: - links_next_key = "next" - - if response.links.get(links_next_key): - return HeaderLinkPaginator() - return None - - -def json_links_detector(response: Response) -> Optional[JSONResponsePaginator]: - dictionary = response.json() - next_path_parts = find_next_page_key(dictionary) - - if not next_path_parts: - return None - - return JSONResponsePaginator(next_url_path=".".join(next_path_parts)) - - -def single_page_detector(response: Response) -> Optional[SinglePagePaginator]: - """This is our fallback paginator, also for results that are single entities""" - return SinglePagePaginator() - - -class PaginatorFactory: - def __init__( - self, detectors: List[Callable[[Response], Optional[BasePaginator]]] = None - ): - if detectors is None: - detectors = [ - header_links_detector, - json_links_detector, - single_page_detector, - ] - self.detectors = detectors - - def create_paginator(self, response: Response) -> Optional[BasePaginator]: - for detector in self.detectors: - paginator = detector(response) - if paginator: - return paginator - return None diff --git a/openapi_python_client/sources/rest_api/exceptions.py b/openapi_python_client/sources/rest_api/exceptions.py deleted file mode 100644 index 4b4d555ca..000000000 --- a/openapi_python_client/sources/rest_api/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -from dlt.common.exceptions import DltException - - -class IgnoreResponseException(DltException): - pass diff --git a/openapi_python_client/sources/rest_api/paginators.py b/openapi_python_client/sources/rest_api/paginators.py deleted file mode 100644 index 59c4f0cbc..000000000 --- a/openapi_python_client/sources/rest_api/paginators.py +++ /dev/null @@ -1,179 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Optional, Sequence, Union, Any - -from dlt.sources.helpers.requests import Response, Request -from dlt.common import jsonpath - -from .utils import create_nested_accessor - - -class BasePaginator(ABC): - def __init__(self) -> None: - self._has_next_page = True - self._next_reference: Optional[str] = None - - @property - def has_next_page(self) -> bool: - """ - Check if there is a next page available. - - Returns: - bool: True if there is a next page available, False otherwise. - """ - return self._has_next_page - - @property - def next_reference(self) -> Optional[str]: - return self._next_reference - - @next_reference.setter - def next_reference(self, value: Optional[str]) -> None: - self._next_reference = value - self._has_next_page = value is not None - - @abstractmethod - def update_state(self, response: Response) -> None: - """Update the paginator state based on the response. - - Args: - response (Response): The response object from the API. - """ - ... - - @abstractmethod - def update_request(self, request: Request) -> None: - """ - Update the request object with the next arguments for the API request. - - Args: - request (Request): The request object to be updated. - """ - ... - - -class SinglePagePaginator(BasePaginator): - """A paginator for single-page API responses.""" - - def update_state(self, response: Response) -> None: - self._has_next_page = False - - def update_request(self, request: Request) -> None: - return - - -class OffsetPaginator(BasePaginator): - """A paginator that uses the 'offset' parameter for pagination.""" - - def __init__( - self, - initial_offset: int, - initial_limit: int, - offset_key: str = "offset", - limit_key: str = "limit", - total_key: str = "total", - ) -> None: - super().__init__() - self.offset_key = offset_key - self.limit_key = limit_key - self._total_accessor = create_nested_accessor(total_key) - - self.offset = initial_offset - self.limit = initial_limit - - def update_state(self, response: Response) -> None: - total = self._total_accessor(response.json()) - - if total is None: - raise ValueError( - f"Total count not found in response for {self.__class__.__name__}" - ) - - self.offset += self.limit - - if self.offset >= total: - self._has_next_page = False - - def update_request(self, request: Request) -> None: - if request.params is None: - request.params = {} - - request.params[self.offset_key] = self.offset - request.params[self.limit_key] = self.limit - - -class BaseNextUrlPaginator(BasePaginator): - def update_request(self, request: Request) -> None: - request.url = self.next_reference - - -class HeaderLinkPaginator(BaseNextUrlPaginator): - """A paginator that uses the 'Link' header in HTTP responses - for pagination. - - A good example of this is the GitHub API: - https://docs.github.com/en/rest/guides/traversing-with-pagination - """ - - def __init__(self, links_next_key: str = "next") -> None: - """ - Args: - links_next_key (str, optional): The key (rel ) in the 'Link' header - that contains the next page URL. Defaults to 'next'. - """ - super().__init__() - self.links_next_key = links_next_key - - def update_state(self, response: Response) -> None: - self.next_reference = response.links.get(self.links_next_key, {}).get("url") - - -class JSONResponsePaginator(BaseNextUrlPaginator): - """A paginator that uses a specific key in the JSON response to find - the next page URL. - """ - - def __init__( - self, - next_url_path: jsonpath.TJsonPath = "next", - ): - """ - Args: - next_url_path: The JSON path to the key that contains the next page URL in the response. - Defaults to 'next'. - """ - super().__init__() - self.next_url_path = jsonpath.compile_path(next_url_path) - - def update_state(self, response: Response) -> None: - values = jsonpath.find_values(self.next_url_path, response.json()) - self.next_reference = values[0] if values else None - - -class JSONResponseCursorPaginator(BasePaginator): - """A paginator that uses a cursor query param to paginate. The cursor for the - next page is found in the JSON response. - """ - - def __init__( - self, - cursor_path: jsonpath.TJsonPath = "cursors.next", - cursor_param: str = "after", - ): - """ - Args: - cursor_path: The JSON path to the key that contains the cursor in the response. - cursor_param: The name of the query parameter to be used in the request to get the next page. - """ - super().__init__() - self.cursor_path = jsonpath.compile_path(cursor_path) - self.cursor_param = cursor_param - - def update_state(self, response: Response) -> None: - values = jsonpath.find_values(self.cursor_path, response.json()) - self.next_reference = values[0] if values else None - - def update_request(self, request: Request) -> None: - if request.params is None: - request.params = {} - - request.params[self.cursor_param] = self._next_reference diff --git a/openapi_python_client/sources/rest_api/requirements.txt b/openapi_python_client/sources/rest_api/requirements.txt deleted file mode 100644 index 68076b836..000000000 --- a/openapi_python_client/sources/rest_api/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -dlt>=0.4.7a0 \ No newline at end of file diff --git a/openapi_python_client/sources/rest_api/typing.py b/openapi_python_client/sources/rest_api/typing.py deleted file mode 100644 index 50a46d43a..000000000 --- a/openapi_python_client/sources/rest_api/typing.py +++ /dev/null @@ -1,110 +0,0 @@ -from typing import ( - Any, - Dict, - List, - NamedTuple, - Optional, - TypedDict, - Union, - Literal, -) - -from dlt.common import jsonpath -from dlt.common.typing import TSortOrder -from dlt.extract.items import TTableHintTemplate -from dlt.extract.incremental.typing import LastValueFunc - -from .paginators import BasePaginator -from .auth import AuthConfigBase - -from dlt.common.schema.typing import ( - TColumnNames, - # TSchemaContract, - TTableFormat, - TTableSchemaColumns, - TWriteDisposition, -) - -PaginatorConfigDict = Dict[str, Any] -PaginatorType = Union[BasePaginator, str, PaginatorConfigDict] - -HTTPMethodBasic = Literal["GET", "POST"] -HTTPMethodExtended = Literal["PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"] -HTTPMethod = Union[HTTPMethodBasic, HTTPMethodExtended] - - -class AuthConfig(TypedDict, total=False): - token: str - - -class ClientConfig(TypedDict, total=False): - base_url: str - auth: Optional[Union[AuthConfig, AuthConfigBase]] - paginator: Optional[PaginatorType] - - -class IncrementalArgs(TypedDict, total=False): - cursor_path: str - initial_value: Optional[str] - last_value_func: LastValueFunc[str] - primary_key: Optional[TTableHintTemplate[TColumnNames]] - end_value: Optional[str] - row_order: Optional[TSortOrder] - - -class IncrementalConfig(IncrementalArgs, total=False): - param: str - - -class ResolveConfig(NamedTuple): - resource_name: str - field_path: str - - -class ResolvedParam(NamedTuple): - param_name: str - resolve_config: ResolveConfig - - -class ResponseAction(TypedDict, total=False): - status_code: Optional[Union[int, str]] - content: Optional[str] - action: str - - -class Endpoint(TypedDict, total=False): - path: Optional[str] - method: Optional[HTTPMethodBasic] - params: Optional[Dict[str, Any]] - json: Optional[Dict[str, Any]] - paginator: Optional[PaginatorType] - data_selector: Optional[jsonpath.TJsonPath] - response_actions: Optional[List[ResponseAction]] - - -class EndpointResourceBase(TypedDict, total=False): - endpoint: Optional[Union[str, Endpoint]] - write_disposition: Optional[TTableHintTemplate[TWriteDisposition]] - parent: Optional[TTableHintTemplate[str]] - columns: Optional[TTableHintTemplate[TTableSchemaColumns]] - primary_key: Optional[TTableHintTemplate[TColumnNames]] - merge_key: Optional[TTableHintTemplate[TColumnNames]] - incremental: Optional[IncrementalConfig] - table_format: Optional[TTableHintTemplate[TTableFormat]] - include_from_parent: Optional[List[str]] - selected: Optional[bool] - - -# NOTE: redefining properties of TypedDict is not allowed -class EndpointResource(EndpointResourceBase, total=False): - name: TTableHintTemplate[str] - - -class DefaultEndpointResource(EndpointResourceBase, total=False): - name: Optional[TTableHintTemplate[str]] - - -class RESTAPIConfig(TypedDict): - client: ClientConfig - resource_defaults: Optional[DefaultEndpointResource] - resources: List[Union[str, EndpointResource]] diff --git a/openapi_python_client/sources/rest_api/utils.py b/openapi_python_client/sources/rest_api/utils.py deleted file mode 100644 index 61640ba31..000000000 --- a/openapi_python_client/sources/rest_api/utils.py +++ /dev/null @@ -1,15 +0,0 @@ -from functools import reduce -from operator import getitem -from typing import Any, Dict, Mapping, Sequence, Union - - -def join_url(base_url: str, path: str) -> str: - if not base_url.endswith("/"): - base_url += "/" - return base_url + path.lstrip("/") - - -def create_nested_accessor(path: Union[str, Sequence[str]]) -> Any: - if isinstance(path, (list, tuple)): - return lambda d: reduce(getitem, path, d) - return lambda d: d.get(path) diff --git a/openapi_python_client/sources/rest_api_pipeline.py b/openapi_python_client/sources/rest_api_pipeline.py deleted file mode 100644 index 67d3e569d..000000000 --- a/openapi_python_client/sources/rest_api_pipeline.py +++ /dev/null @@ -1,131 +0,0 @@ -import dlt -from rest_api import RESTAPIConfig, check_connection, rest_api_source - - -def load_github() -> None: - pipeline = dlt.pipeline( - pipeline_name="rest_api_github_v3", - destination='postgres', - dataset_name="rest_api_data", - ) - - github_config: RESTAPIConfig = { - "client": { - "base_url": "https://api.github.com/repos/dlt-hub/dlt/", - "auth": { - "token": dlt.secrets["github_token"], - }, - }, - # Default params for all resouces and their endpoints - "resource_defaults": { - "primary_key": "id", - "write_disposition": "merge", - "endpoint": { - "params": { - "per_page": 100, - }, - }, - }, - "resources": [ - # "pulls", <- This is both name and endpoint path - # { - # "name": "pulls", - # "endpoint": "pulls", # <- This is the endpoint path - # } - { - "name": "issues", - "endpoint": { - "path": "issues", - "params": { - "sort": "updated", - "direction": "desc", - "state": "open", - "since": { - "type": "incremental", - "cursor_path": "updated_at", - "initial_value": "2024-01-25T11:21:28Z", - }, - }, - }, - }, - { - "name": "issue_comments", - "endpoint": { - "path": "issues/{issue_number}/comments", - "params": { - "issue_number": { - "type": "resolve", - "resource": "issues", - "field": "number", - } - }, - }, - "include_from_parent": ["id"], - }, - ], - } - - not_connecting_config: RESTAPIConfig = { - **github_config, - "client": { - "base_url": "https://api.github.com/repos/dlt-hub/dlt/", - "auth": {"token": "invalid token"}, - }, - } - - not_connecting_gh_source = rest_api_source(not_connecting_config) - (can_connect, error_msg) = check_connection(not_connecting_gh_source, "issues") - assert not can_connect, "A miracle happened. Token should be invalid" - - github_source = rest_api_source(github_config) - - load_info = pipeline.run(github_source) - print(load_info) - - -def load_pokemon() -> None: - pipeline = dlt.pipeline( - pipeline_name="rest_api_pokemon", - destination='postgres', - dataset_name="rest_api_data", - ) - - pokemon_source = rest_api_source( - { - "client": { - "base_url": "https://pokeapi.co/api/v2/", - # If you leave out the paginator, it will be inferred from the API: - # paginator: "json_links", - }, - "resource_defaults": { - "endpoint": { - "params": { - "limit": 1000, - }, - }, - }, - "resources": [ - "pokemon", - "berry", - "location", - ], - } - ) - - def check_network_and_authentication() -> None: - (can_connect, error_msg) = check_connection( - pokemon_source, - "not_existing_endpoint", - ) - if not can_connect: - pass # do something with the error message - - check_network_and_authentication() - - load_info = pipeline.run(pokemon_source) - print(load_info) - - -if __name__ == "__main__": - load_github() - load_pokemon() diff --git a/openapi_python_client/templates/pyproject.toml.jinja b/openapi_python_client/templates/pyproject.toml.jinja index 8f544fcd9..01941e3f1 100644 --- a/openapi_python_client/templates/pyproject.toml.jinja +++ b/openapi_python_client/templates/pyproject.toml.jinja @@ -14,7 +14,7 @@ include = ["CHANGELOG.md", "{{ package_name }}/py.typed"] [tool.poetry.dependencies] python = "^3.8" -dlt = "^0.2.6" +dlt = {git = "https://github.com/dlt-hub/dlt.git", rev = "b8fb7fd2"} [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/openapi_python_client/templates/source.py.jinja b/openapi_python_client/templates/source.py.jinja index 4b89a3b3d..3c170e728 100644 --- a/openapi_python_client/templates/source.py.jinja +++ b/openapi_python_client/templates/source.py.jinja @@ -2,7 +2,7 @@ from typing import List from dlt.extract.source import DltResource -from .rest_api.client import RESTClient +from dlt.sources.helpers.rest_client import RestClient {% for import_string in imports %} {{ import_string }} diff --git a/poetry.lock b/poetry.lock index cb30058fe..279e728fb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -471,21 +471,19 @@ files = [ [[package]] name = "dlt" -version = "0.4.6" +version = "0.4.7" description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run." optional = false python-versions = ">=3.8.1,<3.13" -files = [ - {file = "dlt-0.4.6-py3-none-any.whl", hash = "sha256:ab1f9f4cdb645316a9e66170e8d2dec0571426d781253456ff90d2238894adab"}, - {file = "dlt-0.4.6.tar.gz", hash = "sha256:320d4f34c304eb20f3b0eec2b7ee78415bb8605d540528131ccfa67fba5fb59a"}, -] +files = [] +develop = false [package.dependencies] astunparse = ">=1.6.3" click = ">=7.1" duckdb = [ - {version = ">=0.6.1,<0.10.0", optional = true, markers = "python_version >= \"3.8\" and python_version < \"3.12\" and (extra == \"duckdb\" or extra == \"motherduck\")"}, - {version = ">=0.10.0,<0.11.0", optional = true, markers = "python_version >= \"3.12\" and (extra == \"duckdb\" or extra == \"motherduck\")"}, + {version = ">=0.6.1,<0.10.0", optional = true, markers = "python_version >= \"3.8\" and python_version < \"3.12\""}, + {version = ">=0.10.0,<0.11.0", optional = true, markers = "python_version >= \"3.12\""}, ] fsspec = ">=2022.4.0" gitpython = ">=3.1.29" @@ -533,6 +531,12 @@ snowflake = ["snowflake-connector-python (>=3.5.0)"] synapse = ["adlfs (>=2022.4.0)", "pyarrow (>=12.0.0)", "pyodbc (>=4.0.39,<5.0.0)"] weaviate = ["weaviate-client (>=3.22)"] +[package.source] +type = "git" +url = "https://github.com/dlt-hub/dlt.git" +reference = "b8fb7fd2" +resolved_reference = "b8fb7fd23e7ed81981c41eb1eb89e68d4cdb757c" + [[package]] name = "dparse" version = "0.6.2" @@ -2137,4 +2141,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8.1,<3.13" -content-hash = "8a9a25ec594d7a872a48d4c8f86a96beac5ed452b11a56dee2c6879119e70106" +content-hash = "45f32def51315920572396ac57fdec70976153cd28bcde5f462609f7938446e7" diff --git a/pyproject.toml b/pyproject.toml index 7adfc9bc2..58a199b98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ python-dateutil = "^2.8.1" httpx = ">=0.15.4,<0.25.0" autoflake = "^1.4 || ^2.0.0" PyYAML = "^6.0" -dlt = {extras = ["duckdb"], version = "^0.4.0"} +dlt = {git = "https://github.com/dlt-hub/dlt.git", rev = "b8fb7fd2", extras = ["duckdb"]} requests = "^2.30.0" questionary = "^1.10.0" enlighten = "^1.11.2" From 2e417c37567d335c7f151f3d0053662d1084bb36 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Tue, 26 Mar 2024 14:06:47 +0530 Subject: [PATCH 07/18] Fixed dependency --- poetry.lock | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 279e728fb..9cdbc6e18 100644 --- a/poetry.lock +++ b/poetry.lock @@ -534,8 +534,8 @@ weaviate = ["weaviate-client (>=3.22)"] [package.source] type = "git" url = "https://github.com/dlt-hub/dlt.git" -reference = "b8fb7fd2" -resolved_reference = "b8fb7fd23e7ed81981c41eb1eb89e68d4cdb757c" +reference = "sthor/remove-rest-client-cyclic-import" +resolved_reference = "09e306ac3e071403a0d7365c741b30109876ee2b" [[package]] name = "dparse" @@ -2141,4 +2141,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8.1,<3.13" -content-hash = "45f32def51315920572396ac57fdec70976153cd28bcde5f462609f7938446e7" +content-hash = "2ea49a42c60916ca621a5dc118bdca8be787fd7c5ce6ae0ce305565abaeceaad" diff --git a/pyproject.toml b/pyproject.toml index 58a199b98..0059f5c09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ python-dateutil = "^2.8.1" httpx = ">=0.15.4,<0.25.0" autoflake = "^1.4 || ^2.0.0" PyYAML = "^6.0" -dlt = {git = "https://github.com/dlt-hub/dlt.git", rev = "b8fb7fd2", extras = ["duckdb"]} +dlt = {git = "https://github.com/dlt-hub/dlt.git", branch="sthor/remove-rest-client-cyclic-import", extras = ["duckdb"]} requests = "^2.30.0" questionary = "^1.10.0" enlighten = "^1.11.2" From 88918c05994a8de6af8ac39a1a35ee83c1c713db Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Tue, 26 Mar 2024 14:07:01 +0530 Subject: [PATCH 08/18] Detect cursor and offset pagination --- openapi_python_client/parser/models.py | 90 ++++++------- openapi_python_client/parser/pagination.py | 119 +++++++++++------- openapi_python_client/parser/parameters.py | 4 + .../templates/endpoint_macros.py.jinja | 2 +- .../templates/pyproject.toml.jinja | 2 +- .../templates/source.py.jinja | 3 +- 6 files changed, 131 insertions(+), 89 deletions(-) diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index af6141174..65731e65f 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -90,6 +90,9 @@ class SchemaWrapper: type_format: Optional[str] = None """Format (e.g. datetime, uuid) as an extension of the data type""" + maximum: Optional[float] = None + """Maximum value for number type if applicable""" + array_item: Optional["SchemaWrapper"] = None all_of: List["SchemaWrapper"] = field(default_factory=list) any_of: List["SchemaWrapper"] = field(default_factory=list) @@ -262,6 +265,7 @@ def from_reference( crawled_properties=crawler, hash_key=digest128(schema.json(sort_keys=True)), type_format=schema.schema_format, + maximum=schema.maximum, ) crawler.crawl(result) return result @@ -363,49 +367,49 @@ def is_optional(self, path: Tuple[str, ...]) -> bool: check_path.pop() return False - def find_property_by_name(self, name: str, fallback: Optional[str] = None) -> Optional[DataPropertyPath]: - """Find a property with the given name somewhere in the object tree. - - Prefers paths higher up in the object over deeply nested paths. - - Args: - name: The name of the property to look for - fallback: Optional fallback property to get when `name` is not found - - Returns: - If property is found, `DataPropertyPath` object containing the corresponding schema and path tuple/json path - """ - named = [] - fallbacks = [] - named_optional = [] - fallbacks_optional = [] - for path, prop in self.all_properties.items(): - if name in path: - if self._is_optional(path): - named_optional.append((path, prop)) - else: - named.append((path, prop)) - if fallback and fallback in path: - if self._is_optional(path): - fallbacks_optional.append((path, prop)) - else: - fallbacks.append((path, prop)) - # Prefer the least nested path - named.sort(key=lambda item: len(item[0])) - fallbacks.sort(key=lambda item: len(item[0])) - named_optional.sort(key=lambda item: len(item[0])) - fallbacks_optional.sort(key=lambda item: len(item[0])) - # Prefer required property and required fallback over optional properties - # If not required props found, assume the spec is wrong and optional properties are required in practice - if named: - return DataPropertyPath(*named[0]) - elif fallbacks: - return DataPropertyPath(*fallbacks[0]) - elif named_optional: - return DataPropertyPath(*named_optional[0]) - elif fallbacks_optional: - return DataPropertyPath(*fallbacks_optional[0]) - return None + # def find_property_by_name(self, name: str, fallback: Optional[str] = None) -> Optional[DataPropertyPath]: + # """Find a property with the given name somewhere in the object tree. + + # Prefers paths higher up in the object over deeply nested paths. + + # Args: + # name: The name of the property to look for + # fallback: Optional fallback property to get when `name` is not found + + # Returns: + # If property is found, `DataPropertyPath` object containing the corresponding schema and path tuple/json path + # """ + # named = [] + # fallbacks = [] + # named_optional = [] + # fallbacks_optional = [] + # for path, prop in self.all_properties.items(): + # if name in path: + # if self._is_optional(path): + # named_optional.append((path, prop)) + # else: + # named.append((path, prop)) + # if fallback and fallback in path: + # if self._is_optional(path): + # fallbacks_optional.append((path, prop)) + # else: + # fallbacks.append((path, prop)) + # # Prefer the least nested path + # named.sort(key=lambda item: len(item[0])) + # fallbacks.sort(key=lambda item: len(item[0])) + # named_optional.sort(key=lambda item: len(item[0])) + # fallbacks_optional.sort(key=lambda item: len(item[0])) + # # Prefer required property and required fallback over optional properties + # # If not required props found, assume the spec is wrong and optional properties are required in practice + # if named: + # return DataPropertyPath(*named[0]) + # elif fallbacks: + # return DataPropertyPath(*fallbacks[0]) + # elif named_optional: + # return DataPropertyPath(*named_optional[0]) + # elif fallbacks_optional: + # return DataPropertyPath(*fallbacks_optional[0]) + # return None def crawl(self, schema: SchemaWrapper, path: Tuple[str, ...] = ()) -> None: self.all_properties[path] = schema diff --git a/openapi_python_client/parser/pagination.py b/openapi_python_client/parser/pagination.py index 42b3bbfb9..f525bb90c 100644 --- a/openapi_python_client/parser/pagination.py +++ b/openapi_python_client/parser/pagination.py @@ -1,66 +1,99 @@ import re -from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Optional, List, Dict, Tuple, Any + +from openapi_python_client.parser.models import DataPropertyPath if TYPE_CHECKING: from openapi_python_client.parser.endpoints import Endpoint, Parameter RE_OFFSET_PARAM = re.compile(r"(?i)(page|start|offset)") +RE_LIMIT_PARAM = re.compile(r"(?i)(limit|per_page|page_size)") RE_CURSOR_PARAM = re.compile(r"(?i)(cursor|after|since)") -OFFSET_PARAM_KEYWORDS = {"offset", "page", "start"} - @dataclass class Pagination: + pagination_params: List["Parameter"] = field(default_factory=list) + paginator_class: Optional[str] = None + paginator_config: Optional[Dict[str, str]] = None + @classmethod - def from_endpoint(cls, endpoint: "Endpoint") -> None: + def from_endpoint(cls, endpoint: "Endpoint") -> "Pagination": resp = endpoint.data_response if not resp or not resp.content_schema: raise ValueError(f"Endpoint {endpoint.path} does not have a content response") - crawler = resp.content_schema.crawled_properties - offset_param: Optional["Parameter"] = None - cursor_param: Optional["Parameter"] = None - for name, param in endpoint.parameters.items(): - if not offset_param and "integer" in param.schema.types and RE_OFFSET_PARAM.search(name): - offset_param = param - if not cursor_param and "string" in param.schema.types and RE_CURSOR_PARAM.search(name): - cursor_param = param + offset_params: List["Parameter"] = [] + cursor_params: List["Parameter"] = [] + limit_params: List["Parameter"] = [] + # Find params matching regexes + for param_name, param in endpoint.parameters.items(): + if RE_OFFSET_PARAM.match(param_name): + # Offset is a number type + if "integer" in param.types: + offset_params.append(param) + if RE_LIMIT_PARAM.match(param_name): + # Limit should be a number type + if "integer" in param.types: + limit_params.append(param) + if RE_CURSOR_PARAM.match(param_name): + # Cursor should be a string or integer type + if "string" in param.types or "integer" in param.types: + cursor_params.append(param) - parameter_names = set(endpoint.parameters.keys()) - property_paths = [".".join(k) for k in crawler.all_properties.keys()] + cursor_props: List[Tuple["Parameter", DataPropertyPath]] = [] + for cursor_param in cursor_params: + # Try to response property to feed into the cursor param + prop = cursor_param.find_input_property(resp.content_schema, fallback=None) + if prop: + cursor_props.append((cursor_param, prop)) - prompt_params = [] - for param in endpoint.parameters.values(): - prompt_params.append( - dict( - name=param.name, - location=param.location, - description=param.description, - types=param.schema.types, - ) + pagination_config: Optional[Dict[str, Any]] = None + # Prefer the least nested cursor prop + if cursor_props: + cursor_props.sort(key=lambda x: len(x[1].path)) + cursor_param, cursor_prop = cursor_props[0] + pagination_config = { + "cursor_path": cursor_prop.json_path, + "cursor_param": cursor_param.name, + } + return cls( + paginator_class="JSONResponseCursorPaginator", + paginator_config=pagination_config, + pagination_params=[cursor_param], ) - prompt_props = [] - for path, prop in crawler.all_properties.items(): - prompt_props.append( - dict( - json_path=".".join(path), - description=prop.description, - types=prop.types, - ) + offset_props: List[Tuple["Parameter", DataPropertyPath]] = [] + offset_prop: Optional[DataPropertyPath] = None + offset_param: Optional["Parameter"] = None + limit_param: Optional["Parameter"] = None + limit_initial: Optional[int] = None + for offset_param in offset_params: + # Try to response property to feed into the offset param + prop = offset_param.find_input_property(resp.content_schema, fallback=None) + if prop: + offset_props.append((offset_param, prop)) + # Prefer least nested offset prop + if offset_props: + offset_props.sort(key=lambda x: len(x[1].path)) + offset_param, offset_prop = offset_props[0] + for limit_param in limit_params: + # When spec doesn't provide default/max limit, fallback to a conservative default + # 20 should be safe for most APIs + limit_initial = int(limit_param.maximum) if limit_param.maximum else (limit_param.default or 20) + if offset_param and offset_prop and limit_param and limit_initial: + pagination_config = { + "initial_limit": limit_initial, + "offset_param": offset_param.name, + "limit_param": limit_param.name, + } + return cls( + paginator_class="OffsetPaginator", + paginator_config=pagination_config, + pagination_params=[offset_param, limit_param], ) - # import json - - # print(json.dumps(prompt_params, indent=2)) - # print(json.dumps(prompt_props, indent=2)) - - # if offset_param: - # print("OFFSET PARAM", offset_param.name, offset_param.description) - # if cursor_param: - # if cursor_param.name == "cursor": - # breakpoint() - # print("CURSOR PARAM", cursor_param.name, cursor_param.description) + # No pagination detected + return cls() diff --git a/openapi_python_client/parser/parameters.py b/openapi_python_client/parser/parameters.py index f318cab44..3ed094303 100644 --- a/openapi_python_client/parser/parameters.py +++ b/openapi_python_client/parser/parameters.py @@ -49,6 +49,10 @@ def default(self) -> Optional[Any]: def nullable(self) -> bool: return self.schema.nullable + @property + def maximum(self) -> Optional[float]: + return self.schema.maximum + @property def type_hint(self) -> str: return DataType.from_schema(self.schema, required=self.required).type_hint diff --git a/openapi_python_client/templates/endpoint_macros.py.jinja b/openapi_python_client/templates/endpoint_macros.py.jinja index d1f50c392..ad2776b7b 100644 --- a/openapi_python_client/templates/endpoint_macros.py.jinja +++ b/openapi_python_client/templates/endpoint_macros.py.jinja @@ -89,7 +89,7 @@ credentials: {{ endpoint.credentials.type_hint }} = dlt.secrets.value, data_json_path: Optional[str]="{{endpoint.data_json_path}}", {% if endpoint.transformer %} {# Render the dict of path params #} -path_parameter_paths={{ endpoint.transformer.path_params_mapping }} +path_parameter_paths: Dict[str, TJsonPath]={{ endpoint.transformer.path_params_mapping }} {% endif %} {% endmacro %} diff --git a/openapi_python_client/templates/pyproject.toml.jinja b/openapi_python_client/templates/pyproject.toml.jinja index 01941e3f1..a49054a14 100644 --- a/openapi_python_client/templates/pyproject.toml.jinja +++ b/openapi_python_client/templates/pyproject.toml.jinja @@ -14,7 +14,7 @@ include = ["CHANGELOG.md", "{{ package_name }}/py.typed"] [tool.poetry.dependencies] python = "^3.8" -dlt = {git = "https://github.com/dlt-hub/dlt.git", rev = "b8fb7fd2"} +dlt = {git = "https://github.com/dlt-hub/dlt.git", branch = "sthor/remove-rest-client-cyclic-import"} [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/openapi_python_client/templates/source.py.jinja b/openapi_python_client/templates/source.py.jinja index 3c170e728..39ab27a1c 100644 --- a/openapi_python_client/templates/source.py.jinja +++ b/openapi_python_client/templates/source.py.jinja @@ -2,7 +2,8 @@ from typing import List from dlt.extract.source import DltResource -from dlt.sources.helpers.rest_client import RestClient +from dlt.sources.helpers.rest_client import RESTClient +from dlt.common.jsonpath import TJsonPath {% for import_string in imports %} {{ import_string }} From 84aebd1ed4474e9d97fee62894d33036dd041a57 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Tue, 26 Mar 2024 16:21:43 +0530 Subject: [PATCH 09/18] Todo pagination in template --- openapi_python_client/parser/endpoints.py | 18 ++++++++++- openapi_python_client/parser/pagination.py | 15 +++++++--- .../templates/endpoint_macros.py.jinja | 30 +++---------------- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index 2e2296011..d7fff2d52 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -125,6 +125,8 @@ class Endpoint: path_description: Optional[str] = None """Description applying to all methods of the path""" + pagination: Optional[Pagination] = None + rank: int = 0 def get_imports(self) -> List[str]: @@ -189,6 +191,18 @@ def required_parameters(self) -> Dict[str, Parameter]: def optional_parameters(self) -> Dict[str, Parameter]: return {name: p for name, p in self.parameters.items() if not p.required} + def resource_parameters( + self, required: bool = True, optional: bool = True, pagination: bool = False + ) -> Dict[str, Parameter]: + result = {} + if required: + result.update(self.required_parameters) + if optional: + result.update(self.optional_parameters) + if not pagination and self.pagination: + result = {name: p for name, p in result.items() if p.name not in self.pagination.param_names} + return result + @property def request_args_meta(self) -> Dict[str, Dict[str, Dict[str, str]]]: """Mapping of how to translate python arguments to request parameters""" @@ -324,7 +338,7 @@ def from_operation( credentials = CredentialsProperty.from_requirements(operation.security, context) if operation.security else None - return cls( + endpoint = cls( method=method, path=path, raw_schema=operation, @@ -339,6 +353,8 @@ def from_operation( path_description=path_description, credentials=credentials, ) + endpoint.pagination = Pagination.from_endpoint(endpoint) + return endpoint @dataclass diff --git a/openapi_python_client/parser/pagination.py b/openapi_python_client/parser/pagination.py index f525bb90c..287d2b533 100644 --- a/openapi_python_client/parser/pagination.py +++ b/openapi_python_client/parser/pagination.py @@ -16,8 +16,13 @@ @dataclass class Pagination: pagination_params: List["Parameter"] = field(default_factory=list) - paginator_class: Optional[str] = None - paginator_config: Optional[Dict[str, str]] = None + paginator_class: str = None + paginator_config: Dict[str, str] = None + + @property + def param_names(self) -> List[str]: + """All params used for pagination""" + return [param.name for param in self.pagination_params] @classmethod def from_endpoint(cls, endpoint: "Endpoint") -> "Pagination": @@ -79,11 +84,13 @@ def from_endpoint(cls, endpoint: "Endpoint") -> "Pagination": if offset_props: offset_props.sort(key=lambda x: len(x[1].path)) offset_param, offset_prop = offset_props[0] + elif offset_params: # No matching property found in response, fallback to use the first param detected + offset_param = offset_params[0] for limit_param in limit_params: # When spec doesn't provide default/max limit, fallback to a conservative default # 20 should be safe for most APIs limit_initial = int(limit_param.maximum) if limit_param.maximum else (limit_param.default or 20) - if offset_param and offset_prop and limit_param and limit_initial: + if offset_param and limit_param and limit_initial: pagination_config = { "initial_limit": limit_initial, "offset_param": offset_param.name, @@ -96,4 +103,4 @@ def from_endpoint(cls, endpoint: "Endpoint") -> "Pagination": ) # No pagination detected - return cls() + return None diff --git a/openapi_python_client/templates/endpoint_macros.py.jinja b/openapi_python_client/templates/endpoint_macros.py.jinja index ad2776b7b..871c689e0 100644 --- a/openapi_python_client/templates/endpoint_macros.py.jinja +++ b/openapi_python_client/templates/endpoint_macros.py.jinja @@ -53,35 +53,16 @@ params = {k: v for k, v in params.items() if v is not UNSET and v is not None} {% endmacro %} -{# The all the kwargs passed into an endpoint (and variants thereof)) #} -{% macro arguments(endpoint) %} -rest_client: RESTClient, -{% for parameter in endpoint.required_parameters.values() %} -{{ parameter.to_string() }}, -{% endfor %} -{% for parameter in endpoint.optional_parameters.values() %} -{{ parameter.to_string() }}, -{% endfor %} -{# Security #} -{% if endpoint.credentials_parameter %} -{{ endpoint.credentials_parameter.to_string('dlt.secrets.value') }}, -{% endif %} -{% endmacro %} - {% macro resource_arguments(endpoint) %} {% if endpoint.transformer %} data: Any, rest_client: RESTClient, {% else %} rest_client: RESTClient, -{% for parameter in endpoint.required_parameters.values() %} +{% for parameter in endpoint.resource_parameters().values() %} {{ parameter.to_string() }}, {% endfor %} {% endif %} -{% for parameter in endpoint.optional_parameters.values() %} -{{ parameter.to_string() }}, -{% endfor %} -base_url: str = dlt.config.value, {# Security #} {% if endpoint.credentials %} credentials: {{ endpoint.credentials.type_hint }} = dlt.secrets.value, @@ -95,19 +76,16 @@ path_parameter_paths: Dict[str, TJsonPath]={{ endpoint.transformer.path_params_m {# Just lists all kwargs to endpoints as name=name for passing to other functions #} {% macro kwargs(endpoint) %} -{% for parameter in endpoint.required_parameters.values() %} -{{ parameter.python_name }}={{ parameter.python_name }}, +{% for parameter in endpoint.resource_parameters().values() %} +{{ parameter.python_name }}={{ parameter.python_name }}, {% endfor %} -{% for parameter in endpoint.optional_parameters.values() %} -{{ parameter.python_name }}={{ parameter.python_name }}, -{% endfor %} {% endmacro %} {% macro transformer_kwargs(endpoint) %} {#{% for parameter in endpoint.required_parameters.values() %} #} {#{{ parameter.python_name }}={{ parameter.python_name }}, #] {#{% endfor %} #} -{% for parameter in endpoint.optional_parameters.values() %} +{% for parameter in endpoint.resource_parameters(required=False).values() %} {{ parameter.python_name }}={{ parameter.python_name }}, {% endfor %} {% endmacro %} From b6616b1f23eeb0861f5932e18846a3ac5428d564 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Tue, 26 Mar 2024 16:51:00 +0530 Subject: [PATCH 10/18] paging client data selector --- openapi_python_client/parser/endpoints.py | 2 +- openapi_python_client/templates/endpoint_module.py.jinja | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index d7fff2d52..9784bedbe 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -256,7 +256,7 @@ def table_name(self) -> str: @property def data_json_path(self) -> str: payload = self.payload - return payload.json_path if payload else "" + return (payload.json_path if payload else "") or "$" @property def is_transformer(self) -> bool: diff --git a/openapi_python_client/templates/endpoint_module.py.jinja b/openapi_python_client/templates/endpoint_module.py.jinja index 3ea2bc24f..6e8bc5b17 100644 --- a/openapi_python_client/templates/endpoint_module.py.jinja +++ b/openapi_python_client/templates/endpoint_module.py.jinja @@ -51,6 +51,7 @@ def {{ endpoint.python_name }}({{ resource_arguments(endpoint) | indent(4) }}) - {% if endpoint.credentials %} auth=credentials, {% endif %} + data_selector=data_json_path, ) {% else %} params_dict: Dict[str, Any] = dict({{ kwargs(endpoint) }}) @@ -63,6 +64,7 @@ def {{ endpoint.python_name }}({{ resource_arguments(endpoint) | indent(4) }}) - {% if endpoint.credentials %} auth=credentials, {% endif %} + data_selector=data_json_path, ) {% endif %} {% endmacro %} From c9e8458e250efb98fbf244fe1640cb43b81c4445 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Wed, 27 Mar 2024 09:35:55 +0530 Subject: [PATCH 11/18] Fix transformer params finder --- openapi_python_client/parser/parameters.py | 4 ++-- openapi_python_client/parser/types.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/openapi_python_client/parser/parameters.py b/openapi_python_client/parser/parameters.py index 3ed094303..2a72896d8 100644 --- a/openapi_python_client/parser/parameters.py +++ b/openapi_python_client/parser/parameters.py @@ -5,7 +5,7 @@ from openapi_python_client.utils import PythonIdentifier from openapi_python_client.parser.models import SchemaWrapper, TSchemaType, DataPropertyPath -from openapi_python_client.parser.types import DataType +from openapi_python_client.parser.types import DataType, compare_openapi_types from openapi_python_client.parser.context import OpenapiContext TParamIn = Literal["query", "header", "path", "cookie"] @@ -76,7 +76,7 @@ def to_docstring(self) -> str: return doc def _matches_type(self, schema: SchemaWrapper) -> bool: - return schema.types == self.types and schema.type_format == self.type_format + return compare_openapi_types(self.types, self.type_format, schema.types, schema.type_format) def find_input_property(self, schema: SchemaWrapper, fallback: Optional[str] = None) -> Optional[DataPropertyPath]: """Find property in the given schema that's potentially an input to this parameter""" diff --git a/openapi_python_client/parser/types.py b/openapi_python_client/parser/types.py index e7cd15b37..95432af88 100644 --- a/openapi_python_client/parser/types.py +++ b/openapi_python_client/parser/types.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Literal, TYPE_CHECKING, Dict +from typing import Literal, TYPE_CHECKING, Dict, Optional, List if TYPE_CHECKING: from openapi_python_client.parser.models import SchemaWrapper, Property @@ -54,3 +54,17 @@ def from_property(cls, prop: "Property") -> "DataType": """Create a DataType from a Property. Properties may be required or not, so we need to pass that information.""" return cls(type_hint=schema_to_type_hint(prop.schema, required=prop.required)) + + +def compare_openapi_types( + types: List[TOpenApiType], type_format: Optional[str], other_types: List[TOpenApiType], other_format: Optional[str] +) -> bool: + types = sorted(types) + other_types = sorted(other_types) + if types == other_types: + if type_format == other_format: + return True + elif None in (type_format, other_format): + # One side has format unset, assume it's equivalent + return True + return False From ebb859fa4f78d8ed2e5067ff971c4ae59d9ed827 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Wed, 27 Mar 2024 12:54:06 +0530 Subject: [PATCH 12/18] Fix nullable types in params, transformer arg order --- openapi_python_client/parser/endpoints.py | 37 ++++++++----------- openapi_python_client/parser/models.py | 8 ++-- openapi_python_client/parser/pagination.py | 6 +-- openapi_python_client/parser/types.py | 4 +- .../templates/endpoint_macros.py.jinja | 22 +++++++---- .../templates/endpoint_module.py.jinja | 2 +- tests/parser/test_parser.py | 37 +++++++++++++++++-- 7 files changed, 75 insertions(+), 41 deletions(-) diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index 9784bedbe..567418941 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -183,25 +183,19 @@ def cookie_parameters(self) -> Dict[str, Parameter]: def list_all_parameters(self) -> List[Parameter]: return list(self.parameters.values()) - @property - def required_parameters(self) -> Dict[str, Parameter]: - return {name: p for name, p in self.parameters.items() if p.required} + def positional_arguments(self) -> List[Parameter]: + include_path_params = not self.transformer + ret = (p for p in self.parameters.values() if p.required and p.default is None) + if not include_path_params: + ret = (p for p in ret if p.location != "path") + return list(ret) - @property - def optional_parameters(self) -> Dict[str, Parameter]: - return {name: p for name, p in self.parameters.items() if not p.required} - - def resource_parameters( - self, required: bool = True, optional: bool = True, pagination: bool = False - ) -> Dict[str, Parameter]: - result = {} - if required: - result.update(self.required_parameters) - if optional: - result.update(self.optional_parameters) - if not pagination and self.pagination: - result = {name: p for name, p in result.items() if p.name not in self.pagination.param_names} - return result + def keyword_arguments(self) -> List[Parameter]: + ret = (p for p in self.parameters.values() if p.default is not None) + return list(ret) + + def all_arguments(self) -> List[Parameter]: + return self.positional_arguments() + self.keyword_arguments() @property def request_args_meta(self) -> Dict[str, Dict[str, Dict[str, str]]]: @@ -258,12 +252,13 @@ def data_json_path(self) -> str: payload = self.payload return (payload.json_path if payload else "") or "$" - @property - def is_transformer(self) -> bool: - return not not self.required_parameters + # @property + # def is_transformer(self) -> bool: + # return not not self.path_parameters @property def transformer(self) -> Optional[TransformerSetting]: + # TODO: compute once when generating endpoints if not self.parent: return None # if not self.parent.is_list: diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index 65731e65f..539e30943 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -235,13 +235,15 @@ def from_reference( schema_types = [schema.type] else: schema_types = schema.type.copy() - if "null" in schema_types: - nullable = True - schema_types.remove("null") else: schema_types = [] # No types, they may be taken from all_of/one_of/any_of for obj in all_of + one_of + any_of: schema_types.extend(obj.types) + if obj.nullable: + nullable = True + if "null" in schema_types: + nullable = True + schema_types.remove("null") default = schema.default # Only do string escaping, other types can go as-is diff --git a/openapi_python_client/parser/pagination.py b/openapi_python_client/parser/pagination.py index 287d2b533..43fc28fc4 100644 --- a/openapi_python_client/parser/pagination.py +++ b/openapi_python_client/parser/pagination.py @@ -9,7 +9,7 @@ from openapi_python_client.parser.endpoints import Endpoint, Parameter RE_OFFSET_PARAM = re.compile(r"(?i)(page|start|offset)") -RE_LIMIT_PARAM = re.compile(r"(?i)(limit|per_page|page_size)") +RE_LIMIT_PARAM = re.compile(r"(?i)(limit|per_page|page_size|size)") RE_CURSOR_PARAM = re.compile(r"(?i)(cursor|after|since)") @@ -25,10 +25,10 @@ def param_names(self) -> List[str]: return [param.name for param in self.pagination_params] @classmethod - def from_endpoint(cls, endpoint: "Endpoint") -> "Pagination": + def from_endpoint(cls, endpoint: "Endpoint") -> Optional["Pagination"]: resp = endpoint.data_response if not resp or not resp.content_schema: - raise ValueError(f"Endpoint {endpoint.path} does not have a content response") + return None offset_params: List["Parameter"] = [] cursor_params: List["Parameter"] = [] diff --git a/openapi_python_client/parser/types.py b/openapi_python_client/parser/types.py index 95432af88..630311a9a 100644 --- a/openapi_python_client/parser/types.py +++ b/openapi_python_client/parser/types.py @@ -28,8 +28,8 @@ def schema_to_type_hint(schema: SchemaWrapper, required: bool = True) -> str: item_type = schema_to_type_hint(schema.array_item) py_type = f"List[{item_type}]" union_types[py_type] = None - for one_of in schema.one_of + schema.any_of: - union_types[schema_to_type_hint(one_of)] = None + # for one_of in schema.one_of + schema.any_of: # all these are already included in top level schema + # union_types[schema_to_type_hint(one_of)] = None final_type = "" if len(union_types) == 1: diff --git a/openapi_python_client/templates/endpoint_macros.py.jinja b/openapi_python_client/templates/endpoint_macros.py.jinja index 871c689e0..aa490c07a 100644 --- a/openapi_python_client/templates/endpoint_macros.py.jinja +++ b/openapi_python_client/templates/endpoint_macros.py.jinja @@ -53,16 +53,16 @@ params = {k: v for k, v in params.items() if v is not UNSET and v is not None} {% endmacro %} +{# Resource function signature #} {% macro resource_arguments(endpoint) %} {% if endpoint.transformer %} data: Any, +{% endif %} rest_client: RESTClient, -{% else %} -rest_client: RESTClient, -{% for parameter in endpoint.resource_parameters().values() %} +{# Positional arguments #} +{% for parameter in endpoint.positional_arguments() %} {{ parameter.to_string() }}, {% endfor %} -{% endif %} {# Security #} {% if endpoint.credentials %} credentials: {{ endpoint.credentials.type_hint }} = dlt.secrets.value, @@ -70,13 +70,17 @@ credentials: {{ endpoint.credentials.type_hint }} = dlt.secrets.value, data_json_path: Optional[str]="{{endpoint.data_json_path}}", {% if endpoint.transformer %} {# Render the dict of path params #} -path_parameter_paths: Dict[str, TJsonPath]={{ endpoint.transformer.path_params_mapping }} +path_parameter_paths: Dict[str, TJsonPath]={{ endpoint.transformer.path_params_mapping }}, {% endif %} +{# Keyword arguments #} +{% for parameter in endpoint.keyword_arguments() %} +{{ parameter.to_string() }}, +{% endfor %} {% endmacro %} {# Just lists all kwargs to endpoints as name=name for passing to other functions #} {% macro kwargs(endpoint) %} -{% for parameter in endpoint.resource_parameters().values() %} +{% for parameter in endpoint.all_arguments() %} {{ parameter.python_name }}={{ parameter.python_name }}, {% endfor %} {% endmacro %} @@ -85,11 +89,13 @@ path_parameter_paths: Dict[str, TJsonPath]={{ endpoint.transformer.path_params_m {#{% for parameter in endpoint.required_parameters.values() %} #} {#{{ parameter.python_name }}={{ parameter.python_name }}, #] {#{% endfor %} #} -{% for parameter in endpoint.resource_parameters(required=False).values() %} +{% for parameter in endpoint.all_arguments() %} {{ parameter.python_name }}={{ parameter.python_name }}, {% endfor %} {% endmacro %} + +{# Docstring #} {% macro docstring_content(endpoint, return_string, is_detailed) %} {% if endpoint.summary %}{{ endpoint.summary | wordwrap(100)}} @@ -105,7 +111,7 @@ Parent endpoint: {{ endpoint.parent.path }} {# Leave extra space so that Args or Returns isn't at the top #} {% endif %} -{% set all_parameters = endpoint.list_all_parameters %} +{% set all_parameters = endpoint.all_arguments() %} {% if all_parameters %} Args: {% for parameter in all_parameters %} diff --git a/openapi_python_client/templates/endpoint_module.py.jinja b/openapi_python_client/templates/endpoint_module.py.jinja index 6e8bc5b17..85f0065dc 100644 --- a/openapi_python_client/templates/endpoint_module.py.jinja +++ b/openapi_python_client/templates/endpoint_module.py.jinja @@ -34,7 +34,7 @@ def {{ endpoint.python_name }}({{ resource_arguments(endpoint) | indent(4) }}) - params_dict: Dict[str, Any] = dict( {{ transformer_kwargs(endpoint) }} ) - for child_kwargs in extract_iterate_parent(data, path_parameter_paths, "{{ endpoint.path }}"): + for child_kwargs in extract_iterate_parent(data, path_parameter_paths, endpoint_url): formatted_url = build_formatted_url( endpoint_url, dict(params_dict, **child_kwargs), diff --git a/tests/parser/test_parser.py b/tests/parser/test_parser.py index a3abd296c..b0854b8cf 100644 --- a/tests/parser/test_parser.py +++ b/tests/parser/test_parser.py @@ -2,6 +2,8 @@ from openapi_python_client.parser.openapi_parser import OpenapiParser from openapi_schema_pydantic import Reference +from openapi_python_client.parser.models import SchemaWrapper +import openapi_schema_pydantic as osp from tests.cases import case_path @@ -75,9 +77,38 @@ def test_extract_payload(spotify_parser: OpenapiParser) -> None: def test_find_path_param(pokemon_parser: OpenapiParser) -> None: endpoints = pokemon_parser.endpoints - endpoint = endpoints.endpoints_by_path["/api/v2/pokemon-species/"] + parent_endpoint = endpoints.endpoints_by_path["/api/v2/pokemon-species/"] + endpoint = endpoints.endpoints_by_path["/api/v2/pokemon-species/{id}/"] - schema = endpoint.payload.prop - result = schema.crawled_properties.find_property_by_name("id", fallback="id") + schema = parent_endpoint.payload.prop + param = endpoint.parameters["id"] + result = param.find_input_property(schema, fallback="id") assert result.path == ("[*]", "id") + + +def test_schema_parse_types(pokemon_parser: OpenapiParser) -> None: + osp_schema = osp.Schema.parse_obj( + { + "anyOf": [ + {"type": "string"}, + {"type": "null"}, + ], + "title": "Pathname", + } + ) + + parsed = SchemaWrapper.from_reference(osp_schema, context=pokemon_parser.context) + + assert parsed.nullable is True + assert parsed.types == ["string"] + + +def test_resource_arguments(pokemon_parser: OpenapiParser) -> None: + path = "/api/v2/pokemon-species/{id}/" + endpoint = pokemon_parser.endpoints.endpoints_by_path[path] + + pos_args = endpoint.positional_arguments() + + # Does not include the id arg from transformer parent + assert pos_args == [] From 816be618491143dcf05b9924e2e05ca8b0ce110a Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Wed, 27 Mar 2024 13:37:41 +0530 Subject: [PATCH 13/18] offset and cursor paginator code gen --- openapi_python_client/parser/models.py | 31 ++++++++++++++++++- openapi_python_client/parser/pagination.py | 15 ++++++++- .../templates/api_helpers.py.jinja | 2 ++ .../templates/endpoint_macros.py.jinja | 13 ++++++-- .../templates/endpoint_module.py.jinja | 6 ++-- 5 files changed, 61 insertions(+), 6 deletions(-) diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index 539e30943..480a4da77 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -16,11 +16,12 @@ ) from itertools import chain from dataclasses import dataclass, field +import re from dlt.common.utils import digest128 import openapi_schema_pydantic as osp -from openapi_python_client.parser.types import DataType +from openapi_python_client.parser.types import DataType, TOpenApiType from openapi_python_client.parser.properties.converter import convert from openapi_python_client.utils import unique_list @@ -369,6 +370,34 @@ def is_optional(self, path: Tuple[str, ...]) -> bool: check_path.pop() return False + def find_property( + self, pattern: re.Pattern[str], require_type: Optional[TOpenApiType] = None + ) -> Optional[DataPropertyPath]: + candidates = [] + unknown_type_candidates = [] + for path, schema in self.items(): + if not path: + continue + if not pattern.match(path[-1]): + continue + if require_type: + if require_type in schema.types: + candidates.append((path, schema)) + elif not schema.types: + unknown_type_candidates.append((path, schema)) + else: + candidates.append((path, schema)) + + # prefer least nested path + candidates.sort(key=lambda item: len(item[0])) + unknown_type_candidates.sort(key=lambda item: len(item[0])) + + if candidates: + return DataPropertyPath(*candidates[0]) + elif unknown_type_candidates: + return DataPropertyPath(*unknown_type_candidates[0]) + return None + # def find_property_by_name(self, name: str, fallback: Optional[str] = None) -> Optional[DataPropertyPath]: # """Find a property with the given name somewhere in the object tree. diff --git a/openapi_python_client/parser/pagination.py b/openapi_python_client/parser/pagination.py index 43fc28fc4..6319d983a 100644 --- a/openapi_python_client/parser/pagination.py +++ b/openapi_python_client/parser/pagination.py @@ -10,6 +10,7 @@ RE_OFFSET_PARAM = re.compile(r"(?i)(page|start|offset)") RE_LIMIT_PARAM = re.compile(r"(?i)(limit|per_page|page_size|size)") +RE_TOTAL_PROPERTY = re.compile(r"(?i)(total|count)") RE_CURSOR_PARAM = re.compile(r"(?i)(cursor|after|since)") @@ -19,6 +20,15 @@ class Pagination: paginator_class: str = None paginator_config: Dict[str, str] = None + def to_string(self) -> str: + assert self.paginator_class + assert self.paginator_config is not None + ret = self.paginator_class + "(\n" + for key, value in self.paginator_config.items(): + ret += f"{key}={repr(value)},\n" + ret += ")" + return ret + @property def param_names(self) -> List[str]: """All params used for pagination""" @@ -90,11 +100,14 @@ def from_endpoint(cls, endpoint: "Endpoint") -> Optional["Pagination"]: # When spec doesn't provide default/max limit, fallback to a conservative default # 20 should be safe for most APIs limit_initial = int(limit_param.maximum) if limit_param.maximum else (limit_param.default or 20) - if offset_param and limit_param and limit_initial: + total_prop = resp.content_schema.crawled_properties.find_property(RE_TOTAL_PROPERTY, require_type="integer") + + if offset_param and limit_param and limit_initial and total_prop: pagination_config = { "initial_limit": limit_initial, "offset_param": offset_param.name, "limit_param": limit_param.name, + "total_path": total_prop.json_path, } return cls( paginator_class="OffsetPaginator", diff --git a/openapi_python_client/templates/api_helpers.py.jinja b/openapi_python_client/templates/api_helpers.py.jinja index 8d5403126..905342b7a 100644 --- a/openapi_python_client/templates/api_helpers.py.jinja +++ b/openapi_python_client/templates/api_helpers.py.jinja @@ -58,6 +58,8 @@ def build_query_parameters(parameters: Dict[str, Any], parameter_config: TParame query_params = {} for python_name, param_info in parameter_config.get("query", {}).items(): + if python_name not in parameters: # may be pagination params, etc that are not included + continue query_params.update(build_query_param(param_info, parameters[python_name])) return query_params diff --git a/openapi_python_client/templates/endpoint_macros.py.jinja b/openapi_python_client/templates/endpoint_macros.py.jinja index aa490c07a..99ccfc6cf 100644 --- a/openapi_python_client/templates/endpoint_macros.py.jinja +++ b/openapi_python_client/templates/endpoint_macros.py.jinja @@ -90,12 +90,12 @@ path_parameter_paths: Dict[str, TJsonPath]={{ endpoint.transformer.path_params_m {#{{ parameter.python_name }}={{ parameter.python_name }}, #] {#{% endfor %} #} {% for parameter in endpoint.all_arguments() %} -{{ parameter.python_name }}={{ parameter.python_name }}, +{{ parameter.python_name }}={{ parameter.python_name }}, {% endfor %} {% endmacro %} -{# Docstring #} +{# Docstring #} {% macro docstring_content(endpoint, return_string, is_detailed) %} {% if endpoint.summary %}{{ endpoint.summary | wordwrap(100)}} @@ -126,3 +126,12 @@ Returns: {% macro docstring(endpoint, return_string, is_detailed) %} {{ safe_docstring(docstring_content(endpoint, return_string, is_detailed)) }} {% endmacro %} + + +{% macro paginator(endpoint) %} +{% if endpoint.pagination %} +{{ endpoint.pagination.to_string() }} +{% else %} +None +{% endif %} +{% endmacro %} diff --git a/openapi_python_client/templates/endpoint_module.py.jinja b/openapi_python_client/templates/endpoint_module.py.jinja index 85f0065dc..8e2119ba2 100644 --- a/openapi_python_client/templates/endpoint_module.py.jinja +++ b/openapi_python_client/templates/endpoint_module.py.jinja @@ -1,9 +1,9 @@ {% macro generate_resource(endpoint) %} from http import HTTPStatus from typing import Any, Dict, List, Optional, Union, cast, Iterator -import dlt from dlt.sources.helpers import requests +from dlt.sources.helpers.rest_client.paginators import JSONResponseCursorPaginator, OffsetPaginator from .types import UNSET, Unset from .utils import extract_nested_data, extract_iterate_parent @@ -14,7 +14,7 @@ from .api_helpers import build_query_parameters, build_formatted_url, TParameter {% endfor %} {% from "endpoint_macros.py.jinja" import header_params, cookie_params, query_params, - arguments, resource_arguments, kwargs, transformer_kwargs, parse_response, docstring %} + arguments, resource_arguments, kwargs, transformer_kwargs, parse_response, docstring, paginator %} {# {% set return_string = endpoint.response_type() %} #} {% set return_string = "Any" %} @@ -52,6 +52,7 @@ def {{ endpoint.python_name }}({{ resource_arguments(endpoint) | indent(4) }}) - auth=credentials, {% endif %} data_selector=data_json_path, + paginator={{ paginator(endpoint) }} ) {% else %} params_dict: Dict[str, Any] = dict({{ kwargs(endpoint) }}) @@ -65,6 +66,7 @@ def {{ endpoint.python_name }}({{ resource_arguments(endpoint) | indent(4) }}) - auth=credentials, {% endif %} data_selector=data_json_path, + paginator={{ paginator(endpoint) }} ) {% endif %} {% endmacro %} From 84b5dc0897aad61ca665b23c94df1a8dc42259f5 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Wed, 27 Mar 2024 14:06:21 +0530 Subject: [PATCH 14/18] Fix resource arguments --- openapi_python_client/parser/endpoints.py | 8 +++++++- tests/parser/test_parser.py | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index 567418941..eb1423bb5 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -186,12 +186,18 @@ def list_all_parameters(self) -> List[Parameter]: def positional_arguments(self) -> List[Parameter]: include_path_params = not self.transformer ret = (p for p in self.parameters.values() if p.required and p.default is None) + # exclude pagination params + if self.pagination: + ret = (p for p in ret if p.name not in self.pagination.param_names) if not include_path_params: ret = (p for p in ret if p.location != "path") return list(ret) def keyword_arguments(self) -> List[Parameter]: - ret = (p for p in self.parameters.values() if p.default is not None) + ret = (p for p in self.parameters.values() if not p.required) + # exclude pagination params + if self.pagination: + ret = (p for p in ret if p.name not in self.pagination.param_names) return list(ret) def all_arguments(self) -> List[Parameter]: diff --git a/tests/parser/test_parser.py b/tests/parser/test_parser.py index b0854b8cf..3128ce650 100644 --- a/tests/parser/test_parser.py +++ b/tests/parser/test_parser.py @@ -112,3 +112,8 @@ def test_resource_arguments(pokemon_parser: OpenapiParser) -> None: # Does not include the id arg from transformer parent assert pos_args == [] + + +def test_parent_endpoints(spotify_parser: OpenapiParser) -> None: + # TODO: test e.g. /browse/categories -> /browse/categories/{category_id}/playlists + pass From 0875eaf7a7909002e7bc7b324e00bb2680b5c001 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Thu, 28 Mar 2024 16:59:15 +0530 Subject: [PATCH 15/18] Detect primay key, slightly improve payload detect and table name --- openapi_python_client/parser/endpoints.py | 19 +++++++ openapi_python_client/parser/models.py | 43 ++++++++++++++- openapi_python_client/parser/responses.py | 39 +++++++------ .../templates/endpoint_module.py.jinja | 14 ++++- tests/parser/test_parser.py | 55 ++++++++++++------- 5 files changed, 126 insertions(+), 44 deletions(-) diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index eb1423bb5..e8b3e1c3d 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -43,6 +43,8 @@ class Response: content_schema: Optional[SchemaWrapper] list_property: Optional[DataPropertyPath] = None payload: Optional[DataPropertyPath] = None + initial_payload: Optional[DataPropertyPath] = None + """Payload set initially before comparing other endpoints""" @property def has_content(self) -> bool: @@ -96,6 +98,7 @@ def from_reference( raw_schema=raw_schema, content_schema=content_schema, payload=payload, + initial_payload=payload, ) @@ -163,6 +166,11 @@ def parent(self, value: Optional["Endpoint"]) -> None: if value: value.children.append(self) + @property + def primary_key(self) -> Optional[str]: + payload = self.payload + return payload.schema.primary_key if payload else None + @property def path_parameters(self) -> Dict[str, Parameter]: return {p.name: p for p in self.parameters.values() if p.location == "path"} @@ -475,3 +483,14 @@ def build_endpoint_tree(endpoints: Iterable[Endpoint]) -> Tree: current_node = current_node[part] # type: ignore current_node[""] = endpoint return tree + + def _endpoint_tree_to_str(self) -> str: + # Pretty print presentation of the tree + def _tree_to_str(node: Union["Endpoint", "Tree"], indent: int = 0) -> str: + if isinstance(node, dict): + return "\n".join( + f"{' ' * indent}{key}\n{_tree_to_str(value, indent + 1)}" for key, value in node.items() + ) + return f"{' ' * indent}{node.path} -> {node.operation_id}" + + return _tree_to_str(self.endpoint_tree) diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index 480a4da77..fa3b650e9 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -99,6 +99,9 @@ class SchemaWrapper: any_of: List["SchemaWrapper"] = field(default_factory=list) one_of: List["SchemaWrapper"] = field(default_factory=list) + enum_values: Optional[List[Any]] = None + primary_key: Optional[str] = None + def __getitem__(self, item: str) -> "Property": try: return next(prop for prop in self.properties if prop.name == item) @@ -111,6 +114,31 @@ def __contains__(self, item: str) -> bool: def __iter__(self) -> Iterable["str"]: return (prop.name for prop in self.properties) + def _find_primary_key(self) -> Optional[str]: + """Attempt to find the name of primary key field in the schema""" + # Regex pattern finding the words "unique", "id" or "identifier" in description + # Use word boundaries + desc_pattern = re.compile(r"\b(unique|id|identifier)\b", re.IGNORECASE) + + description_paths = [] + uuid_paths = [] + + for prop in self.all_properties: + if not set(prop.schema.types) & {"string", "integer"}: + continue + if prop.name.lower() == "id": + return prop.name + elif prop.schema.description and desc_pattern.search(prop.schema.description): + description_paths.append(prop.name) + elif prop.schema.type_format == "uuid": + uuid_paths.append(prop.name) + + if description_paths: + return description_paths[0] + elif uuid_paths: + return uuid_paths[0] + return None + @property def has_properties(self) -> bool: return bool(self.properties or self.any_of or self.all_of) @@ -191,6 +219,12 @@ def from_reference( all_of = _remove_nones([cls.from_reference_guarded(ref, context, level=level) for ref in schema.allOf or []]) + if not name: + for sub in all_of: + name = sub.name + if name: + break + # Properties from all_of child schemas should be merged property_map = {prop.name: prop for prop in parent_properties or []} property_map.update({prop.name: prop for prop in chain.from_iterable(s.properties for s in all_of)}) @@ -269,7 +303,9 @@ def from_reference( hash_key=digest128(schema.json(sort_keys=True)), type_format=schema.schema_format, maximum=schema.maximum, + enum_values=schema.enum, ) + result.primary_key = result._find_primary_key() crawler.crawl(result) return result @@ -348,7 +384,9 @@ def __bool__(self) -> bool: def items(self) -> Iterable[Tuple[Tuple[str, ...], SchemaWrapper]]: return self.all_properties.items() - def paths_with_types(self) -> Iterator[tuple[tuple[str, ...], tuple[tuple[TSchemaType, ...], Optional[str]]]]: + def paths_with_types( + self, + ) -> Iterator[tuple[tuple[str, ...], tuple[tuple[TSchemaType, ...], Optional[str], tuple[Any, ...]]]]: """ yields a tuple of (path, ( (types, ...), "format")) for each property in the schema """ @@ -357,7 +395,8 @@ def paths_with_types(self) -> Iterator[tuple[tuple[str, ...], tuple[tuple[TSchem # # Include the array item type for full comparison # yield path, tuple(schema.types + schema.array_item.types]) # else: - yield path, (tuple(schema.types), schema.type_format) + # yield path, (tuple(schema.types), schema.type_format, tuple(schema.enum_values or [])) + yield path, (tuple(schema.types), schema.type_format, tuple(schema.enum_values or [])) def is_optional(self, path: Tuple[str, ...]) -> bool: """Check whether the property itself or any of its parents is nullable""" diff --git a/openapi_python_client/parser/responses.py b/openapi_python_client/parser/responses.py index 5a98e9419..2d1c305d2 100644 --- a/openapi_python_client/parser/responses.py +++ b/openapi_python_client/parser/responses.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Tuple +from typing import TYPE_CHECKING, Dict, Tuple, Optional if TYPE_CHECKING: from openapi_python_client.parser.endpoints import EndpointCollection, Endpoint, Response @@ -45,35 +45,20 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl for other in sorted(endpoints.all_endpoints_to_render, key=lambda ep: ep.path): if other.path == endpoint.path: continue - other_payload = other.data_response.payload + other_payload = other.data_response.initial_payload if not other_payload: continue other_schema = other_payload.prop - # other_props = set(other_schema.crawled_properties) # type: ignore[call-overload] other_props = set(other_schema.crawled_properties.paths_with_types()) - # Remove all common props from the payload, assume most of those are metadata common_props = response_props & other_props - # remaining_props = response_props - common_props - # remaining props including their ancestry starting at top level parents new_props = payload_props - common_props - # # Check if new props contain any object type props - # has_object_props = any( - # prop_path in schema.crawled_properties.object_properties - # or prop_path in schema.crawled_properties.list_properties - # for prop_path in new_props - # ) - # if len(new_props) == 1: - # prop_obj = schema.crawled_properties[list(new_props)[0]] - # if not prop_obj.is_object and not prop_obj.is_list: - # new_props.add(()) - if not new_props: # Don't remove all props continue payload_props = new_props - payload_path: Tuple[str, ...] = () + payload_path: Optional[Tuple[str, ...]] = None if len(payload_props) == 1: # When there's only one remaining prop it can mean the payload is just @@ -85,16 +70,30 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl prop_path = tuple(list(prop_path)[:-1]) payload_path = prop_path # type: ignore[assignment] - if not payload_path: + if payload_path is None: # Payload path is the deepest nested parent of all remaining props payload_path = find_longest_common_prefix([path for path, _ in payload_props]) - payload_path = payload_path while payload_path and payload_path[-1] == "[*]": # We want the path to point to the list property, not the list item # so that payload is correctly detected as list payload_path = payload_path[:-1] payload_schema = schema.crawled_properties[payload_path] + + # If no primary key in the payload, try climbing up the tree and prefer the nearest schema with pk + if not payload_schema.primary_key: + new_path = list(payload_path) + new_schema = payload_schema + while new_path: + new_path.pop() + while new_path and new_path[-1] == "[*]": + new_path.pop() + new_schema = schema.crawled_properties[tuple(new_path)] + if new_schema.primary_key: + payload_path = tuple(new_path) + payload_schema = new_schema + break + ret = DataPropertyPath(root_path + payload_path, payload_schema) print(endpoint.path) print(ret.path) diff --git a/openapi_python_client/templates/endpoint_module.py.jinja b/openapi_python_client/templates/endpoint_module.py.jinja index 8e2119ba2..950933e84 100644 --- a/openapi_python_client/templates/endpoint_module.py.jinja +++ b/openapi_python_client/templates/endpoint_module.py.jinja @@ -21,10 +21,20 @@ from .api_helpers import build_query_parameters, build_formatted_url, TParameter {% set parsed_responses = (endpoint.responses | length > 0) and return_string != "Any" %} {% if endpoint.transformer %} -@dlt.transformer(table_name="{{ endpoint.table_name}}") +@dlt.transformer( + table_name="{{ endpoint.table_name}}" + {% if endpoint.primary_key %} + , primary_key="{{ endpoint.primary_key }}" + {% endif %} +) def {{ endpoint.python_name }}({{ resource_arguments(endpoint) | indent(4) }}) -> Iterator[{{ return_string}}]: {% else %} -@dlt.resource(table_name="{{ endpoint.table_name }}") +@dlt.resource( + table_name="{{ endpoint.table_name}}" + {% if endpoint.primary_key %} + , primary_key="{{ endpoint.primary_key }}" + {% endif %} +) def {{ endpoint.python_name }}({{ resource_arguments(endpoint) | indent(4) }}) -> Iterator[{{ return_string}}]: {% endif %} {{ docstring(endpoint, return_string, is_detailed=false) | indent(4) }} diff --git a/tests/parser/test_parser.py b/tests/parser/test_parser.py index 3128ce650..00a6ebd1e 100644 --- a/tests/parser/test_parser.py +++ b/tests/parser/test_parser.py @@ -51,28 +51,26 @@ def test_new_releases_list_property(spotify_parser: OpenapiParser) -> None: # schema = endpoint.data_response.content_schema -def test_extract_payload(spotify_parser: OpenapiParser) -> None: +@pytest.mark.parametrize( + "endpoint_path,payload_path,payload_name", + [ + ("/browse/new-releases", ("albums", "items"), "SimplifiedAlbumObject"), + # ("/playlists/{playlist_id}/tracks", ("items",), "PlaylistTrackObject"), + ("/me/tracks", ("items", "[*]", "track"), "TrackObject"), + ("/me/albums", ("items", "[*]", "album"), "AlbumObject"), + ("/artists/{id}/related-artists", ("artists",), "ArtistObject"), + # ("/browse/categories/{category_id}", (), "CategoryObject"), + ], +) +def test_extract_payload( + endpoint_path: str, payload_path: tuple[str], payload_name: str, spotify_parser: OpenapiParser +) -> None: endpoints = spotify_parser.endpoints - pl_tr_endpoint = endpoints.endpoints_by_path["/playlists/{playlist_id}/tracks"] - new_releases_endpoint = endpoints.endpoints_by_path["/browse/new-releases"] - saved_tracks_endpoint = endpoints.endpoints_by_path["/me/tracks"] - related_artists_endpoint = endpoints.endpoints_by_path["/artists/{id}/related-artists"] - - assert new_releases_endpoint.data_response.payload.path == ( - "albums", - "items", - ) - assert new_releases_endpoint.data_response.payload.name == "SimplifiedAlbumObject" - - # TODO: - # assert pl_tr_endpoint.data_response.payload.path == ("items",) - # assert pl_tr_endpoint.data_response.payload.name == "PlaylistTrackObject" - assert saved_tracks_endpoint.data_response.payload.path == ("items",) - assert saved_tracks_endpoint.data_response.payload.name == "SavedTrackObject" + endpoint = endpoints.endpoints_by_path[endpoint_path] - assert related_artists_endpoint.data_response.payload.path == ("artists",) - assert related_artists_endpoint.data_response.payload.name == "ArtistObject" + assert endpoint.payload.path == payload_path + assert endpoint.payload.name == payload_name def test_find_path_param(pokemon_parser: OpenapiParser) -> None: @@ -116,4 +114,21 @@ def test_resource_arguments(pokemon_parser: OpenapiParser) -> None: def test_parent_endpoints(spotify_parser: OpenapiParser) -> None: # TODO: test e.g. /browse/categories -> /browse/categories/{category_id}/playlists - pass + parent_path = "/browse/categories" + child_path = "/browse/categories/{category_id}/playlists" + + parent = spotify_parser.endpoints.endpoints_by_path[parent_path] + child = spotify_parser.endpoints.endpoints_by_path[child_path] + + assert child.parent is parent + + +def test_schema_name(spotify_parser: OpenapiParser) -> None: + path = "/me/albums" + endpoint = spotify_parser.endpoints.endpoints_by_path[path] + + schema = endpoint.data_response.content_schema + + album_schema = schema["items"].schema.array_item["album"].schema + # Name taken from nested schema in all_of + assert album_schema.name == "AlbumObject" From 12824848efec856bd03659bd46872d9c451a391e Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Thu, 28 Mar 2024 17:21:09 +0530 Subject: [PATCH 16/18] Primary payload with pk --- openapi_python_client/parser/responses.py | 23 +++++++++-------- tests/parser/test_parser.py | 30 +++++++++++++++++++++-- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/openapi_python_client/parser/responses.py b/openapi_python_client/parser/responses.py index 2d1c305d2..2119b5b74 100644 --- a/openapi_python_client/parser/responses.py +++ b/openapi_python_client/parser/responses.py @@ -74,24 +74,19 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl # Payload path is the deepest nested parent of all remaining props payload_path = find_longest_common_prefix([path for path, _ in payload_props]) - while payload_path and payload_path[-1] == "[*]": - # We want the path to point to the list property, not the list item - # so that payload is correctly detected as list - payload_path = payload_path[:-1] + payload_path = _unnest_array(payload_path) payload_schema = schema.crawled_properties[payload_path] # If no primary key in the payload, try climbing up the tree and prefer the nearest schema with pk - if not payload_schema.primary_key: + if (not payload_schema.primary_key) or (payload_schema.is_list and not payload_schema.array_item.primary_key): new_path = list(payload_path) new_schema = payload_schema while new_path: new_path.pop() - while new_path and new_path[-1] == "[*]": - new_path.pop() new_schema = schema.crawled_properties[tuple(new_path)] - if new_schema.primary_key: - payload_path = tuple(new_path) - payload_schema = new_schema + if new_schema.primary_key or (new_schema.is_list and new_schema.array_item.primary_key): + payload_path = _unnest_array(tuple(new_path)) + payload_schema = schema.crawled_properties[payload_path] break ret = DataPropertyPath(root_path + payload_path, payload_schema) @@ -102,6 +97,14 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl response.payload = ret +def _unnest_array(path: Tuple[str, ...]) -> Tuple[str, ...]: + while path and path[-1] == "[*]": + # We want the path to point to the list property, not the list item + # so that payload is correctly detected as list + path = path[:-1] + return path + + # def _process_response_list( # response: Response, # endpoint: Endpoint, diff --git a/tests/parser/test_parser.py b/tests/parser/test_parser.py index 00a6ebd1e..681fc7a16 100644 --- a/tests/parser/test_parser.py +++ b/tests/parser/test_parser.py @@ -59,10 +59,10 @@ def test_new_releases_list_property(spotify_parser: OpenapiParser) -> None: ("/me/tracks", ("items", "[*]", "track"), "TrackObject"), ("/me/albums", ("items", "[*]", "album"), "AlbumObject"), ("/artists/{id}/related-artists", ("artists",), "ArtistObject"), - # ("/browse/categories/{category_id}", (), "CategoryObject"), + ("/browse/categories/{category_id}", (), "CategoryObject"), ], ) -def test_extract_payload( +def test_extract_payload_spotify( endpoint_path: str, payload_path: tuple[str], payload_name: str, spotify_parser: OpenapiParser ) -> None: endpoints = spotify_parser.endpoints @@ -73,6 +73,25 @@ def test_extract_payload( assert endpoint.payload.name == payload_name +@pytest.mark.parametrize( + "endpoint_path,payload_path,payload_name", + [ + ("/api/v2/pokemon/{id}/", (), "Pokemon"), + ("/api/v2/pokemon-species/", ("results",), "PokemonSpecies"), + ("/api/v2/egg-group/", (), "EggGroup"), + ], +) +def test_extract_payload_pokeapi( + endpoint_path: str, payload_path: tuple[str], payload_name: str, pokemon_parser: OpenapiParser +) -> None: + endpoints = pokemon_parser.endpoints + + endpoint = endpoints.endpoints_by_path[endpoint_path] + + assert endpoint.payload.path == payload_path + assert endpoint.payload.name == payload_name + + def test_find_path_param(pokemon_parser: OpenapiParser) -> None: endpoints = pokemon_parser.endpoints parent_endpoint = endpoints.endpoints_by_path["/api/v2/pokemon-species/"] @@ -132,3 +151,10 @@ def test_schema_name(spotify_parser: OpenapiParser) -> None: album_schema = schema["items"].schema.array_item["album"].schema # Name taken from nested schema in all_of assert album_schema.name == "AlbumObject" + + +def test_detect_primary_key(pokemon_parser: OpenapiParser) -> None: + path = "/api/v2/egg-group/" + endpoint = pokemon_parser.endpoints.endpoints_by_path[path] + + assert endpoint.primary_key == "id" From f8ac60f9f1a75d6785a69234bd9bac38a676b90c Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Sat, 13 Apr 2024 14:30:06 -0400 Subject: [PATCH 17/18] Schema name from component ref only, update dlt, cleanup --- openapi_python_client/parser/context.py | 39 +++-- openapi_python_client/parser/endpoints.py | 24 +-- .../parser/openapi_parser.py | 9 +- openapi_python_client/parser/paths.py | 4 +- openapi_python_client/parser/responses.py | 33 ---- .../templates/pyproject.toml.jinja | 2 +- poetry.lock | 157 +++++++++++++++--- pyproject.toml | 4 +- 8 files changed, 183 insertions(+), 89 deletions(-) diff --git a/openapi_python_client/parser/context.py b/openapi_python_client/parser/context.py index d53e2d347..ac2483b51 100644 --- a/openapi_python_client/parser/context.py +++ b/openapi_python_client/parser/context.py @@ -1,7 +1,9 @@ -from typing import Dict, Union, cast, Tuple, Optional +from typing import Dict, Union, Tuple, Optional, Any from dataclasses import dataclass import openapi_schema_pydantic as osp +import referencing +import referencing.jsonschema from openapi_python_client.parser.config import Config from openapi_python_client.utils import ClassName @@ -43,30 +45,40 @@ def location(self) -> str: class OpenapiContext: spec: osp.OpenAPI + spec_raw: dict[str, Any] - _component_cache: Dict[str, TComponentClass] + _component_cache: Dict[str, dict[str, Any]] security_schemes: Dict[str, SecurityScheme] - def __init__(self, config: Config) -> None: + def __init__(self, config: Config, spec: osp.OpenAPI, spec_raw: dict[str, Any]) -> None: self.config = config + self.spec = spec + self.spec_raw = spec_raw self._component_cache = {} self.security_schemes = {} + resource = referencing.Resource( # type: ignore[var-annotated, call-arg] + contents=self.spec_raw, specification=referencing.jsonschema.DRAFT202012 + ) + registry = referencing.Registry().with_resource(resource=resource, uri="") + self._resolver = registry.resolver() - def _component_from_reference(self, ref: osp.Reference) -> TComponentClass: + def _component_from_reference(self, ref: osp.Reference) -> dict[str, Any]: url = ref.ref if url in self._component_cache: return self._component_cache[url] - if not url.startswith("#/components/"): - raise ValueError(f"Unsupported ref {url} Only #/components/... refs are supported") - section, name = url.split("/components/")[-1].split("/") - obj = getattr(self.spec.components, section)[name] + obj = self._resolver.lookup(url).contents self._component_cache[url] = obj return obj def schema_and_name_from_reference(self, ref: Union[osp.Reference, osp.Schema]) -> Tuple[str, osp.Schema]: name: Optional[str] = None if isinstance(ref, osp.Reference): - name = ref.ref.split("/components/")[-1].split("/")[-1] + if ref.ref.startswith("#/components/"): + # Refs to random places in the spec, e.g. #/paths/some~path/responses/.../schema + # don't generate useful names, so only take names from #/components/schemas/SchemaName refs + name = ref.ref.split("/components/")[-1].split("/")[-1] + else: + name = "" schema = self.schema_from_reference(ref) name = name or schema.title return name, schema @@ -74,17 +86,20 @@ def schema_and_name_from_reference(self, ref: Union[osp.Reference, osp.Schema]) def response_from_reference(self, ref: osp.Reference | osp.Response) -> osp.Response: if isinstance(ref, osp.Response): return ref - return cast(osp.Response, self._component_from_reference(ref)) + return osp.Response.parse_obj(self._component_from_reference(ref)) + # return cast(osp.Response, self._component_from_reference(ref)) def schema_from_reference(self, ref: osp.Reference | osp.Schema) -> osp.Schema: if isinstance(ref, osp.Schema): return ref - return cast(osp.Schema, self._component_from_reference(ref)) + return osp.Schema.parse_obj(self._component_from_reference(ref)) + # return cast(osp.Schema, self._component_from_reference(ref)) def parameter_from_reference(self, ref: Union[osp.Reference, osp.Parameter]) -> osp.Parameter: if isinstance(ref, osp.Parameter): return ref - return cast(osp.Parameter, self._component_from_reference(ref)) + return osp.Parameter.parse_obj(self._component_from_reference(ref)) + # return cast(osp.Parameter, self._component_from_reference(ref)) def get_security_scheme(self, name: str) -> SecurityScheme: # TODO: The security scheme might be a Reference diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index e8b3e1c3d..67892ba1a 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -10,12 +10,11 @@ from openapi_python_client.parser.context import OpenapiContext from openapi_python_client.parser.paths import table_names_from_paths -from openapi_python_client.parser.models import SchemaWrapper, DataPropertyPath, TSchemaType +from openapi_python_client.parser.models import SchemaWrapper, DataPropertyPath from openapi_python_client.utils import PythonIdentifier from openapi_python_client.parser.responses import process_responses from openapi_python_client.parser.credentials import CredentialsProperty from openapi_python_client.parser.pagination import Pagination -from openapi_python_client.parser.types import DataType from openapi_python_client.parser.parameters import Parameter TMethod = Literal["get", "post", "put", "patch"] @@ -335,13 +334,16 @@ def from_operation( all_params.update( {p.name: p for p in (Parameter.from_reference(param, context) for param in operation.parameters or [])} ) - responses = { - resp.status_code: resp - for resp in [ - Response.from_reference(status_code, response_ref, context) - for status_code, response_ref in operation.responses.items() - ] - } + + parsed_responses = ( + Response.from_reference(status_code, response_ref, context) + for status_code, response_ref in operation.responses.items() + ) + responses = {} + for parsed_response in parsed_responses: + if parsed_response.status_code in responses: + continue + responses[parsed_response.status_code] = parsed_response operation_id = operation.operationId or f"{method}_{path}" @@ -385,8 +387,8 @@ def all_endpoints_to_render(self) -> List[Endpoint]: key=lambda e: e.table_name, ) groups = groupby(to_render, key=lambda e: e.table_name) - groups = [(name, list(group)) for name, group in groups] - groups = sorted(groups, key=lambda g: max(e.rank for e in g[1]), reverse=True) + groups = [(name, list(group)) for name, group in groups] # type: ignore[assignment] + groups = sorted(groups, key=lambda g: max(e.rank for e in g[1]), reverse=True) # type: ignore[assignment] return [e for _, group in groups for e in group] @property diff --git a/openapi_python_client/parser/openapi_parser.py b/openapi_python_client/parser/openapi_parser.py index d0ed371ac..4e0e4118f 100644 --- a/openapi_python_client/parser/openapi_parser.py +++ b/openapi_python_client/parser/openapi_parser.py @@ -25,10 +25,12 @@ class OpenapiParser: spec_raw: Dict[str, Any] info: OpenApiInfo credentials: Optional[CredentialsProperty] = None + context: OpenapiContext def __init__(self, spec_file: Union[Path, str], config: Config = Config()) -> None: self.spec_file = spec_file - self.context = OpenapiContext(config=config) + # self.context = OpenapiContext(config=config) + self.config = config def load_spec_raw(self) -> Dict[str, Any]: p = self.spec_file @@ -58,8 +60,9 @@ def _find_references(self, dictionary: Dict[str, Any]) -> Iterator[str]: def parse(self) -> None: self.spec_raw = self.load_spec_raw() - self.context.spec = osp.OpenAPI.parse_obj(self.spec_raw) - log.info("Pydantic parse") + # log.info("Pydantic parse") + spec = osp.OpenAPI.parse_obj(self.spec_raw) + self.context = OpenapiContext(self.config, spec, self.spec_raw) self.info = OpenApiInfo.from_context(self.context) log.info("Parse endpoints") self.endpoints = EndpointCollection.from_context(self.context) diff --git a/openapi_python_client/parser/paths.py b/openapi_python_client/parser/paths.py index 6bfdc108b..ebfd6aa10 100644 --- a/openapi_python_client/parser/paths.py +++ b/openapi_python_client/parser/paths.py @@ -1,6 +1,4 @@ -from collections import defaultdict - -from typing import Iterable, Dict, Tuple, Sequence +from typing import Iterable, Dict, Tuple import os.path diff --git a/openapi_python_client/parser/responses.py b/openapi_python_client/parser/responses.py index 2119b5b74..b1e9b6fc2 100644 --- a/openapi_python_client/parser/responses.py +++ b/openapi_python_client/parser/responses.py @@ -36,9 +36,6 @@ def find_payload(response: Response, endpoint: Endpoint, endpoints: EndpointColl schema = response.payload.prop root_path = response.payload.path - # response_props = set(response.content_schema.crawlepd_properties.all_properties) - # response_props = set(schema.crawled_properties.all_properties) - # payload_props = set(response_props) response_props = set(schema.crawled_properties.paths_with_types()) payload_props = set(response_props) @@ -103,33 +100,3 @@ def _unnest_array(path: Tuple[str, ...]) -> Tuple[str, ...]: # so that payload is correctly detected as list path = path[:-1] return path - - -# def _process_response_list( -# response: Response, -# endpoint: Endpoint, -# endpoints: EndpointCollection, -# ) -> None: -# if not response.list_properties: -# return -# if () in response.list_properties: # Response is a top level list -# response.list_property = DataPropertyPath((), response.list_properties[()]) -# return - -# level_counts = count_by_length(response.list_properties.keys()) - -# # Get list properties max 2 levels down -# props_first_levels = [ -# (path, prop) for path, prop in sorted(response.list_properties.items(), key=lambda k: len(k)) if len(path) <= 2 -# ] - -# # If there is only one list property 1 or 2 levels down, this is the list -# for path, prop in props_first_levels: -# if not prop.is_object: # Only looking for object lists -# continue -# levels = len(path) -# if level_counts[levels] == 1: -# response.list_property = DataPropertyPath(path, prop) -# parent = endpoints.find_immediate_parent(endpoint.path) -# if parent and not parent.required_parameters: -# response.list_property = None diff --git a/openapi_python_client/templates/pyproject.toml.jinja b/openapi_python_client/templates/pyproject.toml.jinja index a49054a14..493cdfac7 100644 --- a/openapi_python_client/templates/pyproject.toml.jinja +++ b/openapi_python_client/templates/pyproject.toml.jinja @@ -14,7 +14,7 @@ include = ["CHANGELOG.md", "{{ package_name }}/py.typed"] [tool.poetry.dependencies] python = "^3.8" -dlt = {git = "https://github.com/dlt-hub/dlt.git", branch = "sthor/remove-rest-client-cyclic-import"} +dlt = {extras = ["duckdb"], version = "^0.4.8"} [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/poetry.lock b/poetry.lock index 9cdbc6e18..0cba9cb0f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -471,19 +471,21 @@ files = [ [[package]] name = "dlt" -version = "0.4.7" +version = "0.4.8" description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run." optional = false -python-versions = ">=3.8.1,<3.13" -files = [] -develop = false +python-versions = "<3.13,>=3.8.1" +files = [ + {file = "dlt-0.4.8-py3-none-any.whl", hash = "sha256:ade57b2745986c8aada7b2e28856df20164a7738a281c8a87596af62a378be06"}, + {file = "dlt-0.4.8.tar.gz", hash = "sha256:fc46d2ee61bd8d128db84a8214081f78ff642bad5677fdac210080a1f1bcbcf8"}, +] [package.dependencies] astunparse = ">=1.6.3" click = ">=7.1" duckdb = [ - {version = ">=0.6.1,<0.10.0", optional = true, markers = "python_version >= \"3.8\" and python_version < \"3.12\""}, - {version = ">=0.10.0,<0.11.0", optional = true, markers = "python_version >= \"3.12\""}, + {version = ">=0.6.1,<0.10.0", optional = true, markers = "python_version >= \"3.8\" and python_version < \"3.12\" and (extra == \"duckdb\" or extra == \"motherduck\")"}, + {version = ">=0.10.0,<0.11.0", optional = true, markers = "python_version >= \"3.12\" and (extra == \"duckdb\" or extra == \"motherduck\")"}, ] fsspec = ">=2022.4.0" gitpython = ">=3.1.29" @@ -516,6 +518,7 @@ bigquery = ["gcsfs (>=2022.4.0)", "google-cloud-bigquery (>=2.26.0)", "grpcio (> cli = ["cron-descriptor (>=1.2.32)", "pipdeptree (>=2.9.0,<2.10)"] databricks = ["databricks-sql-connector (>=2.9.3,<3.0.0)"] dbt = ["dbt-athena-community (>=1.2.0)", "dbt-bigquery (>=1.2.0)", "dbt-core (>=1.2.0)", "dbt-databricks (>=1.7.3,<2.0.0)", "dbt-duckdb (>=1.2.0)", "dbt-redshift (>=1.2.0)", "dbt-snowflake (>=1.2.0)"] +dremio = ["pyarrow (>=12.0.0)"] duckdb = ["duckdb (>=0.10.0,<0.11.0)", "duckdb (>=0.6.1,<0.10.0)"] filesystem = ["botocore (>=1.28)", "s3fs (>=2022.4.0)"] gcp = ["gcsfs (>=2022.4.0)", "google-cloud-bigquery (>=2.26.0)", "grpcio (>=1.50.0)"] @@ -531,12 +534,6 @@ snowflake = ["snowflake-connector-python (>=3.5.0)"] synapse = ["adlfs (>=2022.4.0)", "pyarrow (>=12.0.0)", "pyodbc (>=4.0.39,<5.0.0)"] weaviate = ["weaviate-client (>=3.22)"] -[package.source] -type = "git" -url = "https://github.com/dlt-hub/dlt.git" -reference = "sthor/remove-rest-client-cyclic-import" -resolved_reference = "09e306ac3e071403a0d7365c741b30109876ee2b" - [[package]] name = "dparse" version = "0.6.2" @@ -1625,6 +1622,21 @@ prompt_toolkit = ">=2.0,<4.0" [package.extras] docs = ["Sphinx (>=3.3,<4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphinx-autodoc-typehints (>=1.11.1,<2.0.0)", "sphinx-copybutton (>=0.3.1,<0.4.0)", "sphinx-rtd-theme (>=0.5.0,<0.6.0)"] +[[package]] +name = "referencing" +version = "0.34.0" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.34.0-py3-none-any.whl", hash = "sha256:d53ae300ceddd3169f1ffa9caf2cb7b769e92657e4fafb23d34b93679116dfd4"}, + {file = "referencing-0.34.0.tar.gz", hash = "sha256:5773bd84ef41799a5a8ca72dc34590c041eb01bf9aa02632b4a973fb0181a844"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + [[package]] name = "requests" version = "2.31.0" @@ -1660,6 +1672,114 @@ files = [ [package.dependencies] types-setuptools = ">=57.0.0" +[[package]] +name = "rpds-py" +version = "0.18.0" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.18.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:5b4e7d8d6c9b2e8ee2d55c90b59c707ca59bc30058269b3db7b1f8df5763557e"}, + {file = "rpds_py-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c463ed05f9dfb9baebef68048aed8dcdc94411e4bf3d33a39ba97e271624f8f7"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01e36a39af54a30f28b73096dd39b6802eddd04c90dbe161c1b8dbe22353189f"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d62dec4976954a23d7f91f2f4530852b0c7608116c257833922a896101336c51"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd18772815d5f008fa03d2b9a681ae38d5ae9f0e599f7dda233c439fcaa00d40"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:923d39efa3cfb7279a0327e337a7958bff00cc447fd07a25cddb0a1cc9a6d2da"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39514da80f971362f9267c600b6d459bfbbc549cffc2cef8e47474fddc9b45b1"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a34d557a42aa28bd5c48a023c570219ba2593bcbbb8dc1b98d8cf5d529ab1434"}, + {file = "rpds_py-0.18.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:93df1de2f7f7239dc9cc5a4a12408ee1598725036bd2dedadc14d94525192fc3"}, + {file = "rpds_py-0.18.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:34b18ba135c687f4dac449aa5157d36e2cbb7c03cbea4ddbd88604e076aa836e"}, + {file = "rpds_py-0.18.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c0b5dcf9193625afd8ecc92312d6ed78781c46ecbf39af9ad4681fc9f464af88"}, + {file = "rpds_py-0.18.0-cp310-none-win32.whl", hash = "sha256:c4325ff0442a12113a6379af66978c3fe562f846763287ef66bdc1d57925d337"}, + {file = "rpds_py-0.18.0-cp310-none-win_amd64.whl", hash = "sha256:7223a2a5fe0d217e60a60cdae28d6949140dde9c3bcc714063c5b463065e3d66"}, + {file = "rpds_py-0.18.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3a96e0c6a41dcdba3a0a581bbf6c44bb863f27c541547fb4b9711fd8cf0ffad4"}, + {file = "rpds_py-0.18.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30f43887bbae0d49113cbaab729a112251a940e9b274536613097ab8b4899cf6"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcb25daa9219b4cf3a0ab24b0eb9a5cc8949ed4dc72acb8fa16b7e1681aa3c58"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d68c93e381010662ab873fea609bf6c0f428b6d0bb00f2c6939782e0818d37bf"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b34b7aa8b261c1dbf7720b5d6f01f38243e9b9daf7e6b8bc1fd4657000062f2c"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e6d75ab12b0bbab7215e5d40f1e5b738aa539598db27ef83b2ec46747df90e1"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b8612cd233543a3781bc659c731b9d607de65890085098986dfd573fc2befe5"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:aec493917dd45e3c69d00a8874e7cbed844efd935595ef78a0f25f14312e33c6"}, + {file = "rpds_py-0.18.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:661d25cbffaf8cc42e971dd570d87cb29a665f49f4abe1f9e76be9a5182c4688"}, + {file = "rpds_py-0.18.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1df3659d26f539ac74fb3b0c481cdf9d725386e3552c6fa2974f4d33d78e544b"}, + {file = "rpds_py-0.18.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a1ce3ba137ed54f83e56fb983a5859a27d43a40188ba798993812fed73c70836"}, + {file = "rpds_py-0.18.0-cp311-none-win32.whl", hash = "sha256:69e64831e22a6b377772e7fb337533c365085b31619005802a79242fee620bc1"}, + {file = "rpds_py-0.18.0-cp311-none-win_amd64.whl", hash = "sha256:998e33ad22dc7ec7e030b3df701c43630b5bc0d8fbc2267653577e3fec279afa"}, + {file = "rpds_py-0.18.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7f2facbd386dd60cbbf1a794181e6aa0bd429bd78bfdf775436020172e2a23f0"}, + {file = "rpds_py-0.18.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1d9a5be316c15ffb2b3c405c4ff14448c36b4435be062a7f578ccd8b01f0c4d8"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd5bf1af8efe569654bbef5a3e0a56eca45f87cfcffab31dd8dde70da5982475"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5417558f6887e9b6b65b4527232553c139b57ec42c64570569b155262ac0754f"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:56a737287efecafc16f6d067c2ea0117abadcd078d58721f967952db329a3e5c"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8f03bccbd8586e9dd37219bce4d4e0d3ab492e6b3b533e973fa08a112cb2ffc9"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4457a94da0d5c53dc4b3e4de1158bdab077db23c53232f37a3cb7afdb053a4e3"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0ab39c1ba9023914297dd88ec3b3b3c3f33671baeb6acf82ad7ce883f6e8e157"}, + {file = "rpds_py-0.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9d54553c1136b50fd12cc17e5b11ad07374c316df307e4cfd6441bea5fb68496"}, + {file = "rpds_py-0.18.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0af039631b6de0397ab2ba16eaf2872e9f8fca391b44d3d8cac317860a700a3f"}, + {file = "rpds_py-0.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:84ffab12db93b5f6bad84c712c92060a2d321b35c3c9960b43d08d0f639d60d7"}, + {file = "rpds_py-0.18.0-cp312-none-win32.whl", hash = "sha256:685537e07897f173abcf67258bee3c05c374fa6fff89d4c7e42fb391b0605e98"}, + {file = "rpds_py-0.18.0-cp312-none-win_amd64.whl", hash = "sha256:e003b002ec72c8d5a3e3da2989c7d6065b47d9eaa70cd8808b5384fbb970f4ec"}, + {file = "rpds_py-0.18.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:08f9ad53c3f31dfb4baa00da22f1e862900f45908383c062c27628754af2e88e"}, + {file = "rpds_py-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c0013fe6b46aa496a6749c77e00a3eb07952832ad6166bd481c74bda0dcb6d58"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e32a92116d4f2a80b629778280103d2a510a5b3f6314ceccd6e38006b5e92dcb"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e541ec6f2ec456934fd279a3120f856cd0aedd209fc3852eca563f81738f6861"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bed88b9a458e354014d662d47e7a5baafd7ff81c780fd91584a10d6ec842cb73"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2644e47de560eb7bd55c20fc59f6daa04682655c58d08185a9b95c1970fa1e07"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e8916ae4c720529e18afa0b879473049e95949bf97042e938530e072fde061d"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:465a3eb5659338cf2a9243e50ad9b2296fa15061736d6e26240e713522b6235c"}, + {file = "rpds_py-0.18.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ea7d4a99f3b38c37eac212dbd6ec42b7a5ec51e2c74b5d3223e43c811609e65f"}, + {file = "rpds_py-0.18.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:67071a6171e92b6da534b8ae326505f7c18022c6f19072a81dcf40db2638767c"}, + {file = "rpds_py-0.18.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:41ef53e7c58aa4ef281da975f62c258950f54b76ec8e45941e93a3d1d8580594"}, + {file = "rpds_py-0.18.0-cp38-none-win32.whl", hash = "sha256:fdea4952db2793c4ad0bdccd27c1d8fdd1423a92f04598bc39425bcc2b8ee46e"}, + {file = "rpds_py-0.18.0-cp38-none-win_amd64.whl", hash = "sha256:7cd863afe7336c62ec78d7d1349a2f34c007a3cc6c2369d667c65aeec412a5b1"}, + {file = "rpds_py-0.18.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:5307def11a35f5ae4581a0b658b0af8178c65c530e94893345bebf41cc139d33"}, + {file = "rpds_py-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77f195baa60a54ef9d2de16fbbfd3ff8b04edc0c0140a761b56c267ac11aa467"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39f5441553f1c2aed4de4377178ad8ff8f9d733723d6c66d983d75341de265ab"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a00312dea9310d4cb7dbd7787e722d2e86a95c2db92fbd7d0155f97127bcb40"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f2fc11e8fe034ee3c34d316d0ad8808f45bc3b9ce5857ff29d513f3ff2923a1"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:586f8204935b9ec884500498ccc91aa869fc652c40c093bd9e1471fbcc25c022"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddc2f4dfd396c7bfa18e6ce371cba60e4cf9d2e5cdb71376aa2da264605b60b9"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ddcba87675b6d509139d1b521e0c8250e967e63b5909a7e8f8944d0f90ff36f"}, + {file = "rpds_py-0.18.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7bd339195d84439cbe5771546fe8a4e8a7a045417d8f9de9a368c434e42a721e"}, + {file = "rpds_py-0.18.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:d7c36232a90d4755b720fbd76739d8891732b18cf240a9c645d75f00639a9024"}, + {file = "rpds_py-0.18.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6b0817e34942b2ca527b0e9298373e7cc75f429e8da2055607f4931fded23e20"}, + {file = "rpds_py-0.18.0-cp39-none-win32.whl", hash = "sha256:99f70b740dc04d09e6b2699b675874367885217a2e9f782bdf5395632ac663b7"}, + {file = "rpds_py-0.18.0-cp39-none-win_amd64.whl", hash = "sha256:6ef687afab047554a2d366e112dd187b62d261d49eb79b77e386f94644363294"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ad36cfb355e24f1bd37cac88c112cd7730873f20fb0bdaf8ba59eedf8216079f"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:36b3ee798c58ace201289024b52788161e1ea133e4ac93fba7d49da5fec0ef9e"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8a2f084546cc59ea99fda8e070be2fd140c3092dc11524a71aa8f0f3d5a55ca"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e4461d0f003a0aa9be2bdd1b798a041f177189c1a0f7619fe8c95ad08d9a45d7"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8db715ebe3bb7d86d77ac1826f7d67ec11a70dbd2376b7cc214199360517b641"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:793968759cd0d96cac1e367afd70c235867831983f876a53389ad869b043c948"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66e6a3af5a75363d2c9a48b07cb27c4ea542938b1a2e93b15a503cdfa8490795"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ef0befbb5d79cf32d0266f5cff01545602344eda89480e1dd88aca964260b18"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1d4acf42190d449d5e89654d5c1ed3a4f17925eec71f05e2a41414689cda02d1"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:a5f446dd5055667aabaee78487f2b5ab72e244f9bc0b2ffebfeec79051679984"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9dbbeb27f4e70bfd9eec1be5477517365afe05a9b2c441a0b21929ee61048124"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:22806714311a69fd0af9b35b7be97c18a0fc2826e6827dbb3a8c94eac6cf7eeb"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:b34ae4636dfc4e76a438ab826a0d1eed2589ca7d9a1b2d5bb546978ac6485461"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c8370641f1a7f0e0669ddccca22f1da893cef7628396431eb445d46d893e5cd"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c8362467a0fdeccd47935f22c256bec5e6abe543bf0d66e3d3d57a8fb5731863"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11a8c85ef4a07a7638180bf04fe189d12757c696eb41f310d2426895356dcf05"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b316144e85316da2723f9d8dc75bada12fa58489a527091fa1d5a612643d1a0e"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf1ea2e34868f6fbf070e1af291c8180480310173de0b0c43fc38a02929fc0e3"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e546e768d08ad55b20b11dbb78a745151acbd938f8f00d0cfbabe8b0199b9880"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4901165d170a5fde6f589acb90a6b33629ad1ec976d4529e769c6f3d885e3e80"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:618a3d6cae6ef8ec88bb76dd80b83cfe415ad4f1d942ca2a903bf6b6ff97a2da"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ed4eb745efbff0a8e9587d22a84be94a5eb7d2d99c02dacf7bd0911713ed14dd"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6c81e5f372cd0dc5dc4809553d34f832f60a46034a5f187756d9b90586c2c307"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:43fbac5f22e25bee1d482c97474f930a353542855f05c1161fd804c9dc74a09d"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d7faa6f14017c0b1e69f5e2c357b998731ea75a442ab3841c0dbbbfe902d2c4"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:08231ac30a842bd04daabc4d71fddd7e6d26189406d5a69535638e4dcb88fe76"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:044a3e61a7c2dafacae99d1e722cc2d4c05280790ec5a05031b3876809d89a5c"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f26b5bd1079acdb0c7a5645e350fe54d16b17bfc5e71f371c449383d3342e17"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:482103aed1dfe2f3b71a58eff35ba105289b8d862551ea576bd15479aba01f66"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1374f4129f9bcca53a1bba0bb86bf78325a0374577cf7e9e4cd046b1e6f20e24"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:635dc434ff724b178cb192c70016cc0ad25a275228f749ee0daf0eddbc8183b1"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:bc362ee4e314870a70f4ae88772d72d877246537d9f8cb8f7eacf10884862432"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:4832d7d380477521a8c1644bbab6588dfedea5e30a7d967b5fb75977c45fd77f"}, + {file = "rpds_py-0.18.0.tar.gz", hash = "sha256:42821446ee7a76f5d9f71f9e33a4fb2ffd724bb3e7f93386150b61a43115788d"}, +] + [[package]] name = "ruamel-yaml" version = "0.17.31" @@ -2008,17 +2128,6 @@ files = [ {file = "types_certifi-2020.4.0-py3-none-any.whl", hash = "sha256:0ffdbe451d3b02f6d2cfd87bcfb2f086a4ff1fa76a35d51cfc3771e261d7a8fd"}, ] -[[package]] -name = "types-jsonschema" -version = "4.17.0.8" -description = "Typing stubs for jsonschema" -optional = false -python-versions = "*" -files = [ - {file = "types-jsonschema-4.17.0.8.tar.gz", hash = "sha256:96a56990910f405e62de58862c0bbb3ac29ee6dba6d3d99aa0ba7f874cc547de"}, - {file = "types_jsonschema-4.17.0.8-py3-none-any.whl", hash = "sha256:f5958eb7b53217dfb5125f0412aeaef226a8a9013eac95816c95b5b523f6796b"}, -] - [[package]] name = "types-python-dateutil" version = "2.8.19.13" @@ -2141,4 +2250,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8.1,<3.13" -content-hash = "2ea49a42c60916ca621a5dc118bdca8be787fd7c5ce6ae0ce305565abaeceaad" +content-hash = "7aa46fcf2b2821ab7e1f4b4d289bcc4abbb0a68ba1a5f7c637d2adbb0c7dbac6" diff --git a/pyproject.toml b/pyproject.toml index 0059f5c09..450421b92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,13 +32,14 @@ python-dateutil = "^2.8.1" httpx = ">=0.15.4,<0.25.0" autoflake = "^1.4 || ^2.0.0" PyYAML = "^6.0" -dlt = {git = "https://github.com/dlt-hub/dlt.git", branch="sthor/remove-rest-client-cyclic-import", extras = ["duckdb"]} +dlt = {extras = ["duckdb"], version = "^0.4.8"} requests = "^2.30.0" questionary = "^1.10.0" enlighten = "^1.11.2" openapi-schema-pydantic = "^1.2.4" pyjwt = "^2.8.0" cryptography = "^42.0.5" +referencing = "^0.34.0" [tool.poetry.scripts] dlt-init = "openapi_python_client.cli:app" @@ -57,7 +58,6 @@ types-python-dateutil = "^2.0.0" [tool.poetry.group.dev.dependencies] flake8 = "^6.0.0" -types-jsonschema = "^4.17.0.8" [tool.taskipy.tasks] check = """ From 482958fb4b0a4eb6a08eef0180261db76d83c972 Mon Sep 17 00:00:00 2001 From: Steinthor Palsson Date: Sat, 13 Apr 2024 15:35:42 -0400 Subject: [PATCH 18/18] Func generate endpoint info for prompts --- openapi_python_client/parser/context.py | 7 +- openapi_python_client/parser/endpoints.py | 4 +- openapi_python_client/parser/models.py | 3 + openapi_python_client/parser/prompts.py | 163 ++++++++++++++++++++++ 4 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 openapi_python_client/parser/prompts.py diff --git a/openapi_python_client/parser/context.py b/openapi_python_client/parser/context.py index ac2483b51..9e71750f9 100644 --- a/openapi_python_client/parser/context.py +++ b/openapi_python_client/parser/context.py @@ -62,14 +62,17 @@ def __init__(self, config: Config, spec: osp.OpenAPI, spec_raw: dict[str, Any]) registry = referencing.Registry().with_resource(resource=resource, uri="") self._resolver = registry.resolver() - def _component_from_reference(self, ref: osp.Reference) -> dict[str, Any]: - url = ref.ref + def _component_from_reference_url(self, url: str) -> dict[str, Any]: if url in self._component_cache: return self._component_cache[url] obj = self._resolver.lookup(url).contents self._component_cache[url] = obj return obj + def _component_from_reference(self, ref: osp.Reference) -> dict[str, Any]: + url = ref.ref + return self._component_from_reference_url(url) + def schema_and_name_from_reference(self, ref: Union[osp.Reference, osp.Schema]) -> Tuple[str, osp.Schema]: name: Optional[str] = None if isinstance(ref, osp.Reference): diff --git a/openapi_python_client/parser/endpoints.py b/openapi_python_client/parser/endpoints.py index 67892ba1a..9c6dad40f 100644 --- a/openapi_python_client/parser/endpoints.py +++ b/openapi_python_client/parser/endpoints.py @@ -17,7 +17,7 @@ from openapi_python_client.parser.pagination import Pagination from openapi_python_client.parser.parameters import Parameter -TMethod = Literal["get", "post", "put", "patch"] +TMethod = Literal["GET", "POST", "PUT", "PATCH"] Tree = Dict[str, Union["Endpoint", "Tree"]] @@ -425,7 +425,7 @@ def from_context(cls, context: OpenapiContext) -> "EndpointCollection": continue endpoints.append( Endpoint.from_operation( - cast(TMethod, op_name), + cast(TMethod, op_name.upper()), path, operation, path_table_names[path], diff --git a/openapi_python_client/parser/models.py b/openapi_python_client/parser/models.py index fa3b650e9..176c10e20 100644 --- a/openapi_python_client/parser/models.py +++ b/openapi_python_client/parser/models.py @@ -101,6 +101,7 @@ class SchemaWrapper: enum_values: Optional[List[Any]] = None primary_key: Optional[str] = None + examples: List[Any] = field(default_factory=list) def __getitem__(self, item: str) -> "Property": try: @@ -285,6 +286,7 @@ def from_reference( if isinstance(default, str): default = convert("str", default) + examples = ([schema.example] if schema.example else schema.examples) or [] crawler = SchemaCrawler() result = cls( schema=schema, @@ -304,6 +306,7 @@ def from_reference( type_format=schema.schema_format, maximum=schema.maximum, enum_values=schema.enum, + examples=examples, ) result.primary_key = result._find_primary_key() crawler.crawl(result) diff --git a/openapi_python_client/parser/prompts.py b/openapi_python_client/parser/prompts.py new file mode 100644 index 000000000..2fc0e7384 --- /dev/null +++ b/openapi_python_client/parser/prompts.py @@ -0,0 +1,163 @@ +from dataclasses import dataclass +from typing import Any + +from openapi_python_client.parser.endpoints import Endpoint + + +@dataclass +class PropInfo: + json_path: str + is_optional: bool + types: list[str] + type_format: str | None + description: str | None + examples: list[Any] + maximum: float | None + default: Any | None + + def __str__(self) -> str: + ret = "" + ret += f"json_path: {self.json_path}\n" + ret += f"is_optional: {self.is_optional}\n" + ret += f"types: {', '.join(self.types)}\n" + if self.type_format: + ret += f"type_format: {self.type_format}\n" + if self.default: + ret += f"default: {self.default}\n" + if self.maximum: + ret += f"maximum: {self.maximum}\n" + if self.description: + ret += f"description: {self.description.strip()}\n" + if self.examples: + examples = ", ".join(str(e) for e in self.examples) + ret += f"examples: {examples}\n" + return ret + + +@dataclass +class ResponseInfo: + status_code: str + description: str | None + schema_description: str | None + properties: list[PropInfo] + + def __str__(self) -> str: + ret = "" + ret += f"status_code: {self.status_code}\n" + if self.description: + ret += f"description: {self.description.strip()}\n" + if self.schema_description: + ret += f"schema_description: {self.schema_description.strip()}\n" + if self.properties: + ret += "\nProperties:\n\n" + for prop in self.properties: + ret += str(prop) + "\n" + return ret + + +@dataclass +class ParamInfo: + name: str + required: bool + types: list[str] + type_format: str | None + description: str | None + examples: list[Any] + maximum: float | None + default: Any | None + location: str + + def __str__(self) -> str: + ret = "" + ret += f"name: {self.name}\n" + ret += f"required: {self.required}\n" + ret += f"location: {self.location}\n" + ret += f"types: {', '.join(self.types)}\n" + if self.type_format: + ret += f"type_format: {self.type_format}\n" + if self.default: + ret += f"default: {self.default}\n" + if self.maximum: + ret += f"maximum: {self.maximum}\n" + if self.description: + ret += f"description: {self.description.strip()}\n" + if self.examples: + examples = ", ".join(str(e) for e in self.examples) + ret += f"examples: {examples}\n" + return ret + + +@dataclass +class EndpointInfo: + path: str + method: str + description: str | None + response: ResponseInfo + parameters: list[ParamInfo] + + def __str__(self) -> str: + ret = "endpoint: " + ret += f"{self.method} {self.path}\n" + if self.description: + ret += f"description: {self.description.strip()}\n" + if self.parameters: + ret += "\nParameters:\n\n" + for param in self.parameters: + ret += str(param) + "\n" + ret += "\nResponse:\n\n" + ret += str(self.response) + return ret + + +def create_endpoint_info(endpoint: Endpoint) -> EndpointInfo: + """Collect a full presentation of params and response properties in the endpoint. + params and properties include their names, descriptions, and types. + """ + response_schema = endpoint.data_response.content_schema + params = endpoint.parameters + + prop_infos: list[PropInfo] = [] + + for path, schema in response_schema.crawled_properties.items(): + is_optional = response_schema.crawled_properties.is_optional(path) + info = PropInfo( + json_path=".".join(path), + is_optional=is_optional, + types=schema.types, # type: ignore[arg-type] + type_format=schema.type_format, + description=schema.description, + examples=schema.examples, + maximum=schema.maximum, + default=schema.default, + ) + prop_infos.append(info) + + resp_info = ResponseInfo( + status_code=endpoint.data_response.status_code, + description=endpoint.data_response.description, + schema_description=response_schema.description, + properties=prop_infos, + ) + + param_infos = [] + for param in endpoint.parameters.values(): + param_info = ParamInfo( + name=param.name, + required=param.required, + types=param.schema.types, # type: ignore[arg-type] + type_format=param.schema.type_format, + description=param.schema.description, + examples=param.schema.examples, + maximum=param.schema.maximum, + default=param.schema.default, + location=param.location, + ) + param_infos.append(param_info) + + return EndpointInfo( + path=endpoint.path, + method=endpoint.method, + description=endpoint.description, + response=resp_info, + parameters=param_infos, + )