From 4a91d1ff5cacca9aa4a523c3f10619beec50b1a4 Mon Sep 17 00:00:00 2001 From: Tim O'Guin Date: Wed, 9 Jun 2021 20:19:53 -0500 Subject: [PATCH] chore: Cleanup for first beta release (#4) ### Changed - APIClient class moved to `aws_data_tools.client.APIClient` - README clean-up - Bump version to 0.1.0-beta1 - Adds a CI config for Semantic Pull Requests --- .github/semantic.yml | 2 + CHANGELOG.md | 12 +++-- README.md | 26 ++++------- aws_data_tools/__init__.py | 56 +----------------------- aws_data_tools/builders/organizations.py | 2 +- aws_data_tools/client.py | 52 ++++++++++++++++++++++ aws_data_tools/utils.py | 2 +- pyproject.toml | 2 +- 8 files changed, 75 insertions(+), 79 deletions(-) create mode 100644 .github/semantic.yml create mode 100644 aws_data_tools/client.py diff --git a/.github/semantic.yml b/.github/semantic.yml new file mode 100644 index 0000000..6d4ca7d --- /dev/null +++ b/.github/semantic.yml @@ -0,0 +1,2 @@ +titleAndCommits: true +scopes: ["cli", "organizations"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 058880a..f00fbd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,16 @@ Notes for any unreleased changes do here. When a new release is cut, move these the unreleased section to the section for the new release. --> -Upcoming changes. +No unreleased changes. -### Added +## [0.1.0-beta1] - 2020-06-09 ### Changed -### Removed +- Moves APIClient class to `aws_data_tools.client.APIClient` +- Cleans up README +- Bumps version to 0.1.0-beta1 +- Adds a CI config for Semantic Pull Requests ## [0.1.0-alpha4] - 2020-06-09 @@ -41,5 +44,6 @@ Initial alpha release These Markdown anchors provide a link to the diff for each release. They should be updated any time a new release is cut. --> -[Unreleased]: https://github.com/timoguin/aws-org-tools-py/compare/v0.1.0-alpha4...HEAD +[Unreleased]: https://github.com/timoguin/aws-org-tools-py/compare/v0.1.0-beta-1...HEAD +[0.1.0-beta1]: https://github.com/timoguin/aws-org-tools-py/compare/v0.1.0-alpha4...v0.1.0-beta1 [0.1.0-alpha4]: https://github.com/timoguin/aws-org-tools-py/releases/tag/v0.1.0-alpha4 diff --git a/README.md b/README.md index 01f74bd..f82f38d 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,9 @@ -# AWS Data Tools - An set of opinioned (but flexible) Python libraries for querying and transforming data from various AWS APIs, as well as a CLI interface. This is in early development. -## Installation +# Installation Using pip should work on any system with at least Python 3.9: @@ -19,16 +17,16 @@ By default, the CLI is not installed. To include it, you can specify it as an ex $ pip install aws-data-tools[cli] ``` -## Usage +# Usage There are currently 4 main components of the package: helpers for working with AWS session and APIs, data models for API data types, builders to query AWS APIs and perform deserialization and ETL operations of raw data, and a CLI tool to further abstract some of these operations. -### API Client +## API Client -The [APIClient](aws_data_models/__init__.py) class wraps the initialization of a boto3 +The [APIClient](aws_data_models/client.py) class wraps the initialization of a boto3 session and a low-level client for a named service. It contains a single `api()` function that takes the name of an API operation and any necessary request data as kwargs. @@ -40,7 +38,7 @@ passed for any desired customizations. When initializing the class, it will create a session and a client. ```python -from aws_data_tools import APIClient +from aws_data_tools.client import APIClient client = APIClient("organizations") org = client.api("describe_organization").get("organization") @@ -62,15 +60,13 @@ format that the APIs utilize. Any returned data has any keys converted to `snake The raw boto3 session is available as the `session` field, and the raw, low-level client is available as the `client` field. -### Data Models +## Data Models The [models](aws_data_tools/models) package contains a collection of opinionated models implemented as data classes. There is a package for each available service. Each one is named after the service that would be passed when creating a boto3 client using `boto3.client('service_name')`. -#### Organizations - Most data types used with the Organizations APIs are supported. The top-level `Organization` class is the most useful, as it also acts as a container for all other related data in the organization. @@ -87,7 +83,7 @@ The following data types and operations are currently not supported: All other data types are supported. ```python -from aws_data_tools import APIClient +from aws_data_tools.client import APIClient from aws_data_tools.models.organizations import Organization client = APIClient("organizations") @@ -99,15 +95,13 @@ org.as_json() View the [package](aws_data_tools/models/organization/__init__.py) for the full list of models. -### Builders +## Builders While it is possible to directly utilize and interact with the data models, probably the largest benefit is the [builders](aws_data_tools/builders) package. It abstracts any API operations and data transformations required to build data models. The models can then be serialized to dicts, as well as JSON or YAML strings. -### Organizations - A full model of an AWS Organization can be constructed using the `OrganizationDataBuilder` class. It handles recursing the organizational tree and populating any relational data between the various nodes, e.g., parent-child @@ -150,7 +144,7 @@ org.init_policy_targets() org.init_effective_policies() ``` -### CLI +## CLI As noted above, the CLI is an optional component that can be installed using pip's bracket notation for extras: @@ -177,8 +171,6 @@ Commands: organization Interact with data from AWS Organizations APIs ``` -#### Organizations - Here is how to dump a JSON representation of an AWS Organization to stdout: The `organization` subcommand allows dumping organization data to a file or to stdout: diff --git a/aws_data_tools/__init__.py b/aws_data_tools/__init__.py index 8013b5f..d1cd127 100644 --- a/aws_data_tools/__init__.py +++ b/aws_data_tools/__init__.py @@ -1,60 +1,6 @@ -from dataclasses import dataclass, field, InitVar -from typing import Any, ClassVar, Dict, List, Union - -from boto3.session import Session -from botocore.client import BaseClient -from botocore.paginate import PageIterator, Paginator -from humps import depascalize, pascalize - - -__VERSION__ = "0.1.0-alpha4" - - -_DEFAULT_PAGINATION_CONFIG = {"MaxItems": 500} +__VERSION__ = "0.1.0-beta1" def get_version() -> str: """Return the version of the package""" return __VERSION__ - - -@dataclass -class APIClient: - """ - Service client for interacting with named AWS API services. When initialized, it - establishes a boto3 session and client for the specified service. Loads - """ - - service: str - client: BaseClient = field(default=None) - session: Session = field(default_factory=Session) - - def api(self, func: str, **kwargs) -> Union[Dict[str, Any], List[Dict[str, Any]]]: - """ - Call a named API action by string. All arguments to the action should be passed - as kwargs. The returned data has keys normalized to snake_case. Similarly, all - kwargs can be passed in snake_case as well. - - If the API action is one that supports pagination, it is handled automaticaly. - All paginated responses are fully aggregated and then returned. - """ - kwargs = pascalize(kwargs) - paginate = self.client.can_paginate(func) - if paginate: - paginator = self.client.get_paginator(func) - if kwargs.get("PaginationConfig") is None: - kwargs.update(PaginationConfig=_DEFAULT_PAGINATION_CONFIG) - page_iterator = paginator.paginate(**kwargs) - responses = [] - for page in page_iterator: - page = depascalize(page) - metakeys = ["next_token", "response_metadata"] - key = [k for k in page.keys() if k not in metakeys][0] - responses.extend(page.get(key)) - return responses - else: - response = getattr(self.client, func)(**kwargs) - return depascalize(response) - - def __post_init__(self): - self.client = self.session.client(self.service) diff --git a/aws_data_tools/builders/organizations.py b/aws_data_tools/builders/organizations.py index 40c9184..9e94492 100644 --- a/aws_data_tools/builders/organizations.py +++ b/aws_data_tools/builders/organizations.py @@ -4,7 +4,7 @@ from botocore.session import Session from botocore.client import BaseClient -from .. import APIClient +from ..client import APIClient from ..models import ModelBase from ..utils import tag_list_to_dict, query_tags diff --git a/aws_data_tools/client.py b/aws_data_tools/client.py new file mode 100644 index 0000000..3dbc560 --- /dev/null +++ b/aws_data_tools/client.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass, field, InitVar +from typing import Any, ClassVar, Dict, List, Union + +from boto3.session import Session +from botocore.client import BaseClient +from botocore.paginate import PageIterator, Paginator +from humps import depascalize, pascalize + + +_DEFAULT_PAGINATION_CONFIG = {"MaxItems": 500} + + +@dataclass +class APIClient: + """ + Service client for interacting with named AWS API services. When initialized, it + establishes a boto3 session and client for the specified service. Loads + """ + + service: str + client: BaseClient = field(default=None) + session: Session = field(default_factory=Session) + + def api(self, func: str, **kwargs) -> Union[Dict[str, Any], List[Dict[str, Any]]]: + """ + Call a named API action by string. All arguments to the action should be passed + as kwargs. The returned data has keys normalized to snake_case. Similarly, all + kwargs can be passed in snake_case as well. + + If the API action is one that supports pagination, it is handled automaticaly. + All paginated responses are fully aggregated and then returned. + """ + kwargs = pascalize(kwargs) + paginate = self.client.can_paginate(func) + if paginate: + paginator = self.client.get_paginator(func) + if kwargs.get("PaginationConfig") is None: + kwargs.update(PaginationConfig=_DEFAULT_PAGINATION_CONFIG) + page_iterator = paginator.paginate(**kwargs) + responses = [] + for page in page_iterator: + page = depascalize(page) + metakeys = ["next_token", "response_metadata"] + key = [k for k in page.keys() if k not in metakeys][0] + responses.extend(page.get(key)) + return responses + else: + response = getattr(self.client, func)(**kwargs) + return depascalize(response) + + def __post_init__(self): + self.client = self.session.client(self.service) diff --git a/aws_data_tools/utils.py b/aws_data_tools/utils.py index 1d9aed1..18336c5 100644 --- a/aws_data_tools/utils.py +++ b/aws_data_tools/utils.py @@ -3,7 +3,7 @@ from json import JSONEncoder from typing import Dict, List -from . import APIClient +from .client import APIClient def tag_list_to_dict(tags: List[Dict[str, str]]) -> Dict[str, str]: diff --git a/pyproject.toml b/pyproject.toml index bbb3914..6bbc6e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "aws_data_tools" -version = "0.1.0-alpha4" +version = "0.1.0-beta1" description = "A set of Python libraries for querying and transforming data from AWS APIs" authors = ["Tim O'Guin "] license = "MIT"