From 9a84f46e7fb91afcf176acbd1e2c2660a9dcddf3 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Tue, 11 Apr 2023 17:34:58 -0400 Subject: [PATCH 1/3] add key properties --- tap_dynamodb/dynamo.py | 4 ++++ tap_dynamodb/streams.py | 3 +++ tests/test_dynamodb.py | 32 ++++++++++++++++++++++---------- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/tap_dynamodb/dynamo.py b/tap_dynamodb/dynamo.py index 5a530da..719da85 100644 --- a/tap_dynamodb/dynamo.py +++ b/tap_dynamodb/dynamo.py @@ -85,6 +85,10 @@ def get_table_json_schema(self, table_name: str, strategy: str = "infer"): raise Exception(f"Strategy {strategy} not supported") return schema + def get_table_key_properties(self, table_name): + key_schema = self.resource.Table(table_name).key_schema + return [key.get("AttributeName") for key in key_schema] + def recursively_drop_required(self, schema: dict) -> None: """Recursively drop the required property from a schema. diff --git a/tap_dynamodb/streams.py b/tap_dynamodb/streams.py index ddc9318..ed6cd71 100644 --- a/tap_dynamodb/streams.py +++ b/tap_dynamodb/streams.py @@ -47,4 +47,7 @@ def schema(self) -> dict: # TODO: SDC columns if not self._schema: self._schema = self.dynamodb_obj.get_table_json_schema(self.table_name) + self.primary_keys = self.dynamodb_obj.get_table_key_properties( + self.table_name + ) return self._schema diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py index 9acc96f..03e42e9 100644 --- a/tests/test_dynamodb.py +++ b/tests/test_dynamodb.py @@ -6,9 +6,10 @@ SAMPLE_CONFIG = { "aws_access_key_id": "foo", "aws_secret_access_key": "bar", - "aws_default_region": "us-west-2" + "aws_default_region": "us-west-2", } + def create_table(moto_conn, name): return moto_conn.create_table( TableName=name, @@ -38,6 +39,7 @@ def test_list_tables(): assert tables[0] == "table_1" assert tables[-1] == "table_105" + @mock_dynamodb def test_list_tables_filtered(): # PREP @@ -56,6 +58,7 @@ def test_list_tables_filtered(): tables = db_obj.list_tables([]) assert len(tables) == 0 + @mock_dynamodb def test_get_items(): # PREP @@ -121,15 +124,24 @@ def test_get_table_json_schema(): } +@mock_dynamodb +def test_get_table_json_schema(): + # PREP + moto_conn = boto3.resource("dynamodb", region_name="us-west-2") + table = create_table(moto_conn, "table") + for num in range(5): + table.put_item( + Item={"year": 2023, "title": f"foo_{num}", "info": {"plot": "bar"}} + ) + # END PREP + + db_obj = DynamoDB(SAMPLE_CONFIG) + assert ["year", "title"] == db_obj.get_table_key_properties("table") + + def test_coerce_types(): import decimal - db_obj = DynamoDB(SAMPLE_CONFIG) - coerced = db_obj._coerce_types( - { - "foo": decimal.Decimal("1.23") - } - ) - assert coerced == { - "foo": "1.23" - } + db_obj = DynamoDB(SAMPLE_CONFIG) + coerced = db_obj._coerce_types({"foo": decimal.Decimal("1.23")}) + assert coerced == {"foo": "1.23"} From ceb35df5d01364502b098bcacfadf5a5c505e560 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Tue, 11 Apr 2023 17:35:06 -0400 Subject: [PATCH 2/3] fix sorting --- tests/test_aws_authenticators.py | 4 +++- tox.ini | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_aws_authenticators.py b/tests/test_aws_authenticators.py index adc64b7..ccfa800 100644 --- a/tests/test_aws_authenticators.py +++ b/tests/test_aws_authenticators.py @@ -1,7 +1,9 @@ +from unittest.mock import patch + import pytest from moto import mock_dynamodb, mock_sts + from tap_dynamodb.aws_authenticators import AWSBotoAuthenticator -from unittest.mock import patch @patch( diff --git a/tox.ini b/tox.ini index 3134dc8..faed144 100644 --- a/tox.ini +++ b/tox.ini @@ -10,8 +10,8 @@ commands = poetry install -v poetry run coverage run -m pytest --capture=no {posargs} poetry run coverage html -d tests/codecoverage - poetry run black --check tap_dynamodb/ - poetry run isort --check tap_dynamodb + poetry run black --check tap_dynamodb/ tests/ + poetry run isort --check tap_dynamodb tests poetry run flake8 tap_dynamodb ; poetry run pydocstyle tap_dynamodb poetry run mypy . --exclude='tests' @@ -30,8 +30,8 @@ commands = # To execute, run `tox -e format` commands = poetry install -v - poetry run black tap_dynamodb/ - poetry run isort tap_dynamodb + poetry run black tap_dynamodb/ tests/ + poetry run isort tap_dynamodb tests [testenv:lint] # Raise an error if lint and style standards are not met. @@ -39,8 +39,8 @@ commands = commands = poetry install -v poetry run black --check --diff tap_dynamodb/ - poetry run isort --check tap_dynamodb - poetry run flake8 tap_dynamodb + poetry run isort --check tap_dynamodb tests + poetry run flake8 tap_dynamodb tests # refer to mypy.ini for specific settings poetry run mypy . --exclude='tests' From 8d44ac7cc5015fec24c9fbac1d24dec658404836 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Tue, 11 Apr 2023 18:08:02 -0400 Subject: [PATCH 3/3] fix linting errors from CI --- tests/test_aws_authenticators.py | 6 +++--- tests/test_dynamodb.py | 2 +- tox.ini | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_aws_authenticators.py b/tests/test_aws_authenticators.py index ccfa800..9cd9c6a 100644 --- a/tests/test_aws_authenticators.py +++ b/tests/test_aws_authenticators.py @@ -86,7 +86,7 @@ def test_get_session_assume_role(): }, "dynamodb", ) - session = auth.get_session() + auth.get_session() @mock_dynamodb @@ -100,7 +100,7 @@ def test_get_client(): "dynamodb", ) session = auth.get_session() - client = auth.get_client(session, "dynamodb") + auth.get_client(session, "dynamodb") @mock_dynamodb @@ -114,4 +114,4 @@ def test_get_resource(): "dynamodb", ) session = auth.get_session() - resource = auth.get_resource(session, "dynamodb") + auth.get_resource(session, "dynamodb") diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py index 03e42e9..93803ff 100644 --- a/tests/test_dynamodb.py +++ b/tests/test_dynamodb.py @@ -125,7 +125,7 @@ def test_get_table_json_schema(): @mock_dynamodb -def test_get_table_json_schema(): +def test_get_table_key_properties(): # PREP moto_conn = boto3.resource("dynamodb", region_name="us-west-2") table = create_table(moto_conn, "table") diff --git a/tox.ini b/tox.ini index faed144..a00b950 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ commands = poetry run coverage html -d tests/codecoverage poetry run black --check tap_dynamodb/ tests/ poetry run isort --check tap_dynamodb tests - poetry run flake8 tap_dynamodb + poetry run flake8 tap_dynamodb tests ; poetry run pydocstyle tap_dynamodb poetry run mypy . --exclude='tests' @@ -38,7 +38,7 @@ commands = # To execute, run `tox -e lint` commands = poetry install -v - poetry run black --check --diff tap_dynamodb/ + poetry run black --check --diff tap_dynamodb/ tests/ poetry run isort --check tap_dynamodb tests poetry run flake8 tap_dynamodb tests # refer to mypy.ini for specific settings