Skip to content

Commit

Permalink
Merge pull request #8 from MeltanoLabs/key_properties
Browse files Browse the repository at this point in the history
feat: Key properties
  • Loading branch information
pnadolny13 authored Apr 11, 2023
2 parents 49c056d + 8d44ac7 commit 0218348
Show file tree
Hide file tree
Showing 5 changed files with 43 additions and 22 deletions.
4 changes: 4 additions & 0 deletions tap_dynamodb/dynamo.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@ def get_table_json_schema(self, table_name: str, strategy: str = "infer"):
raise Exception(f"Strategy {strategy} not supported")
return schema

def get_table_key_properties(self, table_name):
key_schema = self.resource.Table(table_name).key_schema
return [key.get("AttributeName") for key in key_schema]

def recursively_drop_required(self, schema: dict) -> None:
"""Recursively drop the required property from a schema.
Expand Down
3 changes: 3 additions & 0 deletions tap_dynamodb/streams.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,7 @@ def schema(self) -> dict:
# TODO: SDC columns
if not self._schema:
self._schema = self.dynamodb_obj.get_table_json_schema(self.table_name)
self.primary_keys = self.dynamodb_obj.get_table_key_properties(
self.table_name
)
return self._schema
10 changes: 6 additions & 4 deletions tests/test_aws_authenticators.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from unittest.mock import patch

import pytest
from moto import mock_dynamodb, mock_sts

from tap_dynamodb.aws_authenticators import AWSBotoAuthenticator
from unittest.mock import patch


@patch(
Expand Down Expand Up @@ -84,7 +86,7 @@ def test_get_session_assume_role():
},
"dynamodb",
)
session = auth.get_session()
auth.get_session()


@mock_dynamodb
Expand All @@ -98,7 +100,7 @@ def test_get_client():
"dynamodb",
)
session = auth.get_session()
client = auth.get_client(session, "dynamodb")
auth.get_client(session, "dynamodb")


@mock_dynamodb
Expand All @@ -112,4 +114,4 @@ def test_get_resource():
"dynamodb",
)
session = auth.get_session()
resource = auth.get_resource(session, "dynamodb")
auth.get_resource(session, "dynamodb")
32 changes: 22 additions & 10 deletions tests/test_dynamodb.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@
SAMPLE_CONFIG = {
"aws_access_key_id": "foo",
"aws_secret_access_key": "bar",
"aws_default_region": "us-west-2"
"aws_default_region": "us-west-2",
}


def create_table(moto_conn, name):
return moto_conn.create_table(
TableName=name,
Expand Down Expand Up @@ -38,6 +39,7 @@ def test_list_tables():
assert tables[0] == "table_1"
assert tables[-1] == "table_105"


@mock_dynamodb
def test_list_tables_filtered():
# PREP
Expand All @@ -56,6 +58,7 @@ def test_list_tables_filtered():
tables = db_obj.list_tables([])
assert len(tables) == 0


@mock_dynamodb
def test_get_items():
# PREP
Expand Down Expand Up @@ -121,15 +124,24 @@ def test_get_table_json_schema():
}


@mock_dynamodb
def test_get_table_key_properties():
# PREP
moto_conn = boto3.resource("dynamodb", region_name="us-west-2")
table = create_table(moto_conn, "table")
for num in range(5):
table.put_item(
Item={"year": 2023, "title": f"foo_{num}", "info": {"plot": "bar"}}
)
# END PREP

db_obj = DynamoDB(SAMPLE_CONFIG)
assert ["year", "title"] == db_obj.get_table_key_properties("table")


def test_coerce_types():
import decimal
db_obj = DynamoDB(SAMPLE_CONFIG)
coerced = db_obj._coerce_types(
{
"foo": decimal.Decimal("1.23")
}
)
assert coerced == {
"foo": "1.23"
}

db_obj = DynamoDB(SAMPLE_CONFIG)
coerced = db_obj._coerce_types({"foo": decimal.Decimal("1.23")})
assert coerced == {"foo": "1.23"}
16 changes: 8 additions & 8 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ commands =
poetry install -v
poetry run coverage run -m pytest --capture=no {posargs}
poetry run coverage html -d tests/codecoverage
poetry run black --check tap_dynamodb/
poetry run isort --check tap_dynamodb
poetry run flake8 tap_dynamodb
poetry run black --check tap_dynamodb/ tests/
poetry run isort --check tap_dynamodb tests
poetry run flake8 tap_dynamodb tests
; poetry run pydocstyle tap_dynamodb
poetry run mypy . --exclude='tests'

Expand All @@ -30,17 +30,17 @@ commands =
# To execute, run `tox -e format`
commands =
poetry install -v
poetry run black tap_dynamodb/
poetry run isort tap_dynamodb
poetry run black tap_dynamodb/ tests/
poetry run isort tap_dynamodb tests

[testenv:lint]
# Raise an error if lint and style standards are not met.
# To execute, run `tox -e lint`
commands =
poetry install -v
poetry run black --check --diff tap_dynamodb/
poetry run isort --check tap_dynamodb
poetry run flake8 tap_dynamodb
poetry run black --check --diff tap_dynamodb/ tests/
poetry run isort --check tap_dynamodb tests
poetry run flake8 tap_dynamodb tests
# refer to mypy.ini for specific settings
poetry run mypy . --exclude='tests'

Expand Down

0 comments on commit 0218348

Please sign in to comment.