Skip to content

Commit

Permalink
refactor: remove testable mangled names
Browse files Browse the repository at this point in the history
  • Loading branch information
gouline committed Dec 22, 2024
1 parent ea0693d commit c542a42
Show file tree
Hide file tree
Showing 6 changed files with 71 additions and 66 deletions.
2 changes: 1 addition & 1 deletion dbtmetabase/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
]

try:
from ._version import __version__ as version # type: ignore
from ._version import __version__ as version

__version__ = version
except ModuleNotFoundError:
Expand Down
64 changes: 33 additions & 31 deletions dbtmetabase/_exposures.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def dbname(details: Mapping) -> str:
return details[key]
return ""

ctx = self.__Context(
ctx = _Context(
model_refs={m.alias_path.lower(): m.ref for m in models if m.ref},
database_names={
d["id"]: dbname(d["details"]) for d in self.metabase.get_databases()
Expand Down Expand Up @@ -127,7 +127,7 @@ def dbname(details: Mapping) -> str:
uid=collection["id"],
models=("card", "dashboard"),
):
exposure = self.__Exposure(
exposure = _Exposure(
model=item["model"],
uid=item["id"],
label="Exposure [Unresolved Name]",
Expand All @@ -153,7 +153,7 @@ def dbname(details: Mapping) -> str:
f"Visualization: {entity.get('display', 'Unknown').title()}"
)

self.__exposure_card(ctx, exposure, entity)
self._exposure_card(ctx, exposure, entity)

if average_query_time_ms := entity.get("average_query_time"):
average_query_time_s = average_query_time_ms / 1000
Expand All @@ -179,7 +179,7 @@ def dbname(details: Mapping) -> str:
continue

if card := self.metabase.find_card(uid=card["id"]):
self.__exposure_card(ctx, exposure, card)
self._exposure_card(ctx, exposure, card)
else:
_logger.warning("Unexpected collection item '%s'", item["model"])
continue
Expand Down Expand Up @@ -236,7 +236,7 @@ def dbname(details: Mapping) -> str:

return exposures

def __exposure_card(self, ctx: __Context, exposure: __Exposure, card: Mapping):
def _exposure_card(self, ctx: _Context, exposure: _Exposure, card: Mapping):
"""Extracts exposures from Metabase questions."""

dataset_query = card.get("dataset_query", {})
Expand All @@ -248,7 +248,7 @@ def __exposure_card(self, ctx: __Context, exposure: __Exposure, card: Mapping):
else:
_logger.warning("Unsupported card type '%s'", card_type)

def __exposure_query(self, ctx: __Context, exposure: __Exposure, card: Mapping):
def __exposure_query(self, ctx: _Context, exposure: _Exposure, card: Mapping):
"""Extracts exposures from Metabase GUI queries."""

dataset_query = card.get("dataset_query", {})
Expand All @@ -259,7 +259,7 @@ def __exposure_query(self, ctx: __Context, exposure: __Exposure, card: Mapping):
# Question based on another question
source_card_uid = query_source.split("__")[-1]
if source_card := self.metabase.find_card(uid=source_card_uid):
self.__exposure_card(ctx, exposure, source_card)
self._exposure_card(ctx, exposure, source_card)

elif isinstance(query_source, int) and query_source in ctx.table_names:
# Question based on table
Expand All @@ -274,7 +274,7 @@ def __exposure_query(self, ctx: __Context, exposure: __Exposure, card: Mapping):
# Question based on another question
source_card_uid = join_source.split("__")[-1]
if source_card := self.metabase.find_card(uid=source_card_uid):
self.__exposure_card(ctx, exposure, source_card)
self._exposure_card(ctx, exposure, source_card)

continue

Expand All @@ -284,7 +284,7 @@ def __exposure_query(self, ctx: __Context, exposure: __Exposure, card: Mapping):
exposure.depends.add(joined_table)
_logger.info("Extracted model '%s' from join", joined_table)

def __exposure_native(self, ctx: __Context, exposure: __Exposure, card: Mapping):
def __exposure_native(self, ctx: _Context, exposure: _Exposure, card: Mapping):
"""Extracts exposures from Metabase native queries."""

dataset_query = card.get("dataset_query", {})
Expand Down Expand Up @@ -410,8 +410,8 @@ def __format_exposure(

return exposure

@staticmethod
def __write_exposures(
self,
exposures: Iterable[Mapping],
output_path: str,
output_grouping: Optional[str],
Expand Down Expand Up @@ -448,24 +448,26 @@ def __write_exposures(
stream=f,
)

@dc.dataclass
class __Context:
model_refs: Mapping[str, str]
database_names: Mapping[int, str]
table_names: Mapping[int, str]

@dc.dataclass
class __Exposure:
model: str
uid: str
label: str
name: str = ""
description: str = ""
created_at: str = ""
header: str = ""
creator_name: str = ""
creator_email: str = ""
average_query_time: Optional[str] = None
last_used_at: Optional[str] = None
native_query: Optional[str] = None
depends: Set[str] = dc.field(default_factory=set)

@dc.dataclass
class _Context:
model_refs: Mapping[str, str]
database_names: Mapping[int, str]
table_names: Mapping[int, str]


@dc.dataclass
class _Exposure:
model: str
uid: str
label: str
name: str = ""
description: str = ""
created_at: str = ""
header: str = ""
creator_name: str = ""
creator_email: str = ""
average_query_time: Optional[str] = None
last_used_at: Optional[str] = None
native_query: Optional[str] = None
depends: Set[str] = dc.field(default_factory=set)
59 changes: 30 additions & 29 deletions dbtmetabase/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@

_logger = logging.getLogger(__name__)

_SYNC_PERIOD = 5


class ModelsMixin(metaclass=ABCMeta):
"""Abstraction for exporting models."""

__SYNC_PERIOD = 5

DEFAULT_MODELS_SYNC_TIMEOUT = 30

@property
Expand Down Expand Up @@ -57,7 +57,7 @@ def export_models(
order_fields (bool, optional): Preserve column order in dbt project.
"""

ctx = self.__Context()
ctx = _Context()
success = True

database = self.metabase.find_database(name=metabase_database)
Expand All @@ -77,9 +77,9 @@ def export_models(
deadline = int(time.time()) + sync_timeout
synced = False
while not synced:
time.sleep(self.__SYNC_PERIOD)
time.sleep(_SYNC_PERIOD)

tables = self.__get_tables(database["id"])
tables = self._get_metabase_tables(database["id"])

synced = True
for model in models:
Expand Down Expand Up @@ -124,7 +124,7 @@ def export_models(
raise MetabaseStateError("Unable to sync models with Metabase")

for model in models:
success &= self.__export_model(
success &= self._export_model(
ctx=ctx,
model=model,
append_tags=append_tags,
Expand Down Expand Up @@ -159,9 +159,9 @@ def export_models(
if not success:
raise MetabaseStateError("Non-critical errors encountered, see above")

def __export_model(
def _export_model(
self,
ctx: __Context,
ctx: _Context,
model: Model,
append_tags: bool,
docs_url: Optional[str],
Expand Down Expand Up @@ -250,7 +250,7 @@ def __export_model(

def __export_model_column_order(
self,
ctx: __Context,
ctx: _Context,
model: Model,
api_table: Mapping,
table_key: str,
Expand Down Expand Up @@ -303,7 +303,7 @@ def __export_model_column_order(

def __export_column(
self,
ctx: __Context,
ctx: _Context,
schema_name: str,
model_name: str,
column: Column,
Expand Down Expand Up @@ -430,7 +430,7 @@ def __export_column(

return success

def __get_tables(self, database_id: str) -> Mapping[str, MutableMapping]:
def _get_metabase_tables(self, database_id: str) -> Mapping[str, MutableMapping]:
tables = {}

metadata = self.metabase.get_database_metadata(database_id)
Expand Down Expand Up @@ -461,8 +461,8 @@ def __get_tables(self, database_id: str) -> Mapping[str, MutableMapping]:

return tables

@staticmethod
def __filtered_models(
self,
models: Iterable[Model],
database_filter: Optional[Filter],
schema_filter: Optional[Filter],
Expand All @@ -479,25 +479,26 @@ def selected(m: Model) -> bool:

return list(filter(selected, models))

@dc.dataclass
class __Context:
tables: Mapping[str, MutableMapping] = dc.field(default_factory=dict)
updates: MutableMapping[str, MutableMapping] = dc.field(default_factory=dict)

def get_field(self, table_key: str, field_key: str) -> MutableMapping:
return self.tables.get(table_key, {}).get("fields", {}).get(field_key, {})
@dc.dataclass
class _Context:
tables: Mapping[str, MutableMapping] = dc.field(default_factory=dict)
updates: MutableMapping[str, MutableMapping] = dc.field(default_factory=dict)

def get_field(self, table_key: str, field_key: str) -> MutableMapping:
return self.tables.get(table_key, {}).get("fields", {}).get(field_key, {})

def update(self, entity: MutableMapping, change: Mapping, label: str):
entity.update(change)
def update(self, entity: MutableMapping, change: Mapping, label: str):
entity.update(change)

key = f"{entity['kind']}.{entity['id']}"
update = self.updates.get(key, {})
update["kind"] = entity["kind"]
update["id"] = entity["id"]
update["label"] = label
key = f"{entity['kind']}.{entity['id']}"
update = self.updates.get(key, {})
update["kind"] = entity["kind"]
update["id"] = entity["id"]
update["label"] = label

body = update.get("body", {})
body.update(change)
update["body"] = body
body = update.get("body", {})
body.update(change)
update["body"] = body

self.updates[key] = update
self.updates[key] = update
3 changes: 2 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import pytest

import dbtmetabase._models
from tests._mocks import TMP_PATH, MockDbtMetabase, MockMetabase


@pytest.fixture(name="core")
def fixture_core() -> MockDbtMetabase:
c = MockDbtMetabase()
c._ModelsMixin__SYNC_PERIOD = 1 # type: ignore
dbtmetabase._models._SYNC_PERIOD = 1
return c


Expand Down
7 changes: 4 additions & 3 deletions tests/test_exposures.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import pytest
import yaml

from dbtmetabase._exposures import _Context, _Exposure
from tests._mocks import FIXTURES_PATH, TMP_PATH, MockDbtMetabase


Expand Down Expand Up @@ -96,20 +97,20 @@ def test_extract_exposures_native_depends(
query: str,
expected: set,
):
ctx = MockDbtMetabase._ExposuresMixin__Context( # type: ignore
ctx = _Context(
model_refs={
"database.schema.table0": "model0",
"database.public.table1": "model1",
},
database_names={1: "database"},
table_names={},
)
exposure = MockDbtMetabase._ExposuresMixin__Exposure( # type: ignore
exposure = _Exposure(
model="card",
uid="",
label="",
)
core._ExposuresMixin__exposure_card( # type: ignore
core._exposure_card(
ctx=ctx,
exposure=exposure,
card={
Expand Down
2 changes: 1 addition & 1 deletion tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def test_build_lookups(core: MockDbtMetabase):
"INVENTORY.SKUS": {"SKU_ID", "PRODUCT"},
}

actual_tables = core._ModelsMixin__get_tables(database_id="2") # type: ignore
actual_tables = core._get_metabase_tables(database_id="2")

assert set(actual_tables.keys()) == set(expected.keys())

Expand Down

0 comments on commit c542a42

Please sign in to comment.