From 1b790dbaab142eba8498fc93ef9e5420067df360 Mon Sep 17 00:00:00 2001 From: Maxime Armstrong <46797220+maximearmstrong@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:53:47 -0400 Subject: [PATCH] [dagster-looker] Update Looker documentation to showcase new API patterns (#25601) ## Summary Updates our Looker docs to incorporate the new specs-building pattern, norm of passing the resource to `Definitions`, and the new pattern for building executable asset definitions. --- docs/content/integrations/looker.mdx | 48 ++++++++++++------- .../api/apidocs/libraries/dagster-looker.rst | 4 ++ .../looker/customize-looker-assets.py | 8 +++- .../looker/materializing-looker-pdts.py | 27 +++++++---- .../looker/representing-looker-assets.py | 7 +-- 5 files changed, 65 insertions(+), 29 deletions(-) diff --git a/docs/content/integrations/looker.mdx b/docs/content/integrations/looker.mdx index e7a6f6b82b165..b21fe63cd5b9d 100644 --- a/docs/content/integrations/looker.mdx +++ b/docs/content/integrations/looker.mdx @@ -40,25 +40,26 @@ pip install dagster dagster-looker To load Looker assets into the Dagster asset graph, you must first construct a , which allows Dagster to communicate with your Looker instance. You'll need to supply your Looker instance URL and API credentials, which can be passed directly or accessed from the environment using . -Dagster can automatically load all views, explores, and dashboards from your Looker instance. Call the function, which returns a object containing all the asset definitions for these Looker assets. +Dagster can automatically load all views, explores, and dashboards from your Looker instance as asset specs. Call the function, which returns a list of s representing your Looker assets. You can then include these asset specs in your object: ```python file=/integrations/looker/representing-looker-assets.py -from dagster_looker import LookerResource +from dagster_looker import LookerResource, load_looker_asset_specs import dagster as dg -resource = LookerResource( +looker_resource = LookerResource( client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"), client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"), base_url=dg.EnvVar("LOOKERSDK_HOST_URL"), ) -defs = resource.build_defs() +looker_specs = load_looker_asset_specs(looker_resource=looker_resource) +defs = dg.Definitions(assets=[*looker_specs], resources={"looker": looker_resource}) ``` ### Customize asset definition metadata for Looker assets -By default, Dagster will generate asset keys for each Looker asset based on its type and name and populate default metadata. You can further customize asset properties by passing a custom subclass to the function. This subclass can implement methods to customize the asset keys or specs for each Looker asset type. +By default, Dagster will generate asset keys for each Looker asset based on its type and name and populate default metadata. You can further customize asset properties by passing a custom subclass to the function. This subclass can implement methods to customize the asset keys or specs for each Looker asset type. ```python file=/integrations/looker/customize-looker-assets.py from dagster_looker import ( @@ -66,11 +67,12 @@ from dagster_looker import ( LookerResource, LookerStructureData, LookerStructureType, + load_looker_asset_specs, ) import dagster as dg -resource = LookerResource( +looker_resource = LookerResource( client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"), client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"), base_url=dg.EnvVar("LOOKERSDK_HOST_URL"), @@ -91,31 +93,45 @@ class CustomDagsterLookerApiTranslator(DagsterLookerApiTranslator): return asset_spec -defs = resource.build_defs(dagster_looker_translator=CustomDagsterLookerApiTranslator()) +looker_specs = load_looker_asset_specs( + looker_resource, dagster_looker_translator=CustomDagsterLookerApiTranslator +) +defs = dg.Definitions(assets=[*looker_specs], resources={"looker": looker_resource}) ``` ### Materialize Looker PDTs from Dagster -You can use Dagster to orchestrate the materialization of Looker PDTs. To model a PDT as an asset, pass its definition by passing to function. +You can use Dagster to orchestrate the materialization of Looker PDTs. To model PDTs as assets, build their asset definitions by passing a list of to function. ```python file=/integrations/looker/materializing-looker-pdts.py -from dagster_looker import LookerResource, RequestStartPdtBuild +from dagster_looker import ( + LookerResource, + RequestStartPdtBuild, + build_looker_pdt_assets_definitions, + load_looker_asset_specs, +) import dagster as dg -resource = LookerResource( +looker_resource = LookerResource( client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"), client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"), base_url=dg.EnvVar("LOOKERSDK_HOST_URL"), ) -defs = resource.build_defs( +looker_specs = load_looker_asset_specs(looker_resource=looker_resource) + +pdts = build_looker_pdt_assets_definitions( + resource_key="looker", request_start_pdt_builds=[ - RequestStartPdtBuild( - model_name="analytics", - view_name="page_keyword_performance", - ), - ] + RequestStartPdtBuild(model_name="my_model", view_name="my_view") + ], +) + + +defs = dg.Definitions( + assets=[*pdts, *looker_specs], + resources={"looker": looker_resource}, ) ``` diff --git a/docs/sphinx/sections/api/apidocs/libraries/dagster-looker.rst b/docs/sphinx/sections/api/apidocs/libraries/dagster-looker.rst index 88f4b1acbffc9..409e6568c27ea 100644 --- a/docs/sphinx/sections/api/apidocs/libraries/dagster-looker.rst +++ b/docs/sphinx/sections/api/apidocs/libraries/dagster-looker.rst @@ -27,6 +27,10 @@ Assets (Looker API) .. autoclass:: RequestStartPdtBuild +.. autofunction:: load_looker_asset_specs + +.. autofunction:: build_looker_pdt_assets_definitions + ************* lkml (LookML) ************* diff --git a/examples/docs_snippets/docs_snippets/integrations/looker/customize-looker-assets.py b/examples/docs_snippets/docs_snippets/integrations/looker/customize-looker-assets.py index 17c6da0a3a28c..4117976f056bb 100644 --- a/examples/docs_snippets/docs_snippets/integrations/looker/customize-looker-assets.py +++ b/examples/docs_snippets/docs_snippets/integrations/looker/customize-looker-assets.py @@ -3,11 +3,12 @@ LookerResource, LookerStructureData, LookerStructureType, + load_looker_asset_specs, ) import dagster as dg -resource = LookerResource( +looker_resource = LookerResource( client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"), client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"), base_url=dg.EnvVar("LOOKERSDK_HOST_URL"), @@ -28,4 +29,7 @@ def get_asset_spec(self, looker_structure: LookerStructureData) -> dg.AssetSpec: return asset_spec -defs = resource.build_defs(dagster_looker_translator=CustomDagsterLookerApiTranslator()) +looker_specs = load_looker_asset_specs( + looker_resource, dagster_looker_translator=CustomDagsterLookerApiTranslator +) +defs = dg.Definitions(assets=[*looker_specs], resources={"looker": looker_resource}) diff --git a/examples/docs_snippets/docs_snippets/integrations/looker/materializing-looker-pdts.py b/examples/docs_snippets/docs_snippets/integrations/looker/materializing-looker-pdts.py index 756a2ae7e9cce..ac0203f135720 100644 --- a/examples/docs_snippets/docs_snippets/integrations/looker/materializing-looker-pdts.py +++ b/examples/docs_snippets/docs_snippets/integrations/looker/materializing-looker-pdts.py @@ -1,18 +1,29 @@ -from dagster_looker import LookerResource, RequestStartPdtBuild +from dagster_looker import ( + LookerResource, + RequestStartPdtBuild, + build_looker_pdt_assets_definitions, + load_looker_asset_specs, +) import dagster as dg -resource = LookerResource( +looker_resource = LookerResource( client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"), client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"), base_url=dg.EnvVar("LOOKERSDK_HOST_URL"), ) -defs = resource.build_defs( +looker_specs = load_looker_asset_specs(looker_resource=looker_resource) + +pdts = build_looker_pdt_assets_definitions( + resource_key="looker", request_start_pdt_builds=[ - RequestStartPdtBuild( - model_name="analytics", - view_name="page_keyword_performance", - ), - ] + RequestStartPdtBuild(model_name="my_model", view_name="my_view") + ], +) + + +defs = dg.Definitions( + assets=[*pdts, *looker_specs], + resources={"looker": looker_resource}, ) diff --git a/examples/docs_snippets/docs_snippets/integrations/looker/representing-looker-assets.py b/examples/docs_snippets/docs_snippets/integrations/looker/representing-looker-assets.py index 073c1f5afa633..25baa798148aa 100644 --- a/examples/docs_snippets/docs_snippets/integrations/looker/representing-looker-assets.py +++ b/examples/docs_snippets/docs_snippets/integrations/looker/representing-looker-assets.py @@ -1,11 +1,12 @@ -from dagster_looker import LookerResource +from dagster_looker import LookerResource, load_looker_asset_specs import dagster as dg -resource = LookerResource( +looker_resource = LookerResource( client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"), client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"), base_url=dg.EnvVar("LOOKERSDK_HOST_URL"), ) -defs = resource.build_defs() +looker_specs = load_looker_asset_specs(looker_resource=looker_resource) +defs = dg.Definitions(assets=[*looker_specs], resources={"looker": looker_resource})