Skip to content

Commit

Permalink
[components] Add env to the default ComponentLoadContext renderer (#2…
Browse files Browse the repository at this point in the history
…6496)

## Summary & Motivation

Copies over work from: https://app.graphite.dev/github/pr/dagster-io/dagster/26340/componenet-Enable-dsl-access-to-os-environ-as-var


## How I Tested These Changes

## Changelog

NOCHANGELOG
  • Loading branch information
OwenKephart authored Dec 18, 2024
1 parent 76c878b commit e3e0b8c
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ def for_test(
resources=resources or {},
registry=registry or ComponentRegistry.empty(),
decl_node=decl_node,
templated_value_resolver=TemplatedValueResolver(context={}),
templated_value_resolver=TemplatedValueResolver.default(),
)

@property
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
Component,
ComponentLoadContext,
ComponentRegistry,
TemplatedValueResolver,
get_component_name,
is_registered_component,
)
Expand Down Expand Up @@ -97,7 +98,12 @@ def build_defs_from_component_path(
if not decl_node:
raise Exception(f"No component found at path {path}")

context = ComponentLoadContext(resources=resources, registry=registry, decl_node=decl_node)
context = ComponentLoadContext(
resources=resources,
registry=registry,
decl_node=decl_node,
templated_value_resolver=TemplatedValueResolver.default(),
)
components = load_components_from_context(context)
return defs_from_components(resources=resources, context=context, components=components)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import os
from typing import AbstractSet, Any, Mapping, Optional, Sequence, Type, TypeVar, Union

import dagster._check as check
Expand All @@ -15,7 +16,7 @@
CONTEXT_KEY = "required_rendering_context"


def add_required_rendering_context(field: FieldInfo, context: AbstractSet[str]) -> Any:
def add_required_rendering_context(field: Any, context: AbstractSet[str]) -> Any:
return FieldInfo.merge_field_infos(
field, FieldInfo(json_schema_extra={CONTEXT_KEY: json.dumps(list(context))})
)
Expand All @@ -26,10 +27,18 @@ def get_required_rendering_context(subschema: Mapping[str, Any]) -> Optional[Abs
return set(json.loads(raw)) if raw else None


def _env(key: str) -> Optional[str]:
return os.environ.get(key)


@record
class TemplatedValueResolver:
context: Mapping[str, Any]

@staticmethod
def default() -> "TemplatedValueResolver":
return TemplatedValueResolver(context={"env": _env})

def with_context(self, **additional_context) -> "TemplatedValueResolver":
return TemplatedValueResolver(context={**self.context, **additional_context})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import pytest
from dagster import AssetKey
from dagster._utils.env import environ
from dagster_components.core.component_decl_builder import ComponentFileModel
from dagster_components.core.component_defs_builder import (
YamlComponentDecl,
Expand Down Expand Up @@ -116,3 +117,42 @@ def test_load_from_path(dbt_path: Path) -> None:
)

assert defs.get_asset_graph().get_all_asset_keys() == JAFFLE_SHOP_KEYS_WITH_PREFIX


def test_render_vars_root(dbt_path: Path) -> None:
with environ({"GROUP_AS_ENV": "group_in_env"}):
decl_node = YamlComponentDecl(
path=dbt_path / COMPONENT_RELPATH,
component_file_model=ComponentFileModel(
type="dbt_project",
params={
"dbt": {"project_dir": "jaffle_shop"},
"translator": {
"group": "{{ env('GROUP_AS_ENV') }}",
},
},
),
)
comp = DbtProjectComponent.load(context=script_load_context(decl_node))
assert get_asset_keys(comp) == JAFFLE_SHOP_KEYS
defs: Definitions = comp.build_defs(script_load_context())
for key in get_asset_keys(comp):
assert defs.get_assets_def(key).get_asset_spec(key).group_name == "group_in_env"


def test_render_vars_asset_key(dbt_path: Path) -> None:
with environ({"ASSET_KEY_PREFIX": "some_prefix"}):
decl_node = YamlComponentDecl(
path=dbt_path / COMPONENT_RELPATH,
component_file_model=ComponentFileModel(
type="dbt_project",
params={
"dbt": {"project_dir": "jaffle_shop"},
"translator": {
"key": "{{ env('ASSET_KEY_PREFIX') }}/{{ node.name }}",
},
},
),
)
comp = DbtProjectComponent.load(context=script_load_context(decl_node))
assert get_asset_keys(comp) == JAFFLE_SHOP_KEYS_WITH_PREFIX
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def registry() -> ComponentRegistry:


def script_load_context(decl_node: Optional[ComponentDeclNode] = None) -> ComponentLoadContext:
return ComponentLoadContext(registry=registry(), resources={}, decl_node=decl_node)
return ComponentLoadContext.for_test(registry=registry(), decl_node=decl_node)


def get_asset_keys(component: Component) -> AbstractSet[AssetKey]:
Expand Down

0 comments on commit e3e0b8c

Please sign in to comment.