Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[docathon] pass tox for asset factories #23909

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,19 +1,24 @@
import dagster as dg
import dagster_aws.s3 as s3
import yaml
import pydantic
import jinja2
import os
from pathlib import Path
from typing import List

import dagster_aws.s3 as s3
import jinja2
import pydantic
import yaml

import dagster as dg


def build_etl_job(
s3_resource: s3.S3Resource,
bucket: str,
source_object: str,
target_object: str,
sql: str,
) -> dg.Definitions: ...
) -> dg.Definitions:
# Code from previous example omitted
return dg.Definitions()


# highlight-start
Expand Down Expand Up @@ -64,4 +69,6 @@ def load_etl_jobs_from_yaml(yaml_path: str) -> dg.Definitions:
# highlight-end


defs = load_etl_jobs_from_yaml("etl_jobs_with_jinja.yaml")
defs = load_etl_jobs_from_yaml(
"docs_beta_snippets/guides/data-modeling/asset-factories/etl_jobs_with_jinja.yaml"
)
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import dagster as dg
import tempfile

import dagster_aws.s3 as s3
import duckdb
import tempfile

import dagster as dg


def build_etl_job(
Expand All @@ -11,7 +13,10 @@ def build_etl_job(
target_object: str,
sql: str,
) -> dg.Definitions:
@dg.asset(name=f"etl_{bucket}_{target_object}")
# asset keys cannot contain '.'
name = f"etl_{bucket}_{target_object}".replace(".", "_")
cmpadden marked this conversation as resolved.
Show resolved Hide resolved

@dg.asset(name=name)
def etl_asset(context):
with tempfile.TemporaryDirectory() as root:
source_path = f"{root}/{source_object}"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
import dagster as dg
import dagster_aws.s3 as s3
import yaml

import dagster as dg


def build_etl_job(
s3_resource: s3.S3Resource,
bucket: str,
source_object: str,
target_object: str,
sql: str,
) -> dg.Definitions: ...
) -> dg.Definitions:
# Code from previous example omitted
return dg.Definitions()


# highlight-start
Expand All @@ -33,5 +36,7 @@ def load_etl_jobs_from_yaml(yaml_path: str) -> dg.Definitions:
return dg.Definitions.merge(*defs)


defs = load_etl_jobs_from_yaml("etl_jobs.yaml")
defs = load_etl_jobs_from_yaml(
"docs_beta_snippets/guides/data-modeling/asset-factories/etl_jobs.yaml"
)
# highlight-end
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,15 @@
snippets_folder = file_relative_path(__file__, "../docs_beta_snippets/")


EXCLUDED_FILES = [
"plus-references.py" # No module named 'dagster_cloud'
]


def get_python_files(directory):
for root, _, files in os.walk(directory):
for file in files:
if file.endswith(".py"):
if file.endswith(".py") and file not in EXCLUDED_FILES:
yield os.path.join(root, file)


Expand Down
2 changes: 1 addition & 1 deletion examples/docs_beta_snippets/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"Operating System :: OS Independent",
],
packages=find_packages(exclude=["test"]),
install_requires=["dagster-cloud"],
install_requires=[],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

wonder if tox.ini can point at another git repo so we can target latest master

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do think it's possible, but complicated slightly because it's in a private repo.

extras_require={
"test": [
"pytest",
Expand Down
11 changes: 6 additions & 5 deletions examples/docs_beta_snippets/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,22 @@ passenv =
BUILDKITE*
install_command = uv pip install {opts} {packages}
deps =
source: -e ../../python_modules/dagster[test]
source: -e ../../python_modules/dagster-pipes
pypi: dagster[test]
jinja2
pyyaml
duckdb
pandas
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-pipes
-e ../../python_modules/dagster-graphql
-e ../../python_modules/dagster-webserver
-e ../../python_modules/libraries/dagster-duckdb-pandas
-e ../../python_modules/libraries/dagster-embedded-elt
-e ../../python_modules/libraries/dagster-aws
-e ../../python_modules/libraries/dagster-duckdb-pandas
-e ../../python_modules/libraries/dagster-duckdb
-e .[test]
allowlist_externals =
/bin/bash
uv
commands =
source: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
pytest -c ../../pyproject.toml -vv {posargs}