Metadata docs

docs/docs-beta/.gitignore

@@ -21,7 +21,7 @@ yarn-error.log*
 *.sqlite
 
 .pnp.*
-.yarn/*
+.yarn
 !.yarn/patches
 !.yarn/plugins
 !.yarn/releases

docs/docs-beta/README.md

@@ -25,6 +25,12 @@ Install Vale with:
 brew install vale
 ```
 
+or
+
+```bash
+pip install vale
+```
+
 ---
 
 ## Overview of the docs

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/custom-local-references.py

@@ -0,0 +1,29 @@
+from dagster import (
+    CodeReferencesMetadataValue,
+    Definitions,
+    LocalFileCodeReference,
+    asset,
+    with_source_code_references,
+)
+
+
+@asset(
+    metadata={
+        "dagster/code_references": CodeReferencesMetadataValue(
+            code_references=[
+                LocalFileCodeReference(
+                    file_path="/path/to/source.yaml",
+                    # Label and line number are optional
+                    line_number=1,
+                    label="Model YAML",
+                )
+            ]
+        )
+    }
+)
+def my_asset_modeled_in_yaml(): ...
+
+
+defs = Definitions(
+    assets=with_source_code_references([my_asset_modeled_in_yaml])
+)

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/definition-metadata.py

@@ -0,0 +1,18 @@
+from dagster import asset, AssetSpec, MetadataValue
+
+# You can attach metadata via the `metadata` argument on the `@asset` decorator.
+@asset(metadata={
+    "link_to_docs": MetadataValue.url("https://..."),
+    "snippet": MetadataValue.md("# Embedded markdown\n...")
+})
+def my_asset():
+    ...
+
+# You can also use `metadata` with `AssetSpec`
+my_external_asset = AssetSpec(
+    "my_external_asset",
+    metadata={
+        "link_to_docs": MetadataValue.url("https://..."),
+        "snippet": MetadataValue.md("# Embedded markdown\n...")
+    }
+)

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/kinds.py

@@ -0,0 +1,12 @@
+from dagster import asset, AssetSpec
+
+# You can attach kinds via the `kinds` argument on the `@asset` decorator.
+@asset(kinds=["snowflake", "dbt"])
+def my_asset():
+    ...
+
+# You can also use `kinds` with `AssetSpec`
+my_external_asset = AssetSpec(
+    "my_external_asset",
+    kinds=["snowflake", "dbt"]
+)

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/oss-references.py

@@ -0,0 +1,36 @@
+import os
+from pathlib import Path
+
+from dagster import (
+    AnchorBasedFilePathMapping,
+    Definitions,
+    asset,
+    link_code_references_to_git,
+    with_source_code_references,
+)
+
+
+@asset
+def my_asset(): ...
+
+
+@asset
+def another_asset(): ...
+
+
+assets = with_source_code_references([my_asset, another_asset])
+
+defs = Definitions(
+    assets=link_code_references_to_git(
+        assets_defs=assets,
+        git_url="https://github.com/dagster-io/dagster",
+        git_branch="main",
+        file_path_mapping=AnchorBasedFilePathMapping(
+            local_file_anchor=Path(__file__),
+            file_anchor_path_in_repository="src/repo.py",
+        ),
+    )
+    # only link to github in production. link locally otherwise
+    if bool(os.getenv("IS_PRODUCTION"))
+    else assets
+)

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/owners.py

@@ -0,0 +1,12 @@
+from dagster import asset, AssetSpec
+
+# You can attach owners via the `owners` argument on the `@asset` decorator.
+@asset(owners=["[email protected]", "team:data-eng"])
+def my_asset():
+    ...
+
+# You can also use `owners` with `AssetSpec`
+my_external_asset = AssetSpec(
+    "my_external_asset",
+    owners=["[email protected]", "team:roof", "team:corpdev"]
+)

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/plus-references.py

@@ -0,0 +1,24 @@
+from dagster_cloud.metadata.source_code import link_code_references_to_git_if_cloud
+
+from dagster import (
+    Definitions,
+    asset,
+    with_source_code_references,
+)
+
+
+@asset
+def my_asset(): ...
+
+
+@asset
+def another_asset(): ...
+
+
+defs = Definitions(
+    assets=link_code_references_to_git_if_cloud(
+        assets_defs=with_source_code_references([my_asset, another_asset]),
+        # This function also supports customizable path mapping, but usually
+        # the defaults are fine.
+    )
+)

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/python-local-references.py

@@ -0,0 +1,15 @@
+from dagster import Definitions, asset, with_source_code_references
+
+
+@asset
+def my_asset(): ...
+
+
+@asset
+def another_asset(): ...
+
+
+defs = Definitions(
+    # with_source_code_references() automatically attaches the proper metadata
+    assets=with_source_code_references([my_asset, another_asset])
+)

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/runtime-metadata.py

@@ -0,0 +1,11 @@
+from dagster import asset, MetadataValue, MaterializeResult
+
+@asset
+def my_asset():
+    # Asset logic goes here
+    return MaterializeResult(
+        metadata={
+            # file_size_kb will be rendered as a time series
+            "file_size_kb": MetadataValue.int(...)
+        }
+    )

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/table-column-lineage-metadata.py

@@ -0,0 +1,43 @@
+from dagster import (
+    AssetKey,
+    MaterializeResult,
+    TableColumnDep,
+    TableColumnLineage,
+    asset,
+)
+
+
+@asset(deps=["source_bar", "source_baz"])
+def my_asset():
+    # The following TableColumnLineage models 3 tables:
+    # source_bar, source_baz, and my_asset
+    # With the following column dependencies:
+    # - my_asset.new_column_foo depends on:
+    #   - source_bar.column_bar
+    #   - source_baz.column_baz
+    # - my_asset.new_column_qux depends on:
+    #   - source_bar.column_quuz
+    return MaterializeResult(
+        metadata={
+            "dagster/column_lineage": TableColumnLineage(
+                deps_by_column={
+                    "new_column_foo": [
+                        TableColumnDep(
+                            asset_key=AssetKey("source_bar"),
+                            column_name="column_bar",
+                        ),
+                        TableColumnDep(
+                            asset_key=AssetKey("source_baz"),
+                            column_name="column_baz",
+                        ),
+                    ],
+                    "new_column_qux": [
+                        TableColumnDep(
+                            asset_key=AssetKey("source_bar"),
+                            column_name="column_quuz",
+                        ),
+                    ],
+                }
+            )
+        }
+    )

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/table-schema-metadata.py

@@ -0,0 +1,49 @@
+from dagster import (
+    MaterializeResult,
+    TableColumn,
+    TableSchema,
+    asset,
+)
+
+
+# Definition-time metadata
+# Here, we know the schema of the asset, so we can attach it to the asset decorator
+@asset(
+    deps=["source_bar", "source_baz"],
+    metadata={
+        "dagster/column_schema": TableSchema(
+            columns=[
+                TableColumn(
+                    "name",
+                    "string",
+                    description="The name of the person",
+                ),
+                TableColumn(
+                    "age",
+                    "int",
+                    description="The age of the person",
+                ),
+            ]
+        )
+    },
+)
+def my_asset(): ...
+
+
+# Runtime metadata
+# Here, the schema isn't known until the asset is materialized
+@asset(deps=["source_bar", "source_baz"])
+def my_other_asset():
+    column_names = ...
+    column_types = ...
+
+    columns = [
+        TableColumn(name, column_type)
+        for name, column_type in zip(column_names, column_types)
+    ]
+
+    return MaterializeResult(
+        metadata={
+            "dagster/column_schema": TableSchema(columns=columns)
+        }
+    )

docs/docs-beta/docs/code_examples/guides/data-modeling/metadata/tags.py

@@ -0,0 +1,12 @@
+from dagster import asset, AssetSpec
+
+# You can attach tags via the `tags` argument on the `@asset` decorator.
+@asset(tags={"domain": "marketing", "pii": "true"})
+def my_asset():
+    ...
+
+# You can also use `tags` with `AssetSpec`
+my_external_asset = AssetSpec(
+    "my_external_asset",
+    tags={"domain": "legal", "sensitive": ""}
+)