Automation Part 2

docs/docs-beta/.vscode/extensions.json

@@ -0,0 +1,9 @@
+{
+    "recommendations": [
+        "dbaeumer.vscode-eslint",
+        "unifiedjs.vscode-mdx",
+        "esbenp.prettier-vscode",
+        "mrmlnc.vscode-remark",
+        "chrischinchilla.vale-vscode"
+    ]
+}

docs/docs-beta/content-templates/guide-no-steps.md

@@ -1,40 +1,38 @@
 ---
-title: ''
-description: ''
+title: 'Title that briefly describes what the guide is for'
+description: 'Description of the guide, useful for SEO and social media links'
 ---
 
-# [Title that briefly describes what the guide is for]
+Provide a brief introduction to the how-to guide. View [this article](https://diataxis.fr/how-to-guides/) for more information on how to write effective how-to guides. The intro should be no more than a few sentences.
+The title from the frontmatter will be used as the first heading in the guide, you don't need to include it in the intro.
 
-<!-- This section is a brief introduction into the guide's topic. -->
+## What you'll learn
 
-By the end of this guide, you will [a sentence or list about what will be achieved by the end of the guide].
+- A thing you'll learn, ex: "How to generate a token"
+- Another thing you'll learn, ex: "How to install this library"
+- One more thing you'll learn
 
----
-
-## Prerequisites
-
-<!-- This section lists the prerequisites users must complete before they should/can proceed with the guide. -->
+<details>
+  <summary>Prerequisites</summary>
 
 To follow the steps in this guide, you'll need:
 
-- A prerequisite, ex: "Familiarity with Asset definitions"
-- Another prerequisite, ex: "To install <this> library"
+- A prerequisite, ex: "Familiarity with [Asset definitions](/concepts/assets)"
+- Another prerequisite, ex: "To install this library"
 - One more
 
----
+</details>
 
 ## Title that describes this section
 
-<!--
-For section heaings:
+For section headings:
 
 - Guides can (and should) contain multiple sections, with each one being a small chunk of information. Break large topics into smaller topics, using subsequent headings (H3, H4, etc) as needed
 - Titles should describe an action, ex: "Generate a token"
 - Don't use gerunds (-ing) in titles, as it can cause issues with translation + SEO
--->
-
----
 
-## Related
+## Next steps
 
-[List of links to related content]
+- Add links to related content
+- Go deeper into [Understanding Automation](/concepts/understanding-automation)
+- Explore [Related Example](/)

docs/docs-beta/content-templates/guide-with-steps.md

@@ -1,49 +1,45 @@
 ---
-title: ''
-description: ''
+title: 'Title that briefly describes what the guide is for'
+description: 'Description of the guide that is useful for SEO and social media links'
 ---
 
-# [Title that briefly describes what the guide is for]
+Provide a brief introduction to the how-to guide. View [this article](https://diataxis.fr/how-to-guides/) for more information on how to write effective how-to guides. The intro should be no more than a few sentences.
+The title from the frontmatter will be used as the first heading in the guide, you don't need to include it in the intro.
 
-<!-- This section is a brief introduction into the guide's topic. -->
+## What you'll learn
 
-By the end of this guide, you will [a sentence or list about what will be achieved by the end of the guide].
+- A thing you'll learn, ex: "How to generate a token"
+- Another thing you'll learn, ex: "How to install this library"
+- One more thing you'll learn
 
----
-
-## Prerequisites
-
-<!-- This section lists the prerequisites users must complete before they should/can proceed with the guide. -->
+<details>
+  <summary>Prerequisites</summary>
 
 To follow the steps in this guide, you'll need:
 
-- A prerequisite, ex: "Familiarity with Asset definitions"
-- Another prerequisite, ex: "To install <this> library"
+- A prerequisite, ex: "Familiarity with [Asset definitions](/concepts/assets)"
+- Another prerequisite, ex: "To install this library"
 - One more
 
----
+</details>
 
 ## Step 1: Title that describes what this step will do {#step-1}
 
-<!--
-For section / step heaings:
+For section / step headings:
 
 - Titles should describe an action, ex: "Generate a token"
 - Don't use gerunds (-ing) in titles, as it can cause issues with translation + SEO
-- Each section heading should have an ID that includes the word 'step' and the number of the step, ex: {#step-1}
--->
+- Each section heading should have an identifier that includes the word 'step' and the number of the step, ex: {#step-1}
 
 ### Step 1.1: Title that describes a substep {#step-1-1}
 
-<!-- - If a step would benefit by being broken into smaller steps, follow this section's formatting
-- Each substep should get an H3 and start with Step N., followed by the number of the substep -->
-
----
+If a step would benefit by being broken into smaller steps, follow this section's formatting
+Each substep should get an H3 and start with Step N., followed by the number of the substep
 
 ## Step 2: Another step {#step-2}
 
----
-
-## Related
+## Next steps
 
-[List of links to related content]
+- Add links to related content
+- Go deeper into [Understanding Automation](/concepts/understanding-automation)
+- Explore [Related Example](/)

docs/docs-beta/docs/code_examples/guides/automation/asset-sensor-custom-eval.py

@@ -0,0 +1,51 @@
+from dagster import (
+    AssetExecutionContext,
+    AssetKey,
+    AssetMaterialization,
+    Definitions,
+    MaterializeResult,
+    RunRequest,
+    SensorEvaluationContext,
+    SkipReason,
+    asset,
+    asset_sensor,
+    define_asset_job,
+)
+
+
+@asset
+def daily_sales_data(context: AssetExecutionContext):
+    context.log.info("Asset to watch, perhaps some function sets metadata here")
+    yield MaterializeResult(metadata={"specific_property": "value"})
+
+
+@asset
+def weekly_report(context: AssetExecutionContext):
+    context.log.info("Running weekly report")
+
+
+my_job = define_asset_job("my_job", [weekly_report])
+
+
+@asset_sensor(asset_key=AssetKey("daily_sales_data"), job=my_job)
+def daily_sales_data_sensor(context: SensorEvaluationContext, asset_event):
+    # Provide a type hint on the underlying event
+    materialization: AssetMaterialization = (
+        asset_event.dagster_event.event_specific_data.materialization
+    )
+
+    # Example custom logic: Check if the asset metadata has a specific property
+    # highlight-start
+    if "specific_property" in materialization.metadata:
+        context.log.info("Triggering job based on custom evaluation logic")
+        yield RunRequest(run_key=context.cursor)
+    else:
+        yield SkipReason("Asset materialization does not have the required property")
+    # highlight-end
+
+
+defs = Definitions(
+    assets=[daily_sales_data, weekly_report],
+    jobs=[my_job],
+    sensors=[daily_sales_data_sensor],
+)

docs/docs-beta/docs/code_examples/guides/automation/asset-sensor-with-config.py

@@ -1,8 +1,11 @@
 from dagster import (
     AssetExecutionContext,
     AssetKey,
+    AssetMaterialization,
+    Config,
     Definitions,
-    EventLogEntry,
+    MaterializeResult,
+    RunConfig,
     RunRequest,
     SensorEvaluationContext,
     asset,
@@ -11,38 +14,49 @@
 )
 
 
+class MyConfig(Config):
+    param1: str
+
+
 @asset
 def daily_sales_data(context: AssetExecutionContext):
     context.log.info("Asset to watch")
+    # highlight-next-line
+    yield MaterializeResult(metadata={"specific_property": "value"})
 
 
 @asset
-def weekly_report(context: AssetExecutionContext):
-    context.log.info("Asset to trigger")
+def weekly_report(context: AssetExecutionContext, config: MyConfig):
+    context.log.info(f"Running weekly report with param1: {config.param1}")
 
 
-my_job = define_asset_job("my_job", [weekly_report])
+my_job = define_asset_job(
+    "my_job",
+    [weekly_report],
+    config=RunConfig(ops={"weekly_report": MyConfig(param1="value")}),
+)
 
 
-# highlight-start
 @asset_sensor(asset_key=AssetKey("daily_sales_data"), job=my_job)
-def daily_sales_data_sensor(context: SensorEvaluationContext, asset_event: EventLogEntry):
-    # This satisifies the type checker. Asset events are guaranteed to have a dagster_event and asset_key.
-    assert asset_event.dagster_event is not None
-    assert asset_event.dagster_event.asset_key is not None
-
-    return RunRequest(
-        run_key=context.cursor,
-        run_config={
-            "ops": {
-                "read_materialization": {
-                    "config": {
-                        "asset_key": asset_event.dagster_event.asset_key.path,
-                    }
+def daily_sales_data_sensor(context: SensorEvaluationContext, asset_event):
+    materialization: AssetMaterialization = (
+        asset_event.dagster_event.event_specific_data.materialization
+    )
+
+    # Example custom logic: Check if the asset metadata has a specific property
+    # highlight-start
+    if "specific_property" in materialization.metadata:
+        yield RunRequest(
+            run_key=context.cursor,
+            run_config=RunConfig(
+                ops={
+                    "weekly_report": MyConfig(
+                        param1=str(materialization.metadata.get("specific_property"))
+                    )
                 }
-            }
-        },
-    )  # highlight-end
+            ),
+        )
+    # highlight-end
 
 
 defs = Definitions(

docs/docs-beta/docs/code_examples/guides/automation/multi-asset-sensor.py

@@ -0,0 +1,32 @@
+from dagster import (
+    AssetKey,
+    MultiAssetSensorEvaluationContext,
+    RunRequest,
+    asset,
+    define_asset_job,
+    multi_asset_sensor,
+)
+
+
+@asset
+def target_asset():
+    pass
+
+
+downstream_job = define_asset_job("downstream_job", [target_asset])
+
+
+@multi_asset_sensor(
+    monitored_assets=[
+        AssetKey("upstream_asset_1"),
+        AssetKey("upstream_asset_2"),
+    ],
+    job=downstream_job,
+)
+def my_multi_asset_sensor(context: MultiAssetSensorEvaluationContext):
+    run_requests = []
+    for asset_key, materialization in context.latest_materialization_records_by_key().items():
+        if materialization:
+            run_requests.append(RunRequest(asset_selection=[asset_key]))
+            context.advance_cursor({asset_key: materialization})
+    return run_requests

docs/docs-beta/docs/code_examples/guides/automation/schedule-with-partition.py

@@ -0,0 +1,22 @@
+from dagster import (
+    DailyPartitionsDefinition,
+    asset,
+    build_schedule_from_partitioned_job,
+    define_asset_job,
+)
+
+daily_partition = DailyPartitionsDefinition(start_date="2024-05-20")
+
+
+@asset(partitions_def=daily_partition)
+def daily_asset(): ...
+
+
+partitioned_asset_job = define_asset_job("partitioned_job", selection=[daily_asset])
+
+# highlight-start
+# This partition will run daily
+asset_partitioned_schedule = build_schedule_from_partitioned_job(
+    partitioned_asset_job,
+)
+# highlight-end