From e7235cce16cf49731f92a5ad571914b464d96118 Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Sat, 17 Aug 2024 17:02:08 -0700 Subject: [PATCH 01/13] Add automation content part 2 --- .../automation/asset-sensor-custom-eval.py | 51 ++++++++++++ .../automation/asset-sensor-with-config.py | 56 ++++++++----- .../guides/automation/multi-asset-sensor.py | 32 ++++++++ .../automation/schedule-with-partition.py | 22 ++++++ docs/docs-next/docs/concepts/automation.md | 69 +++++++++++++++- .../docs/concepts/automation/schedules.md | 6 -- docs/docs-next/docs/guides/automation.md | 40 +++++----- .../docs/guides/automation/asset-sensors.md | 78 ++++++++++++++++++- ...ynamic-pipelines-based-on-external-data.md | 6 -- .../docs/guides/automation/schedules.md | 52 ++++--------- .../docs/guides/automation/sensors.md | 7 +- .../triggering-pipeline-runs-using-events.md | 4 - docs/vale/styles/Dagster/dagster_terms.yml | 3 +- .../config/vocabularies/Dagster/accept.txt | 2 +- 14 files changed, 325 insertions(+), 103 deletions(-) create mode 100644 docs/docs-next/docs/code_examples/guides/automation/asset-sensor-custom-eval.py create mode 100644 docs/docs-next/docs/code_examples/guides/automation/multi-asset-sensor.py create mode 100644 docs/docs-next/docs/code_examples/guides/automation/schedule-with-partition.py delete mode 100644 docs/docs-next/docs/guides/automation/creating-dynamic-pipelines-based-on-external-data.md delete mode 100644 docs/docs-next/docs/guides/automation/triggering-pipeline-runs-using-events.md diff --git a/docs/docs-next/docs/code_examples/guides/automation/asset-sensor-custom-eval.py b/docs/docs-next/docs/code_examples/guides/automation/asset-sensor-custom-eval.py new file mode 100644 index 0000000000000..8a0f006206861 --- /dev/null +++ b/docs/docs-next/docs/code_examples/guides/automation/asset-sensor-custom-eval.py @@ -0,0 +1,51 @@ +from dagster import ( + AssetExecutionContext, + AssetKey, + AssetMaterialization, + Definitions, + MaterializeResult, + RunRequest, + SensorEvaluationContext, + SkipReason, + asset, + asset_sensor, + define_asset_job, +) + + +@asset +def daily_sales_data(context: AssetExecutionContext): + context.log.info("Asset to watch, perhaps some function sets metadata here") + yield MaterializeResult(metadata={"specific_property": "value"}) + + +@asset +def weekly_report(context: AssetExecutionContext): + context.log.info("Running weekly report") + + +my_job = define_asset_job("my_job", [weekly_report]) + + +@asset_sensor(asset_key=AssetKey("daily_sales_data"), job=my_job) +def daily_sales_data_sensor(context: SensorEvaluationContext, asset_event): + # Provide a type hint on the underlying event + materialization: AssetMaterialization = ( + asset_event.dagster_event.event_specific_data.materialization + ) + + # Example custom logic: Check if the asset metadata has a specific property + # highlight-start + if "specific_property" in materialization.metadata: + context.log.info("Triggering job based on custom evaluation logic") + yield RunRequest(run_key=context.cursor) + else: + yield SkipReason("Asset materialization does not have the required property") + # highlight-end + + +defs = Definitions( + assets=[daily_sales_data, weekly_report], + jobs=[my_job], + sensors=[daily_sales_data_sensor], +) diff --git a/docs/docs-next/docs/code_examples/guides/automation/asset-sensor-with-config.py b/docs/docs-next/docs/code_examples/guides/automation/asset-sensor-with-config.py index b1c01c332b728..3c729f898acc5 100644 --- a/docs/docs-next/docs/code_examples/guides/automation/asset-sensor-with-config.py +++ b/docs/docs-next/docs/code_examples/guides/automation/asset-sensor-with-config.py @@ -1,8 +1,11 @@ from dagster import ( AssetExecutionContext, AssetKey, + AssetMaterialization, + Config, Definitions, - EventLogEntry, + MaterializeResult, + RunConfig, RunRequest, SensorEvaluationContext, asset, @@ -11,38 +14,49 @@ ) +class MyConfig(Config): + param1: str + + @asset def daily_sales_data(context: AssetExecutionContext): context.log.info("Asset to watch") + # highlight-next-line + yield MaterializeResult(metadata={"specific_property": "value"}) @asset -def weekly_report(context: AssetExecutionContext): - context.log.info("Asset to trigger") +def weekly_report(context: AssetExecutionContext, config: MyConfig): + context.log.info(f"Running weekly report with param1: {config.param1}") -my_job = define_asset_job("my_job", [weekly_report]) +my_job = define_asset_job( + "my_job", + [weekly_report], + config=RunConfig(ops={"weekly_report": MyConfig(param1="value")}), +) -# highlight-start @asset_sensor(asset_key=AssetKey("daily_sales_data"), job=my_job) -def daily_sales_data_sensor(context: SensorEvaluationContext, asset_event: EventLogEntry): - # This satisifies the type checker. Asset events are guaranteed to have a dagster_event and asset_key. - assert asset_event.dagster_event is not None - assert asset_event.dagster_event.asset_key is not None - - return RunRequest( - run_key=context.cursor, - run_config={ - "ops": { - "read_materialization": { - "config": { - "asset_key": asset_event.dagster_event.asset_key.path, - } +def daily_sales_data_sensor(context: SensorEvaluationContext, asset_event): + materialization: AssetMaterialization = ( + asset_event.dagster_event.event_specific_data.materialization + ) + + # Example custom logic: Check if the asset metadata has a specific property + # highlight-start + if "specific_property" in materialization.metadata: + yield RunRequest( + run_key=context.cursor, + run_config=RunConfig( + ops={ + "weekly_report": MyConfig( + param1=str(materialization.metadata.get("specific_property")) + ) } - } - }, - ) # highlight-end + ), + ) + # highlight-end defs = Definitions( diff --git a/docs/docs-next/docs/code_examples/guides/automation/multi-asset-sensor.py b/docs/docs-next/docs/code_examples/guides/automation/multi-asset-sensor.py new file mode 100644 index 0000000000000..d3c35ba8b761f --- /dev/null +++ b/docs/docs-next/docs/code_examples/guides/automation/multi-asset-sensor.py @@ -0,0 +1,32 @@ +from dagster import ( + AssetKey, + MultiAssetSensorEvaluationContext, + RunRequest, + asset, + define_asset_job, + multi_asset_sensor, +) + + +@asset +def target_asset(): + pass + + +downstream_job = define_asset_job("downstream_job", [target_asset]) + + +@multi_asset_sensor( + monitored_assets=[ + AssetKey("upstream_asset_1"), + AssetKey("upstream_asset_2"), + ], + job=downstream_job, +) +def my_multi_asset_sensor(context: MultiAssetSensorEvaluationContext): + run_requests = [] + for asset_key, materialization in context.latest_materialization_records_by_key().items(): + if materialization: + run_requests.append(RunRequest(asset_selection=[asset_key])) + context.advance_cursor({asset_key: materialization}) + return run_requests diff --git a/docs/docs-next/docs/code_examples/guides/automation/schedule-with-partition.py b/docs/docs-next/docs/code_examples/guides/automation/schedule-with-partition.py new file mode 100644 index 0000000000000..2ec9852fb5151 --- /dev/null +++ b/docs/docs-next/docs/code_examples/guides/automation/schedule-with-partition.py @@ -0,0 +1,22 @@ +from dagster import ( + DailyPartitionsDefinition, + asset, + build_schedule_from_partitioned_job, + define_asset_job, +) + +daily_partition = DailyPartitionsDefinition(start_date="2024-05-20") + + +@asset(partitions_def=daily_partition) +def daily_asset(): ... + + +partitioned_asset_job = define_asset_job("partitioned_job", selection=[daily_asset]) + +# highlight-start +# This partition will run daily +asset_partitioned_schedule = build_schedule_from_partitioned_job( + partitioned_asset_job, +) +# highlight-end diff --git a/docs/docs-next/docs/concepts/automation.md b/docs/docs-next/docs/concepts/automation.md index 420e995b0d090..9d49bc739d563 100644 --- a/docs/docs-next/docs/concepts/automation.md +++ b/docs/docs-next/docs/concepts/automation.md @@ -1 +1,68 @@ -# Automation \ No newline at end of file +--- +title: About Automation +--- + +Dagster has support for several types of automation. All automation in Dagster responds to some external event. + +The first system, and the most basic, is the [Schedule](/guides/automation/schedules), which responds to time. + +[Sensors](/guides/automation/sensors) are like schedules, but they respond to an external event defined by the user. + +[Asset Sensors](/guides/automation/asset-sensors) are a special case of sensor that responds to changes in asset materialization +as reported by the Event Log. + +Finally, the Declarative Automation system is a +more complex system that uses conditions on the assets to determine when to execute. + +## About Schedules + +In Dagster, a schedule is defined by the `ScheduleDefinition` class, or through the `@schedule` decorator. The `@schedule` +decorator is more flexible than the `ScheduleDefinition` class, allowing you to configure job behaviour or emit log messages +as the schedule is processed. + +Schedules were one of the first types of automation in Dagster, created before the introduction of Software-Defined Assets. +As such, you may find that many of the examples can seem foreign if you are used to only working within the asset framework. + +For more on how assets and ops inter-relate, read about [Assets and Ops](/concepts/assets#assets-and-ops) + +The `dagster-daemon` process is responsible for submitting runs by checking each schedule at a regular interval to determine +if it's time to execute the underlying job. + +A schedule can be thought of as a wrapper around two pieces: + +- A `JobDefinition`, which is a set of assets to materialize or ops to execute. +- A `cron` string, which describes the schedule. + +### Defining a schedule using `ScheduleDefinition` + +```python +ecommerce_schedule = ScheduleDefinition( + job=ecommerce_job, + cron_schedule="15 5 * * 1-5", +) +``` + +By default, schedules aren't enabled. You can enable them by visiting the Automation tab and toggling the schedule, +or set a default status to `RUNNING` when you define the schedule. + +```python +ecommerce_schedule = ScheduleDefinition( + job=ecommerce_job, + cron_schedule="15 5 * * 1-5", + default_status=DefaultScheduleStatus.RUNNING, +) +``` + +### Defining a schedule using `@schedule` + +If you want more control over the schedule, you can use the `@schedule` decorator. In doing so, you are then responsible for either +emitting a `RunRequest` or a `SkipReason`. You can also emit logs, which will be visible in the Dagster UI for a given schedule's tick history. + +```python +@schedule(cron_schedule="15 5 * * 1-5") +def ecommerce_schedule(context): + context.log.info("This log message will be visible in the Dagster UI.") + return RunRequest() +``` + +### Understanding Timezones diff --git a/docs/docs-next/docs/concepts/automation/schedules.md b/docs/docs-next/docs/concepts/automation/schedules.md index 280d14cb0761f..e69de29bb2d1d 100644 --- a/docs/docs-next/docs/concepts/automation/schedules.md +++ b/docs/docs-next/docs/concepts/automation/schedules.md @@ -1,6 +0,0 @@ ---- -title: "Schedules" -sidebar_position: 10 ---- - -# Schedules diff --git a/docs/docs-next/docs/guides/automation.md b/docs/docs-next/docs/guides/automation.md index 915eb4e32afe1..5eda1daaf01c5 100644 --- a/docs/docs-next/docs/guides/automation.md +++ b/docs/docs-next/docs/guides/automation.md @@ -35,8 +35,8 @@ Dagster offers several ways to automate pipeline execution: ## Schedules Schedules allow you to run jobs at specified times, like "every Monday at 9 AM" or "daily at midnight." -A schedule combines a selection of assets, known as a [Job](/concepts/ops-jobs), and a [cron expression](https://en.wikipedia.org/wiki/Cron) -in order to define when the job should be run. +A schedule combines a selection of assets, known as a [Job](/concepts/ops-jobs), and a [cron expression](https://en.wikipedia.org/wiki/Cron) +to define when the job should be run. To make creating cron expressions easier, you can use an online tool like [Crontab Guru](https://crontab.guru/). @@ -51,36 +51,38 @@ For more information about how Schedules work, see [About Schedules](/concepts/s ## Sensors -Sensors allow you to trigger runs based on events or conditions, like a new file arriving or an external system status change. +Sensors allow you to trigger runs based on events or conditions that you define, like a new file arriving or an external system status change. -Like schedules, sensors operate on a selection of assets, known as [Jobs](/concepts/ops-jobs) and can either start a pipeline -through a Run or log a reason for not starting a pipeline using a SkipReason. - -However, unlike schedules, sensors are triggered by events that you define. You must provide a function that the sensor will use to determine if it should trigger a run. -### When to use Sensors +Like schedules, sensors operate on a selection of assets, known as [Jobs](/concepts/ops-jobs) and can either start a pipeline through a Run or log a reason for not starting a pipeline using a SkipReason. + + +### When to use sensors - You need event-driven automation - You want to react to changes in external systems For more examples of how to create sensors, see the [How-To Use Sensors](/guides/automation/sensors) guide. -For more information about how Sensors work, see the [About Sensors](/concepts/sensors) concept page. +For more information about how sensors work, see the [About Sensors](/concepts/sensors) concept page. ## Asset sensors Asset Sensors trigger jobs when specified assets are materialized, allowing you to create dependencies between jobs or code locations. -### When to use Asset Sensors +### When to use Asset sensors - You need to trigger jobs based on asset materializations - You want to create dependencies between different jobs or code locations For more examples of how to create asset sensors, see the [How-To Use Asset Sensors](/guides/automation/asset-sensors) guide. +## Declarative automation + +TODO: Add content -## Choosing the Right Automation Method +## How to choose the right automation method Consider these factors when selecting an automation method: @@ -91,17 +93,17 @@ Consider these factors when selecting an automation method: Use this table to help guide your decision: -| Method | Best For | Works With | -|--------|----------|------------| -| Schedules | Regular, time-based job runs | Assets, Ops, Graphs | -| Sensors | Event-driven automation | Assets, Ops, Graphs | -| Declarative Automation | Asset-centric, condition-based updates | Assets only | -| Asset Sensors | Cross-job/location asset dependencies | Assets only | +| Method | Best For | Works With | +| ---------------------- | -------------------------------------- | ------------------- | +| Schedules | Regular, time-based job runs | Assets, Ops, Graphs | +| Sensors | Event-driven automation | Assets, Ops, Graphs | +| Declarative Automation | Asset-centric, condition-based updates | Assets only | +| Asset Sensors | Cross-job/location asset dependencies | Assets only | -## Next Steps +## Next steps - Learn more about [advanced scheduling patterns] - TODO ADD LINK - Explore [complex sensor examples] - TODO ADD LINK - Dive into [Declarative Automation best practices] - TODO ADD LINK -By understanding and effectively using these automation methods, you can build more efficient data pipelines that respond to your specific needs and constraints. \ No newline at end of file +By understanding and effectively using these automation methods, you can build more efficient data pipelines that respond to your specific needs and constraints. diff --git a/docs/docs-next/docs/guides/automation/asset-sensors.md b/docs/docs-next/docs/guides/automation/asset-sensors.md index 1d928c3cc59bc..5890d8f325b7f 100644 --- a/docs/docs-next/docs/guides/automation/asset-sensors.md +++ b/docs/docs-next/docs/guides/automation/asset-sensors.md @@ -1,10 +1,80 @@ --- -title: Asset Sensors -sidebar_position: 50 +title: Scheduling pipelines across projects and jobs with Asset Sensors +sidebar_label: Scheduling pipelines across projects and jobs +sidebar_position: 30 --- -### Basic Asset Sensor Example +Asset sensors in Dagster provide a powerful mechanism for monitoring asset materializations and triggering downstream computations or notifications based on those events. + +There are many use cases for asset sensors, however the most common is to trigger a job when an asset is materialized in a different job or code location, or to provide custom logic to determine when to trigger a run following an asset materialization. + +### Cross-job and cross-code location dependencies + +Asset Sensors enable dependencies across different jobs and even different code locations. This flexibility allows for more modular and decoupled workflows. + + +```mermaid +graph LR; + +AssetToWatch --> AssetSensor; +AssetSensor --> Job; +Job --> Asset1; +Job --> Asset2; + +subgraph CodeLocationA + AssetToWatch +end + +subgraph CodeLocationB + AssetSensor + Job + Asset1 + Asset2 +end + +``` + +Here's a minimal example of an asset sensor that triggers a job when an asset is materialized. The `daily_sales_data` asset is in the same code location for this example, but the same pattern can be applied to assets in different code locations. -This Asset Sensor will trigger a run of `my_job` whenever the `asset_to_watch` asset is materialized. +### Custom evaluation logic + +The evaluation function of an asset sensor can be customized to include custom logic for determining when to trigger a run. This allows for fine-grained control over the conditions under which downstream jobs are executed. + +```mermaid +stateDiagram-v2 + direction LR + + + + [*] --> AssetMaterialized + AssetMaterialized --> [*] + + AssetMaterialized --> UserEvaluationFunction + UserEvaluationFunction --> RunRequest + UserEvaluationFunction --> SkipReason + SkipReason --> [*] + RunRequest --> [*] + + class UserEvaluationFunction userDefined + classDef userDefined fill: lightblue +``` + +In this example, the `@asset_sensor` decorator allows us to define a custom evaluation function that returns a `RunRequest` object when the asset is materialized and certain metadata is present, +otherwise it skips the run. + + + +## Trigger a job with configuration + +By providing a configuration to the `RunRequest` object, you can trigger a job with a specific configuration. This is useful when you want to trigger a job with custom parameters based on custom logic you define. For example, you might use a sensor to trigger a job when an asset is materialized, but also pass metadata about that materialization to the job. + + + + +## Monitor multiple assets + +This example shows how to use a multi-asset sensor to monitor multiple assets and trigger a job when any of the monitored assets are materialized. + + \ No newline at end of file diff --git a/docs/docs-next/docs/guides/automation/creating-dynamic-pipelines-based-on-external-data.md b/docs/docs-next/docs/guides/automation/creating-dynamic-pipelines-based-on-external-data.md deleted file mode 100644 index 98576d9d32794..0000000000000 --- a/docs/docs-next/docs/guides/automation/creating-dynamic-pipelines-based-on-external-data.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Creating dynamic pipelines based on external data" -sidebar_position: 30 ---- - -# Creating dynamic pipelines based on external data diff --git a/docs/docs-next/docs/guides/automation/schedules.md b/docs/docs-next/docs/guides/automation/schedules.md index db5105529916e..4f7cefd61aae0 100644 --- a/docs/docs-next/docs/guides/automation/schedules.md +++ b/docs/docs-next/docs/guides/automation/schedules.md @@ -1,57 +1,35 @@ --- -title: "Scheduling pipelines" +title: "Scheduling cron-based pipelines" sidebar_label: "Running pipelines on a schedule" sidebar_position: 10 --- -## Basic Schedule Example +Schedules enable automated execution of jobs at specified intervals. These intervals can range from common frequencies like hourly, daily, or weekly, to more intricate patterns defined using cron expressions. -A basic schedule is defined by a `JobDefinition` and a `cron_schedule` using the `ScheduleDefinition` class. +## Basic schedule + +A basic schedule is defined by a `JobDefinition` and a `cron_schedule` using the `ScheduleDefinition` class. A job can be thought of as a selection of assets or operations executed together. -## How to Set Custom Timezones +## Run schedules in a different timezone -By default, schedules without a timezone will run in UTC. If you want to run a schedule in a different timezone, you can -set the `timezone` parameter. +By default, schedules without a timezone will run in Coordinated Universal Time (UTC). If you want to run a schedule in a different timezone, you can set the `timezone` parameter. ```python -ecommerce_schedule = ScheduleDefinition( - job=ecommerce_job, - cron_schedule="15 5 * * 1-5", -timezone="America/Los_Angeles", +daily_schedule = ScheduleDefinition( + job=daily_refresh_job, + cron_schedule="0 0 * * *", + timezone="America/Los_Angeles", ) ``` -## How to Create Partitioned Schedules +## Run schedules on a partitioned asset -If you have a partitioned asset and job, you can create a schedule from the partition using `build_schedule_from_partitioned_job`. +If you have a partitioned asset and job, you can create a schedule using the partition with `build_schedule_from_partitioned_job`. The schedule will execute as the same cadence specified by the partition definition. -```python -from dagster import ( - asset, - build_schedule_from_partitioned_job, - define_asset_job, - DailyPartitionsDefinition, -) - -daily_partition = DailyPartitionsDefinition(start_date="2024-05-20") - - -@asset(partitions_def=daily_partition) -def daily_asset(): ... - -partitioned_asset_job = define_asset_job("partitioned_job", selection=[daily_asset]) - -# highlight-start -# This partition will run daily -asset_partitioned_schedule = build_schedule_from_partitioned_job( - partitioned_asset_job, -) -# highlight-end - -``` + If you have a partitioned job, you can create a schedule from the partition using `build_schedule_from_partitioned_job`. @@ -71,4 +49,4 @@ partitioned_op_schedule = build_schedule_from_partitioned_job( --- -For more information about how Schedules work, see the [About Schedules](/concepts/schedules) concept page. +For more information about how Schedules work, see the [About Schedules](/concepts/schedules). \ No newline at end of file diff --git a/docs/docs-next/docs/guides/automation/sensors.md b/docs/docs-next/docs/guides/automation/sensors.md index fcaa64aa5be7c..ba0e5548977a6 100644 --- a/docs/docs-next/docs/guides/automation/sensors.md +++ b/docs/docs-next/docs/guides/automation/sensors.md @@ -1,8 +1,9 @@ --- -title: Sensor Examples ---- +title: Creating event-based pipelines with sensors +sidebar_label: Creating event-based pipelines +sidebar_position: 20 -### Basic Sensor Example +--- This example includes a `check_for_new_files` function that simulates finding new files. In a real scenario, this function would check an actual system or directory. diff --git a/docs/docs-next/docs/guides/automation/triggering-pipeline-runs-using-events.md b/docs/docs-next/docs/guides/automation/triggering-pipeline-runs-using-events.md deleted file mode 100644 index 17826f1c536e4..0000000000000 --- a/docs/docs-next/docs/guides/automation/triggering-pipeline-runs-using-events.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: "Creating event-based pipelines" -sidebar_position: 20 ---- diff --git a/docs/vale/styles/Dagster/dagster_terms.yml b/docs/vale/styles/Dagster/dagster_terms.yml index 4204f3d5fc3f3..9359242e2f7aa 100644 --- a/docs/vale/styles/Dagster/dagster_terms.yml +++ b/docs/vale/styles/Dagster/dagster_terms.yml @@ -15,5 +15,6 @@ swap: public preview: Public Preview run id: Run Id '\bdagster': Dagster - dagster cloud: Dagster Plus + '[Dd]agster [Cc]loud': Dagster Plus '\bworker\b': Worker + '[Aa]sset Sensor': Asset sensor \ No newline at end of file diff --git a/docs/vale/styles/config/vocabularies/Dagster/accept.txt b/docs/vale/styles/config/vocabularies/Dagster/accept.txt index ca940b3243c3c..0128ab9b0b513 100644 --- a/docs/vale/styles/config/vocabularies/Dagster/accept.txt +++ b/docs/vale/styles/config/vocabularies/Dagster/accept.txt @@ -12,4 +12,4 @@ dataframes DataFrame cron materializations -webserver \ No newline at end of file +webserver From be0daa66ad2056a5e8854483b934035aae897f41 Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Sat, 17 Aug 2024 17:04:18 -0700 Subject: [PATCH 02/13] update concept --- docs/docs-next/docs/concepts/automation.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/docs-next/docs/concepts/automation.md b/docs/docs-next/docs/concepts/automation.md index 9d49bc739d563..a568dc5196802 100644 --- a/docs/docs-next/docs/concepts/automation.md +++ b/docs/docs-next/docs/concepts/automation.md @@ -64,5 +64,3 @@ def ecommerce_schedule(context): context.log.info("This log message will be visible in the Dagster UI.") return RunRequest() ``` - -### Understanding Timezones From 7ec8d7bdbb15ada51a0a182aa45e6e6c5a7ae424 Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Sun, 18 Aug 2024 14:43:08 -0700 Subject: [PATCH 03/13] Add readme and update package.json --- docs/docs-next/README.md | 50 +++++++++++++++++++++++++++++++------ docs/docs-next/package.json | 7 +++--- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/docs/docs-next/README.md b/docs/docs-next/README.md index 9b9e6b6696c1f..2de7fbd8d4174 100644 --- a/docs/docs-next/README.md +++ b/docs/docs-next/README.md @@ -1,32 +1,66 @@ -# Website +# Dagster Docs - Beta -This website is built using [Docusaurus](https://docusaurus.io/), a modern static website generator. +This is the home of the new Dagster documentation. It is currently in beta and incomplete. +The documentation site is built using [Docusaurus](https://docusaurus.io/), a modern static website generator. ### Installation +The site uses [pnpm](https://pnpm.io/) for package management. +It also uses [vale](https://vale.sh/) to check for issues in the documentation. + +Install dependencies with: + ```bash -brew install pnpm +brew install pnpm vale pnpm install ``` +### Overview of the docs + +Code in `./src` contains custom components, styles, themes, and layouts. +Code `./content-templates` contains the templates for the documentation pages. +Code in `./docs/` is the source of truth for the documentation. + +`./docs/code_examples` contains all code examples for the documentation. + +The docs are broken down into the following sections: +- [Tutorials](./docs/tutorials/) +- [Guides](./docs/guides/) +- [Concepts](./docs/concepts/) + +`sidebar.ts` and `docusaurus.config.ts` are the main configuration files for the documentation. + ### Local Development +To start the local development server: + ```bash pnpm start ``` This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. Access the website at [http://localhost:3050](http://localhost:3050). -### Build + +To lint the documentation for issues: ```bash -pnpm build +pnpm lint +``` + +To autofix linting issues and format with prettier: + +```bash +pnpm lint:fix ``` -This command generates static content into the `build` directory and can be served using any static contents hosting service. -### Deployment + +### Build + +To build the site for production: ```bash -pnpm deploy +pnpm build ``` + +This command generates static content into the `build` directory and can be served using any static contents hosting service. \ No newline at end of file diff --git a/docs/docs-next/package.json b/docs/docs-next/package.json index 658ecdea16134..2470f4a7c6b76 100644 --- a/docs/docs-next/package.json +++ b/docs/docs-next/package.json @@ -12,10 +12,11 @@ "serve": "docusaurus serve", "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids", - "typecheck": "tsc", + "lint:ts": "tsc --noEmit", + "lint:vale": "vale ./docs --ext=.md,.mdx", "lint:eslint": "eslint . --ext=.tsx,.ts,.js,.md,.mdx", - "lint": "npx tsc --noEmit && prettier . --check && npm run lint:eslint", - "lint:fix": "prettier . --write && npm run lint:eslint --fix" + "lint": "prettier . --check && pnpm run lint:tsc && pnpm run lint:eslint && pnpm run lint:vale", + "lint:fix": "prettier . --write && pnpm run lint:eslint --fix" }, "dependencies": { "@docusaurus/core": "3.5.2", From 33b182633eb181ce3979e79cc3cbd7099a7dd79d Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:05:20 -0700 Subject: [PATCH 04/13] update automation --- docs/docs-beta/docs/concepts/automation.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/docs-beta/docs/concepts/automation.md b/docs/docs-beta/docs/concepts/automation.md index a568dc5196802..de4a9b3269bf1 100644 --- a/docs/docs-beta/docs/concepts/automation.md +++ b/docs/docs-beta/docs/concepts/automation.md @@ -2,7 +2,7 @@ title: About Automation --- -Dagster has support for several types of automation. All automation in Dagster responds to some external event. +There are several ways to automate the execution of your data pipelines with Dagster. The first system, and the most basic, is the [Schedule](/guides/automation/schedules), which responds to time. @@ -14,10 +14,10 @@ as reported by the Event Log. Finally, the Declarative Automation system is a more complex system that uses conditions on the assets to determine when to execute. -## About Schedules +## Schedules In Dagster, a schedule is defined by the `ScheduleDefinition` class, or through the `@schedule` decorator. The `@schedule` -decorator is more flexible than the `ScheduleDefinition` class, allowing you to configure job behaviour or emit log messages +decorator is more flexible than the `ScheduleDefinition` class, allowing you to configure job behavior or emit log messages as the schedule is processed. Schedules were one of the first types of automation in Dagster, created before the introduction of Software-Defined Assets. @@ -33,7 +33,7 @@ A schedule can be thought of as a wrapper around two pieces: - A `JobDefinition`, which is a set of assets to materialize or ops to execute. - A `cron` string, which describes the schedule. -### Defining a schedule using `ScheduleDefinition` +### Define a schedule using `ScheduleDefinition` ```python ecommerce_schedule = ScheduleDefinition( @@ -53,7 +53,7 @@ ecommerce_schedule = ScheduleDefinition( ) ``` -### Defining a schedule using `@schedule` +### Define a schedule using `@schedule` If you want more control over the schedule, you can use the `@schedule` decorator. In doing so, you are then responsible for either emitting a `RunRequest` or a `SkipReason`. You can also emit logs, which will be visible in the Dagster UI for a given schedule's tick history. From c47dc46d2cbd32159d8c0cf43f6f61ec997b86d8 Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:34:47 -0700 Subject: [PATCH 05/13] update templates to match automation guide --- .../content-templates/guide-no-steps.md | 38 ++++++++------- .../content-templates/guide-with-steps.md | 46 +++++++++---------- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/docs/docs-beta/content-templates/guide-no-steps.md b/docs/docs-beta/content-templates/guide-no-steps.md index e49ae9405a926..9ffb954e78ccc 100644 --- a/docs/docs-beta/content-templates/guide-no-steps.md +++ b/docs/docs-beta/content-templates/guide-no-steps.md @@ -1,40 +1,38 @@ --- -title: '' -description: '' +title: 'Title that briefly describes what the guide is for' +description: 'Description of the guide, useful for SEO and social media links' --- -# [Title that briefly describes what the guide is for] +Provide a brief introduction to the how-to guide. View [this article](https://diataxis.fr/how-to-guides/) for more information on how to write effective how-to guides. The intro should be no more than a few sentences. +The title from the frontmatter will be used as the first heading in the guide, you don't need to include it in the intro. - +## What you'll learn -By the end of this guide, you will [a sentence or list about what will be achieved by the end of the guide]. +- A thing you'll learn, ex: "How to generate a token" +- Another thing you'll learn, ex: "How to install this library" +- One more thing you'll learn ---- - -## Prerequisites - - +
+ Prerequisites To follow the steps in this guide, you'll need: -- A prerequisite, ex: "Familiarity with Asset definitions" -- Another prerequisite, ex: "To install library" +- A prerequisite, ex: "Familiarity with [Asset definitions](/concepts/assets)" +- Another prerequisite, ex: "To install this library" - One more ---- +
## Title that describes this section - - ---- -## Related +## Next steps -[List of links to related content] +- Add links to related content +- Go deeper into [Understanding Automation](/concepts/understanding-automation) +- Explore [Related Example](/) diff --git a/docs/docs-beta/content-templates/guide-with-steps.md b/docs/docs-beta/content-templates/guide-with-steps.md index c23598f5092cc..3e96b770524a0 100644 --- a/docs/docs-beta/content-templates/guide-with-steps.md +++ b/docs/docs-beta/content-templates/guide-with-steps.md @@ -1,49 +1,45 @@ --- -title: '' -description: '' +title: 'Title that briefly describes what the guide is for' +description: 'Description of the guide that is useful for SEO and social media links' --- -# [Title that briefly describes what the guide is for] +Provide a brief introduction to the how-to guide. View [this article](https://diataxis.fr/how-to-guides/) for more information on how to write effective how-to guides. The intro should be no more than a few sentences. +The title from the frontmatter will be used as the first heading in the guide, you don't need to include it in the intro. - +## What you'll learn -By the end of this guide, you will [a sentence or list about what will be achieved by the end of the guide]. +- A thing you'll learn, ex: "How to generate a token" +- Another thing you'll learn, ex: "How to install this library" +- One more thing you'll learn ---- - -## Prerequisites - - +
+ Prerequisites To follow the steps in this guide, you'll need: -- A prerequisite, ex: "Familiarity with Asset definitions" -- Another prerequisite, ex: "To install library" +- A prerequisite, ex: "Familiarity with [Asset definitions](/concepts/assets)" +- Another prerequisite, ex: "To install this library" - One more ---- +
## Step 1: Title that describes what this step will do {#step-1} - +- Each section heading should have an identifier that includes the word 'step' and the number of the step, ex: {#step-1} ### Step 1.1: Title that describes a substep {#step-1-1} - - ---- +If a step would benefit by being broken into smaller steps, follow this section's formatting +Each substep should get an H3 and start with Step N., followed by the number of the substep ## Step 2: Another step {#step-2} ---- - -## Related +## Next steps -[List of links to related content] +- Add links to related content +- Go deeper into [Understanding Automation](/concepts/understanding-automation) +- Explore [Related Example](/) From d2279991de2f21a70e8c3dd1b94b65fe1d59a72e Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Mon, 19 Aug 2024 12:40:37 -0400 Subject: [PATCH 06/13] add visible hr --- docs/docs-beta/src/styles/custom.scss | 4 ++++ docs/docs-beta/src/styles/theme-dark.scss | 3 +++ docs/docs-beta/src/styles/theme-light.scss | 3 +++ 3 files changed, 10 insertions(+) diff --git a/docs/docs-beta/src/styles/custom.scss b/docs/docs-beta/src/styles/custom.scss index 4e8f9bc211c6d..8e1b505736d66 100644 --- a/docs/docs-beta/src/styles/custom.scss +++ b/docs/docs-beta/src/styles/custom.scss @@ -21,6 +21,10 @@ article { margin: 0 auto; } +hr { + height: 1px; +} + .breadcrumbs { display: flex; flex-direction: row; diff --git a/docs/docs-beta/src/styles/theme-dark.scss b/docs/docs-beta/src/styles/theme-dark.scss index c07eef110edd6..3622df270594d 100644 --- a/docs/docs-beta/src/styles/theme-dark.scss +++ b/docs/docs-beta/src/styles/theme-dark.scss @@ -60,4 +60,7 @@ --ifm-menu-color: var(--theme-color-text-light); --ifm-hover-overlay: var(--theme-color-background-blue); --ifm-menu-color-active: var(--theme-color-background-blue); + + //hr + --ifm-hr-background-color: var(--theme-color-background-lighter); } diff --git a/docs/docs-beta/src/styles/theme-light.scss b/docs/docs-beta/src/styles/theme-light.scss index 0bcccfaa0917b..ca7c54ea169b3 100644 --- a/docs/docs-beta/src/styles/theme-light.scss +++ b/docs/docs-beta/src/styles/theme-light.scss @@ -76,6 +76,9 @@ --ifm-hover-overlay: var(--theme-color-background-blue); --ifm-menu-color-active: var(--theme-color-background-blue); + //hr + --ifm-hr-background-color: var(--theme-color-background-light); + // docusaurus --docusaurus-highlighted-code-line-bg: var(--theme-color-background-blue); From 765fd1f4cb999e0f3f4167d5bdea2f41e44e36ad Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Mon, 19 Aug 2024 12:58:49 -0400 Subject: [PATCH 07/13] add top borders to all markdown h2 headers --- docs/docs-beta/src/styles/custom.scss | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/docs-beta/src/styles/custom.scss b/docs/docs-beta/src/styles/custom.scss index 8e1b505736d66..80d3c8d91ad19 100644 --- a/docs/docs-beta/src/styles/custom.scss +++ b/docs/docs-beta/src/styles/custom.scss @@ -108,6 +108,15 @@ a.pyobject { --ifm-h5-font-size: 0.8rem; } + // Emulate horizontal rule above h2 headers + h2 { + border-top: 1px; + border-top-style: solid; + border-top-color: var(--theme-color-text-light); + margin-top: 0px; + padding-top: calc(var(--ifm-heading-vertical-rhythm-bottom) * var(--ifm-leading)) + } + // We want this to only apply to inline code, so don't apply // this color to ``` code blocks nor any headings. :not(pre):not(h2):not(h3):not(h4):not(h5):not(h6) > code { From 8158134fc1596c14c01be10afc905793b5925b41 Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Mon, 19 Aug 2024 13:03:20 -0400 Subject: [PATCH 08/13] h2 border color keyline --- docs/docs-beta/src/styles/custom.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs-beta/src/styles/custom.scss b/docs/docs-beta/src/styles/custom.scss index 80d3c8d91ad19..2acc97b1f0580 100644 --- a/docs/docs-beta/src/styles/custom.scss +++ b/docs/docs-beta/src/styles/custom.scss @@ -112,7 +112,7 @@ a.pyobject { h2 { border-top: 1px; border-top-style: solid; - border-top-color: var(--theme-color-text-light); + border-top-color: var(--theme-color-keyline); margin-top: 0px; padding-top: calc(var(--ifm-heading-vertical-rhythm-bottom) * var(--ifm-leading)) } From 8d436d19edd8048bcfdde7176cf1dff78d543be9 Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:05:56 -0700 Subject: [PATCH 09/13] add vscode recommendations; --- docs/docs-beta/.vscode/extensions.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/docs-beta/.vscode/extensions.json diff --git a/docs/docs-beta/.vscode/extensions.json b/docs/docs-beta/.vscode/extensions.json new file mode 100644 index 0000000000000..ca4a4c709abe9 --- /dev/null +++ b/docs/docs-beta/.vscode/extensions.json @@ -0,0 +1,9 @@ +{ + "recommendations": [ + "dbaeumer.vscode-eslint", + "unifiedjs.vscode-mdx", + "esbenp.prettier-vscode", + "mrmlnc.vscode-remark", + "chrischinchilla.vale-vscode" + ] +} \ No newline at end of file From f267e5248de2cddc2d9f5d63e196bdfcad5126ca Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:29:31 -0700 Subject: [PATCH 10/13] update data modeling --- .../docs/guides/{data-assets.md => data-modeling.md} | 0 docs/docs-beta/sidebars.ts | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename docs/docs-beta/docs/guides/{data-assets.md => data-modeling.md} (100%) diff --git a/docs/docs-beta/docs/guides/data-assets.md b/docs/docs-beta/docs/guides/data-modeling.md similarity index 100% rename from docs/docs-beta/docs/guides/data-assets.md rename to docs/docs-beta/docs/guides/data-modeling.md diff --git a/docs/docs-beta/sidebars.ts b/docs/docs-beta/sidebars.ts index 9ed1a026afa71..11c337db6fc98 100644 --- a/docs/docs-beta/sidebars.ts +++ b/docs/docs-beta/sidebars.ts @@ -30,10 +30,10 @@ const sidebars: SidebarsConfig = { items: [ { type: 'category', - label: 'Data assets', + label: 'Data modeling', link: { type: 'doc', - id: 'guides/data-assets', + id: 'guides/data-modeling', }, items: [ { From 985f58300a078b2701085a90544b96db76ba8afb Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Mon, 19 Aug 2024 14:37:06 -0700 Subject: [PATCH 11/13] add prelude/next-steps from templates --- docs/docs-beta/docs/guides/automation.md | 2 +- .../docs/guides/automation/asset-sensors.md | 29 ++++++++++++++----- ...ynamic-pipelines-based-on-external-data.md | 6 ---- .../docs/guides/automation/schedules.md | 17 +++++++++-- .../triggering-pipeline-runs-using-events.md | 4 --- docs/docs-beta/src/theme/MDXComponents.tsx | 3 ++ 6 files changed, 41 insertions(+), 20 deletions(-) delete mode 100644 docs/docs-beta/docs/guides/automation/creating-dynamic-pipelines-based-on-external-data.md delete mode 100644 docs/docs-beta/docs/guides/automation/triggering-pipeline-runs-using-events.md diff --git a/docs/docs-beta/docs/guides/automation.md b/docs/docs-beta/docs/guides/automation.md index 5eda1daaf01c5..cfa58783c771f 100644 --- a/docs/docs-beta/docs/guides/automation.md +++ b/docs/docs-beta/docs/guides/automation.md @@ -80,7 +80,7 @@ For more examples of how to create asset sensors, see the [How-To Use Asset Sens ## Declarative automation -TODO: Add content +TODO: add content ## How to choose the right automation method diff --git a/docs/docs-beta/docs/guides/automation/asset-sensors.md b/docs/docs-beta/docs/guides/automation/asset-sensors.md index 5890d8f325b7f..491ad30a051e7 100644 --- a/docs/docs-beta/docs/guides/automation/asset-sensors.md +++ b/docs/docs-beta/docs/guides/automation/asset-sensors.md @@ -4,14 +4,25 @@ sidebar_label: Scheduling pipelines across projects and jobs sidebar_position: 30 --- + Asset sensors in Dagster provide a powerful mechanism for monitoring asset materializations and triggering downstream computations or notifications based on those events. There are many use cases for asset sensors, however the most common is to trigger a job when an asset is materialized in a different job or code location, or to provide custom logic to determine when to trigger a run following an asset materialization. -### Cross-job and cross-code location dependencies +
+Prerequisites -Asset Sensors enable dependencies across different jobs and even different code locations. This flexibility allows for more modular and decoupled workflows. +- Familiarity with [Assets](/concepts/assets) +- Familiarity with [Ops and Jobs](/concepts/ops-and-jobs) +
+ +In this guide, you'll learn how to: + + +## Cross-job and cross-code location dependencies + +Asset Sensors enable dependencies across different jobs and even different code locations. This flexibility allows for more modular and decoupled workflows. ```mermaid graph LR; @@ -31,14 +42,13 @@ subgraph CodeLocationB Asset1 Asset2 end - ``` Here's a minimal example of an asset sensor that triggers a job when an asset is materialized. The `daily_sales_data` asset is in the same code location for this example, but the same pattern can be applied to assets in different code locations. -### Custom evaluation logic +## Custom evaluation logic The evaluation function of an asset sensor can be customized to include custom logic for determining when to trigger a run. This allows for fine-grained control over the conditions under which downstream jobs are executed. @@ -61,7 +71,7 @@ stateDiagram-v2 classDef userDefined fill: lightblue ``` -In this example, the `@asset_sensor` decorator allows us to define a custom evaluation function that returns a `RunRequest` object when the asset is materialized and certain metadata is present, +In this example, the `@asset_sensor` decorator allows you to define a custom evaluation function that returns a `RunRequest` object when the asset is materialized and certain metadata is present, otherwise it skips the run. @@ -75,6 +85,11 @@ By providing a configuration to the `RunRequest` object, you can trigger a job w ## Monitor multiple assets -This example shows how to use a multi-asset sensor to monitor multiple assets and trigger a job when any of the monitored assets are materialized. +The previous examples showed how to use a single asset sensor to monitor a single asset and trigger a job when it's materialized. This example uses a multi-asset sensor to monitor multiple assets and trigger a job when any of the monitored assets are materialized. + + + +## Next steps - \ No newline at end of file +- Learn more about asset sensors in [Understanding Automation](/concepts/automation) +- Explore [Declarative Automation](/concepts/declarative-automation) as an alternative to asset sensors \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/automation/creating-dynamic-pipelines-based-on-external-data.md b/docs/docs-beta/docs/guides/automation/creating-dynamic-pipelines-based-on-external-data.md deleted file mode 100644 index 98576d9d32794..0000000000000 --- a/docs/docs-beta/docs/guides/automation/creating-dynamic-pipelines-based-on-external-data.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Creating dynamic pipelines based on external data" -sidebar_position: 30 ---- - -# Creating dynamic pipelines based on external data diff --git a/docs/docs-beta/docs/guides/automation/schedules.md b/docs/docs-beta/docs/guides/automation/schedules.md index 4f7cefd61aae0..5cf68d428b14a 100644 --- a/docs/docs-beta/docs/guides/automation/schedules.md +++ b/docs/docs-beta/docs/guides/automation/schedules.md @@ -6,6 +6,13 @@ sidebar_position: 10 Schedules enable automated execution of jobs at specified intervals. These intervals can range from common frequencies like hourly, daily, or weekly, to more intricate patterns defined using cron expressions. +
+Prerequisites + +- Familiarity with [Assets](/concepts/assets) +- Familiarity with [Ops and Jobs](/concepts/ops-and-jobs) +
+ ## Basic schedule A basic schedule is defined by a `JobDefinition` and a `cron_schedule` using the `ScheduleDefinition` class. A job can be thought of as a selection of assets or operations executed together. @@ -47,6 +54,12 @@ partitioned_op_schedule = build_schedule_from_partitioned_job( # highlight-end ``` ---- -For more information about how Schedules work, see the [About Schedules](/concepts/schedules). \ No newline at end of file + +## Next steps + +- Learn more about schedules in [Understanding Automation](/concepts/automation) +- React to events with [sensors](/guides/automation/sensors) +- Explore [Declarative Automation](/concepts/declarative-automation) as an alternative to schedules + +By understanding and effectively using these automation methods, you can build more efficient data pipelines that respond to your specific needs and constraints. \ No newline at end of file diff --git a/docs/docs-beta/docs/guides/automation/triggering-pipeline-runs-using-events.md b/docs/docs-beta/docs/guides/automation/triggering-pipeline-runs-using-events.md deleted file mode 100644 index 17826f1c536e4..0000000000000 --- a/docs/docs-beta/docs/guides/automation/triggering-pipeline-runs-using-events.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: "Creating event-based pipelines" -sidebar_position: 20 ---- diff --git a/docs/docs-beta/src/theme/MDXComponents.tsx b/docs/docs-beta/src/theme/MDXComponents.tsx index f4b9d9a17a51d..7f4f8361523c5 100644 --- a/docs/docs-beta/src/theme/MDXComponents.tsx +++ b/docs/docs-beta/src/theme/MDXComponents.tsx @@ -4,6 +4,8 @@ import {PyObject} from '../components/PyObject'; import CodeExample from '../components/CodeExample'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import TOCInline from '@theme/TOCInline'; + export default { // Re-use the default mapping ...MDXComponents, @@ -11,4 +13,5 @@ export default { Tabs, TabItem, CodeExample, + TOCInline, }; From 18f065abfabd130d85e940dc601303b43ff56df7 Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:11:26 -0700 Subject: [PATCH 12/13] update sensor how-to --- .../docs/guides/automation/asset-sensors.md | 4 ++-- docs/docs-beta/docs/guides/automation/sensors.md | 14 ++++++++++++++ docs/docs-beta/src/styles/custom.scss | 9 +++++---- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/docs-beta/docs/guides/automation/asset-sensors.md b/docs/docs-beta/docs/guides/automation/asset-sensors.md index 491ad30a051e7..6a29a70c31aec 100644 --- a/docs/docs-beta/docs/guides/automation/asset-sensors.md +++ b/docs/docs-beta/docs/guides/automation/asset-sensors.md @@ -1,6 +1,6 @@ --- -title: Scheduling pipelines across projects and jobs with Asset Sensors -sidebar_label: Scheduling pipelines across projects and jobs +title: Triggering jobs with Asset Sensors +sidebar_label: Triggering jobs with Asset Sensors sidebar_position: 30 --- diff --git a/docs/docs-beta/docs/guides/automation/sensors.md b/docs/docs-beta/docs/guides/automation/sensors.md index ba0e5548977a6..f30e16b94379e 100644 --- a/docs/docs-beta/docs/guides/automation/sensors.md +++ b/docs/docs-beta/docs/guides/automation/sensors.md @@ -4,11 +4,25 @@ sidebar_label: Creating event-based pipelines sidebar_position: 20 --- +Sensors are a way to trigger runs in response to events in Dagster. Sensors +run on a regular interval and can either trigger a run, or provide a reason why a run was skipped. + +Sensors allow you to react events in external systems. For example, you can trigger a run when a new file arrives in an S3 bucket, or when a row is updated in a database. + +
+Prerequisites + +- Familiarity with [Assets](/concepts/assets) +- Familiarity with [Ops and Jobs](/concepts/ops-and-jobs) +
+ +## Basic sensor example This example includes a `check_for_new_files` function that simulates finding new files. In a real scenario, this function would check an actual system or directory. The sensor runs every 5 seconds. If it finds new files, it starts a run of `my_job`. If not, it skips the run and logs "No new files found" in the Dagster UI. + :::tip diff --git a/docs/docs-beta/src/styles/custom.scss b/docs/docs-beta/src/styles/custom.scss index 2acc97b1f0580..212e2a42dd264 100644 --- a/docs/docs-beta/src/styles/custom.scss +++ b/docs/docs-beta/src/styles/custom.scss @@ -114,13 +114,14 @@ a.pyobject { border-top-style: solid; border-top-color: var(--theme-color-keyline); margin-top: 0px; - padding-top: calc(var(--ifm-heading-vertical-rhythm-bottom) * var(--ifm-leading)) + padding-top: calc(var(--ifm-heading-vertical-rhythm-bottom) * var(--ifm-leading)); } - // We want this to only apply to inline code, so don't apply - // this color to ``` code blocks nor any headings. + // We want this to only apply to inline code :not(pre):not(h2):not(h3):not(h4):not(h5):not(h6) > code { - border: none; + background-color: var(--theme-color-background-blue); + border: 0.5px solid var(--theme-color-keyline); + padding: 0.1rem; } // don't apply --dagster-inline-code colors to admonitions From c0eb46a7aa8db7d38dc55d6e373c77a2339d70e3 Mon Sep 17 00:00:00 2001 From: Pedram Navid <1045990+PedramNavid@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:31:23 -0700 Subject: [PATCH 13/13] refine asset sensors guide and improve TOC styling --- .../docs/guides/automation/asset-sensors.md | 10 +++------- docs/docs-beta/src/styles/custom.scss | 20 ++++++++++++++++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/docs-beta/docs/guides/automation/asset-sensors.md b/docs/docs-beta/docs/guides/automation/asset-sensors.md index 6a29a70c31aec..21d29857980d2 100644 --- a/docs/docs-beta/docs/guides/automation/asset-sensors.md +++ b/docs/docs-beta/docs/guides/automation/asset-sensors.md @@ -7,7 +7,7 @@ sidebar_position: 30 Asset sensors in Dagster provide a powerful mechanism for monitoring asset materializations and triggering downstream computations or notifications based on those events. -There are many use cases for asset sensors, however the most common is to trigger a job when an asset is materialized in a different job or code location, or to provide custom logic to determine when to trigger a run following an asset materialization. +This guide covers the most common use cases for asset sensors such as defining cross-job and cross-code location dependencies.
Prerequisites @@ -16,11 +16,7 @@ There are many use cases for asset sensors, however the most common is to trigge - Familiarity with [Ops and Jobs](/concepts/ops-and-jobs)
-In this guide, you'll learn how to: - - - -## Cross-job and cross-code location dependencies +## Define cross-job and cross-code location dependencies Asset Sensors enable dependencies across different jobs and even different code locations. This flexibility allows for more modular and decoupled workflows. @@ -48,7 +44,7 @@ Here's a minimal example of an asset sensor that triggers a job when an asset is -## Custom evaluation logic +## Customize evaluation logic The evaluation function of an asset sensor can be customized to include custom logic for determining when to trigger a run. This allows for fine-grained control over the conditions under which downstream jobs are executed. diff --git a/docs/docs-beta/src/styles/custom.scss b/docs/docs-beta/src/styles/custom.scss index 212e2a42dd264..8a06dc9f77f5a 100644 --- a/docs/docs-beta/src/styles/custom.scss +++ b/docs/docs-beta/src/styles/custom.scss @@ -216,10 +216,24 @@ a.pyobject { border: 1px solid rgba(200, 200, 200, 0.3); } -.table-of-contents__link--active { - font-weight: var(--ifm-font-weight-bold); +.markdown .table-of-contents { + li { + list-style: none; + padding-top: 4px; + line-height: 1; + } + li a { + font-weight: var(--ifm-font-weight-normal); + } +} +.table-of-contents { + &__link:hover { + background-color: var(--theme-color-background-gray); + } + &__link--active { + font-weight: 500; + } } - .pagination-nav { &__link { border: 0;