diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/trigger.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/trigger.py index 3f6087cdbfd38..fff5f317adb89 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/trigger.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/trigger.py @@ -28,7 +28,7 @@ def build_trigger_step( dagster_commit_hash = safe_getenv("BUILDKITE_COMMIT") step: TriggerStep = { "trigger": pipeline, - "label": f":link: {pipeline} from dagster@{dagster_commit_hash[:6]}", + "label": f":link: {pipeline} from dagster@{dagster_commit_hash[:10]}", "async": async_step, "build": { "env": env or {}, diff --git a/CHANGES.md b/CHANGES.md index 911f8fe2c6e34..1d6dddc474e12 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,38 @@ # Changelog +## 1.9.6 (core) / 0.25.6 (libraries) + +### New + +- Updated `cronitor` pin to allow versions `>= 5.0.1` to enable use of `DayOfWeek` as 7. Cronitor `4.0.0` is still disallowed. (Thanks, [@joshuataylor](https://github.com/joshuataylor)!) +- Added flag `checkDbReadyInitContainer` to optionally disable db check initContainer. +- [ui] Added Google Drive icon for `kind` tags. (Thanks, [@dragos-pop](https://github.com/dragos-pop)!) +- [ui] Renamed the run lineage sidebar on the Run details page to `Re-executions`. +- [ui] Sensors and schedules that appear in the Runs page are now clickable. +- [ui] Runs targeting assets now show more of the assets in the Runs page. +- [dagster-airbyte] The destination type for an Airbyte asset is now added as a `kind` tag for display in the UI. +- [dagster-gcp] `DataprocResource` now receives an optional parameter `labels` to be attached to Dataproc clusters. (Thanks, [@thiagoazcampos](https://github.com/thiagoazcampos)!) +- [dagster-k8s] Added a `checkDbReadyInitContainer` flag to the Dagster Helm chart to allow disabling the default init container behavior. (Thanks, [@easontm](https://github.com/easontm)!) +- [dagster-k8s] K8s pod logs are now logged when a pod fails. (Thanks, [@apetryla](https://github.com/apetryla)!) +- [dagster-sigma] Introduced `build_materialize_workbook_assets_definition` which can be used to build assets that run materialize schedules for a Sigma workbook. +- [dagster-snowflake] `SnowflakeResource` and `SnowflakeIOManager` both accept `additional_snowflake_connection_args` config. This dictionary of arguments will be passed to the `snowflake.connector.connect` method. This config will be ignored if you are using the `sqlalchemy` connector. +- [helm] Added the ability to set user-deployments labels on k8s deployments as well as pods. + +### Bugfixes + +- Assets with self dependencies and `BackfillPolicy` are now evaluated correctly during backfills. Self dependent assets no longer result in serial partition submissions or disregarded upstream dependencies. +- Previously, the freshness check sensor would not re-evaluate freshness checks if an in-flight run was planning on evaluating that check. Now, the freshness check sensor will kick off an independent run of the check, even if there's already an in flight run, as long as the freshness check can potentially fail. +- Previously, if the freshness check was in a failing state, the sensor would wait for a run to update the freshness check before re-evaluating. Now, if there's a materialization later than the last evaluation of the freshness check and no planned evaluation, we will re-evaluate the freshness check automatically. +- [ui] Fixed run log streaming for runs with a large volume of logs. +- [ui] Fixed a bug in the Backfill Preview where a loading spinner would spin forever if an asset had no valid partitions targeted by the backfill. +- [dagster-aws] `PipesCloudWatchMessageReader` correctly identifies streams which are not ready yet and doesn't fail on `ThrottlingException`. (Thanks, [@jenkoian](https://github.com/jenkoian)!) +- [dagster-fivetran] Column metadata can now be fetched for Fivetran assets using `FivetranWorkspace.sync_and_poll(...).fetch_column_metadata()`. +- [dagster-k8s] The k8s client now waits for the main container to be ready instead of only waiting for sidecar init containers. (Thanks, [@OrenLederman](https://github.com/OrenLederman)!) + +### Documentation + +- Fixed a typo in the `dlt_assets` API docs. (Thanks, [@zilto](https://github.com/zilto)!) + ## 1.9.5 (core) / 0.25.5 (libraries) ### New diff --git a/docs/content/guides/limiting-concurrency-in-data-pipelines.mdx b/docs/content/guides/limiting-concurrency-in-data-pipelines.mdx index 010eb5a11e278..ab45e8b7e61e1 100644 --- a/docs/content/guides/limiting-concurrency-in-data-pipelines.mdx +++ b/docs/content/guides/limiting-concurrency-in-data-pipelines.mdx @@ -391,11 +391,6 @@ height={1638} ### Limiting op/asset concurrency across runs - - This feature is experimental and is only supported with Postgres/MySQL - storages. - - #### For specific ops/assets Limits can be specified on the Dagster instance using the special op tag `dagster/concurrency_key`. If this instance limit would be exceeded by launching an op/asset, then the op/asset will be queued. diff --git a/docs/docs-beta/CONTRIBUTING.md b/docs/docs-beta/CONTRIBUTING.md index c0b8e52ba5712..3d446d91f5e60 100644 --- a/docs/docs-beta/CONTRIBUTING.md +++ b/docs/docs-beta/CONTRIBUTING.md @@ -102,6 +102,10 @@ After: | `DAGSTER_CLOUD_DEPLOYMENT_NAME` | The name of the Dagster+ deployment.

**Example:** `prod`. | | `DAGSTER_CLOUD_IS_BRANCH_DEPLOYMENT` | `1` if the deployment is a [branch deployment](/dagster-plus/features/ci-cd/branch-deployments/index.md). | +#### Line breaks and lists in tables + +[Use HTML](https://www.markdownguide.org/hacks/#table-formatting) to add line breaks and lists to tables. + ### Whitespace via `{" "}` Forcing empty space using the `{" "}` interpolation is not supported, and must be removed. diff --git a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/dagster-cloud-yaml.md b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/dagster-cloud-yaml.md index 7790f2cf5230a..09ebc00dbc154 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/dagster-cloud-yaml.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/dagster-cloud-yaml.md @@ -1,7 +1,305 @@ --- title: dagster_cloud.yaml reference sidebar_position: 200 -unlisted: true --- -{/* TODO move content from https://docs.dagster.io/dagster-plus/managing-deployments/dagster-cloud-yaml */} \ No newline at end of file +:::note +This reference is applicable to Dagster+. +::: + + + + + + + + + + + + + + + + + + + + + + + + +
+ Name + dagster_cloud.yaml
+ Status + Active
+ Required + Required for Dagster+
+ Description + + {" "} + Similar to the workspace.yaml in open source to define code + locations for Dagster+. +
+ Uses + + Defines multiple code locations for Dagster+. For Hybrid deployments, this file can be used + to manage + environment variables/secrets. +
    +
    + +## File location + +The `dagster_cloud.yaml` file should be placed in the root of your Dagster project. Below is an example of a file structure modified from the [Dagster+ ETL quickstart](https://github.com/dagster-io/dagster/tree/master/examples/quickstart_etl). + +```shell +quickstart_etl +├── README.md +├── quickstart_etl +│   ├── __init__.py +│   ├── assets +│   ├── docker_image +├── ml_project +│ ├── quickstart_ml +│ ├── __init__.py +│ ├── ml_assets +├── random_assets.py +├── quickstart_etl_tests +├── dagster_cloud.yaml +├── pyproject.toml +├── setup.cfg +└── setup.py +``` + +If your repository contains multiple Dagster projects in subdirectories - otherwise known as a monorepository - add the `dagster_cloud.yaml` file to the root of where the Dagster projects are stored. + +## File structure + +Settings are formatted using YAML. For example, using the file structure above as an example: + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: data-eng-pipeline + code_source: + package_name: quickstart_etl + build: + directory: ./quickstart_etl + registry: localhost:5000/docker_image + - location_name: ml-pipeline + code_source: + package_name: quickstart_ml + working_directory: ./ml_project + executable_path: venvs/path/to/ml_tensorflow/bin/python + - location_name: my_random_assets + code_source: + python_file: random_assets.py + container_context: + k8s: + env_vars: + - database_name + - database_username=hooli_testing + env_secrets: + - database_password +``` + +## Settings + +The `dagster_cloud.yaml` file contains a single top-level key, `locations`. This key accepts a list of code locations; for each code location, you can configure the following: + +- [Location name](#location-name) +- [Code source](#code-source) +- [Working directory](#working-directory) +- [Build](#build) +- [Python executable](#python-executable) +- [Container context](#container-context) + +### Location name + +**This key is required.** The `location_name` key specifies the name of the code location. The location name will always be paired with a [code source](#code-source). + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: data-eng-pipeline + code_source: + package_name: quickstart_etl +``` + +| Property | Description | Format | +|-----------------|----------------------------------------------------------------------------------------|----------| +| `location_name` | The name of your code location that will appear in the Dagster UI Code locations page. | `string` | + +### Code source + +**This section is required.** The `code_source` defines how a code location is sourced. + +A `code_source` key must contain either a `module_name`, `package_name`, or `file_name` parameter that specifies where to find the definitions in the code location. + + + + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: data-eng-pipeline + code_source: + package_name: quickstart_etl +``` + + + + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: data-eng-pipeline + code_source: + package_name: quickstart_etl + - location_name: machine_learning + code_source: + python_file: ml/ml_model.py +``` + + + + +| Property | Description | Format | +|----------------------------|-----------------------------------------------------------------------------------|--------------------------| +| `code_source.package_name` | The name of a package containing Dagster code | `string` (folder name) | +| `code_source.python_file` | The name of a Python file containing Dagster code (e.g. `analytics_pipeline.py` ) | `string` (.py file name) | +| `code_source.module_name` | The name of a Python module containing Dagster code (e.g. `analytics_etl`) | `string` (module name) | + +### Working directory + +Use the `working_directory` setting to load Dagster code from a different directory than the root of your code repository. This setting allows you to specify the directory you want to load your code from. + +Consider the following project: + +```shell +quickstart_etl +├── README.md +├── project_directory +│   ├── quickstart_etl +│   ├── __init__.py +│   ├── assets +│   ├── quickstart_etl_tests +├── dagster_cloud.yaml +├── pyproject.toml +├── setup.cfg +└── setup.py +``` + +To load from `/project_directory`, the `dagster_cloud.yaml` code location would look like this: + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: data-eng-pipeline + code_source: + package_name: quickstart_etl + working_directory: ./project_directory +``` + +| Property | Description | Format | +|---------------------|-------------------------------------------------------------------------|-----------------| +| `working_directory` | The path of the directory that Dagster should load the code source from | `string` (path) | + +### Build + +The `build` section contains two parameters: + +- `directory` - Setting a build directory is useful if your `setup.py` or `requirements.txt` is in a subdirectory instead of the project root. This is common if you have multiple Python modules within a single Dagster project. +- `registry` - **Applicable only to Hybrid deployments.** Specifies the Docker registry to push the code location to. + +In the example below, the Docker image for the code location is in the root directory and the registry and image defined: + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: data-eng-pipeline + code_source: + package_name: quickstart_etl + build: + directory: ./ + registry: your-docker-image-registry/image-name # e.g. localhost:5000/myimage +``` + + +| Property | Description | Format | Default | +|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|---------| +| `build.directory` | The path to the directory in your project that you want to deploy. If there are subdirectories, you can specify the path to only deploy a specific project directory. | `string` (path) | `.` | +| `build.registry` | **Applicable to Hybrid deployments.** The Docker registry to push your code location to | `string` (docker registry) | | + + +### Python executable + +For Dagster+ Hybrid deployments, the Python executable that is installed globally in the image, or the default Python executable on the local system if you use the local agent, will be used. To use a different Python executable, specify it using the `executable_path` setting. It can be useful to have different Python executables for different code locations. + +{/* For Dagster+ Serverless deployments, you can specify a different Python version by [following these instructions](/dagster-plus/deployment/deployment-types/serverless/runtime-environment#python-version). */} +For Dagster+ Serverless deployments, you can specify a different Python version by [following these instructions](/todo). + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: data-eng-pipeline + code_source: + package_name: quickstart_etl + executable_path: venvs/path/to/dataengineering_spark_team/bin/python + - location_name: machine_learning + code_source: + python_file: ml_model.py + executable_path: venvs/path/to/ml_tensorflow/bin/python +``` + +| Property | Description | Format | +|-------------------|-----------------------------------------------|-----------------| +| `executable_path` | The file path of the Python executable to use | `string` (path) | + +### Container context + +If using Hybrid deployment, you can define additional configuration options for code locations using the `container_context` parameter. Depending on the Hybrid agent you're using, the configuration settings under `container_context` will vary. + +Refer to the configuration reference for your agent for more info: + +{/* - [Docker agent configuration reference](/dagster-plus/deployment/agents/docker/configuration-reference) */} +- [Docker agent configuration reference](/todo) +{/* - [Amazon ECS agent configuration reference](/dagster-plus/deployment/agents/amazon-ecs/configuration-reference) */} +- [Amazon ECS agent configuration reference](/todo) +{/* - [Kubernetes agent configuration reference](/dagster-plus/deployment/agents/kubernetes/configuration-reference) */} +- [Kubernetes agent configuration reference](/todo) diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/configuration.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/configuration.md index 1294b30313cfb..99111c0fa0616 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/configuration.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/configuration.md @@ -1,7 +1,41 @@ --- title: Docker agent configuration sidebar_position: 200 -unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/docker/configuration-reference */} \ No newline at end of file +:::note +This guide is applicable to Dagster+. +::: + +{/* This reference describes the various configuration options Dagster+ currently supports for [Docker agents](/dagster-plus/deployment/agents/docker/configuring-running-docker-agent). */} +This reference describes the various configuration options Dagster+ currently supports for [Docker agents](/todo). + +--- + +## Environment variables and secrets + +Using the `container_context.docker.env_vars` property, you can include environment variables and secrets in the Docker container associated with a specific code location. For example: + +```yaml +# dagster_cloud.yaml +locations: + - location_name: cloud-examples + image: dagster/dagster-cloud-examples:latest + code_source: + package_name: dagster_cloud_examples + container_context: + docker: + env_vars: + - DATABASE_NAME + - DATABASE_USERNAME=hooli_testing +``` + +The `container_context.docker.env_vars` property is a list, where each item can be either `KEY` or `KEY=VALUE`. If only `KEY` is specified, the value will be pulled from the local environment. + +Refer to the following guides for more info about environment variables: + +{/* - [Dagster+ environment variables and secrets](/dagster-plus/managing-deployments/environment-variables-and-secrets) */} +- [Dagster+ environment variables and secrets](/todo) +{/* - [Using environment variables and secrets in Dagster code](/guides/dagster/using-environment-variables-and-secrets) */} +- [Using environment variables and secrets in Dagster code](/todo) + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/setup.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/setup.md index f16e0dc8e25a9..9144b7cb623cf 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/setup.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/setup.md @@ -1,7 +1,125 @@ --- title: Docker agent setup sidebar_position: 100 -unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/docker/configuring-running-docker-agent */} \ No newline at end of file +:::note +This guide is applicable to Dagster+. +::: + +In this guide, you'll configure and run a Docker agent. Docker agents are used to launch your code in Docker containers. + +## Prerequisites + +To complete the steps in this guide, you'll need: + +{/* - **Permissions in Dagster+ that allow you to manage agent tokens**. Refer to the [User permissions documentation](/dagster-plus/account/managing-users) for more info. */} +- **Permissions in Dagster+ that allow you to manage agent tokens**. Refer to the [User permissions documentation](/todo) for more info. +- **To have Docker installed** +- **Access to a container registry to which you can push images with Dagster code.** Additionally, your Docker agent must have the permissions required to pull images from the registry. + + This can be: + + - A self-hosted registry, + - A public registry such as [DockerHub](https://hub.docker.com/), or + - A managed offering such as [Amazon ECR](https://aws.amazon.com/ecr/), [Azure CR](https://azure.microsoft.com/en-us/services/container-registry/#overview), or [Google CR](https://cloud.google.com/container-registry) + +## Step 1: Generate a Dagster+ agent token + +In this step, you'll generate a token for the Dagster+ agent. The Dagster+ agent will use this to authenticate to the agent API. + +1. Sign in to your Dagster+ instance. +2. Click the **user menu (your icon) > Organization Settings**. +3. In the **Organization Settings** page, click the **Tokens** tab. +4. Click the **+ Create agent token** button. +5. After the token has been created, click **Reveal token**. + +Keep the token somewhere handy - you'll need it to complete the setup. + +## Step 2: Create a Docker agent + +1. Create a Docker network for your agent: + + ```shell + docker network create dagster_cloud_agent + ``` + +2. Create a `dagster.yaml` file: + + ```yaml + instance_class: + module: dagster_cloud.instance + class: DagsterCloudAgentInstance + + dagster_cloud_api: + agent_token: + branch_deployments: true # enables branch deployments + deployment: prod + + user_code_launcher: + module: dagster_cloud.workspace.docker + class: DockerUserCodeLauncher + config: + networks: + - dagster_cloud_agent + ``` + +3. In the file, fill in the following: + + - `agent_token` - Add the agent token you created in [Step 1](#step-1-generate-a-dagster-agent-token) + - `deployment` - Enter the deployment associated with this instance of the agent. + + In the above example, we specified `prod` as the deployment. This is present when Dagster+ organizations are first created. + +4. Save the file. + +## Step 3: Start the agent + +Next, you'll start the agent as a container. Run the following command in the same folder as your `dagster.yaml` file: + +```shell +docker run \ + --network=dagster_cloud_agent \ + --volume $PWD/dagster.yaml:/opt/dagster/app/dagster.yaml:ro \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + -it docker.io/dagster/dagster-cloud-agent:latest \ + dagster-cloud agent run /opt/dagster/app +``` + +This command: + +- Starts the agent with your local `dagster.yaml` mounted as a volume +- Starts the system Docker socket mounted as a volume, allowing the agent to launch containers. + +To view the agent in Dagster+, click the Dagster icon in the top left to navigate to the **Status** page and click the **Agents** tab. You should see the agent running in the **Agent statuses** section: + +![Instance Status](/images/dagster-cloud/agents/dagster-cloud-instance-status.png) + +## Credential Helpers + +If your images are stored in a private registry, configuring a [Docker credentials helper](https://docs.docker.com/engine/reference/commandline/login/#credential-helpers) allows the agent to log in to your registry. The agent image comes with several popular credentials helpers preinstalled: + +- [docker-credential-ecr-login](https://github.com/awslabs/amazon-ecr-credential-helper) +- [docker-credential-gcr](https://github.com/GoogleCloudPlatform/docker-credential-gcr) + +These credential helpers generally are configured in `~/.docker.config.json`. To use one, make sure you mount that file as a volume when you start your agent: + +```shell + docker run \ + --network=dagster_cloud_agent \ + --volume $PWD/dagster.yaml:/opt/dagster/app/dagster.yaml:ro \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume ~/.docker/config.json:/root/.docker/config.json:ro \ + -it docker.io/dagster/dagster-cloud-agent:latest \ + dagster-cloud agent run /opt/dagster/app +``` + +## Next steps + +Now that you've got your agent running, what's next? + +{/* - **If you're getting Dagster+ set up**, the next step is to [add a code location](/dagster-plus/managing-deployments/code-locations) using the agent. */} +- **If you're getting Dagster+ set up**, the next step is to [add a code location](/todo) using the agent. + +{/* - **If you're ready to load your Dagster code**, refer to the [Adding Code to Dagster+](/dagster-plus/managing-deployments/code-locations) guide for more info. */} +- **If you're ready to load your Dagster code**, refer to the [Adding Code to Dagster+](/todo) guide for more info. diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/configuration.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/configuration.md index f3053801d3c1c..44932d1d246ac 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/configuration.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/configuration.md @@ -4,4 +4,187 @@ sidebar_position: 200 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/kubernetes/configuration-reference */} \ No newline at end of file +:::note +This guide is applicable to Dagster+. +::: + +{/* This reference describes the various configuration options Dagster+ currently supports for [Kubernetes agents](/dagster-plus/deployment/agents/kubernetes/configuring-running-kubernetes-agent). */} +This reference describes the various configuration options Dagster+ currently supports for [Kubernetes agents](/todo). + +## Viewing the Helm chart + +To see the different customizations that can be applied to the Kubernetes agent, you can view the chart's default values: + +```shell +helm repo add dagster-plus https://dagster-io.github.io/helm-user-cloud +helm repo update +helm show values dagster-plus/dagster-plus-agent +``` + +You can also view the chart values on [ArtifactHub](https://artifacthub.io/packages/helm/dagster-cloud/dagster-cloud-agent?modal=values). + +## Per-deployment configuration + +The [`workspace`](https://artifacthub.io/packages/helm/dagster-cloud/dagster-cloud-agent?modal=values) value of the Helm chart provides the ability to add configuration for all jobs that are spun up by the agent, across all repositories. To add secrets or mounted volumes to all Kubernetes Pods, you can specify your desired configuration under this value. + +Additionally, the [`imagePullSecrets`](https://artifacthub.io/packages/helm/dagster-cloud/dagster-cloud-agent?modal=values) value allows you to specify a list of secrets that should be included when pulling the images for your containers. + +## Per-location configuration + +{/* When [adding a code location](/dagster-plus/managing-deployments/code-locations) to Dagster+ with a Kubernetes agent, you can use the `container_context` key on the location configuration to add additional Kubernetes-specific configuration. If you're using the Dagster+ Github action, the `container_context` key can also be set for each location in your `dagster_cloud.yaml` file, using the same format. */} +When [adding a code location](/todo) to Dagster+ with a Kubernetes agent, you can use the `container_context` key on the location configuration to add additional Kubernetes-specific configuration. If you're using the Dagster+ Github action, the `container_context` key can also be set for each location in your `dagster_cloud.yaml` file, using the same format. + +{/* The following example [`dagster_cloud.yaml`](/dagster-plus/managing-deployments/dagster-cloud-yaml) file illustrates the available fields: */} +The following example [`dagster_cloud.yaml`](/todo) file illustrates the available fields: + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: cloud-examples + image: dagster/dagster-cloud-examples:latest + code_source: + package_name: dagster_cloud_examples + container_context: + k8s: + env_config_maps: + - my_config_map + env_secrets: + - my_secret + env_vars: + - FOO_ENV_VAR=foo_value + - BAR_ENV_VAR + image_pull_policy: Always + image_pull_secrets: + - name: my_image_pull_secret + labels: + my_label_key: my_label_value + namespace: my_k8s_namespace + service_account_name: my_service_account_name + volume_mounts: + - mount_path: /opt/dagster/test_mount_path/volume_mounted_file.yaml + name: test-volume + sub_path: volume_mounted_file.yaml + volumes: + - name: test-volume + config_map: + name: test-volume-configmap + server_k8s_config: # Raw kubernetes config for code servers launched by the agent + pod_spec_config: # Config for the code server pod spec + node_selector: + disktype: standard + pod_template_spec_metadata: # Metadata for the code server pod + annotations: + mykey: myvalue + deployment_metadata: # Metadata for the code server deployment + annotations: + mykey: myvalue + service_metadata: # Metadata for the code server service + annotations: + mykey: myvalue + container_config: # Config for the main dagster container in the code server pod + resources: + limits: + cpu: 100m + memory: 128Mi + run_k8s_config: # Raw kubernetes config for runs launched by the agent + pod_spec_config: # Config for the run's PodSpec + node_selector: + disktype: ssd + container_config: # Config for the main dagster container in the run pod + resources: + limits: + cpu: 500m + memory: 1024Mi + pod_template_spec_metadata: # Metadata for the run pod + annotations: + mykey: myvalue + job_spec_config: # Config for the Kubernetes job for the run + ttl_seconds_after_finished: 7200 + job_metadata: # Metadata for the Kubernetes job for the run + annotations: + mykey: myvalue +``` + +### Environment variables and secrets + +Using the `container_context.k8s.env_vars` and `container_context.k8s.env_secrets` properties, you can specify environment variables and secrets for a specific code location. For example: + +```yaml +# dagster_cloud.yaml + +location: + - location_name: cloud-examples + image: dagster/dagster-cloud-examples:latest + code_source: + package_name: dagster_cloud_examples + container_context: + k8s: + env_vars: + - database_name + - database_username=hooli_testing + env_secrets: + - database_password +``` + + | Property | Description | + |---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | `env_vars` | A list of environment variable names to inject into the job, formatted as `KEY` or `KEY=VALUE`. If only `KEY` is specified, the value will be pulled from the current process. | + | `env_secrets` | A list of secret names, from which environment variables for a job are drawn using `envFrom`. Refer to the [Kubernetes documentation](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables) for more info. | + +Refer to the following guides for more info about environment variables: + +{/* - [Dagster+ environment variables and secrets](/dagster-plus/managing-deployments/environment-variables-and-secrets) */} +- [Dagster+ environment variables and secrets](/todo) +{/* - [Using environment variables and secrets in Dagster code](/guides/dagster/using-environment-variables-and-secrets) */} +- [Using environment variables and secrets in Dagster code](/todo) + +## Op isolation + +By default, each Dagster job will run in its own Kubernetes pod, with each op running in its own subprocess within the pod. + +You can also configure your Dagster job with the [`k8s_job_executor`](https://docs.dagster.io/\_apidocs/libraries/dagster-k8s#dagster_k8s.k8s_job_executor) to run each op in its own Kubernetes pod. For example: + +```python +from dagster import job +from dagster_k8s import k8s_job_executor + +@job(executor_def=k8s_job_executor) +def k8s_job(): + ... +``` + +## Per-job and per-op configuration + +{/* To add configuration to specific Dagster jobs, ops, or assets, use the `dagster-k8s/config` tag. For example, to specify that a job should have certain resource limits when it runs. Refer to [Customizing your Kubernetes deployment for Dagster Open Source](/deployment/guides/kubernetes/customizing-your-deployment#per-job-kubernetes-configuration) for more info. */} +To add configuration to specific Dagster jobs, ops, or assets, use the `dagster-k8s/config` tag. For example, to specify that a job should have certain resource limits when it runs. Refer to [Customizing your Kubernetes deployment for Dagster Open Source](/todo) for more info. + +## Running as a non-root user + +Starting in 0.14.0, the provided `docker.io/dagster/dagster-cloud-agent` image offers a non-root user with id `1001`. To run the agent with this user, you can specify the [`dagsterCloudAgent`](https://artifacthub.io/packages/helm/dagster-cloud/dagster-cloud-agent?modal=values) value in the Helm chart to be: + +```yaml +dagsterCloudAgent: + podSecurityContext: + runAsUser: 1001 +``` + +We plan to make this user the default in a future release. + +## Grant AWS permissions + +{/* You can provide your Dagster pods with [permissions to assume an AWS IAM role](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) using a [Service Account](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/). For example, you might do this to [configure an S3 IO Manager](/deployment/guides/aws#using-s3-for-io-management). */} +You can provide your Dagster pods with [permissions to assume an AWS IAM role](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) using a [Service Account](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/). For example, you might do this to [configure an S3 IO Manager](/todo). + +1. [Create an IAM OIDC provider for your EKS cluster](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) +2. [Create an IAM role and and attach IAM policies](https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html) +3. Update the [ Helm chart](#viewing-the-helm-chart) to associate the IAM role with a service account: + + ```bash + serviceAccount: + create: true + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam::1234567890:role/my_service_account_role" + ``` + +This will allow your agent and the pods it creates to assume the `my_service_account_role` IAM role. diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/dagster-cloud-cli-reference.md b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/dagster-cloud-cli-reference.md index 90d74f2c112e5..6c7521800425f 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/dagster-cloud-cli-reference.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/dagster-cloud-cli-reference.md @@ -1,7 +1,24 @@ --- title: dagster-cloud CLI reference sidebar_position: 200 -unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/managing-deployments/dagster-plus-cli#reference */} \ No newline at end of file +## Custom configuration file path + +Point the CLI at an alternate config location by specifying the `DAGSTER_CLOUD_CLI_CONFIG` environment variable. + +## Environment variables and CLI options + +Environment variables and CLI options can be used in place of or to override the CLI configuration file. + +The priority of these items is as follows: + +- **CLI options** - highest +- **Environment variables** +- **CLI configuration** - lowest + +| Setting | Environment variable | CLI flag | CLI config value | +| ------------ | ---------------------------- | ---------------------- | -------------------- | +| Organization | `DAGSTER_CLOUD_ORGANIZATION` | `--organization`, `-o` | `organization` | +| Deployment | `DAGSTER_CLOUD_DEPLOYMENT` | `--deployment`, `-d` | `default_deployment` | +| User Token | `DAGSTER_CLOUD_API_TOKEN` | `--user-token`, `-u` | `user_token` | diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/installing-and-configuring.md b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/installing-and-configuring.md index ff5280c31f89c..faf818e9e88c3 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/installing-and-configuring.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/installing-and-configuring.md @@ -1,8 +1,131 @@ --- title: Installing and configuring the dagster-cloud CLI sidebar_position: 100 -unlisted: true --- -{/* TODO copy from "Installing the CLI" and "Configuring the CLI" sections of https://docs.dagster.io/dagster-plus/managing-deployments/dagster-plus-cli */} +:::note +This guide is applicable to Dagster+. +::: +The `dagster-cloud` CLI is a command-line toolkit designed to work with Dagster+. + +In this guide, we'll cover how to install and configure the `dagster-cloud` CLI, get help, and use some helpful environment variables and CLI options. + +## Installing the CLI + +The Dagster+ Agent library is available in PyPi. To install, run: + +```shell +pip install dagster-cloud +``` + +Refer to the [configuration section](#configuring-the-cli) for next steps. + +### Completions + +Optionally, you can install command-line completions to make using the `dagster-cloud` CLI easier. + +To have the CLI install these completions to your shell, run: + +```shell +dagster-cloud --install-completion +``` + +To print out the completion for copying or manual installation: + +```shell +dagster-cloud --show-completion +``` + +## Configuring the CLI + +The recommended way to set up your CLI's config for long-term use is through the configuration file, located by default at `~/.dagster_cloud_cli/config`. + +### Setting up the configuration file + +Set up the config file: + +```shell +dagster-cloud config setup +``` + +Select your authentication method. **Note**: Browser authentication is the easiest method to configure. + +
    +BROWSER AUTHENTICATION + +The easiest way to set up is to authenticate through the browser. + +```shell +$ dagster-cloud config setup +? How would you like to authenticate the CLI? (Use arrow keys) + » Authenticate in browser + Authenticate using token +Authorized for organization `hooli` + +? Default deployment: prod +``` + +When prompted, you can specify a default deployment. If specified, a deployment won't be required in subsequent `dagster-cloud` commands. The default deployment for a new Dagster+ organization is `prod`. + +
    + +
    +TOKEN AUTHENTICATION + +Alternatively, you may authenticate using a user token. Refer to the [User tokens guide](/dagster-plus/deployment/management/tokens/user-tokens) for more info. + +```shell +$ dagster-cloud config setup +? How would you like to authenticate the CLI? (Use arrow keys) + Authenticate in browser + » Authenticate using token + +? Dagster+ organization: hooli +? Dagster+ user token: ************************************* +? Default deployment: prod +``` + +When prompted, specify the following: + +- **Organization** - Your organization name as it appears in your Dagster+ URL. For example, if your Dagster+ instance is `https://hooli.dagster.cloud/`, this would be `hooli`. +- **User token** - The user token. +- **Default deployment** - **Optional**. A default deployment. If specified, a deployment won't be required in subsequent `dagster-cloud` commands. The default deployment for a new Dagster+ organization is `prod`. + +
    + +### Viewing and modifying the configuration file + +To view the contents of the CLI configuration file, run: + +```shell +$ dagster-cloud config view + +default_deployment: prod +organization: hooli +user_token: '*******************************8214fe' +``` + +Specify the `--show-token` flag to show the full user token. + +To modify the existing config, re-run: + +```shell +dagster-cloud config setup +``` + +## Toggling between deployments + +To quickly toggle between deployments, run: + +```shell +dagster-cloud config set-deployment +``` + +## Getting help + +To view help options in the CLI: + +```shell +dagster-cloud --help +``` diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.md b/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.md deleted file mode 100644 index a800e113a83ba..0000000000000 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: "Set environment variables using agent config" -sidebar_position: 300 -sidebar_label: "Set with agent config" -unlisted: true ---- - -{/* TODO move from https://docs.dagster.io/dagster-plus/managing-deployments/setting-environment-variables-agents */} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.mdx b/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.mdx new file mode 100644 index 0000000000000..29d8d0b845016 --- /dev/null +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.mdx @@ -0,0 +1,262 @@ +--- +title: "Set environment variables using agent config" +sidebar_position: 300 +sidebar_label: "Set with agent config" +--- + +:::note +This guide is applicable to Dagster+. +::: + +In this guide, we'll walk you through setting environment variables for a Dagster+ [Hybrid deployment](/dagster-plus/deployment/deployment-types/hybrid) using the Hybrid agent's configuration. + +There are two ways to set environment variables: + +- **On a per-code location basis**, which involves modifying the `dagster_cloud.yaml` file. **Note**: This approach is functionally the same as [setting environment variables using the Dagster+ UI](/dagster-plus/deployment/management/environment-variables/dagster-ui). Values will pass through Dagster+. +- **For a full deployment and all the code locations it contains**. This approach makes variables available for all code locations in a full Dagster+ deployment. As values are pulled from the user cluster, values will bypass Dagster+ entirely. + +## Prerequisites + +To complete the steps in this guide, you'll need: + +- A Dagster+ account using [Hybrid deployment](/dagster-plus/deployment/deployment-types/hybrid/) +- An existing [Hybrid agent](/dagster-plus/deployment/deployment-types/hybrid/#dagster-hybrid-agents) +- **Editor**, **Admin**, or **Organization Admin** permissions in Dagster+ + +## Setting environment variables for a code location + +:::note + To set environment variables, you need one of the following user roles in Dagster+: + +::: + +Setting environment variables for specific code locations is accomplished by adding them to your agent's configuration in your project's [`dagster_cloud.yaml` file](/dagster-plus/deployment/management/settings/). The `container_context` property in this file sets the variables in the agent's environment. + +**Note**: This approach is functionally the same as [setting environment variables using the Dagster+ UI](/dagster-plus/deployment/management/environment-variables/dagster-ui). + +How `container_context` is configured depends on the agent type. Click the tab for your agent type to view instructions. + + + + +### Amazon ECS agents + +Using the `container_context.ecs.env_vars` and `container_context.ecs.secrets` properties, you can configure environment variables and secrets for a specific code location. + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: cloud-examples + image: dagster/dagster-cloud-examples:latest + code_source: + package_name: dagster_cloud_examples + container_context: + ecs: + env_vars: + - DATABASE_NAME=testing + - DATABASE_PASSWORD + secrets: + - name: "MY_API_TOKEN" + valueFrom: "arn:aws:secretsmanager:us-east-1:123456789012:secret:FOO-AbCdEf:token::" + - name: "MY_PASSWORD" + valueFrom: "arn:aws:secretsmanager:us-east-1:123456789012:secret:FOO-AbCdEf:password::" + secrets_tags: + - "my_tag_name" +``` + +| Key | Description | +|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `container_context.ecs.env_vars` | A list of keys or key-value pairs. If a value is not specified, it pulls from the agent task. E.g., `FOO_ENV_VAR` = `foo_value`, `BAR_ENV_VAR` = agent task value. | +| `container_context.ecs.secrets` | Individual secrets using the [ECS API structure](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Secret.html). | +| `container_context.ecs.secrets_tags` | A list of tag names; secrets tagged with these in AWS Secrets Manager will be environment variables. The variable name is the secret name, the value is the secret's value. | + +After you've modified `dagster_cloud.yaml`, redeploy the code location in Dagster+ to apply the changes: + +!["Highlighted Redeploy option in the dropdown menu next to a code location in Dagster+"](/images/dagster-cloud/developing-testing/code-locations/redeploy-code-location.png) + + + + +### Docker agents + +Using the `container_context.docker.env_vars` property, you can include environment variables and secrets in the Docker container associated with a specific code location. For example: + +```yaml +# dagster_cloud.yaml +locations: + - location_name: cloud-examples + image: dagster/dagster-cloud-examples:latest + code_source: + package_name: dagster_cloud_examples + container_context: + docker: + env_vars: + - DATABASE_NAME + - DATABASE_USERNAME=hooli_testing +``` + +The `container_context.docker.env_vars` property is a list, where each item can be either `KEY` or `KEY=VALUE`. If only `KEY` is specified, the value will be pulled from the local environment. + +After you've modified `dagster_cloud.yaml`, redeploy the code location in Dagster+ to apply the changes: + +![Highlighted Redeploy option in the dropdown menu next to a code location in Dagster+](/images/dagster-cloud/developing-testing/code-locations/redeploy-code-location.png) + + + + +### Kubernetes agents + +Using the `container_context.k8s.env_vars` and `container_context.k8s.env_secrets` properties, you can specify environment variables and secrets for a specific code location. For example: + +```yaml +# dagster_cloud.yaml + +locations: + - location_name: cloud-examples + image: dagster/dagster-cloud-examples:latest + code_source: + package_name: dagster_cloud_examples + container_context: + k8s: + env_vars: + - database_name # value pulled from agent's environment + - database_username=hooli_testing + env_secrets: + - database_password +``` + + | Key | Description | + |---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | `env_vars` | A list of environment variable names to inject into the job, formatted as KEY or KEY=VALUE. If only KEY is specified, the value will be pulled from the current process. | + | `env_secrets` | A list of secret names, from which environment variables for a job are drawn using envFrom. Refer to the Kubernetes documentation for more info. | + + +After you've modified `dagster_cloud.yaml`, redeploy the code location in Dagster+ to apply the changes: + +![Highlighted Redeploy option in the dropdown menu next to a code location in Dagster+](/images/dagster-cloud/developing-testing/code-locations/redeploy-code-location.png) + + + + +## Setting environment variables for full deployments + +:::note + If you're a Dagster+ Editor or + Admin , you can only set environment variables for full deployments where + you're an Editor + or Admin. +::: + +Setting environment variables for a full deployment will make the variables available for all code locations in the full deployment. Using this approach will pull variable values from your user cluster, bypassing Dagster+ entirely. + +Click the tab for your agent type to view instructions. + + + + +### Amazon ECS agents + +To make environment variables accessible to a full deployment with an Amazon ECS agent, you'll need to modify the agent's CloudFormation template as follows: + +1. Sign in to your AWS account. + +2. Navigate to **CloudFormation** and open the stack for the agent. + +3. Click **Update**. + +4. Click **Edit template in designer**. + +5. In the section that displays, click **View in Designer**. The AWS template designer will display. + +6. In the section displaying the template YAML, locate the `AgentTaskDefinition` section: + + ![Highlighted AgentTaskDefinition section of the AWS ECS agent CloudFormation template in the AWS Console](/images/dagster-cloud/developing-testing/environment-variables/aws-ecs-cloudformation-template.png) + + +7. In the `user_code_launcher.config` portion of the `AgentTaskDefinition` section, add the environment variables as follows: + + ```yaml + user_code_launcher: + module: dagster_cloud.workspace.ecs + class: EcsUserCodeLauncher + config: + cluster: ${ConfigCluster} + subnets: [${ConfigSubnet}] + service_discovery_namespace_id: ${ServiceDiscoveryNamespace} + execution_role_arn: ${TaskExecutionRole.Arn} + task_role_arn: ${AgentRole} + log_group: ${AgentLogGroup} + env_vars: + - SNOWFLAKE_USERNAME=dev + - SNOWFLAKE_PASSWORD ## pulled from agent environment + ' > $DAGSTER_HOME/dagster.yaml && cat $DAGSTER_HOME/dagster.yaml && dagster-cloud agent run" + ``` + +8. When finished, click the **Create Stack** button: + + ![Highlighted Create Stack button in the AWS Console](/images/dagster-cloud/developing-testing/environment-variables/aws-ecs-save-template.png) + +9. You'll be redirected back to the **Update stack** wizard, where the new template will be populated. Click **Next**. + +10. Continue to click **Next** until you reach the **Review** page. + +11. Click **Submit** to update the stack. + + + + +### Docker agents + +To make environment variables accessible to a full deployment with a Docker agent, you'll need to modify your project's `dagster.yaml` file. + +In the `user_code_launcher` section, add an `env_vars` property as follows: + +```yaml +# dagster.yaml + +user_code_launcher: + module: dagster_cloud.workspace.docker + class: DockerUserCodeLauncher + config: + networks: + - dagster_cloud_agent + env_vars: + - SNOWFLAKE_PASSWORD # value pulled from agent's environment + - SNOWFLAKE_USERNAME=dev +``` + +In `env_vars`, specify the environment variables as keys (`SNOWFLAKE_PASSWORD`) or key-value pairs (`SNOWFLAKE_USERNAME=dev`). If only `KEY` is provided, the value will be pulled from the agent's environment. + + + + +### Kubernetes agents + +To make environment variables available to a full deployment with a Kubernetes agent, you'll need to modify and upgrade the Helm chart's `values.yaml`. + +1. In `values.yaml`, add or locate the `workspace` value. + +2. Add an `envVars` property as follows: + + ```yaml + # values.yaml + + workspace: + envVars: + - SNOWFLAKE_PASSWORD # value pulled from agent's environment + - SNOWFLAKE_USERNAME=dev + ``` + +3. In `envVars`, specify the environment variables as keys (`SNOWFLAKE_PASSWORD`) or key-value pairs (`SNOWFLAKE_USERNAME=dev`). If only `KEY` is provided, the value will be pulled from the local (agent's) environment. + +4. Upgrade the Helm chart. + + + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/managing-compute-logs-and-error-messages.md b/docs/docs-beta/docs/dagster-plus/deployment/management/managing-compute-logs-and-error-messages.md index 01279273021f2..ec61dc4fbbee3 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/managing-compute-logs-and-error-messages.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/managing-compute-logs-and-error-messages.md @@ -3,4 +3,106 @@ title: Managing compute logs and error messages unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/managing-deployments/controlling-logs */} \ No newline at end of file +import ThemedImage from '@theme/ThemedImage'; + +:::note +This guide is applicable to Dagster+. +::: + +In this guide, we'll cover how to adjust where Dagster+ compute logs are stored and manage masking of error messages in the Dagster+ UI. + +{/* By default, Dagster+ ingests [structured event logs and compute logs](/concepts/logging#log-types) from runs and surfaces error messages from [code locations](/guides/deploy/code-locations/) in the UI. */} +By default, Dagster+ ingests [structured event logs and compute logs](/todo) from runs and surfaces error messages from [code locations](/todo) in the UI. + +Depending on your organization's needs, you may want to retain these logs in your own infrastructure or mask error message contents. + +--- + +## Modifying compute log storage + +Dagster's compute logs are handled by the configured [`ComputeLogManager`](/api/python-api/internals#compute-log-manager). By default, Dagster+ utilizes the `CloudComputeLogManager` which stores logs in a Dagster+-managed Amazon S3 bucket, but you can customize this behavior to store logs in a destination of your choice. + +### Writing to your own S3 bucket + +If using the Kubernetes agent, you can instead forward logs to your own S3 bucket by using the [`S3ComputeLogManager`](/api/python-api/libraries/dagster-aws#dagster_aws.s3.S3ComputeLogManager). + +{/* You can configure the `S3ComputeLogManager` in your [`dagster.yaml` file](/dagster-plus/deployment/agents/customizing-configuration): */} +You can configure the `S3ComputeLogManager` in your [`dagster.yaml` file](/todo): + +```yaml +compute_logs: + module: dagster_aws.s3.compute_log_manager + class: S3ComputeLogManager + config: + show_url_only: true + bucket: your-compute-log-storage-bucket + region: your-bucket-region +``` + +If you are using Helm to deploy the Kubernetes agent, you can provide the following configuration in your `values.yaml` file: + +```yaml +computeLogs: + enabled: true + custom: + module: dagster_aws.s3.compute_log_manager + class: S3ComputeLogManager + config: + show_url_only: true + bucket: your-compute-log-storage-bucket + region: your-bucket-region +``` + +### Disabling compute log upload + +If your organization has its own logging solution which ingests `stdout` and `stderr` from your compute environment, you may want to disable compute log upload entirely. You can do this with the . + +{/* You can configure the `NoOpComputeLogManager` in your [`dagster.yaml` file](/dagster-plus/deployment/agents/customizing-configuration): */} +You can configure the `NoOpComputeLogManager` in your [`dagster.yaml` file](/todo): + +```yaml +compute_logs: + module: dagster.core.storage.noop_compute_log_manager + class: NoOpComputeLogManager +``` + +If you are using Helm to deploy the Kubernetes agent, use the `enabled` flag to disable compute log upload: + +```yaml +computeLogs: + enabled: false +``` + +### Other compute log storage options + +{/* For a full list of available compute log storage options, refer to the [Compute log storage docs](/deployment/dagster-instance#compute-log-storage). */} +For a full list of available compute log storage options, refer to the [Compute log storage docs](/todo). + +--- + +## Masking error messages + +By default, Dagster+ surfaces error messages from your code locations in the UI, including when runs fail, sensors or schedules throw an exception, or code locations fail to load. You can mask these error messages in the case that their contents are sensitive. + +To mask error messages in a Dagster+ Deployment, set the environment variable `DAGSTER_REDACT_USER_CODE_ERRORS` equal to `1` using the [**Environment variables** page](/dagster-plus/deployment/management/environment-variables/) in the UI: + + + +Once set, error messages from your code locations will be masked in the UI. A unique error ID will be generated, which you can use to look up the error message in your own logs. This error ID will appear in place of the error message in UI dialogs or in a run's event logs. + + + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/managing-multiple-projects-and-teams.md b/docs/docs-beta/docs/dagster-plus/deployment/management/managing-multiple-projects-and-teams.md index cc59bd13364a9..6b88b8104a216 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/managing-multiple-projects-and-teams.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/managing-multiple-projects-and-teams.md @@ -1,6 +1,393 @@ --- title: Managing multiple projects and teams -unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/best-practices/managing-multiple-projects-and-teams */} \ No newline at end of file +In this guide, we'll cover some strategies for managing multiple projects/code bases and teams in a Dagster+ account. + +## Separating code bases + +:::note +In this section, repository refers to a version control system, such as Git or Mercurial. +::: + +If you want to manage complexity or divide your work into areas of responsibility, consider isolating your code bases into multiple projects with: + +- Multiple directories in a single repository, or +- Multiple repositories + +Refer to the following table for more information, including the pros and cons of each approach. + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Approach + + Multiple directories in a single repository + Multiple repositories
    + How it works + + You can use a single repository to manage multiple projects by placing + each project in a separate directory. Depending on your VCS, you may be + able to set{" "} + + code owners + {" "} + to restrict who can modify each project. + + For stronger isolation, you can use multiple repositories to manage + multiple projects. +
    + Pros + +
      +
    • + Simple to implement +
      • +
    • Facilitates code sharing between projects
      • +
    +     		+
      +
    • + Stronger isolation between projects and teams +
      • +
    • + Each project has its own CI/CD pipeline and be deployed + independently +
      • +
    • Dependencies between projects can be managed independently
      • +
    +
    + Cons + +
      +
    • + All projects share the same CI/CD pipeline and cannot be deployed + independently +
      • +
    • + Shared dependencies between projects may cause conflicts and require + coordination between teams +
      • +
    +     		+ Code sharing between projects require additional coordination to publish + and reuse packages between projects +
    + +### Deployment configuration + +{/* Whether you use a single repository or multiple, you can use a [`dagster_cloud.yaml` file](/dagster-plus/managing-deployments/dagster-cloud-yaml) to define the code locations to deploy. For each repository, follow the [steps appropriate to your CI/CD provider](/dagster-plus/getting-started#step-4-configure-cicd-for-your-project) and include only the code locations that are relevant to the repository in your CI/CD workflow. */} +Whether you use a single repository or multiple, you can use a [`dagster_cloud.yaml` file](/todo) to define the code locations to deploy. For each repository, follow the [steps appropriate to your CI/CD provider](/todo) and include only the code locations that are relevant to the repository in your CI/CD workflow. + +#### Example with GitHub CI/CD on Hybrid deployment + +1. **For each repository**, use the CI/CD workflow provided in [Dagster+ Hybrid quickstart repository](https://github.com/dagster-io/dagster-cloud-hybrid-quickstart/blob/main/.github/workflows/dagster-cloud-deploy.yml). + +{/* 2. **For each project in the repository**, configure a code location in the [`dagster_cloud.yaml` file](/dagster-plus/managing-deployments/dagster-cloud-yaml): */} +2. **For each project in the repository**, configure a code location in the [`dagster_cloud.yaml` file](/todo): + + ```yaml + # dagster_cloud.yml + + locations: + - location_name: project_a + code_source: + package_name: project_a + build: + # ... + - location_name: project_b + code_source: + package_name: project_b + build: + # ... + ``` + +3. In the repository's `dagster-cloud-deploy.yml` file, modify the CI/CD workflow to deploy all code locations for the repository: + + ```yaml + # .github/workflows/dagster-cloud-deploy.yml + + jobs: + dagster-cloud-deploy: + # ... + steps: + - name: Update build session with image tag for "project_a" code location + id: ci-set-build-output-project-a + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci set-build-output --location-name=project_a --image-tag=$IMAGE_TAG" + + - name: Update build session with image tag for "project_b" code location + id: ci-set-build-output-project-b + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci set-build-output --location-name=project_b --image-tag=$IMAGE_TAG" + # ... + ``` + +--- + +## Isolating execution context between projects + +Separating execution context between projects can have several motivations: + +- Facilitating separation of duty between teams to prevent access to sensitive data +- Differing compute environments and requirements, such as different architecture, cloud provider, etc. +- Reducing impact on other projects. For example, a project with a large number of runs can impact the performance of other projects. + +In order from least to most isolated, there are three levels of isolation: + +- [Code location](#code-location-isolation) +- [Agent](#agent-isolation) +- [Deployment](#deployment-isolation) + +### Code location isolation + +If you have no specific requirements for isolation beyond the ability to deploy and run multiple projects, you can use a single agent and deployment to manage all your projects as individual code locations. + +![Diagram of isolation at the code location level](/images/dagster-cloud/managing-deployments/isolation-level-code-locations.png) + + + + + + + + + + + + + + +
    + Pros + Cons
    +
      +
    • + Simplest and most cost-effective solution +
      • +
    • User access control can be set at the code location level
      • +
    • Single glass pane to view all assets
      • +
    +     		+
      +
    • + No isolation between execution environments +
      • +
    +
    + +### Agent isolation + +:::note +Agent queues are a Dagster+ Pro feature available on hybrid deployment. +::: + +{/* Using the [agent routing feature](/dagster-plus/deployment/agents/running-multiple-agents#routing-requests-to-specific-agents), you can effectively isolate execution environments between projects by using a separate agent for each project. */} +Using the [agent routing feature](/todo), you can effectively isolate execution environments between projects by using a separate agent for each project. + +Motivations for utilizing this approach could include: + +- Different compute requirements, such as different cloud providers or architectures +- Optimizing for locality or access, such as running the data processing closer or in environment with access to the storage locations + +![Diagram of isolation at the agent level](/images/dagster-cloud/managing-deployments/isolation-level-agents.png) + + + + + + + + + + + + + + +
    + Pros + Cons
    +
      +
    • + Isolation between execution environments +
      • +
    • User access control can be set at the code location level
      • +
    • Single glass pane to view all assets
      • +
    +     		Extra work to set up additional agents and agent queues
    + +### Deployment isolation + +:::note +Multiple deployments are only available in Dagster+ Pro. +::: + +Of the approaches outlined in this guide, multiple deployments are the most isolated solution. The typical motivation for this isolation level is to separate production and non-production environments. It may be considered to satisfy other organization specific requirements. + +![Diagram of isolation at the Dagster+ deployment level](/images/dagster-cloud/managing-deployments/isolation-level-deployments.png) + + + + + + + + + + + + + + +
    + Pros + Cons
    +
      +
    • + Isolation between assets and execution environments +
      • +
    • + User access control can be set at the code location and deployment + level +
      • +
    +     		+ No single glass pane to view all assets (requires switching between + multiple deployments in the UI) +
    + + +{/* +## Related + +- [Dagster+ Hybrid deployments](/dagster-plus/deployment/hybrid) +- [Dagster+ Hybrid agents](/dagster-plus/deployment/agents) +- [Managing deployments in Dagster+](/dagster-plus/managing-deployments) +(/* - [Running multiple Dagster+ Hybrid agents](/dagster-plus/deployment/agents/running-multiple-agents#routing-requests-to-specific-agents) */} +- [Running multiple Dagster+ Hybrid agents](/todo) +{/* - [dagster_cloud.yaml](/dagster-plus/managing-deployments/dagster-cloud-yaml) */} +- [dagster_cloud.yaml](/todo) diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/rate-limits.md b/docs/docs-beta/docs/dagster-plus/deployment/management/rate-limits.md index b4364e74ff276..7c85eb8fdaa11 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/rate-limits.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/rate-limits.md @@ -1,6 +1,13 @@ --- title: Dagster+ rate limits -unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/references/limits */} \ No newline at end of file +Dagster+ enforces several rate limits to smoothly distribute the load. Deployments are limited to: + +- 40,000 user log events (e.g, `context.log.info`) per minute. This limit only applies to custom logs; system events like the ones that drive orchestration or materialize assets are not subject to this limit. +- 35MB of events per minute. This limit applies to both custom events and system events. + +Rate-limited requests return a "429 - Too Many Requests" response. Dagster+ agents automatically retry these requests. + +{/* Switching from [Structured event logs](/concepts/logging#structured-event-logs) to [Raw compute logs](/concepts/logging#raw-compute-logs) or reducing your custom log volume can help you stay within these limits. */} +Switching from [Structured event logs](/todo) to [Raw compute logs](/todo) or reducing your custom log volume can help you stay within these limits. diff --git a/docs/docs-beta/docs/dagster-plus/features/insights/export-metrics.md b/docs/docs-beta/docs/dagster-plus/features/insights/export-metrics.md index 219eee2060f41..1944b206519b8 100644 --- a/docs/docs-beta/docs/dagster-plus/features/insights/export-metrics.md +++ b/docs/docs-beta/docs/dagster-plus/features/insights/export-metrics.md @@ -5,3 +5,153 @@ sidebar_position: 200 unlisted: true --- +{/* Using a GraphQL API endpoint, you can export [Dagster+ Insights](/dagster-plus/insights) metrics from your Dagster+ instance. */} +Using a GraphQL API endpoint, you can export [Dagster+ Insights](/todo) metrics from your Dagster+ instance. + +{/* Refer to the [Available Insights metrics](/dagster-plus/insights#available-metrics) for a list of available metrics. */} +Refer to the [Available Insights metrics](/todo) for a list of available metrics. + +## Prerequisites + +To complete the steps in this guide, you'll need: + +- A Dagster+ account +{/* - Access to the [Dagster+ Insights feature](/dagster-plus/insights) */} +- Access to the [Dagster+ Insights feature](/todo) +{/* - A Dagster+ [user token](/dagster-plus/account/managing-user-agent-tokens#managing-user-tokens) */} +- A Dagster+ [user token](/todo) +- Your deployment-scoped Dagster+ deployment URL. For example: `dagster-university.dagster.cloud/prod` + +## Before you start + +Before you start, note that: + +- Metrics are currently computed once per day +- We don't recommend frequently querying over large time ranges that may download a large amount of data. After an initial data load, we recommend loading data daily for the most recent week or less. + +## Using the API + +{/* In this example, we're using the [GraphQL Python Client](/concepts/webserver/graphql-client) to export the Dagster credits metric for all assets for September 2023: */} +In this example, we're using the [GraphQL Python Client](/todo) to export the Dagster credits metric for all assets for September 2023: + +```python +from datetime import datetime +from dagster_graphql import DagsterGraphQLClient + +ASSET_METRICS_QUERY = """ +query AssetMetrics($metricName: String, $after: Float, $before: Float) { + reportingMetricsByAsset( + metricsSelector: { + metricName: $metricName + after: $after + before: $before + sortAggregationFunction: SUM + granularity: DAILY + } + ) { + __typename + ... on ReportingMetrics { + metrics { + values + entity { + ... on ReportingAsset { + assetKey { + path + } + } + } + } + } + } +} + +""" + + +def get_client(): + url = "YOUR_ORG.dagster.cloud/prod" # Your deployment-scoped url + user_token = "YOUR_TOKEN" # A token generated from Organization Settings > Tokens + return DagsterGraphQLClient(url, headers={"Dagster-Cloud-Api-Token": user_token}) + + +if __name__ == "__main__": + client = get_client() + result = client._execute( + ASSET_METRICS_QUERY, + { + "metricName": "__dagster_dagster_credits", + "after": datetime(2023, 9, 1).timestamp(), + "before": datetime(2023, 10, 1).timestamp(), + }, + ) + + for asset_series in result["reportingMetricsByAsset"]["metrics"]: + print("Asset key:", asset_series["entity"]["assetKey"]["path"]) + print("Daily values:", asset_series["values"]) + +``` + +To use this example yourself, replace the values of `url` and `user_token` in this function: + +```python +def get_client(): + url = "YOUR_ORG.dagster.cloud/prod" # Your deployment-scoped url + user_token = "YOUR_TOKEN" # A token generated from Organization Settings > Tokens + return DagsterGraphQLClient(url, headers={"Dagster-Cloud-Api-Token": user_token}) +``` + +Refer to the [Reference section](#reference) for more info about the endpoints available in the GraphQL API. + +## Reference + +For the full GraphQL API reference: + +1. Navigate to `YOUR_ORG.dagster.cloud/prod/graphql`, replacing `YOUR_ORG` with your organization name. For example: `https://dagster-university.dagster.cloud/prod/graphql` +2. Click the **Schema** tab. + +### Available top-level queries + +```graphql +reportingMetricsByJob( + metricsFilter: JobReportingMetricsFilter + metricsSelector: ReportingMetricsSelector! +): ReportingMetricsOrError! + +reportingMetricsByAsset( + metricsFilter: AssetReportingMetricsFilter + metricsSelector: ReportingMetricsSelector! +): ReportingMetricsOrError! + +reportingMetricsByAssetGroup( + metricsFilter: AssetGroupReportingMetricsFilter + metricsSelector: ReportingMetricsSelector! +): ReportingMetricsOrError! +``` + +### Specifying metrics and time granularity + +Use `metricsSelector` to specify the metric name and time granularity: + +```graphql +input ReportingMetricsSelector { + after: Float # timestamp + before: Float # timestamp + metricName: String # see below for valid values + granularity: ReportingMetricsGranularity +} + +enum ReportingMetricsGranularity { + DAILY + WEEKLY + MONTHLY +} + +# The valid metric names are: +# "__dagster_dagster_credits" +# "__dagster_execution_time_ms" +# "__dagster_materializations" +# "__dagster_step_failures" +# "__dagster_step_retries" +# "__dagster_asset_check_errors" +# "__dagster_asset_check_warnings" +``` diff --git a/docs/docs-beta/static/images/dagster-cloud/agents/dagster-cloud-instance-status.png b/docs/docs-beta/static/images/dagster-cloud/agents/dagster-cloud-instance-status.png new file mode 100644 index 0000000000000..507a67f8f8ad9 Binary files /dev/null and b/docs/docs-beta/static/images/dagster-cloud/agents/dagster-cloud-instance-status.png differ diff --git a/docs/docs-beta/static/images/dagster-cloud/developing-testing/code-locations/redeploy-code-location.png b/docs/docs-beta/static/images/dagster-cloud/developing-testing/code-locations/redeploy-code-location.png new file mode 100644 index 0000000000000..c89e91758ea16 Binary files /dev/null and b/docs/docs-beta/static/images/dagster-cloud/developing-testing/code-locations/redeploy-code-location.png differ diff --git a/docs/docs-beta/static/images/dagster-cloud/developing-testing/environment-variables/aws-ecs-cloudformation-template.png b/docs/docs-beta/static/images/dagster-cloud/developing-testing/environment-variables/aws-ecs-cloudformation-template.png new file mode 100644 index 0000000000000..9b223bc5fccbc Binary files /dev/null and b/docs/docs-beta/static/images/dagster-cloud/developing-testing/environment-variables/aws-ecs-cloudformation-template.png differ diff --git a/docs/docs-beta/static/images/dagster-cloud/developing-testing/environment-variables/aws-ecs-save-template.png b/docs/docs-beta/static/images/dagster-cloud/developing-testing/environment-variables/aws-ecs-save-template.png new file mode 100644 index 0000000000000..399fe9b9b415e Binary files /dev/null and b/docs/docs-beta/static/images/dagster-cloud/developing-testing/environment-variables/aws-ecs-save-template.png differ diff --git a/docs/docs-beta/static/images/dagster-cloud/logs-error-messages/configure-redact-env-var.png b/docs/docs-beta/static/images/dagster-cloud/logs-error-messages/configure-redact-env-var.png new file mode 100644 index 0000000000000..4efe4e0b654a9 Binary files /dev/null and b/docs/docs-beta/static/images/dagster-cloud/logs-error-messages/configure-redact-env-var.png differ diff --git a/docs/docs-beta/static/images/dagster-cloud/logs-error-messages/masked-err-message.png b/docs/docs-beta/static/images/dagster-cloud/logs-error-messages/masked-err-message.png new file mode 100644 index 0000000000000..d5e27296cc0bb Binary files /dev/null and b/docs/docs-beta/static/images/dagster-cloud/logs-error-messages/masked-err-message.png differ diff --git a/docs/docs-beta/static/images/dagster-cloud/managing-deployments/isolation-level-agents.png b/docs/docs-beta/static/images/dagster-cloud/managing-deployments/isolation-level-agents.png new file mode 100644 index 0000000000000..2fd662c04df8a Binary files /dev/null and b/docs/docs-beta/static/images/dagster-cloud/managing-deployments/isolation-level-agents.png differ diff --git a/docs/docs-beta/static/images/dagster-cloud/managing-deployments/isolation-level-code-locations.png b/docs/docs-beta/static/images/dagster-cloud/managing-deployments/isolation-level-code-locations.png new file mode 100644 index 0000000000000..558b74df3d244 Binary files /dev/null and b/docs/docs-beta/static/images/dagster-cloud/managing-deployments/isolation-level-code-locations.png differ diff --git a/docs/docs-beta/static/images/dagster-cloud/managing-deployments/isolation-level-deployments.png b/docs/docs-beta/static/images/dagster-cloud/managing-deployments/isolation-level-deployments.png new file mode 100644 index 0000000000000..27cc815b9bf44 Binary files /dev/null and b/docs/docs-beta/static/images/dagster-cloud/managing-deployments/isolation-level-deployments.png differ diff --git a/docs/next/.versioned_content/_versions_with_static_links.json b/docs/next/.versioned_content/_versions_with_static_links.json index 5b12b5f2ea18d..9b2fdbc33b793 100644 --- a/docs/next/.versioned_content/_versions_with_static_links.json +++ b/docs/next/.versioned_content/_versions_with_static_links.json @@ -610,5 +610,9 @@ { "url": "https://release-1-9-5.dagster.dagster-docs.io/", "version": "1.9.5" + }, + { + "url": "https://release-1-9-6.dagster.dagster-docs.io/", + "version": "1.9.6" } ] \ No newline at end of file diff --git a/helm/dagster/templates/deployment-celery-queues.yaml b/helm/dagster/templates/deployment-celery-queues.yaml index c2f803376c99f..fdb016523066d 100644 --- a/helm/dagster/templates/deployment-celery-queues.yaml +++ b/helm/dagster/templates/deployment-celery-queues.yaml @@ -42,7 +42,7 @@ spec: initContainers: {{- if $celeryK8sRunLauncherConfig.checkDbReadyInitContainer }} - name: check-db-ready - image: "{{- $.Values.postgresql.image.repository -}}:{{- $.Values.postgresql.image.tag -}}" + image: {{ include "dagster.externalPostgresImage.name" $.Values.postgresql.image | quote }} imagePullPolicy: "{{- $.Values.postgresql.image.pullPolicy -}}" command: ['sh', '-c', {{ include "dagster.postgresql.pgisready" $ | squote }}] securityContext: diff --git a/helm/dagster/templates/deployment-daemon.yaml b/helm/dagster/templates/deployment-daemon.yaml index 91b86b5c7ec73..c63f770463254 100644 --- a/helm/dagster/templates/deployment-daemon.yaml +++ b/helm/dagster/templates/deployment-daemon.yaml @@ -51,7 +51,7 @@ spec: initContainers: {{- if .Values.dagsterDaemon.checkDbReadyInitContainer }} - name: check-db-ready - image: {{ include "dagster.externalImage.name" $.Values.postgresql.image | quote }} + image: {{ include "dagster.externalPostgresImage.name" $.Values.postgresql.image | quote }} imagePullPolicy: "{{- $.Values.postgresql.image.pullPolicy -}}" command: ['sh', '-c', {{ include "dagster.postgresql.pgisready" . | squote }}] securityContext: diff --git a/helm/dagster/templates/deployment-flower.yaml b/helm/dagster/templates/deployment-flower.yaml index 63b9c7d44a515..ca0ed535362ab 100644 --- a/helm/dagster/templates/deployment-flower.yaml +++ b/helm/dagster/templates/deployment-flower.yaml @@ -38,7 +38,7 @@ spec: initContainers: {{- if .Values.flower.checkDbReadyInitContainer }} - name: check-db-ready - image: "{{- $.Values.postgresql.image.repository -}}:{{- $.Values.postgresql.image.tag -}}" + image: {{ include "dagster.externalPostgresImage.name" $.Values.postgresql.image | quote }} imagePullPolicy: "{{- $.Values.postgresql.image.pullPolicy -}}" command: ['sh', '-c', {{ include "dagster.postgresql.pgisready" . | squote }}] securityContext: diff --git a/helm/dagster/templates/helpers/_deployment-webserver.tpl b/helm/dagster/templates/helpers/_deployment-webserver.tpl index 9427646bf4999..db97496d81408 100644 --- a/helm/dagster/templates/helpers/_deployment-webserver.tpl +++ b/helm/dagster/templates/helpers/_deployment-webserver.tpl @@ -49,7 +49,7 @@ spec: initContainers: {{- if .Values.dagsterWebserver.checkDbReadyInitContainer }} - name: check-db-ready - image: {{ include "dagster.externalImage.name" .Values.postgresql.image | quote }} + image: {{ include "dagster.externalPostgresImage.name" .Values.postgresql.image | quote }} imagePullPolicy: {{ .Values.postgresql.image.pullPolicy }} command: ['sh', '-c', {{ include "dagster.postgresql.pgisready" . | squote }}] securityContext: diff --git a/helm/dagster/templates/helpers/_helpers.tpl b/helm/dagster/templates/helpers/_helpers.tpl index bc654a1d1a839..33d862b260f0b 100644 --- a/helm/dagster/templates/helpers/_helpers.tpl +++ b/helm/dagster/templates/helpers/_helpers.tpl @@ -30,6 +30,10 @@ If release name contains chart name it will be used as a full name. {{- .repository -}}:{{- .tag -}} {{- end }} +{{- define "dagster.externalPostgresImage.name" }} +{{- .registry -}}/{{- .repository -}}:{{- .tag -}} +{{- end }} + {{- define "dagster.dagsterImage.name" }} {{- $ := index . 0 }} diff --git a/helm/dagster/values.yaml b/helm/dagster/values.yaml index 3e7c6cb41c5da..b16da4a504a99 100644 --- a/helm/dagster/values.yaml +++ b/helm/dagster/values.yaml @@ -767,6 +767,7 @@ postgresql: # Used by init container to check that db is running. (Even if enabled:false) image: + registry: "docker.io" repository: "library/postgres" tag: "14.6" pullPolicy: IfNotPresent diff --git a/js_modules/dagster-ui/packages/ui-core/client.json b/js_modules/dagster-ui/packages/ui-core/client.json index 9b8c430dd2ead..8e9e8ae0c4b40 100644 --- a/js_modules/dagster-ui/packages/ui-core/client.json +++ b/js_modules/dagster-ui/packages/ui-core/client.json @@ -123,7 +123,8 @@ "RunAssetChecksQuery": "6946372fc625c6aba249a54be1943c0858c8efbd5e6f5c64c55723494dc199e4", "RunAssetsQuery": "53c1e7814d451dfd58fb2427dcb326a1e9628c8bbc91b3b9c76f8d6c7b75e278", "RunTabsCountQuery": "5fe1760a3bf0494fb98e3c09f31add5138f9f31d59507a8b25186e2103bebbb4", - "RunRootQuery": "1aa4561b33c2cfb079d7a3ff284096fc3208a46dee748a24c7af827a2cb22919", + "RunStepStatsQuery": "77d73353a4aea095bfa241903122abf14eb38341c5869a9688b70c0d53f5a167", + "RunRootQuery": "4f2633b31ddc71c08d3a985be30dc1bf21fbc462287554f165060c51a3554beb", "RunStatsQuery": "75e80f740a79607de9e1152f9b7074d319197fbc219784c767c1abd5553e9a49", "LaunchPipelineExecution": "292088c4a697aca6be1d3bbc0cfc45d8a13cdb2e75cfedc64b68c6245ea34f89", "LaunchMultipleRuns": "a56d9efdb35e71e0fd1744dd768129248943bc5b23e717458b82c46829661763", @@ -135,7 +136,7 @@ "RunTagValuesQuery": "0c0a9998c215bb801eb0adcd5449c0ac4cf1e8efbc6d0fcc5fb6d76fcc95cb92", "ScheduledRunsListQuery": "2650d8ebdfc444fe76fcf8acd9ff54f9ecacdb680b1d83e3f487cb71dd0c7eae", "TerminateRunIdsQuery": "d38573af47f3ab2f2b11d90cb85ce8426307e2384e67a5b20e2bf67d5c1054bb", - "RunActionButtonsTestQuery": "d85a7e0201a27eb36be5a7471d2724fe5a68b7257e6635f54f120fc40f8169c0", + "RunActionButtonsTestQuery": "5d358c3360e683549b885108c3dbb7c1d21d8afd790a5ee963e6e9640ccdbfe8", "RunsRootQuery": "091646e47ecea81ba4765a3f2cead18880b09ee400d1d7e9dcb6e194ee364e51", "RunsFeedRootQuery": "ef8eb6ca144d661c6bcd409ed878551851f15dd1c0aa8c03ee9c68c1c4c301d1", "OngoingRunTimelineQuery": "055420e85ba799b294bab52c01d3f4a4470580606a40483031c35777d88d527f", diff --git a/js_modules/dagster-ui/packages/ui-core/src/app/FeatureFlags.oss.tsx b/js_modules/dagster-ui/packages/ui-core/src/app/FeatureFlags.oss.tsx index c9244178d1cfe..bb481edf6ea67 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/app/FeatureFlags.oss.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/app/FeatureFlags.oss.tsx @@ -7,6 +7,7 @@ export enum FeatureFlag { flagAssetSelectionSyntax = 'flagAssetSelectionSyntax', flagRunSelectionSyntax = 'flagRunSelectionSyntax', flagAssetSelectionWorker = 'flagAssetSelectionWorker', + flagOpSelectionSyntax = 'flagOpSelectionSyntax', // Flags for tests __TestFlagDefaultNone = '__TestFlagDefaultNone', diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationList.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationList.tsx index 18eeaf18b0f7e..25daf6c8e841f 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationList.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationList.tsx @@ -1,15 +1,16 @@ import {Table} from '@dagster-io/ui-components'; +import {AssetKey} from '../types'; import {EvaluationListRow} from './EvaluationListRow'; import {AssetConditionEvaluationRecordFragment} from './types/GetEvaluationsQuery.types'; -import {AssetViewDefinitionNodeFragment} from '../types/AssetView.types'; interface Props { - definition: AssetViewDefinitionNodeFragment; + assetKey: AssetKey; + isPartitioned: boolean; evaluations: AssetConditionEvaluationRecordFragment[]; } -export const EvaluationList = ({definition, evaluations}: Props) => { +export const EvaluationList = ({assetKey, isPartitioned, evaluations}: Props) => { return ( @@ -25,7 +26,8 @@ export const EvaluationList = ({definition, evaluations}: Props) => { ); })} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationListRow.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationListRow.tsx index 4a52253515d1f..34ba66772ec19 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationListRow.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationListRow.tsx @@ -10,20 +10,21 @@ import { } from '@dagster-io/ui-components'; import {useState} from 'react'; +import {AssetKey} from '../types'; import {EvaluationDetailDialog} from './EvaluationDetailDialog'; import {EvaluationStatusTag} from './EvaluationStatusTag'; import {AssetConditionEvaluationRecordFragment} from './types/GetEvaluationsQuery.types'; import {DEFAULT_TIME_FORMAT} from '../../app/time/TimestampFormat'; import {RunsFeedTableWithFilters} from '../../runs/RunsFeedTable'; import {TimestampDisplay} from '../../schedules/TimestampDisplay'; -import {AssetViewDefinitionNodeFragment} from '../types/AssetView.types'; interface Props { - definition: AssetViewDefinitionNodeFragment; + assetKey: AssetKey; + isPartitioned: boolean; evaluation: AssetConditionEvaluationRecordFragment; } -export const EvaluationListRow = ({evaluation, definition}: Props) => { +export const EvaluationListRow = ({evaluation, assetKey, isPartitioned}: Props) => { const [isOpen, setIsOpen] = useState(false); return ( @@ -39,27 +40,32 @@ export const EvaluationListRow = ({evaluation, definition}: Props) => { ); }; -const EvaluationRunInfo = ({evaluation}: {evaluation: AssetConditionEvaluationRecordFragment}) => { - const {runIds} = evaluation; +interface EvaluationRunInfoProps { + runIds: string[]; + timestamp: number; +} + +const EvaluationRunInfo = ({runIds, timestamp}: EvaluationRunInfoProps) => { const [isOpen, setIsOpen] = useState(false); if (runIds.length === 0) { @@ -95,7 +101,7 @@ const EvaluationRunInfo = ({evaluation}: {evaluation: AssetConditionEvaluationRe <> Runs at{' '} diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationStatusTag.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationStatusTag.tsx index 4e89fbb089f9e..2e2b21a794ff0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationStatusTag.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AutoMaterializePolicyPage/EvaluationStatusTag.tsx @@ -1,17 +1,23 @@ import {Box, Colors, Icon, Popover, Tag} from '@dagster-io/ui-components'; import {useMemo} from 'react'; +import {AssetKey} from '../types'; import {PartitionSubsetList} from './PartitionSubsetList'; import {AssetConditionEvaluationRecordFragment} from './types/GetEvaluationsQuery.types'; -import {AssetViewDefinitionNodeFragment} from '../types/AssetView.types'; interface Props { - definition: AssetViewDefinitionNodeFragment; + assetKey: AssetKey; + isPartitioned: boolean; selectedEvaluation: AssetConditionEvaluationRecordFragment; selectPartition: (partitionKey: string | null) => void; } -export const EvaluationStatusTag = ({definition, selectedEvaluation, selectPartition}: Props) => { +export const EvaluationStatusTag = ({ + assetKey, + isPartitioned, + selectedEvaluation, + selectPartition, +}: Props) => { const evaluation = selectedEvaluation?.evaluation; const rootEvaluationNode = useMemo( () => evaluation?.evaluationNodes.find((node) => node.uniqueId === evaluation.rootUniqueId), @@ -19,8 +25,7 @@ export const EvaluationStatusTag = ({definition, selectedEvaluation, selectParti ); const rootUniqueId = evaluation?.rootUniqueId; - const partitionDefinition = definition?.partitionDefinition; - const assetKeyPath = definition?.assetKey.path || []; + const assetKeyPath = assetKey.path || []; const numRequested = selectedEvaluation?.numRequested; const numTrue = @@ -29,7 +34,7 @@ export const EvaluationStatusTag = ({definition, selectedEvaluation, selectParti : null; if (numRequested) { - if (partitionDefinition && rootUniqueId && numTrue) { + if (isPartitioned && rootUniqueId && numTrue) { return ( { return ( - + ); }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/graph/kindtag-images/tool-googledrive-color.svg b/js_modules/dagster-ui/packages/ui-core/src/graph/kindtag-images/tool-googledrive-color.svg index 2d94beff46945..5575d924a946a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graph/kindtag-images/tool-googledrive-color.svg +++ b/js_modules/dagster-ui/packages/ui-core/src/graph/kindtag-images/tool-googledrive-color.svg @@ -1,3 +1,3 @@ - + diff --git a/js_modules/dagster-ui/packages/ui-core/src/launchpad/OpSelector.tsx b/js_modules/dagster-ui/packages/ui-core/src/launchpad/OpSelector.tsx index 0d367ea3fee26..4d10777dcef95 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/launchpad/OpSelector.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/launchpad/OpSelector.tsx @@ -7,6 +7,7 @@ import {OpSelectorQuery, OpSelectorQueryVariables} from './types/OpSelector.type import {filterByQuery} from '../app/GraphQueryImpl'; import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorFragment'; import {ShortcutHandler} from '../app/ShortcutHandler'; +import {filterOpSelectionByQuery} from '../op-selection/AntlrOpSelection'; import {explodeCompositesInHandleGraph} from '../pipelines/CompositeSupport'; import {GRAPH_EXPLORER_SOLID_HANDLE_FRAGMENT} from '../pipelines/GraphExplorer'; import {GraphQueryInput} from '../ui/GraphQueryInput'; @@ -85,7 +86,7 @@ export const OpSelector = (props: IOpSelectorProps) => { const opsFetchError = (data?.pipelineOrError.__typename !== 'Pipeline' && data?.pipelineOrError.message) || null; - const queryResultOps = filterByQuery(ops, query).all; + const queryResultOps = filterOpSelectionByQuery(ops, query).all; const invalidOpSelection = !loading && queryResultOps.length === 0; const errorMessage = invalidOpSelection diff --git a/js_modules/dagster-ui/packages/ui-core/src/launchpad/useLaunchMultipleRunsWithTelemetry.ts b/js_modules/dagster-ui/packages/ui-core/src/launchpad/useLaunchMultipleRunsWithTelemetry.ts index 9249da1025e91..f9c6f36fe9899 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/launchpad/useLaunchMultipleRunsWithTelemetry.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/launchpad/useLaunchMultipleRunsWithTelemetry.ts @@ -29,6 +29,7 @@ export function useLaunchMultipleRunsWithTelemetry() { const executionParamsList = Array.isArray(variables.executionParamsList) ? variables.executionParamsList : [variables.executionParamsList]; + const jobNames = executionParamsList.map( (params) => params.selector.jobName || params.selector.pipelineName, ); diff --git a/js_modules/dagster-ui/packages/ui-core/src/op-selection/AntlrOpSelection.ts b/js_modules/dagster-ui/packages/ui-core/src/op-selection/AntlrOpSelection.ts index 614bde690e347..80a6042aa9881 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/op-selection/AntlrOpSelection.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/op-selection/AntlrOpSelection.ts @@ -1,10 +1,12 @@ import {CharStreams, CommonTokenStream} from 'antlr4ts'; +import {FeatureFlag} from 'shared/app/FeatureFlags.oss'; import {AntlrOpSelectionVisitor} from './AntlrOpSelectionVisitor'; -import {GraphQueryItem} from '../app/GraphQueryImpl'; +import {GraphQueryItem, filterByQuery} from '../app/GraphQueryImpl'; import {AntlrInputErrorListener} from '../asset-selection/AntlrAssetSelection'; import {OpSelectionLexer} from './generated/OpSelectionLexer'; import {OpSelectionParser} from './generated/OpSelectionParser'; +import {featureEnabled} from '../app/Flags'; type OpSelectionQueryResult = { all: GraphQueryItem[]; @@ -40,3 +42,18 @@ export const parseOpSelectionQuery = ( return e as Error; } }; + +export const filterOpSelectionByQuery = ( + all_ops: GraphQueryItem[], + query: string, +): OpSelectionQueryResult => { + if (featureEnabled(FeatureFlag.flagOpSelectionSyntax)) { + const result = parseOpSelectionQuery(all_ops, query); + if (result instanceof Error) { + // fall back to old behavior + return filterByQuery(all_ops, query); + } + return result; + } + return filterByQuery(all_ops, query); +}; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/RunConfigDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/RunConfigDialog.tsx index 7dad84a6c971d..713a6650a95f2 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/RunConfigDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/RunConfigDialog.tsx @@ -17,7 +17,7 @@ interface Props { runConfigYaml: string; mode: string | null; isJob: boolean; - + jobName?: string; // Optionally provide tags to display them as well. tags?: RunTagsFragment[]; @@ -27,8 +27,18 @@ interface Props { } export const RunConfigDialog = (props: Props) => { - const {isOpen, onClose, copyConfig, runConfigYaml, tags, mode, isJob, request, repoAddress} = - props; + const { + isOpen, + onClose, + copyConfig, + runConfigYaml, + tags, + mode, + isJob, + jobName, + request, + repoAddress, + } = props; const hasTags = !!tags && tags.length > 0; return ( @@ -76,10 +86,12 @@ export const RunConfigDialog = (props: Props) => { topBorder left={ request && - repoAddress && ( + repoAddress && + jobName && ( diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/RunFragments.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/RunFragments.tsx index c0ebb65c183fd..b34d76e07dab3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/RunFragments.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/RunFragments.tsx @@ -46,20 +46,6 @@ export const RUN_FRAGMENT = gql` } stepKeysToExecute updateTime - stepStats { - stepKey - status - startTime - endTime - attempts { - startTime - endTime - } - markers { - startTime - endTime - } - } ...RunTimingFragment } diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx b/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx index c99bfc4106a6f..a3d495d021e7e 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/RunMetadataProvider.tsx @@ -1,11 +1,17 @@ import * as React from 'react'; +import {useMemo} from 'react'; import {LogsProviderLogs} from './LogsProvider'; import {RunContext} from './RunContext'; -import {gql} from '../apollo-client'; +import {gql, useQuery} from '../apollo-client'; import {flattenOneLevel} from '../util/flattenOneLevel'; import {RunFragment} from './types/RunFragments.types'; -import {RunMetadataProviderMessageFragment} from './types/RunMetadataProvider.types'; +import { + RunMetadataProviderMessageFragment, + RunStepStatsFragment, + RunStepStatsQuery, + RunStepStatsQueryVariables, +} from './types/RunMetadataProvider.types'; import {StepEventStatus} from '../graphql/types'; import {METADATA_ENTRY_FRAGMENT} from '../metadata/MetadataEntryFragment'; @@ -103,13 +109,17 @@ export const extractLogCaptureStepsFromLegacySteps = (stepKeys: string[]) => { const fromTimestamp = (ts: number | null) => (ts ? Math.floor(ts * 1000) : undefined); -function extractMetadataFromRun(run?: RunFragment): IRunMetadataDict { +function extractMetadataFromRun( + run: RunFragment | null = null, + stepStats: RunStepStatsFragment['stepStats'] = [], +): IRunMetadataDict { const metadata: IRunMetadataDict = { firstLogAt: 0, mostRecentLogAt: 0, globalMarkers: [], steps: {}, }; + if (!run) { return metadata; } @@ -120,7 +130,7 @@ function extractMetadataFromRun(run?: RunFragment): IRunMetadataDict { metadata.exitedAt = fromTimestamp(run.endTime); } - run.stepStats.forEach((stepStat) => { + stepStats.forEach((stepStat) => { metadata.steps[stepStat.stepKey] = { // state: // current state @@ -370,7 +380,18 @@ interface IRunMetadataProviderProps { export const RunMetadataProvider = ({logs, children}: IRunMetadataProviderProps) => { const run = React.useContext(RunContext); - const runMetadata = React.useMemo(() => extractMetadataFromRun(run), [run]); + + // Step stats can be expensive to load, so we separate them from the main run query. + const {data} = useQuery(RUN_STEP_STATS_QUERY, { + variables: run ? {runId: run.id} : undefined, + skip: !run, + }); + + const stepStats = useMemo(() => { + return data?.pipelineRunOrError.__typename === 'Run' ? data.pipelineRunOrError.stepStats : []; + }, [data]); + + const runMetadata = React.useMemo(() => extractMetadataFromRun(run, stepStats), [run, stepStats]); const metadata = React.useMemo( () => logs.loading ? runMetadata : extractMetadataFromLogs(flattenOneLevel(logs.allNodeChunks)), @@ -379,6 +400,35 @@ export const RunMetadataProvider = ({logs, children}: IRunMetadataProviderProps) return <>{children(metadata)}; }; +const RUN_STEP_STATS_QUERY = gql` + query RunStepStatsQuery($runId: ID!) { + pipelineRunOrError(runId: $runId) { + ... on Run { + id + ...RunStepStatsFragment + } + } + } + + fragment RunStepStatsFragment on Run { + id + stepStats { + stepKey + status + startTime + endTime + attempts { + startTime + endTime + } + markers { + startTime + endTime + } + } + } +`; + export const RUN_METADATA_PROVIDER_MESSAGE_FRAGMENT = gql` fragment RunMetadataProviderMessageFragment on DagsterRunEvent { ... on MessageEvent { diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/types/RunActionButtonsTestQuery.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/types/RunActionButtonsTestQuery.types.ts index ea344cf423ea4..cc5a89e67c1fb 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/types/RunActionButtonsTestQuery.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/__tests__/types/RunActionButtonsTestQuery.types.ts @@ -56,25 +56,8 @@ export type RunActionButtonsTestQuery = { }>; }>; } | null; - stepStats: Array<{ - __typename: 'RunStepStats'; - stepKey: string; - status: Types.StepEventStatus | null; - startTime: number | null; - endTime: number | null; - attempts: Array<{ - __typename: 'RunMarker'; - startTime: number | null; - endTime: number | null; - }>; - markers: Array<{ - __typename: 'RunMarker'; - startTime: number | null; - endTime: number | null; - }>; - }>; } | {__typename: 'RunNotFoundError'}; }; -export const RunActionButtonsTestQueryVersion = 'd85a7e0201a27eb36be5a7471d2724fe5a68b7257e6635f54f120fc40f8169c0'; +export const RunActionButtonsTestQueryVersion = '5d358c3360e683549b885108c3dbb7c1d21d8afd790a5ee963e6e9640ccdbfe8'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts index a50b77c1b0de8..e75aab9ccde18 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunFragments.types.ts @@ -49,15 +49,6 @@ export type RunFragment = { }>; }>; } | null; - stepStats: Array<{ - __typename: 'RunStepStats'; - stepKey: string; - status: Types.StepEventStatus | null; - startTime: number | null; - endTime: number | null; - attempts: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; - markers: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; - }>; }; export type RunDagsterRunEventFragment_AlertFailureEvent = { @@ -3186,13 +3177,4 @@ export type RunPageFragment = { }>; }>; } | null; - stepStats: Array<{ - __typename: 'RunStepStats'; - stepKey: string; - status: Types.StepEventStatus | null; - startTime: number | null; - endTime: number | null; - attempts: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; - markers: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; - }>; }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts index db950cff62923..9c7d0454624ba 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunMetadataProvider.types.ts @@ -2,6 +2,52 @@ import * as Types from '../../graphql/types'; +export type RunStepStatsQueryVariables = Types.Exact<{ + runId: Types.Scalars['ID']['input']; +}>; + +export type RunStepStatsQuery = { + __typename: 'Query'; + pipelineRunOrError: + | {__typename: 'PythonError'} + | { + __typename: 'Run'; + id: string; + stepStats: Array<{ + __typename: 'RunStepStats'; + stepKey: string; + status: Types.StepEventStatus | null; + startTime: number | null; + endTime: number | null; + attempts: Array<{ + __typename: 'RunMarker'; + startTime: number | null; + endTime: number | null; + }>; + markers: Array<{ + __typename: 'RunMarker'; + startTime: number | null; + endTime: number | null; + }>; + }>; + } + | {__typename: 'RunNotFoundError'}; +}; + +export type RunStepStatsFragment = { + __typename: 'Run'; + id: string; + stepStats: Array<{ + __typename: 'RunStepStats'; + stepKey: string; + status: Types.StepEventStatus | null; + startTime: number | null; + endTime: number | null; + attempts: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; + markers: Array<{__typename: 'RunMarker'; startTime: number | null; endTime: number | null}>; + }>; +}; + export type RunMetadataProviderMessageFragment_AlertFailureEvent = { __typename: 'AlertFailureEvent'; message: string; @@ -488,3 +534,5 @@ export type RunMetadataProviderMessageFragment = | RunMetadataProviderMessageFragment_StepExpectationResultEvent | RunMetadataProviderMessageFragment_StepWorkerStartedEvent | RunMetadataProviderMessageFragment_StepWorkerStartingEvent; + +export const RunStepStatsQueryVersion = '77d73353a4aea095bfa241903122abf14eb38341c5869a9688b70c0d53f5a167'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunRoot.types.ts b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunRoot.types.ts index d32da61f71afd..8c196998cc90a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunRoot.types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/runs/types/RunRoot.types.ts @@ -58,25 +58,8 @@ export type RunRootQuery = { }>; }>; } | null; - stepStats: Array<{ - __typename: 'RunStepStats'; - stepKey: string; - status: Types.StepEventStatus | null; - startTime: number | null; - endTime: number | null; - attempts: Array<{ - __typename: 'RunMarker'; - startTime: number | null; - endTime: number | null; - }>; - markers: Array<{ - __typename: 'RunMarker'; - startTime: number | null; - endTime: number | null; - }>; - }>; } | {__typename: 'RunNotFoundError'}; }; -export const RunRootQueryVersion = '1aa4561b33c2cfb079d7a3ff284096fc3208a46dee748a24c7af827a2cb22919'; +export const RunRootQueryVersion = '4f2633b31ddc71c08d3a985be30dc1bf21fbc462287554f165060c51a3554beb'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/search/SearchDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/search/SearchDialog.tsx index e9855337ff37c..e8605d16b4161 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/search/SearchDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/search/SearchDialog.tsx @@ -72,6 +72,14 @@ const initialState: State = { const DEBOUNCE_MSEC = 100; +// sort by Fuse score ascending, lower is better +const sortResultsByFuseScore = ( + a: Fuse.FuseResult, + b: Fuse.FuseResult, +) => { + return (a.score ?? 0) - (b.score ?? 0); +}; + export const SearchDialog = () => { const history = useHistory(); const {initialize, loading, searchPrimary, searchSecondary} = useGlobalSearch({ @@ -82,9 +90,11 @@ export const SearchDialog = () => { const [state, dispatch] = React.useReducer(reducer, initialState); const {shown, queryString, primaryResults, secondaryResults, highlight} = state; - const results = [...primaryResults, ...secondaryResults]; - const renderedResults = results.slice(0, MAX_DISPLAYED_RESULTS); - const numRenderedResults = renderedResults.length; + const {renderedResults, numRenderedResults} = React.useMemo(() => { + const results = [...primaryResults, ...secondaryResults].sort(sortResultsByFuseScore); + const renderedResults = results.slice(0, MAX_DISPLAYED_RESULTS); + return {renderedResults, numRenderedResults: renderedResults.length}; + }, [primaryResults, secondaryResults]); const openSearch = React.useCallback(() => { trackEvent('open-global-search'); diff --git a/js_modules/dagster-ui/packages/ui-core/src/search/useGlobalSearch.tsx b/js_modules/dagster-ui/packages/ui-core/src/search/useGlobalSearch.tsx index c17bb0674470e..013e1435c8093 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/search/useGlobalSearch.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/search/useGlobalSearch.tsx @@ -307,6 +307,7 @@ const fuseOptions = { threshold: 0.3, useExtendedSearch: true, includeMatches: true, + includeScore: true, // Allow searching to continue to the end of the string. ignoreLocation: true, diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/DryRunRequestTable.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/DryRunRequestTable.tsx index b9859d06be68a..ca7e773b9c6e1 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/DryRunRequestTable.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/DryRunRequestTable.tsx @@ -71,6 +71,7 @@ export const RunRequestTable = ({runRequests, isJob, repoAddress, mode, jobName} runConfigYaml={selectedRequest.runConfigYaml} tags={selectedRequest.tags} isJob={isJob} + jobName={jobName} request={selectedRequest} repoAddress={repoAddress} /> diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateScheduleDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateScheduleDialog.tsx index 7262979f9473a..cdb83e8aa6066 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateScheduleDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/EvaluateScheduleDialog.tsx @@ -170,9 +170,9 @@ const EvaluateSchedule = ({repoAddress, name, onClose, jobName}: Props) => { const executionParamsList = useMemo( () => scheduleExecutionData && scheduleSelector - ? buildExecutionParamsListSchedule(scheduleExecutionData, scheduleSelector) + ? buildExecutionParamsListSchedule(scheduleExecutionData, scheduleSelector, jobName) : [], - [scheduleSelector, scheduleExecutionData], + [scheduleSelector, scheduleExecutionData, jobName], ); const canLaunchAll = useMemo(() => { diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/SensorDryRunDialog.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/SensorDryRunDialog.tsx index 1047d13d9a391..d54ec777a3f9a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/SensorDryRunDialog.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/SensorDryRunDialog.tsx @@ -105,9 +105,9 @@ const SensorDryRun = ({repoAddress, name, currentCursor, onClose, jobName}: Prop const executionParamsList = useMemo( () => sensorExecutionData && sensorSelector - ? buildExecutionParamsListSensor(sensorExecutionData, sensorSelector) + ? buildExecutionParamsListSensor(sensorExecutionData, sensorSelector, jobName) : [], - [sensorSelector, sensorExecutionData], + [sensorSelector, sensorExecutionData, jobName], ); const submitTest = useCallback(async () => { diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/__fixtures__/EvaluateScheduleDialog.fixtures.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/__fixtures__/EvaluateScheduleDialog.fixtures.tsx index 724b62da00125..da76071aa8ea9 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/__fixtures__/EvaluateScheduleDialog.fixtures.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/__fixtures__/EvaluateScheduleDialog.fixtures.tsx @@ -79,6 +79,44 @@ export const scheduleDryWithWithRunRequest = { }), }; +export const scheduleDryWithWithRunRequestUndefinedName = { + __typename: 'Mutation' as const, + scheduleDryRun: buildDryRunInstigationTick({ + timestamp: 1674950400, + evaluationResult: buildTickEvaluation({ + runRequests: [ + buildRunRequest({ + jobName: undefined, + runConfigYaml: + 'ops:\n configurable_op:\n config:\n scheduled_date: 2023-01-29\n', + tags: [ + buildPipelineTag({ + key: 'dagster/schedule_name', + value: 'configurable_job_schedule', + }), + buildPipelineTag({ + key: 'date', + value: '2023-01-29', + __typename: 'PipelineTag' as const, + }), + buildPipelineTag({ + key: 'github_test', + value: 'test', + }), + buildPipelineTag({ + key: 'okay_t2', + value: 'okay', + }), + ], + runKey: 'EvaluateScheduleDialog.test.tsx:1675705668.993122345', + }), + ], + skipReason: null, + error: null, + }), + }), +}; + export const ScheduleDryRunMutationRunRequests: MockedResponse = { request: { query: SCHEDULE_DRY_RUN_MUTATION, @@ -94,6 +132,22 @@ export const ScheduleDryRunMutationRunRequests: MockedResponse = + { + request: { + query: SCHEDULE_DRY_RUN_MUTATION, + variables: { + selectorData: { + scheduleName: 'test', + repositoryLocationName: 'testLocation', + repositoryName: 'testName', + }, + timestamp: 5, + }, + }, + result: {data: scheduleDryWithWithRunRequestUndefinedName}, + }; + export const ScheduleDryRunMutationError: MockedResponse = { request: { query: SCHEDULE_DRY_RUN_MUTATION, @@ -253,3 +307,87 @@ export const ScheduleLaunchAllMutation: MockedResponse = + { + request: { + query: LAUNCH_MULTIPLE_RUNS_MUTATION, + variables: { + executionParamsList: [ + { + runConfigData: + 'ops:\n configurable_op:\n config:\n scheduled_date: 2023-01-29', + selector: { + jobName: 'testJobName', // fallback + repositoryLocationName: 'testLocation', + repositoryName: 'testName', + assetSelection: [], + assetCheckSelection: [], + solidSelection: undefined, + }, + mode: 'default', + executionMetadata: { + tags: [ + { + key: 'dagster/schedule_name', + value: 'configurable_job_schedule', + }, + { + key: 'date', + value: '2023-01-29', + }, + { + key: 'github_test', + value: 'test', + }, + { + key: 'okay_t2', + value: 'okay', + }, + ], + }, + }, + ], + }, + }, + result: { + data: { + __typename: 'Mutation', + launchMultipleRuns: buildLaunchMultipleRunsResult({ + launchMultipleRunsResult: [ + buildLaunchRunSuccess({ + run: buildRun({ + id: '504b3a77-d6c4-440c-a128-7f59c9d75d59', + pipeline: buildPipelineSnapshot({ + name: 'testJobName', // fallback + }), + tags: [ + buildPipelineTag({ + key: 'dagster/schedule_name', + value: 'configurable_job_schedule', + }), + buildPipelineTag({ + key: 'date', + value: '2023-01-29', + }), + buildPipelineTag({ + key: 'github_test', + value: 'test', + }), + buildPipelineTag({ + key: 'okay_t2', + value: 'okay', + }), + ], + status: RunStatus.QUEUED, + runConfigYaml: + 'ops:\n configurable_op:\n config:\n scheduled_date: 2023-01-29', + mode: 'default', + resolvedOpSelection: null, + }), + }), + ], + }), + }, + }, + }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/__fixtures__/SensorDryRunDialog.fixtures.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/__fixtures__/SensorDryRunDialog.fixtures.tsx index 98e691e5086a9..68d62e5701303 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/__fixtures__/SensorDryRunDialog.fixtures.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/__fixtures__/SensorDryRunDialog.fixtures.tsx @@ -55,6 +55,19 @@ export const runRequests: RunRequest[] = [ }), ]; +export const runRequestWithUndefinedJobName: RunRequest[] = [ + buildRunRequest({ + jobName: undefined, // undefined jobName + runKey: 'DryRunRequestTable.test.tsx:1675705668.9931223', + runConfigYaml: + 'solids:\n read_file:\n config:\n directory: /Users/marcosalazar/code/dagster/js_modules/dagster-ui/packages/ui-core/src/ticks/tests\n filename: DryRunRequestTable.test.tsx\n', + tags: [ + buildPipelineTag({key: 'dagster2', value: 'test'}), + buildPipelineTag({key: 'marco2', value: 'salazar2'}), + ], + }), +]; + export const SensorDryRunMutationRunRequests: MockedResponse = { request: { query: EVALUATE_SENSOR_MUTATION, @@ -81,6 +94,33 @@ export const SensorDryRunMutationRunRequests: MockedResponse = + { + request: { + query: EVALUATE_SENSOR_MUTATION, + variables: { + selectorData: { + sensorName: 'test', + repositoryLocationName: 'testLocation', + repositoryName: 'testName', + }, + cursor: 'testCursortesting123', + }, + }, + result: { + data: { + __typename: 'Mutation', + sensorDryRun: buildDryRunInstigationTick({ + evaluationResult: buildTickEvaluation({ + cursor: 'a new cursor', + runRequests: runRequestWithUndefinedJobName, + error: null, + }), + }), + }, + }, + }; + export const SensorDryRunMutationError: MockedResponse = { request: { query: EVALUATE_SENSOR_MUTATION, @@ -335,3 +375,67 @@ export const SensorLaunchAllMutation: MockedResponse }, }, }; + +export const SensorLaunchAllMutation1JobWithUndefinedJobName: MockedResponse = + { + request: { + query: LAUNCH_MULTIPLE_RUNS_MUTATION, + variables: { + executionParamsList: [ + { + runConfigData: + 'solids:\n read_file:\n config:\n directory: /Users/marcosalazar/code/dagster/js_modules/dagster-ui/packages/ui-core/src/ticks/tests\n filename: DryRunRequestTable.test.tsx', + selector: { + jobName: 'testJobName', // fallback + repositoryLocationName: 'testLocation', + repositoryName: 'testName', + assetSelection: [], + assetCheckSelection: [], + solidSelection: undefined, + }, + mode: 'default', + executionMetadata: { + tags: [ + { + key: 'dagster2', + value: 'test', + }, + { + key: 'marco2', + value: 'salazar2', + }, + ], + }, + }, + ], + }, + }, + result: { + data: { + __typename: 'Mutation', + launchMultipleRuns: buildLaunchMultipleRunsResult({ + launchMultipleRunsResult: [ + buildLaunchRunSuccess({ + __typename: 'LaunchRunSuccess', + run: buildRun({ + __typename: 'Run', + id: '504b3a77-d6c4-440c-a128-7f59c9d75d59', + pipeline: buildPipelineSnapshot({ + name: 'testJobName', + }), + tags: [ + buildPipelineTag({key: 'dagster2', value: 'test'}), + buildPipelineTag({key: 'marco2', value: 'salazar2'}), + ], + status: RunStatus.QUEUED, + runConfigYaml: + 'solids:\n read_file:\n config:\n directory: /Users/marcosalazar/code/dagster/js_modules/dagster-ui/packages/ui-core/src/ticks/tests\n filename: DryRunRequestTable.test.tsx\n', + mode: 'default', + resolvedOpSelection: null, + }), + }), + ], + }), + }, + }, + }; diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/EvaluateScheduleDialog.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/EvaluateScheduleDialog.test.tsx index a8a2b55c06eb5..10eb96d2d2f3d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/EvaluateScheduleDialog.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/EvaluateScheduleDialog.test.tsx @@ -10,8 +10,10 @@ import { GetScheduleQueryMock, ScheduleDryRunMutationError, ScheduleDryRunMutationRunRequests, + ScheduleDryRunMutationRunRequestsWithUndefinedName, ScheduleDryRunMutationSkipped, ScheduleLaunchAllMutation, + ScheduleLaunchAllMutationWithUndefinedName, } from '../__fixtures__/EvaluateScheduleDialog.fixtures'; // This component is unit tested separately so mocking it out @@ -160,4 +162,49 @@ describe('EvaluateScheduleTest', () => { expect(pushSpy).toHaveBeenCalled(); }); }); + + it('launches all runs for 1 runrequest with undefined job name in the runrequest', async () => { + const pushSpy = jest.fn(); + const createHrefSpy = jest.fn(); + + (useHistory as jest.Mock).mockReturnValue({ + push: pushSpy, + createHref: createHrefSpy, + }); + + (useTrackEvent as jest.Mock).mockReturnValue(jest.fn()); + + render( + + + , + ); + const selectButton = await screen.findByTestId('tick-selection'); + await userEvent.click(selectButton); + await waitFor(() => { + expect(screen.getByTestId('tick-5')).toBeVisible(); + }); + await userEvent.click(screen.getByTestId('tick-5')); + await userEvent.click(screen.getByTestId('continue')); + await waitFor(() => { + expect(screen.getByText(/1\s+run request/i)).toBeVisible(); + expect(screen.getByTestId('launch-all')).not.toBeDisabled(); + }); + + userEvent.click(screen.getByTestId('launch-all')); + + await waitFor(() => { + expect(screen.getByText(/Launching runs/i)).toBeVisible(); + }); + + await waitFor(() => { + expect(pushSpy).toHaveBeenCalled(); + }); + }); }); diff --git a/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/SensorDryRunDialog.test.tsx b/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/SensorDryRunDialog.test.tsx index efb95aa30ed1e..b217a38741e75 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/SensorDryRunDialog.test.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/ticks/__tests__/SensorDryRunDialog.test.tsx @@ -95,7 +95,7 @@ describe('SensorDryRunTest', () => { expect(screen.getByTestId('cursor-input')).toBeVisible(); }); - it('launches all runs', async () => { + it('launches all runs with well defined job names', async () => { const pushSpy = jest.fn(); const createHrefSpy = jest.fn(); @@ -140,4 +140,50 @@ describe('SensorDryRunTest', () => { expect(pushSpy).toHaveBeenCalled(); }); }); + + it('launches all runs for 1 runrequest with undefined job name in the runrequest', async () => { + const pushSpy = jest.fn(); + const createHrefSpy = jest.fn(); + + (useHistory as jest.Mock).mockReturnValue({ + push: pushSpy, + createHref: createHrefSpy, + }); + + (useTrackEvent as jest.Mock).mockReturnValue(jest.fn()); + + render( + + + , + ); + const cursorInput = await screen.findByTestId('cursor-input'); + await userEvent.type(cursorInput, 'testing123'); + await userEvent.click(screen.getByTestId('continue')); + await waitFor(() => { + expect(screen.getByText(/1\srun requests/g)).toBeVisible(); + expect(screen.queryByText('Skipped')).toBe(null); + expect(screen.queryByText('Failed')).toBe(null); + }); + + await waitFor(() => { + expect(screen.getByTestId('launch-all')).not.toBeDisabled(); + }); + + userEvent.click(screen.getByTestId('launch-all')); + + await waitFor(() => { + expect(screen.getByText(/Launching runs/i)).toBeVisible(); + }); + + await waitFor(() => { + expect(pushSpy).toHaveBeenCalled(); + }); + }); }); diff --git a/js_modules/dagster-ui/packages/ui-core/src/util/buildExecutionParamsList.ts b/js_modules/dagster-ui/packages/ui-core/src/util/buildExecutionParamsList.ts index d5264bf96a6b6..41f621f63f131 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/util/buildExecutionParamsList.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/util/buildExecutionParamsList.ts @@ -15,6 +15,7 @@ const onlyKeyAndValue = ({key, value}: {key: string; value: string}) => ({key, v export const buildExecutionParamsListSensor = ( sensorExecutionData: SensorDryRunInstigationTick, sensorSelector: SensorSelector, + jobName: string, ) => { if (!sensorExecutionData) { return []; @@ -36,7 +37,7 @@ export const buildExecutionParamsListSensor = ( const executionParams: ExecutionParams = { runConfigData: configYamlOrEmpty, selector: { - jobName: request.jobName, // get jobName from runRequest + jobName: request.jobName ?? jobName, // get jobName from runRequest, fallback to jobName repositoryLocationName, repositoryName, assetSelection: [], @@ -57,6 +58,7 @@ export const buildExecutionParamsListSensor = ( export const buildExecutionParamsListSchedule = ( scheduleExecutionData: ScheduleDryRunInstigationTick, scheduleSelector: ScheduleSelector, + jobName: string, ) => { if (!scheduleExecutionData) { return []; @@ -78,7 +80,7 @@ export const buildExecutionParamsListSchedule = ( const executionParams: ExecutionParams = { runConfigData: configYamlOrEmpty, selector: { - jobName: request.jobName, // get jobName from runRequest + jobName: request.jobName ?? jobName, // get jobName from runRequest, fallback to jobName repositoryLocationName, repositoryName, assetSelection: [], diff --git a/pyright/alt-1/requirements-pinned.txt b/pyright/alt-1/requirements-pinned.txt index 9c55cd90ee281..e7d36fbca5958 100644 --- a/pyright/alt-1/requirements-pinned.txt +++ b/pyright/alt-1/requirements-pinned.txt @@ -1,8 +1,8 @@ agate==1.9.1 -aiobotocore==2.15.2 +aiobotocore==2.16.0 aiofile==3.9.0 aiohappyeyeballs==2.4.4 -aiohttp==3.11.10 +aiohttp==3.11.11 aioitertools==0.12.0 aiosignal==1.3.2 alembic==1.14.0 @@ -24,10 +24,10 @@ backoff==2.2.1 backports-tarfile==1.2.0 beautifulsoup4==4.12.3 bleach==6.2.0 -boto3==1.35.36 +boto3==1.35.81 boto3-stubs-lite==1.35.70 -botocore==1.35.36 -botocore-stubs==1.35.82 +botocore==1.35.81 +botocore-stubs==1.35.84 buildkite-test-collector==0.1.9 cachetools==5.5.0 caio==0.9.17 @@ -66,12 +66,12 @@ daff==1.3.46 -e python_modules/libraries/dagster-spark -e python_modules/dagster-webserver db-dtypes==1.3.1 -dbt-adapters==1.10.4 +dbt-adapters==1.12.0 dbt-common==1.14.0 -dbt-core==1.8.9 +dbt-core==1.9.1 dbt-duckdb==1.9.1 dbt-extractor==0.5.1 -dbt-semantic-interfaces==0.5.1 +dbt-semantic-interfaces==0.7.4 dbt-snowflake==1.9.0 debugpy==1.8.11 decopatch==1.4.10 @@ -95,7 +95,7 @@ frozenlist==1.5.0 fsspec==2024.3.0 gcsfs==0.8.0 google-api-core==2.24.0 -google-api-python-client==2.155.0 +google-api-python-client==2.156.0 google-auth==2.37.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.1 @@ -145,13 +145,12 @@ jupyter-events==0.11.0 jupyter-lsp==2.2.5 jupyter-server==2.14.2 jupyter-server-terminals==0.5.3 -jupyterlab==4.3.3 +jupyterlab==4.3.4 jupyterlab-pygments==0.3.0 jupyterlab-server==2.27.3 keyring==25.5.0 kiwisolver==1.4.7 leather==0.4.0 -logbook==1.5.3 makefun==1.15.6 mako==1.3.8 markdown-it-py==3.0.0 @@ -161,7 +160,6 @@ matplotlib==3.10.0 matplotlib-inline==0.1.7 mccabe==0.7.0 mdurl==0.1.2 -minimal-snowplow-tracker==0.0.2 mistune==3.0.2 more-itertools==10.5.0 morefs==0.2.2 @@ -169,7 +167,7 @@ msgpack==1.1.0 multidict==6.1.0 multimethod==1.12 mypy==1.13.0 -mypy-boto3-ecs==1.35.77 +mypy-boto3-ecs==1.35.83 mypy-boto3-emr==1.35.68 mypy-boto3-emr-serverless==1.35.79 mypy-boto3-glue==1.35.80 @@ -210,7 +208,7 @@ prometheus-client==0.21.1 prompt-toolkit==3.0.48 propcache==0.2.1 proto-plus==1.25.0 -protobuf==5.29.1 +protobuf==5.29.2 psutil==6.1.0 psycopg2-binary==2.9.10 ptyprocess==0.7.0 @@ -220,8 +218,8 @@ pyarrow==18.1.0 pyasn1==0.6.1 pyasn1-modules==0.4.1 pycparser==2.22 -pydantic==2.10.3 -pydantic-core==2.27.1 +pydantic==2.10.4 +pydantic-core==2.27.2 pygments==2.18.0 pyjwt==2.10.1 pylint==3.3.2 @@ -244,7 +242,7 @@ pytimeparse==1.1.8 pytz==2024.2 pyyaml==6.0.2 pyzmq==26.2.0 -rapidfuzz==3.10.1 +rapidfuzz==3.11.0 referencing==0.35.1 requests==2.32.3 requests-oauthlib==2.0.0 @@ -264,14 +262,15 @@ send2trash==1.8.3 setuptools==75.6.0 shellingham==1.5.4 six==1.17.0 -slack-sdk==3.33.5 +slack-sdk==3.34.0 sniffio==1.3.1 snowflake-connector-python==3.12.4 snowflake-sqlalchemy==1.5.1 +snowplow-tracker==1.0.4 sortedcontainers==2.4.0 soupsieve==2.6 sqlalchemy==1.4.54 -sqlglot==26.0.0 +sqlglot==26.0.1 sqlglotrs==0.3.0 sqlparse==0.5.3 stack-data==0.6.3 @@ -292,7 +291,7 @@ tqdm==4.67.1 traitlets==5.14.3 typeguard==4.4.1 typer==0.15.1 -types-awscrt==0.23.5 +types-awscrt==0.23.6 types-backports==0.1.3 types-certifi==2021.10.8.3 types-cffi==1.16.0.20240331 diff --git a/pyright/master/requirements-pinned.txt b/pyright/master/requirements-pinned.txt index f3383f88c3e96..4705a9adbdaee 100644 --- a/pyright/master/requirements-pinned.txt +++ b/pyright/master/requirements-pinned.txt @@ -1,4 +1,4 @@ -acryl-datahub==0.14.1.12 +acryl-datahub==0.15.0 agate==1.9.1 aiofile==3.9.0 aiohappyeyeballs==2.4.4 @@ -58,10 +58,10 @@ billiard==4.2.1 bleach==6.2.0 blinker==1.9.0 bokeh==3.6.2 -boto3==1.35.82 +boto3==1.35.84 boto3-stubs-lite==1.35.70 -botocore==1.35.82 -botocore-stubs==1.35.82 +botocore==1.35.84 +botocore-stubs==1.35.84 buildkite-test-collector==0.1.9 cachecontrol==0.14.1 cached-property==2.0.1 @@ -89,7 +89,7 @@ coloredlogs==14.0 colorlog==4.8.0 comm==0.2.2 configupdater==3.2 -confluent-kafka==2.6.1 +confluent-kafka==2.6.2 connexion==2.14.2 contourpy==1.3.1 coverage==7.6.9 @@ -173,8 +173,8 @@ dagster-contrib-modal==0.0.2 -e python_modules/libraries/dagster-wandb -e python_modules/dagster-webserver -e python_modules/libraries/dagstermill -dask==2024.12.0 -dask-expr==1.1.20 +dask==2024.12.1 +dask-expr==1.1.21 dask-jobqueue==0.9.0 dask-kubernetes==2022.9.0 dask-yarn==0.9 @@ -184,13 +184,13 @@ dataclasses-json==0.6.7 datadog==0.50.2 dataproperty==1.0.1 db-dtypes==1.3.1 -dbt-adapters==1.10.2 +dbt-adapters==1.12.0 dbt-common==1.14.0 -dbt-core==1.8.9 +dbt-core==1.9.1 dbt-duckdb==1.9.1 -e examples/starlift-demo dbt-extractor==0.5.1 -dbt-semantic-interfaces==0.5.1 +dbt-semantic-interfaces==0.7.4 debugpy==1.8.11 decopatch==1.4.10 decorator==5.1.1 @@ -202,10 +202,10 @@ deprecated==1.2.15 dict2css==0.3.0.post1 dill==0.3.9 distlib==0.3.9 -distributed==2024.12.0 +distributed==2024.12.1 distro==1.9.0 -e examples/experimental/dagster-dlift/kitchen-sink -dlt==1.4.1 +dlt==1.5.0 dnspython==2.7.0 docker==7.1.0 docker-image-py==0.1.13 @@ -223,7 +223,7 @@ execnet==2.1.1 executing==2.1.0 expandvars==0.12.0 faiss-cpu==1.8.0 -fastapi==0.1.17 +fastapi==0.115.6 fastavro==1.9.7 fastjsonschema==2.21.1 -e examples/feature_graph_backed_assets @@ -252,7 +252,7 @@ gitdb==4.0.11 gitpython==3.1.43 giturlparse==0.12.0 google-api-core==2.24.0 -google-api-python-client==2.155.0 +google-api-python-client==2.156.0 google-auth==2.37.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.1 @@ -327,7 +327,7 @@ jupyter-events==0.11.0 jupyter-lsp==2.2.5 jupyter-server==2.14.2 jupyter-server-terminals==0.5.3 -jupyterlab==4.3.3 +jupyterlab==4.3.4 jupyterlab-pygments==0.3.0 jupyterlab-server==2.27.3 jupyterlab-widgets==3.0.13 @@ -337,12 +337,12 @@ kiwisolver==1.4.7 kombu==5.4.2 kopf==1.37.4 kubernetes==31.0.0 -kubernetes-asyncio==31.1.1 +kubernetes-asyncio==32.0.0 langchain==0.3.7 langchain-community==0.3.5 -langchain-core==0.3.25 +langchain-core==0.3.27 langchain-openai==0.2.5 -langchain-text-splitters==0.3.3 +langchain-text-splitters==0.3.4 langsmith==0.1.147 lazy-object-proxy==1.10.0 leather==0.4.0 @@ -352,7 +352,6 @@ linkify-it-py==2.0.3 lkml==1.3.6 locket==1.0.0 lockfile==0.12.2 -logbook==1.5.3 looker-sdk==24.16.2 lxml==5.3.0 makefun==1.15.6 @@ -363,17 +362,16 @@ markupsafe==3.0.2 marshmallow==3.23.1 marshmallow-oneofschema==3.1.1 marshmallow-sqlalchemy==0.26.1 -mashumaro==3.15 +mashumaro==3.14 matplotlib==3.10.0 matplotlib-inline==0.1.3 mbstrdecoder==1.1.3 mdit-py-plugins==0.4.2 mdurl==0.1.2 -minimal-snowplow-tracker==0.0.2 mistune==3.0.2 mixpanel==4.10.1 mlflow==1.27.0 -modal==0.68.26 +modal==0.68.41 more-itertools==10.5.0 morefs==0.2.2 moto==4.2.14 @@ -383,7 +381,7 @@ msal-extensions==1.2.0 msgpack==1.1.0 multidict==6.1.0 multimethod==1.12 -mypy-boto3-ecs==1.35.77 +mypy-boto3-ecs==1.35.83 mypy-boto3-emr==1.35.68 mypy-boto3-emr-serverless==1.35.79 mypy-boto3-glue==1.35.80 @@ -409,7 +407,7 @@ objgraph==3.6.2 onnx==1.17.0 onnxconverter-common==1.13.0 onnxruntime==1.20.1 -openai==1.57.4 +openai==1.58.1 openapi-schema-validator==0.6.2 openapi-spec-validator==0.7.1 opentelemetry-api==1.29.0 @@ -461,7 +459,7 @@ prometheus-flask-exporter==0.23.1 prompt-toolkit==3.0.48 propcache==0.2.1 proto-plus==1.25.0 -protobuf==5.29.1 +protobuf==5.29.2 psutil==6.1.0 psycopg2-binary==2.9.10 ptyprocess==0.7.0 @@ -473,8 +471,8 @@ pyarrow-hotfix==0.6 pyasn1==0.6.1 pyasn1-modules==0.4.1 pycparser==2.22 -pydantic==2.10.3 -pydantic-core==2.27.1 +pydantic==2.10.4 +pydantic-core==2.27.2 pydantic-settings==2.7.0 pydata-google-auth==1.9.0 pyflakes==3.2.0 @@ -513,7 +511,7 @@ pytzdata==2020.1 pyyaml==6.0.2 pyzmq==26.2.0 querystring-parser==1.2.4 -rapidfuzz==3.10.1 +rapidfuzz==3.11.0 readme-renderer==44.0 referencing==0.35.1 regex==2024.11.6 @@ -552,15 +550,16 @@ sigtools==4.0.1 simplejson==3.19.3 six==1.17.0 skein==0.8.2 -skl2onnx==1.17.0 -slack-sdk==3.33.5 +skl2onnx==1.18.0 +slack-sdk==3.34.0 sling==1.3.3 sling-mac-arm64==1.3.3 smmap==5.0.1 sniffio==1.3.1 snowballstemmer==2.2.0 snowflake-connector-python==3.12.4 -snowflake-sqlalchemy==1.7.1 +snowflake-sqlalchemy==1.7.2 +snowplow-tracker==1.0.4 sortedcontainers==2.4.0 soupsieve==2.6 sphinx==8.1.3 @@ -578,16 +577,16 @@ sphinxcontrib-serializinghtml==2.0.0 sqlalchemy==1.4.54 sqlalchemy-jsonfield==1.0.2 sqlalchemy-utils==0.41.2 -sqlglot==26.0.0 +sqlglot==26.0.1 sqlglotrs==0.3.0 sqlparse==0.5.3 sshpubkeys==3.3.1 sshtunnel==0.4.0 stack-data==0.6.3 -starlette==0.42.0 +starlette==0.41.3 structlog==24.4.0 sympy==1.13.1 -synchronicity==0.9.6 +synchronicity==0.9.7 syrupy==4.8.0 tableauserverclient==0.34 tabledata==1.3.3 @@ -620,7 +619,7 @@ twine==6.0.1 typeguard==4.4.1 typepy==1.3.2 typer==0.15.1 -types-awscrt==0.23.5 +types-awscrt==0.23.6 types-backports==0.1.3 types-certifi==2021.10.8.3 types-cffi==1.16.0.20240331 diff --git a/pyright/master/requirements.txt b/pyright/master/requirements.txt index 1d9b6d64ff895..2cba16896ccbc 100644 --- a/pyright/master/requirements.txt +++ b/pyright/master/requirements.txt @@ -107,7 +107,7 @@ wordcloud # (quickstart-*) apache-airflow>2.7 pendulum<3 types-sqlalchemy==1.4.53.34 - +fastapi>=0.115.6 # producing a bizarrely early version of fastapi without this ### EXAMPLES diff --git a/python_modules/dagster/dagster/_cli/asset.py b/python_modules/dagster/dagster/_cli/asset.py index e1589f6514876..89dd9f21ebaf6 100644 --- a/python_modules/dagster/dagster/_cli/asset.py +++ b/python_modules/dagster/dagster/_cli/asset.py @@ -3,9 +3,11 @@ import click import dagster._check as check +from dagster._cli.job import get_config_from_args from dagster._cli.utils import get_instance_for_cli, get_possibly_temporary_instance_for_cli from dagster._cli.workspace.cli_target import ( get_repository_python_origin_from_kwargs, + python_job_config_argument, python_origin_target_argument, ) from dagster._core.definitions.asset_selection import AssetSelection @@ -38,6 +40,12 @@ def asset_cli(): help="Asset partition range to target i.e. ...", required=False, ) +@python_job_config_argument("materialize") +@click.option( + "--config-json", + type=click.STRING, + help="JSON string of run config to use for this job run. Cannot be used with -c / --config.", +) def asset_materialize_command(**kwargs): with capture_interrupts(): with get_possibly_temporary_instance_for_cli( @@ -48,6 +56,7 @@ def asset_materialize_command(**kwargs): @telemetry_wrapper def execute_materialize_command(instance: DagsterInstance, kwargs: Mapping[str, str]) -> None: + config = get_config_from_args(kwargs) repository_origin = get_repository_python_origin_from_kwargs(kwargs) recon_repo = recon_repository_from_origin(repository_origin) @@ -137,7 +146,11 @@ def execute_materialize_command(instance: DagsterInstance, kwargs: Mapping[str, tags = {} result = execute_job( - job=reconstructable_job, asset_selection=list(asset_keys), instance=instance, tags=tags + job=reconstructable_job, + asset_selection=list(asset_keys), + instance=instance, + tags=tags, + run_config=config, ) if not result.success: raise click.ClickException("Materialization failed.") diff --git a/python_modules/dagster/dagster/_cli/job.py b/python_modules/dagster/dagster/_cli/job.py index fea9f907916fe..134a3843b3c1b 100644 --- a/python_modules/dagster/dagster/_cli/job.py +++ b/python_modules/dagster/dagster/_cli/job.py @@ -16,11 +16,13 @@ ClickArgValue, ClickOption, get_code_location_from_workspace, + get_config_from_args, get_job_python_origin_from_kwargs, get_remote_job_from_kwargs, get_remote_job_from_remote_repo, get_remote_repository_from_code_location, get_remote_repository_from_kwargs, + get_run_config_from_file_list, get_workspace_from_kwargs, job_repository_target_argument, job_target_argument, @@ -60,7 +62,7 @@ from dagster._utils.interrupts import capture_interrupts from dagster._utils.merger import merge_dicts from dagster._utils.tags import normalize_tags -from dagster._utils.yaml_utils import dump_run_config_yaml, load_yaml_from_glob_list +from dagster._utils.yaml_utils import dump_run_config_yaml T = TypeVar("T") T_Callable = TypeVar("T_Callable", bound=Callable[..., Any]) @@ -235,11 +237,6 @@ def print_op( printer.line(output_def_snap.name) -def get_run_config_from_file_list(file_list: Optional[Sequence[str]]) -> Mapping[str, object]: - check.opt_sequence_param(file_list, "file_list", of_type=str) - return cast(Mapping[str, object], load_yaml_from_glob_list(file_list) if file_list else {}) - - @job_cli.command( name="execute", help="Execute a job.\n\n{instructions}".format( @@ -306,33 +303,6 @@ def get_tags_from_args(kwargs: ClickArgMapping) -> Mapping[str, str]: ) from e -def get_config_from_args(kwargs: Mapping[str, str]) -> Mapping[str, object]: - config = cast(Tuple[str, ...], kwargs.get("config")) # files - config_json = kwargs.get("config_json") - - if not config and not config_json: - return {} - - elif config and config_json: - raise click.UsageError("Cannot specify both -c / --config and --config-json") - - elif config: - config_file_list = list(check.opt_tuple_param(config, "config", of_type=str)) - return get_run_config_from_file_list(config_file_list) - - elif config_json: - config_json = cast(str, config_json) - try: - return json.loads(config_json) - - except JSONDecodeError: - raise click.UsageError( - f"Invalid JSON-string given for `--config-json`: {config_json}\n\n{serializable_error_info_from_exc_info(sys.exc_info()).to_string()}" - ) - else: - check.failed("Unhandled case getting config from kwargs") - - def get_op_selection_from_args(kwargs: ClickArgMapping) -> Optional[Sequence[str]]: op_selection_str = kwargs.get("op_selection") if not isinstance(op_selection_str, str): diff --git a/python_modules/dagster/dagster/_cli/workspace/cli_target.py b/python_modules/dagster/dagster/_cli/workspace/cli_target.py index 9a8587c1182f7..28ecece2accab 100644 --- a/python_modules/dagster/dagster/_cli/workspace/cli_target.py +++ b/python_modules/dagster/dagster/_cli/workspace/cli_target.py @@ -47,7 +47,10 @@ WorkspaceLoadTarget, ) from dagster._grpc.utils import get_loadable_targets +from dagster._seven import JSONDecodeError, json +from dagster._utils.error import serializable_error_info_from_exc_info from dagster._utils.hosted_user_process import recon_repository_from_origin +from dagster._utils.yaml_utils import load_yaml_from_glob_list if TYPE_CHECKING: from dagster._core.workspace.context import WorkspaceProcessContext @@ -800,3 +803,35 @@ def get_remote_job_from_kwargs(instance: DagsterInstance, version: str, kwargs: def _sorted_quoted(strings: Iterable[str]) -> str: return "[" + ", ".join([f"'{s}'" for s in sorted(list(strings))]) + "]" + + +def get_run_config_from_file_list(file_list: Optional[Sequence[str]]) -> Mapping[str, object]: + check.opt_sequence_param(file_list, "file_list", of_type=str) + return cast(Mapping[str, object], load_yaml_from_glob_list(file_list) if file_list else {}) + + +def get_config_from_args(kwargs: Mapping[str, str]) -> Mapping[str, object]: + config = cast(Tuple[str, ...], kwargs.get("config")) # files + config_json = kwargs.get("config_json") + + if not config and not config_json: + return {} + + elif config and config_json: + raise click.UsageError("Cannot specify both -c / --config and --config-json") + + elif config: + config_file_list = list(check.opt_tuple_param(config, "config", of_type=str)) + return get_run_config_from_file_list(config_file_list) + + elif config_json: + config_json = cast(str, config_json) + try: + return json.loads(config_json) + + except JSONDecodeError: + raise click.UsageError( + f"Invalid JSON-string given for `--config-json`: {config_json}\n\n{serializable_error_info_from_exc_info(sys.exc_info()).to_string()}" + ) + else: + check.failed("Unhandled case getting config from kwargs") diff --git a/python_modules/dagster/dagster/_core/definitions/asset_graph.py b/python_modules/dagster/dagster/_core/definitions/asset_graph.py index fdf5b528385eb..57ac569366530 100644 --- a/python_modules/dagster/dagster/_core/definitions/asset_graph.py +++ b/python_modules/dagster/dagster/_core/definitions/asset_graph.py @@ -90,11 +90,11 @@ def owners(self) -> Sequence[str]: @property def is_partitioned(self) -> bool: - return self.assets_def.partitions_def is not None + return self.partitions_def is not None @property def partitions_def(self) -> Optional[PartitionsDefinition]: - return self.assets_def.partitions_def + return self.assets_def.specs_by_key[self.key].partitions_def @property def partition_mappings(self) -> Mapping[AssetKey, PartitionMapping]: diff --git a/python_modules/dagster/dagster/_core/definitions/asset_in.py b/python_modules/dagster/dagster/_core/definitions/asset_in.py index 79b06ec0647f2..448ec1d3476ea 100644 --- a/python_modules/dagster/dagster/_core/definitions/asset_in.py +++ b/python_modules/dagster/dagster/_core/definitions/asset_in.py @@ -77,3 +77,10 @@ def __new__( else resolve_dagster_type(dagster_type) ), ) + + @classmethod + def from_coercible(cls, coercible: "CoercibleToAssetIn") -> "AssetIn": + return coercible if isinstance(coercible, AssetIn) else AssetIn(key=coercible) + + +CoercibleToAssetIn = Union[AssetIn, CoercibleToAssetKey] diff --git a/python_modules/dagster/dagster/_core/definitions/asset_out.py b/python_modules/dagster/dagster/_core/definitions/asset_out.py index 050e1d9e03ea8..8d81cf665bcba 100644 --- a/python_modules/dagster/dagster/_core/definitions/asset_out.py +++ b/python_modules/dagster/dagster/_core/definitions/asset_out.py @@ -22,6 +22,7 @@ from dagster._core.definitions.freshness_policy import FreshnessPolicy from dagster._core.definitions.input import NoValueSentinel from dagster._core.definitions.output import Out +from dagster._core.definitions.partition import PartitionsDefinition from dagster._core.definitions.utils import resolve_automation_condition from dagster._core.errors import DagsterInvalidDefinitionError from dagster._core.types.dagster_type import DagsterType @@ -217,12 +218,17 @@ def to_out(self) -> Out: ) def to_spec( - self, key: AssetKey, deps: Sequence[AssetDep], additional_tags: Mapping[str, str] = {} + self, + key: AssetKey, + deps: Sequence[AssetDep], + additional_tags: Mapping[str, str] = {}, + partitions_def: Optional[PartitionsDefinition] = ..., ) -> AssetSpec: return self._spec.replace_attributes( key=key, tags={**additional_tags, **self.tags} if self.tags else additional_tags, deps=[*self._spec.deps, *deps], + partitions_def=partitions_def, ) @public diff --git a/python_modules/dagster/dagster/_core/definitions/assets.py b/python_modules/dagster/dagster/_core/definitions/assets.py index 064507025685d..1eb903545f868 100644 --- a/python_modules/dagster/dagster/_core/definitions/assets.py +++ b/python_modules/dagster/dagster/_core/definitions/assets.py @@ -36,7 +36,7 @@ AssetSpec, ) from dagster._core.definitions.auto_materialize_policy import AutoMaterializePolicy -from dagster._core.definitions.backfill_policy import BackfillPolicy, BackfillPolicyType +from dagster._core.definitions.backfill_policy import BackfillPolicy from dagster._core.definitions.declarative_automation.automation_condition import ( AutomationCondition, ) @@ -107,9 +107,9 @@ class AssetsDefinition(ResourceAddable, IHasInternalInit): "descriptions_by_key", "asset_deps", "owners_by_key", + "partitions_def", } - _partitions_def: Optional[PartitionsDefinition] # partition mappings are also tracked inside the AssetSpecs, but this enables faster access by # upstream asset key _partition_mappings: Mapping[AssetKey, PartitionMapping] @@ -229,24 +229,10 @@ def __init__( execution_type=execution_type or AssetExecutionType.MATERIALIZATION, ) - self._partitions_def = _resolve_partitions_def(specs, partitions_def) - self._resource_defs = wrap_resources_for_execution( check.opt_mapping_param(resource_defs, "resource_defs") ) - if self._partitions_def is None: - # check if backfill policy is BackfillPolicyType.SINGLE_RUN if asset is not partitioned - check.param_invariant( - ( - backfill_policy.policy_type is BackfillPolicyType.SINGLE_RUN - if backfill_policy - else True - ), - "backfill_policy", - "Non partitioned asset can only have single run backfill policy", - ) - if specs is not None: check.invariant(group_names_by_key is None) check.invariant(metadata_by_key is None) @@ -258,6 +244,7 @@ def __init__( check.invariant(owners_by_key is None) check.invariant(partition_mappings is None) check.invariant(asset_deps is None) + check.invariant(partitions_def is None) resolved_specs = specs else: @@ -297,6 +284,7 @@ def __init__( metadata_by_key=metadata_by_key, descriptions_by_key=descriptions_by_key, code_versions_by_key=None, + partitions_def=partitions_def, ) normalized_specs: List[AssetSpec] = [] @@ -333,11 +321,11 @@ def __init__( check.invariant( not ( spec.freshness_policy - and self._partitions_def is not None - and not isinstance(self._partitions_def, TimeWindowPartitionsDefinition) + and spec.partitions_def is not None + and not isinstance(spec.partitions_def, TimeWindowPartitionsDefinition) ), "FreshnessPolicies are currently unsupported for assets with partitions of type" - f" {type(self._partitions_def)}.", + f" {spec.partitions_def}.", ) normalized_specs.append( @@ -347,10 +335,19 @@ def __init__( metadata=metadata, description=description, skippable=skippable, - partitions_def=self._partitions_def, ) ) + unique_partitions_defs = { + spec.partitions_def for spec in normalized_specs if spec.partitions_def is not None + } + if len(unique_partitions_defs) > 1 and not can_subset: + raise DagsterInvalidDefinitionError( + "If different AssetSpecs have different partitions_defs, can_subset must be True" + ) + + _validate_self_deps(normalized_specs) + self._specs_by_key = {spec.key: spec for spec in normalized_specs} self._partition_mappings = get_partition_mappings_from_deps( @@ -363,27 +360,11 @@ def __init__( spec.key: spec for spec in self._check_specs_by_output_name.values() } - if self._computation: - _validate_self_deps( - input_keys=[ - key - # filter out the special inputs which are used for cases when a multi-asset is - # subsetted, as these are not the same as self-dependencies and are never loaded - # in the same step that their corresponding output is produced - for input_name, key in self._computation.keys_by_input_name.items() - if not input_name.startswith(ASSET_SUBSET_INPUT_PREFIX) - ], - output_keys=self._computation.selected_asset_keys, - partition_mappings=self._partition_mappings, - partitions_def=self._partitions_def, - ) - def dagster_internal_init( *, keys_by_input_name: Mapping[str, AssetKey], keys_by_output_name: Mapping[str, AssetKey], node_def: NodeDefinition, - partitions_def: Optional[PartitionsDefinition], selected_asset_keys: Optional[AbstractSet[AssetKey]], can_subset: bool, resource_defs: Optional[Mapping[str, object]], @@ -400,7 +381,6 @@ def dagster_internal_init( keys_by_input_name=keys_by_input_name, keys_by_output_name=keys_by_output_name, node_def=node_def, - partitions_def=partitions_def, selected_asset_keys=selected_asset_keys, can_subset=can_subset, resource_defs=resource_defs, @@ -771,17 +751,13 @@ def _output_dict_to_asset_dict( metadata_by_key=_output_dict_to_asset_dict(metadata_by_output_name), descriptions_by_key=_output_dict_to_asset_dict(descriptions_by_output_name), code_versions_by_key=_output_dict_to_asset_dict(code_versions_by_output_name), + partitions_def=partitions_def, ) return AssetsDefinition.dagster_internal_init( keys_by_input_name=keys_by_input_name, keys_by_output_name=keys_by_output_name_with_prefix, node_def=node_def, - partitions_def=check.opt_inst_param( - partitions_def, - "partitions_def", - PartitionsDefinition, - ), resource_defs=resource_defs, backfill_policy=check.opt_inst_param( backfill_policy, "backfill_policy", BackfillPolicy @@ -1044,10 +1020,20 @@ def backfill_policy(self) -> Optional[BackfillPolicy]: return self._computation.backfill_policy if self._computation else None @public - @property + @cached_property def partitions_def(self) -> Optional[PartitionsDefinition]: """Optional[PartitionsDefinition]: The PartitionsDefinition for this AssetsDefinition (if any).""" - return self._partitions_def + partitions_defs = { + spec.partitions_def for spec in self.specs if spec.partitions_def is not None + } + if len(partitions_defs) == 1: + return next(iter(partitions_defs)) + elif len(partitions_defs) == 0: + return None + else: + check.failed( + "Different assets within this AssetsDefinition have different PartitionsDefinitions" + ) @property def metadata_by_key(self) -> Mapping[AssetKey, ArbitraryMetadataMapping]: @@ -1138,12 +1124,17 @@ def get_partition_mapping_for_dep(self, dep_key: AssetKey) -> Optional[Partition return self._partition_mappings.get(dep_key) def infer_partition_mapping( - self, upstream_asset_key: AssetKey, upstream_partitions_def: Optional[PartitionsDefinition] + self, + asset_key: AssetKey, + upstream_asset_key: AssetKey, + upstream_partitions_def: Optional[PartitionsDefinition], ) -> PartitionMapping: with disable_dagster_warnings(): partition_mapping = self._partition_mappings.get(upstream_asset_key) return infer_partition_mapping( - partition_mapping, self._partitions_def, upstream_partitions_def + partition_mapping, + self.specs_by_key[asset_key].partitions_def, + upstream_partitions_def, ) def has_output_for_asset_key(self, key: AssetKey) -> bool: @@ -1398,7 +1389,7 @@ def _output_to_source_asset(self, output_name: str) -> SourceAsset: io_manager_key=output_def.io_manager_key, description=spec.description, resource_defs=self.resource_defs, - partitions_def=self.partitions_def, + partitions_def=spec.partitions_def, group_name=spec.group_name, tags=spec.tags, io_manager_def=None, @@ -1504,7 +1495,6 @@ def get_attributes_dict(self) -> Dict[str, Any]: keys_by_input_name=self.node_keys_by_input_name, keys_by_output_name=self.node_keys_by_output_name, node_def=self._computation.node_def if self._computation else None, - partitions_def=self._partitions_def, selected_asset_keys=self.keys, can_subset=self.can_subset, resource_defs=self._resource_defs, @@ -1700,6 +1690,7 @@ def _asset_specs_from_attr_key_params( code_versions_by_key: Optional[Mapping[AssetKey, str]], descriptions_by_key: Optional[Mapping[AssetKey, str]], owners_by_key: Optional[Mapping[AssetKey, Sequence[str]]], + partitions_def: Optional[PartitionsDefinition], ) -> Sequence[AssetSpec]: validated_group_names_by_key = check.opt_mapping_param( group_names_by_key, "group_names_by_key", key_type=AssetKey, value_type=str @@ -1772,27 +1763,24 @@ def _asset_specs_from_attr_key_params( # NodeDefinition skippable=False, auto_materialize_policy=None, - partitions_def=None, kinds=None, + partitions_def=check.opt_inst_param( + partitions_def, "partitions_def", PartitionsDefinition + ), ) ) return result -def _validate_self_deps( - input_keys: Iterable[AssetKey], - output_keys: Iterable[AssetKey], - partition_mappings: Mapping[AssetKey, PartitionMapping], - partitions_def: Optional[PartitionsDefinition], -) -> None: - output_keys_set = set(output_keys) - for input_key in input_keys: - if input_key in output_keys_set: - if input_key in partition_mappings: - partition_mapping = partition_mappings[input_key] +def _validate_self_deps(specs: Iterable[AssetSpec]) -> None: + for spec in specs: + for dep in spec.deps: + if dep.asset_key != spec.key: + continue + if dep.partition_mapping: time_window_partition_mapping = get_self_dep_time_window_partition_mapping( - partition_mapping, partitions_def + dep.partition_mapping, spec.partitions_def ) if ( time_window_partition_mapping is not None @@ -1802,7 +1790,7 @@ def _validate_self_deps( continue raise DagsterInvalidDefinitionError( - f'Asset "{input_key.to_user_string()}" depends on itself. Assets can only depend' + f'Asset "{spec.key.to_user_string()}" depends on itself. Assets can only depend' " on themselves if they are:\n(a) time-partitioned and each partition depends on" " earlier partitions\n(b) multipartitioned, with one time dimension that depends" " on earlier time partitions" @@ -1834,38 +1822,6 @@ def get_self_dep_time_window_partition_mapping( return None -def _resolve_partitions_def( - specs: Optional[Sequence[AssetSpec]], partitions_def: Optional[PartitionsDefinition] -) -> Optional[PartitionsDefinition]: - if specs: - asset_keys_by_partitions_def = defaultdict(set) - for spec in specs: - asset_keys_by_partitions_def[spec.partitions_def].add(spec.key) - if len(asset_keys_by_partitions_def) > 1: - partition_1_asset_keys, partition_2_asset_keys, *_ = ( - asset_keys_by_partitions_def.values() - ) - check.failed( - f"All AssetSpecs must have the same partitions_def, but " - f"{next(iter(partition_1_asset_keys)).to_user_string()} and " - f"{next(iter(partition_2_asset_keys)).to_user_string()} have different " - "partitions_defs." - ) - common_partitions_def = next(iter(asset_keys_by_partitions_def.keys())) - if ( - common_partitions_def is not None - and partitions_def is not None - and common_partitions_def != partitions_def - ): - check.failed( - f"AssetSpec for {next(iter(specs)).key.to_user_string()} has partitions_def which is different " - "than the partitions_def provided to AssetsDefinition.", - ) - return partitions_def or common_partitions_def - else: - return partitions_def - - def get_partition_mappings_from_deps( partition_mappings: Dict[AssetKey, PartitionMapping], deps: Iterable[AssetDep], asset_name: str ) -> Mapping[AssetKey, PartitionMapping]: diff --git a/python_modules/dagster/dagster/_core/definitions/automation_tick_evaluation_context.py b/python_modules/dagster/dagster/_core/definitions/automation_tick_evaluation_context.py index c6bdc353e0006..416851a12540e 100644 --- a/python_modules/dagster/dagster/_core/definitions/automation_tick_evaluation_context.py +++ b/python_modules/dagster/dagster/_core/definitions/automation_tick_evaluation_context.py @@ -197,19 +197,6 @@ def evaluate( ] -def _get_mapping_from_asset_partitions( - asset_partitions: AbstractSet[AssetKeyPartitionKey], asset_graph: BaseAssetGraph -) -> _PartitionsDefKeyMapping: - mapping: _PartitionsDefKeyMapping = defaultdict(set) - - for asset_partition in asset_partitions: - mapping[ - asset_graph.get(asset_partition.asset_key).partitions_def, asset_partition.partition_key - ].add(asset_partition.asset_key) - - return mapping - - def _get_mapping_from_entity_subsets( entity_subsets: Iterable[EntitySubset], asset_graph: BaseAssetGraph ) -> _PartitionsDefKeyMapping: @@ -236,18 +223,6 @@ def _get_mapping_from_entity_subsets( return mapping -def build_run_requests_from_asset_partitions( - asset_partitions: AbstractSet[AssetKeyPartitionKey], - asset_graph: BaseAssetGraph, - run_tags: Optional[Mapping[str, str]], -) -> Sequence[RunRequest]: - return _build_run_requests_from_partitions_def_mapping( - _get_mapping_from_asset_partitions(asset_partitions, asset_graph), - asset_graph, - run_tags, - ) - - def _build_backfill_request( entity_subsets: Sequence[EntitySubset[EntityKey]], asset_graph: BaseAssetGraph, @@ -373,9 +348,7 @@ def build_run_requests_with_backfill_policies( asset_graph: BaseAssetGraph, dynamic_partitions_store: DynamicPartitionsStore, ) -> Sequence[RunRequest]: - """If all assets have backfill policies, we should respect them and materialize them according - to their backfill policies. - """ + """Build run requests for a selection of asset partitions based on the associated BackfillPolicies.""" run_requests = [] asset_partition_keys: Mapping[AssetKey, Set[str]] = { @@ -406,9 +379,9 @@ def build_run_requests_with_backfill_policies( asset_check_keys = asset_graph.get_check_keys_for_assets(asset_keys) if partitions_def is None and partition_keys is not None: check.failed("Partition key provided for unpartitioned asset") - if partitions_def is not None and partition_keys is None: + elif partitions_def is not None and partition_keys is None: check.failed("Partition key missing for partitioned asset") - if partitions_def is None and partition_keys is None: + elif partitions_def is None and partition_keys is None: # non partitioned assets will be backfilled in a single run run_requests.append( RunRequest( @@ -417,6 +390,15 @@ def build_run_requests_with_backfill_policies( tags={}, ) ) + elif backfill_policy is None: + # just use the normal single-partition behavior + entity_keys = cast(Set[EntityKey], asset_keys) + mapping: _PartitionsDefKeyMapping = { + (partitions_def, pk): entity_keys for pk in (partition_keys or [None]) + } + run_requests.extend( + _build_run_requests_from_partitions_def_mapping(mapping, asset_graph, run_tags={}) + ) else: run_requests.extend( _build_run_requests_with_backfill_policy( diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/asset_check_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/asset_check_decorator.py index c52ee4735a1df..a04248af80be0 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/asset_check_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/asset_check_decorator.py @@ -1,4 +1,4 @@ -from typing import AbstractSet, Any, Callable, Iterable, Mapping, Optional, Sequence, Set, Union +from typing import Any, Callable, Iterable, Mapping, Optional, Sequence, Set, Union from typing_extensions import TypeAlias @@ -8,8 +8,8 @@ from dagster._core.definitions.asset_check_result import AssetCheckResult from dagster._core.definitions.asset_check_spec import AssetCheckSpec from dagster._core.definitions.asset_checks import AssetChecksDefinition -from dagster._core.definitions.asset_dep import CoercibleToAssetDep -from dagster._core.definitions.asset_in import AssetIn +from dagster._core.definitions.asset_dep import AssetDep, CoercibleToAssetDep +from dagster._core.definitions.asset_in import AssetIn, CoercibleToAssetIn from dagster._core.definitions.asset_key import AssetCheckKey from dagster._core.definitions.assets import AssetsDefinition from dagster._core.definitions.declarative_automation.automation_condition import ( @@ -20,9 +20,10 @@ DecoratorAssetsDefinitionBuilder, DecoratorAssetsDefinitionBuilderArgs, NamedIn, - build_named_ins, + build_and_validate_named_ins, compute_required_resource_keys, get_function_params_without_context_or_config_or_resources, + validate_named_ins_subset_of_deps, ) from dagster._core.definitions.decorators.op_decorator import _Op from dagster._core.definitions.events import AssetKey, CoercibleToAssetKey @@ -44,7 +45,7 @@ def _build_asset_check_named_ins( asset_key: AssetKey, fn: Callable[..., Any], additional_ins: Mapping[str, AssetIn], - additional_deps: Optional[AbstractSet[AssetKey]], + additional_deps: Mapping[AssetKey, AssetDep], ) -> Mapping[AssetKey, NamedIn]: fn_params = get_function_params_without_context_or_config_or_resources(fn) @@ -66,9 +67,9 @@ def _build_asset_check_named_ins( f"'{in_name}' is specified in 'additional_ins' but isn't a parameter." ) - # if all the fn_params are in additional_ins, then we add the prmary asset as a dep + # if all the fn_params are in additional_ins, then we add the primary asset as a dep if len(fn_params) == len(additional_ins): - all_deps = {*(additional_deps if additional_deps else set()), asset_key} + all_deps = {**additional_deps, **{asset_key: AssetDep(asset_key)}} all_ins = additional_ins # otherwise there should be one extra fn_param, which is the primary asset. Add that as an input elif len(fn_params) == len(additional_ins) + 1: @@ -87,10 +88,10 @@ def _build_asset_check_named_ins( " the target asset or be specified in 'additional_ins'." ) - return build_named_ins( + return build_and_validate_named_ins( fn=fn, asset_ins=all_ins, - deps=all_deps, + deps=all_deps.values(), ) @@ -189,7 +190,11 @@ def inner(fn: AssetCheckFunction) -> AssetChecksDefinition: resolved_name = name or fn.__name__ asset_key = AssetKey.from_coercible_or_definition(asset) - additional_dep_keys = set([dep.asset_key for dep in make_asset_deps(additional_deps) or []]) + additional_dep_keys = ( + {dep.asset_key: dep for dep in make_asset_deps(additional_deps) or []} + if additional_deps + else {} + ) named_in_by_asset_key = _build_asset_check_named_ins( resolved_name, asset_key, @@ -283,6 +288,7 @@ def multi_asset_check( required_resource_keys: Optional[Set[str]] = None, retry_policy: Optional[RetryPolicy] = None, config_schema: Optional[UserConfigSchema] = None, + ins: Optional[Mapping[str, CoercibleToAssetIn]] = None, ) -> Callable[[Callable[..., Any]], AssetChecksDefinition]: """Defines a set of asset checks that can be executed together with the same op. @@ -306,6 +312,8 @@ def multi_asset_check( retry_policy (Optional[RetryPolicy]): The retry policy for the op that executes the checks. can_subset (bool): Whether the op can emit results for a subset of the asset checks keys, based on the context.selected_asset_check_keys argument. Defaults to False. + ins (Optional[Mapping[str, Union[AssetKey, AssetIn]]]): A mapping from input name to AssetIn depended upon by + a given asset check. If an AssetKey is provided, it will be converted to an AssetIn with the same key. Examples: @@ -345,12 +353,21 @@ def inner(fn: MultiAssetCheckFunction) -> AssetChecksDefinition: outs = { spec.get_python_identifier(): Out(None, is_required=not can_subset) for spec in specs } - named_ins_by_asset_key = build_named_ins( + all_deps_by_key = { + **{spec.asset_key: AssetDep(spec.asset_key) for spec in specs}, + **{dep.asset_key: dep for spec in specs for dep in (spec.additional_deps or [])}, + } + + named_ins_by_asset_key = build_and_validate_named_ins( fn=fn, - asset_ins={}, - deps={spec.asset_key for spec in specs} - | {dep.asset_key for spec in specs for dep in spec.additional_deps or []}, + asset_ins={ + inp_name: AssetIn.from_coercible(coercible) for inp_name, coercible in ins.items() + } + if ins + else {}, + deps=all_deps_by_key.values(), ) + validate_named_ins_subset_of_deps(named_ins_by_asset_key, all_deps_by_key) with disable_dagster_warnings(): op_def = _Op( diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py index 3aaa35ba4a0c3..4f53f0668b03b 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py @@ -40,7 +40,7 @@ from dagster._core.definitions.decorators.decorator_assets_definition_builder import ( DecoratorAssetsDefinitionBuilder, DecoratorAssetsDefinitionBuilderArgs, - build_named_ins, + build_and_validate_named_ins, build_named_outs, create_check_specs_by_output_name, validate_and_assign_output_names_to_check_specs, @@ -911,7 +911,7 @@ def graph_asset_no_defaults( kinds: Optional[AbstractSet[str]], ) -> AssetsDefinition: ins = ins or {} - named_ins = build_named_ins(compose_fn, ins or {}, set()) + named_ins = build_and_validate_named_ins(compose_fn, ins or {}, set()) out_asset_key, _asset_name = resolve_asset_key_and_name_for_decorator( key=key, key_prefix=key_prefix, @@ -1030,7 +1030,7 @@ def inner(fn: Callable[..., Any]) -> AssetsDefinition: if asset_in.partition_mapping } - named_ins = build_named_ins(fn, ins or {}, set()) + named_ins = build_and_validate_named_ins(fn, ins or {}, set()) keys_by_input_name = { input_name: asset_key for asset_key, (input_name, _) in named_ins.items() } diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/decorator_assets_definition_builder.py b/python_modules/dagster/dagster/_core/definitions/decorators/decorator_assets_definition_builder.py index 0eab760bc93e7..cd5d477265db9 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/decorator_assets_definition_builder.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/decorator_assets_definition_builder.py @@ -75,13 +75,39 @@ def get_function_params_without_context_or_config_or_resources( return new_input_args -def build_named_ins( +def validate_can_coexist(asset_in: AssetIn, asset_dep: AssetDep) -> None: + """Validates that the asset_in and asset_dep can coexist peacefully on the same asset key. + If both asset_in and asset_dep are set on the same asset key, expect that _no_ properties + are set on AssetIn except for the key itself. + """ + if ( + asset_in.metadata + or asset_in.key_prefix + or asset_in.dagster_type != NoValueSentinel + or asset_in.partition_mapping is not None + ): + raise DagsterInvalidDefinitionError( + f"Asset key '{asset_dep.asset_key.to_user_string()}' is used as both an input (via AssetIn) and a dependency (via AssetDep). If an asset key is used as an input and also set as a dependency, the input should only define the relationship between the asset key and the input name, or optionally set the input_manager_key. Any other properties should either not be set, or should be set on the dependency." + ) + + +def build_and_validate_named_ins( fn: Callable[..., Any], asset_ins: Mapping[str, AssetIn], - deps: Optional[AbstractSet[AssetKey]], + deps: Optional[Iterable[AssetDep]], ) -> Mapping[AssetKey, "NamedIn"]: """Creates a mapping from AssetKey to (name of input, In object).""" - deps = check.opt_set_param(deps, "deps", AssetKey) + deps_by_key = {dep.asset_key: dep for dep in deps} if deps else {} + ins_by_asset_key = { + asset_in.key if asset_in.key else AssetKey.from_coercible(input_name): asset_in + for input_name, asset_in in asset_ins.items() + } + shared_keys_between_ins_and_deps = set(ins_by_asset_key.keys()) & set(deps_by_key.keys()) + if shared_keys_between_ins_and_deps: + for shared_key in shared_keys_between_ins_and_deps: + validate_can_coexist(ins_by_asset_key[shared_key], deps_by_key[shared_key]) + + deps = check.opt_iterable_param(deps, "deps", AssetDep) new_input_args = get_function_params_without_context_or_config_or_resources(fn) @@ -126,16 +152,12 @@ def build_named_ins( In(metadata=metadata, input_manager_key=input_manager_key, dagster_type=dagster_type), ) - for asset_key in deps: - if asset_key in named_ins_by_asset_key: - raise DagsterInvalidDefinitionError( - f"deps value {asset_key} also declared as input/AssetIn" + for dep in deps: + if dep.asset_key not in named_ins_by_asset_key: + named_ins_by_asset_key[dep.asset_key] = NamedIn( + stringify_asset_key_to_input_name(dep.asset_key), + In(cast(type, Nothing)), ) - # mypy doesn't realize that Nothing is a valid type here - named_ins_by_asset_key[asset_key] = NamedIn( - stringify_asset_key_to_input_name(asset_key), - In(cast(type, Nothing)), - ) return named_ins_by_asset_key @@ -348,25 +370,23 @@ def from_multi_asset_specs( ), ) - upstream_keys = set() + upstream_deps = {} for spec in asset_specs: for dep in spec.deps: if dep.asset_key not in named_outs_by_asset_key: - upstream_keys.add(dep.asset_key) + upstream_deps[dep.asset_key] = dep if dep.asset_key in named_outs_by_asset_key and dep.partition_mapping is not None: # self-dependent asset also needs to be considered an upstream_key - upstream_keys.add(dep.asset_key) + upstream_deps[dep.asset_key] = dep # get which asset keys have inputs set - loaded_upstreams = build_named_ins(fn, asset_in_map, deps=set()) - unexpected_upstreams = {key for key in loaded_upstreams.keys() if key not in upstream_keys} - if unexpected_upstreams: - raise DagsterInvalidDefinitionError( - f"Asset inputs {unexpected_upstreams} do not have dependencies on the passed" - " AssetSpec(s). Set the deps on the appropriate AssetSpec(s)." - ) - remaining_upstream_keys = {key for key in upstream_keys if key not in loaded_upstreams} - named_ins_by_asset_key = build_named_ins(fn, asset_in_map, deps=remaining_upstream_keys) + named_ins_by_asset_key = build_and_validate_named_ins( + fn, asset_in_map, deps=upstream_deps.values() + ) + # We expect that asset_ins are a subset of asset_deps. The reason we do not check this in + # `build_and_validate_named_ins` is because in other decorator pathways, we allow for argument-based + # dependencies which are not specified in deps (such as the asset decorator). + validate_named_ins_subset_of_deps(named_ins_by_asset_key, upstream_deps) internal_deps = { spec.key: {dep.asset_key for dep in spec.deps} @@ -401,10 +421,10 @@ def from_asset_outs_in_asset_centric_decorator( check.param_invariant( not passed_args.specs, "args", "This codepath for non-spec based create" ) - named_ins_by_asset_key = build_named_ins( + named_ins_by_asset_key = build_and_validate_named_ins( fn, asset_in_map, - deps=({dep.asset_key for dep in upstream_asset_deps} if upstream_asset_deps else set()), + deps=upstream_asset_deps or set(), ) named_outs_by_asset_key = build_named_outs(asset_out_map) @@ -560,7 +580,6 @@ def create_assets_definition(self) -> AssetsDefinition: keys_by_input_name=self.asset_keys_by_input_names, keys_by_output_name=self.asset_keys_by_output_name, node_def=self.create_op_definition(), - partitions_def=self.args.partitions_def, can_subset=self.args.can_subset, resource_defs=self.args.assets_def_resource_defs, backfill_policy=self.args.backfill_policy, @@ -574,18 +593,61 @@ def create_assets_definition(self) -> AssetsDefinition: @cached_property def specs(self) -> Sequence[AssetSpec]: - specs = self.args.specs if self.args.specs else self._synthesize_specs() - - if not self.group_name: - return specs + if self.args.specs: + specs = self.args.specs + self._validate_spec_partitions_defs(specs, self.args.partitions_def) + else: + specs = self._synthesize_specs() check.invariant( - all((spec.group_name is None or spec.group_name == self.group_name) for spec in specs), + not self.group_name + or all( + (spec.group_name is None or spec.group_name == self.group_name) for spec in specs + ), "Cannot set group_name parameter on multi_asset if one or more of the" " AssetSpecs/AssetOuts supplied to this multi_asset have a group_name defined.", ) - return [spec.replace_attributes(group_name=self.group_name) for spec in specs] + if not self.group_name and not self.args.partitions_def: + return specs + + return [ + spec.replace_attributes( + group_name=self.group_name, + partitions_def=spec.partitions_def or self.args.partitions_def, + ) + for spec in specs + ] + + def _validate_spec_partitions_defs( + self, specs: Sequence[AssetSpec], partitions_def: Optional[PartitionsDefinition] + ) -> Optional[PartitionsDefinition]: + any_spec_has_partitions_def = False + any_spec_has_no_partitions_def = False + if partitions_def is not None: + for spec in specs: + if spec.partitions_def is not None and spec.partitions_def != partitions_def: + check.failed( + f"AssetSpec for {spec.key.to_user_string()} has partitions_def " + f"(type={type(spec.partitions_def)}) which is different than the " + f"partitions_def provided to AssetsDefinition (type={type(partitions_def)}).", + ) + + any_spec_has_partitions_def = ( + any_spec_has_partitions_def or spec.partitions_def is not None + ) + any_spec_has_no_partitions_def = ( + any_spec_has_no_partitions_def or spec.partitions_def is None + ) + + if ( + partitions_def is not None + and any_spec_has_partitions_def + and any_spec_has_no_partitions_def + ): + check.failed( + "If partitions_def is provided, then either all specs must have that PartitionsDefinition or none." + ) def _synthesize_specs(self) -> Sequence[AssetSpec]: resolved_specs = [] @@ -610,7 +672,9 @@ def _synthesize_specs(self) -> Sequence[AssetSpec]: else: deps = input_deps - resolved_specs.append(asset_out.to_spec(key, deps=deps)) + resolved_specs.append( + asset_out.to_spec(key, deps=deps, partitions_def=self.args.partitions_def) + ) specs = resolved_specs return specs @@ -653,3 +717,22 @@ def _validate_check_specs_target_relevant_asset_keys( f"Invalid asset key {spec.asset_key} in check spec {spec.name}. Must be one of" f" {valid_asset_keys}" ) + + +def validate_named_ins_subset_of_deps( + named_ins_per_key: Mapping[AssetKey, NamedIn], + asset_deps_by_key: Mapping[AssetKey, AssetDep], +) -> None: + """Validates that the asset_ins are a subset of the asset_deps. This is a common validation + that we need to do in multiple places, so we've factored it out into a helper function. + """ + asset_dep_keys = set(asset_deps_by_key.keys()) + asset_in_keys = set(named_ins_per_key.keys()) + + if asset_in_keys - asset_dep_keys: + invalid_asset_in_keys = asset_in_keys - asset_dep_keys + raise DagsterInvalidDefinitionError( + f"Invalid asset dependencies: `{invalid_asset_in_keys}` specified as AssetIns, but" + " are not specified as `AssetDep` objects on any constituent AssetSpec objects. Asset inputs must be associated with an" + " output produced by the asset." + ) diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/sensor_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/sensor_decorator.py index e571e075c4193..9c98321c1c9bb 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/sensor_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/sensor_decorator.py @@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Sequence, Set, Union import dagster._check as check -from dagster._annotations import deprecated from dagster._core.definitions.asset_selection import AssetSelection, CoercibleToAssetSelection from dagster._core.definitions.asset_sensor_definition import AssetSensorDefinition from dagster._core.definitions.events import AssetKey @@ -248,7 +247,6 @@ def _wrapped_fn(*args, **kwargs) -> Any: return inner -@deprecated(breaking_version="2.0.0", additional_warn_text="use `AutomationConditions` instead") def multi_asset_sensor( monitored_assets: Union[Sequence[AssetKey], AssetSelection], *, diff --git a/python_modules/dagster/dagster/_core/definitions/external_asset.py b/python_modules/dagster/dagster/_core/definitions/external_asset.py index 57e7f3ad6fb4e..ef42bee63e384 100644 --- a/python_modules/dagster/dagster/_core/definitions/external_asset.py +++ b/python_modules/dagster/dagster/_core/definitions/external_asset.py @@ -153,13 +153,13 @@ def create_external_asset_from_source_asset(source_asset: SourceAsset) -> Assets automation_condition=source_asset.automation_condition, deps=[], owners=[], + partitions_def=source_asset.partitions_def, ) return AssetsDefinition( specs=[spec], keys_by_output_name=keys_by_output_name, node_def=node_def, - partitions_def=source_asset.partitions_def, # We don't pass the `io_manager_def` because it will already be present in # `resource_defs` (it is added during `SourceAsset` initialization). resource_defs=source_asset.resource_defs, diff --git a/python_modules/dagster/dagster/_core/definitions/multi_asset_sensor_definition.py b/python_modules/dagster/dagster/_core/definitions/multi_asset_sensor_definition.py index b98f6ba5a8253..a30322a421fe5 100644 --- a/python_modules/dagster/dagster/_core/definitions/multi_asset_sensor_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/multi_asset_sensor_definition.py @@ -265,13 +265,11 @@ def __init__( self._partitions_def_by_asset_key: Dict[AssetKey, Optional[PartitionsDefinition]] = {} asset_graph = self._repository_def.asset_graph for asset_key in self._monitored_asset_keys: - assets_def = ( - asset_graph.get(asset_key).assets_def if asset_graph.has(asset_key) else None - ) - self._assets_by_key[asset_key] = assets_def + asset_node = asset_graph.get(asset_key) if asset_graph.has(asset_key) else None + self._assets_by_key[asset_key] = asset_node.assets_def if asset_node else None self._partitions_def_by_asset_key[asset_key] = ( - assets_def.partitions_def if assets_def else None + asset_node.partitions_def if asset_node else None ) # Cursor object with utility methods for updating and retrieving cursor information. @@ -744,24 +742,25 @@ def get_downstream_partition_keys( to_asset = self._get_asset(to_asset_key, fn_name="get_downstream_partition_keys") from_asset = self._get_asset(from_asset_key, fn_name="get_downstream_partition_keys") - to_partitions_def = to_asset.partitions_def + to_partitions_def = to_asset.specs_by_key[to_asset_key].partitions_def + from_partitions_def = from_asset.specs_by_key[from_asset_key].partitions_def if not isinstance(to_partitions_def, PartitionsDefinition): raise DagsterInvalidInvocationError( f"Asset key {to_asset_key} is not partitioned. Cannot get partition keys." ) - if not isinstance(from_asset.partitions_def, PartitionsDefinition): + if not isinstance(from_partitions_def, PartitionsDefinition): raise DagsterInvalidInvocationError( f"Asset key {from_asset_key} is not partitioned. Cannot get partition keys." ) partition_mapping = to_asset.infer_partition_mapping( - from_asset_key, from_asset.partitions_def + to_asset_key, from_asset_key, from_partitions_def ) downstream_partition_key_subset = ( partition_mapping.get_downstream_partitions_for_partitions( - from_asset.partitions_def.empty_subset().with_partition_keys([partition_key]), - from_asset.partitions_def, + from_partitions_def.empty_subset().with_partition_keys([partition_key]), + from_partitions_def, downstream_partitions_def=to_partitions_def, dynamic_partitions_store=self.instance, ) diff --git a/python_modules/dagster/dagster/_core/definitions/repository_definition/repository_data_builder.py b/python_modules/dagster/dagster/_core/definitions/repository_definition/repository_data_builder.py index da29ce7539dcb..938957c149f04 100644 --- a/python_modules/dagster/dagster/_core/definitions/repository_definition/repository_data_builder.py +++ b/python_modules/dagster/dagster/_core/definitions/repository_definition/repository_data_builder.py @@ -255,14 +255,15 @@ def build_caching_repository_data_from_list( asset_check_keys.update(definition.check_keys) asset_checks_defs.append(definition) elif isinstance(definition, AssetsDefinition): - for key in definition.keys: - if key in asset_keys: - raise DagsterInvalidDefinitionError(f"Duplicate asset key: {key}") + for spec in definition.specs: + if spec.key in asset_keys: + raise DagsterInvalidDefinitionError(f"Duplicate asset key: {spec.key}") + + if spec.partitions_def is not None: + partitions_defs.add(spec.partitions_def) for key in definition.check_keys: if key in asset_check_keys: raise DagsterInvalidDefinitionError(f"Duplicate asset check key: {key}") - if definition.partitions_def is not None: - partitions_defs.add(definition.partitions_def) asset_keys.update(definition.keys) asset_check_keys.update(definition.check_keys) diff --git a/python_modules/dagster/dagster/_core/definitions/selector.py b/python_modules/dagster/dagster/_core/definitions/selector.py index 5bff31e3a57cd..7f3ce30d3b36c 100644 --- a/python_modules/dagster/dagster/_core/definitions/selector.py +++ b/python_modules/dagster/dagster/_core/definitions/selector.py @@ -130,7 +130,7 @@ def repository_selector(self) -> "RepositorySelector": @whitelist_for_serdes @record -class RepositorySelector(IHaveNew): +class RepositorySelector: location_name: str repository_name: str @@ -167,17 +167,10 @@ def from_graphql_input(graphql_data): ) -@record_custom -class CodeLocationSelector(IHaveNew): +@record(kw_only=False) +class CodeLocationSelector: location_name: str - # allow posargs to avoid breaking change - def __new__(cls, location_name: str): - return super().__new__( - cls, - location_name=location_name, - ) - def to_repository_selector(self) -> RepositorySelector: return RepositorySelector( location_name=self.location_name, @@ -318,21 +311,13 @@ def to_graphql_input(self): } -@record_custom -class PartitionRangeSelector(IHaveNew): +@record(kw_only=False) +class PartitionRangeSelector: """The information needed to resolve a partition range.""" start: str end: str - # allow posargs - def __new__(cls, start: str, end: str): - return super().__new__( - cls, - start=start, - end=end, - ) - def to_graphql_input(self): return { "start": self.start, @@ -347,19 +332,12 @@ def from_graphql_input(graphql_data): ) -@record_custom -class PartitionsSelector(IHaveNew): +@record(kw_only=False) +class PartitionsSelector: """The information needed to define selection partitions.""" ranges: Sequence[PartitionRangeSelector] - # allow posargs - def __new__(cls, ranges: Sequence[PartitionRangeSelector]): - return super().__new__( - cls, - ranges=ranges, - ) - def to_graphql_input(self): return {"ranges": [partition_range.to_graphql_input() for partition_range in self.ranges]} diff --git a/python_modules/dagster/dagster/_core/execution/asset_backfill.py b/python_modules/dagster/dagster/_core/execution/asset_backfill.py index 5d33dc5246b6c..e66b7f08a05cc 100644 --- a/python_modules/dagster/dagster/_core/execution/asset_backfill.py +++ b/python_modules/dagster/dagster/_core/execution/asset_backfill.py @@ -25,7 +25,6 @@ from dagster._core.definitions.asset_graph_subset import AssetGraphSubset from dagster._core.definitions.asset_selection import KeysAssetSelection from dagster._core.definitions.automation_tick_evaluation_context import ( - build_run_requests_from_asset_partitions, build_run_requests_with_backfill_policies, ) from dagster._core.definitions.base_asset_graph import BaseAssetGraph, BaseAssetNode @@ -1533,37 +1532,11 @@ def _format_keys(keys: Iterable[AssetKeyPartitionKey]): f"The following assets were considered for materialization but not requested:\n\n{not_requested_str}" ) - # check if all assets have backfill policies if any of them do, otherwise, raise error - asset_backfill_policies = [ - asset_graph.get(asset_key).backfill_policy - for asset_key in { - asset_partition.asset_key for asset_partition in asset_partitions_to_request - } - ] - all_assets_have_backfill_policies = all( - backfill_policy is not None for backfill_policy in asset_backfill_policies + run_requests = build_run_requests_with_backfill_policies( + asset_partitions=asset_partitions_to_request, + asset_graph=asset_graph, + dynamic_partitions_store=instance_queryer, ) - if all_assets_have_backfill_policies: - run_requests = build_run_requests_with_backfill_policies( - asset_partitions=asset_partitions_to_request, - asset_graph=asset_graph, - dynamic_partitions_store=instance_queryer, - ) - else: - if not all(backfill_policy is None for backfill_policy in asset_backfill_policies): - # if some assets have backfill policies, but not all of them, raise error - raise DagsterBackfillFailedError( - "Either all assets must have backfill policies or none of them must have backfill" - " policies. To backfill these assets together, either add backfill policies to all" - " assets, or remove backfill policies from all assets." - ) - # When any of the assets do not have backfill policies, we fall back to the default behavior of - # backfilling them partition by partition. - run_requests = build_run_requests_from_asset_partitions( - asset_partitions=asset_partitions_to_request, - asset_graph=asset_graph, - run_tags={}, - ) if request_roots: check.invariant( diff --git a/python_modules/dagster/dagster/_core/execution/context/op_execution_context.py b/python_modules/dagster/dagster/_core/execution/context/op_execution_context.py index 6b3d90105c8b8..8b1466ba00f23 100644 --- a/python_modules/dagster/dagster/_core/execution/context/op_execution_context.py +++ b/python_modules/dagster/dagster/_core/execution/context/op_execution_context.py @@ -292,7 +292,7 @@ def an_asset(context: AssetExecutionContext): # ["2023-08-21", "2023-08-22", "2023-08-23", "2023-08-24", "2023-08-25"] """ key_range = self.partition_key_range - partitions_def = self.assets_def.partitions_def + partitions_def = self._step_execution_context.run_partitions_def if partitions_def is None: raise DagsterInvariantViolationError( "Cannot access partition_keys for a non-partitioned run" diff --git a/python_modules/dagster/dagster/_core/execution/stats.py b/python_modules/dagster/dagster/_core/execution/stats.py index 7ae0756838f5d..bced58da9aa75 100644 --- a/python_modules/dagster/dagster/_core/execution/stats.py +++ b/python_modules/dagster/dagster/_core/execution/stats.py @@ -1,6 +1,6 @@ from collections import defaultdict from enum import Enum -from typing import Any, Dict, Iterable, NamedTuple, Optional, Sequence, cast +from typing import Any, Dict, Iterable, Mapping, Optional, Sequence, cast import dagster._check as check from dagster._core.definitions import ExpectationResult @@ -12,6 +12,7 @@ ) from dagster._core.events.log import EventLogEntry from dagster._core.storage.dagster_run import DagsterRunStatsSnapshot +from dagster._record import IHaveNew, record, record_custom from dagster._serdes import whitelist_for_serdes RUN_STATS_EVENT_TYPES = { @@ -37,7 +38,9 @@ def build_run_stats_from_events( - run_id: str, entries: Iterable[EventLogEntry] + run_id: str, + entries: Iterable[EventLogEntry], + previous_stats: Optional[DagsterRunStatsSnapshot] = None, ) -> DagsterRunStatsSnapshot: try: iter(entries) @@ -45,17 +48,27 @@ def build_run_stats_from_events( raise check.ParameterCheckError( "Invariant violation for parameter 'records'. Description: Expected iterable." ) from exc - for i, record in enumerate(entries): - check.inst_param(record, f"records[{i}]", EventLogEntry) - - steps_succeeded = 0 - steps_failed = 0 - materializations = 0 - expectations = 0 - enqueued_time = None - launch_time = None - start_time = None - end_time = None + for i, entry in enumerate(entries): + check.inst_param(entry, f"entries[{i}]", EventLogEntry) + + if previous_stats: + steps_succeeded = previous_stats.steps_succeeded + steps_failed = previous_stats.steps_failed + materializations = previous_stats.materializations + expectations = previous_stats.expectations + enqueued_time = previous_stats.enqueued_time + launch_time = previous_stats.launch_time + start_time = previous_stats.start_time + end_time = previous_stats.end_time + else: + steps_succeeded = 0 + steps_failed = 0 + materializations = 0 + expectations = 0 + enqueued_time = None + launch_time = None + start_time = None + end_time = None for event in entries: if not event.is_dagster_event: @@ -103,14 +116,154 @@ class StepEventStatus(Enum): IN_PROGRESS = "IN_PROGRESS" +@whitelist_for_serdes +@record_custom +class RunStepMarker(IHaveNew): + start_time: Optional[float] + end_time: Optional[float] + key: Optional[str] + + def __new__( + cls, + start_time: Optional[float] = None, + end_time: Optional[float] = None, + key: Optional[str] = None, + ): + return super().__new__( + cls, + start_time=check.opt_float_param(start_time, "start_time"), + end_time=check.opt_float_param(end_time, "end_time"), + key=check.opt_str_param(key, "key"), + ) + + +@whitelist_for_serdes +@record_custom +class RunStepKeyStatsSnapshot(IHaveNew): + run_id: str + step_key: str + status: Optional[StepEventStatus] + start_time: Optional[float] + end_time: Optional[float] + materialization_events: Sequence[EventLogEntry] + expectation_results: Sequence[ExpectationResult] + attempts: Optional[int] + attempts_list: Sequence[RunStepMarker] + markers: Sequence[RunStepMarker] + partial_attempt_start: Optional[float] + + def __new__( + cls, + run_id: str, + step_key: str, + status: Optional[StepEventStatus] = None, + start_time: Optional[float] = None, + end_time: Optional[float] = None, + materialization_events: Optional[Sequence[EventLogEntry]] = None, + expectation_results: Optional[Sequence[ExpectationResult]] = None, + attempts: Optional[int] = None, + attempts_list: Optional[Sequence[RunStepMarker]] = None, + markers: Optional[Sequence[RunStepMarker]] = None, + partial_attempt_start: Optional[float] = None, + ): + return super().__new__( + cls, + run_id=check.str_param(run_id, "run_id"), + step_key=check.str_param(step_key, "step_key"), + status=check.opt_inst_param(status, "status", StepEventStatus), + start_time=check.opt_float_param(start_time, "start_time"), + end_time=check.opt_float_param(end_time, "end_time"), + materialization_events=check.opt_sequence_param( + materialization_events, + "materialization_events", + EventLogEntry, + ), + expectation_results=check.opt_sequence_param( + expectation_results, "expectation_results", ExpectationResult + ), + attempts=check.opt_int_param(attempts, "attempts"), + attempts_list=check.opt_sequence_param(attempts_list, "attempts_list", RunStepMarker), + markers=check.opt_sequence_param(markers, "markers", RunStepMarker), + # used to calculate incremental step stats using batches of event logs + partial_attempt_start=check.opt_float_param( + partial_attempt_start, "partial_attempt_start" + ), + ) + + +@whitelist_for_serdes +@record +class RunStepStatsSnapshot: + run_id: str + step_key_stats: Sequence[RunStepKeyStatsSnapshot] + partial_markers: Optional[Mapping[str, Sequence[RunStepMarker]]] + + def build_run_step_stats_from_events( - run_id: str, records: Iterable[EventLogEntry] -) -> Sequence["RunStepKeyStatsSnapshot"]: + run_id: str, + entries: Iterable[EventLogEntry], +) -> Sequence[RunStepKeyStatsSnapshot]: + snapshot = build_run_step_stats_snapshot_from_events(run_id, entries) + return snapshot.step_key_stats + + +def build_run_step_stats_snapshot_from_events( + run_id: str, + entries: Iterable[EventLogEntry], + previous_snapshot: Optional["RunStepStatsSnapshot"] = None, +) -> "RunStepStatsSnapshot": by_step_key: Dict[str, Dict[str, Any]] = defaultdict(dict) attempts = defaultdict(list) - attempt_events = defaultdict(list) markers: Dict[str, Dict[str, Any]] = defaultdict(dict) - for event in records: + + if previous_snapshot: + for step_stats in previous_snapshot.step_key_stats: + check.invariant(step_stats.run_id == run_id) + by_step_key[step_stats.step_key] = { + "start_time": step_stats.start_time, + "end_time": step_stats.end_time, + "status": step_stats.status, + "materialization_events": step_stats.materialization_events, + "expectation_results": step_stats.expectation_results, + "attempts": step_stats.attempts, + "partial_attempt_start": step_stats.partial_attempt_start, + } + for attempt in step_stats.attempts_list: + attempts[step_stats.step_key].append(attempt) + + for marker in step_stats.markers: + assert marker.key + markers[step_stats.step_key][marker.key] = { + "key": marker.key, + "start": marker.start_time, + "end": marker.end_time, + } + + # handle the partial markers + if previous_snapshot.partial_markers: + for step_key, partial_markers in previous_snapshot.partial_markers.items(): + for marker in partial_markers: + assert marker.key + markers[step_key][marker.key] = { + "key": marker.key, + "start": marker.start_time, + "end": marker.end_time, + } + + def _open_attempt(step_key: str, event: EventLogEntry) -> None: + by_step_key[step_key]["attempts"] = int(by_step_key[step_key].get("attempts") or 0) + 1 + by_step_key[step_key]["partial_attempt_start"] = event.timestamp + + def _close_attempt(step_key: str, event: EventLogEntry) -> None: + attempts[step_key].append( + RunStepMarker( + start_time=by_step_key[step_key].get("partial_attempt_start"), + end_time=event.timestamp, + ) + ) + by_step_key[step_key]["partial_attempt_start"] = None + + for event in entries: if not event.is_dagster_event: continue dagster_event = event.get_dagster_event() @@ -123,19 +276,25 @@ def build_run_step_stats_from_events( continue if dagster_event.event_type == DagsterEventType.STEP_START: + by_step_key[step_key]["status"] = StepEventStatus.IN_PROGRESS by_step_key[step_key]["start_time"] = event.timestamp - by_step_key[step_key]["attempts"] = 1 + _open_attempt(step_key, event) + if dagster_event.event_type == DagsterEventType.STEP_RESTARTED: + _open_attempt(step_key, event) + if dagster_event.event_type == DagsterEventType.STEP_UP_FOR_RETRY: + _close_attempt(step_key, event) if dagster_event.event_type == DagsterEventType.STEP_FAILURE: by_step_key[step_key]["end_time"] = event.timestamp by_step_key[step_key]["status"] = StepEventStatus.FAILURE - if dagster_event.event_type == DagsterEventType.STEP_RESTARTED: - by_step_key[step_key]["attempts"] = int(by_step_key[step_key].get("attempts") or 0) + 1 + _close_attempt(step_key, event) if dagster_event.event_type == DagsterEventType.STEP_SUCCESS: by_step_key[step_key]["end_time"] = event.timestamp by_step_key[step_key]["status"] = StepEventStatus.SUCCESS + _close_attempt(step_key, event) if dagster_event.event_type == DagsterEventType.STEP_SKIPPED: by_step_key[step_key]["end_time"] = event.timestamp by_step_key[step_key]["status"] = StepEventStatus.SKIPPED + _close_attempt(step_key, event) if dagster_event.event_type == DagsterEventType.ASSET_MATERIALIZATION: materialization_events = by_step_key[step_key].get("materialization_events", []) materialization_events.append(event) @@ -146,129 +305,50 @@ def build_run_step_stats_from_events( step_expectation_results = by_step_key[step_key].get("expectation_results", []) step_expectation_results.append(expectation_result) by_step_key[step_key]["expectation_results"] = step_expectation_results - if dagster_event.event_type in ( - DagsterEventType.STEP_UP_FOR_RETRY, - DagsterEventType.STEP_RESTARTED, - ): - attempt_events[step_key].append(event) + if dagster_event.event_type in MARKER_EVENTS: if dagster_event.engine_event_data.marker_start: - key = dagster_event.engine_event_data.marker_start - if key not in markers[step_key]: - markers[step_key][key] = {"key": key, "start": event.timestamp} + marker_key = dagster_event.engine_event_data.marker_start + if marker_key not in markers[step_key]: + markers[step_key][marker_key] = {"key": marker_key, "start": event.timestamp} else: - markers[step_key][key]["start"] = event.timestamp + markers[step_key][marker_key]["start"] = event.timestamp if dagster_event.engine_event_data.marker_end: - key = dagster_event.engine_event_data.marker_end - if key not in markers[step_key]: - markers[step_key][key] = {"key": key, "end": event.timestamp} + marker_key = dagster_event.engine_event_data.marker_end + if marker_key not in markers[step_key]: + markers[step_key][marker_key] = {"key": marker_key, "end": event.timestamp} else: - markers[step_key][key]["end"] = event.timestamp + markers[step_key][marker_key]["end"] = event.timestamp + snapshots = [] for step_key, step_stats in by_step_key.items(): - events = attempt_events[step_key] - step_attempts = [] - attempt_start = step_stats.get("start_time") - - for event in events: - if not event.dagster_event: - continue - if event.dagster_event.event_type == DagsterEventType.STEP_UP_FOR_RETRY: - step_attempts.append( - RunStepMarker(start_time=attempt_start, end_time=event.timestamp) - ) - elif event.dagster_event.event_type == DagsterEventType.STEP_RESTARTED: - attempt_start = event.timestamp - if step_stats.get("end_time"): - step_attempts.append( - RunStepMarker(start_time=attempt_start, end_time=step_stats["end_time"]) + snapshots.append( + RunStepKeyStatsSnapshot( + run_id=run_id, + step_key=step_key, + **step_stats, + markers=[ + RunStepMarker( + start_time=marker.get("start"), + end_time=marker.get("end"), + key=marker.get("key"), + ) + for marker in markers[step_key].values() + ], + attempts_list=attempts[step_key], ) - else: - by_step_key[step_key]["status"] = StepEventStatus.IN_PROGRESS - attempts[step_key] = step_attempts - - return [ - RunStepKeyStatsSnapshot( - run_id=run_id, - step_key=step_key, - attempts_list=attempts[step_key], - markers=[ - RunStepMarker(start_time=marker.get("start"), end_time=marker.get("end")) - for marker in markers[step_key].values() - ], - **value, ) - for step_key, value in by_step_key.items() - ] - -@whitelist_for_serdes -class RunStepMarker( - NamedTuple( - "_RunStepMarker", - [("start_time", Optional[float]), ("end_time", Optional[float])], + return RunStepStatsSnapshot( + run_id=run_id, + step_key_stats=snapshots, + partial_markers={ + step_key: [ + RunStepMarker(start_time=marker.get("start"), end_time=marker.get("end"), key=key) + for key, marker in markers.items() + ] + for step_key, markers in markers.items() + if step_key not in by_step_key + }, ) -): - def __new__( - cls, - start_time: Optional[float] = None, - end_time: Optional[float] = None, - ): - return super(RunStepMarker, cls).__new__( - cls, - start_time=check.opt_float_param(start_time, "start_time"), - end_time=check.opt_float_param(end_time, "end_time"), - ) - - -@whitelist_for_serdes -class RunStepKeyStatsSnapshot( - NamedTuple( - "_RunStepKeyStatsSnapshot", - [ - ("run_id", str), - ("step_key", str), - ("status", Optional[StepEventStatus]), - ("start_time", Optional[float]), - ("end_time", Optional[float]), - ("materialization_events", Sequence[EventLogEntry]), - ("expectation_results", Sequence[ExpectationResult]), - ("attempts", Optional[int]), - ("attempts_list", Sequence[RunStepMarker]), - ("markers", Sequence[RunStepMarker]), - ], - ) -): - def __new__( - cls, - run_id: str, - step_key: str, - status: Optional[StepEventStatus] = None, - start_time: Optional[float] = None, - end_time: Optional[float] = None, - materialization_events: Optional[Sequence[EventLogEntry]] = None, - expectation_results: Optional[Sequence[ExpectationResult]] = None, - attempts: Optional[int] = None, - attempts_list: Optional[Sequence[RunStepMarker]] = None, - markers: Optional[Sequence[RunStepMarker]] = None, - ): - return super(RunStepKeyStatsSnapshot, cls).__new__( - cls, - run_id=check.str_param(run_id, "run_id"), - step_key=check.str_param(step_key, "step_key"), - status=check.opt_inst_param(status, "status", StepEventStatus), - start_time=check.opt_float_param(start_time, "start_time"), - end_time=check.opt_float_param(end_time, "end_time"), - materialization_events=check.opt_sequence_param( - materialization_events, - "materialization_events", - EventLogEntry, - ), - expectation_results=check.opt_sequence_param( - expectation_results, "expectation_results", ExpectationResult - ), - attempts=check.opt_int_param(attempts, "attempts"), - attempts_list=check.opt_sequence_param(attempts_list, "attempts_list", RunStepMarker), - markers=check.opt_sequence_param(markers, "markers", RunStepMarker), - ) diff --git a/python_modules/dagster/dagster/_core/instance/__init__.py b/python_modules/dagster/dagster/_core/instance/__init__.py index 25d251bc30679..92c177d1d97ce 100644 --- a/python_modules/dagster/dagster/_core/instance/__init__.py +++ b/python_modules/dagster/dagster/_core/instance/__init__.py @@ -491,13 +491,6 @@ def __init__( " run worker will be marked as failed, but will not be resumed.", ) - if self.run_retries_enabled: - check.invariant( - self.event_log_storage.supports_event_consumer_queries(), - "Run retries are enabled, but the configured event log storage does not support" - " them. Consider switching to Postgres or Mysql.", - ) - # Used for batched event handling self._event_buffer: Dict[str, List[EventLogEntry]] = defaultdict(list) @@ -831,6 +824,9 @@ def get_settings(self, settings_key: str) -> Any: return self._settings.get(settings_key) return {} + def get_backfill_settings(self) -> Mapping[str, Any]: + return self.get_settings("backfills") + def get_scheduler_settings(self) -> Mapping[str, Any]: return self.get_settings("schedules") diff --git a/python_modules/dagster/dagster/_core/instance/config.py b/python_modules/dagster/dagster/_core/instance/config.py index 24dcd46c01ad1..2da173d909254 100644 --- a/python_modules/dagster/dagster/_core/instance/config.py +++ b/python_modules/dagster/dagster/_core/instance/config.py @@ -267,6 +267,20 @@ def get_tick_retention_settings( return default_retention_settings +def backfills_daemon_config() -> Field: + return Field( + { + "use_threads": Field(Bool, is_required=False, default_value=False), + "num_workers": Field( + int, + is_required=False, + description="How many threads to use to process multiple backfills in parallel", + ), + }, + is_required=False, + ) + + def sensors_daemon_config() -> Field: return Field( { @@ -389,6 +403,7 @@ def dagster_instance_config_schema() -> Mapping[str, Field]: ), "secrets": secrets_loader_config_schema(), "retention": retention_config_schema(), + "backfills": backfills_daemon_config(), "sensors": sensors_daemon_config(), "schedules": schedules_daemon_config(), "auto_materialize": Field( diff --git a/python_modules/dagster/dagster/_core/instance/ref.py b/python_modules/dagster/dagster/_core/instance/ref.py index 6a160b8d217a4..5c29d1d85b5de 100644 --- a/python_modules/dagster/dagster/_core/instance/ref.py +++ b/python_modules/dagster/dagster/_core/instance/ref.py @@ -451,6 +451,7 @@ def from_dir( "run_retries", "code_servers", "retention", + "backfills", "sensors", "schedules", "nux", diff --git a/python_modules/dagster/dagster/_core/remote_representation/handle.py b/python_modules/dagster/dagster/_core/remote_representation/handle.py index cc59a21ef5bc2..4807570492c8b 100644 --- a/python_modules/dagster/dagster/_core/remote_representation/handle.py +++ b/python_modules/dagster/dagster/_core/remote_representation/handle.py @@ -11,7 +11,7 @@ RegisteredCodeLocationOrigin, RemoteRepositoryOrigin, ) -from dagster._record import IHaveNew, record, record_custom +from dagster._record import record from dagster._serdes.serdes import whitelist_for_serdes if TYPE_CHECKING: @@ -84,19 +84,11 @@ def for_test( ) -@record_custom -class JobHandle(IHaveNew): +@record(kw_only=False) +class JobHandle: job_name: str repository_handle: RepositoryHandle - # allow posargs - def __new__(cls, job_name: str, repository_handle: RepositoryHandle): - return super().__new__( - cls, - job_name=job_name, - repository_handle=repository_handle, - ) - def to_string(self): return f"{self.location_name}.{self.repository_name}.{self.job_name}" @@ -123,19 +115,11 @@ def to_selector(self) -> JobSubsetSelector: ) -@record_custom -class InstigatorHandle(IHaveNew): +@record(kw_only=False) +class InstigatorHandle: instigator_name: str repository_handle: RepositoryHandle - # allow posargs - def __new__(cls, instigator_name: str, repository_handle: RepositoryHandle): - return super().__new__( - cls, - instigator_name=instigator_name, - repository_handle=repository_handle, - ) - @property def repository_name(self) -> str: return self.repository_handle.repository_name diff --git a/python_modules/dagster/dagster/_core/snap/node.py b/python_modules/dagster/dagster/_core/snap/node.py index e56fe51fc32dd..2da0681b7a665 100644 --- a/python_modules/dagster/dagster/_core/snap/node.py +++ b/python_modules/dagster/dagster/_core/snap/node.py @@ -208,7 +208,7 @@ def get_output_snap(self, name: str) -> OutputDefSnap: }, ) @record -class NodeDefsSnapshot(IHaveNew): +class NodeDefsSnapshot: op_def_snaps: Sequence[OpDefSnap] graph_def_snaps: Sequence[GraphDefSnap] diff --git a/python_modules/dagster/dagster/_core/storage/event_log/base.py b/python_modules/dagster/dagster/_core/storage/event_log/base.py index b82eae23ad458..86d70c8069387 100644 --- a/python_modules/dagster/dagster/_core/storage/event_log/base.py +++ b/python_modules/dagster/dagster/_core/storage/event_log/base.py @@ -336,9 +336,6 @@ def get_event_records( ) -> Sequence[EventLogRecord]: pass - def supports_event_consumer_queries(self) -> bool: - return False - def get_logs_for_all_runs_by_log_id( self, after_cursor: int = -1, diff --git a/python_modules/dagster/dagster/_core/storage/event_log/sql_event_log.py b/python_modules/dagster/dagster/_core/storage/event_log/sql_event_log.py index 1bb1398ec7ef3..94e6d21d02ac6 100644 --- a/python_modules/dagster/dagster/_core/storage/event_log/sql_event_log.py +++ b/python_modules/dagster/dagster/_core/storage/event_log/sql_event_log.py @@ -1017,9 +1017,6 @@ def _get_event_records( return event_records - def supports_event_consumer_queries(self) -> bool: - return True - def _get_event_records_result( self, event_records_filter: EventRecordsFilter, diff --git a/python_modules/dagster/dagster/_core/storage/event_log/sqlite/sqlite_event_log.py b/python_modules/dagster/dagster/_core/storage/event_log/sqlite/sqlite_event_log.py index 69807391d07c9..7cfcc2bfecfc2 100644 --- a/python_modules/dagster/dagster/_core/storage/event_log/sqlite/sqlite_event_log.py +++ b/python_modules/dagster/dagster/_core/storage/event_log/sqlite/sqlite_event_log.py @@ -434,9 +434,6 @@ def fetch_run_status_changes( has_more = len(records) == limit return EventRecordsResult(records, cursor=new_cursor, has_more=has_more) - def supports_event_consumer_queries(self) -> bool: - return False - def wipe(self) -> None: # should delete all the run-sharded db files and drop the contents of the index for filename in ( diff --git a/python_modules/dagster/dagster/_core/storage/legacy_storage.py b/python_modules/dagster/dagster/_core/storage/legacy_storage.py index 28668cb5fb2e6..b44832c34afc1 100644 --- a/python_modules/dagster/dagster/_core/storage/legacy_storage.py +++ b/python_modules/dagster/dagster/_core/storage/legacy_storage.py @@ -416,6 +416,19 @@ def get_logs_for_run( run_id, cursor, of_type, limit, ascending ) + def get_logs_for_all_runs_by_log_id( + self, + after_cursor: int = -1, + dagster_event_type: Optional[Union["DagsterEventType", Set["DagsterEventType"]]] = None, + limit: Optional[int] = None, + ) -> Mapping[int, "EventLogEntry"]: + return self._storage.event_log_storage.get_logs_for_all_runs_by_log_id( + after_cursor=after_cursor, dagster_event_type=dagster_event_type, limit=limit + ) + + def get_maximum_record_id(self) -> Optional[int]: + return self._storage.event_log_storage.get_maximum_record_id() + def get_stats_for_run(self, run_id: str) -> "DagsterRunStatsSnapshot": return self._storage.event_log_storage.get_stats_for_run(run_id) diff --git a/python_modules/dagster/dagster/_core/test_utils.py b/python_modules/dagster/dagster/_core/test_utils.py index 117dda48a1102..8b76c1474981c 100644 --- a/python_modules/dagster/dagster/_core/test_utils.py +++ b/python_modules/dagster/dagster/_core/test_utils.py @@ -90,6 +90,7 @@ def assert_namedtuple_lists_equal( t2_list: Sequence[T_NamedTuple], exclude_fields: Optional[Sequence[str]] = None, ) -> None: + assert len(t1_list) == len(t2_list) for t1, t2 in zip(t1_list, t2_list): assert_namedtuples_equal(t1, t2, exclude_fields) diff --git a/python_modules/dagster/dagster/_daemon/asset_daemon.py b/python_modules/dagster/dagster/_daemon/asset_daemon.py index 85bed317ee1bd..e4ac46140256c 100644 --- a/python_modules/dagster/dagster/_daemon/asset_daemon.py +++ b/python_modules/dagster/dagster/_daemon/asset_daemon.py @@ -72,7 +72,7 @@ from dagster._serdes import serialize_value from dagster._serdes.serdes import deserialize_value from dagster._time import get_current_datetime, get_current_timestamp -from dagster._utils import SingleInstigatorDebugCrashFlags, check_for_debug_crash +from dagster._utils import SingleInstigatorDebugCrashFlags, check_for_debug_crash, return_as_list _LEGACY_PRE_SENSOR_AUTO_MATERIALIZE_CURSOR_KEY = "ASSET_DAEMON_CURSOR" _PRE_SENSOR_AUTO_MATERIALIZE_CURSOR_KEY = "ASSET_DAEMON_CURSOR_NEW" @@ -655,24 +655,6 @@ def _copy_default_auto_materialize_sensor_states( return result - def _process_auto_materialize_tick( - self, - workspace_process_context: IWorkspaceProcessContext, - repository: Optional[RemoteRepository], - sensor: Optional[RemoteSensor], - debug_crash_flags: SingleInstigatorDebugCrashFlags, - submit_threadpool_executor: Optional[ThreadPoolExecutor], - ): - return list( - self._process_auto_materialize_tick_generator( - workspace_process_context, - repository, - sensor, - debug_crash_flags, - submit_threadpool_executor=submit_threadpool_executor, - ) - ) - def _process_auto_materialize_tick_generator( self, workspace_process_context: IWorkspaceProcessContext, @@ -878,6 +860,8 @@ def _process_auto_materialize_tick_generator( yield error_info + _process_auto_materialize_tick = return_as_list(_process_auto_materialize_tick_generator) + def _evaluate_auto_materialize_tick( self, tick_context: AutoMaterializeLaunchContext, diff --git a/python_modules/dagster/dagster/_daemon/backfill.py b/python_modules/dagster/dagster/_daemon/backfill.py index 1fcc0c24c4199..23b5b8ccfdc68 100644 --- a/python_modules/dagster/dagster/_daemon/backfill.py +++ b/python_modules/dagster/dagster/_daemon/backfill.py @@ -1,8 +1,10 @@ import logging import os import sys +import threading +from concurrent.futures import Future, ThreadPoolExecutor from contextlib import contextmanager -from typing import Iterable, Mapping, Optional, Sequence, cast +from typing import TYPE_CHECKING, Dict, Iterable, Mapping, Optional, Sequence, cast import dagster._check as check from dagster._core.definitions.instigation_logger import InstigationLogger @@ -16,9 +18,13 @@ from dagster._core.execution.job_backfill import execute_job_backfill_iteration from dagster._core.workspace.context import IWorkspaceProcessContext from dagster._daemon.utils import DaemonErrorCapture -from dagster._time import get_current_datetime +from dagster._time import get_current_datetime, get_current_timestamp +from dagster._utils import return_as_list from dagster._utils.error import SerializableErrorInfo +if TYPE_CHECKING: + from dagster._daemon.daemon import DaemonIterator + @contextmanager def _get_instigation_logger_if_log_storage_enabled( @@ -46,9 +52,55 @@ def _get_max_asset_backfill_retries(): return int(os.getenv("DAGSTER_MAX_ASSET_BACKFILL_RETRIES", "5")) +def execute_backfill_iteration_loop( + workspace_process_context: IWorkspaceProcessContext, + logger: logging.Logger, + shutdown_event: threading.Event, + until: Optional[float] = None, + threadpool_executor: Optional[ThreadPoolExecutor] = None, +) -> "DaemonIterator": + from dagster._daemon.controller import DEFAULT_DAEMON_INTERVAL_SECONDS + from dagster._daemon.daemon import SpanMarker + + backfill_futures: Dict[str, Future] = {} + while True: + start_time = get_current_timestamp() + if until and start_time >= until: + # provide a way of organically ending the loop to support test environment + break + + yield SpanMarker.START_SPAN + + try: + yield from execute_backfill_iteration( + workspace_process_context, + logger, + threadpool_executor=threadpool_executor, + backfill_futures=backfill_futures, + ) + except Exception: + error_info = DaemonErrorCapture.on_exception( + exc_info=sys.exc_info(), + logger=logger, + log_message="BackfillDaemon caught an error", + ) + yield error_info + + yield SpanMarker.END_SPAN + + end_time = get_current_timestamp() + loop_duration = end_time - start_time + sleep_time = max(0, DEFAULT_DAEMON_INTERVAL_SECONDS - loop_duration) + shutdown_event.wait(sleep_time) + + yield None + + def execute_backfill_iteration( workspace_process_context: IWorkspaceProcessContext, logger: logging.Logger, + threadpool_executor: Optional[ThreadPoolExecutor] = None, + backfill_futures: Optional[Dict[str, Future]] = None, debug_crash_flags: Optional[Mapping[str, int]] = None, ) -> Iterable[Optional[SerializableErrorInfo]]: instance = workspace_process_context.instance @@ -68,7 +120,12 @@ def execute_backfill_iteration( backfill_jobs = [*in_progress_backfills, *canceling_backfills] yield from execute_backfill_jobs( - workspace_process_context, logger, backfill_jobs, debug_crash_flags + workspace_process_context, + logger, + backfill_jobs, + threadpool_executor, + backfill_futures, + debug_crash_flags, ) @@ -86,6 +143,8 @@ def execute_backfill_jobs( workspace_process_context: IWorkspaceProcessContext, logger: logging.Logger, backfill_jobs: Sequence[PartitionBackfill], + threadpool_executor: Optional[ThreadPoolExecutor] = None, + backfill_futures: Optional[Dict[str, Future]] = None, debug_crash_flags: Optional[Mapping[str, int]] = None, ) -> Iterable[Optional[SerializableErrorInfo]]: instance = workspace_process_context.instance @@ -103,18 +162,49 @@ def execute_backfill_jobs( ) try: - if backfill.is_asset_backfill: - yield from execute_asset_backfill_iteration( - backfill, backfill_logger, workspace_process_context, instance - ) + if threadpool_executor: + if backfill_futures is None: + check.failed( + "backfill_futures dict must be passed with threadpool_executor" + ) + + # only allow one backfill per backfill job to be in flight + if backfill_id in backfill_futures and not backfill_futures[backfill_id].done(): + continue + + if backfill.is_asset_backfill: + future = threadpool_executor.submit( + return_as_list(execute_asset_backfill_iteration), + backfill, + backfill_logger, + workspace_process_context, + instance, + ) + else: + future = threadpool_executor.submit( + return_as_list(execute_job_backfill_iteration), + backfill, + backfill_logger, + workspace_process_context, + debug_crash_flags, + instance, + ) + backfill_futures[backfill_id] = future + yield + else: - yield from execute_job_backfill_iteration( - backfill, - backfill_logger, - workspace_process_context, - debug_crash_flags, - instance, - ) + if backfill.is_asset_backfill: + yield from execute_asset_backfill_iteration( + backfill, backfill_logger, workspace_process_context, instance + ) + else: + yield from execute_job_backfill_iteration( + backfill, + backfill_logger, + workspace_process_context, + debug_crash_flags, + instance, + ) except Exception as e: backfill = check.not_none(instance.get_backfill(backfill.backfill_id)) if ( diff --git a/python_modules/dagster/dagster/_daemon/controller.py b/python_modules/dagster/dagster/_daemon/controller.py index e0016460d5921..0371c07128180 100644 --- a/python_modules/dagster/dagster/_daemon/controller.py +++ b/python_modules/dagster/dagster/_daemon/controller.py @@ -362,7 +362,7 @@ def create_daemon_of_type(daemon_type: str, instance: DagsterInstance) -> Dagste interval_seconds=instance.run_coordinator.dequeue_interval_seconds # type: ignore # (??) ) elif daemon_type == BackfillDaemon.daemon_type(): - return BackfillDaemon(interval_seconds=DEFAULT_DAEMON_INTERVAL_SECONDS) + return BackfillDaemon(settings=instance.get_backfill_settings()) elif daemon_type == MonitoringDaemon.daemon_type(): return MonitoringDaemon(interval_seconds=instance.run_monitoring_poll_interval_seconds) elif daemon_type == EventLogConsumerDaemon.daemon_type(): diff --git a/python_modules/dagster/dagster/_daemon/daemon.py b/python_modules/dagster/dagster/_daemon/daemon.py index 6a461618c0a69..9ae368c903cf2 100644 --- a/python_modules/dagster/dagster/_daemon/daemon.py +++ b/python_modules/dagster/dagster/_daemon/daemon.py @@ -22,7 +22,7 @@ from dagster._core.telemetry import DAEMON_ALIVE, log_action from dagster._core.utils import InheritContextThreadPoolExecutor from dagster._core.workspace.context import IWorkspaceProcessContext -from dagster._daemon.backfill import execute_backfill_iteration +from dagster._daemon.backfill import execute_backfill_iteration_loop from dagster._daemon.monitoring import ( execute_concurrency_slots_iteration, execute_run_monitoring_iteration, @@ -333,16 +333,39 @@ def core_loop( ) -class BackfillDaemon(IntervalDaemon): +class BackfillDaemon(DagsterDaemon): + def __init__(self, settings: Mapping[str, Any]) -> None: + super().__init__() + self._exit_stack = ExitStack() + self._threadpool_executor: Optional[InheritContextThreadPoolExecutor] = None + + if settings.get("use_threads"): + self._threadpool_executor = self._exit_stack.enter_context( + InheritContextThreadPoolExecutor( + max_workers=settings.get("num_workers"), + thread_name_prefix="backfill_daemon_worker", + ) + ) + @classmethod def daemon_type(cls) -> str: return "BACKFILL" - def run_iteration( + def __exit__(self, _exception_type, _exception_value, _traceback): + self._exit_stack.close() + super().__exit__(_exception_type, _exception_value, _traceback) + + def core_loop( self, workspace_process_context: IWorkspaceProcessContext, + shutdown_event: Event, ) -> DaemonIterator: - yield from execute_backfill_iteration(workspace_process_context, self._logger) + yield from execute_backfill_iteration_loop( + workspace_process_context, + self._logger, + shutdown_event, + threadpool_executor=self._threadpool_executor, + ) class MonitoringDaemon(IntervalDaemon): diff --git a/python_modules/dagster/dagster/_daemon/sensor.py b/python_modules/dagster/dagster/_daemon/sensor.py index f9daf37f65a3b..1611f911cca73 100644 --- a/python_modules/dagster/dagster/_daemon/sensor.py +++ b/python_modules/dagster/dagster/_daemon/sensor.py @@ -66,7 +66,12 @@ from dagster._daemon.utils import DaemonErrorCapture from dagster._scheduler.stale import resolve_stale_or_missing_assets from dagster._time import get_current_datetime, get_current_timestamp -from dagster._utils import DebugCrashFlags, SingleInstigatorDebugCrashFlags, check_for_debug_crash +from dagster._utils import ( + DebugCrashFlags, + SingleInstigatorDebugCrashFlags, + check_for_debug_crash, + return_as_list, +) from dagster._utils.error import SerializableErrorInfo from dagster._utils.merger import merge_dicts @@ -344,7 +349,6 @@ def execute_sensor_iteration_loop( yield SpanMarker.END_SPAN end_time = get_current_timestamp() - loop_duration = end_time - start_time sleep_time = max(0, MIN_INTERVAL_LOOP_TIME - loop_duration) shutdown_event.wait(sleep_time) @@ -453,30 +457,6 @@ def execute_sensor_iteration( ) -def _process_tick( - workspace_process_context: IWorkspaceProcessContext, - logger: logging.Logger, - remote_sensor: RemoteSensor, - sensor_state: InstigatorState, - sensor_debug_crash_flags: Optional[SingleInstigatorDebugCrashFlags], - tick_retention_settings, - submit_threadpool_executor: Optional[ThreadPoolExecutor], -): - # evaluate the tick immediately, but from within a thread. The main thread should be able to - # heartbeat to keep the daemon alive - return list( - _process_tick_generator( - workspace_process_context, - logger, - remote_sensor, - sensor_state, - sensor_debug_crash_flags, - tick_retention_settings, - submit_threadpool_executor, - ) - ) - - def _get_evaluation_tick( instance: DagsterInstance, sensor: RemoteSensor, @@ -630,6 +610,11 @@ def _process_tick_generator( yield error_info +# evaluate the tick immediately, but from within a thread. The main thread should be able to +# heartbeat to keep the daemon alive +_process_tick = return_as_list(_process_tick_generator) + + def _sensor_instigator_data(state: InstigatorState) -> Optional[SensorInstigatorData]: instigator_data = state.instigator_data if instigator_data is None or isinstance(instigator_data, SensorInstigatorData): diff --git a/python_modules/dagster/dagster/_record/__init__.py b/python_modules/dagster/dagster/_record/__init__.py index 58e1c5c1e07ed..66d819823a76c 100644 --- a/python_modules/dagster/dagster/_record/__init__.py +++ b/python_modules/dagster/dagster/_record/__init__.py @@ -35,6 +35,7 @@ _NAMED_TUPLE_BASE_NEW_FIELD = "__nt_new__" _REMAPPING_FIELD = "__field_remap__" _ORIGINAL_CLASS_FIELD = "__original_class__" +_KW_ONLY_FIELD = "__kw_only__" _sample_nt = namedtuple("_canary", "x") @@ -44,10 +45,12 @@ def _get_field_set_and_defaults( cls: Type, + kw_only: bool, ) -> Tuple[Mapping[str, Any], Mapping[str, Any]]: field_set = getattr(cls, "__annotations__", {}) defaults = {} + last_defaulted_field = None for name in field_set.keys(): if hasattr(cls, name): attr_val = getattr(cls, name) @@ -57,11 +60,9 @@ def _get_field_set_and_defaults( f"Conflicting non-abstract @property for field {name} on record {cls.__name__}." "Add the the @abstractmethod decorator to make it abstract.", ) - elif isinstance(attr_val, _tuple_getter_type): - # When doing record inheritance, filter out tuplegetters from parents. - # This workaround only seems needed for py3.8 - continue - else: + # When doing record inheritance, filter out tuplegetters from parents. + # This workaround only seems needed for py3.9 + elif not isinstance(attr_val, _tuple_getter_type): check.invariant( not inspect.isfunction(attr_val), f"Conflicting function for field {name} on record {cls.__name__}. " @@ -69,11 +70,25 @@ def _get_field_set_and_defaults( "you will have to override __new__.", ) defaults[name] = attr_val + last_defaulted_field = name + continue + + # fall through here means no default set + if last_defaulted_field and not kw_only: + check.failed( + "Fields without defaults cannot appear after fields with default values. " + f"Field {name} has no default after {last_defaulted_field} with default value." + ) for base in cls.__bases__: if is_record(base): original_base = getattr(base, _ORIGINAL_CLASS_FIELD) - base_field_set, base_defaults = _get_field_set_and_defaults(original_base) + base_kw_only = getattr(base, _KW_ONLY_FIELD) + check.invariant( + kw_only == base_kw_only, + "Can not inherit from a parent @record with different kw_only setting.", + ) + base_field_set, base_defaults = _get_field_set_and_defaults(original_base, kw_only) field_set = {**base_field_set, **field_set} defaults = {**base_defaults, **defaults} @@ -87,13 +102,14 @@ def _namedtuple_record_transform( with_new: bool, decorator_frames: int, field_to_new_mapping: Optional[Mapping[str, str]], + kw_only: bool, ) -> TType: """Transforms the input class in to one that inherits a generated NamedTuple base class and: * bans tuple methods that don't make sense for a record object * creates a run time checked __new__ (optional). """ - field_set, defaults = _get_field_set_and_defaults(cls) + field_set, defaults = _get_field_set_and_defaults(cls, kw_only) base = NamedTuple(f"_{cls.__name__}", field_set.items()) nt_new = base.__new__ @@ -109,7 +125,8 @@ def _namedtuple_record_transform( field_set, defaults, eval_ctx, - 1 if with_new else 0, + new_frames=1 if with_new else 0, + kw_only=kw_only, ) elif defaults: # allow arbitrary ordering of default values by generating a kwarg only __new__ impl @@ -120,7 +137,7 @@ def _namedtuple_record_transform( lazy_imports={}, ) generated_new = eval_ctx.compile_fn( - _build_defaults_new(field_set, defaults), + _build_defaults_new(field_set, defaults, kw_only), _DEFAULTS_NEW, ) @@ -145,6 +162,7 @@ def _namedtuple_record_transform( _NAMED_TUPLE_BASE_NEW_FIELD: nt_new, _REMAPPING_FIELD: field_to_new_mapping or {}, _ORIGINAL_CLASS_FIELD: cls, + _KW_ONLY_FIELD: kw_only, "__reduce__": _reduce, # functools doesn't work, so manually update_wrapper "__module__": cls.__module__, @@ -219,6 +237,7 @@ def record( def record( *, checked: bool = True, + kw_only: bool = True, ) -> Callable[[TType], TType]: ... # Overload for using decorator used with args. @@ -230,11 +249,13 @@ def record( cls: Optional[TType] = None, *, checked: bool = True, + kw_only: bool = True, ) -> Union[TType, Callable[[TType], TType]]: """A class decorator that will create an immutable record class based on the defined fields. Args: - checked: Whether or not to generate runtime type checked construction. + checked: Whether or not to generate runtime type checked construction (default True). + kw_only: Whether or not the generated __new__ is kwargs only (default True). """ if cls: return _namedtuple_record_transform( @@ -243,6 +264,7 @@ def record( with_new=False, decorator_frames=1, field_to_new_mapping=None, + kw_only=kw_only, ) else: return partial( @@ -251,6 +273,7 @@ def record( with_new=False, decorator_frames=0, field_to_new_mapping=None, + kw_only=kw_only, ) @@ -303,6 +326,7 @@ def __new__(cls, name: Optional[str] = None) with_new=True, decorator_frames=1, field_to_new_mapping=field_to_new_mapping, + kw_only=True, ) else: return partial( @@ -311,6 +335,7 @@ def __new__(cls, name: Optional[str] = None) with_new=True, decorator_frames=0, field_to_new_mapping=field_to_new_mapping, + kw_only=True, ) @@ -429,12 +454,14 @@ def __init__( defaults: Mapping[str, Any], eval_ctx: EvalContext, new_frames: int, + kw_only: bool, ): self._field_set = field_set self._defaults = defaults self._eval_ctx = eval_ctx self._new_frames = new_frames # how many frames of __new__ there are self._compiled = False + self._kw_only = kw_only def __call__(self, cls, *args, **kwargs): if _do_defensive_checks(): @@ -470,7 +497,11 @@ def __call__(self, cls, *args, **kwargs): return compiled_fn(cls, *args, **kwargs) def _build_checked_new_str(self) -> str: - kw_args_str, set_calls_str = build_args_and_assignment_strs(self._field_set, self._defaults) + args_str, set_calls_str = build_args_and_assignment_strs( + self._field_set, + self._defaults, + self._kw_only, + ) check_calls = [] for name, ttype in self._field_set.items(): call_str = build_check_call_str( @@ -487,7 +518,7 @@ def _build_checked_new_str(self) -> str: ) checked_new_str = f""" -def __checked_new__(cls{kw_args_str}): +def __checked_new__(cls{args_str}): {lazy_imports_str} {set_calls_str} return cls.{_NAMED_TUPLE_BASE_NEW_FIELD}( @@ -501,9 +532,10 @@ def __checked_new__(cls{kw_args_str}): def _build_defaults_new( field_set: Mapping[str, Type], defaults: Mapping[str, Any], + kw_only: bool, ) -> str: """Build a __new__ implementation that handles default values.""" - kw_args_str, set_calls_str = build_args_and_assignment_strs(field_set, defaults) + kw_args_str, set_calls_str = build_args_and_assignment_strs(field_set, defaults, kw_only) assign_str = ",\n ".join([f"{name}={name}" for name in field_set.keys()]) return f""" def __defaults_new__(cls{kw_args_str}): @@ -518,39 +550,40 @@ def __defaults_new__(cls{kw_args_str}): def build_args_and_assignment_strs( field_set: Mapping[str, Type], defaults: Mapping[str, Any], + kw_only: bool, ) -> Tuple[str, str]: """Utility funciton shared between _defaults_new and _checked_new to create the arguments to the function as well as any assignment calls that need to happen. """ - kw_args = [] + args = [] set_calls = [] for arg in field_set.keys(): if arg in defaults: default = defaults[arg] if default is None: - kw_args.append(f"{arg} = None") + args.append(f"{arg} = None") # dont share class instance of default empty containers elif default == []: - kw_args.append(f"{arg} = None") + args.append(f"{arg} = None") set_calls.append(f"{arg} = {arg} if {arg} is not None else []") elif default == {}: - kw_args.append(f"{arg} = None") + args.append(f"{arg} = None") set_calls.append(f"{arg} = {arg} if {arg} is not None else {'{}'}") # fallback to direct reference if unknown else: - kw_args.append(f"{arg} = {_INJECTED_DEFAULT_VALS_LOCAL_VAR}['{arg}']") + args.append(f"{arg} = {_INJECTED_DEFAULT_VALS_LOCAL_VAR}['{arg}']") else: - kw_args.append(arg) + args.append(arg) - kw_args_str = "" - if kw_args: - kw_args_str = f", *, {', '.join(kw_args)}" + args_str = "" + if args: + args_str = f", {'*,' if kw_only else ''} {', '.join(args)}" set_calls_str = "" if set_calls: set_calls_str = "\n ".join(set_calls) - return kw_args_str, set_calls_str + return args_str, set_calls_str def _banned_iter(*args, **kwargs): diff --git a/python_modules/dagster/dagster/_scheduler/scheduler.py b/python_modules/dagster/dagster/_scheduler/scheduler.py index 4c83de5acc5e0..22e3882bb3b0d 100644 --- a/python_modules/dagster/dagster/_scheduler/scheduler.py +++ b/python_modules/dagster/dagster/_scheduler/scheduler.py @@ -197,8 +197,8 @@ def execute_scheduler_iteration_loop( scheduler_run_futures: Dict[str, Future] = {} iteration_times: Dict[str, ScheduleIterationTimes] = {} - submit_threadpool_executor = None threadpool_executor = None + submit_threadpool_executor = None with ExitStack() as stack: settings = workspace_process_context.instance.get_scheduler_settings() @@ -225,6 +225,7 @@ def execute_scheduler_iteration_loop( next_interval_time = _get_next_scheduler_iteration_time(start_time) yield SpanMarker.START_SPAN + try: yield from launch_scheduled_runs( workspace_process_context, @@ -248,8 +249,8 @@ def execute_scheduler_iteration_loop( next_interval_time = min(start_time + ERROR_INTERVAL_TIME, next_interval_time) yield SpanMarker.END_SPAN - end_time = get_current_timestamp() + end_time = get_current_timestamp() if next_interval_time > end_time: # Sleep until the beginning of the next minute, plus a small epsilon to # be sure that we're past the start of the minute diff --git a/python_modules/dagster/dagster/_utils/__init__.py b/python_modules/dagster/dagster/_utils/__init__.py index fce1525d93856..39808e9453cfa 100644 --- a/python_modules/dagster/dagster/_utils/__init__.py +++ b/python_modules/dagster/dagster/_utils/__init__.py @@ -844,3 +844,13 @@ def run_with_concurrent_update_guard( return update_fn(**kwargs) return + + +def return_as_list(func: Callable[..., Iterable[T]]) -> Callable[..., List[T]]: + """A decorator that returns a list from the output of a function.""" + + @functools.wraps(func) + def inner(*args, **kwargs): + return list(func(*args, **kwargs)) + + return inner diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/partition_mapping_tests/test_asset_partition_mappings.py b/python_modules/dagster/dagster_tests/asset_defs_tests/partition_mapping_tests/test_asset_partition_mappings.py index 0e7c8881a5af3..c5e56d3748d37 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/partition_mapping_tests/test_asset_partition_mappings.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/partition_mapping_tests/test_asset_partition_mappings.py @@ -251,7 +251,7 @@ def load_input(self, context): exclude_fields=["tags"], ) assert_namedtuple_lists_equal( - result.asset_materializations_for_node("downstream"), + result.asset_materializations_for_node("downstream_a_b"), [AssetMaterialization(AssetKey(["downstream_a_b"]))], exclude_fields=["tags"], ) diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_deps.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_deps.py index 8b9f9056b95cd..a3cfde7ba0541 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_deps.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_deps.py @@ -4,6 +4,7 @@ AssetOut, FilesystemIOManager, IOManager, + Nothing, SourceAsset, TimeWindowPartitionMapping, asset, @@ -12,7 +13,9 @@ ) from dagster._check import ParameterCheckError from dagster._core.definitions.asset_dep import AssetDep +from dagster._core.definitions.asset_in import AssetIn from dagster._core.definitions.asset_spec import AssetSpec +from dagster._core.definitions.partition_mapping import IdentityPartitionMapping from dagster._core.errors import DagsterInvalidDefinitionError, DagsterInvariantViolationError from dagster._core.types.dagster_type import DagsterTypeKind @@ -522,20 +525,158 @@ def my_asset(): def test_dep_via_deps_and_fn(): + """Test combining deps and ins in the same @asset-decorated function.""" + @asset def the_upstream_asset(): return 1 - with pytest.raises( - DagsterInvalidDefinitionError, - match=r"deps value .* also declared as input/AssetIn", - ): + # When deps and ins are both set, expect that deps is only used for the asset key and potentially input name. + for param_dict in [ + {"partition_mapping": IdentityPartitionMapping()}, + {"metadata": {"foo": "bar"}}, + {"key_prefix": "prefix"}, + {"dagster_type": Nothing}, + ]: + with pytest.raises(DagsterInvalidDefinitionError): + + @asset( + deps=[AssetDep(the_upstream_asset)], + ins={"the_upstream_asset": AssetIn(**param_dict)}, + ) + def _(the_upstream_asset): + return None + + # We allow the asset key to be set via deps and ins as long as no additional information is set. + @asset(deps=[the_upstream_asset]) + def depends_on_upstream_asset_implicit_remap(the_upstream_asset): + assert the_upstream_asset == 1 + + @asset( + deps=[AssetDep(the_upstream_asset)], ins={"remapped": AssetIn(key=the_upstream_asset.key)} + ) + def depends_on_upstream_asset_explicit_remap(remapped): + assert remapped == 1 + + res = materialize( + [ + the_upstream_asset, + depends_on_upstream_asset_implicit_remap, + depends_on_upstream_asset_explicit_remap, + ], + ) + assert res.success + + @asset + def upstream2(): + return 2 + + # As an unfortunate consequence of the many iterations of dependency specification and the fact that they were all additive with each other, + # we have to support the case where deps are specified separately in both the function signature and the decorator. + # This is not recommended, but it is supported. + @asset(deps=[the_upstream_asset]) + def some_explicit_and_implicit_deps(the_upstream_asset, upstream2): + assert the_upstream_asset == 1 + assert upstream2 == 2 + + @asset(deps=[the_upstream_asset], ins={"remapped": AssetIn(key=upstream2.key)}) + def deps_disjoint_between_args(the_upstream_asset, remapped): + assert the_upstream_asset == 1 + assert remapped == 2 + + res = materialize( + [ + the_upstream_asset, + upstream2, + some_explicit_and_implicit_deps, + deps_disjoint_between_args, + ], + ) + assert res.success + - @asset(deps=[the_upstream_asset]) - def depends_on_upstream_asset(the_upstream_asset): +def test_multi_asset_specs_deps_and_fn(): + @asset + def the_upstream_asset(): + return 1 + + # When deps and ins are both set, expect that deps is only used for the asset key and potentially input name. + for param_dict in [ + {"partition_mapping": IdentityPartitionMapping()}, + {"metadata": {"foo": "bar"}}, + {"key_prefix": "prefix"}, + {"dagster_type": Nothing}, + ]: + with pytest.raises(DagsterInvalidDefinitionError): + + @multi_asset( + specs=[AssetSpec("the_asset", deps=[AssetDep(the_upstream_asset)])], + ins={"the_upstream_asset": AssetIn(**param_dict)}, + ) + def _(the_upstream_asset): + return None + + # We allow the asset key to be set via deps and ins as long as no additional information is set. + @multi_asset(specs=[AssetSpec("the_asset", deps=[the_upstream_asset])]) + def depends_on_upstream_asset_implicit_remap(the_upstream_asset): + assert the_upstream_asset == 1 + + @multi_asset( + specs=[AssetSpec("other_asset", deps=[AssetDep(the_upstream_asset)])], + ins={"remapped": AssetIn(key=the_upstream_asset.key)}, + ) + def depends_on_upstream_asset_explicit_remap(remapped): + assert remapped == 1 + + res = materialize( + [ + the_upstream_asset, + depends_on_upstream_asset_implicit_remap, + depends_on_upstream_asset_explicit_remap, + ], + ) + assert res.success + + # We do not allow you to set a dependency purely via input if you're opting in to the spec pattern. + with pytest.raises(DagsterInvalidDefinitionError): + + @multi_asset( + specs=[AssetSpec("the_asset")], + ) + def _(the_upstream_asset): return None +def test_allow_remapping_io_manager_key() -> None: + @asset + def the_upstream_asset(): + return 1 + + @asset( + deps=[the_upstream_asset], + ins={"the_upstream_asset": AssetIn(input_manager_key="custom_io")}, + ) + def depends_on_upstream_asset(the_upstream_asset): + assert the_upstream_asset == 1 + + calls = [] + + class MyIOManager(IOManager): + def handle_output(self, context, obj): + raise Exception("Should not be called") + + def load_input(self, context): + calls.append("load_input") + return 1 + + res = materialize( + [the_upstream_asset, depends_on_upstream_asset], + resources={"custom_io": MyIOManager()}, + ) + assert res.success + assert calls == ["load_input"] + + def test_duplicate_deps(): @asset def the_upstream_asset(): diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py index 1f29844b42a83..5fbc36e955cbb 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py @@ -1976,7 +1976,7 @@ def also_input(table_A): ... with pytest.raises( DagsterInvalidDefinitionError, - match="do not have dependencies on the passed AssetSpec", + match="specified as AssetIns", ): @multi_asset(specs=[table_b, table_c]) @@ -1984,7 +1984,7 @@ def rogue_input(table_X): ... with pytest.raises( DagsterInvalidDefinitionError, - match="do not have dependencies on the passed AssetSpec", + match="specified as AssetIns", ): @multi_asset(specs=[table_b_no_dep, table_c_no_dep]) diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py index 8b446f2b36baa..398d9dc796167 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py @@ -16,6 +16,7 @@ InputContext, IOManager, IOManagerDefinition, + MaterializeResult, MultiPartitionKey, MultiPartitionsDefinition, Output, @@ -58,6 +59,7 @@ def error_on_warning(): def get_upstream_partitions_for_partition_range( downstream_assets_def: AssetsDefinition, + downstream_asset_key: AssetKey, upstream_partitions_def: PartitionsDefinition, upstream_asset_key: AssetKey, downstream_partition_key_range: Optional[PartitionKeyRange], @@ -66,7 +68,7 @@ def get_upstream_partitions_for_partition_range( check.failed("upstream asset is not partitioned") downstream_partition_mapping = downstream_assets_def.infer_partition_mapping( - upstream_asset_key, upstream_partitions_def + downstream_asset_key, upstream_asset_key, upstream_partitions_def ) downstream_partitions_def = downstream_assets_def.partitions_def downstream_partitions_subset = ( @@ -92,6 +94,7 @@ def get_upstream_partitions_for_partition_range( def get_downstream_partitions_for_partition_range( downstream_assets_def: AssetsDefinition, + downstream_asset_key: AssetKey, upstream_assets_def: AssetsDefinition, upstream_asset_key: AssetKey, upstream_partition_key_range: PartitionKeyRange, @@ -103,7 +106,7 @@ def get_downstream_partitions_for_partition_range( check.failed("upstream asset is not partitioned") downstream_partition_mapping = downstream_assets_def.infer_partition_mapping( - upstream_asset_key, upstream_assets_def.partitions_def + downstream_asset_key, upstream_asset_key, upstream_assets_def.partitions_def ) upstream_partitions_def = upstream_assets_def.partitions_def upstream_partitions_subset = upstream_partitions_def.empty_subset().with_partition_keys( @@ -136,6 +139,7 @@ def downstream_asset(upstream_asset): assert get_upstream_partitions_for_partition_range( downstream_asset, + downstream_asset.key, upstream_asset.partitions_def, # pyright: ignore[reportArgumentType] AssetKey("upstream_asset"), PartitionKeyRange("a", "c"), @@ -143,6 +147,7 @@ def downstream_asset(upstream_asset): assert get_downstream_partitions_for_partition_range( downstream_asset, + downstream_asset.key, upstream_asset, AssetKey("upstream_asset"), PartitionKeyRange("a", "c"), @@ -427,6 +432,7 @@ def downstream_asset_2(upstream_asset_2: int): assert get_upstream_partitions_for_partition_range( downstream_asset_1, + downstream_asset_1.key, upstream_asset.partitions_def, # pyright: ignore[reportArgumentType] AssetKey("upstream_asset_1"), PartitionKeyRange("a", "c"), @@ -434,6 +440,7 @@ def downstream_asset_2(upstream_asset_2: int): assert get_upstream_partitions_for_partition_range( downstream_asset_2, + downstream_asset_2.key, upstream_asset.partitions_def, # pyright: ignore[reportArgumentType] AssetKey("upstream_asset_2"), PartitionKeyRange("a", "c"), @@ -441,6 +448,7 @@ def downstream_asset_2(upstream_asset_2: int): assert get_downstream_partitions_for_partition_range( downstream_asset_1, + downstream_asset_1.key, upstream_asset, AssetKey("upstream_asset_1"), PartitionKeyRange("a", "c"), @@ -448,6 +456,7 @@ def downstream_asset_2(upstream_asset_2: int): assert get_downstream_partitions_for_partition_range( downstream_asset_2, + downstream_asset_2.key, upstream_asset, AssetKey("upstream_asset_2"), PartitionKeyRange("a", "c"), @@ -490,6 +499,84 @@ def my_asset(): ) +def test_multi_asset_with_different_partitions_defs(): + partitions_def1 = StaticPartitionsDefinition(["a", "b", "c", "d"]) + partitions_def2 = StaticPartitionsDefinition(["1", "2", "3"]) + + @multi_asset( + specs=[ + AssetSpec("my_asset_1", partitions_def=partitions_def1), + AssetSpec("my_asset_2", partitions_def=partitions_def2), + ], + can_subset=True, + ) + def my_assets(context): + assert context.partition_key == "b" + assert context.partition_keys == ["b"] + for asset_key in context.selected_asset_keys: + yield MaterializeResult(asset_key=asset_key) + + result = materialize(assets=[my_assets], partition_key="b", selection=["my_asset_1"]) + assert result.success + + assert_namedtuple_lists_equal( + result.asset_materializations_for_node("my_assets"), + [ + AssetMaterialization(asset_key=AssetKey(["my_asset_1"]), partition="b"), + ], + exclude_fields=["tags"], + ) + + with pytest.raises( + DagsterInvalidDefinitionError, + match="Selected assets must have the same partitions definitions, but the selected assets ", + ): + materialize(assets=[my_assets], partition_key="b") + + +def test_multi_asset_with_different_partitions_defs_partition_key_range(): + partitions_def1 = DailyPartitionsDefinition(start_date="2020-01-01") + partitions_def2 = StaticPartitionsDefinition(["1", "2", "3"]) + + @multi_asset( + specs=[ + AssetSpec("my_asset_1", partitions_def=partitions_def1), + AssetSpec("my_asset_2", partitions_def=partitions_def2), + ], + can_subset=True, + ) + def my_assets(context): + assert context.partition_keys == ["2020-01-01", "2020-01-02", "2020-01-03"] + assert context.partition_key_range == PartitionKeyRange("2020-01-01", "2020-01-03") + assert context.partition_time_window == TimeWindow( + partitions_def1.time_window_for_partition_key("2020-01-01").start, + partitions_def1.time_window_for_partition_key("2020-01-03").end, + ) + for asset_key in context.selected_asset_keys: + yield MaterializeResult(asset_key=asset_key) + + result = materialize( + assets=[my_assets], + selection=["my_asset_1"], + tags={ + ASSET_PARTITION_RANGE_START_TAG: "2020-01-01", + ASSET_PARTITION_RANGE_END_TAG: "2020-01-03", + }, + ) + assert result.success + + materializations = result.asset_materializations_for_node("my_assets") + assert len(materializations) == 3 + assert { + (materialization.asset_key, materialization.partition) + for materialization in materializations + } == { + (AssetKey(["my_asset_1"]), "2020-01-01"), + (AssetKey(["my_asset_1"]), "2020-01-02"), + (AssetKey(["my_asset_1"]), "2020-01-03"), + } + + def test_two_partitioned_multi_assets_job(): partitions_def = StaticPartitionsDefinition(["a", "b", "c", "d"]) @@ -786,7 +873,7 @@ def assets2(): ... with pytest.raises( CheckError, - match="AssetSpec for asset1 has partitions_def which is different than the partitions_def provided to AssetsDefinition.", + match="which is different than the partitions_def provided to AssetsDefinition", ): @multi_asset( @@ -796,8 +883,8 @@ def assets2(): ... def assets3(): ... with pytest.raises( - CheckError, - match="All AssetSpecs must have the same partitions_def, but asset1 and asset2 have different partitions_defs.", + DagsterInvalidDefinitionError, + match="If different AssetSpecs have different partitions_defs, can_subset must be True", ): @multi_asset( @@ -807,3 +894,17 @@ def assets3(): ... ], ) def assets4(): ... + + with pytest.raises( + CheckError, + match="If partitions_def is provided, then either all specs must have that PartitionsDefinition or none", + ): + + @multi_asset( + specs=[ + AssetSpec("asset1", partitions_def=partitions_def), + AssetSpec("asset2"), + ], + partitions_def=partitions_def, + ) + def assets5(): ... diff --git a/python_modules/dagster/dagster_tests/cli_tests/command_tests/assets.py b/python_modules/dagster/dagster_tests/cli_tests/command_tests/assets.py index 3cc4565e0f711..59dd764b890af 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/command_tests/assets.py +++ b/python_modules/dagster/dagster_tests/cli_tests/command_tests/assets.py @@ -1,4 +1,4 @@ -from dagster import StaticPartitionsDefinition, asset +from dagster import AssetExecutionContext, Config, StaticPartitionsDefinition, asset from dagster._core.definitions.backfill_policy import BackfillPolicy from dagster._core.definitions.time_window_partitions import DailyPartitionsDefinition @@ -37,6 +37,21 @@ def single_run_partitioned_asset() -> None: ... def multi_run_partitioned_asset() -> None: ... +class MyConfig(Config): + some_prop: str + + +@asset +def asset_with_config(context: AssetExecutionContext, config: MyConfig): + context.log.info(f"some_prop:{config.some_prop}") + + +@asset +def asset_assert_with_config(context: AssetExecutionContext, config: MyConfig): + assert config.some_prop == "foo" + context.log.info(f"some_prop:{config.some_prop}") + + @asset def fail_asset() -> None: raise Exception("failure") diff --git a/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_asset_list_command.py b/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_asset_list_command.py index ae1f33fcecc16..482d04b7e5364 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_asset_list_command.py +++ b/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_asset_list_command.py @@ -28,6 +28,8 @@ def test_no_selection(): == "\n".join( [ "asset1", + "asset_assert_with_config", + "asset_with_config", "differently_partitioned_asset", "downstream_asset", "fail_asset", diff --git a/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_materialize_command.py b/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_materialize_command.py index 4d16a01ade5eb..1a730fa956b42 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_materialize_command.py +++ b/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_materialize_command.py @@ -1,7 +1,7 @@ from typing import Optional from click.testing import CliRunner -from dagster import AssetKey +from dagster import AssetKey, DagsterInvalidConfigError from dagster._cli.asset import asset_materialize_command from dagster._core.test_utils import instance_for_test from dagster._utils import file_relative_path @@ -177,3 +177,22 @@ def test_partition_range_multi_run_backfill_policy(): def test_failure(): result = invoke_materialize("fail_asset") assert result.exit_code == 1 + + +def test_run_cli_config_json(): + with instance_for_test() as instance: + asset_key = "asset_assert_with_config" + runner = CliRunner() + options = [ + "-f", + file_relative_path(__file__, "assets.py"), + "--select", + asset_key, + "--config-json", + '{"ops": {"asset_assert_with_config": {"config": {"some_prop": "foo"}}}}', + ] + + result = runner.invoke(asset_materialize_command, options) + assert not isinstance(result, DagsterInvalidConfigError) + assert instance.get_latest_materialization_event(AssetKey(asset_key)) is not None + assert result.exit_code == 0 diff --git a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill.py b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill.py index b0c71d4084069..765204ab65ebe 100644 --- a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill.py +++ b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill.py @@ -17,20 +17,24 @@ import pytest from dagster import ( AssetCheckResult, + AssetDep, AssetIn, AssetKey, AssetOut, AssetsDefinition, + AssetSpec, BackfillPolicy, DagsterInstance, DagsterRunStatus, DailyPartitionsDefinition, HourlyPartitionsDefinition, LastPartitionMapping, + MaterializeResult, Nothing, PartitionKeyRange, PartitionsDefinition, RunRequest, + StaticPartitionMapping, StaticPartitionsDefinition, TimeWindowPartitionMapping, WeeklyPartitionsDefinition, @@ -562,7 +566,9 @@ def get_asset_graph( ) as get_builtin_partition_mapping_types: get_builtin_partition_mapping_types.return_value = tuple( assets_def.infer_partition_mapping( - dep_key, assets_defs_by_key[dep_key].partitions_def + next(iter(assets_def.keys)), + dep_key, + assets_defs_by_key[dep_key].specs_by_key[dep_key].partitions_def, ).__class__ for assets in assets_by_repo_name.values() for assets_def in assets @@ -1685,6 +1691,56 @@ def my_multi_asset(): assert AssetKeyPartitionKey(AssetKey("c"), "1") in backfill_data.requested_subset +def test_multi_asset_internal_deps_different_partitions_asset_backfill() -> None: + @multi_asset( + specs=[ + AssetSpec( + "asset1", partitions_def=StaticPartitionsDefinition(["a", "b"]), skippable=True + ), + AssetSpec( + "asset2", + partitions_def=StaticPartitionsDefinition(["1"]), + deps=[ + AssetDep( + "asset1", + partition_mapping=StaticPartitionMapping({"a": {"1"}, "b": {"1"}}), + ) + ], + skippable=True, + ), + ], + can_subset=True, + ) + def my_multi_asset(context): + for asset_key in context.selected_asset_keys: + yield MaterializeResult(asset_key=asset_key) + + instance = DagsterInstance.ephemeral() + repo_dict = {"repo": [my_multi_asset]} + asset_graph = get_asset_graph(repo_dict) + current_time = create_datetime(2024, 1, 9, 0, 0, 0) + asset_backfill_data = AssetBackfillData.from_asset_graph_subset( + asset_graph_subset=AssetGraphSubset.all( + asset_graph, dynamic_partitions_store=MagicMock(), current_time=current_time + ), + backfill_start_timestamp=current_time.timestamp(), + dynamic_partitions_store=MagicMock(), + ) + backfill_data_after_iter1 = _single_backfill_iteration( + "fake_id", asset_backfill_data, asset_graph, instance, repo_dict + ) + after_iter1_requested_subset = backfill_data_after_iter1.requested_subset + assert AssetKeyPartitionKey(AssetKey("asset1"), "a") in after_iter1_requested_subset + assert AssetKeyPartitionKey(AssetKey("asset1"), "b") in after_iter1_requested_subset + assert AssetKeyPartitionKey(AssetKey("asset2"), "1") not in after_iter1_requested_subset + + backfill_data_after_iter2 = _single_backfill_iteration( + "fake_id", backfill_data_after_iter1, asset_graph, instance, repo_dict + ) + after_iter2_requested_subset = backfill_data_after_iter2.requested_subset + assert AssetKeyPartitionKey(AssetKey("asset2"), "1") in after_iter2_requested_subset + + def test_multi_asset_internal_and_external_deps_asset_backfill() -> None: pd = StaticPartitionsDefinition(["1", "2", "3"]) diff --git a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill_with_backfill_policies.py b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill_with_backfill_policies.py index 68a9a2cb714a3..08930878ff2f1 100644 --- a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill_with_backfill_policies.py +++ b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill_with_backfill_policies.py @@ -15,7 +15,6 @@ asset, ) from dagster._core.definitions.partition import StaticPartitionsDefinition -from dagster._core.errors import DagsterBackfillFailedError from dagster._core.execution.asset_backfill import AssetBackfillData, AssetBackfillStatus from dagster._core.instance_for_test import instance_for_test from dagster._core.storage.tags import ( @@ -37,7 +36,7 @@ ) -def test_asset_backfill_not_all_asset_have_backfill_policy(): +def test_asset_backfill_not_all_asset_have_backfill_policy() -> None: @asset(backfill_policy=None) def unpartitioned_upstream_of_partitioned(): return 1 @@ -45,6 +44,7 @@ def unpartitioned_upstream_of_partitioned(): @asset( partitions_def=DailyPartitionsDefinition("2023-01-01"), backfill_policy=BackfillPolicy.single_run(), + deps=[unpartitioned_upstream_of_partitioned], ) def upstream_daily_partitioned_asset(): return 1 @@ -69,19 +69,35 @@ def upstream_daily_partitioned_asset(): backfill_start_timestamp=get_current_timestamp(), ) - with pytest.raises( - DagsterBackfillFailedError, - match=( - "Either all assets must have backfill policies or none of them must have backfill" - " policies" - ), - ): - execute_asset_backfill_iteration_consume_generator( - backfill_id="test_backfill_id", - asset_backfill_data=backfill_data, - asset_graph=asset_graph, - instance=DagsterInstance.ephemeral(), - ) + instance = DagsterInstance.ephemeral() + _, materialized, failed = run_backfill_to_completion( + asset_graph, + assets_by_repo_name, + backfill_data=backfill_data, + fail_asset_partitions=set(), + instance=instance, + ) + + assert len(failed) == 0 + assert {akpk.asset_key for akpk in materialized} == { + unpartitioned_upstream_of_partitioned.key, + upstream_daily_partitioned_asset.key, + } + + runs = instance.get_runs(ascending=True) + + # separate runs for the assets (different partitions_def / backfill policy) + assert len(runs) == 2 + + unpartitioned = runs[0] + assert unpartitioned.tags == {"dagster/backfill": "backfillid_x"} + + partitioned = runs[1] + assert partitioned.tags.keys() == { + "dagster/asset_partition_range_end", + "dagster/asset_partition_range_start", + "dagster/backfill", + } def test_asset_backfill_parent_and_children_have_different_backfill_policy(): diff --git a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_run.py b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_run.py index 6db5285f05648..192d26c9bb9a1 100644 --- a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_run.py +++ b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_run.py @@ -2908,12 +2908,6 @@ def test_repository_namespacing(executor): assert len(ticks) == 2 -def test_settings(): - settings = {"use_threads": True, "num_workers": 4} - with instance_for_test(overrides={"sensors": settings}) as thread_inst: - assert thread_inst.get_settings("sensors") == settings - - @pytest.mark.parametrize("sensor_name", ["logging_sensor", "multi_asset_logging_sensor"]) def test_sensor_logging(executor, instance, workspace_context, remote_repo, sensor_name) -> None: sensor = remote_repo.get_sensor(sensor_name) diff --git a/python_modules/dagster/dagster_tests/daemon_tests/test_backfill.py b/python_modules/dagster/dagster_tests/daemon_tests/test_backfill.py index 740e8bb9ae0a3..7b88aeecb6dd2 100644 --- a/python_modules/dagster/dagster_tests/daemon_tests/test_backfill.py +++ b/python_modules/dagster/dagster_tests/daemon_tests/test_backfill.py @@ -4,6 +4,9 @@ import string import sys import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import cast from unittest import mock import dagster._check as check @@ -58,11 +61,11 @@ from dagster._core.execution.backfill import BulkActionStatus, PartitionBackfill from dagster._core.execution.plan.resume_retry import ReexecutionStrategy from dagster._core.remote_representation import ( + CodeLocation, InProcessCodeLocationOrigin, RemoteRepository, RemoteRepositoryOrigin, ) -from dagster._core.remote_representation.code_location import CodeLocation from dagster._core.storage.compute_log_manager import ComputeIOType from dagster._core.storage.dagster_run import ( IN_PROGRESS_RUN_STATUSES, @@ -81,13 +84,17 @@ ) from dagster._core.test_utils import ( create_run_for_test, + create_test_daemon_workspace_context, environ, + instance_for_test, step_did_not_run, step_failed, step_succeeded, + wait_for_futures, ) from dagster._core.types.loadable_target_origin import LoadableTargetOrigin from dagster._core.workspace.context import WorkspaceProcessContext +from dagster._core.workspace.load_target import ModuleTarget from dagster._daemon import get_default_daemon_logger from dagster._daemon.auto_run_reexecution.auto_run_reexecution import ( consume_new_runs_for_automatic_reexecution, @@ -580,10 +587,12 @@ def wait_for_all_runs_to_finish(instance, timeout=10): break +@pytest.mark.parametrize("parallel", [True, False]) def test_simple_backfill( instance: DagsterInstance, workspace_context: WorkspaceProcessContext, remote_repo: RemoteRepository, + parallel: bool, ): partition_set = remote_repo.get_partition_set("the_job_partition_set") instance.add_backfill( @@ -600,7 +609,24 @@ def test_simple_backfill( ) assert instance.get_runs_count() == 0 - list(execute_backfill_iteration(workspace_context, get_default_daemon_logger("BackfillDaemon"))) + if parallel: + backfill_daemon_futures = {} + list( + execute_backfill_iteration( + workspace_context, + get_default_daemon_logger("BackfillDaemon"), + threadpool_executor=ThreadPoolExecutor(2), + backfill_futures=backfill_daemon_futures, + ) + ) + + wait_for_futures(backfill_daemon_futures) + else: + list( + execute_backfill_iteration( + workspace_context, get_default_daemon_logger("BackfillDaemon") + ) + ) assert instance.get_runs_count() == 3 runs = instance.get_runs() @@ -613,6 +639,105 @@ def test_simple_backfill( assert three.tags[PARTITION_NAME_TAG] == "three" +@pytest.mark.parametrize("parallel", [True, False]) +def test_two_backfills_at_the_same_time( + tmp_path: Path, + parallel: bool, +): + # In order to avoid deadlock, we need to ensure that the instance we + # are using will launch runs in separate subprocesses rather than in + # the same in-memory process. This is akin to the context created in + # https://github.com/dagster-io/dagster/blob/a116c44/python_modules/dagster/dagster_tests/scheduler_tests/conftest.py#L53-L71 + with instance_for_test( + overrides={ + "event_log_storage": { + "module": "dagster._core.storage.event_log", + "class": "ConsolidatedSqliteEventLogStorage", + "config": {"base_dir": str(tmp_path)}, + }, + "run_retries": {"enabled": True}, + } + ) as instance: + with create_test_daemon_workspace_context( + workspace_load_target=ModuleTarget( + module_name="dagster_tests.daemon_tests.test_backfill", + attribute="the_repo", + working_directory=os.path.join(os.path.dirname(__file__), "..", ".."), + location_name="test_location", + ), + instance=instance, + ) as workspace_context: + remote_repo = cast( + CodeLocation, + next( + iter( + workspace_context.create_request_context() + .get_code_location_entries() + .values() + ) + ).code_location, + ).get_repository("the_repo") + + first_partition_set = remote_repo.get_partition_set("the_job_partition_set") + second_partition_keys = my_config.partitions_def.get_partition_keys() + second_partition_set = remote_repo.get_partition_set( + "comp_always_succeed_partition_set" + ) + instance.add_backfill( + PartitionBackfill( + backfill_id="simple", + partition_set_origin=first_partition_set.get_remote_origin(), + status=BulkActionStatus.REQUESTED, + partition_names=["one", "two", "three"], + from_failure=False, + reexecution_steps=None, + tags=None, + backfill_timestamp=get_current_timestamp(), + ) + ) + instance.add_backfill( + PartitionBackfill( + backfill_id="partition_schedule_from_job", + partition_set_origin=second_partition_set.get_remote_origin(), + status=BulkActionStatus.REQUESTED, + partition_names=second_partition_keys[:3], + from_failure=False, + reexecution_steps=None, + tags=None, + backfill_timestamp=get_current_timestamp(), + ) + ) + assert instance.get_runs_count() == 0 + + if parallel: + threadpool_executor = ThreadPoolExecutor(4) + backfill_daemon_futures = {} + list( + execute_backfill_iteration( + workspace_context, + get_default_daemon_logger("BackfillDaemon"), + threadpool_executor=threadpool_executor, + backfill_futures=backfill_daemon_futures, + ) + ) + + wait_for_futures(backfill_daemon_futures) + else: + list( + execute_backfill_iteration( + workspace_context, get_default_daemon_logger("BackfillDaemon") + ) + ) + + assert instance.get_runs_count() == 6 + + runs = list(instance.get_runs()) + backfill_ids = sorted(run.tags[BACKFILL_ID_TAG] for run in runs) + partition_names = {run.tags[PARTITION_NAME_TAG] for run in runs} + assert backfill_ids == ["partition_schedule_from_job"] * 3 + ["simple"] * 3 + assert partition_names == {"one", "two", "three", *second_partition_keys[:3]} + + def test_canceled_backfill( instance: DagsterInstance, workspace_context: WorkspaceProcessContext, @@ -648,10 +773,12 @@ def test_canceled_backfill( assert instance.get_runs_count() == 1 +@pytest.mark.parametrize("parallel", [True, False]) def test_failure_backfill( instance: DagsterInstance, workspace_context: WorkspaceProcessContext, remote_repo: RemoteRepository, + parallel: bool, ): output_file = _failure_flag_file() partition_set = remote_repo.get_partition_set("conditional_failure_job_partition_set") @@ -671,11 +798,25 @@ def test_failure_backfill( try: touch_file(output_file) - list( - execute_backfill_iteration( - workspace_context, get_default_daemon_logger("BackfillDaemon") + if parallel: + backfill_daemon_futures = {} + list( + execute_backfill_iteration( + workspace_context, + get_default_daemon_logger("BackfillDaemon"), + threadpool_executor=ThreadPoolExecutor(2), + backfill_futures=backfill_daemon_futures, + ) ) - ) + + wait_for_futures(backfill_daemon_futures) + else: + list( + execute_backfill_iteration( + workspace_context, get_default_daemon_logger("BackfillDaemon") + ) + ) + wait_for_all_runs_to_start(instance) finally: os.remove(output_file) @@ -718,7 +859,25 @@ def test_failure_backfill( ) assert not os.path.isfile(_failure_flag_file()) - list(execute_backfill_iteration(workspace_context, get_default_daemon_logger("BackfillDaemon"))) + if parallel: + backfill_daemon_futures = {} + list( + execute_backfill_iteration( + workspace_context, + get_default_daemon_logger("BackfillDaemon"), + threadpool_executor=ThreadPoolExecutor(2), + backfill_futures=backfill_daemon_futures, + ) + ) + + wait_for_futures(backfill_daemon_futures) + else: + list( + execute_backfill_iteration( + workspace_context, get_default_daemon_logger("BackfillDaemon") + ) + ) + wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 6 @@ -925,6 +1084,61 @@ def test_large_backfill( assert instance.get_runs_count() == 3 +def test_backfill_is_processed_only_once( + instance: DagsterInstance, + workspace_context: WorkspaceProcessContext, + remote_repo: RemoteRepository, +): + backfill_id = "simple" + partition_set = remote_repo.get_partition_set("config_job_partition_set") + instance.add_backfill( + PartitionBackfill( + backfill_id=backfill_id, + partition_set_origin=partition_set.get_remote_origin(), + status=BulkActionStatus.REQUESTED, + partition_names=["one", "two", "three"], + from_failure=False, + reexecution_steps=None, + tags=None, + backfill_timestamp=get_current_timestamp(), + ) + ) + assert instance.get_runs_count() == 0 + + threadpool_executor = ThreadPoolExecutor(2) + backfill_daemon_futures = {} + list( + execute_backfill_iteration( + workspace_context, + get_default_daemon_logger("BackfillDaemon"), + threadpool_executor=threadpool_executor, + backfill_futures=backfill_daemon_futures, + ) + ) + + assert instance.get_runs_count() == 0 + future = backfill_daemon_futures[backfill_id] + + with mock.patch.object( + threadpool_executor, "submit", side_effect=AssertionError("Should not be called") + ): + list( + execute_backfill_iteration( + workspace_context, + get_default_daemon_logger("BackfillDaemon"), + threadpool_executor=threadpool_executor, + backfill_futures=backfill_daemon_futures, + ) + ) + + assert instance.get_runs_count() == 0 + assert backfill_daemon_futures[backfill_id] is future + + wait_for_futures(backfill_daemon_futures) + + assert instance.get_runs_count() == 3 + + def test_unloadable_backfill(instance, workspace_context): unloadable_origin = _unloadable_partition_set_origin() instance.add_backfill( @@ -2828,10 +3042,12 @@ def test_asset_backfill_logs( assert record_dict.get("msg") +@pytest.mark.parametrize("parallel", [True, False]) def test_asset_backfill_from_asset_graph_subset( instance: DagsterInstance, workspace_context: WorkspaceProcessContext, remote_repo: RemoteRepository, + parallel: bool, ): del remote_repo @@ -2860,7 +3076,24 @@ def test_asset_backfill_from_asset_graph_subset( assert backfill assert backfill.status == BulkActionStatus.REQUESTED - list(execute_backfill_iteration(workspace_context, get_default_daemon_logger("BackfillDaemon"))) + if parallel: + backfill_daemon_futures = {} + list( + execute_backfill_iteration( + workspace_context, + get_default_daemon_logger("BackfillDaemon"), + threadpool_executor=ThreadPoolExecutor(2), + backfill_futures=backfill_daemon_futures, + ) + ) + + wait_for_futures(backfill_daemon_futures) + else: + list( + execute_backfill_iteration( + workspace_context, get_default_daemon_logger("BackfillDaemon") + ) + ) assert instance.get_runs_count() == 3 wait_for_all_runs_to_start(instance, timeout=30) assert instance.get_runs_count() == 3 @@ -2875,7 +3108,24 @@ def test_asset_backfill_from_asset_graph_subset( assert step_succeeded(instance, run, "reusable") assert step_succeeded(instance, run, "bar") - list(execute_backfill_iteration(workspace_context, get_default_daemon_logger("BackfillDaemon"))) + if parallel: + backfill_daemon_futures = {} + list( + execute_backfill_iteration( + workspace_context, + get_default_daemon_logger("BackfillDaemon"), + threadpool_executor=ThreadPoolExecutor(2), + backfill_futures=backfill_daemon_futures, + ) + ) + + wait_for_futures(backfill_daemon_futures) + else: + list( + execute_backfill_iteration( + workspace_context, get_default_daemon_logger("BackfillDaemon") + ) + ) backfill = instance.get_backfill("backfill_from_asset_graph_subset") assert backfill assert backfill.status == BulkActionStatus.COMPLETED_SUCCESS diff --git a/python_modules/dagster/dagster_tests/daemon_tests/test_dagster_daemon.py b/python_modules/dagster/dagster_tests/daemon_tests/test_dagster_daemon.py index 4c656c6077505..1b90b97fea1bc 100644 --- a/python_modules/dagster/dagster_tests/daemon_tests/test_dagster_daemon.py +++ b/python_modules/dagster/dagster_tests/daemon_tests/test_dagster_daemon.py @@ -11,6 +11,13 @@ from dagster._utils.log import get_structlog_json_formatter +@pytest.mark.parametrize("daemon", ["backfills", "schedules", "sensors"]) +def test_settings(daemon): + settings = {"use_threads": True, "num_workers": 4} + with instance_for_test(overrides={daemon: settings}) as thread_inst: + assert thread_inst.get_settings(daemon) == settings + + def test_scheduler_instance(): with instance_for_test( overrides={ diff --git a/python_modules/dagster/dagster_tests/definitions_tests/declarative_automation_tests/daemon_tests/test_e2e.py b/python_modules/dagster/dagster_tests/definitions_tests/declarative_automation_tests/daemon_tests/test_e2e.py index 4a664fda6e6c4..402c0133cfcd4 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/declarative_automation_tests/daemon_tests/test_e2e.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/declarative_automation_tests/daemon_tests/test_e2e.py @@ -176,15 +176,20 @@ def _execute_ticks( ) ) + backfill_daemon_futures = {} list( execute_backfill_iteration( context, get_default_daemon_logger("BackfillDaemon"), + threadpool_executor=threadpool_executor, + backfill_futures=backfill_daemon_futures, + debug_crash_flags=debug_crash_flags or {}, ) ) wait_for_futures(asset_daemon_futures) wait_for_futures(sensor_daemon_futures) + wait_for_futures(backfill_daemon_futures) def _get_current_state(context: WorkspaceRequestContext) -> Mapping[str, InstigatorState]: diff --git a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py index 09b0151444fa4..3299b270c437c 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py @@ -28,6 +28,7 @@ ) from dagster._core.definitions.asset_check_spec import AssetCheckKey from dagster._core.definitions.asset_checks import AssetChecksDefinition +from dagster._core.definitions.asset_in import AssetIn from dagster._core.errors import ( DagsterInvalidDefinitionError, DagsterInvalidSubsetError, @@ -118,19 +119,105 @@ def my_check() -> AssetCheckResult: assert spec.asset_key == AssetKey(["prefix", "asset1"]) +def test_asset_check_input() -> None: + @asset + def asset1() -> int: + return 5 + + @asset_check(asset=asset1) + def my_check1(asset1: int) -> AssetCheckResult: + return AssetCheckResult(passed=asset1 == 5) + + @asset_check(asset=asset1) + def my_check2(random_name: int) -> AssetCheckResult: + return AssetCheckResult(passed=random_name == 5) + + @asset_check(asset=asset1) + def my_check3(context, random_name: int) -> AssetCheckResult: + return AssetCheckResult(passed=random_name == 5) + + @asset_check(asset=asset1) + def my_check4(context, asset1: int) -> AssetCheckResult: + return AssetCheckResult(passed=asset1 == 5) + + result = execute_assets_and_checks( + assets=[asset1], asset_checks=[my_check1, my_check2, my_check3, my_check4] + ) + + assert result.success + assert len(result.get_asset_check_evaluations()) == 4 + assert all(check.passed for check in result.get_asset_check_evaluations()) + + +def test_asset_check_additional_ins() -> None: + @asset + def asset1() -> int: + return 5 + + @asset + def asset2() -> int: + return 4 + + @asset_check(asset=asset1, additional_ins={"asset2": AssetIn(key=asset2.key)}) + def my_check(asset1: int, asset2: int) -> AssetCheckResult: + return AssetCheckResult(passed=asset1 == 5 and asset2 == 4) + + @asset_check(asset=asset1, additional_ins={"asset2": AssetIn(key=asset2.key)}) + def my_check2(asset2: int) -> AssetCheckResult: + return AssetCheckResult(passed=asset2 == 4) + + @asset_check(asset=asset1, additional_ins={"asset2": AssetIn(key=asset2.key)}) + def my_check3(context, asset2: int) -> AssetCheckResult: + return AssetCheckResult(passed=asset2 == 4) + + # Error bc asset2 is in additional_ins but not in the function signature + with pytest.raises(DagsterInvalidDefinitionError): + + @asset_check(asset=asset1, additional_ins={"asset2": AssetIn(key=asset2.key)}) + def my_check4(): + return AssetCheckResult(passed=True) + + # Error bc asset1 is in both additional_ins and the function signature + with pytest.raises(DagsterInvalidDefinitionError): + + @asset_check(asset=asset1, additional_ins={"asset1": AssetIn(key=asset1.key)}) + def my_check5(asset1: int) -> AssetCheckResult: + return AssetCheckResult(passed=asset1 == 5) + + # Error bc asset2 is in the function signature but not additional_ins + with pytest.raises(DagsterInvalidDefinitionError): + + @asset_check(asset=asset1, additional_ins={"asset2": AssetIn(key=asset2.key)}) + def my_check6(asset1: int) -> AssetCheckResult: + return AssetCheckResult(passed=asset1 == 5) + + result = execute_assets_and_checks( + assets=[asset1, asset2], asset_checks=[my_check, my_check2, my_check3] + ) + + assert result.success + assert len(result.get_asset_check_evaluations()) == 3 + assert all(check.passed for check in result.get_asset_check_evaluations()) + + def test_asset_check_input_with_prefix() -> None: @asset(key_prefix="prefix") - def asset1() -> None: ... + def asset1() -> int: + return 5 @asset_check(asset=asset1) - def my_check(asset1) -> AssetCheckResult: - return AssetCheckResult(passed=True) + def my_check(unrelated_name: int) -> AssetCheckResult: + return AssetCheckResult(passed=unrelated_name == 5) spec = my_check.get_spec_for_check_key( AssetCheckKey(AssetKey(["prefix", "asset1"]), "my_check") ) assert spec.asset_key == AssetKey(["prefix", "asset1"]) + result = execute_assets_and_checks(assets=[asset1], asset_checks=[my_check]) + assert result.success + assert len(result.get_asset_check_evaluations()) == 1 + def test_execute_asset_and_check() -> None: @asset @@ -862,6 +949,26 @@ def check1(context) -> AssetCheckResult: assert result.passed +def test_direct_invocation_with_input() -> None: + @asset_check(asset="asset1") + def check1(asset1) -> AssetCheckResult: + return AssetCheckResult(passed=True) + + result = check1(5) + assert isinstance(result, AssetCheckResult) + assert result.passed + + +def test_direct_invocation_with_context_and_input() -> None: + @asset_check(asset="asset1") + def check1(context, asset1) -> AssetCheckResult: + return AssetCheckResult(passed=True) + + result = check1(build_op_context(), 5) + assert isinstance(result, AssetCheckResult) + assert result.passed + + def test_multi_check_direct_invocation() -> None: @multi_asset_check( specs=[ @@ -881,3 +988,210 @@ def checks() -> Iterable[AssetCheckResult]: assert results[0].passed assert not results[1].passed assert results[2].passed + + +def test_direct_invocation_with_inputs() -> None: + @asset + def asset1() -> int: + return 4 + + @asset + def asset2() -> int: + return 5 + + @multi_asset_check( + specs=[ + AssetCheckSpec("check1", asset=asset1), + AssetCheckSpec("check2", asset=asset2), + ] + ) + def multi_check(asset1: int, asset2: int) -> Iterable[AssetCheckResult]: + yield AssetCheckResult(passed=asset1 == 4, asset_key="asset1", check_name="check1") + yield AssetCheckResult(passed=asset2 == 5, asset_key="asset2", check_name="check2") + + result = list(multi_check(4, 5)) # type: ignore + assert len(result) == 2 + assert all(isinstance(r, AssetCheckResult) for r in result) + assert all(r.passed for r in result) + + +def test_direct_invocation_remapped_inputs() -> None: + @asset + def asset1() -> int: + return 4 + + @asset + def asset2() -> int: + return 5 + + @multi_asset_check( + specs=[ + AssetCheckSpec("check1", asset=asset1), + AssetCheckSpec("check2", asset=asset2), + ], + ins={"remapped": asset1.key}, + ) + def multi_check(remapped: int, asset2: int) -> Iterable[AssetCheckResult]: + yield AssetCheckResult(passed=remapped == 4, asset_key="asset1", check_name="check1") + yield AssetCheckResult(passed=asset2 == 5, asset_key="asset2", check_name="check2") + + result = list(multi_check(4, 5)) # type: ignore + assert len(result) == 2 + assert all(isinstance(r, AssetCheckResult) for r in result) + assert all(r.passed for r in result) + + +def test_multi_check_asset_with_inferred_inputs() -> None: + """Test automatic inference of asset inputs in a multi-check.""" + + @asset + def asset1() -> int: + return 4 + + @asset + def asset2() -> int: + return 5 + + @multi_asset_check( + specs=[ + AssetCheckSpec("check1", asset=asset1), + AssetCheckSpec("check2", asset=asset2), + ] + ) + def multi_check(asset1: int, asset2: int) -> Iterable[AssetCheckResult]: + yield AssetCheckResult(passed=asset1 == 4, asset_key="asset1", check_name="check1") + yield AssetCheckResult(passed=asset2 == 5, asset_key="asset2", check_name="check2") + + result = execute_assets_and_checks(assets=[asset1, asset2], asset_checks=[multi_check]) + assert result.success + + check_evals = result.get_asset_check_evaluations() + assert len(check_evals) == 2 + assert all(check_eval.passed for check_eval in check_evals) + + +def test_multi_check_input_remapping() -> None: + """Test remapping an asset input to a different name in a multi-check.""" + + @asset + def asset1() -> int: + return 4 + + @asset + def asset2() -> int: + return 5 + + @multi_asset_check( + specs=[ + AssetCheckSpec("check1", asset=asset1), + AssetCheckSpec("check2", asset=asset2), + ], + ins={"remapped": asset1.key}, + ) + def multi_check(remapped: int, asset2: int) -> Iterable[AssetCheckResult]: + yield AssetCheckResult(passed=remapped == 4, asset_key="asset1", check_name="check1") + yield AssetCheckResult(passed=asset2 == 5, asset_key="asset2", check_name="check2") + + result = execute_assets_and_checks(assets=[asset1, asset2], asset_checks=[multi_check]) + assert result.success + + check_evals = result.get_asset_check_evaluations() + assert len(check_evals) == 2 + assert all(check_eval.passed for check_eval in check_evals) + + +def test_multi_check_input_remapping_with_context() -> None: + """Test remapping an asset input to a different name in a multi-check.""" + + @asset + def asset1() -> int: + return 4 + + @asset + def asset2() -> int: + return 5 + + @multi_asset_check( + specs=[ + AssetCheckSpec("check1", asset=asset1), + AssetCheckSpec("check2", asset=asset2), + ], + ins={"remapped": asset1.key}, + ) + def multi_check(context, remapped: int, asset2: int) -> Iterable[AssetCheckResult]: + assert isinstance(context, AssetCheckExecutionContext) + yield AssetCheckResult(passed=remapped == 4, asset_key="asset1", check_name="check1") + yield AssetCheckResult(passed=asset2 == 5, asset_key="asset2", check_name="check2") + + result = execute_assets_and_checks(assets=[asset1, asset2], asset_checks=[multi_check]) + assert result.success + + check_evals = result.get_asset_check_evaluations() + assert len(check_evals) == 2 + assert all(check_eval.passed for check_eval in check_evals) + + +def test_input_manager_overrides_multi_asset_check_decorator() -> None: + """Test overriding input manager key for a particular asset in a multi-check, ensure that it is correctly mapped.""" + + @asset + def asset1() -> int: + return 4 + + @asset + def asset2() -> int: + return 5 + + @multi_asset_check( + specs=[ + AssetCheckSpec("check1", asset=asset1), + AssetCheckSpec("check2", asset=asset2), + ], + ins={"asset1": AssetIn(key="asset1", input_manager_key="override_manager")}, + ) + def my_check(asset1: int, asset2: int) -> Iterable[AssetCheckResult]: + yield AssetCheckResult(passed=asset1 == 4, asset_key="asset1", check_name="check1") + yield AssetCheckResult(passed=asset2 == 5, asset_key="asset2", check_name="check2") + + called = [] + + class MyIOManager(IOManager): + def load_input(self, context) -> int: + called.append(context.asset_key) + return 4 + + def handle_output(self, context, obj) -> None: + raise NotImplementedError() + + result = execute_assets_and_checks( + assets=[asset1, asset2], + asset_checks=[my_check], + resources={"override_manager": MyIOManager()}, + ) + + assert result.success + assert called == [AssetKey("asset1")] + assert all(check_eval.passed for check_eval in result.get_asset_check_evaluations()) + + +def test_nonsense_input_name() -> None: + """Test a nonsensical input name in a multi-check.""" + + @asset + def asset1() -> int: + return 4 + + @asset + def asset2() -> int: + return 5 + + with pytest.raises(DagsterInvalidDefinitionError): + + @multi_asset_check( + specs=[ + AssetCheckSpec("check1", asset=asset1), + AssetCheckSpec("check2", asset=asset2), + ], + ) + def my_check(nonsense: int, asset2: int): + pass diff --git a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator_secondary_assets.py b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator_secondary_assets.py index d06db18106ede..f171e92961452 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator_secondary_assets.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator_secondary_assets.py @@ -98,18 +98,18 @@ def check1(asset_1): def test_additional_ins_and_deps_overlap(): - with pytest.raises( - DagsterInvalidDefinitionError, - match=re.escape("deps value AssetKey(['asset2']) also declared as input/AssetIn"), - ): + @asset_check( + asset=asset1, + additional_ins={"asset_2": AssetIn("asset2")}, + additional_deps=[asset2], + ) + def check1(asset_2) -> AssetCheckResult: + return AssetCheckResult(passed=asset_2 == 5) - @asset_check( # pyright: ignore[reportArgumentType] - asset=asset1, - additional_ins={"asset_2": AssetIn("asset2")}, - additional_deps=[asset2], - ) - def check1(asset_2): - pass + result = execute_assets_and_checks(assets=[asset1, asset2], asset_checks=[check1]) + assert result.success + assert len(result.get_asset_check_evaluations()) == 1 + assert all(e.passed for e in result.get_asset_check_evaluations()) def test_additional_ins_must_correspond_to_params(): diff --git a/python_modules/dagster/dagster_tests/general_tests/test_record.py b/python_modules/dagster/dagster_tests/general_tests/test_record.py index 08af55d1fc1dc..8d15cfa20ca4a 100644 --- a/python_modules/dagster/dagster_tests/general_tests/test_record.py +++ b/python_modules/dagster/dagster_tests/general_tests/test_record.py @@ -364,7 +364,7 @@ class OtherSample: def test_build_args_and_assign(fields, defaults, expected): # tests / documents shared utility fn # don't hesitate to delete this upon refactor - assert build_args_and_assignment_strs(fields, defaults) == expected + assert build_args_and_assignment_strs(fields, defaults, kw_only=True) == expected @record @@ -830,3 +830,58 @@ def boop(self): def test_defensive_checks_running(): # make sure we have enabled defensive checks in test, ideally as broadly as possible assert os.getenv("DAGSTER_RECORD_DEFENSIVE_CHECKS") == "true" + + +def test_allow_posargs(): + @record(kw_only=False) + class Foo: + a: int + + assert Foo(2) + + @record(kw_only=False) + class Bar: + a: int + b: int + c: int = 4 + + assert Bar(1, 2) + + with pytest.raises(CheckError): + + @record(kw_only=False) + class Baz: + a: int = 4 + b: int # type: ignore # good job type checker + + +def test_posargs_inherit(): + @record(kw_only=False) + class Parent: + name: str + + @record(kw_only=False) + class Child(Parent): + parent: Parent + + p = Parent("Alex") + assert p + c = Child("Lyra", p) + assert c + + # test kw_only not being aligned + with pytest.raises(CheckError): + + @record + class Bad(Parent): + other: str + + with pytest.raises(CheckError): + + @record + class A: + a: int + + @record(kw_only=False) + class B(A): + b: int diff --git a/python_modules/dagster/dagster_tests/scheduler_tests/conftest.py b/python_modules/dagster/dagster_tests/scheduler_tests/conftest.py index 80c87a0a281ea..0679622ec42f8 100644 --- a/python_modules/dagster/dagster_tests/scheduler_tests/conftest.py +++ b/python_modules/dagster/dagster_tests/scheduler_tests/conftest.py @@ -1,10 +1,10 @@ import os import sys -from typing import Iterator, Optional +from typing import Iterator, Optional, cast import pytest from dagster._core.instance import DagsterInstance -from dagster._core.remote_representation.external import RemoteRepository +from dagster._core.remote_representation import CodeLocation, RemoteRepository from dagster._core.test_utils import ( SingleThreadPoolExecutor, create_test_daemon_workspace_context, @@ -73,11 +73,12 @@ def workspace_fixture( @pytest.fixture(name="remote_repo", scope="session") def remote_repo_fixture(workspace_context: WorkspaceProcessContext) -> RemoteRepository: - return next( - iter(workspace_context.create_request_context().get_code_location_entries().values()) - ).code_location.get_repository( # type: ignore # (possible none) - "the_repo" - ) + return cast( + CodeLocation, + next( + iter(workspace_context.create_request_context().get_code_location_entries().values()) + ).code_location, + ).get_repository("the_repo") def loadable_target_origin() -> LoadableTargetOrigin: diff --git a/python_modules/dagster/dagster_tests/storage_tests/test_event_log.py b/python_modules/dagster/dagster_tests/storage_tests/test_event_log.py index b111e213861fb..8db0dc8f8b240 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/test_event_log.py +++ b/python_modules/dagster/dagster_tests/storage_tests/test_event_log.py @@ -9,8 +9,14 @@ import pytest import sqlalchemy import sqlalchemy as db -from dagster import DagsterInstance +from dagster import AssetKey, AssetMaterialization, DagsterInstance, Out, Output, RetryRequested, op from dagster._core.errors import DagsterEventLogInvalidForRun +from dagster._core.execution.stats import ( + StepEventStatus, + build_run_stats_from_events, + build_run_step_stats_from_events, + build_run_step_stats_snapshot_from_events, +) from dagster._core.storage.event_log import ( ConsolidatedSqliteEventLogStorage, SqlEventLogStorageMetadata, @@ -28,7 +34,10 @@ from sqlalchemy import __version__ as sqlalchemy_version from sqlalchemy.engine import Connection -from dagster_tests.storage_tests.utils.event_log_storage import TestEventLogStorage +from dagster_tests.storage_tests.utils.event_log_storage import ( + TestEventLogStorage, + _synthesize_events, +) class TestInMemoryEventLogStorage(TestEventLogStorage): @@ -66,6 +75,9 @@ def event_log_storage(self, instance): assert isinstance(event_log_storage, SqliteEventLogStorage) yield instance.event_log_storage + def supports_multiple_event_type_queries(self): + return False + def test_filesystem_event_log_storage_run_corrupted(self, storage): # URL begins sqlite:/// @@ -182,6 +194,9 @@ def event_log_storage(self, instance): def is_sqlite(self, storage): return True + def supports_multiple_event_type_queries(self): + return False + @pytest.mark.parametrize("dagster_event_type", ["dummy"]) def test_get_latest_tags_by_partition(self, storage, instance, dagster_event_type): pytest.skip("skip this since legacy storage is harder to mock.patch") @@ -279,3 +294,92 @@ def test_concurrency_reconcile(): assert _get_slot_count(conn, "bar") == 3 assert _get_limit_row_num(conn, "foo") == 5 assert _get_limit_row_num(conn, "bar") == 3 + + +def test_run_stats(): + @op + def op_success(_): + return 1 + + @op + def asset_op(_): + yield AssetMaterialization(asset_key=AssetKey("asset_1")) + yield Output(1) + + @op + def op_failure(_): + raise ValueError("failing") + + def _ops(): + op_success() + asset_op() + op_failure() + + events, result = _synthesize_events(_ops, check_success=False) + + run_stats = build_run_stats_from_events(result.run_id, events) + + assert run_stats.run_id == result.run_id + assert run_stats.materializations == 1 + assert run_stats.steps_succeeded == 2 + assert run_stats.steps_failed == 1 + assert ( + run_stats.start_time is not None + and run_stats.end_time is not None + and run_stats.end_time > run_stats.start_time + ) + + # build up run stats through incremental events + incremental_run_stats = None + for event in events: + incremental_run_stats = build_run_stats_from_events( + result.run_id, [event], incremental_run_stats + ) + + assert incremental_run_stats == run_stats + + +def test_step_stats(): + @op + def op_success(_): + return 1 + + @op + def asset_op(_): + yield AssetMaterialization(asset_key=AssetKey("asset_1")) + yield Output(1) + + @op(out=Out(str)) + def op_failure(_): + time.sleep(0.001) + raise RetryRequested(max_retries=3) + + def _ops(): + op_success() + asset_op() + op_failure() + + events, result = _synthesize_events(_ops, check_success=False) + + step_stats = build_run_step_stats_from_events(result.run_id, events) + assert len(step_stats) == 3 + assert len([step for step in step_stats if step.status == StepEventStatus.SUCCESS]) == 2 + assert len([step for step in step_stats if step.status == StepEventStatus.FAILURE]) == 1 + assert all([step.run_id == result.run_id for step in step_stats]) + + op_failure_stats = next( + iter([step for step in step_stats if step.step_key == "op_failure"]), None + ) + assert op_failure_stats + assert op_failure_stats.attempts == 4 + assert len(op_failure_stats.attempts_list) == 4 + + # build up run stats through incremental events + incremental_snapshot = None + for event in events: + incremental_snapshot = build_run_step_stats_snapshot_from_events( + result.run_id, [event], incremental_snapshot + ) + + assert incremental_snapshot + assert incremental_snapshot.step_key_stats == step_stats diff --git a/python_modules/dagster/dagster_tests/storage_tests/utils/event_log_storage.py b/python_modules/dagster/dagster_tests/storage_tests/utils/event_log_storage.py index 1bfbd2ec23fbc..0bbab79ac9b3b 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/utils/event_log_storage.py +++ b/python_modules/dagster/dagster_tests/storage_tests/utils/event_log_storage.py @@ -504,6 +504,9 @@ def can_set_concurrency_defaults(self): def supports_offset_cursor_queries(self): return True + def supports_get_logs_for_all_runs_by_log_id(self): + return True + def supports_multiple_event_type_queries(self): return True @@ -4142,8 +4145,8 @@ def never_materializes_asset(): assert result.run_id == records[0].asset_entry.last_run_id def test_get_logs_for_all_runs_by_log_id_of_type(self, storage: EventLogStorage): - if not storage.supports_event_consumer_queries(): - pytest.skip("storage does not support event consumer queries") + if not self.supports_get_logs_for_all_runs_by_log_id(): + pytest.skip("storage does not support get_logs_for_all_runs_by_log_id") @op def return_one(_): @@ -4164,8 +4167,8 @@ def _ops(): ) == [DagsterEventType.RUN_SUCCESS, DagsterEventType.RUN_SUCCESS] def test_get_logs_for_all_runs_by_log_id_by_multi_type(self, storage: EventLogStorage): - if not storage.supports_event_consumer_queries(): - pytest.skip("storage does not support event consumer queries") + if not self.supports_get_logs_for_all_runs_by_log_id(): + pytest.skip("storage does not support get_logs_for_all_runs_by_log_id") if not self.supports_multiple_event_type_queries(): pytest.skip("storage does not support deprecated multi-event-type queries") @@ -4197,8 +4200,8 @@ def _ops(): ] def test_get_logs_for_all_runs_by_log_id_cursor(self, storage: EventLogStorage): - if not storage.supports_event_consumer_queries(): - pytest.skip("storage does not support event consumer queries") + if not self.supports_get_logs_for_all_runs_by_log_id(): + pytest.skip("storage does not support get_logs_for_all_runs_by_log_id") @op def return_one(_): @@ -4234,8 +4237,8 @@ def _ops(): ] def test_get_logs_for_all_runs_by_log_id_cursor_multi_type(self, storage: EventLogStorage): - if not storage.supports_event_consumer_queries(): - pytest.skip("storage does not support event consumer queries") + if not self.supports_get_logs_for_all_runs_by_log_id(): + pytest.skip("storage does not support get_logs_for_all_runs_by_log_id") if not self.supports_multiple_event_type_queries(): pytest.skip("storage does not support deprecated multi-event-type queries") @@ -4281,8 +4284,8 @@ def _ops(): ] def test_get_logs_for_all_runs_by_log_id_limit(self, storage: EventLogStorage): - if not storage.supports_event_consumer_queries(): - pytest.skip("storage does not support event consumer queries") + if not self.supports_get_logs_for_all_runs_by_log_id(): + pytest.skip("storage does not support get_logs_for_all_runs_by_log_id") @op def return_one(_): @@ -4314,8 +4317,8 @@ def _ops(): ] def test_get_logs_for_all_runs_by_log_id_limit_multi_type(self, storage: EventLogStorage): - if not storage.supports_event_consumer_queries(): - pytest.skip("storage does not support event consumer queries") + if not self.supports_get_logs_for_all_runs_by_log_id(): + pytest.skip("storage does not support get_logs_for_all_runs_by_log_id") if not self.supports_multiple_event_type_queries(): pytest.skip("storage does not support deprecated multi-event-type queries") @@ -4347,9 +4350,6 @@ def _ops(): ] def test_get_maximum_record_id(self, storage: EventLogStorage): - if not storage.supports_event_consumer_queries(): - pytest.skip("storage does not support event consumer queries") - storage.wipe() storage.store_event( @@ -4360,9 +4360,8 @@ def test_get_maximum_record_id(self, storage: EventLogStorage): run_id=make_new_run_id(), timestamp=time.time(), dagster_event=DagsterEvent( - DagsterEventType.ENGINE_EVENT.value, - "nonce", - event_specific_data=EngineEventData.in_process(999), + DagsterEventType.RUN_SUCCESS.value, + "my_job", ), ) ) @@ -4379,9 +4378,8 @@ def test_get_maximum_record_id(self, storage: EventLogStorage): run_id=make_new_run_id(), timestamp=time.time(), dagster_event=DagsterEvent( - DagsterEventType.ENGINE_EVENT.value, - "nonce", - event_specific_data=EngineEventData.in_process(999), + DagsterEventType.RUN_SUCCESS.value, + "my_job", ), ) ) diff --git a/python_modules/dagster/dagster_tests/storage_tests/utils/partition_status_cache.py b/python_modules/dagster/dagster_tests/storage_tests/utils/partition_status_cache.py index 9f4e8a9b715f4..fdc37412975db 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/utils/partition_status_cache.py +++ b/python_modules/dagster/dagster_tests/storage_tests/utils/partition_status_cache.py @@ -10,6 +10,7 @@ EventLogEntry, MultiPartitionKey, MultiPartitionsDefinition, + PartitionsDefinition, StaticPartitionsDefinition, asset, define_asset_job, @@ -79,36 +80,31 @@ def test_get_cached_partition_status_changed_time_partitions(self, instance): original_partitions_def = HourlyPartitionsDefinition(start_date="2022-01-01-00:00") new_partitions_def = DailyPartitionsDefinition(start_date="2022-01-01") - @asset(partitions_def=original_partitions_def) - def asset1(): - return 1 - - asset_key = AssetKey("asset1") - asset_graph = AssetGraph.from_assets([asset1]) - asset_job = define_asset_job("asset_job").resolve(asset_graph=asset_graph) + def make_asset_job_and_graph(partitions_def: PartitionsDefinition): + @asset(partitions_def=partitions_def) + def asset1(): + return 1 - def _swap_partitions_def(new_partitions_def, asset, asset_graph, asset_job): - asset._partitions_def = new_partitions_def # noqa: SLF001 - asset_graph = AssetGraph.from_assets([asset]) + asset_graph = AssetGraph.from_assets([asset1]) asset_job = define_asset_job("asset_job").resolve(asset_graph=asset_graph) - return asset, asset_job, asset_graph + return asset1, asset_job, asset_graph + + asset1, asset_job, asset_graph = make_asset_job_and_graph(original_partitions_def) counter = Counter() traced_counter.set(counter) - asset_records = list(instance.get_asset_records([asset_key])) + asset_records = list(instance.get_asset_records([asset1.key])) assert len(asset_records) == 0 asset_job.execute_in_process(instance=instance, partition_key="2022-02-01-00:00") # swap the partitions def and kick off a run before we try to get the cached status - asset1, asset_job, asset_graph = _swap_partitions_def( - new_partitions_def, asset1, asset_graph, asset_job - ) + asset1, asset_job, asset_graph = make_asset_job_and_graph(new_partitions_def) asset_job.execute_in_process(instance=instance, partition_key="2022-02-02") cached_status = get_and_update_asset_status_cache_value( - instance, asset_key, asset_graph.get(asset_key).partitions_def + instance, asset1.key, asset_graph.get(asset1.key).partitions_def ) assert cached_status @@ -127,34 +123,31 @@ def _swap_partitions_def(new_partitions_def, asset, asset_graph, asset_job): def test_get_cached_partition_status_by_asset(self, instance): partitions_def = DailyPartitionsDefinition(start_date="2022-01-01") - @asset(partitions_def=partitions_def) - def asset1(): - return 1 + def make_asset_job_and_graph(partitions_def: PartitionsDefinition): + @asset(partitions_def=partitions_def) + def asset1(): + return 1 - asset_key = AssetKey("asset1") - asset_graph = AssetGraph.from_assets([asset1]) - asset_job = define_asset_job("asset_job").resolve(asset_graph=asset_graph) - - def _swap_partitions_def(new_partitions_def, asset, asset_graph, asset_job): - asset._partitions_def = new_partitions_def # noqa: SLF001 - asset_graph = AssetGraph.from_assets([asset]) + asset_graph = AssetGraph.from_assets([asset1]) asset_job = define_asset_job("asset_job").resolve(asset_graph=asset_graph) - return asset, asset_job, asset_graph + return asset1, asset_job, asset_graph + + asset1, asset_job, asset_graph = make_asset_job_and_graph(partitions_def) traced_counter.set(Counter()) - asset_records = list(instance.get_asset_records([asset_key])) + asset_records = list(instance.get_asset_records([asset1.key])) assert len(asset_records) == 0 cached_status = get_and_update_asset_status_cache_value( - instance, asset_key, asset_graph.get(asset_key).partitions_def + instance, asset1.key, asset_graph.get(asset1.key).partitions_def ) assert not cached_status asset_job.execute_in_process(instance=instance, partition_key="2022-02-01") cached_status = get_and_update_asset_status_cache_value( - instance, asset_key, asset_graph.get(asset_key).partitions_def + instance, asset1.key, asset_graph.get(asset1.key).partitions_def ) assert cached_status assert cached_status.latest_storage_id @@ -173,7 +166,7 @@ def _swap_partitions_def(new_partitions_def, asset, asset_graph, asset_job): asset_job.execute_in_process(instance=instance, partition_key="2022-02-02") cached_status = get_and_update_asset_status_cache_value( - instance, asset_key, asset_graph.get(asset_key).partitions_def + instance, asset1.key, asset_graph.get(asset1.key).partitions_def ) assert cached_status assert cached_status.latest_storage_id @@ -190,12 +183,10 @@ def _swap_partitions_def(new_partitions_def, asset, asset_graph, asset_job): ) static_partitions_def = StaticPartitionsDefinition(["a", "b", "c"]) - asset1, asset_job, asset_graph = _swap_partitions_def( - static_partitions_def, asset1, asset_graph, asset_job - ) + asset1, asset_job, asset_graph = make_asset_job_and_graph(static_partitions_def) asset_job.execute_in_process(instance=instance, partition_key="a") cached_status = get_and_update_asset_status_cache_value( - instance, asset_key, asset_graph.get(asset_key).partitions_def + instance, asset1.key, asset_graph.get(asset1.key).partitions_def ) assert cached_status assert cached_status.serialized_materialized_partition_subset @@ -795,36 +786,35 @@ def asset1(context): def test_failed_partitioned_asset_converted_to_multipartitioned(self, instance): daily_def = DailyPartitionsDefinition("2023-01-01") - @asset( - partitions_def=daily_def, - ) - def my_asset(): - raise Exception("oops") + def make_asset_job_and_graph(partitions_def: PartitionsDefinition): + @asset(partitions_def=partitions_def) + def my_asset(): + raise Exception("oops") - asset_graph = AssetGraph.from_assets([my_asset]) - my_job = define_asset_job("asset_job", partitions_def=daily_def).resolve( - asset_graph=asset_graph - ) + asset_graph = AssetGraph.from_assets([my_asset]) + asset_job = define_asset_job("asset_job").resolve(asset_graph=asset_graph) + return my_asset, asset_job, asset_graph + + my_asset, my_job, asset_graph = make_asset_job_and_graph(daily_def) my_job.execute_in_process( instance=instance, partition_key="2023-01-01", raise_on_error=False ) - my_asset._partitions_def = MultiPartitionsDefinition( # noqa: SLF001 - partitions_defs={ - "a": DailyPartitionsDefinition("2023-01-01"), - "b": StaticPartitionsDefinition(["a", "b"]), - } + my_asset, my_job, asset_graph = make_asset_job_and_graph( + MultiPartitionsDefinition( + partitions_defs={ + "a": DailyPartitionsDefinition("2023-01-01"), + "b": StaticPartitionsDefinition(["a", "b"]), + } + ) ) - asset_graph = AssetGraph.from_assets([my_asset]) - my_job = define_asset_job("asset_job").resolve(asset_graph=asset_graph) - asset_key = AssetKey("my_asset") cached_status = get_and_update_asset_status_cache_value( - instance, asset_key, asset_graph.get(asset_key).partitions_def + instance, my_asset.key, asset_graph.get(my_asset.key).partitions_def ) failed_subset = cached_status.deserialize_failed_partition_subsets( # pyright: ignore[reportOptionalMemberAccess] - asset_graph.get(asset_key).partitions_def # pyright: ignore[reportArgumentType] + asset_graph.get(my_asset.key).partitions_def # pyright: ignore[reportArgumentType] ) assert failed_subset.get_partition_keys() == set() diff --git a/python_modules/libraries/dagster-components/dagster_components/cli/generate.py b/python_modules/libraries/dagster-components/dagster_components/cli/generate.py index 8405b5ec04e88..4da9481d312b2 100644 --- a/python_modules/libraries/dagster-components/dagster_components/cli/generate.py +++ b/python_modules/libraries/dagster-components/dagster_components/cli/generate.py @@ -1,6 +1,6 @@ import sys from pathlib import Path -from typing import Optional, Tuple +from typing import Optional import click from pydantic import TypeAdapter @@ -23,14 +23,12 @@ def generate_cli() -> None: @click.argument("component_type", type=str) @click.argument("component_name", type=str) @click.option("--json-params", type=str, default=None) -@click.argument("extra_args", nargs=-1, type=str) @click.pass_context def generate_component_command( ctx: click.Context, component_type: str, component_name: str, json_params: Optional[str], - extra_args: Tuple[str, ...], ) -> None: builtin_component_lib = ctx.obj.get(CLI_BUILTIN_COMPONENT_LIB_KEY, False) if not is_inside_code_location_project(Path.cwd()): @@ -52,18 +50,11 @@ def generate_component_command( sys.exit(1) component_type_cls = context.get_component_type(component_type) - generate_params_schema = component_type_cls.generate_params_schema - generate_params_cli = getattr(generate_params_schema, "cli", None) - if generate_params_schema is None: - generate_params = None - elif json_params is not None: + if json_params: + generate_params_schema = component_type_cls.generate_params_schema generate_params = TypeAdapter(generate_params_schema).validate_json(json_params) - elif generate_params_cli is not None: - inner_ctx = click.Context(generate_params_cli) - generate_params_cli.parse_args(inner_ctx, list(extra_args)) - generate_params = inner_ctx.invoke(generate_params_schema.cli, **inner_ctx.params) else: - generate_params = None + generate_params = {} generate_component_instance( context.component_instances_root_path, diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component.py b/python_modules/libraries/dagster-components/dagster_components/core/component.py index dc8ca5e0dab20..dc1d77f6dcf72 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component.py @@ -22,6 +22,7 @@ TypeVar, ) +import click from dagster import _check as check from dagster._core.definitions.definitions_class import Definitions from dagster._core.errors import DagsterError @@ -48,6 +49,10 @@ class Component(ABC): params_schema: ClassVar = None generate_params_schema: ClassVar = None + @classmethod + def get_rendering_scope(cls) -> Mapping[str, Any]: + return {} + @classmethod def generate_files(cls, request: ComponentGenerateRequest, params: Any) -> None: ... @@ -87,6 +92,16 @@ def _clean_docstring(docstring: str) -> str: return f"{first_line}\n{rest}" +def _get_click_cli_help(command: click.Command) -> str: + with click.Context(command) as ctx: + formatter = click.formatting.HelpFormatter() + param_records = [ + p.get_help_record(ctx) for p in command.get_params(ctx) if p.name != "help" + ] + formatter.write_dl([pr for pr in param_records if pr]) + return formatter.getvalue() + + class ComponentInternalMetadata(TypedDict): summary: Optional[str] description: Optional[str] @@ -222,6 +237,12 @@ def path(self) -> Path: return self.decl_node.path + def with_rendering_scope(self, rendering_scope: Mapping[str, Any]) -> "ComponentLoadContext": + return dataclasses.replace( + self, + templated_value_resolver=self.templated_value_resolver.with_context(**rendering_scope), + ) + def for_decl_node(self, decl_node: ComponentDeclNode) -> "ComponentLoadContext": return dataclasses.replace(self, decl_node=decl_node) diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py b/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py index 04a1ce93ed03a..6021344d9cf39 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py @@ -42,6 +42,7 @@ def load_module_from_path(module_name, path) -> ModuleType: def load_components_from_context(context: ComponentLoadContext) -> Sequence[Component]: if isinstance(context.decl_node, YamlComponentDecl): component_type = component_type_from_yaml_decl(context.registry, context.decl_node) + context = context.with_rendering_scope(component_type.get_rendering_scope()) return [component_type.load(context)] elif isinstance(context.decl_node, ComponentFolder): components = [] diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component_rendering.py b/python_modules/libraries/dagster-components/dagster_components/core/component_rendering.py index 02b47e71be6d2..b766edb09186d 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component_rendering.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component_rendering.py @@ -4,7 +4,7 @@ import dagster._check as check from dagster._record import record -from jinja2 import Template +from jinja2.nativetypes import NativeTemplate from pydantic import BaseModel, Field from pydantic.fields import FieldInfo @@ -51,8 +51,8 @@ def default() -> "TemplatedValueResolver": def with_context(self, **additional_context) -> "TemplatedValueResolver": return TemplatedValueResolver(context={**self.context, **additional_context}) - def resolve(self, val: str) -> str: - return Template(val).render(**self.context) + def resolve(self, val: Any) -> Any: + return NativeTemplate(val).render(**self.context) if isinstance(val, str) else val def _should_render( @@ -70,7 +70,7 @@ def _should_render( # Optional[ComplexType] (e.g.) will contain multiple schemas in the "anyOf" field if "anyOf" in subschema: - return any(_should_render(valpath, json_schema, inner) for inner in subschema["anyOf"]) + return all(_should_render(valpath, json_schema, inner) for inner in subschema["anyOf"]) el = valpath[0] if isinstance(el, str): @@ -84,9 +84,9 @@ def _should_render( else: check.failed(f"Unexpected valpath element: {el}") - # the path wasn't valid + # the path wasn't valid, or unspecified if not inner: - return False + return subschema.get("additionalProperties", True) _, *rest = valpath return _should_render(rest, json_schema, inner) diff --git a/python_modules/libraries/dagster-components/dagster_components/generate.py b/python_modules/libraries/dagster-components/dagster_components/generate.py index 5f6e9a369bf83..56318d59e98c0 100644 --- a/python_modules/libraries/dagster-components/dagster_components/generate.py +++ b/python_modules/libraries/dagster-components/dagster_components/generate.py @@ -32,7 +32,7 @@ def generate_component_instance( name: str, component_type: Type[Component], component_type_name: str, - generate_params: Any, + generate_params: Mapping[str, Any], ) -> None: component_instance_root_path = Path(os.path.join(root_path, name)) click.echo(f"Creating a Dagster component instance folder at {component_instance_root_path}.") diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/dbt_project.py b/python_modules/libraries/dagster-components/dagster_components/lib/dbt_project.py index 02f8ca82273c4..4222ef7ab9f89 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/dbt_project.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/dbt_project.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Any, Iterator, Mapping, Optional, Sequence -import click import dagster._check as check from dagster._core.definitions.asset_key import AssetKey from dagster._core.definitions.definitions_class import Definitions @@ -42,13 +41,6 @@ class DbtGenerateParams(BaseModel): init: bool = Field(default=False) project_path: Optional[str] = None - @staticmethod - @click.command - @click.option("--project-path", "-p", type=click.Path(resolve_path=True), default=None) - @click.option("--init", "-i", is_flag=True, default=False) - def cli(project_path: Optional[str], init: bool) -> "DbtGenerateParams": - return DbtGenerateParams(project_path=project_path, init=init) - class DbtProjectComponentTranslator(DagsterDbtTranslator): def __init__( diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/__init__.py b/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/custom_scope_component/component.py b/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/custom_scope_component/component.py new file mode 100644 index 0000000000000..55075ba403c82 --- /dev/null +++ b/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/custom_scope_component/component.py @@ -0,0 +1,42 @@ +from typing import Any, Mapping + +from dagster import AssetSpec, AutomationCondition, Definitions +from dagster_components import Component, ComponentLoadContext, component +from pydantic import BaseModel + + +def my_custom_fn(a: str, b: str) -> str: + return a + "|" + b + + +def my_custom_automation_condition(cron_schedule: str) -> AutomationCondition: + return AutomationCondition.cron_tick_passed(cron_schedule) & ~AutomationCondition.in_progress() + + +class CustomScopeParams(BaseModel): + attributes: Mapping[str, Any] + + +@component(name="custom_scope_component") +class HasCustomScope(Component): + params_schema = CustomScopeParams + + @classmethod + def get_rendering_scope(cls) -> Mapping[str, Any]: + return { + "custom_str": "xyz", + "custom_dict": {"a": "b"}, + "custom_fn": my_custom_fn, + "custom_automation_condition": my_custom_automation_condition, + } + + def __init__(self, attributes: Mapping[str, Any]): + self.attributes = attributes + + @classmethod + def load(cls, context: ComponentLoadContext): + loaded_params = context.load_params(cls.params_schema) + return cls(attributes=loaded_params.attributes) + + def build_defs(self, context: ComponentLoadContext): + return Definitions(assets=[AssetSpec(key="key", **self.attributes)]) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/custom_scope_component/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/custom_scope_component/component.yaml new file mode 100644 index 0000000000000..a23aca62cf535 --- /dev/null +++ b/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/custom_scope_component/component.yaml @@ -0,0 +1,9 @@ +type: .custom_scope_component + +params: + attributes: + group_name: "{{ custom_str }}" + tags: "{{ custom_dict }}" + metadata: + prefixed: "prefixed_{{ custom_fn('a', custom_str) }}" + automation_condition: "{{ custom_automation_condition('@daily') }}" diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_component_rendering.py b/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/test_component_rendering.py similarity index 98% rename from python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_component_rendering.py rename to python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/test_component_rendering.py index 6e036492363c1..e93fbd4a9dbe6 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_component_rendering.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/test_component_rendering.py @@ -41,7 +41,6 @@ class Outer(BaseModel): (["inner_optional", 0, "deferred"], False), (["inner_deferred_optional", 0], False), (["inner_deferred_optional", 0, "a"], False), - (["NONEXIST", 0, "deferred"], False), ], ) def test_should_render(path, expected: bool) -> None: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/test_custom_scope.py b/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/test_custom_scope.py new file mode 100644 index 0000000000000..8a8af764d332e --- /dev/null +++ b/python_modules/libraries/dagster-components/dagster_components_tests/rendering_tests/test_custom_scope.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from dagster import AssetSpec, AutomationCondition +from dagster_components.core.component_defs_builder import build_defs_from_component_path + +from dagster_components_tests.utils import registry + + +def test_custom_scope() -> None: + defs = build_defs_from_component_path( + path=Path(__file__).parent / "custom_scope_component", + registry=registry(), + resources={}, + ) + + assets = list(defs.assets or []) + assert len(assets) == 1 + spec = assets[0] + assert isinstance(spec, AssetSpec) + + assert spec.group_name == "xyz" + assert spec.tags == {"a": "b"} + assert spec.metadata == {"prefixed": "prefixed_a|xyz"} + assert ( + spec.automation_condition + == AutomationCondition.cron_tick_passed("@daily") & ~AutomationCondition.in_progress() + ) diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/asset_specs.py b/python_modules/libraries/dagster-dbt/dagster_dbt/asset_specs.py index 8c80ff498c353..0f3ed7eca908c 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/asset_specs.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/asset_specs.py @@ -62,6 +62,7 @@ def build_dbt_asset_specs( key=check.inst(asset_out.key, AssetKey), deps=[AssetDep(asset=dep) for dep in internal_asset_deps.get(output_name, set())], additional_tags={f"{KIND_PREFIX}dbt": ""}, + partitions_def=None, ) # Allow specs to be represented as external assets by adhering to external asset invariants. ._replace( diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py index d8cd4041ec23c..de007a61d537e 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py @@ -7,6 +7,7 @@ from dagster_dg.cli.info import info_cli from dagster_dg.cli.list import list_cli from dagster_dg.config import DgConfig, set_config_on_cli_context +from dagster_dg.utils import DgClickGroup from dagster_dg.version import __version__ @@ -22,6 +23,7 @@ def create_dg_cli(): commands=commands, context_settings={"max_content_width": 120, "help_option_names": ["-h", "--help"]}, invoke_without_command=True, + cls=DgClickGroup, ) @click.option( "--builtin-component-lib", diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/generate.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/generate.py index e8111606e41a5..e3812213cdc59 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/generate.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/generate.py @@ -1,10 +1,12 @@ import os import sys from pathlib import Path -from typing import Optional, Tuple +from typing import Any, Mapping, Optional import click +from click.core import ParameterSource +from dagster_dg.component import RemoteComponentType from dagster_dg.context import ( CodeLocationDirectoryContext, DeploymentDirectoryContext, @@ -18,14 +20,20 @@ generate_component_type, generate_deployment, ) +from dagster_dg.utils import ( + DgClickCommand, + DgClickGroup, + json_schema_property_to_click_option, + parse_json_option, +) -@click.group(name="generate") +@click.group(name="generate", cls=DgClickGroup) def generate_cli() -> None: """Commands for generating Dagster components and related entities.""" -@generate_cli.command(name="deployment") +@generate_cli.command(name="deployment", cls=DgClickCommand) @click.argument("path", type=Path) def generate_deployment_command(path: Path) -> None: """Generate a Dagster deployment file structure. @@ -43,7 +51,7 @@ def generate_deployment_command(path: Path) -> None: generate_deployment(path) -@generate_cli.command(name="code-location") +@generate_cli.command(name="code-location", cls=DgClickCommand) @click.argument("name", type=str) @click.option( "--use-editable-dagster", @@ -112,7 +120,7 @@ def generate_code_location_command( generate_code_location(code_location_path, editable_dagster_root) -@generate_cli.command(name="component-type") +@generate_cli.command(name="component-type", cls=DgClickCommand) @click.argument("name", type=str) @click.pass_context def generate_component_type_command(cli_context: click.Context, name: str) -> None: @@ -138,82 +146,150 @@ def generate_component_type_command(cli_context: click.Context, name: str) -> No generate_component_type(context, name) -@generate_cli.command(name="component") -@click.argument( - "component_type", - type=str, -) -@click.argument("component_name", type=str) -@click.option("--json-params", type=str, default=None, help="JSON string of component parameters.") -@click.argument("extra_args", nargs=-1, type=str) -@click.pass_context -def generate_component_command( - cli_context: click.Context, - component_type: str, - component_name: str, - json_params: Optional[str], - extra_args: Tuple[str, ...], -) -> None: - """Generate a scaffold of a Dagster component. +# The `dg generate component` command is special because its subcommands are dynamically generated +# from the registered component types in the code location. Because the registered component types +# depend on the built-in component library we are using, we cannot resolve them until we have the +# built-in component library, which can be set via a global option, e.g.: +# +# dg --builtin-component-lib dagster_components.test ... +# +# To handle this, we define a custom click.Group subclass that loads the commands on demand. +class GenerateComponentGroup(DgClickGroup): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._commands_defined = False + + def get_command(self, cli_context: click.Context, cmd_name: str) -> Optional[click.Command]: + if not self._commands_defined: + self._define_commands(cli_context) + return super().get_command(cli_context, cmd_name) + + def list_commands(self, cli_context): + if not self._commands_defined: + self._define_commands(cli_context) + return super().list_commands(cli_context) + + def _define_commands(self, cli_context: click.Context) -> None: + """Dynamically define a command for each registered component type.""" + app_context = DgContext.from_cli_context(cli_context) + + if not is_inside_code_location_directory(Path.cwd()): + click.echo( + click.style( + "This command must be run inside a Dagster code location directory.", fg="red" + ) + ) + sys.exit(1) - This command must be run inside a Dagster code location directory. The component scaffold will be - generated in submodule `.components.`. + context = CodeLocationDirectoryContext.from_path(Path.cwd(), app_context) + for key, component_type in context.iter_component_types(): + command = _create_generate_component_subcommand(key, component_type) + self.add_command(command) - The COMPONENT_TYPE must be a registered component type in the code location environment. - You can view all registered component types with `dg list component-types`. The COMPONENT_NAME - will be used to name the submodule created under .components. - Components can optionally be passed generate parameters. There are two ways to do this: +@generate_cli.group(name="component", cls=GenerateComponentGroup) +def generate_component_group() -> None: + """Generate a scaffold of a Dagster component.""" - - Passing --json-params with a JSON string of parameters. For example: - dg generate component foo.bar my_component --json-params '{"param1": "value", "param2": "value"}'`. +def _create_generate_component_subcommand( + component_key: str, component_type: RemoteComponentType +) -> DgClickCommand: + @click.command(name=component_key, cls=DgClickCommand) + @click.argument("component_name", type=str) + @click.option( + "--json-params", + type=str, + default=None, + help="JSON string of component parameters.", + callback=parse_json_option, + ) + @click.pass_context + def generate_component_command( + cli_context: click.Context, + component_name: str, + json_params: Mapping[str, Any], + **key_value_params: Any, + ) -> None: + f"""Generate a scaffold of a {component_type.name} component. - - Passing key-value pairs as space-separated EXTRA_ARGS after `--`. For example: + This command must be run inside a Dagster code location directory. The component scaffold will be + generated in submodule `.components.`. - dg generate component foo.bar my_component -- param1=value param2=value + Components can optionally be passed generate parameters. There are two ways to do this: - When key-value pairs are used, the value type will be inferred from the - underlying component generation schema. + (1) Passing a single --json-params option with a JSON string of parameters. For example: - It is an error to pass both --json-params and EXTRA_ARGS. - """ - dg_context = DgContext.from_cli_context(cli_context) - if not is_inside_code_location_directory(Path.cwd()): - click.echo( - click.style( - "This command must be run inside a Dagster code location directory.", fg="red" + dg generate component foo.bar my_component --json-params '{{"param1": "value", "param2": "value"}}'`. + + (2) Passing each parameter as an option. For example: + + dg generate component foo.bar my_component --param1 value1 --param2 value2` + + It is an error to pass both --json-params and key-value pairs as options. + """ + dg_context = DgContext.from_cli_context(cli_context) + if not is_inside_code_location_directory(Path.cwd()): + click.echo( + click.style( + "This command must be run inside a Dagster code location directory.", fg="red" + ) ) - ) - sys.exit(1) + sys.exit(1) - context = CodeLocationDirectoryContext.from_path(Path.cwd(), dg_context) - if not context.has_component_type(component_type): - click.echo( - click.style(f"No component type `{component_type}` could be resolved.", fg="red") - ) - sys.exit(1) - elif context.has_component_instance(component_name): - click.echo( - click.style(f"A component instance named `{component_name}` already exists.", fg="red") - ) - sys.exit(1) + context = CodeLocationDirectoryContext.from_path(Path.cwd(), dg_context) + if not context.has_component_type(component_key): + click.echo( + click.style(f"No component type `{component_key}` could be resolved.", fg="red") + ) + sys.exit(1) + elif context.has_component_instance(component_name): + click.echo( + click.style( + f"A component instance named `{component_name}` already exists.", fg="red" + ) + ) + sys.exit(1) - if json_params is not None and extra_args: - click.echo( - click.style( - "Detected both --json-params and EXTRA_ARGS. These are mutually exclusive means of passing" - " component generation parameters. Use only one.", - fg="red", + # Specified key-value params will be passed to this function with their default value of + # `None` even if the user did not set them. Filter down to just the ones that were set by + # the user. + user_provided_key_value_params = { + k: v + for k, v in key_value_params.items() + if cli_context.get_parameter_source(k) == ParameterSource.COMMANDLINE + } + if json_params is not None and user_provided_key_value_params: + click.echo( + click.style( + "Detected params passed as both --json-params and individual options. These are mutually exclusive means of passing" + " component generation parameters. Use only one.", + fg="red", + ) ) + sys.exit(1) + elif json_params: + generate_params = json_params + elif user_provided_key_value_params: + generate_params = user_provided_key_value_params + else: + generate_params = None + + generate_component_instance( + Path(context.component_instances_root_path), + component_name, + component_key, + generate_params, + dg_context, ) - sys.exit(1) - generate_component_instance( - Path(context.component_instances_root_path), - component_name, - component_type, - json_params, - extra_args, - dg_context, - ) + # If there are defined generate params, add them to the command + schema = component_type.generate_params_schema + if schema: + for key, field_info in schema["properties"].items(): + # All fields are currently optional because they can also be passed under + # `--json-params` + option = json_schema_property_to_click_option(key, field_info, required=False) + generate_component_command.params.append(option) + + return generate_component_command diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/info.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/info.py index f509963606b3d..4152567a74611 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/info.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/info.py @@ -10,9 +10,10 @@ DgContext, is_inside_code_location_directory, ) +from dagster_dg.utils import DgClickCommand, DgClickGroup -@click.group(name="info") +@click.group(name="info", cls=DgClickGroup) def info_cli(): """Commands for listing Dagster components and related entities.""" @@ -21,7 +22,7 @@ def _serialize_json_schema(schema: Mapping[str, Any]) -> str: return json.dumps(schema, indent=4) -@info_cli.command(name="component-type") +@info_cli.command(name="component-type", cls=DgClickCommand) @click.argument("component_type", type=str) @click.option("--description", is_flag=True, default=False) @click.option("--generate-params-schema", is_flag=True, default=False) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/list.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/list.py index 055c5a827079f..fa8679f3dc728 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/list.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/list.py @@ -10,14 +10,15 @@ is_inside_code_location_directory, is_inside_deployment_directory, ) +from dagster_dg.utils import DgClickCommand, DgClickGroup -@click.group(name="list") +@click.group(name="list", cls=DgClickGroup) def list_cli(): """Commands for listing Dagster components and related entities.""" -@list_cli.command(name="code-locations") +@list_cli.command(name="code-locations", cls=DgClickCommand) @click.pass_context def list_code_locations_command(cli_context: click.Context) -> None: """List code locations in the current deployment.""" @@ -33,7 +34,7 @@ def list_code_locations_command(cli_context: click.Context) -> None: click.echo(code_location) -@list_cli.command(name="component-types") +@list_cli.command(name="component-types", cls=DgClickCommand) @click.pass_context def list_component_types_command(cli_context: click.Context) -> None: """List registered Dagster components in the current code location environment.""" @@ -53,7 +54,7 @@ def list_component_types_command(cli_context: click.Context) -> None: click.echo(f" {component_type.summary}") -@list_cli.command(name="components") +@list_cli.command(name="components", cls=DgClickCommand) @click.pass_context def list_components_command(cli_context: click.Context) -> None: """List Dagster component instances defined in the current code location.""" diff --git a/python_modules/libraries/dagster-dg/dagster_dg/generate.py b/python_modules/libraries/dagster-dg/dagster_dg/generate.py index 46a198f56f7a7..788c9c898dd8c 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/generate.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/generate.py @@ -1,8 +1,9 @@ +import json import os import subprocess import textwrap from pathlib import Path -from typing import Optional, Tuple +from typing import Any, Mapping, Optional import click @@ -148,8 +149,7 @@ def generate_component_instance( root_path: Path, name: str, component_type: str, - json_params: Optional[str], - extra_args: Tuple[str, ...], + generate_params: Optional[Mapping[str, Any]], dg_context: "DgContext", ) -> None: component_instance_root_path = root_path / name @@ -160,8 +160,7 @@ def generate_component_instance( "component", component_type, name, - *(["--json-params", json_params] if json_params else []), - *(["--", *extra_args] if extra_args else []), + *(["--json-params", json.dumps(generate_params)] if generate_params else []), ) execute_code_location_command( Path(component_instance_root_path), diff --git a/python_modules/libraries/dagster-dg/dagster_dg/utils.py b/python_modules/libraries/dagster-dg/dagster_dg/utils.py index 57be0cc34e53f..8bcf29db6b277 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/utils.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/utils.py @@ -1,4 +1,5 @@ import contextlib +import json import os import posixpath import re @@ -6,12 +7,26 @@ import sys from fnmatch import fnmatch from pathlib import Path -from typing import TYPE_CHECKING, Any, Final, Iterator, List, Mapping, Optional, Sequence, Union +from typing import ( + TYPE_CHECKING, + Any, + Final, + Iterator, + List, + Mapping, + Optional, + Sequence, + Tuple, + TypeVar, + Union, +) import click import jinja2 +from click.formatting import HelpFormatter from typing_extensions import TypeAlias +from dagster_dg.error import DgError from dagster_dg.version import __version__ as dagster_version # There is some weirdness concerning the availabilty of hashlib.HASH between different Python @@ -211,3 +226,151 @@ def hash_file_metadata(hasher: Hash, path: Union[str, Path]) -> None: hasher.update(str(path).encode()) hasher.update(str(stat.st_mtime).encode()) # Last modified time hasher.update(str(stat.st_size).encode()) # File size + + +T = TypeVar("T") + + +def not_none(value: Optional[T]) -> T: + if value is None: + raise DgError("Expected non-none value.") + return value + + +# ######################## +# ##### CUSTOM CLICK SUBCLASSES +# ######################## + +# Here we subclass click.Command and click.Group to customize the help output. We do this in order +# to show the options for each parent group in the help output of a subcommand. The form of the +# output can be seen in dagster_dg_tests.test_custom_help_format. + +# When rendering options for parent groups, exclude these options since they are not used when +# executing a subcommand. +_EXCLUDE_PARENT_OPTIONS = ["help", "version"] + + +class DgClickHelpMixin: + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self._commands: List[str] = [] + + def format_help(self, context: click.Context, formatter: click.HelpFormatter): + """Customizes the help to include hierarchical usage.""" + if not isinstance(self, click.Command): + raise ValueError("This mixin is only intended for use with click.Command instances.") + self.format_usage(context, formatter) + self.format_help_text(context, formatter) + if isinstance(self, click.MultiCommand): + self.format_commands(context, formatter) + self.format_options(context, formatter) + + # Add section for each parent option group + for ctx, cmd in self._walk_parents(context): + cmd.format_options(ctx, formatter, as_parent=True) + + def format_options( + self, ctx: click.Context, formatter: HelpFormatter, as_parent: bool = False + ) -> None: + """Writes all the options into the formatter if they exist. + + If `as_parent` is True, the header will include the command path and the `--help` option + will be excluded. + """ + if not isinstance(self, click.Command): + raise ValueError("This mixin is only intended for use with click.Command instances.") + + params = [ + p + for p in self.get_params(ctx) + if p.name and not (as_parent and p.name in _EXCLUDE_PARENT_OPTIONS) + ] + opts = [rv for p in params if (rv := p.get_help_record(ctx)) is not None] + if as_parent: + opts = [opt for opt in opts if not opt[0].startswith("--help")] + if opts: + header = f"Options ({ctx.command_path})" if as_parent else "Options" + with formatter.section(header): + formatter.write_dl(opts) + + def format_usage(self, context: click.Context, formatter: HelpFormatter) -> None: + if not isinstance(self, click.Command): + raise ValueError("This mixin is only intended for use with click.Command instances.") + arg_pieces = self.collect_usage_pieces(context) + + path_parts: List[str] = [not_none(context.info_name)] + for ctx, cmd in self._walk_parents(context): + if cmd.has_visible_options_as_parent(ctx): + path_parts.append("[OPTIONS]") + path_parts.append(not_none(ctx.info_name)) + path_parts.reverse() + return formatter.write_usage(" ".join(path_parts), " ".join(arg_pieces)) + + def has_visible_options_as_parent(self, ctx: click.Context) -> bool: + """Returns True if the command has options that are not help-related.""" + if not isinstance(self, click.Command): + raise ValueError("This mixin is only intended for use with click.Command instances.") + return any( + p for p in self.get_params(ctx) if (p.name and p.name not in _EXCLUDE_PARENT_OPTIONS) + ) + + def _walk_parents( + self, ctx: click.Context + ) -> Iterator[Tuple[click.Context, "DgClickHelpMixin"]]: + while ctx.parent: + if not isinstance(ctx.parent.command, DgClickHelpMixin): + raise DgError("Parent command must be an instance of DgClickHelpMixin.") + yield ctx.parent, ctx.parent.command + ctx = ctx.parent + + +class DgClickCommand(DgClickHelpMixin, click.Command): + pass + + +class DgClickGroup(DgClickHelpMixin, click.Group): + pass + + +# ######################## +# ##### JSON SCHEMA +# ######################## + +_JSON_SCHEMA_TYPE_TO_CLICK_TYPE = {"string": str, "integer": int, "number": float, "boolean": bool} + + +def json_schema_property_to_click_option( + key: str, field_info: Mapping[str, Any], required: bool +) -> click.Option: + field_type = field_info.get("type", "string") + option_name = f"--{key.replace('_', '-')}" + + # Handle object type fields as JSON strings + if field_type == "object": + option_type = str # JSON string input + help_text = f"{key} (JSON string)" + callback = parse_json_option + + # Handle other basic types + else: + option_type = _JSON_SCHEMA_TYPE_TO_CLICK_TYPE[field_type] + help_text = key + callback = None + + return click.Option( + [option_name], + type=option_type, + required=required, + help=help_text, + callback=callback, + ) + + +def parse_json_option(context: click.Context, param: click.Option, value: str): + """Callback to parse JSON string options into Python objects.""" + if value: + try: + return json.loads(value) + except json.JSONDecodeError: + raise click.BadParameter(f"Invalid JSON string for '{param.name}'.") + return value diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py new file mode 100644 index 0000000000000..7819f4935d5a0 --- /dev/null +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py @@ -0,0 +1,222 @@ +import textwrap + +import click +from click.testing import CliRunner +from dagster_dg.utils import DgClickCommand, DgClickGroup, ensure_dagster_dg_tests_import + +ensure_dagster_dg_tests_import() + +from dagster_dg_tests.utils import assert_runner_result + +# ######################## +# ##### TEST CLI +# ######################## + + +@click.group(name="test", cls=DgClickGroup) +@click.option("--test-opt", type=str, default="test", help="Test option.") +def test_cli(test_opt): + """Test CLI group.""" + pass + + +@test_cli.group(name="sub-test-1", cls=DgClickGroup) +@click.option("--sub-test-1-opt", type=str, default="sub-test-1", help="Sub-test 1 option.") +def sub_test_1(sub_test_1_opt): + """Sub-test 1 group.""" + pass + + +@sub_test_1.command(name="alpha", cls=DgClickCommand) +@click.option("--alpha-opt", type=str, default="alpha", help="Alpha option.") +def alpha(alpha_opt): + """Alpha command.""" + pass + + +@test_cli.group(name="sub-test-2", cls=DgClickGroup) +def sub_test_2(): + """Sub-test 2 group.""" + pass + + +@click.option("--beta-opt", type=str, default="alpha", help="Beta option.") +@sub_test_2.command(name="beta", cls=DgClickCommand) +def beta(beta_opt): + """Beta command.""" + pass + + +@click.option("--delta-opt", type=str, default="delta", help="Delta option.") +@test_cli.command(name="delta", cls=DgClickCommand) +def delta(delta_opt): + """Delta command.""" + pass + + +@test_cli.command(name="gamma", cls=DgClickCommand) +def gamma(gamma_opt): + """Gamma command.""" + pass + + +# ######################## +# ##### TESTS +# ######################## + + +def test_root_group_help_message(): + runner = CliRunner() + result = runner.invoke(test_cli, ["--help"]) + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + Usage: test [OPTIONS] COMMAND [ARGS]... + + Test CLI group. + + Commands: + delta Delta command. + gamma Gamma command. + sub-test-1 Sub-test 1 group. + sub-test-2 Sub-test 2 group. + + Options: + --test-opt TEXT Test option. + --help Show this message and exit. + """).strip() + ) + + +def test_sub_group_with_option_help_message(): + runner = CliRunner() + result = runner.invoke(test_cli, ["sub-test-1", "--help"]) + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + Usage: test [OPTIONS] sub-test-1 [OPTIONS] COMMAND [ARGS]... + + Sub-test 1 group. + + Commands: + alpha Alpha command. + + Options: + --sub-test-1-opt TEXT Sub-test 1 option. + --help Show this message and exit. + + Options (test): + --test-opt TEXT Test option. + """).strip() + ) + + +def test_command_in_sub_group_with_option_help_message(): + runner = CliRunner() + result = runner.invoke(test_cli, ["sub-test-1", "alpha", "--help"]) + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + Usage: test [OPTIONS] sub-test-1 [OPTIONS] alpha [OPTIONS] + + Alpha command. + + Options: + --alpha-opt TEXT Alpha option. + --help Show this message and exit. + + Options (test sub-test-1): + --sub-test-1-opt TEXT Sub-test 1 option. + + Options (test): + --test-opt TEXT Test option. + """).strip() + ) + + +def test_sub_group_with_no_option_help_message(): + runner = CliRunner() + result = runner.invoke(test_cli, ["sub-test-2", "--help"]) + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + Usage: test [OPTIONS] sub-test-2 [OPTIONS] COMMAND [ARGS]... + + Sub-test 2 group. + + Commands: + beta Beta command. + + Options: + --help Show this message and exit. + + Options (test): + --test-opt TEXT Test option. + """).strip() + ) + + +def test_command_in_sub_group_with_no_option_help_message(): + runner = CliRunner() + result = runner.invoke(test_cli, ["sub-test-2", "beta", "--help"]) + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + Usage: test [OPTIONS] sub-test-2 beta [OPTIONS] + + Beta command. + + Options: + --beta-opt TEXT Beta option. + --help Show this message and exit. + + Options (test): + --test-opt TEXT Test option. + """).strip() + ) + + +def test_command_with_option_in_root_group_help_message(): + runner = CliRunner() + result = runner.invoke(test_cli, ["delta", "--help"]) + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + Usage: test [OPTIONS] delta [OPTIONS] + + Delta command. + + Options: + --delta-opt TEXT Delta option. + --help Show this message and exit. + + Options (test): + --test-opt TEXT Test option. + """).strip() + ) + + +def test_command_with_no_option_in_root_group_help_message(): + runner = CliRunner() + result = runner.invoke(test_cli, ["gamma", "--help"]) + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + Usage: test [OPTIONS] gamma [OPTIONS] + + Gamma command. + + Options: + --help Show this message and exit. + + Options (test): + --test-opt TEXT Test option. + """).strip() + ) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_generate_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_generate_commands.py index 7db275b8b0b92..7bc7ebc634b0b 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_generate_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_generate_commands.py @@ -1,6 +1,7 @@ import json import os import subprocess +import textwrap from pathlib import Path import pytest @@ -153,6 +154,21 @@ def test_generate_component_type_already_exists_fails(in_deployment: bool) -> No assert "already exists" in result.output +def test_generate_component_dynamic_subcommand_generation() -> None: + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): + result = runner.invoke("generate", "component", "--help") + assert_runner_result(result) + assert ( + textwrap.dedent(""" + Commands: + dagster_components.test.all_metadata_empty_asset + dagster_components.test.simple_asset + dagster_components.test.simple_pipes_script_asset + """).strip() + in result.output + ) + + @pytest.mark.parametrize("in_deployment", [True, False]) def test_generate_component_no_params_success(in_deployment: bool) -> None: with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): @@ -195,14 +211,13 @@ def test_generate_component_json_params_success(in_deployment: bool) -> None: @pytest.mark.parametrize("in_deployment", [True, False]) -def test_generate_component_extra_args_success(in_deployment: bool) -> None: +def test_generate_component_key_value_params_success(in_deployment: bool) -> None: with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): result = runner.invoke( "generate", "component", "dagster_components.test.simple_pipes_script_asset", "qux", - "--", "--asset-key=foo", "--filename=hello.py", ) @@ -217,7 +232,7 @@ def test_generate_component_extra_args_success(in_deployment: bool) -> None: ) -def test_generate_component_json_params_and_extra_args_fails() -> None: +def test_generate_component_json_params_and_key_value_params_fails() -> None: with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): result = runner.invoke( "generate", @@ -226,11 +241,12 @@ def test_generate_component_json_params_and_extra_args_fails() -> None: "qux", "--json-params", '{"filename": "hello.py"}', - "--", "--filename=hello.py", ) assert_runner_result(result, exit_0=False) - assert "Detected both --json-params and EXTRA_ARGS" in result.output + assert ( + "Detected params passed as both --json-params and individual options" in result.output + ) def test_generate_component_outside_code_location_fails() -> None: @@ -297,7 +313,7 @@ def test_generate_sling_replication_instance() -> None: "params", [ ["--json-params", json.dumps({"project_path": str(dbt_project_path)})], - ["--", "--project-path", dbt_project_path], + ["--project-path", dbt_project_path], ], ) def test_generate_dbt_project_instance(params) -> None: diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_integrity.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_integrity.py new file mode 100644 index 0000000000000..5588c3dec1c10 --- /dev/null +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_integrity.py @@ -0,0 +1,16 @@ +from dagster_dg.cli import cli +from dagster_dg.utils import DgClickCommand, DgClickGroup + + +# Important that all nodes of the command tree inherit from one of our customized click +# Command/Group subclasses to ensure that the help formatting is consistent. +def test_all_commands_custom_subclass(): + def crawl(command): + assert isinstance( + command, (DgClickGroup, DgClickCommand) + ), f"Group is not a DgClickGroup or DgClickCommand: {command}" + if isinstance(command, DgClickGroup): + for command in command.commands.values(): + crawl(command) + + crawl(cli)
    {}} /> - +