From fd2612223fbdf305f63db251aae5c55c401a235d Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Fri, 13 Dec 2024 15:46:44 -0500 Subject: [PATCH 1/3] [docs-beta] create format directives via prettier --- docs/docs-beta/.prettierignore | 4 ++-- docs/docs-beta/package.json | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/docs-beta/.prettierignore b/docs/docs-beta/.prettierignore index 93ca2364b8c27..49d82661ba5e9 100644 --- a/docs/docs-beta/.prettierignore +++ b/docs/docs-beta/.prettierignore @@ -1,4 +1,4 @@ -node_modules .docusaurus +.yarn/ build/ -docs/ \ No newline at end of file +node_modules diff --git a/docs/docs-beta/package.json b/docs/docs-beta/package.json index 59e159c6c878d..2a5384bb259db 100644 --- a/docs/docs-beta/package.json +++ b/docs/docs-beta/package.json @@ -16,7 +16,9 @@ "vale": "vale ./docs --ext=.md,.mdx", "lint": "eslint . --ext=.tsx,.ts,.js,.md,.mdx --fix", "lint-and-vale": "yarn run lint && yarn run vale", - "sync-api-docs": "/bin/sh scripts/vercel-sync-api-docs.sh" + "sync-api-docs": "/bin/sh scripts/vercel-sync-api-docs.sh", + "format": "prettier --write .", + "format:diff": "prettier --list-different ." }, "dependencies": { "@docusaurus/core": "3.5.2", From 8e3008fbba5a65f2f8b290d848c7a64b6ddaa9f6 Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Fri, 13 Dec 2024 17:04:51 -0500 Subject: [PATCH 2/3] yarn format --- docs/docs-beta/docs/about/community.md | 10 +- docs/docs-beta/docs/api/api-lifecycle.md | 23 +- docs/docs-beta/docs/changelog.md | 2 +- .../code-locations/code-location-history.md | 2 +- .../code-locations/dagster-cloud-yaml.md | 2 +- .../deployment/code-locations/index.md | 149 ++++---- .../deployment/code-requirements.md | 4 +- .../amazon-ecs/configuration-reference.md | 2 +- .../hybrid/amazon-ecs/existing-vpc.md | 2 +- .../hybrid/amazon-ecs/index.md | 2 +- .../hybrid/amazon-ecs/manual-provision.md | 2 +- .../hybrid/amazon-ecs/new-vpc.md | 2 +- .../amazon-ecs/upgrading-cloudformation.md | 2 +- .../hybrid/docker/configuration.md | 2 +- .../deployment-types/hybrid/docker/index.md | 2 +- .../deployment-types/hybrid/docker/setup.md | 2 +- .../deployment-types/hybrid/index.md | 14 +- .../hybrid/kubernetes/configuration.md | 2 +- .../hybrid/kubernetes/index.md | 2 +- .../hybrid/kubernetes/setup.md | 49 ++- .../deployment-types/hybrid/local.md | 19 +- .../deployment-types/hybrid/multiple.md | 2 +- .../deployment/deployment-types/index.md | 2 +- .../serverless/run-isolation.md | 1 + .../serverless/runtime-environment.md | 110 ++++-- .../dagster-cloud-cli-reference.md | 2 +- .../management/dagster-cloud-cli/index.md | 2 +- .../installing-and-configuring.md | 3 +- .../environment-variables/agent-config.md | 6 +- .../environment-variables/built-in.md | 33 +- .../environment-variables/dagster-ui.md | 29 +- .../deployment/management/index.md | 2 +- ...anaging-compute-logs-and-error-messages.md | 2 +- .../managing-multiple-projects-and-teams.md | 2 +- .../deployment/management/rate-limits.md | 2 +- .../settings/customizing-agent-settings.md | 2 +- .../settings/dagster-plus-settings.md | 3 +- .../settings/deployment-settings.md | 4 +- .../settings/hybrid-agent-settings.md | 5 +- .../deployment/management/settings/index.md | 2 +- .../management/tokens/agent-tokens.md | 2 +- .../deployment/management/tokens/index.md | 2 +- .../management/tokens/user-tokens.md | 2 +- .../deployment/migration/index.md | 2 +- ...nfiguring-an-alert-notification-service.md | 5 +- .../features/alerts/creating-alerts.md | 354 ++++++++++++------ .../index.md | 4 +- .../rbac/audit-logs.md | 52 +-- .../rbac/index.md | 4 +- .../rbac/teams.md | 15 +- .../rbac/user-roles-permissions.md | 86 ++--- .../rbac/users.md | 8 +- .../scim/enabling-scim-provisioning.md | 2 +- .../scim/index.md | 6 +- .../scim/okta-scim.md | 2 +- .../sso/authentication.md | 3 +- .../sso/azure-ad-sso.md | 2 + .../sso/google-workspace-sso.md | 46 +-- .../sso/index.md | 4 +- .../sso/okta-sso.md | 8 +- .../sso/onelogin-sso.md | 9 +- .../dagster-plus/features/catalog-views.md | 6 +- .../branch-deployments/change-tracking.md | 4 +- .../branch-deployments/dagster-cloud-cli.md | 2 +- .../ci-cd/branch-deployments/index.md | 2 +- .../setting-up-branch-deployments.md | 75 ++-- .../ci-cd/branch-deployments/testing.md | 2 +- .../features/ci-cd/ci-cd-file-reference.md | 2 +- .../docs/dagster-plus/features/ci-cd/index.md | 4 +- .../dagster-plus/features/data-catalog.md | 3 +- .../features/insights/asset-metadata.md | 8 +- .../features/insights/export-metrics.md | 5 +- .../features/insights/google-bigquery.md | 34 +- .../features/insights/snowflake.md | 24 +- .../docs/dagster-plus/getting-started.md | 8 +- docs/docs-beta/docs/dagster-plus/index.md | 2 +- .../docs/getting-started/glossary.md | 16 +- .../docs/getting-started/installation.md | 18 +- .../docs/getting-started/quickstart.md | 23 +- .../docs/guides/automate/asset-sensors.md | 2 +- .../guides/automate/declarative-automation.md | 4 +- docs/docs-beta/docs/guides/automate/index.md | 11 +- .../docs/guides/automate/schedules.md | 5 +- .../docs-beta/docs/guides/automate/sensors.md | 2 +- .../guides/deploy/code-locations/index.mdx | 2 +- .../code-locations/managing-code-locations.md | 2 +- .../deploy/code-locations/workspace-yaml.md | 21 +- .../docs/guides/deploy/dagster-yaml.md | 6 +- .../deployment-options/amazon-web-services.md | 2 +- .../deploy/deployment-options/docker.md | 47 ++- .../google-cloud-platform.md | 2 +- .../deploy/deployment-options/index.mdx | 4 +- .../deploy/deployment-options/kubernetes.md | 36 +- .../deployment-options/microsoft-azure.md | 2 +- .../running-local-ui-development.md | 11 +- .../guides/deploy/execution/dagster-daemon.md | 2 +- .../docs/guides/deploy/execution/index.mdx | 4 +- .../deploy/execution/run-coordinators.md | 2 +- .../guides/deploy/execution/run-executors.md | 2 +- .../guides/deploy/execution/run-launchers.md | 2 +- docs/docs-beta/docs/guides/deploy/index.md | 2 +- docs/docs-beta/docs/guides/index.md | 2 +- .../docs-beta/docs/guides/monitor/alerting.md | 2 +- docs/docs-beta/docs/guides/monitor/index.md | 4 +- .../guides/monitor/logging/custom-logging.md | 24 +- .../monitor/logging/custom-metrics-logs.md | 2 +- .../docs/guides/monitor/logging/index.mdx | 4 +- .../docs/guides/test/asset-checks.md | 30 +- .../guides/test/data-freshness-testing.md | 22 +- docs/docs-beta/docs/guides/test/index.md | 4 +- .../guides/test/unit-tests-assets-and-ops.md | 68 +++- .../guides/multi-asset-integration.md | 2 +- .../docs/integrations/libraries/airbyte.md | 2 - .../docs/integrations/libraries/aws/athena.md | 2 - .../integrations/libraries/aws/cloudwatch.md | 2 - .../docs/integrations/libraries/aws/ecr.md | 2 - .../docs/integrations/libraries/aws/emr.md | 2 - .../docs/integrations/libraries/aws/glue.md | 2 - .../docs/integrations/libraries/aws/index.md | 2 +- .../docs/integrations/libraries/aws/lambda.md | 2 - .../integrations/libraries/aws/redshift.md | 2 - .../docs/integrations/libraries/aws/s3.md | 2 - .../libraries/aws/secretsmanager.md | 2 - .../docs/integrations/libraries/aws/ssm.md | 2 - .../integrations/libraries/azure-adls2.md | 2 - .../docs/integrations/libraries/census.md | 2 - .../docs/integrations/libraries/cube.md | 2 - .../docs/integrations/libraries/databricks.md | 2 - .../docs/integrations/libraries/datadog.md | 2 - .../docs/integrations/libraries/dbt-cloud.md | 2 - .../docs/integrations/libraries/dbt.md | 2 - .../docs/integrations/libraries/deltalake.md | 2 - .../docs/integrations/libraries/dlt.md | 2 - .../docs/integrations/libraries/docker.md | 2 - .../docs/integrations/libraries/duckdb.md | 2 - .../docs/integrations/libraries/fivetran.md | 2 - .../integrations/libraries/gcp/bigquery.md | 2 - .../integrations/libraries/gcp/dataproc.md | 2 - .../docs/integrations/libraries/gcp/gcs.md | 2 - .../docs/integrations/libraries/github.md | 2 - .../docs/integrations/libraries/hashicorp.md | 2 - .../docs/integrations/libraries/hightouch.md | 2 - .../docs/integrations/libraries/index.md | 2 +- .../docs/integrations/libraries/kubernetes.md | 2 - .../docs/integrations/libraries/lakefs.md | 2 - .../docs/integrations/libraries/looker.md | 2 - .../docs/integrations/libraries/meltano.md | 2 - .../integrations/libraries/open-metadata.md | 2 - .../docs/integrations/libraries/openai.md | 2 - .../docs/integrations/libraries/pagerduty.md | 2 - .../docs/integrations/libraries/pandas.md | 2 - .../docs/integrations/libraries/pandera.md | 2 - .../docs/integrations/libraries/prometheus.md | 2 - .../docs/integrations/libraries/sdf.md | 2 - .../docs/integrations/libraries/secoda.md | 2 - .../docs/integrations/libraries/shell.md | 2 - .../docs/integrations/libraries/slack.md | 2 - .../docs/integrations/libraries/sling.md | 2 - .../docs/integrations/libraries/snowflake.md | 2 - .../docs/integrations/libraries/spark.md | 2 - .../docs/integrations/libraries/ssh-sftp.md | 2 - .../docs/integrations/libraries/twilio.md | 2 - .../docs/integrations/libraries/wandb.md | 2 - docs/docs-beta/docs/intro.md | 42 ++- docs/docs-beta/docs/partials/_Experimental.md | 2 +- docs/docs-beta/docs/partials/_TestSSO.md | 2 +- docs/docs-beta/docs/tutorial/introduction.md | 4 +- docs/docs-beta/docusaurus.config.ts | 2 +- docs/docs-beta/sidebars.ts | 49 +-- docs/docs-beta/src/styles/custom.scss | 47 ++- docs/docs-beta/src/theme/MDXComponents.tsx | 2 +- 171 files changed, 1075 insertions(+), 880 deletions(-) diff --git a/docs/docs-beta/docs/about/community.md b/docs/docs-beta/docs/about/community.md index 830fd0f01dc44..0bd1d08c2c709 100644 --- a/docs/docs-beta/docs/about/community.md +++ b/docs/docs-beta/docs/about/community.md @@ -76,11 +76,11 @@ Instances of abusive, harassing, or otherwise unacceptable behavior may be repor The Dagster core team will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: -| Level | Community impact | Consequence | -|---|----|----| -| Reminder | Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. | A private, written reminder from the Dagster core team, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. | -| Warning | A violation through a single incident or series of actions. | A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces and external channels like social media. Violating these terms will lead to a permanent ban. | -| Permanent ban | Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. | A permanent ban from any sort of public interaction within the community. | +| Level | Community impact | Consequence | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Reminder | Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. | A private, written reminder from the Dagster core team, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. | +| Warning | A violation through a single incident or series of actions. | A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces and external channels like social media. Violating these terms will lead to a permanent ban. | +| Permanent ban | Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. | A permanent ban from any sort of public interaction within the community. | ### Attribution diff --git a/docs/docs-beta/docs/api/api-lifecycle.md b/docs/docs-beta/docs/api/api-lifecycle.md index 5b91b22fe3d39..352008d652637 100644 --- a/docs/docs-beta/docs/api/api-lifecycle.md +++ b/docs/docs-beta/docs/api/api-lifecycle.md @@ -10,49 +10,54 @@ The lifecycle stages described below provide a clear framework for understanding ## API lifecycle stages -| Stage | Description | Lifetime | -|---------|------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| -| Preview | API in the design phase, can change significantly, or be removed completely. Not for production use. | Until design is complete, or implementation cancelled | -| Beta | Features that is still being tested and may change. More stable than Preview, but still subject to change. | At most, two 1.x releases before it is either considered stable or returned to preview | -| Generally Available (GA)| Ready for production use, with minimal risk of breaking changes. | Supported until at least 2.0 -| Superseded | This API is still available, but is no longer the best practice. A better alternative is available. | Supported until at least 2.0 -| Deprecated | API is still available but will be removed in the future; avoid new usage. | Will be removed in a minor release, the DeprecationWarning will indicate the next release that will remove the API. - +| Stage | Description | Lifetime | +| ------------------------ | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| Preview | API in the design phase, can change significantly, or be removed completely. Not for production use. | Until design is complete, or implementation cancelled | +| Beta | Features that is still being tested and may change. More stable than Preview, but still subject to change. | At most, two 1.x releases before it is either considered stable or returned to preview | +| Generally Available (GA) | Ready for production use, with minimal risk of breaking changes. | Supported until at least 2.0 | +| Superseded | This API is still available, but is no longer the best practice. A better alternative is available. | Supported until at least 2.0 | +| Deprecated | API is still available but will be removed in the future; avoid new usage. | Will be removed in a minor release, the DeprecationWarning will indicate the next release that will remove the API. | ## Understanding the stages ### Preview + - **Purpose**: For early testing and feedback - **Stability**: Highly unstable, expect frequent changes - **Usage**: Not recommended for production environments - **Documentation**: Minimal, typically just a README or unlisted documentation ### Beta + - **Purpose**: Feature testing with wider audience - **Stability**: More stable than Preview, but still subject to changes - **Usage**: Can be used in non-critical production environments - **Documentation**: How-to guides and API documentation available ### GA (General Availability) + - **Purpose**: Production-ready features - **Stability**: Stable with minimal risk of breaking changes - **Usage**: Recommended for all production environments - **Documentation**: Comprehensive documentation available ### Superseded + - **Purpose**: Maintains backwards compatibility while promoting newer alternatives - **Stability**: Stable but no longer recommended - **Usage**: Existing implementations can continue, but new projects should use the recommended alternative - **Documentation**: API docs remain, but usage is discouraged in favor of newer alternatives ### Deprecated + - **Purpose**: Signals upcoming removal of the API - **Stability**: Stable but scheduled for removal - **Usage**: Existing implementations should plan migration - **Documentation**: API docs remain, with clear warnings about deprecation. Arguments may be removed from function signature ### Dead + - **Purpose**: Removed APIs - **Stability**: N/A - **Usage**: No longer available for use -- **Documentation**: Removed from all documentation \ No newline at end of file +- **Documentation**: Removed from all documentation diff --git a/docs/docs-beta/docs/changelog.md b/docs/docs-beta/docs/changelog.md index 94f067f740a34..b2be5394a665f 100644 --- a/docs/docs-beta/docs/changelog.md +++ b/docs/docs-beta/docs/changelog.md @@ -1,5 +1,5 @@ --- -title: "Changelog" +title: 'Changelog' unlisted: true --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/code-location-history.md b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/code-location-history.md index 33c575030c05a..b454b4b70e29d 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/code-location-history.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/code-location-history.md @@ -1,5 +1,5 @@ --- -title: "Code location history and rollbacks" +title: 'Code location history and rollbacks' sidebar_position: 100 --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/dagster-cloud-yaml.md b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/dagster-cloud-yaml.md index 7790f2cf5230a..a9fda07bf97d0 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/dagster-cloud-yaml.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/dagster-cloud-yaml.md @@ -4,4 +4,4 @@ sidebar_position: 200 unlisted: true --- -{/* TODO move content from https://docs.dagster.io/dagster-plus/managing-deployments/dagster-cloud-yaml */} \ No newline at end of file +{/** TODO move content from https://docs.dagster.io/dagster-plus/managing-deployments/dagster-cloud-yaml **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/index.md b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/index.md index facde90edf2aa..91e4b66eed08d 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/code-locations/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/code-locations/index.md @@ -1,33 +1,34 @@ --- -title: "Code locations" +title: 'Code locations' sidebar_position: 40 --- Separate code locations allow you to deploy different projects that still roll up into a single Dagster+ deployment with one global lineage graph. This guide will cover three options for adding a new code location: + - Adding a code location manually - Adding a code location in a new Git repository - Adding a new code location to an existing Git monorepo
-Prerequisites + Prerequisites 1. An existing Dagster project. Refer to the [recommended project structure](/tutorial/create-new-project) and [code requirements](/dagster-plus/deployment/code-requirements) pages for more information. 2. Editor, Admin, or Organization Admin permissions in Dagster+. -3. To know your Dagster+ deployment type. An administrator can help find this information, or you can locate it by clicking *Deployment > Agents tab* in the Dagster UI. Dagster+ Serverless organizations will have a *Managed by Dagster+* label next to the agent. +3. To know your Dagster+ deployment type. An administrator can help find this information, or you can locate it by clicking _Deployment > Agents tab_ in the Dagster UI. Dagster+ Serverless organizations will have a _Managed by Dagster+_ label next to the agent.
Adding a code location follows two steps: + - For Dagster+ Hybrid, ensuring the Dagster code is in a place accessible by your agent, usually by building a Docker image with your code that's pushed to a registry. For Dagster+ Serverless you can skip this step. - Notifying Dagster+ of the new or updated code location. This will be done by using the Dagster+ Python client. Often these two steps are handled by CI/CD connected to your Git repository. - ## Add a new code location manually Start by installing the `dagster-cloud` Python client: @@ -62,6 +63,7 @@ Now add the code location. The following example assumes you are running the com ``` The commands below take two main arguments: + - `module_name` is determined by your code structure - `location_name` is the unique label for this code location used in Dagster+ @@ -74,7 +76,8 @@ If you are using Dagster+ Serverless, run the following command to add a code lo dagster-cloud serverless deploy-python-executable --deployment prod --location-name quickstart --module-name quickstart ``` -Running the command multiple times with the same location name will *update* the code location. Running the command with a new location name will *add* a code location. +Running the command multiple times with the same location name will _update_ the code location. Running the command with a new location name will _add_ a code location. + @@ -83,10 +86,11 @@ If you are using Dagster+ Hybrid, make sure you have deployed the code appropria ```bash dagster-cloud deployment add-location --deployment prod --location-name quickstart --module-name quickstart --image 764506304434.dkr.ecr.us-west-2.amazonaws.com/hooli-data-science-prod:latest ``` + -After running the command you can verify the code location was deployed by navigating to the *Deployments* tab on Dagster+. +After running the command you can verify the code location was deployed by navigating to the _Deployments_ tab on Dagster+. ## Adding a code location in a new Git repository @@ -99,82 +103,81 @@ To get started, review the appropriate example repository and then create your G - [GitLab CI/CD for Dagster+ Serverless](https://github.com/dagster-io/dagster-cloud-action/blob/main/gitlab/serverless-ci.yml) - [GitLab CI/CD for Dagster+ Hybrid](https://github.com/dagster-io/dagster-cloud-action/blob/main/gitlab/hybrid-ci.yml) - Overall, the Git repository should contain: 1. Your Dagster code, structured as a Python module. For Dagter+ Hybrid you may need a [Dockerfile](https://github.com/dagster-io/dagster-cloud-hybrid-quickstart/blob/main/Dockerfile) as well. The repository might look like this: - ```bash - README.md - dagster_cloud.yaml - Dockerfile - /.github - /workflows - dagster-cloud-deploy.yml - setup.py - pyproject.toml - /quickstart - __init__.py - definitions.py - /assets - ... - /resources - ... - ``` + ```bash + README.md + dagster_cloud.yaml + Dockerfile + /.github + /workflows + dagster-cloud-deploy.yml + setup.py + pyproject.toml + /quickstart + __init__.py + definitions.py + /assets + ... + /resources + ... + ``` 2. A [`dagster_cloud.yaml` file](/todo) with the settings for your code location. Here is an example: - ```yaml title="dagster_cloud.yaml - locations: - - location_name: quickstart - code_source: - package_name: quickstart - ``` + ```yaml title="dagster_cloud.yaml + locations: + - location_name: quickstart + code_source: + package_name: quickstart + ``` 3. A CI/CD workflow file that contains the steps for adding your code location. These are the same steps outlined in the preceding section. Here is a minimal example workflow file for a Dagster+ Hybrid organization based on [this GitLab template](https://github.com/dagster-io/dagster-cloud-action/blob/main/gitlab/hybrid-ci.yml). - ```yaml - variables: - DAGSTER_CLOUD_ORGANIZATION: - DAGSTER_PROJECT_DIR: . - IMAGE_REGISTRY: .dkr.ecr.us-west-2.amazonaws.com/ - IMAGE_TAG: $CI_COMMIT_SHORT_SHA-$CI_PIPELINE_ID - - stages: - - build - - deploy + ```yaml + variables: + DAGSTER_CLOUD_ORGANIZATION: + DAGSTER_PROJECT_DIR: . + IMAGE_REGISTRY: .dkr.ecr.us-west-2.amazonaws.com/ + IMAGE_TAG: $CI_COMMIT_SHORT_SHA-$CI_PIPELINE_ID + + stages: + - build + - deploy + + build: + stage: build + image: docker:latest + services: + - docker:dind + before_script: + # # For Gitlab Container Registry + # - echo $CI_JOB_TOKEN | docker login --username $CI_REGISTRY_USER --password-stdin $REGISTRY_URL + # # For DockerHub + # - echo $DOCKERHUB_TOKEN | docker login --username $DOCKERHUB_USERNAME --password-stdin $REGISTRY_URL + # # For AWS Elastic Container Registry (ECR) + # - apk add --no-cache curl jq python3 py3-pip + # - pip install awscli + # - echo $AWS_ECR_PASSWORD | docker login --username AWS --password-stdin $IMAGE_REGISTRY + # # For Google Container Registry (GCR) + # - echo $GCR_JSON_KEY | docker login --username _json_key --password-stdin $REGISTRY_URL + script: + - docker build . -t $IMAGE_REGISTRY:$IMAGE_TAG + - docker push $IMAGE_REGISTRY:$IMAGE_TAG + + deploy: + stage: deploy + dependencies: + - build + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.43 + script: + - dagster-cloud deployment add-location --deployment prod --image + $IMAGE_REGISTRY:$IMAGE_TAG --location-name quickstart --package-name quickstart + ``` - build: - stage: build - image: docker:latest - services: - - docker:dind - before_script: - # # For Gitlab Container Registry - # - echo $CI_JOB_TOKEN | docker login --username $CI_REGISTRY_USER --password-stdin $REGISTRY_URL - # # For DockerHub - # - echo $DOCKERHUB_TOKEN | docker login --username $DOCKERHUB_USERNAME --password-stdin $REGISTRY_URL - # # For AWS Elastic Container Registry (ECR) - # - apk add --no-cache curl jq python3 py3-pip - # - pip install awscli - # - echo $AWS_ECR_PASSWORD | docker login --username AWS --password-stdin $IMAGE_REGISTRY - # # For Google Container Registry (GCR) - # - echo $GCR_JSON_KEY | docker login --username _json_key --password-stdin $REGISTRY_URL - script: - - docker build . -t $IMAGE_REGISTRY:$IMAGE_TAG - - docker push $IMAGE_REGISTRY:$IMAGE_TAG - - deploy: - stage: deploy - dependencies: - - build - image: ghcr.io/dagster-io/dagster-cloud-action:0.1.43 - script: - - dagster-cloud deployment add-location --deployment prod --image - $IMAGE_REGISTRY:$IMAGE_TAG --location-name quickstart --package-name quickstart - ``` - -Once your Git repository has this structure, you will want to run your CI/CD process. The CI/CD process will add the code location to Dagster+ which can be verified by viewing the *Deployments* tab. +Once your Git repository has this structure, you will want to run your CI/CD process. The CI/CD process will add the code location to Dagster+ which can be verified by viewing the _Deployments_ tab. ## Adding a new code location to a Git monorepo @@ -220,7 +223,7 @@ locations: package_name: core build: directory: ./core - registry: your-registry/image # eg 764506304434.dkr.ecr.us-west-2.amazonaws.com/core + registry: your-registry/image # eg 764506304434.dkr.ecr.us-west-2.amazonaws.com/core - location_name: new code_source: package_name: new @@ -229,7 +232,7 @@ locations: registry: your-registry/image # eg 764506304434.dkr.ecr.us-west-2.amazonaws.com/new ``` -The monorepo should have CI/CD configured to deploy your changes and add or update your new code location. After adding your code and updating the `dagster_cloud.yaml` file, trigger the CI/CD process to add your code location to Dagster+. Navigate to the *Deployments* tab in Dagster+ to confirm your code location was added. +The monorepo should have CI/CD configured to deploy your changes and add or update your new code location. After adding your code and updating the `dagster_cloud.yaml` file, trigger the CI/CD process to add your code location to Dagster+. Navigate to the _Deployments_ tab in Dagster+ to confirm your code location was added. ## Next steps diff --git a/docs/docs-beta/docs/dagster-plus/deployment/code-requirements.md b/docs/docs-beta/docs/dagster-plus/deployment/code-requirements.md index ad75ee2411dd0..9f3f814215700 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/code-requirements.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/code-requirements.md @@ -1,6 +1,6 @@ --- title: 'Dagster+ code requirements' -sidebar_label: "Code requirements" +sidebar_label: 'Code requirements' sidebar_position: 10 --- @@ -12,6 +12,7 @@ Your Dagster project must meet a few requirements to run in Dagster+. To follow the steps in this guide, you'll need: - A basic understanding of Python project structure and Docker + ## General requirements @@ -42,4 +43,3 @@ If you're using [Hybrid Deployment](/dagster-plus/deployment/deployment-types/hy - **If using a local agent**, your code must be in a Python environment that can be accessed on the same machine as your agent. Additionally, your code doesn't need to use the same version of Dagster as your agent. - diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/configuration-reference.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/configuration-reference.md index 62d9f00dcf50d..97e34051bbf07 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/configuration-reference.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/configuration-reference.md @@ -4,4 +4,4 @@ sidebar_position: 400 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/configuration-reference */} +{/** TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/configuration-reference **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/existing-vpc.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/existing-vpc.md index edddddfbded2f..6793537ca240f 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/existing-vpc.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/existing-vpc.md @@ -4,4 +4,4 @@ sidebar_position: 200 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/creating-ecs-agent-existing-vpc */} \ No newline at end of file +{/** TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/creating-ecs-agent-existing-vpc **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/index.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/index.md index ade5c5b935ffa..c18e972b652cb 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/index.md @@ -5,4 +5,4 @@ sidebar_position: 50 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/manual-provision.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/manual-provision.md index 8604054fa0824..336be5a58e818 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/manual-provision.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/manual-provision.md @@ -4,4 +4,4 @@ sidebar_position: 300 unlisted: true --- -{/* TODO move content from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/manually-provisioning-ecs-agent */} \ No newline at end of file +{/** TODO move content from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/manually-provisioning-ecs-agent **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/new-vpc.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/new-vpc.md index d68622e6d7eb0..8a9bdd5c4758b 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/new-vpc.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/new-vpc.md @@ -4,4 +4,4 @@ sidebar_position: 100 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/creating-ecs-agent-new-vpc */} \ No newline at end of file +{/** TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/creating-ecs-agent-new-vpc **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/upgrading-cloudformation.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/upgrading-cloudformation.md index ee612a4ba34d4..5e73e2f529228 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/upgrading-cloudformation.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/upgrading-cloudformation.md @@ -5,4 +5,4 @@ sidebar_position: 500 unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/upgrading-cloudformation-template */} \ No newline at end of file +{/** TODO move from https://docs.dagster.io/dagster-plus/deployment/agents/amazon-ecs/upgrading-cloudformation-template **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/configuration.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/configuration.md index 1294b30313cfb..8d75b1e654e7c 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/configuration.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/configuration.md @@ -4,4 +4,4 @@ sidebar_position: 200 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/docker/configuration-reference */} \ No newline at end of file +{/** TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/docker/configuration-reference **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/index.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/index.md index cb00c69f21296..45b3d38431382 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/index.md @@ -5,4 +5,4 @@ sidebar_position: 30 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/setup.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/setup.md index f16e0dc8e25a9..d8682c6ba16c3 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/setup.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/docker/setup.md @@ -4,4 +4,4 @@ sidebar_position: 100 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/docker/configuring-running-docker-agent */} \ No newline at end of file +{/** TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/docker/configuring-running-docker-agent **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/index.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/index.md index 3d96293d0c923..cd0a8c302265e 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/index.md @@ -1,5 +1,5 @@ --- -title: "Hybrid deployment" +title: 'Hybrid deployment' sidebar_label: Hybrid sidebar_position: 20 --- @@ -21,11 +21,11 @@ To get started with a Hybrid deployment you'll need to: The Dagster+ agent is a long-lived process that polls Dagster+'s API servers for new work. See the following guides for setting up an agent: - - [Kubernetes](/dagster-plus/deployment/deployment-types/hybrid/kubernetes) - - [AWS ECS](/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/new-vpc) - - [Docker](/dagster-plus/deployment/deployment-types/hybrid/docker) - - [Locally](/dagster-plus/deployment/deployment-types/hybrid/local) +- [Kubernetes](/dagster-plus/deployment/deployment-types/hybrid/kubernetes) +- [AWS ECS](/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/new-vpc) +- [Docker](/dagster-plus/deployment/deployment-types/hybrid/docker) +- [Locally](/dagster-plus/deployment/deployment-types/hybrid/local) ## What you'll see in your environment @@ -35,7 +35,6 @@ Dagster+ runs your Dagster projects through code locations. To get started, foll When you inform Dagster+ about a new code location, we enqueue instructions for your agent to launch a new code server. The agent uses your container image to launch a code server that interacts with your Dagster definitions. The agent will run one long-standing code server for each code location. Once the code server is running, the agent will send Dagster+ metadata about your Dagster definitions that Dagster+ uses to make orchestration decisions. - ### Runs Your definitions might include [automations](/guides/automate) that launch runs or materialize assets. Or your developers might launch runs directly with the web UI. @@ -49,15 +48,18 @@ Your agent will send Dagster+ metadata letting us know the run has been launched Dagster+ hybrid relies on a shared security model. The Dagster+ control plane is SOC 2 Type II certified and follows best practices such as: + - encrypting data at rest (AES 256) and in transit (TLS 1.2+) - highly available, with disaster recovery and backup strategies - only manages metadata such as pipeline names, execution status, and run duration The execution environment is managed by the customer: + - your code never leaves your environment - all connections to databases, file systems, and other resources are made from your environment - the execution environment only requires egress access to Dagster+ Common security considerations in Dagster+ hybrid include: + - [disabling log forwarding](/todo) - [managing tokens](/todo) diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/configuration.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/configuration.md index f3053801d3c1c..98e9ebd9385f1 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/configuration.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/configuration.md @@ -4,4 +4,4 @@ sidebar_position: 200 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/kubernetes/configuration-reference */} \ No newline at end of file +{/** TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/kubernetes/configuration-reference **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/index.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/index.md index d744a447c3b87..12fda0c54e321 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/index.md @@ -5,4 +5,4 @@ sidebar_position: 40 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/setup.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/setup.md index 619f5ffa72326..94a50395aa59c 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/setup.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/kubernetes/setup.md @@ -7,7 +7,6 @@ This page provides instructions for running the [Dagster+ agent](/todo) on a [Ku ## Installation - ### Prerequisites You'll need a Kubernetes cluster. This can be a self-hosted Kubernetes cluster or a managed offering like [Amazon EKS](https://aws.amazon.com/eks/), [Azure AKS](https://azure.microsoft.com/en-us/products/kubernetes-service), or [Google GKE](https://cloud.google.com/kubernetes-engine). @@ -54,8 +53,8 @@ We recommend upgrading your Dagster+ agent every 6 months. The version of your a ```yaml # values.yaml dagsterCloudAgent: - image: - tag: latest + image: + tag: latest ``` ```shell @@ -76,10 +75,10 @@ You can see basic health information about your agent in the Dagster+ UI: kubectl --namespace dagster-cloud logs -l deployment=agent ``` - ## Common configurations There are three places to customize how Dagster interacts with Kubernetes: + - **Per Deployment** by configuring the Dagster+ agent using [Helm values](https://artifacthub.io/packages/helm/dagster-cloud/dagster-cloud-agent?modal=values) - **Per Project** by configuring the `dagster_cloud.yaml` file for your [code location](/dagster-plus/deployment/code-locations) - **Per Asset or Job** by adding tags to the [asset](/todo), [job](/todo), or [customizing the Kubernetes pipes invocation](/todo) @@ -88,7 +87,6 @@ Changes apply in a hierarchy, for example, a customization for an asset will ove An exhaustive list of settings is available [here](/dagster-plus/deployment/management/settings/hybrid-agent-settings), but common options are presented below. - ### Configure your agents to serve branch deployments [Branch deployments](/dagster-plus/features/ci-cd/branch-deployments/index.md) are lightweight staging environments created for each code change. To configure your Dagster+ agent to manage them: @@ -96,7 +94,7 @@ An exhaustive list of settings is available [here](/dagster-plus/deployment/mana ```yaml # values.yaml dagsterCloud: - branchDeployment: true + branchDeployment: true ``` ```shell @@ -112,7 +110,7 @@ You can configure your Dagster+ agent to run with multiple replicas. Work will b ```yaml # values.yaml dagsterCloudAgent: - replicas: 2 + replicas: 2 ``` ```shell @@ -138,12 +136,10 @@ helm --namespace dagster-cloud upgrade agent \ The agent is responsible for managing the lifecycle of your code locations and will typically need to pull images after your CI/CD process builds them and pushes them to your registry. You can specify a secret the agent will use to authenticate to your image registry. - :::tip For cloud-based Kubernetes deployments such as AWS EKS, AKS, or GCP, you don't need an image pull secret. The role used by Kubernetes will have permission to access the registry, so you can skip this configuration. ::: - First create the secret. This step will vary based on the registry you use, but for DockerHub: ``` @@ -213,7 +209,6 @@ helm --namespace dagster-cloud upgrade agent \ --values ./values.yaml ``` - @@ -227,7 +222,6 @@ kubectl create secret generic database-password-kubernetes-secret \ --namespace dagster-plus ``` - Next, determine if the secret should be available to all code locations or a single code location. @@ -268,15 +262,13 @@ location: `env_secrets` will make the secret available in an environment variable, see the Kubernetes docs on [`envFrom` for details](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables). In this example the environment variable `DATABASE_PASSWORD` would have the value `your_password`. - :::note -If you need to request secrets from a secret manager like AWS Secrets Manager or HashiCorp Vault, follow one of the prior methods to give your code access to vault credentials. Then, inside your Dagster code, use those credentials to authenticate a Python client that requests secrets from the secret manager. +If you need to request secrets from a secret manager like AWS Secrets Manager or HashiCorp Vault, follow one of the prior methods to give your code access to vault credentials. Then, inside your Dagster code, use those credentials to authenticate a Python client that requests secrets from the secret manager. ::: - ### Use a different service account for a specific code location Modify the [`dagster_cloud.yaml` file](/todo) in your project's Git repository: @@ -320,18 +312,22 @@ You may wish to run data pipelines from project A in Kubernetes cluster A, and p - Deploy an agent into each environment and use the `agentQueue` configuration: Specify an agent queue for the on-prem agent: + ```yaml file=values.yaml dagsterCloud: agentQueues: additionalQueues: - on-prem-agent-queue ``` + Deploy onto the on-prem Kubernetes cluster: + ```shell helm --namespace dagster-cloud upgrade agent \ dagster-cloud/dagster-cloud-agent \ --values ./values.yaml ``` + Specify an agent queue for the agent on AWS: ```yaml file=values.yaml @@ -340,12 +336,15 @@ You may wish to run data pipelines from project A in Kubernetes cluster A, and p additionalQueues: - aws-agent-queue ``` + Deploy onto the AWS Kubernetes cluster: + ```shell helm --namespace dagster-cloud upgrade agent \ dagster-cloud/dagster-cloud-agent \ --values ./values.yaml ``` + - Create separate code locations for each project - Update the `dagster_cloud.yaml` file for each code location ```yaml file=dagster_cloud.yaml @@ -366,6 +365,7 @@ dagsterCloud: agentQueues: includeDefaultQueue: true ``` + ::: @@ -432,11 +432,19 @@ The units for CPU and memory resources are described [in this document](https:// The default behavior in Dagster+ is to create one pod for a run. Each asset targeted by that run is executed in subprocess within the pod. Use a job tag to request resources for this pod, which in turn makes those resources available to the targeted assets. - + Another option is to launch a pod for each asset by telling Dagster to use the Kubernetes job executor. In this case, you can specify resources for each individual asset. - + @@ -444,10 +452,11 @@ Another option is to launch a pod for each asset by telling Dagster to use the K Dagster can launch and manage existing Docker images as Kubernetes jobs using the [Dagster kubernetes pipes integration](/integrations/libraries/kubernetes). To request resources for these jobs by supplying the appropriate Kubernetes pod spec. - - - + - diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/local.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/local.md index 53c43ba09d468..370c98efb3c20 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/local.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/local.md @@ -11,7 +11,7 @@ Local agents aren't well suited for most production use cases. If you're running - You've set up a supervisor to automatically restart the agent process if it crashes - You're alerted if the VM or container dies, or to automatically restart it -::: + :::
Prerequisites @@ -21,9 +21,10 @@ To follow the steps in this guide, you'll need: - **Organization Admin** permissions in your Dagster+ account - **To install the `dagster-cloud` CLI** in the same environment where the agent will run. We recommend using a Python virtual environment for your Dagster application code and its dependencies. - ```bash - pip install dagster-cloud - ``` +```bash +pip install dagster-cloud +``` +
## Step 1: Generate an agent token @@ -41,10 +42,14 @@ Your local agent will need a token to authenticate with your Dagster+ account. T 1. Create a directory to act as your Dagster home. This guide uses `~/dagster_home`, but the directory can be located wherever you want. 2. In the new directory, create a `dagster.yaml` file with the following: - + 3. In the file, fill in the following: - - `agent_token.env` - The name of the environment variable storing the agent token you created in Step 1. - - `deployment` - The name of the deployment associated with this instance of the agent. In the preceding example, `prod` was used as the deployment. + - `agent_token.env` - The name of the environment variable storing the agent token you created in Step 1. + - `deployment` - The name of the deployment associated with this instance of the agent. In the preceding example, `prod` was used as the deployment. 4. Save the file. For more information about `dagster.yaml` configuration options, check out the [`dagster.yaml` reference](/todo). diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/multiple.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/multiple.md index 0df411c9e2b52..3d2f1dbb39229 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/multiple.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/hybrid/multiple.md @@ -5,4 +5,4 @@ sidebar_label: Multiple agents unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/running-multiple-agents */} \ No newline at end of file +{/** TODO copy from https://docs.dagster.io/dagster-plus/deployment/agents/running-multiple-agents **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/index.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/index.md index bd70a65d90ffd..11714e83761d1 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/index.md @@ -5,4 +5,4 @@ sidebar_position: 20 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/run-isolation.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/run-isolation.md index c9c5d04ab6756..9c42efb9bb5fe 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/run-isolation.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/run-isolation.md @@ -13,6 +13,7 @@ To follow the steps in this guide, you'll need: - An active Dagster+ Serverless deployment - An understanding of [Dagster+ deployment settings](/dagster-plus/deployment/management/settings/deployment-settings) + --- diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/runtime-environment.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/runtime-environment.md index 750eefa59919a..a81af91fd6b30 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/runtime-environment.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/runtime-environment.md @@ -17,7 +17,11 @@ By default, Dagster+ Serverless will package your code as PEX files and deploys You can add dependencies by including the corresponding Python libraries in your Dagster project's `setup.py` file. These should follow [PEP 508](https://peps.python.org/pep-0508/). - + You can also use a tarball to install a dependency, such as if `pip` is unable to resolve a package using `dependency_links`. For example, `soda` and `soda-snowflake` provide tarballs that you can include in the `install_requires` section: @@ -59,7 +63,11 @@ In your `.github/workflows/deploy.yml` file, update the `PYTHON_VERSION` environ 3. Update the `PYTHON_VERSION` environment variable with your desired Python version - + @@ -98,23 +106,27 @@ Setting a custom base image isn't supported for GitLab CI/CD workflows out of th ``` 4. Specify this base image tag in you GitHub workflow, or using the `dagster-cloud` CLI: - + + + In your `.github/workflows/deploy.yml` file, add the `SERVERLESS_BASE_IMAGE_TAG` environment variable and set it to the tag printed out in the previous step: - + + + - You can specify the base image when you deploy your code with the `dagster-cloud serverless deploy-python-executable` command: - ```bash - dagster-cloud serverless deploy-python-executable \ - --base-image-tag=sha256_518ad2f92b078c63c60e89f0310f13f19d3a1c7ea9e1976d67d59fcb7040d0d6 \ - --location-name=my_location - ``` + ```bash + dagster-cloud serverless deploy-python-executable \ + --base-image-tag=sha256_518ad2f92b078c63c60e89f0310f13f19d3a1c7ea9e1976d67d59fcb7040d0d6 \ + --location-name=my_location + ``` + + - ## Include data files \{#data-files} @@ -132,50 +144,60 @@ To add data files to your deployment, use the [Data Files Support](https://setup ``` If you want to include the data folder, modify your `setup.py` to add the `package_data` line: - + + ## Disable PEX deploys \{#disable-pex} You have the option to disable PEX-based deploys and deploy using a Docker image instead of PEX. You can disable PEX in your GitHub or GitLab workflow, or by using the `dagster-cloud` CLI. - -In your `.github/workflows/deploy.yml` file, update the `ENABLE_FAST_DEPLOYS` environment variable to `false`: - + + In your `.github/workflows/deploy.yml` file, update the `ENABLE_FAST_DEPLOYS` environment variable to `false`: + - - -1. Open your `.gitlab-ci.yml` file. If your `.gitlab-ci.yml` contains an `include` with a link to a Dagster provided CI/CD template: - + + + 1. Open your `.gitlab-ci.yml` file. If your `.gitlab-ci.yml` contains an `include` with a link to a Dagster provided CI/CD template: + - Follow the link and replace the contents of your `.gitlab-ci.yml` with the YAML document at the link address. Otherwise, continue to the next step. + Follow the link and replace the contents of your `.gitlab-ci.yml` with the YAML document at the link address. Otherwise, continue to the next step. 3. Update the `DISABLE_FAST_DEPLOYS` variable to `true` - + - - -You can deploy using a Docker image instead of PEX by using the `dagster-cloud serverless deploy` command instead of the `dagster-cloud-serverless deploy-python-executable` command: + + + You can deploy using a Docker image instead of PEX by using the `dagster-cloud serverless deploy` command instead of the `dagster-cloud-serverless deploy-python-executable` command: ```bash dagster-cloud serverless deploy --location-name=my_location ``` - + You can customize the Docker image using lifecycle hooks or by customizing the base image: - -This method is the easiest to set up, and doesn't require setting up any additional infrastructure. + + This method is the easiest to set up, and doesn't require setting up any additional infrastructure. In the root of your repo, you can provide two optional shell scripts: `dagster_cloud_pre_install.sh` and `dagster_cloud_post_install.sh`. These will run before and after Python dependencies are installed. They're useful for installing any non-Python dependencies or otherwise configuring your environment. - - -This method is the most flexible, but requires setting up a pipeline outside of Dagster to build a custom base image. + + + + This method is the most flexible, but requires setting up a pipeline outside of Dagster to build a custom base image. :::note Setting a custom base image isn't supported for GitLab CI/CD workflows out of the box, but you can write a custom GitLab CI/CD yaml file that implements the manual steps noted. @@ -183,21 +205,27 @@ Setting a custom base image isn't supported for GitLab CI/CD workflows out of th 1. Build you base image 2. Specify this base image tag in your GitHub workflow, or using the `dagster-cloud` CLI: + + - - In your `.github/workflows/deploy.yml` file, add the `SERVERLESS_BASE_IMAGE_TAG` environment variable and set it to the tag printed out in the previous step: - - - - You can specify the base image when you deploy your code with the `dagster-cloud serverless deploy` command: - ```bash - dagster-cloud serverless deploy --base-image=my_base_image:latest --location-name=my_location - ``` - + + In your `.github/workflows/deploy.yml` file, add the `SERVERLESS_BASE_IMAGE_TAG` environment variable and set it to the tag printed out in the previous step: + + + + + You can specify the base image when you deploy your code with the `dagster-cloud serverless deploy` command: + + ```bash + dagster-cloud serverless deploy --base-image=my_base_image:latest --location-name=my_location + ``` + + - + + ## Use private Python packages \{#private-packages} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/dagster-cloud-cli-reference.md b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/dagster-cloud-cli-reference.md index 90d74f2c112e5..ff4df16ba70db 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/dagster-cloud-cli-reference.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/dagster-cloud-cli-reference.md @@ -4,4 +4,4 @@ sidebar_position: 200 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/managing-deployments/dagster-plus-cli#reference */} \ No newline at end of file +{/** TODO copy from https://docs.dagster.io/dagster-plus/managing-deployments/dagster-plus-cli#reference **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/index.md b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/index.md index 96f6f0d097bd7..6ef4de02bd902 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/index.md @@ -7,4 +7,4 @@ The dagster-cloud CLI is a command-line toolkit designed to work with Dagster+. import DocCardList from '@theme/DocCardList'; - \ No newline at end of file +{' '} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/installing-and-configuring.md b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/installing-and-configuring.md index ff5280c31f89c..a7afe2901dfe6 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/installing-and-configuring.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/installing-and-configuring.md @@ -4,5 +4,4 @@ sidebar_position: 100 unlisted: true --- -{/* TODO copy from "Installing the CLI" and "Configuring the CLI" sections of https://docs.dagster.io/dagster-plus/managing-deployments/dagster-plus-cli */} - +{/** TODO copy from "Installing the CLI" and "Configuring the CLI" sections of https://docs.dagster.io/dagster-plus/managing-deployments/dagster-plus-cli **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.md b/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.md index a800e113a83ba..abbbffd2382ff 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/agent-config.md @@ -1,8 +1,8 @@ --- -title: "Set environment variables using agent config" +title: 'Set environment variables using agent config' sidebar_position: 300 -sidebar_label: "Set with agent config" +sidebar_label: 'Set with agent config' unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/managing-deployments/setting-environment-variables-agents */} +{/** TODO move from https://docs.dagster.io/dagster-plus/managing-deployments/setting-environment-variables-agents **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/built-in.md b/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/built-in.md index 6bf5398f0088e..191d922f41b0f 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/built-in.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/built-in.md @@ -1,7 +1,7 @@ --- -title: "Built-in environment variables" +title: 'Built-in environment variables' sidebar_position: 100 -sidebar_label: "Built-in variables" +sidebar_label: 'Built-in variables' --- [Dagster+](/todo) provides a set of built-in, automatically populated environment variables, such as the name of a deployment or details about a branch deployment commit, that can be used to modify behavior based on environment. @@ -10,11 +10,10 @@ sidebar_label: "Built-in variables" The following variables are available in every deployment of your Dagster+ instance. -| Key | Value | -|---|---| -| `DAGSTER_CLOUD_DEPLOYMENT_NAME` | The name of the Dagster+ deployment.

**Example:** `prod`. | -| `DAGSTER_CLOUD_IS_BRANCH_DEPLOYMENT` | `1` if the deployment is a branch deployment. | - +| Key | Value | +| ------------------------------------ | -------------------------------------------------------------------- | +| `DAGSTER_CLOUD_DEPLOYMENT_NAME` | The name of the Dagster+ deployment.

**Example:** `prod`. | +| `DAGSTER_CLOUD_IS_BRANCH_DEPLOYMENT` | `1` if the deployment is a branch deployment. | ### Branch deployment variables @@ -22,14 +21,14 @@ The following environment variables are available only in a [branch deployment]( For every commit made to a branch, the following environment variables are available: -| Key | Value | -|---|---| -| `DAGSTER_CLOUD_GIT_SHA` | The SHA of the commit. | -| `DAGSTER_CLOUD_GIT_TIMESTAMP` | The Unix timestamp in seconds when the commit occurred.

**Example:** `1724871941` | -| `DAGSTER_CLOUD_GIT_AUTHOR_EMAIL` | The email of the git user who authored the commit. | -| `DAGSTER_CLOUD_GIT_AUTHOR_NAME` | The name of the git user who authored the commit. | -| `DAGSTER_CLOUD_GIT_MESSAGE` | The message associated with the commit. | -| `DAGSTER_CLOUD_GIT_BRANCH` | The name of the branch associated with the commit. | -| `DAGSTER_CLOUD_GIT_REPO` | The name of the repository associated with the commit. | -| `DAGSTER_CLOUD_PULL_REQUEST_ID` | The ID of the pull request associated with the commit. | +| Key | Value | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `DAGSTER_CLOUD_GIT_SHA` | The SHA of the commit. | +| `DAGSTER_CLOUD_GIT_TIMESTAMP` | The Unix timestamp in seconds when the commit occurred.

**Example:** `1724871941` | +| `DAGSTER_CLOUD_GIT_AUTHOR_EMAIL` | The email of the git user who authored the commit. | +| `DAGSTER_CLOUD_GIT_AUTHOR_NAME` | The name of the git user who authored the commit. | +| `DAGSTER_CLOUD_GIT_MESSAGE` | The message associated with the commit. | +| `DAGSTER_CLOUD_GIT_BRANCH` | The name of the branch associated with the commit. | +| `DAGSTER_CLOUD_GIT_REPO` | The name of the repository associated with the commit. | +| `DAGSTER_CLOUD_PULL_REQUEST_ID` | The ID of the pull request associated with the commit. | | `DAGSTER_CLOUD_PULL_REQUEST_STATUS` | The status of the pull request at the time of the commit.

**Possible values:** `OPEN`, `CLOSED`, and `MERGED`. | diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/dagster-ui.md b/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/dagster-ui.md index 705aae2cba831..3369097d0443d 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/dagster-ui.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/environment-variables/dagster-ui.md @@ -1,7 +1,7 @@ --- -title: "Setting environment variables with the Dagster+ UI" +title: 'Setting environment variables with the Dagster+ UI' sidebar_position: 200 -sidebar_label: "Set with Dagster+ UI" +sidebar_label: 'Set with Dagster+ UI' --- Environment variable are key-value pairs that are set outside of your source code. Using environment variables lets you dynamically change the behavior of your application without modifying source code and securely set up secrets. @@ -24,16 +24,15 @@ Before you begin, use the deployment switcher to select the right deployment. 1. Click the **+ Add environment variable** button. 2. In the modal that displays, fill in the following: - - **Name** - Enter a name for the environment variable. This is how the variable will be referenced in your code. - - **Value** - Enter a value for the environment variable. - - **Deployment Scope** - select the deployment(s) where the variable should be accessible: - - **Full deployment** - The variable will be available to selected code locations in the full deployment. - - **Branch deployments** - The variable will be available to selected code locations in Branch Deployments. - - **Local** - If selected, the variable will be included when [exporting environment variables to a local `.env` file](#export). - - **Code Location Scope** - select the code location(s) where the variable should be accessible. At least one code location is required. + - **Name** - Enter a name for the environment variable. This is how the variable will be referenced in your code. + - **Value** - Enter a value for the environment variable. + - **Deployment Scope** - select the deployment(s) where the variable should be accessible: + - **Full deployment** - The variable will be available to selected code locations in the full deployment. + - **Branch deployments** - The variable will be available to selected code locations in Branch Deployments. + - **Local** - If selected, the variable will be included when [exporting environment variables to a local `.env` file](#export). + - **Code Location Scope** - select the code location(s) where the variable should be accessible. At least one code location is required. - -{/* TODO replace placeholder image */} +{/** TODO replace placeholder image **/} ![Screenshot of adding environment variables](/img/placeholder.svg) @@ -78,11 +77,11 @@ When you [add an environment variable](#add), you can select the deployment scop For example, if you wanted to provide different Snowflake passwords for your production and branch deployments, you would make two environment variables with the same key: - For the **production** environment variable: - - Set the value as the production password, and - - Check only the **Full deployment** box + - Set the value as the production password, and + - Check only the **Full deployment** box - For the **branch deployment** environment variable: - - Set the value as the branch deployment password, and - - Check only the **Branch deployments** box + - Set the value as the branch deployment password, and + - Check only the **Branch deployments** box ![Screenshot of environment variables](/img/placeholder.svg) diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/index.md b/docs/docs-beta/docs/dagster-plus/deployment/management/index.md index 144dd10871fe2..aabb25b2e8a9c 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/index.md @@ -5,4 +5,4 @@ sidebar_position: 30 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/managing-compute-logs-and-error-messages.md b/docs/docs-beta/docs/dagster-plus/deployment/management/managing-compute-logs-and-error-messages.md index 01279273021f2..bc68c199c6471 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/managing-compute-logs-and-error-messages.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/managing-compute-logs-and-error-messages.md @@ -3,4 +3,4 @@ title: Managing compute logs and error messages unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/managing-deployments/controlling-logs */} \ No newline at end of file +{/** TODO move from https://docs.dagster.io/dagster-plus/managing-deployments/controlling-logs **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/managing-multiple-projects-and-teams.md b/docs/docs-beta/docs/dagster-plus/deployment/management/managing-multiple-projects-and-teams.md index cc59bd13364a9..de7c3866654f8 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/managing-multiple-projects-and-teams.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/managing-multiple-projects-and-teams.md @@ -3,4 +3,4 @@ title: Managing multiple projects and teams unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/best-practices/managing-multiple-projects-and-teams */} \ No newline at end of file +{/** TODO move from https://docs.dagster.io/dagster-plus/best-practices/managing-multiple-projects-and-teams **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/rate-limits.md b/docs/docs-beta/docs/dagster-plus/deployment/management/rate-limits.md index b4364e74ff276..ccf1036561bda 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/rate-limits.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/rate-limits.md @@ -3,4 +3,4 @@ title: Dagster+ rate limits unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/references/limits */} \ No newline at end of file +{/** TODO move from https://docs.dagster.io/dagster-plus/references/limits **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/customizing-agent-settings.md b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/customizing-agent-settings.md index df595ee48e153..47dbcab83d9c7 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/customizing-agent-settings.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/customizing-agent-settings.md @@ -4,4 +4,4 @@ sidebar_position: 80 unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/deployment/agents/customizing-configuration */} \ No newline at end of file +{/** TODO move from https://docs.dagster.io/dagster-plus/deployment/agents/customizing-configuration **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/dagster-plus-settings.md b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/dagster-plus-settings.md index ac4d6cfc1af70..131a78976fede 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/dagster-plus-settings.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/dagster-plus-settings.md @@ -1,6 +1,5 @@ --- -title: "Dagster+ settings" +title: 'Dagster+ settings' sidebar_position: 100 unlisted: true --- - diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/deployment-settings.md b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/deployment-settings.md index c0cc4dbecca60..41acbcf0a133d 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/deployment-settings.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/deployment-settings.md @@ -1,7 +1,7 @@ --- -title: "Deployment settings" +title: 'Deployment settings' sidebar_position: 200 unlisted: true --- -{/* TODO move from https://docs.dagster.io/dagster-plus/managing-deployments/deployment-settings-reference */} +{/** TODO move from https://docs.dagster.io/dagster-plus/managing-deployments/deployment-settings-reference **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/hybrid-agent-settings.md b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/hybrid-agent-settings.md index 8b076b2f2e919..42bfc1eca1862 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/hybrid-agent-settings.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/hybrid-agent-settings.md @@ -1,8 +1,7 @@ --- -title: "Hybrid agent settings" +title: 'Hybrid agent settings' sidebar_position: 70 unlisted: true --- -{/* TODO not sure why this page exists, is this covered by "Customizing agent settings"? */} - +{/** TODO not sure why this page exists, is this covered by "Customizing agent settings"? **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/index.md b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/index.md index 0bc19ee72c5b6..72823e4d16669 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/settings/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/settings/index.md @@ -5,4 +5,4 @@ sidebar_position: 70 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/agent-tokens.md b/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/agent-tokens.md index e777a568a9f9f..b4df8fbbd520b 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/agent-tokens.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/agent-tokens.md @@ -4,4 +4,4 @@ sidebar_position: 200 unlisted: true --- -{/* TODO move agent token content from https://docs.dagster.io/dagster-plus/account/managing-user-agent-tokens */} +{/** TODO move agent token content from https://docs.dagster.io/dagster-plus/account/managing-user-agent-tokens **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/index.md b/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/index.md index f42a1600d7018..80d09c7d410e4 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/index.md @@ -5,4 +5,4 @@ sidebar_position: 80 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/user-tokens.md b/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/user-tokens.md index 130e274d49ab7..01b309c6a856b 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/user-tokens.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/tokens/user-tokens.md @@ -4,4 +4,4 @@ sidebar_position: 100 unlisted: true --- -{/* TODO move user token content from https://docs.dagster.io/dagster-plus/account/managing-user-agent-tokens */} \ No newline at end of file +{/** TODO move user token content from https://docs.dagster.io/dagster-plus/account/managing-user-agent-tokens **/} diff --git a/docs/docs-beta/docs/dagster-plus/deployment/migration/index.md b/docs/docs-beta/docs/dagster-plus/deployment/migration/index.md index ea7c4a344ade3..96e7be14941de 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/migration/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/migration/index.md @@ -5,4 +5,4 @@ sidebar_position: 50 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/features/alerts/configuring-an-alert-notification-service.md b/docs/docs-beta/docs/dagster-plus/features/alerts/configuring-an-alert-notification-service.md index 5cd722b49385b..f7b3e7a0ac112 100644 --- a/docs/docs-beta/docs/dagster-plus/features/alerts/configuring-an-alert-notification-service.md +++ b/docs/docs-beta/docs/dagster-plus/features/alerts/configuring-an-alert-notification-service.md @@ -5,7 +5,7 @@ sidebar_position: 100 Dagster+ allows you to configure alerts to fire in response to a range of events. These alerts can be sent to a variety of different services, depending on your organization's needs. -{/* TODO this needs content on the value prop of alerts -- save money, etc */} +{/** TODO this needs content on the value prop of alerts -- save money, etc **/} :::note You must have **Organization**, **Admin**, or **Editor** permissions on Dagster+ to configure an alert notification service. @@ -18,6 +18,7 @@ Before [creating alerts](creating-alerts), you'll need to configure a service to No additional configuration is required to send emails from Dagster+. All alert emails will be sent by `"no-reply@dagster.cloud"` or `"no-reply@.dagster.cloud"`. Alerts can be configured to be sent to any number of emails. + Create an incoming webhook by following the [Microsoft Teams documentation](https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=newteams%2Cdotnet). @@ -49,4 +50,4 @@ Navigate to **Deployment > Alerts** in the Dagster+ UI and click **Connect to Sl When setting up an alert, you can choose a Slack channel to send those alerts to. Make sure to invite the `@Dagster+` bot to any channel that you'd like to receive an alert in. - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/features/alerts/creating-alerts.md b/docs/docs-beta/docs/dagster-plus/features/alerts/creating-alerts.md index 8239359b605fd..566cacfe39993 100644 --- a/docs/docs-beta/docs/dagster-plus/features/alerts/creating-alerts.md +++ b/docs/docs-beta/docs/dagster-plus/features/alerts/creating-alerts.md @@ -5,16 +5,18 @@ sidebar_position: 200 You can create alerts in the Dagster+ UI or using the [`dagster-cloud` CLI](/dagster-plus/deployment/management/dagster-cloud-cli). -{/* TODO link to dagster-cloud CLI tool doc */} +{/** TODO link to dagster-cloud CLI tool doc **/} :::note Before you create alerts, you must [configure an alert notification service](configuring-an-alert-notification-service). ::: ## Alerting when a run fails + You can set up alerts to notify you when a run fails. By default, these alerts will target all runs in the deployment, but they can be scoped to runs with a specific tag. + 1. In the Dagster UI, click **Deployment**. @@ -30,31 +32,47 @@ If desired, add **tags** in the format `{key}:{value}` to filter the runs that w Execute the following command to sync the configured alert policy to your Dagster+ deployment. - ```bash - dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml - ``` - - - - - - - - - - - - - - +```bash +dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml +``` + +{' '} + + + + + + + + + + + + + + ## Alerting when a run is taking too long to complete + You can set up alerts to notify you whenever a run takes more than some threshold amount of time. By default, these alerts will target all runs in the deployment, but they can be scoped to runs with a specific tag. + 1. In the Dagster UI, click **Deployment**. @@ -70,31 +88,47 @@ If desired, add **tags** in the format `{key}:{value}` to filter the runs that w Execute the following command to sync the configured alert policy to your Dagster+ deployment. - ```bash - dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml - ``` - - - - - - - - - - - - - - +```bash +dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml +``` + +{' '} + + + + + + + + + + + + + + ## Alerting when an asset fails to materialize + You can set up alerts to notify you when an asset materialization attempt fails. By default, these alerts will target all assets in the deployment, but they can be scoped to a specific asset or group of assets. + 1. In the Dagster UI, click **Deployment**. @@ -110,31 +144,47 @@ If desired, select a **target** from the dropdown menu to scope this alert to a Execute the following command to sync the configured alert policy to your Dagster+ deployment. - ```bash - dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml - ``` - - - - - - - - - - - - - - +```bash +dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml +``` + +{' '} + + + + + + + + + + + + + + ## Alerting when an asset check fails + You can set up alerts to notify you when an asset check on an asset fails. By default, these alerts will target all assets in the deployment, but they can be scoped to checks on a specific asset or group of assets. + 1. In the Dagster UI, click **Deployment**. @@ -150,31 +200,47 @@ If desired, select a **target** from the dropdown menu to scope this alert to a Execute the following command to sync the configured alert policy to your Dagster+ deployment. - ```bash - dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml - ``` - - - - - - - - - - - - - - +```bash +dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml +``` + +{' '} + + + + + + + + + + + + + + ## Alerting when a schedule or sensor tick fails + You can set up alerts to fire when any schedule or sensor tick across your entire deployment fails. Alerts are sent only when a schedule/sensor transitions from **success** to **failure**, so only the initial failure will trigger the alert. + 1. In the Dagster UI, click **Deployment**. @@ -185,29 +251,45 @@ Alerts are sent only when a schedule/sensor transitions from **success** to **fa Execute the following command to sync the configured alert policy to your Dagster+ deployment. - ```bash - dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml - ``` - - - - - - - - - - - - - - +```bash +dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml +``` + +{' '} + + + + + + + + + + + + + + ## Alerting when a code location fails to load + You can set up alerts to fire when any code location fails to load due to an error. + 1. In the Dagster UI, click **Deployment**. @@ -218,33 +300,49 @@ You can set up alerts to fire when any code location fails to load due to an err Execute the following command to sync the configured alert policy to your Dagster+ deployment. - ```bash - dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml - ``` - - - - - - - - - - - - - - +```bash +dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml +``` + +{' '} + + + + + + + + + + + + + + ## Alerting when a Hybrid agent becomes unavailable + :::note This is only available for [Hybrid](/todo) deployments. ::: You can set up alerts to fire if your Hybrid agent hasn't sent a heartbeat in the last 5 minutes. + 1. In the Dagster UI, click **Deployment**. @@ -255,23 +353,37 @@ You can set up alerts to fire if your Hybrid agent hasn't sent a heartbeat in th Execute the following command to sync the configured alert policy to your Dagster+ deployment. - ```bash - dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml - ``` - - - - - - - - - - - - - - +```bash +dagster-cloud deployment alert-policies sync -a /path/to/alert_policies.yaml +``` + +{' '} + + + + + + + + + + + + + + - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/index.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/index.md index 4b3a0e15bfa4f..73a19478add53 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/index.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/index.md @@ -1,8 +1,8 @@ --- -title: "Authentication and access control" +title: 'Authentication and access control' sidebar_position: 10 --- import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/audit-logs.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/audit-logs.md index fd298db800102..3d9c115b14bf4 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/audit-logs.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/audit-logs.md @@ -1,19 +1,19 @@ --- -title: "Audit logs" +title: 'Audit logs' sidebar_position: 300 --- The Dagster+ audit log enables Dagster+ Pro organizations to track and attribute changes to their Dagster deployment. For large organizations, tracking down when and by whom changes were made can be crucial for maintaining security and compliance. The audit log is also valuable - for tracking operational history, including sensor and schedule updates. +for tracking operational history, including sensor and schedule updates. This guide walks through how to access the audit log and details the interactions which are tracked in the audit log.
-Prerequisites -- A Dagster+ Pro organization -- An [Organization Admin](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) role in your Dagster+ organization + Prerequisites- A Dagster+ Pro organization - An [Organization + Admin](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) role + in your Dagster+ organization
## View audit logs @@ -38,28 +38,32 @@ The **Filter** button near the top left of the page can be used to filter the li ## Audit log entry types -| Event type | Description | Additional details | -|--------------------------------|---------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------| -| Log in | A user logs in to the Dagster+ organization | | -| Update sensor | A user toggles a sensor on or off | The sensor name, code location, and cursor | -| Update schedule | A user toggles a schedule on or off | The schedule name, code location, and cursor | -| Update alert policy | A user modifies an [alert policy](/dagster-plus/features/alerts/creating-alerts) | The new configuration for the alert policy | -| Create deployment | A user creates a new deployment | Whether the deployment is a branch deployment | -| Delete deployment | A user removes an existing deployment | Whether the deployment is a branch deployment | -| Create user token | A user creates a new user token | | -| Revoke user token | A user revokes an existing user token | | -| Change user permissions | A user alters [permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) for another user | The permission grant and targeted deployment | -| Create agent token | A user creates a new agent token | | -| Revoke agent token | A user revokes an existing agent token | | -| Update agent token permissions | A user alters [permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) for an agent token | The permission grant and targeted deployment | -| Create secret | A user creates a new [environment variable](/dagster-plus/deployment/management/environment-variables/dagster-ui) | The created variable name | -| Update secret | A user modifies an existing [environment variable](/dagster-plus/deployment/management/environment-variables/dagster-ui) | The previous and current variable names and whether the value was changed | -| Delete secret | A user removes an [environment variable](/dagster-plus/deployment/management/environment-variables/dagster-ui) | The deleted variable name | -| Update subscription | A user modifies the selected Dagster+ subscription for the organization | The previous and current plan types | +| Event type | Description | Additional details | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------- | +| Log in | A user logs in to the Dagster+ organization | | +| Update sensor | A user toggles a sensor on or off | The sensor name, code location, and cursor | +| Update schedule | A user toggles a schedule on or off | The schedule name, code location, and cursor | +| Update alert policy | A user modifies an [alert policy](/dagster-plus/features/alerts/creating-alerts) | The new configuration for the alert policy | +| Create deployment | A user creates a new deployment | Whether the deployment is a branch deployment | +| Delete deployment | A user removes an existing deployment | Whether the deployment is a branch deployment | +| Create user token | A user creates a new user token | | +| Revoke user token | A user revokes an existing user token | | +| Change user permissions | A user alters [permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) for another user | The permission grant and targeted deployment | +| Create agent token | A user creates a new agent token | | +| Revoke agent token | A user revokes an existing agent token | | +| Update agent token permissions | A user alters [permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) for an agent token | The permission grant and targeted deployment | +| Create secret | A user creates a new [environment variable](/dagster-plus/deployment/management/environment-variables/dagster-ui) | The created variable name | +| Update secret | A user modifies an existing [environment variable](/dagster-plus/deployment/management/environment-variables/dagster-ui) | The previous and current variable names and whether the value was changed | +| Delete secret | A user removes an [environment variable](/dagster-plus/deployment/management/environment-variables/dagster-ui) | The deleted variable name | +| Update subscription | A user modifies the selected Dagster+ subscription for the organization | The previous and current plan types | ## Programmatic access to audit logs Audit logs can be accessed programmatically over the Dagster+ GraphQL API. You can access a visual GraphiQL interface by navigating to `https://.dagster.cloud//graphql` in your browser. You can also query the API directly using the Python client. - + diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/index.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/index.md index 5f6a44b46616a..823ed38b8b21c 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/index.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/index.md @@ -1,8 +1,8 @@ --- -title: "Role-based access control" +title: 'Role-based access control' sidebar_position: 10 --- import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/teams.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/teams.md index 894a4a29f0db9..d8e56dac1c7b6 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/teams.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/teams.md @@ -1,6 +1,6 @@ --- title: Team management in Dagster+ -sidebar_label: "Team management" +sidebar_label: 'Team management' sidebar_position: 100 --- @@ -13,12 +13,11 @@ To complete the steps in this guide, you'll need: - A Dagster+ Pro plan - Dagster+ [Organization Admin permissions](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions): - - In your organization, and - - For the deployments where you want to manage teams + - In your organization, and + - For the deployments where you want to manage teams - ## Adding teams 1. In the Dagster+ UI, click the **user menu (your icon) > Organization Settings**. @@ -57,10 +56,10 @@ Navigate to the **Organization Settings > Teams** tab and locate the team you wa 3. In the window that displays, select the team role for the deployment. This [role](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) will be used as the default for this team for all code locations in the deployment. 4. Click **Save**. 5. To set permissions for individual [code locations](/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions) in a deployment: - 1. Click the toggle to the left of the deployment to open a list of code locations. - 2. Next to a code location, click **Edit team role**. - 3. Select the team role for the code location. - 4. Click **Save**. + 1. Click the toggle to the left of the deployment to open a list of code locations. + 2. Next to a code location, click **Edit team role**. + 3. Select the team role for the code location. + 4. Click **Save**. ## Removing teams diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions.md index d6ce4c96de5e3..e2550010d810d 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/user-roles-permissions.md @@ -108,29 +108,29 @@ TODO: add picture previously at "/images/dagster-cloud/user-token-management/cod ### General -| | Viewer | Launcher | Editor | Admin | Organization
admin | -| ------------------------------------------------------------------------ | ------ | -------- | ------ | ----- | ------------------------ | -| View runs of [jobs](/guides/build/ops-jobs) | ✅ | ✅ | ✅ | ✅ | ✅ | -| Launch, re-execute, terminate, and delete runs of jobs | ❌ | ✅ | ✅ | ✅ | ✅ | -| Start and stop [schedules](/guides/automate/schedules) | ❌ | ❌ | ✅ | ✅ | ✅ | -| Start and stop [schedules](/guides/automate/sensors) | ❌ | ❌ | ✅ | ✅ | ✅ | -| Wipe assets | ❌ | ❌ | ✅ | ✅ | ✅ | -| Launch and cancel [schedules](/guides/build/backfill) | ❌ | ✅ | ✅ | ✅ | ✅ | -| Add dynamic partitions | ❌ | ❌ | ✅ | ✅ | ✅ | +| | Viewer | Launcher | Editor | Admin | Organization
admin | +| ------------------------------------------------------ | ------ | -------- | ------ | ----- | ------------------------ | +| View runs of [jobs](/guides/build/ops-jobs) | ✅ | ✅ | ✅ | ✅ | ✅ | +| Launch, re-execute, terminate, and delete runs of jobs | ❌ | ✅ | ✅ | ✅ | ✅ | +| Start and stop [schedules](/guides/automate/schedules) | ❌ | ❌ | ✅ | ✅ | ✅ | +| Start and stop [schedules](/guides/automate/sensors) | ❌ | ❌ | ✅ | ✅ | ✅ | +| Wipe assets | ❌ | ❌ | ✅ | ✅ | ✅ | +| Launch and cancel [schedules](/guides/build/backfill) | ❌ | ✅ | ✅ | ✅ | ✅ | +| Add dynamic partitions | ❌ | ❌ | ✅ | ✅ | ✅ | ### Deployments Deployment settings are accessed in the UI by navigating to **user menu (your icon) > Organization Settings > Deployments**. -| | Viewer | Launcher | Editor | Admin | Organization
admin | -|----------------------------------------------------------------------------------------------|-------|-----------|--------|-------|-------------------------------| -| View [deployments](/todo) | ✅ | ✅ | ✅ | ✅ | ✅ | -| Modify [deployment](/todo) settings | ❌ | ❌ | ✅ | ✅ | ✅ | -| Create, edit, delete [environment variables](/dagster-plus/deployment/management/environment-variables) | ❌ | ❌ | ✅ | ✅ | ✅ | -| View [environment variable](/dagster-plus/deployment/management/environment-variables) values | ❌ | ❌ | ✅ | ✅ | ✅ | -| Export [environment variables](/dagster-plus/deployment/management/environment-variables) | ❌ | ❌ | ✅ | ✅ | ✅ | -| Create and delete [deployments](/todo) | ❌ | ❌ | ❌ | ❌ | ✅ | -| Create [Branch Deployments](dagster-plus/features/ci-cd/branch-deployments/index.md) | ❌ | ❌ | ✅ | ✅ | ✅ | +| | Viewer | Launcher | Editor | Admin | Organization
admin | +| ------------------------------------------------------------------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | +| View [deployments](/todo) | ✅ | ✅ | ✅ | ✅ | ✅ | +| Modify [deployment](/todo) settings | ❌ | ❌ | ✅ | ✅ | ✅ | +| Create, edit, delete [environment variables](/dagster-plus/deployment/management/environment-variables) | ❌ | ❌ | ✅ | ✅ | ✅ | +| View [environment variable](/dagster-plus/deployment/management/environment-variables) values | ❌ | ❌ | ✅ | ✅ | ✅ | +| Export [environment variables](/dagster-plus/deployment/management/environment-variables) | ❌ | ❌ | ✅ | ✅ | ✅ | +| Create and delete [deployments](/todo) | ❌ | ❌ | ❌ | ❌ | ✅ | +| Create [Branch Deployments](dagster-plus/features/ci-cd/branch-deployments/index.md) | ❌ | ❌ | ✅ | ✅ | ✅ | ### Code locations @@ -146,12 +146,12 @@ Code locations are accessed in the UI by navigating to **Deployment > Code locat Agent tokens are accessed in the UI by navigating to **user menu (your icon) > Organization Settings > Tokens**. -| | Viewer | Launcher | Editor | Admin | Organization
admin | -| ----------------------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | +| | Viewer | Launcher | Editor | Admin | Organization
admin | +| ---------------------------------------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | | View [agent tokens](/dagster-plus/deployment/management/tokens/agent-tokens) | ❌ | ❌ | ❌ | ❌ | ✅ | -| Create agent tokens | ❌ | ❌ | ❌ | ❌ | ✅ | -| Edit agent tokens | ❌ | ❌ | ❌ | ❌ | ✅ | -| Revoke agent tokens | ❌ | ❌ | ❌ | ❌ | ✅ | +| Create agent tokens | ❌ | ❌ | ❌ | ❌ | ✅ | +| Edit agent tokens | ❌ | ❌ | ❌ | ❌ | ✅ | +| Revoke agent tokens | ❌ | ❌ | ❌ | ❌ | ✅ | ### User tokens @@ -167,12 +167,12 @@ User tokens are accessed in the UI by navigating to **user menu (your icon) > Or User management is accessed in the UI by navigating to **user menu (your icon) > Organization Settings > Users**. -| | Viewer | Launcher | Editor | Admin | Organization
admin | -| --------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | +| | Viewer | Launcher | Editor | Admin | Organization
admin | +| --------------------------------------------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | | [View users](/dagster-plus/features/authentication-and-access-control/rbac/users) | ✅ | ✅ | ✅ | ✅ | ✅ | -| Add users | ❌ | ❌ | ❌ | ✅ | ✅ | -| Edit user roles | ❌ | ❌ | ❌ | ❌ | ✅ | -| Remove users | ❌ | ❌ | ❌ | ❌ | ✅ | +| Add users | ❌ | ❌ | ❌ | ✅ | ✅ | +| Edit user roles | ❌ | ❌ | ❌ | ❌ | ✅ | +| Remove users | ❌ | ❌ | ❌ | ❌ | ✅ | ### Teams @@ -180,26 +180,26 @@ Team management is accessed in the UI by navigating to **user menu (your icon) > **Note**: Admin users can modify teams only in deployments where they're an Admin. -| | Viewer | Launcher | Editor | Admin | Organization
admin | -| --------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | +| | Viewer | Launcher | Editor | Admin | Organization
admin | +| --------------------------------------------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | | [View teams](/dagster-plus/features/authentication-and-access-control/rbac/teams) | ✅ | ✅ | ✅ | ✅ | ✅ | -| Modify team permissions | ❌ | ❌ | ❌ | ✅ | ✅ | -| Create teams | ❌ | ❌ | ❌ | ❌ | ✅ | -| Re-name teams | ❌ | ❌ | ❌ | ❌ | ✅ | -| Add/remove team members | ❌ | ❌ | ❌ | ❌ | ✅ | -| Remove teams | ❌ | ❌ | ❌ | ❌ | ✅ | +| Modify team permissions | ❌ | ❌ | ❌ | ✅ | ✅ | +| Create teams | ❌ | ❌ | ❌ | ❌ | ✅ | +| Re-name teams | ❌ | ❌ | ❌ | ❌ | ✅ | +| Add/remove team members | ❌ | ❌ | ❌ | ❌ | ✅ | +| Remove teams | ❌ | ❌ | ❌ | ❌ | ✅ | ### Workspace administration -| | Viewer | Launcher | Editor | Admin | Organization
admin | -| ------------------------------------------------------ | ------ | -------- | ------ | ----- | ------------------------ | -| Manage [alerts](/dagster-plus/features/alerts) | ❌ | ❌ | ✅ | ✅ | ✅ | -| Edit workspace | ❌ | ❌ | ✅ | ✅ | ✅ | +| | Viewer | Launcher | Editor | Admin | Organization
admin | +| ---------------------------------------------------------------------------------------------- | ------ | -------- | ------ | ----- | ------------------------ | +| Manage [alerts](/dagster-plus/features/alerts) | ❌ | ❌ | ✅ | ✅ | ✅ | +| Edit workspace | ❌ | ❌ | ✅ | ✅ | ✅ | | [Administer SAML](/dagster-plus/features/authentication-and-access-control/sso/authentication) | ❌ | ❌ | ❌ | ❌ | ✅ | -| [Manage SCIM](/todo) | ❌ | ❌ | ❌ | ❌ | ✅ | -| View usage | ❌ | ❌ | ❌ | ❌ | ✅ | -| Manage billing | ❌ | ❌ | ❌ | ❌ | ✅ | -| View audit logs | ❌ | ❌ | ❌ | ❌ | ✅ | +| [Manage SCIM](/todo) | ❌ | ❌ | ❌ | ❌ | ✅ | +| View usage | ❌ | ❌ | ❌ | ❌ | ✅ | +| Manage billing | ❌ | ❌ | ❌ | ❌ | ✅ | +| View audit logs | ❌ | ❌ | ❌ | ❌ | ✅ | ## Next steps diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/users.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/users.md index 2f1294cb4fb17..3c60d9f0accdf 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/users.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/rbac/users.md @@ -1,6 +1,6 @@ --- -title: "Managing users in Dagster+" -sidebar_label: "User management" +title: 'Managing users in Dagster+' +sidebar_label: 'User management' sidebar_position: 400 --- @@ -13,8 +13,8 @@ In this guide, you'll learn how to manage users and their permissions using the - A Dagster+ account - The required [Dagster+ permissions](/todo): - - **Organization Admins** can add, manage, and remove users - - **Admins** can add users + - **Organization Admins** can add, manage, and remove users + - **Admins** can add users diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/enabling-scim-provisioning.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/enabling-scim-provisioning.md index 2effe2befed3a..f4c7c5bc05977 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/enabling-scim-provisioning.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/enabling-scim-provisioning.md @@ -4,4 +4,4 @@ sidebar_position: 100 unlisted: true --- -{/* TODO content from https://docs.dagster.io/dagster-plus/account/authentication/utilizing-scim-provisioning#enabling-scim-provisioning should be copied here */} +{/** TODO content from https://docs.dagster.io/dagster-plus/account/authentication/utilizing-scim-provisioning#enabling-scim-provisioning should be copied here **/} diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/index.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/index.md index 392c7b0be278a..a4803f5f1dcb3 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/index.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/index.md @@ -1,11 +1,11 @@ --- -title: "SCIM provisioning" +title: 'SCIM provisioning' sidebar_position: 30 unlisted: true --- -{/* TODO content from top of "Utilizing SCIM provisioning" https://docs.dagster.io/dagster-plus/account/authentication/utilizing-scim-provisioning page and "Understanding SCIM provisioning" section https://docs.dagster.io/dagster-plus/account/authentication/utilizing-scim-provisioning#understanding-scim-provisioning should be copied here. */} +{/** TODO content from top of "Utilizing SCIM provisioning" https://docs.dagster.io/dagster-plus/account/authentication/utilizing-scim-provisioning page and "Understanding SCIM provisioning" section https://docs.dagster.io/dagster-plus/account/authentication/utilizing-scim-provisioning#understanding-scim-provisioning should be copied here. **/} ## User management with SCIM -## Team management with SCIM \ No newline at end of file +## Team management with SCIM diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/okta-scim.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/okta-scim.md index c007683bc3df8..ecb64b49bea98 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/okta-scim.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/scim/okta-scim.md @@ -4,4 +4,4 @@ sidebar_position: 200 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/account/authentication/okta/scim-provisioning */} +{/** TODO copy from https://docs.dagster.io/dagster-plus/account/authentication/okta/scim-provisioning **/} diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/authentication.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/authentication.md index 4a3489acf0fbe..a9ae43a09fa10 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/authentication.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/authentication.md @@ -1,6 +1,5 @@ --- -title: "Authentication" +title: 'Authentication' sidebar_position: 100 unlisted: true --- - diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/azure-ad-sso.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/azure-ad-sso.md index 969014b7520cd..51989ff7638ec 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/azure-ad-sso.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/azure-ad-sso.md @@ -56,7 +56,9 @@ In this step, you'll configure and enable SSO for Azure AD in your Azure portal. - `Email` - `user.userprincipalname` Add these attribute mappings to the SAML assertion. + 8. On the **Set up single sign-on with SAML** page: + 1. Locate the **SAML Signing Certificate** section. 2. Next to **Federation Metadata XML**, click **Download**: diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/google-workspace-sso.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/google-workspace-sso.md index 6c980a33717f7..12c712534b3ef 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/google-workspace-sso.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/google-workspace-sso.md @@ -36,42 +36,46 @@ To complete the steps in this guide, you'll need: ## Step 2: Configure SSO in Google Workspace \{#configure-sso} 1. On the **App details** page: - 1. Fill in the **App name** field. - 2. Fill in the **Description** field. - The page should look similar to the following: + 1. Fill in the **App name** field. + 2. Fill in the **Description** field. - ![Application Details](/img/placeholder.svg) + The page should look similar to the following: - 3. Click **Continue**. + ![Application Details](/img/placeholder.svg) + + 3. Click **Continue**. 2. On the **Google Identity Provider details** page, click **Continue**. No action is required for this page. 3. On the **Service provider details** page: - 1. In the **ACS URL** and **Entity ID** fields: - Copy and paste the following URL, replacing `` with your Dagster+ organization name: + 1. In the **ACS URL** and **Entity ID** fields: + + Copy and paste the following URL, replacing `` with your Dagster+ organization name: + + ``` + https://.dagster.cloud/auth/saml/consume + ``` + + 2. Check the **Signed Response** box. The page should look similar to the image below. In this example, the organization's name is `hooli` and the Dagster+ domain is `https://hooli.dagster.cloud`: - ``` - https://.dagster.cloud/auth/saml/consume - ``` + ![Service Provider Details](/img/placeholder.svg) - 2. Check the **Signed Response** box. The page should look similar to the image below. In this example, the organization's name is `hooli` and the Dagster+ domain is `https://hooli.dagster.cloud`: + 3. When finished, click **Continue**. - ![Service Provider Details](/img/placeholder.svg) +4. On the **Attributes** page: - 3. When finished, click **Continue**. -4. On the **Attributes** page: - 1. Click **Add mapping** to add and configure the following attributes: + 1. Click **Add mapping** to add and configure the following attributes: - - **Basic Information > First Name** - `FirstName` - - **Basic Information > Last Name** - `LastName` - - **Basic Information > Email** - `Email` + - **Basic Information > First Name** - `FirstName` + - **Basic Information > Last Name** - `LastName` + - **Basic Information > Email** - `Email` - The page should look like the following image: + The page should look like the following image: - ![Attribute Mapping](/img/placeholder.svg) + ![Attribute Mapping](/img/placeholder.svg) - 2. Click **Finish**. + 2. Click **Finish**. ## Step 3: Upload the SAML metadata to Dagster+ \{#upload-saml} diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/index.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/index.md index e5711aad31dd8..1f778935b654c 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/index.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/index.md @@ -1,8 +1,8 @@ --- -title: "Single sign-on" +title: 'Single sign-on' sidebar_position: 20 --- import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/okta-sso.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/okta-sso.md index 7161afbccd1cb..a6c8ca62da9d7 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/okta-sso.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/okta-sso.md @@ -20,7 +20,6 @@ To complete the steps in this guide, you'll need: - ## Step 1: Add the Dagster+ app in Okta \{#dagster-app} 1. Sign in to your Okta Admin Dashboard. @@ -29,20 +28,18 @@ To complete the steps in this guide, you'll need: 4. On the **Browse App Integration Catalog** page, search for `Dagster Cloud`. 5. Add and save the application. - ## Step 2: Configure SSO in Okta \{#configure-sso} 1. In Okta, open the Dagster Cloud application and navigate to its **Sign On Settings**. 2. Scroll down to the **Advanced Sign-on settings** section. 3. In the **Organization** field, enter your Dagster+ organization name. This is used to route the SAML response to the correct Dagster+ subdomain. - For example, your organization name is `hooli` and your Dagster+ domain is `https://hooli.dagster.cloud`. To configure this correctly, you'd enter `hooli` into the **Organization** field: +For example, your organization name is `hooli` and your Dagster+ domain is `https://hooli.dagster.cloud`. To configure this correctly, you'd enter `hooli` into the **Organization** field: - ![Okta Subdomain Configuration](/img/placeholder.svg) +![Okta Subdomain Configuration](/img/placeholder.svg) 4. When finished, click **Done**. - ## Step 3: Upload the SAML metadata to Dagster+ \{#upload-saml} Next, you'll save and upload the application's SAML metadata to Dagster+. This will enable single sign-on. @@ -73,7 +70,6 @@ Next, you'll save and upload the application's SAML metadata to Dagster+. This w --url https://.dagster.cloud ``` - ## Step 4: Grant access to users \{#grant-access} Next, you'll assign users to the Dagster+ application in Okta. This will allow them to log in using their Okta credentials when the single sign-on flow is initiated. diff --git a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/onelogin-sso.md b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/onelogin-sso.md index 5c4ec8a246dda..2efd8919bd87e 100644 --- a/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/onelogin-sso.md +++ b/docs/docs-beta/docs/dagster-plus/features/authentication-and-access-control/sso/onelogin-sso.md @@ -5,7 +5,6 @@ sidebar_position: 500 In this guide, you'll configure OneLogin to use single sign-on (SSO) with your Dagster+ organization. -
Prerequisites @@ -22,7 +21,6 @@ To follow the steps in this guide, you'll need:
- ## Step 1: Add the Dagster+ app in OneLogin \{#dagster-app} 1. Sign into your OneLogin portal. @@ -34,17 +32,14 @@ To follow the steps in this guide, you'll need: 5. Add and save the application. - - ## Step 2: Configure SSO in OneLogin \{#configure-sso} 1. In OneLogin, open the application and navigate to its **Configuration**. 2. In the **Dagster+ organisation name** field, enter your Dagster+ organization name. This is used to route the SAML response to the correct Dagster+ subdomain. For example, your organization name is `hooli` and your Dagster+ domain is `https://hooli.dagster.cloud`. To configure this correctly, you'd enter `hooli` into the **Subdomain** field. -3. When finished, click **Done**. - +3. When finished, click **Done**. ## Step 3: Upload the SAML metadata to Dagster+ \{#upload-saml} @@ -61,7 +56,6 @@ Next, you'll save and upload the application's SAML metadata to Dagster+. This w --url https://.dagster.cloud ``` - ## Step 4: Grant access to users \{#grant-access} Next, you'll assign users to the Dagster+ application in OneLogin. This will allow them to log in using their OneLogin credentials with the sign in flow is initiated. @@ -77,7 +71,6 @@ Next, you'll assign users to the Dagster+ application in OneLogin. This will all 6. Click **Save User.** 7. Repeat steps 2-6 for every user you want to access Dagster+. - import TestSSO from '../../../../partials/\_TestSSO.md'; diff --git a/docs/docs-beta/docs/dagster-plus/features/catalog-views.md b/docs/docs-beta/docs/dagster-plus/features/catalog-views.md index 201467bd939ad..dded09eeb7cc1 100644 --- a/docs/docs-beta/docs/dagster-plus/features/catalog-views.md +++ b/docs/docs-beta/docs/dagster-plus/features/catalog-views.md @@ -1,5 +1,5 @@ --- -title: "Catalog views" +title: 'Catalog views' sidebar_position: 100 --- @@ -21,9 +21,6 @@ In this guide, you'll learn how to create, access, and share catalog views with - - - ## Creating catalog views To view the Dagster+ Asset catalog, use the **Catalog** button on the top navigation. @@ -33,6 +30,7 @@ In any Dagster+ catalog page, you can access the current catalog view, or create ![Screenshot of the catalog view dropdown](/img/placeholder.svg) To create a new catalog view, you have two options: + 1. Create a new catalog view from scratch, from the catalog view menu. 2. Create a new catalog view from your current set of filters. diff --git a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/change-tracking.md b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/change-tracking.md index f798ce75e47bd..c038aa8ae8042 100644 --- a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/change-tracking.md +++ b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/change-tracking.md @@ -1,7 +1,7 @@ --- -title: "Change tracking in branch deployments" +title: 'Change tracking in branch deployments' sidebar_position: 200 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/managing-deployments/branch-deployments/change-tracking */} +{/** TODO copy from https://docs.dagster.io/dagster-plus/managing-deployments/branch-deployments/change-tracking **/} diff --git a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/dagster-cloud-cli.md b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/dagster-cloud-cli.md index f5af56c5b9abe..f3adcf903369a 100644 --- a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/dagster-cloud-cli.md +++ b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/dagster-cloud-cli.md @@ -4,4 +4,4 @@ sidebar_position: 300 unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/managing-deployments/dagster-plus-cli#using-the-dagster-cloud-cli */} \ No newline at end of file +{/** TODO copy from https://docs.dagster.io/dagster-plus/managing-deployments/dagster-plus-cli#using-the-dagster-cloud-cli **/} diff --git a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/index.md b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/index.md index 39e2db6935879..64cfd2d5f6e40 100644 --- a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/index.md +++ b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/index.md @@ -5,4 +5,4 @@ sidebar_position: 20 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/setting-up-branch-deployments.md b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/setting-up-branch-deployments.md index 79c27ab7f9383..f59e42bc4bc32 100644 --- a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/setting-up-branch-deployments.md +++ b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/setting-up-branch-deployments.md @@ -1,5 +1,5 @@ --- -title: "Setting up branch deployments" +title: 'Setting up branch deployments' sidebar_position: 100 --- @@ -82,58 +82,59 @@ While you can use your existing production agent, we recommend creating a dedica - 1. **Deploy an ECS agent to serve your branch deployments**. Follow the [ECS agent](/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/new-vpc) setup guide, making sure to set the **Enable Branch Deployments** parameter if using the CloudFormation template. If you are running an existing agent, follow the [upgrade guide](/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/existing-vpc) to ensure your template is up-to-date. Then, turn on the **Enable Branch Deployments** parameter. +1. **Deploy an ECS agent to serve your branch deployments**. Follow the [ECS agent](/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/new-vpc) setup guide, making sure to set the **Enable Branch Deployments** parameter if using the CloudFormation template. If you are running an existing agent, follow the [upgrade guide](/dagster-plus/deployment/deployment-types/hybrid/amazon-ecs/existing-vpc) to ensure your template is up-to-date. Then, turn on the **Enable Branch Deployments** parameter. - 2. **Create a private [Amazon Elastic Registry (ECR) repository](https://console.aws.amazon.com/ecr/repositories).** Refer to the [AWS ECR documentation](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html) for instructions. +2. **Create a private [Amazon Elastic Registry (ECR) repository](https://console.aws.amazon.com/ecr/repositories).** Refer to the [AWS ECR documentation](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html) for instructions. - After the repository has been created, navigate back to the list of [ECR repositories](https://console.aws.amazon.com/ecr/repositories). +After the repository has been created, navigate back to the list of [ECR repositories](https://console.aws.amazon.com/ecr/repositories). - In the list, locate the repository and its **URI**: +In the list, locate the repository and its **URI**: - ![Show this in the UI](/img/placeholder.svg) +![Show this in the UI](/img/placeholder.svg) - Keep this around, as you'll need it in a later step. +Keep this around, as you'll need it in a later step. - 3. [**Create an IAM user.**](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html) This user must: +3. [**Create an IAM user.**](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html) This user must: - - Have push access to the ECR repository, and - - Have programmatic access to AWS using an [access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) + - Have push access to the ECR repository, and + - Have programmatic access to AWS using an [access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) - After the user is created, save the **Access key ID** and **Secret access key** values shown on the confirmation page: + After the user is created, save the **Access key ID** and **Secret access key** values shown on the confirmation page: - ![Show this in the UI](/img/placeholder.svg) + ![Show this in the UI](/img/placeholder.svg) - 1. Set up a new Docker agent. Refer to the [Docker agent setup guide](/dagster-plus/deployment/deployment-types/hybrid/docker) for instructions. - 2. After the agent is set up, modify the `dagster.yaml` file as follows: +1. Set up a new Docker agent. Refer to the [Docker agent setup guide](/dagster-plus/deployment/deployment-types/hybrid/docker) for instructions. +2. After the agent is set up, modify the `dagster.yaml` file as follows: - - Set the `dagster_cloud_api.branch_deployments` field to `true` - - Remove any `deployment` field(s) + - Set the `dagster_cloud_api.branch_deployments` field to `true` + - Remove any `deployment` field(s) - For example: + For example: - 1. Set up a new Kubernetes agent. Refer to the [Kubernetes agent setup guide](/dagster-plus/deployment/deployment-types/hybrid/kubernetes) for instructions. +1. Set up a new Kubernetes agent. Refer to the [Kubernetes agent setup guide](/dagster-plus/deployment/deployment-types/hybrid/kubernetes) for instructions. - 2. After the agent is set up, modify your Helm values file to include the following: +2. After the agent is set up, modify your Helm values file to include the following: -## Step 4: Set up branch deployments +## Step 4: Set up branch deployments ### Step 4.1: Add GitHub CI/CD script to your project + :::note If you used the GitHub app to configure you're repository, this step isn't required. [Skip ahead to Step 4.5](#step-45-verify-github-action-runs) ::: @@ -156,7 +157,10 @@ In the `dagster_cloud.yaml` file, replace `build.registry` with the registry use For example: - + ### Step 4.3: Configure GitHub Action secrets @@ -260,12 +264,14 @@ In the next step, you'll modify these files to work with your Dagster+ setup. If you used the GitLab app to configure you're repository, this step isn't required. [Skip ahead to Step 4.5](#step-45-verify-gitlab-pipeline-runs) ::: - In the `dagster_cloud.yaml` file, replace `build.registry` with the registry used by the [agent you created in Step 2](#step-2-generate-a-dagster-agent-token). For example: - + ### Step 4.3: configure GitLab CI/CD variables @@ -273,7 +279,6 @@ For example: If you used the GitLab app to configure you're repository, this step isn't required. [Skip ahead to Step 4.5](#step-45-verify-gitlab-pipeline-runs) ::: - 1. In your project, click the **Settings** tab. 2. In the **CI/CD** section of the sidebar, expand **Variables**. 3. Click **Add variable**. @@ -371,17 +376,17 @@ The following examples assume the registry URL and image tag are stored in the ` Execute the following command within your CI/CD process: - ```shell - BRANCH_DEPLOYMENT_NAME=$( - dagster-cloud branch-deployment create-or-update \ - --organization $ORGANIZATION_NAME \ - --api-token $DAGSTER_CLOUD_API_TOKEN \ # Agent token from Step 1 - --git-repo-name $REPOSITORY_NAME \ # Git repository name - --branch-name $BRANCH_NAME \ # Git branch name - --commit-hash $COMMIT_SHA \ # Latest commit SHA on the branch - --timestamp $TIMESTAMP # UTC unixtime timestamp of the latest commit - ) - ``` +```shell +BRANCH_DEPLOYMENT_NAME=$( + dagster-cloud branch-deployment create-or-update \ + --organization $ORGANIZATION_NAME \ + --api-token $DAGSTER_CLOUD_API_TOKEN \ # Agent token from Step 1 + --git-repo-name $REPOSITORY_NAME \ # Git repository name + --branch-name $BRANCH_NAME \ # Git branch name + --commit-hash $COMMIT_SHA \ # Latest commit SHA on the branch + --timestamp $TIMESTAMP # UTC unixtime timestamp of the latest commit +) +``` One or more additional parameters can optionally be supplied to the `create-or-update` command to enhance the Branch Deployments UI in Dagster+: diff --git a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/testing.md b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/testing.md index 8106ced129daa..c86ae62b328ef 100644 --- a/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/testing.md +++ b/docs/docs-beta/docs/dagster-plus/features/ci-cd/branch-deployments/testing.md @@ -4,4 +4,4 @@ sidebar_position: 400 unlisted: true --- -{/* TODO move from https://docs.dagster.io/guides/dagster/branch_deployments#testing-against-production-with-dagster-branch-deployments */} \ No newline at end of file +{/** TODO move from https://docs.dagster.io/guides/dagster/branch_deployments#testing-against-production-with-dagster-branch-deployments **/} diff --git a/docs/docs-beta/docs/dagster-plus/features/ci-cd/ci-cd-file-reference.md b/docs/docs-beta/docs/dagster-plus/features/ci-cd/ci-cd-file-reference.md index 21c585c75361c..0b890becf65f0 100644 --- a/docs/docs-beta/docs/dagster-plus/features/ci-cd/ci-cd-file-reference.md +++ b/docs/docs-beta/docs/dagster-plus/features/ci-cd/ci-cd-file-reference.md @@ -3,4 +3,4 @@ title: CI/CD file reference unlisted: true --- -{/* TODO copy from https://docs.dagster.io/dagster-plus/references/ci-cd-file-reference */} \ No newline at end of file +{/** TODO copy from https://docs.dagster.io/dagster-plus/references/ci-cd-file-reference **/} diff --git a/docs/docs-beta/docs/dagster-plus/features/ci-cd/index.md b/docs/docs-beta/docs/dagster-plus/features/ci-cd/index.md index 27f8c4ab52dba..ce3ca8cf085c7 100644 --- a/docs/docs-beta/docs/dagster-plus/features/ci-cd/index.md +++ b/docs/docs-beta/docs/dagster-plus/features/ci-cd/index.md @@ -1,8 +1,8 @@ --- -title: "CI/CD" +title: 'CI/CD' sidebar_position: 30 --- import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/dagster-plus/features/data-catalog.md b/docs/docs-beta/docs/dagster-plus/features/data-catalog.md index bed4e9bfef93e..baf2b1b9f22fd 100644 --- a/docs/docs-beta/docs/dagster-plus/features/data-catalog.md +++ b/docs/docs-beta/docs/dagster-plus/features/data-catalog.md @@ -1,6 +1,5 @@ --- -title: "Data catalog" +title: 'Data catalog' sidebar_position: 200 unlisted: true --- - diff --git a/docs/docs-beta/docs/dagster-plus/features/insights/asset-metadata.md b/docs/docs-beta/docs/dagster-plus/features/insights/asset-metadata.md index ba05db163f29d..079e544c44ee1 100644 --- a/docs/docs-beta/docs/dagster-plus/features/insights/asset-metadata.md +++ b/docs/docs-beta/docs/dagster-plus/features/insights/asset-metadata.md @@ -1,6 +1,6 @@ --- -title: "Integrate asset metadata into Dagster+ Insights" -sidebar_label: "Integrate asset metadata" +title: 'Integrate asset metadata into Dagster+ Insights' +sidebar_label: 'Integrate asset metadata' sidebar_position: 100 --- @@ -31,12 +31,12 @@ Once your assets are emitting numeric metadata values, you'll be able to enable To add your metadata key to the list of metrics shown in Insights, click **Edit** in the sidebar next to the **User provided metrics** header: ![Viewing the Insights tab in the Dagster+ UI](/img/placeholder.svg) -{/* Viewing the Insights tab in the Dagster UI - + - + @@ -55,10 +61,16 @@ First, add a `.with_insights()` call to your `dbt.cli()` command(s). - + - + @@ -66,10 +78,16 @@ Then, add the following to your `dbt_project.yml`: - + - + diff --git a/docs/docs-beta/docs/dagster-plus/features/insights/snowflake.md b/docs/docs-beta/docs/dagster-plus/features/insights/snowflake.md index ae4af2b9f1777..401325a935f47 100644 --- a/docs/docs-beta/docs/dagster-plus/features/insights/snowflake.md +++ b/docs/docs-beta/docs/dagster-plus/features/insights/snowflake.md @@ -1,6 +1,6 @@ --- -title: "Track Snowflake usage with Dagster+ Insights" -sidebar_label: "Snowflake" +title: 'Track Snowflake usage with Dagster+ Insights' +sidebar_label: 'Snowflake' sidebar_position: 300 --- @@ -42,10 +42,16 @@ Only use `create_snowflake_insights_asset_and_schedule` in a single code locatio - + - + @@ -63,10 +69,16 @@ Only use `create_snowflake_insights_asset_and_schedule` in a single code locatio - + - + diff --git a/docs/docs-beta/docs/dagster-plus/getting-started.md b/docs/docs-beta/docs/dagster-plus/getting-started.md index fb4214b62e4ce..7459e707d5906 100644 --- a/docs/docs-beta/docs/dagster-plus/getting-started.md +++ b/docs/docs-beta/docs/dagster-plus/getting-started.md @@ -1,8 +1,9 @@ --- -title: "Getting started with Dagster+" +title: 'Getting started with Dagster+' --- First [create a Dagster+ organization](https://dagster.plus/signup). Note: you can sign up with: + - a Google email address - a GitHub account - a one-time email link, great if you are using a corporate email. You can setup SSO after completing these steps. @@ -23,6 +24,7 @@ We recommend following the steps in Dagster+ to add a new project. ![Screenshot of Dagster+ serverless NUX](/img/placeholder.svg) The Dagster+ on-boarding will guide you through: + - creating a Git repository containing your Dagster code - setting up the necessary CI/CD actions to deploy that repository to Dagster+ @@ -31,6 +33,7 @@ If you don't have any Dagster code yet, you will have the option to select an ex ::: See the guide on [adding code locations](/dagster-plus/deployment/code-locations) for details. + @@ -39,10 +42,10 @@ See the guide on [adding code locations](/dagster-plus/deployment/code-locations Follow [these guides](/dagster-plus/deployment/deployment-types/hybrid) for installing a Dagster+ Hybrid agent. Not sure which agent to pick? We recommend using the Dagster+ Kubernetes agent in most cases. - ## Setup CI/CD In most cases, your CI/CD process will be responsible for: + - building your Dagster code into a Docker image - pushing your Docker image to a container registry you manage - notifying Dagster+ of the new or updated code @@ -52,7 +55,6 @@ Refer to the guide for [adding a code location](/dagster-plus/deployment/code-lo - ## Next steps Your Dagster+ account is automatically enrolled in a trial. You can [pick your plan type and enter your billing information](/dagster-plus/deployment/management/settings/dagster-plus-settings), or [contact the Dagster team](https://dagster.io/contact) if you need support or want to evaluate the Dagster+ Pro plan. diff --git a/docs/docs-beta/docs/dagster-plus/index.md b/docs/docs-beta/docs/dagster-plus/index.md index fa8e16a0df269..01b602d6b0e5c 100644 --- a/docs/docs-beta/docs/dagster-plus/index.md +++ b/docs/docs-beta/docs/dagster-plus/index.md @@ -1,5 +1,5 @@ --- -title: "About Dagster+" +title: 'About Dagster+' sidebar_position: 1 --- diff --git a/docs/docs-beta/docs/getting-started/glossary.md b/docs/docs-beta/docs/getting-started/glossary.md index 8d0b5a8038b65..e9bc05088e751 100644 --- a/docs/docs-beta/docs/getting-started/glossary.md +++ b/docs/docs-beta/docs/getting-started/glossary.md @@ -9,11 +9,11 @@ unlisted: true TODO - link to conceptual content about the following (and other key concepts if needed): -* Assets -* Definitions -* Partitions (and backfills) -* Resources -* Schedules -* Sensors -* I/O managers? -* Ops and jobs (and graphs?) +- Assets +- Definitions +- Partitions (and backfills) +- Resources +- Schedules +- Sensors +- I/O managers? +- Ops and jobs (and graphs?) diff --git a/docs/docs-beta/docs/getting-started/installation.md b/docs/docs-beta/docs/getting-started/installation.md index e35319282fb79..5c6b68df5398a 100644 --- a/docs/docs-beta/docs/getting-started/installation.md +++ b/docs/docs-beta/docs/getting-started/installation.md @@ -19,18 +19,12 @@ After installing Python, it's recommended that you set up a virtual environment. There are many ways to do this, but this guide will use `venv` as it doesn't require additional dependencies. - -```bash -python -m venv venv -source venv/bin/activate -``` - - -```bash -python -m venv venv -source venv\Scripts\activate -``` - + + ```bash python -m venv venv source venv/bin/activate ``` + + + ```bash python -m venv venv source venv\Scripts\activate ``` + :::tip diff --git a/docs/docs-beta/docs/getting-started/quickstart.md b/docs/docs-beta/docs/getting-started/quickstart.md index afb15e582ed45..fdc2bf8ce749d 100644 --- a/docs/docs-beta/docs/getting-started/quickstart.md +++ b/docs/docs-beta/docs/getting-started/quickstart.md @@ -1,8 +1,8 @@ --- -title: "Dagster quickstart" +title: 'Dagster quickstart' description: Learn how to quickly get up and running with Dagster sidebar_position: 30 -sidebar_label: "Quickstart" +sidebar_label: 'Quickstart' --- # Build your first Dagster project @@ -28,6 +28,7 @@ To follow the steps in this guide, you'll need: - Basic Python knowledge - Python 3.9+ installed on your system. Refer to the [Installation guide](/getting-started/installation) for information. + ## Step 1: Set up the Dagster environment @@ -42,18 +43,12 @@ To follow the steps in this guide, you'll need: 2. Create and activate a virtual environment: - - ```bash - python -m venv venv - source venv/bin/activate - ``` - - - ```bash - python -m venv venv - source venv\Scripts\activate - ``` - + + ```bash python -m venv venv source venv/bin/activate ``` + + + ```bash python -m venv venv source venv\Scripts\activate ``` + 3. Install Dagster and the required dependencies: diff --git a/docs/docs-beta/docs/guides/automate/asset-sensors.md b/docs/docs-beta/docs/guides/automate/asset-sensors.md index 2c71bdaca5bd3..6f395879e1b37 100644 --- a/docs/docs-beta/docs/guides/automate/asset-sensors.md +++ b/docs/docs-beta/docs/guides/automate/asset-sensors.md @@ -81,7 +81,7 @@ stateDiagram-v2 In the following example, the `@asset_sensor` decorator defines a custom evaluation function that returns a `RunRequest` object when the asset is materialized and certain metadata is present, otherwise it skips the run. - + ## Trigger a job with configuration diff --git a/docs/docs-beta/docs/guides/automate/declarative-automation.md b/docs/docs-beta/docs/guides/automate/declarative-automation.md index 9a26ff8d2a427..539570724010f 100644 --- a/docs/docs-beta/docs/guides/automate/declarative-automation.md +++ b/docs/docs-beta/docs/guides/automate/declarative-automation.md @@ -1,6 +1,6 @@ --- -title: "Declarative automation" -sidebar_label: "Declarative automation" +title: 'Declarative automation' +sidebar_label: 'Declarative automation' unlisted: true sidebar_position: 400 --- diff --git a/docs/docs-beta/docs/guides/automate/index.md b/docs/docs-beta/docs/guides/automate/index.md index a10399054e850..b8b92ddb1966f 100644 --- a/docs/docs-beta/docs/guides/automate/index.md +++ b/docs/docs-beta/docs/guides/automate/index.md @@ -1,5 +1,5 @@ --- -title: "Automating pipelines" +title: 'Automating pipelines' description: Learn how to automate your data pipelines. sidebar_position: 40 sidebar_class_name: hidden @@ -92,8 +92,7 @@ For more examples of how to create asset sensors, see the [How-To Use Asset Sens ## Declarative Automation -{/* TODO: add content */} - +{/** TODO: add content **/} ## GraphQL Endpoint @@ -230,8 +229,8 @@ response = requests.post( ## Next steps -- Learn more about [advanced scheduling patterns] - {/* TODO ADD LINK */} -- Explore [complex sensor examples] - {/* TODO ADD LINK */} -- Dive into [Declarative Automation best practices] - {/* TODO ADD LINK */} +- Learn more about [advanced scheduling patterns] - {/** TODO ADD LINK **/} +- Explore [complex sensor examples] - {/** TODO ADD LINK **/} +- Dive into [Declarative Automation best practices] - {/** TODO ADD LINK **/} By understanding and effectively using these automation methods, you can build more efficient data pipelines that respond to your specific needs and constraints. diff --git a/docs/docs-beta/docs/guides/automate/schedules.md b/docs/docs-beta/docs/guides/automate/schedules.md index 3d749758ecb87..0bd62c7a877e5 100644 --- a/docs/docs-beta/docs/guides/automate/schedules.md +++ b/docs/docs-beta/docs/guides/automate/schedules.md @@ -1,6 +1,6 @@ --- -title: "Schedule cron-based pipelines" -sidebar_label: "Schedules" +title: 'Schedule cron-based pipelines' +sidebar_label: 'Schedules' sidebar_position: 100 --- @@ -13,6 +13,7 @@ To follow the steps in this guide, you'll need: - Familiarity with [Assets](/guides/build/assets-concepts/index.mdx - Familiarity with [Ops and Jobs](/guides/build/ops-jobs/index.md) + ## Basic schedule diff --git a/docs/docs-beta/docs/guides/automate/sensors.md b/docs/docs-beta/docs/guides/automate/sensors.md index 71be08e940220..4456fccc8b315 100644 --- a/docs/docs-beta/docs/guides/automate/sensors.md +++ b/docs/docs-beta/docs/guides/automate/sensors.md @@ -11,7 +11,7 @@ An alternative to polling with sensors is to push events to Dagster using the [D :::
-Prerequisites + Prerequisites To follow the steps in this guide, you'll need: diff --git a/docs/docs-beta/docs/guides/deploy/code-locations/index.mdx b/docs/docs-beta/docs/guides/deploy/code-locations/index.mdx index 7951c9ba3dbdf..1f3ccd5240edc 100644 --- a/docs/docs-beta/docs/guides/deploy/code-locations/index.mdx +++ b/docs/docs-beta/docs/guides/deploy/code-locations/index.mdx @@ -5,4 +5,4 @@ sidebar_position: 30 import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/guides/deploy/code-locations/managing-code-locations.md b/docs/docs-beta/docs/guides/deploy/code-locations/managing-code-locations.md index b44a57a85cad4..89ed07aef39fb 100644 --- a/docs/docs-beta/docs/guides/deploy/code-locations/managing-code-locations.md +++ b/docs/docs-beta/docs/guides/deploy/code-locations/managing-code-locations.md @@ -1,5 +1,5 @@ --- -title: "Managing code locations with Definitions" +title: 'Managing code locations with Definitions' unlisted: true --- diff --git a/docs/docs-beta/docs/guides/deploy/code-locations/workspace-yaml.md b/docs/docs-beta/docs/guides/deploy/code-locations/workspace-yaml.md index 4ee95d7f1a0eb..3c51fd9df7ecc 100644 --- a/docs/docs-beta/docs/guides/deploy/code-locations/workspace-yaml.md +++ b/docs/docs-beta/docs/guides/deploy/code-locations/workspace-yaml.md @@ -1,11 +1,11 @@ --- -title: "workspace.yaml reference" +title: 'workspace.yaml reference' --- # Workspace file reference :::info - This reference is only applicable to Dagster OSS. For Dagster Cloud see [the Dagster Cloud Code Locations guide](/dagster-plus/deployment/code-locations) +This reference is only applicable to Dagster OSS. For Dagster Cloud see [the Dagster Cloud Code Locations guide](/dagster-plus/deployment/code-locations) ::: The `workspace.yaml` file is used to configure code locations in Dagster. It tells Dagster where to find your code and how to load it. @@ -26,17 +26,15 @@ The `workspace.yaml` file uses the following structure: ```yaml load_from: - - : - + - : ``` Where `` can be one of: + - `python_file` - `python_module` - `grpc_server` - - ## Loading methods We recommend using `python_module` for most use cases. @@ -46,22 +44,24 @@ We recommend using `python_module` for most use cases. Loads a code location from a Python module. **Options:** + - `module_name`: Name of the Python module to load. - Other options are the same as `python_file`. **Example:** + ```yaml load_from: - python_module: module_name: hello_world_module.definitions ``` - ### Python file Loads a code location from a Python file. **Options:** + - `relative_path`: Path to the Python file, relative to the `workspace.yaml` location. - `attribute` (optional): Name of a specific repository or function returning a `RepositoryDefinition`. - `working_directory` (optional): Custom working directory for relative imports. @@ -69,6 +69,7 @@ Loads a code location from a Python file. - `location_name` (optional): Custom name for the code location. **Example:** + ```yaml load_from: - python_file: @@ -84,17 +85,19 @@ load_from: Configures a gRPC server as a code location. **Options:** + - `host`: The host address of the gRPC server. - `port`: The port number of the gRPC server. - `location_name`: Custom name for the code location. **Example:** + ```yaml load_from: - grpc_server: host: localhost port: 4266 - location_name: "my_grpc_server" + location_name: 'my_grpc_server' ``` ## Multiple code locations @@ -117,4 +120,4 @@ load_from: - Each code location is loaded in its own process. - Code locations can mix and match between `Definitions` objects and `@repository` decorators. -- If a code location is renamed or its configuration is modified, running schedules and sensors in that location need to be stopped and restarted. \ No newline at end of file +- If a code location is renamed or its configuration is modified, running schedules and sensors in that location need to be stopped and restarted. diff --git a/docs/docs-beta/docs/guides/deploy/dagster-yaml.md b/docs/docs-beta/docs/guides/deploy/dagster-yaml.md index d7f628b9f533a..cadad8166b068 100644 --- a/docs/docs-beta/docs/guides/deploy/dagster-yaml.md +++ b/docs/docs-beta/docs/guides/deploy/dagster-yaml.md @@ -1,7 +1,9 @@ --- -title: "dagster.yaml reference" +title: 'dagster.yaml reference' sidebar_position: 200 ----- +--- + +- # dagster.yaml reference diff --git a/docs/docs-beta/docs/guides/deploy/deployment-options/amazon-web-services.md b/docs/docs-beta/docs/guides/deploy/deployment-options/amazon-web-services.md index d425e912d4cd3..642cf6eaee7ac 100644 --- a/docs/docs-beta/docs/guides/deploy/deployment-options/amazon-web-services.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/amazon-web-services.md @@ -1,5 +1,5 @@ --- -title: "Deploy to Amazon Web Services" +title: 'Deploy to Amazon Web Services' unlisted: true sidebar_position: 500 --- diff --git a/docs/docs-beta/docs/guides/deploy/deployment-options/docker.md b/docs/docs-beta/docs/guides/deploy/deployment-options/docker.md index 4247a956771c8..eaa96869e5392 100644 --- a/docs/docs-beta/docs/guides/deploy/deployment-options/docker.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/docker.md @@ -1,5 +1,5 @@ --- -title: "Deploying with Docker Compose" +title: 'Deploying with Docker Compose' description: A guide to deploying Dagster with Docker Compose. sidebar_position: 400 --- @@ -7,13 +7,11 @@ sidebar_position: 400 This guide provides instructions for deploying Dagster using Docker Compose. This is useful when you want to, for example, deploy Dagster on an AWS EC2 host. A typical Dagster Docker deployment includes a several long-running containers: one for the webserver, one for the daemon, and one for each code location. It also typically executes each run in its own container.
- Prerequisites -- Familiarity with Docker and Docker Compose -- Familiarity with `dagster.yaml` instance configuration -- Familiarity with `workspace.yaml` code location configuration + Prerequisites- Familiarity with Docker and Docker Compose - Familiarity with + `dagster.yaml` instance configuration - Familiarity with `workspace.yaml` code location + configuration
- ## Define a Docker image for the Dagster webserver and daemon The Dagster webserver and daemon are the two _host processes_ in a Dagster deployment. They typically each run in their own container, using the same Docker image. This image contains Dagster packages and configuration, but no user code. @@ -41,6 +39,7 @@ WORKDIR $DAGSTER_HOME ``` Additionally, the following files should be in the same directory as the Docker file: + - A `workspace.yaml` to tell the webserver and daemon the location of the code servers - A `dagster.yaml` to configure the Dagster instance @@ -74,7 +73,7 @@ CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-f", "de The following `docker-compose.yaml` defines how to run the webserver container, daemon container, code location containers, and database container: ```yaml title="docker-compose.yaml" -version: "3.7" +version: '3.7' services: # This service runs the postgres DB used by dagster for run storage, schedule storage, @@ -83,9 +82,9 @@ services: image: postgres:11 container_name: docker_postgresql environment: - POSTGRES_USER: "postgres_user" - POSTGRES_PASSWORD: "postgres_password" - POSTGRES_DB: "postgres_db" + POSTGRES_USER: 'postgres_user' + POSTGRES_PASSWORD: 'postgres_password' + POSTGRES_DB: 'postgres_db' networks: - docker_network @@ -98,10 +97,10 @@ services: image: docker_user_code_image restart: always environment: - DAGSTER_POSTGRES_USER: "postgres_user" - DAGSTER_POSTGRES_PASSWORD: "postgres_password" - DAGSTER_POSTGRES_DB: "postgres_db" - DAGSTER_CURRENT_IMAGE: "docker_user_code_image" + DAGSTER_POSTGRES_USER: 'postgres_user' + DAGSTER_POSTGRES_PASSWORD: 'postgres_password' + DAGSTER_POSTGRES_DB: 'postgres_db' + DAGSTER_CURRENT_IMAGE: 'docker_user_code_image' networks: - docker_network @@ -113,20 +112,20 @@ services: entrypoint: - dagster-webserver - -h - - "0.0.0.0" + - '0.0.0.0' - -p - - "3000" + - '3000' - -w - workspace.yaml container_name: docker_webserver expose: - - "3000" + - '3000' ports: - - "3000:3000" + - '3000:3000' environment: - DAGSTER_POSTGRES_USER: "postgres_user" - DAGSTER_POSTGRES_PASSWORD: "postgres_password" - DAGSTER_POSTGRES_DB: "postgres_db" + DAGSTER_POSTGRES_USER: 'postgres_user' + DAGSTER_POSTGRES_PASSWORD: 'postgres_password' + DAGSTER_POSTGRES_DB: 'postgres_db' volumes: # Make docker client accessible so we can terminate containers from the webserver - /var/run/docker.sock:/var/run/docker.sock - /tmp/io_manager_storage:/tmp/io_manager_storage @@ -148,9 +147,9 @@ services: container_name: docker_daemon restart: on-failure environment: - DAGSTER_POSTGRES_USER: "postgres_user" - DAGSTER_POSTGRES_PASSWORD: "postgres_password" - DAGSTER_POSTGRES_DB: "postgres_db" + DAGSTER_POSTGRES_USER: 'postgres_user' + DAGSTER_POSTGRES_PASSWORD: 'postgres_password' + DAGSTER_POSTGRES_DB: 'postgres_db' volumes: # Make docker client accessible so we can launch containers using host docker - /var/run/docker.sock:/var/run/docker.sock - /tmp/io_manager_storage:/tmp/io_manager_storage diff --git a/docs/docs-beta/docs/guides/deploy/deployment-options/google-cloud-platform.md b/docs/docs-beta/docs/guides/deploy/deployment-options/google-cloud-platform.md index 0cdd2376cb6b5..4306dc44896a1 100644 --- a/docs/docs-beta/docs/guides/deploy/deployment-options/google-cloud-platform.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/google-cloud-platform.md @@ -1,5 +1,5 @@ --- -title: "Deploy to Google Cloud Platform" +title: 'Deploy to Google Cloud Platform' unlisted: true sidebar_position: 600 --- diff --git a/docs/docs-beta/docs/guides/deploy/deployment-options/index.mdx b/docs/docs-beta/docs/guides/deploy/deployment-options/index.mdx index 757020a9b3852..015c6d12db7a5 100644 --- a/docs/docs-beta/docs/guides/deploy/deployment-options/index.mdx +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/index.mdx @@ -1,8 +1,8 @@ --- -title: "Deployment options" +title: 'Deployment options' sidebar_position: 10 --- import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/guides/deploy/deployment-options/kubernetes.md b/docs/docs-beta/docs/guides/deploy/deployment-options/kubernetes.md index f29729c61072b..d699c582705aa 100644 --- a/docs/docs-beta/docs/guides/deploy/deployment-options/kubernetes.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/kubernetes.md @@ -1,5 +1,5 @@ --- -title: "Deploy to Kubernetes" +title: 'Deploy to Kubernetes' sidebar_position: 300 --- @@ -27,16 +27,21 @@ To follow the steps in this guide, you'll need:
- ## Step 1: Write and build a Docker image containing your Dagster project + ### Step 1.1: Write a Dockerfile + Next, you'll build a Docker image that contains your Dagster project and all of its dependencies. The Dockerfile should: + 1. Copy your Dagster project into the image. 2. Install `dagster`, `dagster-postgres`, and `dagster-k8s`, along with any other libraries your project depends on. The example project has a dependency on `pandas` so it's included in the `pip install` in the following example Dockerfile. 3. Expose port 80, which we'll use to set up port-forwarding later. - - + ### Step 1.2: Build and push a Docker image @@ -45,8 +50,8 @@ To build your Docker image, run the following command from the directory where y ```bash docker build . -t iris_analysis:1 ``` -This builds the Docker image from Step 1.1 and gives it the name `iris_analysis` and tag `1`. You can set custom values for both the name and the tag. We recommend that each time you rebuild your Docker image, you assign a new value for the tag to ensure that the correct image is used when running your code. +This builds the Docker image from Step 1.1 and gives it the name `iris_analysis` and tag `1`. You can set custom values for both the name and the tag. We recommend that each time you rebuild your Docker image, you assign a new value for the tag to ensure that the correct image is used when running your code. If you are using a Docker image registry, push the image to your registry. If you are following along on your local machine, you can skip this command. @@ -55,23 +60,28 @@ docker push iris_analysis:1 ``` If you are pushing your image to an image registry, you can find more information about this process in your registry's documentation: + - [Amazon ECR](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html) - [DockerHub](https://docs.docker.com/docker-hub/quickstart/#step-5-build-and-push-a-container-image-to-docker-hub-from-your-computer) - ## Step 2: Configure `kubectl` to point at a Kubernetes cluster + Before you can deploy Dagster, you need to configure `kubectl` to develop against the Kubernetes cluster where you want Dagster to be deployed. If you are using Docker Desktop and the included Kubernetes server, you will need to create a context first. If you already have a Kubernetes cluster and context created for your Dagster deployment you can skip running this command. + ```bash kubectl config set-context dagster --namespace default --cluster docker-desktop --user=docker-desktop ``` Ensure that `kubectl` is using the correct context by running: + ```bash kubectl config use-context ``` + Where `` is the name of the context you want to use. For example, if you ran the preceding `kubectl config set-context` command, you will run + ```bash kubectl config use-context dagster ``` @@ -105,21 +115,27 @@ helm show values dagster/dagster > values.yaml ``` ### Step 4.2: Modify the `values.yaml` file for your deployment + The `values.yaml` file contains configuration options you can set for your deployment. Different configuration options are explained in [inline comments in `values.yaml`](https://artifacthub.io/packages/helm/dagster/dagster?modal=values). To deploy your project, you'll need to set the following options: + - `dagster-user-deployments.deployments.name`, which should be a unique name for your deployment - `dagster-user-deployments.deployments.image.name` and `dagster-user-deployments.deployments.image.tag`, which should be set to match the Docker image from Step 1 - `dagster-user-deployments.deployments.dagsterApiGrpcArgs`, which should be set to the arguments you would pass to `dagster api grpc` to [run a gRPC server for your project](https://docs.dagster.io/concepts/code-locations/workspace-files#running-your-own-grpc-server). If you are following this guide on your local machine, you will also need to set `pullPolicy: IfNotPresent`. This will use the local version of the image built in Step 1. However, in production use cases when your Docker images are pushed to image registries, this value should remain `pullPolicy: Always`. - + In this example, the image `name` and `tag` are set to `iris_analysis` and `1` to match the image that was pushed in Step 1. To run the gPRC server, the path to the Dagster project needs to be specified, so `--python-file` and `/iris_analysis/definitions.py` are set for `dagsterApiGrpcArgs`. - ## Step 5: Install the Helm chart + Now that you have modified the Helm `values.yaml` file, you can install the changes in your Kubernetes cluster. Run the following command to install the Helm chart and create a [release](https://helm.sh/docs/intro/using_helm/#three-big-concepts). @@ -166,10 +182,10 @@ kubectl logs dagster-webserver-7c5b5c7f5c-rqrf8 - ## Step 6: Connect to your Dagster deployment and materialize your assets ### Step 6.1: Start port-forwarding to the webserver pod + Run the following command to set up port forwarding to the webserver pod: ```bash @@ -188,6 +204,7 @@ The webserver has been port-forwarded to `8080`, so you can visit the Dagster de ![Screenshot of Dagster landing page](/img/placeholder.svg) ### Step 6.3: Materialize an asset + In the Dagster UI, navigate to the Asset catalog and click the **Materialize** button to materialize an asset. Dagster will start a Kubernetes job to materialize the asset. You can introspect on the Kubernetes cluster to see this job: ```bash @@ -197,5 +214,6 @@ dagster-run-5ee8a0b3-7ca5-44e6-97a6-8f4bd86ee630 1/1 4s 11s ``` ## Next steps + - Forwarding Dagster logs from a Kubernetes deployment to AWS, Azure, GCP - Other configuration options for K8s deployment - secrets, diff --git a/docs/docs-beta/docs/guides/deploy/deployment-options/microsoft-azure.md b/docs/docs-beta/docs/guides/deploy/deployment-options/microsoft-azure.md index a33096ceb12c0..b08612619b56c 100644 --- a/docs/docs-beta/docs/guides/deploy/deployment-options/microsoft-azure.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/microsoft-azure.md @@ -1,5 +1,5 @@ --- -title: "Deploy to Microsoft Azure" +title: 'Deploy to Microsoft Azure' unlisted: true sidebar_position: 700 --- diff --git a/docs/docs-beta/docs/guides/deploy/deployment-options/running-local-ui-development.md b/docs/docs-beta/docs/guides/deploy/deployment-options/running-local-ui-development.md index f6e84c3fafe95..f06cd95d3a445 100644 --- a/docs/docs-beta/docs/guides/deploy/deployment-options/running-local-ui-development.md +++ b/docs/docs-beta/docs/guides/deploy/deployment-options/running-local-ui-development.md @@ -6,7 +6,7 @@ sidebar_position: 100 # Running Dagster locally -In this guide, we'll walk you through how to run Dagster on your local machine using the `dagster dev` command. The `dagster dev` command launches the Dagster UI and the Dagster daemon, allowing you to start a full deployment of Dagster from the command line. +In this guide, we'll walk you through how to run Dagster on your local machine using the `dagster dev` command. The `dagster dev` command launches the Dagster UI and the Dagster daemon, allowing you to start a full deployment of Dagster from the command line. :::warning `dagster dev` is intended for local development _only_. If you want to run Dagster for production use cases, see our other [deployment guides](/guides/deploy/deployment-options/index.mdx). @@ -32,7 +32,6 @@ For a refresher on how to set up a Dagster project, follow our [Recommended Dags To load definitions from a module without supplying the `-m` command line argument, you can use a `pyproject.toml` file. This file, included in all Dagster example projects, contains a `tool.dagster` section where you can supply the `module_name`: - Dagster can load a file directly as a code location. @@ -51,10 +50,12 @@ For a refresher on how to set up a Dagster project, follow our [Recommended Dags ## Creating a persistent instance -Running `dagster dev` without any additional configuration starts an ephemeral instance in a temporary directory. You may see log output indicating as such: +Running `dagster dev` without any additional configuration starts an ephemeral instance in a temporary directory. You may see log output indicating as such: + ```shell Using temporary directory /Users/rhendricks/tmpqs_fk8_5 for storage. ``` + This indicates that any runs or materialized assets created during your session won't be persisted once the session ends. To designate a more permanent home for your runs and assets, you can set the `DAGSTER_HOME` environment variable to a folder on your filesystem. Dagster will then use the specified folder for storage on all subsequent runs of `dagster dev`. @@ -70,8 +71,6 @@ dagster dev To configure your Dagster instance, you can create a `dagster.yaml` file in your `$DAGSTER_HOME` folder. For example, to have your local instance limit the number of concurrent runs, you could configure the following `dagster.yaml`: - - + For the full list of options that can be set in the `dagster.yaml` file, refer to the [Dagster instance documentation](/todo). - diff --git a/docs/docs-beta/docs/guides/deploy/execution/dagster-daemon.md b/docs/docs-beta/docs/guides/deploy/execution/dagster-daemon.md index 85da08fe450d3..0ce47fdc8e336 100644 --- a/docs/docs-beta/docs/guides/deploy/execution/dagster-daemon.md +++ b/docs/docs-beta/docs/guides/deploy/execution/dagster-daemon.md @@ -1,5 +1,5 @@ --- -title: "Dagster daemon" +title: 'Dagster daemon' sidebar_position: 100 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/deploy/execution/index.mdx b/docs/docs-beta/docs/guides/deploy/execution/index.mdx index c1bbbc2b8a5e0..8a8a4bd7d6a63 100644 --- a/docs/docs-beta/docs/guides/deploy/execution/index.mdx +++ b/docs/docs-beta/docs/guides/deploy/execution/index.mdx @@ -1,8 +1,8 @@ --- -title: "Execution" +title: 'Execution' sidebar_position: 20 --- import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/guides/deploy/execution/run-coordinators.md b/docs/docs-beta/docs/guides/deploy/execution/run-coordinators.md index 48ce80b1690cc..3d355f890c0d3 100644 --- a/docs/docs-beta/docs/guides/deploy/execution/run-coordinators.md +++ b/docs/docs-beta/docs/guides/deploy/execution/run-coordinators.md @@ -1,5 +1,5 @@ --- -title: "Run coordinators" +title: 'Run coordinators' sidebar_position: 400 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/deploy/execution/run-executors.md b/docs/docs-beta/docs/guides/deploy/execution/run-executors.md index 7351a18aefd9d..331f9e2f494ce 100644 --- a/docs/docs-beta/docs/guides/deploy/execution/run-executors.md +++ b/docs/docs-beta/docs/guides/deploy/execution/run-executors.md @@ -1,5 +1,5 @@ --- -title: "Run executors" +title: 'Run executors' sidebar_position: 300 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/deploy/execution/run-launchers.md b/docs/docs-beta/docs/guides/deploy/execution/run-launchers.md index f4982c170a6b3..b62e720e66228 100644 --- a/docs/docs-beta/docs/guides/deploy/execution/run-launchers.md +++ b/docs/docs-beta/docs/guides/deploy/execution/run-launchers.md @@ -1,5 +1,5 @@ --- -title: "Run launchers" +title: 'Run launchers' sidebar_position: 200 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/deploy/index.md b/docs/docs-beta/docs/guides/deploy/index.md index 3a5ec3fb353d1..37dd3d17d014a 100644 --- a/docs/docs-beta/docs/guides/deploy/index.md +++ b/docs/docs-beta/docs/guides/deploy/index.md @@ -1,5 +1,5 @@ --- -title: "Deploying Dagster" +title: 'Deploying Dagster' sidebar_position: 70 sidebar_class_name: hidden --- diff --git a/docs/docs-beta/docs/guides/index.md b/docs/docs-beta/docs/guides/index.md index cb467c230dadd..d5d3175562f0f 100644 --- a/docs/docs-beta/docs/guides/index.md +++ b/docs/docs-beta/docs/guides/index.md @@ -1,5 +1,5 @@ --- -title: "Guides" +title: 'Guides' unlisted: true --- diff --git a/docs/docs-beta/docs/guides/monitor/alerting.md b/docs/docs-beta/docs/guides/monitor/alerting.md index 92c409cce7b8a..25a73b5dbdd5f 100644 --- a/docs/docs-beta/docs/guides/monitor/alerting.md +++ b/docs/docs-beta/docs/guides/monitor/alerting.md @@ -1,5 +1,5 @@ --- -title: "Setting up alerts" +title: 'Setting up alerts' sidebar_position: 10 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/monitor/index.md b/docs/docs-beta/docs/guides/monitor/index.md index aaf06312205dc..b16b519405ed6 100644 --- a/docs/docs-beta/docs/guides/monitor/index.md +++ b/docs/docs-beta/docs/guides/monitor/index.md @@ -1,7 +1,7 @@ --- -title: "Monitoring pipelines" +title: 'Monitoring pipelines' sidebar_position: 50 sidebar_class_name: hidden --- -This section is about monitoring pipelines. \ No newline at end of file +This section is about monitoring pipelines. diff --git a/docs/docs-beta/docs/guides/monitor/logging/custom-logging.md b/docs/docs-beta/docs/guides/monitor/logging/custom-logging.md index c3bf0380c5621..6de5383b4ba74 100644 --- a/docs/docs-beta/docs/guides/monitor/logging/custom-logging.md +++ b/docs/docs-beta/docs/guides/monitor/logging/custom-logging.md @@ -1,5 +1,5 @@ --- -title: "Setting up custom logging" +title: 'Setting up custom logging' sidebar_position: 100 --- @@ -17,26 +17,30 @@ To follow the steps in this guide, you'll need: - ## Step 1: Add a prebuilt custom logger to your jobs This step shows how to add an existing custom logger, the `json_console_logger`, to your jobs. This will override the default `colored_console_logger` and produce logs in JSON format. - ### Add the custom logger to your asset jobs The following example shows how to add the custom logger to your code location definitions and configure an asset job to use it. - - + ### Add the custom logger to your ops-based jobs Configuring a ops job to use the custom logger slightly differs from the asset job example. The following example shows how: - - + ### Expected `json_console_logger` output @@ -103,7 +107,11 @@ loggers: In this example, we'll create a logger implementation that produces comma separated values from selected fields in the log record. Other examples can be found in the codebase, in the built-in loggers such as `json_console_logger`. - + Sample output: diff --git a/docs/docs-beta/docs/guides/monitor/logging/custom-metrics-logs.md b/docs/docs-beta/docs/guides/monitor/logging/custom-metrics-logs.md index 62476124e059c..91e4129c28f1b 100644 --- a/docs/docs-beta/docs/guides/monitor/logging/custom-metrics-logs.md +++ b/docs/docs-beta/docs/guides/monitor/logging/custom-metrics-logs.md @@ -1,5 +1,5 @@ --- -title: "Use custom metrics in logs" +title: 'Use custom metrics in logs' sidebar_position: 200 unlisted: true --- diff --git a/docs/docs-beta/docs/guides/monitor/logging/index.mdx b/docs/docs-beta/docs/guides/monitor/logging/index.mdx index 8de645e227aec..b6d7d0314bbf0 100644 --- a/docs/docs-beta/docs/guides/monitor/logging/index.mdx +++ b/docs/docs-beta/docs/guides/monitor/logging/index.mdx @@ -1,8 +1,8 @@ --- -title: "Logging" +title: 'Logging' sidebar_position: 20 --- import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/guides/test/asset-checks.md b/docs/docs-beta/docs/guides/test/asset-checks.md index b28bee6e46e3e..f9bae54a9395b 100644 --- a/docs/docs-beta/docs/guides/test/asset-checks.md +++ b/docs/docs-beta/docs/guides/test/asset-checks.md @@ -1,7 +1,7 @@ --- -title: "Testing assets with Asset checks" +title: 'Testing assets with Asset checks' sidebar_position: 100 -sidebar_label: "Asset checks" +sidebar_label: 'Asset checks' --- Asset checks are tests that verify specific properties of your data assets, allowing you to execute data quality checks on your data. For example, you can create checks to: @@ -18,6 +18,7 @@ Each asset check should test only a single asset property to keep tests uncompli To follow this guide, you'll need: - Familiarity with [Assets](/guides/build/assets-concepts/index.mdx + ## Getting started @@ -40,7 +41,10 @@ A asset check is defined using the `@asset_check` decorator. The following example defines an asset check on an asset that fails if the `order_id` column of the asset contains a null value. The asset check will run after the asset has been materialized. - + ## Defining multiple asset checks \{#multiple-checks} @@ -53,13 +57,19 @@ The following example defines two asset checks using the `@multi_asset_check` de In this example, both asset checks will run in a single operation after the asset has been materialized. - + ## Programmatically generating asset checks \{#factory-pattern} Defining multiple checks can also be done using a factory pattern. The example below defines the same two asset checks as in the previous example, but this time using a factory pattern and the `@multi_asset_check` decorator. - + ## Blocking downstream materialization @@ -67,7 +77,10 @@ By default, if a parent's asset check fails during a run, the run will continue In the example bellow, if the `orders_id_has_no_nulls` check fails, the downstream `augmented_orders` asset won't be materialized. - + ## Scheduling and monitoring asset checks @@ -75,7 +88,10 @@ In some cases, running asset checks separately from the job materializing the as In the example below, two jobs are defined: one for the asset and another for the asset check. Schedules are defined to materialize the asset and execute the asset check independently. A sensor is defined to send an email alert when the asset check job fails. - + ## Next steps diff --git a/docs/docs-beta/docs/guides/test/data-freshness-testing.md b/docs/docs-beta/docs/guides/test/data-freshness-testing.md index a738941c2d8f5..86cb0964fecae 100644 --- a/docs/docs-beta/docs/guides/test/data-freshness-testing.md +++ b/docs/docs-beta/docs/guides/test/data-freshness-testing.md @@ -1,7 +1,7 @@ --- -title: "Checking for data freshness" +title: 'Checking for data freshness' sidebar_position: 200 -sidebar_label: "Data freshness" +sidebar_label: 'Data freshness' --- Freshness checks provide a way to identify data assets that are overdue for an update. For example, you can use freshness checks to identify stale assets caused by: @@ -31,6 +31,7 @@ To get started with freshness checks, follow these general steps: 1. **Define a freshness check**: Freshness checks are defined using `build_last_update_freshness_checks`, which utilizes an asset's last updated time to determine freshness. **If using Dagster+ Pro**, you can also use [`build_anomaly_detection_freshness_checks`](#anomaly-detection) to define a freshness check that uses an anomaly detection model to determine freshness. + 2. **Define a schedule or sensor**: Defining a schedule or sensor (`build_sensor_for_freshness_checks`) is required to ensure the freshness check executes. If the check only runs after the asset has been materialized, the check won't be able to detect the times materialization fails. 3. **Pass the freshness check and schedule/sensor to the `Definitions` object**: Freshness checks and the associated schedule or sensor must be added to a `Definitions` object for Dagster to recognize them. 4. **View the freshness check results in the Dagster UI**: Freshness check results will appear in the UI, allowing you to track the results over time. @@ -41,7 +42,10 @@ Materializable assets are assets materialized by Dagster. To calculate whether a The example below defines a freshness check on an asset that fails if the asset's latest materialization occurred more than one hour before the current time. - + ## External asset freshness \{#external-assets} @@ -51,7 +55,10 @@ To run freshness checks on external assets, the checks need to know when the ext The example below defines a freshness check and adds a schedule to run the check periodically. - + ### Testing freshness with anomaly detection \{#anomaly-detection} @@ -61,7 +68,10 @@ Anomaly detection is a Dagster+ Pro feature. Instead of applying policies on an asset-by-asset basis, Dagster+ Pro users can use `build_anomaly_detection_freshness_checks` to take advantage of a time series anomaly detection model to determine if data arrives later than expected. - + :::note If the asset hasn't been updated enough times, the check will pass with a message indicating that more data is needed to detect anomalies. @@ -78,4 +88,4 @@ In Dagster+, you can set up alerts to notify you when assets are overdue for an ## Next steps - Explore more [asset checks](/guides/test/asset-checks) -- Explore how to [raise alerts when assets are overdue](/dagster-plus/features/alerts) (Dagster+ Pro) \ No newline at end of file +- Explore how to [raise alerts when assets are overdue](/dagster-plus/features/alerts) (Dagster+ Pro) diff --git a/docs/docs-beta/docs/guides/test/index.md b/docs/docs-beta/docs/guides/test/index.md index 6284fc0b252bc..e483dd89b3d63 100644 --- a/docs/docs-beta/docs/guides/test/index.md +++ b/docs/docs-beta/docs/guides/test/index.md @@ -1,7 +1,7 @@ --- -title: "Testing assets" +title: 'Testing assets' sidebar_position: 60 sidebar_class_name: hidden --- -This section is about testing assets. \ No newline at end of file +This section is about testing assets. diff --git a/docs/docs-beta/docs/guides/test/unit-tests-assets-and-ops.md b/docs/docs-beta/docs/guides/test/unit-tests-assets-and-ops.md index 057ef145945bd..e34eaf2c2e1ad 100644 --- a/docs/docs-beta/docs/guides/test/unit-tests-assets-and-ops.md +++ b/docs/docs-beta/docs/guides/test/unit-tests-assets-and-ops.md @@ -1,7 +1,7 @@ --- -title: "Unit testing assets and ops" +title: 'Unit testing assets and ops' sidebar_position: 300 -sidebar_label: "Unit testing" +sidebar_label: 'Unit testing' --- Unit testing is essential for ensuring that computations function as intended. In the context of data pipelines, this can be particularly challenging. However, Dagster streamlines the process by enabling direct invocation of computations with specified input values and mocked resources, making it easier to verify that data transformations behave as expected. @@ -11,7 +11,7 @@ While unit tests can't fully replace integration tests or manual review, they ca This guide covers how to write unit tests for assets and ops with a variety of different input requirements.
-Prerequisites + Prerequisites To follow the steps in this guide, you'll need familiarity with: @@ -32,10 +32,16 @@ The simplest assets and ops to test are those with no arguments. In these cases, - + - + @@ -45,10 +51,16 @@ If an asset or op has an upstream dependency, you can directly pass a value for - + - + @@ -58,10 +70,16 @@ If an asset or op uses config, you can construct an instance of the required con - + - + @@ -71,10 +89,16 @@ If an asset or op uses a resource, it can be useful to create a mock instance of - + - + @@ -84,10 +108,16 @@ If an asset or op uses a `context` argument, you can use `build_asset_context()` - + - + @@ -97,10 +127,16 @@ If an asset or op has multiple parameters, it's recommended to use keyword argum - + - + @@ -108,4 +144,4 @@ If an asset or op has multiple parameters, it's recommended to use keyword argum - Learn more about assets in [Understanding Assets](/guides/build/assets-concepts/index.mdx - Learn more about ops in [Understanding Ops](/guides/build/ops-jobs) -- Learn more about resources in [Resources](/guides/build/configure/resources) \ No newline at end of file +- Learn more about resources in [Resources](/guides/build/configure/resources) diff --git a/docs/docs-beta/docs/integrations/guides/multi-asset-integration.md b/docs/docs-beta/docs/integrations/guides/multi-asset-integration.md index df0615e217fab..265391de5d8c4 100644 --- a/docs/docs-beta/docs/integrations/guides/multi-asset-integration.md +++ b/docs/docs-beta/docs/integrations/guides/multi-asset-integration.md @@ -2,4 +2,4 @@ title: Creating a multi-asset integration --- -{/* TODO write this */} \ No newline at end of file +{/** TODO write this **/} diff --git a/docs/docs-beta/docs/integrations/libraries/airbyte.md b/docs/docs-beta/docs/integrations/libraries/airbyte.md index 36e160f2986d9..8fa04e0da1241 100644 --- a/docs/docs-beta/docs/integrations/libraries/airbyte.md +++ b/docs/docs-beta/docs/integrations/libraries/airbyte.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, etl] --- - - Using this integration, you can trigger Airbyte syncs and orchestrate your Airbyte connections from within Dagster, making it easy to chain an Airbyte sync with upstream or downstream steps in your workflow. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/aws/athena.md b/docs/docs-beta/docs/integrations/libraries/aws/athena.md index f28c6a5a8fae3..1711104e90bdc 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/athena.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/athena.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, storage] --- - - This integration allows you to connect to AWS Athena, a serverless interactive query service that makes it easy to analyze data in Amazon S3 using standard SQL. Using this integration, you can issue queries to Athena, fetch results, and handle query execution states within your Dagster pipelines. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/aws/cloudwatch.md b/docs/docs-beta/docs/integrations/libraries/aws/cloudwatch.md index 8f27e767cd512..216aa4ab0d3a9 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/cloudwatch.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/cloudwatch.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, monitoring] --- - - This integration allows you to send Dagster logs to AWS CloudWatch, enabling centralized logging and monitoring of your Dagster jobs. By using AWS CloudWatch, you can take advantage of its powerful log management features, such as real-time log monitoring, log retention policies, and alerting capabilities. Using this integration, you can configure your Dagster jobs to log directly to AWS CloudWatch, making it easier to track and debug your workflows. This is particularly useful for production environments where centralized logging is essential for maintaining observability and operational efficiency. diff --git a/docs/docs-beta/docs/integrations/libraries/aws/ecr.md b/docs/docs-beta/docs/integrations/libraries/aws/ecr.md index a3b2487794a0b..681fcee8c2a9f 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/ecr.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/ecr.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported] --- - - This integration allows you to connect to AWS Elastic Container Registry (ECR). It provides resources to interact with AWS ECR, enabling you to manage your container images. Using this integration, you can seamlessly integrate AWS ECR into your Dagster pipelines, making it easier to manage and deploy containerized applications. diff --git a/docs/docs-beta/docs/integrations/libraries/aws/emr.md b/docs/docs-beta/docs/integrations/libraries/aws/emr.md index db5ba4e4bc91d..cb708a7b39479 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/emr.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/emr.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, compute] --- - - The `dagster-aws` integration provides ways orchestrating data pipelines that leverage AWS services, including AWS EMR (Elastic MapReduce). This integration allows you to run and scale big data workloads using open source tools such as Apache Spark, Hive, Presto, and more. Using this integration, you can: diff --git a/docs/docs-beta/docs/integrations/libraries/aws/glue.md b/docs/docs-beta/docs/integrations/libraries/aws/glue.md index 629df3adc218d..e481e3b34a74f 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/glue.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/glue.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, compute] --- - - The `dagster-aws` integration library provides the `PipesGlueClient` resource, enabling you to launch AWS Glue jobs directly from Dagster assets and ops. This integration allows you to pass parameters to Glue code while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/aws/index.md b/docs/docs-beta/docs/integrations/libraries/aws/index.md index a95102a5e87c8..481931c7b81e2 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/index.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/index.md @@ -4,4 +4,4 @@ title: AWS import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/integrations/libraries/aws/lambda.md b/docs/docs-beta/docs/integrations/libraries/aws/lambda.md index 5ec9c7c9c64ad..48694fbf1c89b 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/lambda.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/lambda.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, compute] --- - - Using this integration, you can leverage AWS Lambda to execute external code as part of your Dagster pipelines. This is particularly useful for running serverless functions that can scale automatically and handle various workloads without the need for managing infrastructure. The `PipesLambdaClient` class allows you to invoke AWS Lambda functions and stream logs and structured metadata back to Dagster's UI and tools. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/aws/redshift.md b/docs/docs-beta/docs/integrations/libraries/aws/redshift.md index 593516db98136..66aabaa85817b 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/redshift.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/redshift.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, storage] --- - - Using this integration, you can connect to an AWS Redshift cluster and issue queries against it directly from your Dagster assets. This allows you to seamlessly integrate Redshift into your data pipelines, leveraging the power of Redshift's data warehousing capabilities within your Dagster workflows. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/aws/s3.md b/docs/docs-beta/docs/integrations/libraries/aws/s3.md index 71e35378e38b1..c4ee5e8362904 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/s3.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/s3.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, storage] --- - - The AWS S3 integration allows data engineers to easily read, and write objects to the durable AWS S3 storage -- enabling engineers to a resilient storage layer when constructing their pipelines. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/aws/secretsmanager.md b/docs/docs-beta/docs/integrations/libraries/aws/secretsmanager.md index 48b3b007bf1cd..bc148eab42f64 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/secretsmanager.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/secretsmanager.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported] --- - - This integration allows you to manage, retrieve, and rotate credentials, API keys, and other secrets using [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/). ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/aws/ssm.md b/docs/docs-beta/docs/integrations/libraries/aws/ssm.md index d0da33d8d7e2f..6537e73001a4c 100644 --- a/docs/docs-beta/docs/integrations/libraries/aws/ssm.md +++ b/docs/docs-beta/docs/integrations/libraries/aws/ssm.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported] --- - - The Dagster AWS Systems Manager (SSM) Parameter Store integration allows you to manage and retrieve parameters stored in AWS SSM Parameter Store directly within your Dagster pipelines. This integration provides resources to fetch parameters by name, tags, or paths, and optionally set them as environment variables for your operations. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/azure-adls2.md b/docs/docs-beta/docs/integrations/libraries/azure-adls2.md index 780b800aa62dc..c036cf868454e 100644 --- a/docs/docs-beta/docs/integrations/libraries/azure-adls2.md +++ b/docs/docs-beta/docs/integrations/libraries/azure-adls2.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, storage] --- - - Dagster helps you use Azure Storage Accounts as part of your data pipeline. Azure Data Lake Storage Gen 2 (ADLS2) is our primary focus but we also provide utilities for Azure Blob Storage. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/census.md b/docs/docs-beta/docs/integrations/libraries/census.md index 1c3e4f8e2c903..d2d269cfa5efa 100644 --- a/docs/docs-beta/docs/integrations/libraries/census.md +++ b/docs/docs-beta/docs/integrations/libraries/census.md @@ -17,8 +17,6 @@ enables: tags: [community-supported, etl] --- - - With the `dagster-census` integration you can execute a Census sync and poll until that sync completes, raising an error if it's unsuccessful. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/cube.md b/docs/docs-beta/docs/integrations/libraries/cube.md index e1e976090adf0..63c595db74584 100644 --- a/docs/docs-beta/docs/integrations/libraries/cube.md +++ b/docs/docs-beta/docs/integrations/libraries/cube.md @@ -17,8 +17,6 @@ enables: tags: [community-supported] --- - - With the `dagster_cube` integration you can setup Cube and Dagster to work together so that Dagster can push changes from upstream data sources to Cube using its integration API. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/databricks.md b/docs/docs-beta/docs/integrations/libraries/databricks.md index ff4736c3c0c4e..033dcf6cf2416 100644 --- a/docs/docs-beta/docs/integrations/libraries/databricks.md +++ b/docs/docs-beta/docs/integrations/libraries/databricks.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, compute] --- - - The `dagster-databricks` integration library provides the `PipesDatabricksClient` resource, enabling you to launch Databricks jobs directly from Dagster assets and ops. This integration allows you to pass parameters to Databricks code while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/datadog.md b/docs/docs-beta/docs/integrations/libraries/datadog.md index 4f2e867eb2fd7..33f45a5b10f39 100644 --- a/docs/docs-beta/docs/integrations/libraries/datadog.md +++ b/docs/docs-beta/docs/integrations/libraries/datadog.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, monitoring] --- - - While Dagster provides comprehensive monitoring and observability of the pipelines it orchestrates, many teams look to centralize all their monitoring across apps, processes and infrastructure using Datadog's 'Cloud Monitoring as a Service'. The `dagster-datadog` integration allows you to publish metrics to Datadog from within Dagster ops. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/dbt-cloud.md b/docs/docs-beta/docs/integrations/libraries/dbt-cloud.md index 80c07e9990d41..8e8b1c8ad536f 100644 --- a/docs/docs-beta/docs/integrations/libraries/dbt-cloud.md +++ b/docs/docs-beta/docs/integrations/libraries/dbt-cloud.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, etl] --- - - Dagster allows you to run dbt Cloud jobs alongside other technologies. You can schedule them to run as a step in a larger pipeline and manage them as a data asset. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/dbt.md b/docs/docs-beta/docs/integrations/libraries/dbt.md index 177dc9f4e45f3..6d7f40f4a9603 100644 --- a/docs/docs-beta/docs/integrations/libraries/dbt.md +++ b/docs/docs-beta/docs/integrations/libraries/dbt.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, etl] --- - - Dagster orchestrates dbt alongside other technologies, so you can schedule dbt with Spark, Python, etc. in a single data pipeline. Dagster assets understand dbt at the level of individual dbt models. This means that you can: diff --git a/docs/docs-beta/docs/integrations/libraries/deltalake.md b/docs/docs-beta/docs/integrations/libraries/deltalake.md index 3a3fc20373fef..028b4eb198a28 100644 --- a/docs/docs-beta/docs/integrations/libraries/deltalake.md +++ b/docs/docs-beta/docs/integrations/libraries/deltalake.md @@ -18,8 +18,6 @@ enables: tags: [community-supported, storage] --- - - Delta Lake is a great storage format for Dagster workflows. With this integration, you can use the Delta Lake I/O Manager to read and write your Dagster assets. Here are some of the benefits that Delta Lake provides Dagster users: diff --git a/docs/docs-beta/docs/integrations/libraries/dlt.md b/docs/docs-beta/docs/integrations/libraries/dlt.md index 19c54360eea56..ef8293769dcf2 100644 --- a/docs/docs-beta/docs/integrations/libraries/dlt.md +++ b/docs/docs-beta/docs/integrations/libraries/dlt.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, etl] --- - - This integration allows you to use [dlt](https://dlthub.com/) to easily ingest and replicate data between systems through Dagster. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/docker.md b/docs/docs-beta/docs/integrations/libraries/docker.md index 6a65e2818cee9..6344cbb01b932 100644 --- a/docs/docs-beta/docs/integrations/libraries/docker.md +++ b/docs/docs-beta/docs/integrations/libraries/docker.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, compute] --- - - The `dagster-docker` integration library provides the `PipesDockerClient` resource, enabling you to launch Docker containers and execute external code directly from Dagster assets and ops. This integration allows you to pass parameters to Docker containers while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/duckdb.md b/docs/docs-beta/docs/integrations/libraries/duckdb.md index e8097b5040ed8..7800bc181dd05 100644 --- a/docs/docs-beta/docs/integrations/libraries/duckdb.md +++ b/docs/docs-beta/docs/integrations/libraries/duckdb.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, storage] --- - - This library provides an integration with the DuckDB database, and allows for an out-of-the-box [I/O Manager](https://docs.dagster.io/concepts/io-management/io-managers) so that you can make DuckDB your storage of choice. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/fivetran.md b/docs/docs-beta/docs/integrations/libraries/fivetran.md index 9edbfbbee1dc3..bc441730d1785 100644 --- a/docs/docs-beta/docs/integrations/libraries/fivetran.md +++ b/docs/docs-beta/docs/integrations/libraries/fivetran.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, etl] --- - - The Dagster-Fivetran integration enables you to orchestrate data ingestion as part of a larger pipeline. Programmatically interact with the Fivetran REST API to initiate syncs and monitor their progress. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/gcp/bigquery.md b/docs/docs-beta/docs/integrations/libraries/gcp/bigquery.md index 986a7f4070de2..da963a18621e5 100644 --- a/docs/docs-beta/docs/integrations/libraries/gcp/bigquery.md +++ b/docs/docs-beta/docs/integrations/libraries/gcp/bigquery.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, storage] --- - - The Google Cloud Platform BigQuery integration allows data engineers to easily query and store data in the BigQuery data warehouse through the use of the `BigQueryResource`. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/gcp/dataproc.md b/docs/docs-beta/docs/integrations/libraries/gcp/dataproc.md index dc168778f798b..d5c52823850d4 100644 --- a/docs/docs-beta/docs/integrations/libraries/gcp/dataproc.md +++ b/docs/docs-beta/docs/integrations/libraries/gcp/dataproc.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, compute] --- - - Using this integration, you can manage and interact with Google Cloud Platform's Dataproc service directly from Dagster. This integration allows you to create, manage, and delete Dataproc clusters, and submit and monitor jobs on these clusters. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/gcp/gcs.md b/docs/docs-beta/docs/integrations/libraries/gcp/gcs.md index 5969fe8f89368..80a6033f487ac 100644 --- a/docs/docs-beta/docs/integrations/libraries/gcp/gcs.md +++ b/docs/docs-beta/docs/integrations/libraries/gcp/gcs.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, storage] --- - - This integration allows you to interact with Google Cloud Storage (GCS) using Dagster. It provides resources, I/O Managers, and utilities to manage and store data in GCS, making it easier to integrate GCS into your data pipelines. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/github.md b/docs/docs-beta/docs/integrations/libraries/github.md index 8d4176eb8e940..e745ff5a84321 100644 --- a/docs/docs-beta/docs/integrations/libraries/github.md +++ b/docs/docs-beta/docs/integrations/libraries/github.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported] --- - - This library provides an integration with _[GitHub Apps](https://docs.github.com/en/developers/apps/getting-started-with-apps/about-apps)_ by providing a thin wrapper on the GitHub v4 GraphQL API. This allows for automating operations within your GitHub repositories and with the tighter permissions scopes that GitHub Apps allow for vs using a personal token. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/hashicorp.md b/docs/docs-beta/docs/integrations/libraries/hashicorp.md index 5d65ae6106959..73e6754601096 100644 --- a/docs/docs-beta/docs/integrations/libraries/hashicorp.md +++ b/docs/docs-beta/docs/integrations/libraries/hashicorp.md @@ -18,8 +18,6 @@ enables: tags: [community-supported] --- - - Package for integrating HashiCorp Vault into Dagster so that you can securely manage tokens and passwords. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/hightouch.md b/docs/docs-beta/docs/integrations/libraries/hightouch.md index eea47836e2390..a12c638e4e943 100644 --- a/docs/docs-beta/docs/integrations/libraries/hightouch.md +++ b/docs/docs-beta/docs/integrations/libraries/hightouch.md @@ -17,8 +17,6 @@ enables: tags: [community-supported, etl] --- - - With this integration you can trigger Hightouch syncs and monitor them from within Dagster. Fine-tune when Hightouch syncs kick-off, visualize their dependencies, and monitor the steps in your data activation workflow. This native integration helps your team more effectively orchestrate the last mile of data analytics—bringing that data from the warehouse back into the SaaS tools your business teams live in. With the `dagster-hightouch` integration, Hightouch users have more granular and sophisticated control over when data gets activated. diff --git a/docs/docs-beta/docs/integrations/libraries/index.md b/docs/docs-beta/docs/integrations/libraries/index.md index 99a53949d12af..82dfed03539bc 100644 --- a/docs/docs-beta/docs/integrations/libraries/index.md +++ b/docs/docs-beta/docs/integrations/libraries/index.md @@ -7,4 +7,4 @@ You can integrate Dagster with external services using our libraries and librari import DocCardList from '@theme/DocCardList'; - \ No newline at end of file + diff --git a/docs/docs-beta/docs/integrations/libraries/kubernetes.md b/docs/docs-beta/docs/integrations/libraries/kubernetes.md index bdff728e10bbf..120a9a0f6c492 100644 --- a/docs/docs-beta/docs/integrations/libraries/kubernetes.md +++ b/docs/docs-beta/docs/integrations/libraries/kubernetes.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, compute] --- - - The `dagster-k8s` integration library provides the `PipesK8sClient` resource, enabling you to launch Kubernetes pods and execute external code directly from Dagster assets and ops. This integration allows you to pass parameters to Kubernetes pods while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/lakefs.md b/docs/docs-beta/docs/integrations/libraries/lakefs.md index 64f3510405c21..5740d6fc53426 100644 --- a/docs/docs-beta/docs/integrations/libraries/lakefs.md +++ b/docs/docs-beta/docs/integrations/libraries/lakefs.md @@ -18,8 +18,6 @@ enables: tags: [community-supported, storage] --- - - By integrating with lakeFS, a big data scale version control system, you can leverage the versioning capabilities of lakeFS to track changes to your data. This integration allows you to have a complete lineage of your data, from the initial raw data to the transformed and processed data, making it easier to understand and reproduce data transformations. With lakeFS and Dagster integration, you can ensure that data flowing through your Dagster jobs is easily reproducible. lakeFS provides a consistent view of your data across different versions, allowing you to troubleshoot pipeline runs and ensure consistent results. diff --git a/docs/docs-beta/docs/integrations/libraries/looker.md b/docs/docs-beta/docs/integrations/libraries/looker.md index 4cff5bf5fddcd..65adc8accbae1 100644 --- a/docs/docs-beta/docs/integrations/libraries/looker.md +++ b/docs/docs-beta/docs/integrations/libraries/looker.md @@ -18,8 +18,6 @@ enables: tags: [dagster-supported, bi] --- - - Dagster allows you to represent your Looker project as assets, alongside other your other technologies like dbt and Sling. This allows you to see how your Looker assets are connected to your other data assets, and how changes to other data assets might impact your Looker project. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/meltano.md b/docs/docs-beta/docs/integrations/libraries/meltano.md index c78e518e8ca24..34ec11d8e5931 100644 --- a/docs/docs-beta/docs/integrations/libraries/meltano.md +++ b/docs/docs-beta/docs/integrations/libraries/meltano.md @@ -18,8 +18,6 @@ enables: tags: [community-supported, etl] --- - - The `dagster-meltano` library allows you to run Meltano using Dagster. Design and configure ingestion jobs using the popular [Singer.io](https://singer.io) specification. **Note** that this integration can also be [managed from the Meltano platform](https://hub.meltano.com/utilities/dagster/) using `meltano add utility dagster` and configured using `meltano config dagster set --interactive`. diff --git a/docs/docs-beta/docs/integrations/libraries/open-metadata.md b/docs/docs-beta/docs/integrations/libraries/open-metadata.md index 65c6c5077fd86..bd9a7f3c78a13 100644 --- a/docs/docs-beta/docs/integrations/libraries/open-metadata.md +++ b/docs/docs-beta/docs/integrations/libraries/open-metadata.md @@ -18,8 +18,6 @@ enables: tags: [community-supported, metadata] --- - - With this integration you can create a Open Metadata service to ingest metadata produced by the Dagster application. View the Ingestion Pipeline running from the Open Metadata Service Page. ### About Open Metadata diff --git a/docs/docs-beta/docs/integrations/libraries/openai.md b/docs/docs-beta/docs/integrations/libraries/openai.md index 532e21a9018eb..b3f2bce0cf52d 100644 --- a/docs/docs-beta/docs/integrations/libraries/openai.md +++ b/docs/docs-beta/docs/integrations/libraries/openai.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported] --- - - The `dagster-openai` library allows you to easily interact with the OpenAI REST API using the OpenAI Python API to build AI steps into your Dagster pipelines. You can also log OpenAI API usage metadata in Dagster Insights, giving you detailed observability on API call credit consumption. When paired with Dagster assets, the resource automatically logs OpenAI usage metadata in asset metadata. diff --git a/docs/docs-beta/docs/integrations/libraries/pagerduty.md b/docs/docs-beta/docs/integrations/libraries/pagerduty.md index 4c3b577b03fcc..535064af1e004 100644 --- a/docs/docs-beta/docs/integrations/libraries/pagerduty.md +++ b/docs/docs-beta/docs/integrations/libraries/pagerduty.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, alerting] --- - - This library provides an integration between Dagster and PagerDuty to support creating alerts from your Dagster code. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/pandas.md b/docs/docs-beta/docs/integrations/libraries/pandas.md index 6f7ea38111aac..3f10f049bc5fb 100644 --- a/docs/docs-beta/docs/integrations/libraries/pandas.md +++ b/docs/docs-beta/docs/integrations/libraries/pandas.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, metadata] --- - - Perform data validation, emit summary statistics, and enable reliable DataFrame serialization/deserialization. The dagster_pandas library provides you with the utilities for implementing validation on Pandas DataFrames. The Dagster type system generates documentation of your DataFrame constraints and makes it accessible in the Dagster UI. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/pandera.md b/docs/docs-beta/docs/integrations/libraries/pandera.md index 1549013311feb..f6a2409ab8c30 100644 --- a/docs/docs-beta/docs/integrations/libraries/pandera.md +++ b/docs/docs-beta/docs/integrations/libraries/pandera.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, metadata] --- - - The `dagster-pandera` integration library provides an API for generating Dagster Types from [Pandera DataFrame schemas](https://pandera.readthedocs.io/en/stable/dataframe_schemas.html). Like all Dagster types, Dagster-Pandera-generated types can be used to annotate op inputs and outputs. This provides runtime type-checking with rich error reporting and allows Dagster UI to display information about a DataFrame's structure. diff --git a/docs/docs-beta/docs/integrations/libraries/prometheus.md b/docs/docs-beta/docs/integrations/libraries/prometheus.md index e90e0fe7fb10b..cd66709440363 100644 --- a/docs/docs-beta/docs/integrations/libraries/prometheus.md +++ b/docs/docs-beta/docs/integrations/libraries/prometheus.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, monitoring] --- - - This integration allows you to push metrics to the Prometheus gateway from within a Dagster pipeline. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/sdf.md b/docs/docs-beta/docs/integrations/libraries/sdf.md index a155f37bfa613..75f714d0bbc73 100644 --- a/docs/docs-beta/docs/integrations/libraries/sdf.md +++ b/docs/docs-beta/docs/integrations/libraries/sdf.md @@ -18,8 +18,6 @@ enables: tags: [community-supported, etl] --- - - SDF can integrate seamlessly with your existing Dagster projects, providing the best-in-class transformation layer while enabling you to schedule, orchestrate, and monitor your dags in Dagster. When it comes time to materialize your Dagster assets, you can be confident that SDF has successfully compiled your workspace, making it safe to execute locally or against your cloud data warehouse. diff --git a/docs/docs-beta/docs/integrations/libraries/secoda.md b/docs/docs-beta/docs/integrations/libraries/secoda.md index 3ec41ce364776..540246f9bd483 100644 --- a/docs/docs-beta/docs/integrations/libraries/secoda.md +++ b/docs/docs-beta/docs/integrations/libraries/secoda.md @@ -18,8 +18,6 @@ enables: tags: [community-supported, metadata] --- - - Connect Dagster to Secoda and see metadata related to your Dagster assets, asset groups and jobs right in Secoda. Simplify your team's access, and remove the need to switch between tools. When you connect Dagster to Secoda, you can use Secoda's tools to add further context to your Dagster assets and jobs. Help your team understand metadata from Dagster by adding context in Secoda, like creating Documents, defining Metrics, and adding Tags. diff --git a/docs/docs-beta/docs/integrations/libraries/shell.md b/docs/docs-beta/docs/integrations/libraries/shell.md index 6b69731c6aeea..63704cdfe50e8 100644 --- a/docs/docs-beta/docs/integrations/libraries/shell.md +++ b/docs/docs-beta/docs/integrations/libraries/shell.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, compute] --- - - Dagster comes with a native `PipesSubprocessClient` resource that enables you to launch shell commands directly from Dagster assets and ops. This integration allows you to pass parameters to external shell scripts while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated external execution. With minimal code changes required on the job side, this integration is both efficient and easy to implement. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/slack.md b/docs/docs-beta/docs/integrations/libraries/slack.md index 3c5d3b0253ec9..22dd470c06e55 100644 --- a/docs/docs-beta/docs/integrations/libraries/slack.md +++ b/docs/docs-beta/docs/integrations/libraries/slack.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, alerting] --- - - This library provides an integration with Slack to support posting messages in your company's Slack workspace. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/sling.md b/docs/docs-beta/docs/integrations/libraries/sling.md index 58cceb2bac261..f6fbe9a5111bc 100644 --- a/docs/docs-beta/docs/integrations/libraries/sling.md +++ b/docs/docs-beta/docs/integrations/libraries/sling.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, etl] --- - - This integration allows you to use [Sling](https://slingdata.io/) to extract and load data from popular data sources to destinations with high performance and ease. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/snowflake.md b/docs/docs-beta/docs/integrations/libraries/snowflake.md index 3e3f45986ff76..2e7198fc37af4 100644 --- a/docs/docs-beta/docs/integrations/libraries/snowflake.md +++ b/docs/docs-beta/docs/integrations/libraries/snowflake.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, storage] --- - - This library provides an integration with the Snowflake data warehouse. Connect to Snowflake as a resource, then use the integration-provided functions to construct an op to establish connections and execute Snowflake queries. Read and write natively to Snowflake from Dagster assets. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/spark.md b/docs/docs-beta/docs/integrations/libraries/spark.md index e14f63f73a10a..d7fcb7fefbfd3 100644 --- a/docs/docs-beta/docs/integrations/libraries/spark.md +++ b/docs/docs-beta/docs/integrations/libraries/spark.md @@ -18,8 +18,6 @@ enables: tags: [dagster-supported, compute] --- - - Spark jobs typically execute on infrastructure that's specialized for Spark. Spark applications are typically not containerized or executed on Kubernetes. Running Spark code often requires submitting code to a Databricks or EMR cluster. `dagster-pyspark` provides a Spark class with methods for configuration and constructing the `spark-submit` command for a Spark job. diff --git a/docs/docs-beta/docs/integrations/libraries/ssh-sftp.md b/docs/docs-beta/docs/integrations/libraries/ssh-sftp.md index be2314fa07cbb..cf1a3a025e8d0 100644 --- a/docs/docs-beta/docs/integrations/libraries/ssh-sftp.md +++ b/docs/docs-beta/docs/integrations/libraries/ssh-sftp.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported] --- - - This integration provides a resource for SSH remote execution using [Paramiko](https://github.com/paramiko/paramiko). It allows you to establish secure connections to networked resources and execute commands remotely. The integration also provides an SFTP client for secure file transfers between the local and remote systems. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/twilio.md b/docs/docs-beta/docs/integrations/libraries/twilio.md index 4584fdb550781..5412d8bfdd959 100644 --- a/docs/docs-beta/docs/integrations/libraries/twilio.md +++ b/docs/docs-beta/docs/integrations/libraries/twilio.md @@ -17,8 +17,6 @@ enables: tags: [dagster-supported, alerting] --- - - Use your Twilio `Account SID` and `Auth Token` to build Twilio tasks right into your Dagster pipeline. ### Installation diff --git a/docs/docs-beta/docs/integrations/libraries/wandb.md b/docs/docs-beta/docs/integrations/libraries/wandb.md index 363950d2fa10e..d5beb5c86bc2b 100644 --- a/docs/docs-beta/docs/integrations/libraries/wandb.md +++ b/docs/docs-beta/docs/integrations/libraries/wandb.md @@ -18,8 +18,6 @@ enables: tags: [community-supported] --- - - Use Dagster and Weights & Biases (W&B) to orchestrate your MLOps pipelines and maintain ML assets. The integration with W&B makes it easy within Dagster to: - use and create W&B Artifacts diff --git a/docs/docs-beta/docs/intro.md b/docs/docs-beta/docs/intro.md index 56b22d535a3c7..582e4f4d040ad 100644 --- a/docs/docs-beta/docs/intro.md +++ b/docs/docs-beta/docs/intro.md @@ -6,7 +6,7 @@ displayed_sidebar: 'docs' hide_table_of_contents: true --- -import { Card, CardGroup } from '@site/src/components/Cards'; +import {Card, CardGroup} from '@site/src/components/Cards'; import ThemedImage from '@theme/ThemedImage'; # Welcome to Dagster @@ -16,7 +16,7 @@ Dagster is a data orchestrator built for data engineers, with integrated lineage - - Build your first Dagster pipeline in our Quickstart tutorial. + + Build your first Dagster pipeline in our Quickstart tutorial. - + New to Dagster? Learn about how thinking in assets can help you manage your data better. - Learn about Dagster Plus, our managed offering that includes a hosted Dagster instance and many more features. + Learn about Dagster Plus, our managed offering that includes a hosted Dagster instance and many + more features. ## Join the Dagster community - - Join our Slack community to talk with other Dagster users, use our AI-powered chatbot, and get help with Dagster. + + Join our Slack community to talk with other Dagster users, use our AI-powered chatbot, and get + help with Dagster. - + Star our GitHub repository and follow our development through GitHub Discussions. - + Watch our latest videos on YouTube. diff --git a/docs/docs-beta/docs/partials/_Experimental.md b/docs/docs-beta/docs/partials/_Experimental.md index a4089ffa6a372..5936bf8b87130 100644 --- a/docs/docs-beta/docs/partials/_Experimental.md +++ b/docs/docs-beta/docs/partials/_Experimental.md @@ -1,3 +1,3 @@ :::warning This feature is considered experimental and is under active development. -::: \ No newline at end of file +::: diff --git a/docs/docs-beta/docs/partials/_TestSSO.md b/docs/docs-beta/docs/partials/_TestSSO.md index 319ac0a41d8a5..45cb7f9ba6242 100644 --- a/docs/docs-beta/docs/partials/_TestSSO.md +++ b/docs/docs-beta/docs/partials/_TestSSO.md @@ -15,4 +15,4 @@ Lastly, you'll test your SSO configuration: ### Testing an identity provider-initiated login \{#test-identity-login} -{/* Copy for this section is in individual guides */} \ No newline at end of file +{/** Copy for this section is in individual guides **/} diff --git a/docs/docs-beta/docs/tutorial/introduction.md b/docs/docs-beta/docs/tutorial/introduction.md index fa350b858b24b..25fa121f7edba 100644 --- a/docs/docs-beta/docs/tutorial/introduction.md +++ b/docs/docs-beta/docs/tutorial/introduction.md @@ -1,6 +1,6 @@ --- -title: "Introduction" -description: "Welcome to the Dagster documentation! If this is your first time developing a Dagster pipeline, read through this Getting Started section to get familiar with the basics. Otherwise, feel free to explore our guides and API documentation!" +title: 'Introduction' +description: 'Welcome to the Dagster documentation! If this is your first time developing a Dagster pipeline, read through this Getting Started section to get familiar with the basics. Otherwise, feel free to explore our guides and API documentation!' slug: introduction hide_title: false unlisted: true diff --git a/docs/docs-beta/docusaurus.config.ts b/docs/docs-beta/docusaurus.config.ts index 7a3f8dd9f75af..eab1dc1c27b85 100644 --- a/docs/docs-beta/docusaurus.config.ts +++ b/docs/docs-beta/docusaurus.config.ts @@ -1,7 +1,7 @@ import {themes as prismThemes} from 'prism-react-renderer'; import type {Config} from '@docusaurus/types'; import type * as Preset from '@docusaurus/preset-classic'; -import { groupCollapsed } from 'console'; +import {groupCollapsed} from 'console'; const config: Config = { title: 'Dagster Docs - Beta', diff --git a/docs/docs-beta/sidebars.ts b/docs/docs-beta/sidebars.ts index 03ca26fa27bcb..3ebf22cbc96b0 100644 --- a/docs/docs-beta/sidebars.ts +++ b/docs/docs-beta/sidebars.ts @@ -5,7 +5,12 @@ const sidebars: SidebarsConfig = { type: 'category', label: 'Getting Started', collapsed: false, - items: ['intro', 'getting-started/quickstart', 'getting-started/installation', 'getting-started/glossary'], + items: [ + 'intro', + 'getting-started/quickstart', + 'getting-started/installation', + 'getting-started/glossary', + ], }, { type: 'category', @@ -21,8 +26,8 @@ const sidebars: SidebarsConfig = { items: [ { type: 'autogenerated', - dirName: 'guides/build' - } + dirName: 'guides/build', + }, ], }, { @@ -33,8 +38,8 @@ const sidebars: SidebarsConfig = { items: [ { type: 'autogenerated', - dirName: 'guides/automate' - } + dirName: 'guides/automate', + }, ], }, { @@ -45,7 +50,7 @@ const sidebars: SidebarsConfig = { items: [ { type: 'autogenerated', - dirName: 'guides/monitor' + dirName: 'guides/monitor', }, ], }, @@ -57,8 +62,8 @@ const sidebars: SidebarsConfig = { items: [ { type: 'autogenerated', - dirName: 'guides/test' - } + dirName: 'guides/test', + }, ], }, { @@ -69,8 +74,8 @@ const sidebars: SidebarsConfig = { items: [ { type: 'autogenerated', - dirName: 'guides/deploy' - } + dirName: 'guides/deploy', + }, ], }, { @@ -90,9 +95,7 @@ const sidebars: SidebarsConfig = { type: 'category', label: 'Guides', collapsed: false, - items: [ - 'integrations/guides/multi-asset-integration' - ] + items: ['integrations/guides/multi-asset-integration'], }, { type: 'category', @@ -102,10 +105,10 @@ const sidebars: SidebarsConfig = { items: [ { type: 'autogenerated', - dirName: 'integrations/libraries' - } - ] - } + dirName: 'integrations/libraries', + }, + ], + }, ], dagsterPlus: [ 'dagster-plus/index', @@ -117,9 +120,9 @@ const sidebars: SidebarsConfig = { items: [ { type: 'autogenerated', - dirName: 'dagster-plus/features' - } - ] + dirName: 'dagster-plus/features', + }, + ], }, { type: 'category', @@ -128,8 +131,8 @@ const sidebars: SidebarsConfig = { items: [ { type: 'autogenerated', - dirName: 'dagster-plus/deployment' - } + dirName: 'dagster-plus/deployment', + }, ], }, ], @@ -147,7 +150,7 @@ const sidebars: SidebarsConfig = { dirName: 'api/python-api', }, ], - } + }, ], }; diff --git a/docs/docs-beta/src/styles/custom.scss b/docs/docs-beta/src/styles/custom.scss index c7a73c0f06af0..a02b3e94cac4e 100644 --- a/docs/docs-beta/src/styles/custom.scss +++ b/docs/docs-beta/src/styles/custom.scss @@ -207,7 +207,6 @@ a.pyobject { margin-top: 8px; } - // We want this to only apply to inline code :not(pre):not(h2):not(h3):not(h4):not(h5):not(h6) > code { background-color: var(--theme-color-background-blue); @@ -687,46 +686,44 @@ div[class^='announcementBar_'] { } } - /* Feedback navigation item */ .feedback-nav-link { color: var(--theme-color-text-light); display: block; font-weight: 500; - padding: 0.5rem .5rem; + padding: 0.5rem 0.5rem; text-align: center; text-decoration: none; border: 1px solid var(--theme-color-border-default); border-radius: 8px; text-decoration: none; - transition: all 0.3s ease; + transition: all 0.3s ease; &:hover { background-color: var(--theme-color-background-light); - color: var(--theme-color-text-light); + color: var(--theme-color-text-light); border-color: var(--theme-color-border-hover); } - } /* Markdown Card */ - .card { - background-color: var(--theme-color-background-default); - border: 1px solid var(--theme-color-keyline) !important; - transition: all .3s ease; - - h2: { - font-size: 1.5rem; - margin-top: 0; - margin-bottom: 0; - font-weight: 600; - display: none; - } - - &:hover { - background-color: var(--theme-color-background-light); - border: 1px solid var(--theme-color-border-default) !important; - transform: translateY(0px); - } - } \ No newline at end of file +.card { + background-color: var(--theme-color-background-default); + border: 1px solid var(--theme-color-keyline) !important; + transition: all 0.3s ease; + + h2: { + font-size: 1.5rem; + margin-top: 0; + margin-bottom: 0; + font-weight: 600; + display: none; + } + + &:hover { + background-color: var(--theme-color-background-light); + border: 1px solid var(--theme-color-border-default) !important; + transform: translateY(0px); + } +} diff --git a/docs/docs-beta/src/theme/MDXComponents.tsx b/docs/docs-beta/src/theme/MDXComponents.tsx index d26d4fe3c81ca..d780d971c00ac 100644 --- a/docs/docs-beta/src/theme/MDXComponents.tsx +++ b/docs/docs-beta/src/theme/MDXComponents.tsx @@ -1,6 +1,6 @@ // Import the original mapper import MDXComponents from '@theme-original/MDXComponents'; -import { PyObject } from '../components/PyObject'; +import {PyObject} from '../components/PyObject'; import CodeExample from '../components/CodeExample'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; From 7ecef59ce52a28152817ce1b62ac515197b9d127 Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Fri, 13 Dec 2024 17:09:09 -0500 Subject: [PATCH 3/3] yarn lint --- .../deployment-types/serverless/runtime-environment.md | 7 +++---- .../deployment/management/dagster-cloud-cli/index.md | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/runtime-environment.md b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/runtime-environment.md index a81af91fd6b30..666ec1b7df33b 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/runtime-environment.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/deployment-types/serverless/runtime-environment.md @@ -206,7 +206,6 @@ Setting a custom base image isn't supported for GitLab CI/CD workflows out of th 1. Build you base image 2. Specify this base image tag in your GitHub workflow, or using the `dagster-cloud` CLI: - @@ -217,9 +216,9 @@ Setting a custom base image isn't supported for GitLab CI/CD workflows out of th You can specify the base image when you deploy your code with the `dagster-cloud serverless deploy` command: - ```bash - dagster-cloud serverless deploy --base-image=my_base_image:latest --location-name=my_location - ``` + ```bash + dagster-cloud serverless deploy --base-image=my_base_image:latest --location-name=my_location + ``` diff --git a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/index.md b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/index.md index 6ef4de02bd902..fdd93aeb581d9 100644 --- a/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/index.md +++ b/docs/docs-beta/docs/dagster-plus/deployment/management/dagster-cloud-cli/index.md @@ -7,4 +7,4 @@ The dagster-cloud CLI is a command-line toolkit designed to work with Dagster+. import DocCardList from '@theme/DocCardList'; -{' '} +