Merge branch 'master' into add-backoff-duckdb-connections

.buildkite/dagster-buildkite/dagster_buildkite/package_spec.py

@@ -232,7 +232,7 @@ def build_steps(self) -> List[BuildkiteTopLevelStep]:
     @property
     def requirements(self):
         # First try to infer requirements from the python package
-        package = PythonPackages.get(self.name)
+        package = PythonPackages.get(self.name)  # pyright: ignore[reportArgumentType]
         if package:
             return set.union(package.install_requires, *package.extras_require.values())
 

.buildkite/dagster-buildkite/dagster_buildkite/pipelines/dagster_oss_nightly_pipeline.py

@@ -2,6 +2,7 @@
 
 from dagster_buildkite.package_spec import PackageSpec
 from dagster_buildkite.python_version import AvailablePythonVersion
+from dagster_buildkite.step_builder import BuildkiteQueue
 from dagster_buildkite.steps.packages import (
     build_steps_from_package_specs,
     gcp_creds_extra_cmds,
@@ -28,6 +29,7 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                 unsupported_python_versions=[
                     AvailablePythonVersion.V3_12,
                 ],
+                always_run_if=lambda: True,
             ),
             PackageSpec(
                 "python_modules/libraries/dagster-k8s",
@@ -41,6 +43,7 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "nightly",
                 ],
                 pytest_extra_cmds=k8s_extra_cmds,
+                always_run_if=lambda: True,
             ),
             PackageSpec(
                 "examples/experimental/dagster-dlift/kitchen-sink",
@@ -52,6 +55,7 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "KS_DBT_CLOUD_ACCESS_URL",
                     "KS_DBT_CLOUD_DISCOVERY_API_URL",
                 ],
+                always_run_if=lambda: True,
             ),
             PackageSpec(
                 "examples/starlift-demo",
@@ -63,6 +67,20 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "KS_DBT_CLOUD_ACCESS_URL",
                     "KS_DBT_CLOUD_DISCOVERY_API_URL",
                 ],
+                queue=BuildkiteQueue.DOCKER,
+                always_run_if=lambda: True,
+            ),
+            PackageSpec(
+                "integration_tests/test_suites/dagster-azure-live-tests",
+                name="azure-live-tests",
+                env_vars=[
+                    "TEST_AZURE_TENANT_ID",
+                    "TEST_AZURE_CLIENT_ID",
+                    "TEST_AZURE_CLIENT_SECRET",
+                    "TEST_AZURE_STORAGE_ACCOUNT_ID",
+                    "TEST_AZURE_CONTAINER_ID",
+                ],
+                always_run_if=lambda: True,
             ),
         ]
     )

.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py

@@ -6,6 +6,7 @@
     GCP_CREDS_LOCAL_FILE,
     LATEST_DAGSTER_RELEASE,
 )
+from dagster_buildkite.git import ChangedFiles
 from dagster_buildkite.package_spec import (
     PackageSpec,
     PytestExtraCommandsFunction,
@@ -42,6 +43,7 @@ def build_integration_steps() -> List[BuildkiteStep]:
     steps += build_k8s_suite_steps()
     steps += build_daemon_suite_steps()
     steps += build_auto_materialize_perf_suite_steps()
+    steps += build_azure_live_test_suite_steps()
 
     return steps
 
@@ -165,6 +167,29 @@ def build_auto_materialize_perf_suite_steps():
     )
 
 
+def skip_if_not_azure_commit():
+    """If no dagster-azure files are changed, skip the azure live tests."""
+    return (
+        None
+        if (any("dagster-azure" in str(path) for path in ChangedFiles.all))
+        else "Not a dagster-azure commit"
+    )
+
+
+def build_azure_live_test_suite_steps() -> List[BuildkiteTopLevelStep]:
+    return PackageSpec(
+        os.path.join("integration_tests", "test_suites", "dagster-azure-live-tests"),
+        skip_if=skip_if_not_azure_commit,
+        env_vars=[
+            "TEST_AZURE_TENANT_ID",
+            "TEST_AZURE_CLIENT_ID",
+            "TEST_AZURE_CLIENT_SECRET",
+            "TEST_AZURE_STORAGE_ACCOUNT_ID",
+            "TEST_AZURE_CONTAINER_ID",
+        ],
+    ).build_steps()
+
+
 def daemon_pytest_extra_cmds(version: AvailablePythonVersion, _):
     return [
         "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value,

.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py

@@ -272,14 +272,6 @@ def k8s_extra_cmds(version: AvailablePythonVersion, _) -> List[str]:
 # Some Dagster packages have more involved test configs or support only certain Python version;
 # special-case those here
 EXAMPLE_PACKAGES_WITH_CUSTOM_CONFIG: List[PackageSpec] = [
-    PackageSpec(
-        "examples/with_airflow",
-        unsupported_python_versions=[
-            AvailablePythonVersion.V3_10,
-            AvailablePythonVersion.V3_11,
-            AvailablePythonVersion.V3_12,
-        ],
-    ),
     PackageSpec(
         "examples/assets_smoke_test",
     ),

.buildkite/dagster-buildkite/dagster_buildkite/utils.py

@@ -363,7 +363,7 @@ def skip_if_no_docs_changes():
     if message_contains("NO_SKIP"):
         return None
 
-    if not is_feature_branch(os.getenv("BUILDKITE_BRANCH")):
+    if not is_feature_branch(os.getenv("BUILDKITE_BRANCH")):  # pyright: ignore[reportArgumentType]
         return None
 
     # If anything changes in the docs directory

CHANGES.md

@@ -1,5 +1,71 @@
 # Changelog
 
+## 1.9.5 (core) / 0.25.5 (libraries)
+
+### New
+
+- The automatic run retry daemon has been updated so that there is a single source of truth for if a run will be retried and if the retry has been launched. Tags are now added to run at failure time indicating if the run will be retried by the automatic retry system. Once the automatic retry has been launched, the run ID of the retry is added to the original run.
+- When canceling a backfill of a job, the backfill daemon will now cancel all runs launched by that backfill before marking the backfill as canceled.
+- Dagster execution info (tags such as `dagster/run-id`, `dagster/code-location`, `dagster/user` and Dagster Cloud environment variables) typically attached to external resources are now available under `DagsterRun.dagster_execution_info`.
+- `SensorReturnTypesUnion` is now exported for typing the output of sensor functions.
+- [dagster-dbt] dbt seeds now get a valid code version (Thanks [@marijncv](https://github.com/marijncv)!).
+- Manual and automatic retries of runs launched by backfills that occur while the backfill is still in progress are now incorporated into the backfill's status.
+- Manual retries of runs launched by backfills are no longer considered part of the backfill if the backfill is complete when the retry is launched.
+- [dagster-fivetran] Fivetran assets can now be materialized using the FivetranWorkspace.sync_and_poll(…) method in the definition of a `@fivetran_assets` decorator.
+- [dagster-fivetran] `load_fivetran_asset_specs` has been updated to accept an instance of `DagsterFivetranTranslator` or custom subclass.
+- [dagster-fivetran] The `fivetran_assets` decorator was added. It can be used with the `FivetranWorkspace` resource and `DagsterFivetranTranslator` translator to load Fivetran tables for a given connector as assets in Dagster. The `build_fivetran_assets_definitions` factory can be used to create assets for all the connectors in your Fivetran workspace.
+- [dagster-aws] `ECSPipesClient.run` now waits up to 70 days for tasks completion (waiter parameters are configurable) (Thanks [@jenkoian](https://github.com/jenkoian)!)
+- [dagster-dbt] Update dagster-dbt scaffold template to be compatible with uv (Thanks [@wingyplus](https://github.com/wingyplus)!).
+- [dagster-airbyte] A `load_airbyte_cloud_asset_specs` function has
+  been added. It can be used with the `AirbyteCloudWorkspace` resource and `DagsterAirbyteTranslator` translator to load your Airbyte Cloud connection streams as external assets in Dagster.
+- [ui] Add an icon for the `icechunk` kind.
+- [ui] Improved ui for manual sensor/schedule evaluation.
+
+### Bugfixes
+
+- Fixed database locking bug for the `ConsolidatedSqliteEventLogStorage`, which is mostly used for tests.
+- [dagster-aws] Fixed a bug in the ECSRunLauncher that prevented it from accepting a user-provided task definition when DAGSTER_CURRENT_IMAGE was not set in the code location.
+- [ui] Fixed an issue that would sometimes cause the asset graph to fail to render on initial load.
+- [ui] Fix global auto-materialize tick timeline when paginating.
+
+## 1.9.4 (core) / 0.25.4 (libraries)
+
+### New
+
+- Global op concurrency is now enabled on the default SQLite storage. Deployments that have not been migrated since `1.6.0` may need to run `dagster instance migrate` to enable.
+- Introduced `map_asset_specs` to enable modifying `AssetSpec`s and `AssetsDefinition`s in bulk.
+- Introduced `AssetSpec.replace_attributes` and `AssetSpec.merge_attributes` to easily alter properties of an asset spec.
+- [ui] Add a "View logs" button to open tick logs in the sensor tick history table.
+- [ui] Add Spanner kind icon.
+- [ui] The asset catalog now supports filtering using the asset selection syntax.
+- [dagster-pipes, dagster-aws] `PipesS3MessageReader` now has a new parameter `include_stdio_in_messages` which enables log forwarding to Dagster via Pipes messages.
+- [dagster-pipes] Experimental: A new Dagster Pipes message type `log_external_stream` has been added. It can be used to forward external logs to Dagster via Pipes messages.
+- [dagster-powerbi] Opts in to using admin scan APIs to pull data from a Power BI instance. This can be disabled by passing `load_powerbi_asset_specs(..., use_workspace_scan=False)`.
+- [dagster-sigma] Introduced an experimental `dagster-sigma snapshot` command, allowing Sigma workspaces to be captured to a file for faster subsequent loading.
+
+### Bugfixes
+
+- Fixed a bug that caused `DagsterExecutionStepNotFoundError` errors when trying to execute an asset check step of a run launched by a backfill.
+- Fixed an issue where invalid cron strings like "0 0 30 2 \*" that represented invalid dates in February were still allowed as Dagster cron strings, but then failed during schedule execution. Now, these invalid cronstrings will raise an exception when they are first loaded.
+- Fixed a bug where `owners` added to `AssetOut`s when defining a `@graph_multi_asset` were not added to the underlying `AssetsDefinition`.
+- Fixed a bug where using the `&` or `|` operators on `AutomationCondition`s with labels would cause that label to be erased.
+- [ui] Launching partitioned asset jobs from the launchpad now warns if no partition is selected.
+- [ui] Fixed unnecessary middle truncation occurring in dialogs.
+- [ui] Fixed timestamp labels and "Now" line rendering bugs on the sensor tick timeline.
+- [ui] Opening Dagster's UI with a single job defined takes you to the Overview page rather than the Job page.
+- [ui] Fix stretched tags in backfill table view for non-partitioned assets.
+- [ui] Open automation sensor evaluation details in a dialog instead of navigating away.
+- [ui] Fix scrollbars in dark mode.
+- [dagster-sigma] Workbooks filtered using a `SigmaFilter` no longer fetch lineage information.
+- [dagster-powerbi] Fixed an issue where reports without an upstream dataset dependency would fail to translate to an asset spec.
+
+### Deprecations
+
+- [dagster-powerbi] `DagsterPowerBITranslator.get_asset_key` is deprecated in favor of `DagsterPowerBITranslator.get_asset_spec().key`
+- [dagster-looker] `DagsterLookerApiTranslator.get_asset_key` is deprecated in favor of `DagsterLookerApiTranslator.get_asset_spec().key`
+- [dagster-sigma] `DagsterSigmaTranslator.get_asset_key` is deprecated in favor of `DagsterSigmaTranslator.get_asset_spec().key`
+- [dagster-tableau] `DagsterTableauTranslator.get_asset_key` is deprecated in favor of `DagsterTableauTranslator.get_asset_spec().key`
+
 ## 1.9.3 (core) / 0.25.3 (libraries)
 
 ### New

docs/Makefile

@@ -35,4 +35,4 @@ mdx:
 	tox -e sphinx-mdx
 
 mdx_copy:
-	cp -rf sphinx/_build/mdx/sections/api/apidocs/* docs-beta/docs/api/
+	cp -rf sphinx/_build/mdx/sections/api/apidocs/* docs-beta/docs/api/python-api/

docs/content/_navigation.json

@@ -179,6 +179,10 @@
               {
                 "title": "Customizing automation conditions",
                 "path": "/concepts/automation/declarative-automation/customizing-automation-conditions"
+              },
+              {
+                "title": "Automation conditions explained (Advanced)",
+                "path": "/concepts/automation/declarative-automation/automation-conditions-explained"
               }
             ]
           },
@@ -548,6 +552,24 @@
               {
                 "title": "Setting environment variables",
                 "path": "/dagster-plus/managing-deployments/setting-environment-variables-agents"
+              },
+              {
+                "title": "Deploying on Azure: Full guide",
+                "path": "/dagster-plus/deployment/azure/overview",
+                "children": [
+                  {
+                    "title": "Part 1: Setting up an AKS agent.",
+                    "path": "/dagster-plus/deployment/azure/aks-agent"
+                  },
+                  {
+                    "title": "Part 2: Deploy user code using ACR.",
+                    "path": "/dagster-plus/deployment/azure/acr-user-code"
+                  },
+                  {
+                    "title": "Part 3: Storing compute logs in Azure Blob Storage / Azure Data Lake",
+                    "path": "/dagster-plus/deployment/azure/blob-compute-logs"
+                  }
+                ]
               }
             ]
           }
@@ -916,6 +938,10 @@
         "title": "Airlift",
         "path": "/integrations/airlift",
         "children": [
+          {
+            "title": "Mapping Airflow Concepts to Dagster",
+            "path": "/integrations/airlift/airflow-dagster-equivalents"
+          },
           {
             "title": "Airflow Migration Tutorial",
             "path": "/integrations/airlift/tutorial/overview",
@@ -967,6 +993,32 @@
           {
             "title": "DAG-level migration",
             "path": "/integrations/airlift/full_dag"
+          },
+          {
+            "title": "Migration with Airflow 1",
+            "path": "/integrations/airlift/airflow-1-migration"
+          },
+          {
+            "title": "Operator Migration Reference",
+            "path": "/integrations/airlift/operator-migration/overview",
+            "children": [
+              {
+                "title": "Migrating a BashOperator for dbt",
+                "path": "/integrations/airlift/operator-migration/bash-operator-dbt"
+              },
+              {
+                "title": "Migrating a KubernetesPodOperator",
+                "path": "/integrations/airlift/operator-migration/kubernetes-pod-operator"
+              },
+              {
+                "title": "Migrating a PythonOperator",
+                "path": "/integrations/airlift/operator-migration/python-operator"
+              },
+              {
+                "title": "Migrating a BashOperator (general)",
+                "path": "/integrations/airlift/operator-migration/bash-operator-general"
+              }
+            ]
           }
         ]
       },
@@ -1076,7 +1128,13 @@
       },
       {
         "title": "Fivetran",
-        "path": "/integrations/fivetran"
+        "path": "/integrations/fivetran/fivetran",
+        "children": [
+          {
+            "title": "Fivetran (Legacy)",
+            "path": "/integrations/fivetran/fivetran-legacy"
+          }
+        ]
       },
       {
         "title": "Google BigQuery",

docs/content/concepts/assets/asset-checks.mdx

@@ -28,7 +28,7 @@ height={802}
 
 Using asset checks helps you:
 
-- Create a single place in the Dagstser UI to learn about and investigate issues in your data pipeline
+- Create a single place in the Dagster UI to learn about and investigate issues in your data pipeline
 - Ensure that data produced by your pipeline is timely and high quality by applying basic suites of tests to all assets
 - Identify problems in the source data that your pipeline depends on, such as checking data quality on source assets where dbt tests aren't possible
 - Communicate what the data produced by your pipeline is expected to look like (aka act as a data contract)
@@ -89,7 +89,7 @@ The following table lists Dagster's built-in utility methods for creating asset
 | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
 | <PyObject object="build_metadata_bounds_checks" />          | Builds asset checks that pass if numeric metadata value falls within a particular range                   |
 | <PyObject object="build_column_schema_change_checks" />     | Builds asset checks that pass if an asset's columns are the same, compared with its prior materialization |
-| <PyObject object="build_last_update_freshness_checks" />    | Builds asset checks that pass if not too much time has elapsed since the latest time an asset was updated |
+| <PyObject object="build_last_update_freshness_checks" />    | Builds asset checks that pass if they were updated within a given threshold                               |
 | <PyObject object="build_time_partition_freshness_checks" /> | Builds asset checks that pass if an asset's most recent partition has been materialized before a deadline |
 
 ### Limitations

docs/content/concepts/metadata-tags/kind-tags.mdx

@@ -137,6 +137,7 @@ Some kinds are given a branded icon in the UI. We currently support nearly 200 u
 | `huggingface`       | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-huggingface-color.svg" width={20} height={20} />       |
 | `huggingfaceapi`    | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-huggingface-color.svg" width={20} height={20} />       |
 | `iceberg`           | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-iceberg-color.svg" width={20} height={20} />           |
+| `icechunk`          | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-icechunk-color.svg" width={20} height={20} />          |
 | `impala`            | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-impala-color.svg" width={20} height={20} />            |
 | `instagram`         | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-instagram-color.svg" width={20} height={20} />         |
 | `ipynb`             | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-jupyter-color.svg" width={20} height={20} />           |