Merge branch 'master' into nikki/docs/integrations-tutorials

dagster-io · Dec 10, 2024 · d4d4164 · d4d4164
2 parents 1f3959b + 76c18dd
commit d4d4164
Show file tree

Hide file tree

Showing 268 changed files with 16,725 additions and 6,664 deletions.
diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/pipelines/dagster_oss_nightly_pipeline.py b/.buildkite/dagster-buildkite/dagster_buildkite/pipelines/dagster_oss_nightly_pipeline.py
@@ -2,6 +2,7 @@
 
 from dagster_buildkite.package_spec import PackageSpec
 from dagster_buildkite.python_version import AvailablePythonVersion
+from dagster_buildkite.step_builder import BuildkiteQueue
 from dagster_buildkite.steps.packages import (
     build_steps_from_package_specs,
     gcp_creds_extra_cmds,
@@ -28,6 +29,7 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                 unsupported_python_versions=[
                     AvailablePythonVersion.V3_12,
                 ],
+                always_run_if=lambda: True,
             ),
             PackageSpec(
                 "python_modules/libraries/dagster-k8s",
@@ -41,6 +43,7 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "nightly",
                 ],
                 pytest_extra_cmds=k8s_extra_cmds,
+                always_run_if=lambda: True,
             ),
             PackageSpec(
                 "examples/experimental/dagster-dlift/kitchen-sink",
@@ -52,6 +55,7 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "KS_DBT_CLOUD_ACCESS_URL",
                     "KS_DBT_CLOUD_DISCOVERY_API_URL",
                 ],
+                always_run_if=lambda: True,
             ),
             PackageSpec(
                 "examples/starlift-demo",
@@ -63,6 +67,8 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "KS_DBT_CLOUD_ACCESS_URL",
                     "KS_DBT_CLOUD_DISCOVERY_API_URL",
                 ],
+                queue=BuildkiteQueue.DOCKER,
+                always_run_if=lambda: True,
             ),
             PackageSpec(
                 "integration_tests/test_suites/dagster-azure-live-tests",
@@ -74,6 +80,7 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "TEST_AZURE_STORAGE_ACCOUNT_ID",
                     "TEST_AZURE_CONTAINER_ID",
                 ],
+                always_run_if=lambda: True,
             ),
         ]
     )

diff --git a/docs/content/_navigation.json b/docs/content/_navigation.json
@@ -934,6 +934,10 @@
         "title": "Airlift",
         "path": "/integrations/airlift",
         "children": [
+          {
+            "title": "Mapping Airflow Concepts to Dagster",
+            "path": "/integrations/airlift/airflow-dagster-equivalents"
+          },
           {
             "title": "Airflow Migration Tutorial",
             "path": "/integrations/airlift/tutorial/overview",
@@ -985,6 +989,32 @@
           {
             "title": "DAG-level migration",
             "path": "/integrations/airlift/full_dag"
+          },
+          {
+            "title": "Migration with Airflow 1",
+            "path": "/integrations/airlift/airflow-1-migration"
+          },
+          {
+            "title": "Operator Migration Reference",
+            "path": "/integrations/airlift/operator-migration/overview",
+            "children": [
+              {
+                "title": "Migrating a BashOperator for dbt",
+                "path": "/integrations/airlift/operator-migration/bash-operator-dbt"
+              },
+              {
+                "title": "Migrating a KubernetesPodOperator",
+                "path": "/integrations/airlift/operator-migration/kubernetes-pod-operator"
+              },
+              {
+                "title": "Migrating a PythonOperator",
+                "path": "/integrations/airlift/operator-migration/python-operator"
+              },
+              {
+                "title": "Migrating a BashOperator (general)",
+                "path": "/integrations/airlift/operator-migration/bash-operator-general"
+              }
+            ]
           }
         ]
       },
@@ -1094,7 +1124,13 @@
       },
       {
         "title": "Fivetran",
-        "path": "/integrations/fivetran"
+        "path": "/integrations/fivetran/fivetran",
+        "children": [
+          {
+            "title": "Fivetran (Legacy)",
+            "path": "/integrations/fivetran/fivetran-legacy"
+          }
+        ]
       },
       {
         "title": "Google BigQuery",

diff --git a/docs/content/api/modules.json.gz b/docs/content/api/modules.json.gz
diff --git a/docs/content/api/searchindex.json.gz b/docs/content/api/searchindex.json.gz
diff --git a/docs/content/api/sections.json.gz b/docs/content/api/sections.json.gz
diff --git a/docs/content/concepts/assets/asset-checks.mdx b/docs/content/concepts/assets/asset-checks.mdx
@@ -28,7 +28,7 @@ height={802}
 
 Using asset checks helps you:
 
-- Create a single place in the Dagstser UI to learn about and investigate issues in your data pipeline
+- Create a single place in the Dagster UI to learn about and investigate issues in your data pipeline
 - Ensure that data produced by your pipeline is timely and high quality by applying basic suites of tests to all assets
 - Identify problems in the source data that your pipeline depends on, such as checking data quality on source assets where dbt tests aren't possible
 - Communicate what the data produced by your pipeline is expected to look like (aka act as a data contract)
@@ -89,7 +89,7 @@ The following table lists Dagster's built-in utility methods for creating asset
 | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
 | <PyObject object="build_metadata_bounds_checks" />          | Builds asset checks that pass if numeric metadata value falls within a particular range                   |
 | <PyObject object="build_column_schema_change_checks" />     | Builds asset checks that pass if an asset's columns are the same, compared with its prior materialization |
-| <PyObject object="build_last_update_freshness_checks" />    | Builds asset checks that pass if not too much time has elapsed since the latest time an asset was updated |
+| <PyObject object="build_last_update_freshness_checks" />    | Builds asset checks that pass if they were updated within a given threshold                               |
 | <PyObject object="build_time_partition_freshness_checks" /> | Builds asset checks that pass if an asset's most recent partition has been materialized before a deadline |
 
 ### Limitations

diff --git a/docs/content/concepts/metadata-tags/kind-tags.mdx b/docs/content/concepts/metadata-tags/kind-tags.mdx
@@ -137,6 +137,7 @@ Some kinds are given a branded icon in the UI. We currently support nearly 200 u
 | `huggingface`       | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-huggingface-color.svg" width={20} height={20} />       |
 | `huggingfaceapi`    | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-huggingface-color.svg" width={20} height={20} />       |
 | `iceberg`           | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-iceberg-color.svg" width={20} height={20} />           |
+| `icechunk`          | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-icechunk-color.svg" width={20} height={20} />          |
 | `impala`            | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-impala-color.svg" width={20} height={20} />            |
 | `instagram`         | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-instagram-color.svg" width={20} height={20} />         |
 | `ipynb`             | <Image src="/images/concepts/metadata-tags/kinds/icons/tool-jupyter-color.svg" width={20} height={20} />           |

diff --git a/docs/content/integrations/airlift.mdx b/docs/content/integrations/airlift.mdx
@@ -54,6 +54,12 @@ In this tutorial, we'll use `dagster-airlift` to observe DAGs from multiple Airf
 
 [Click here to get started](/integrations/airlift/federation-tutorial/overview).
 
+## Airlift Operator Migration Reference
+
+In this reference, we'll explain how to migrate common Airflow operators to Dagster.
+
+[Click here to get started](/integrations/airlift/operator-migration/overview).
+
 ## References
 
 <ArticleList>
@@ -69,4 +75,8 @@ In this tutorial, we'll use `dagster-airlift` to observe DAGs from multiple Airf
     title="Additional Airlift Functionality"
     href="/integrations/airlift/reference"
   ></ArticleListItem>
+  <ArticleListItem
+    title="Migrating Airflow 1 to Dagster"
+    href="/integrations/airlift/airflow-1-migration"
+  ></ArticleListItem>
 </ArticleList>