[docs-beta] Merge the new beta Docs site (#23748)

## Summary & Motivation

This adds the new beta docs site into our main repo. This is still a
work in progress and does not affect the deployment of our existing docs. This
work was originally undertaken in the #23530 branch, however, I have
brought it in as a squashed self-contained PR to keep the history clean.

## How I Tested These Changes

Built locally, BK, and Vercel.

---------

Co-authored-by: colton <[email protected]>

.github/workflows/build-docs-revamp.yml

@@ -0,0 +1,45 @@
+name: Deploy Docs Revamp
+on:
+  pull_request:
+    paths:
+      - docs/docs-beta
+      - .github/workflows/build-docs-revamp.yml
+  push:
+    branches:
+      - docs/revamp
+    paths:
+      - docs/docs-beta
+      - .github/workflows/build-docs-revamp.yml
+
+concurrency:
+  group: ${{ github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout docs/revamp branch
+        uses: actions/checkout@v4
+
+      - name: Publish Preview to Vercel
+        uses: amondnet/vercel-action@v25
+        with:
+          github-comment: ${{ true }}
+          vercel-token: ${{ secrets.VERCEL_TOKEN }}
+          vercel-org-id: ${{ secrets.VERCEL_ORG_ID }}
+          vercel-project-id: ${{ secrets.VERCEL_DOCS_NEXT_PROJECT_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          scope: ${{ secrets.VERCEL_ORG_ID }}
+
+      - name: Publish to Vercel Production
+        uses: amondnet/vercel-action@v25
+        if: github.event_name == 'push' && github.ref == 'refs/heads/docs/revamp'
+        with:
+          vercel-token: ${{ secrets.VERCEL_TOKEN }}
+          vercel-org-id: ${{ secrets.VERCEL_ORG_ID }}
+          vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }}
+          vercel-args: "--prod"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          scope: ${{ secrets.VERCEL_ORG_ID }}
+          alias-domains: dagster-docs-beta.dagster.dagster-docs.io

.github/workflows/build-docs.yml

@@ -18,7 +18,7 @@ on:
       - .github/workflows/build-docs.yml
 concurrency:
   # Cancel in-progress runs on same branch
-  group: ${{ github.ref }}
+  group: ${{ github.workflow}}-${{github.ref}}
   cancel-in-progress: true
 jobs:
   deploy:

.github/workflows/vale.yml

@@ -0,0 +1,50 @@
+name: Vale Docs
+on: 
+  pull_request:
+    paths:
+      - 'docs/**'
+      - .github/workflows/vale.yml
+  push:
+    branches:
+      - master
+      - docs-prod
+    paths:
+      - 'docs/**'
+      - .github/workflows/vale.yml
+
+concurrency:
+  group: ${{ github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  vale:
+    name: runner / vale
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: errata-ai/vale-action@reviewdog
+        env:
+          REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          files: '["docs-beta/docs"]'
+          vale_flags: "--config=docs/.vale.ini"
+          fail_on_error: true
+      - uses: errata-ai/vale-action@reviewdog
+        env:
+          REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          files: '["docs/content", "docs/sphinx"]'
+          vale_flags: "--config=docs/.vale.ini"
+          fail_on_error: true
+          reporter: github-pr-review
+      - uses: errata-ai/vale-action@reviewdog
+        env:
+          REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          files: '["docs/content", "docs/sphinx"]'
+          vale_flags: "--config=docs/.vale.ini"
+          fail_on_error: true
+          reporter: github-check

docs/.gitignore

@@ -1,3 +1,6 @@
 sphinx/_build/
 .env
 next/.env
+docs-beta/.docusaurus
+*.duckdb
+*.sqlite

docs/.vale.ini

@@ -0,0 +1,28 @@
+StylesPath = "vale/styles"
+MinAlertLevel = suggestion
+
+Packages = Google 
+Vocab = Dagster
+
+[*.{md,mdx,rst}]
+BasedOnStyles = Vale, Google, Dagster
+
+; Ignore all :py directives
+IgnorePatterns = (:py:[^`]+`[^`]+`)
+
+; Error on headings that aren't sentence cased.
+Google.Headings = Error
+
+; Google avoids using will, we don't need to check for this
+Google.Will = NO
+Google.WordList = NO
+
+; Exclamation is fine
+Google.Exclamation = NO
+
+; Passive is a bit too noisy
+Google.Passive = NO
+
+
+[formats]
+mdx = md

docs/docs-beta/.eslintrc.js

@@ -0,0 +1,28 @@
+module.exports = {
+  parser: '@babel/eslint-parser',
+  extends: ['plugin:react/recommended', 'plugin:@docusaurus/recommended', 'plugin:mdx/recommended'],
+  rules: {
+    'import/no-default-export': 'off',
+    'react/react-in-jsx-scope': 'off',
+  },
+  ignorePatterns: ['build'],
+  parserOptions: {
+    extensions: ['.md', '.mdx'], // Resolve .md files as .mdx
+    markdownExtensions: ['.md', '.markdown'], // Treat .md and .markdown files as plain markdown
+  },
+  overrides: [
+    {
+      files: ['*.md', '*.mdx'],
+      extends: 'plugin:mdx/recommended',
+      rules: {
+        'react/jsx-no-undef': 'off',
+        'react/no-unescaped-entities': 'off',
+      },
+    },
+  ],
+  settings: {
+    react: {
+      version: 'detect',
+    },
+  },
+};

docs/docs-beta/.gitignore

@@ -0,0 +1,21 @@
+# Dependencies
+/node_modules
+
+# Production
+/build
+
+# Generated files
+.docusaurus
+.cache-loader
+
+# Misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+*.sqlite

docs/docs-beta/.prettierignore

@@ -0,0 +1,4 @@
+node_modules
+.docusaurus
+build/
+docs/

docs/docs-beta/.prettierrc.js

@@ -0,0 +1,6 @@
+module.exports = {
+  bracketSpacing: false,
+  printWidth: 100,
+  singleQuote: true,
+  trailingComma: 'all',
+};

docs/docs-beta/.remarkrc.js

@@ -0,0 +1,5 @@
+module.exports = {
+  plugins: [
+    'remark-frontmatter',
+  ]
+}

docs/docs-beta/.vscode/extensions.json

@@ -0,0 +1,9 @@
+{
+    "recommendations": [
+        "dbaeumer.vscode-eslint",
+        "unifiedjs.vscode-mdx",
+        "esbenp.prettier-vscode",
+        "mrmlnc.vscode-remark",
+        "chrischinchilla.vale-vscode"
+    ]
+}

docs/docs-beta/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "editor.defaultFormatter": "esbenp.prettier-vscode"
+}

docs/docs-beta/README.md

@@ -0,0 +1,66 @@
+# Dagster Docs - Beta
+
+This is the home of the new Dagster documentation. It is currently in beta and incomplete.
+The documentation site is built using [Docusaurus](https://docusaurus.io/), a modern static website generator.
+
+### Installation
+
+The site uses [pnpm](https://pnpm.io/) for package management. 
+It also uses [vale](https://vale.sh/) to check for issues in the documentation.
+
+Install dependencies with:
+
+```bash
+brew install pnpm vale
+pnpm install
+```
+
+### Overview of the docs
+
+Code in `./src` contains custom components, styles, themes, and layouts.
+Code `./content-templates` contains the templates for the documentation pages.
+Code in `./docs/` is the source of truth for the documentation.
+
+`./docs/code_examples` contains all code examples for the documentation. 
+
+The docs are broken down into the following sections:
+- [Tutorials](./docs/tutorials/)
+- [Guides](./docs/guides/)
+- [Concepts](./docs/concepts/)
+
+`sidebar.ts` and `docusaurus.config.ts` are the main configuration files for the documentation.
+
+### Local Development
+
+To start the local development server:
+
+```bash
+pnpm start
+```
+
+This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. Access the website at [http://localhost:3050](http://localhost:3050).
+
+
+To lint the documentation for issues:
+
+```bash
+pnpm lint
+```
+
+To autofix linting issues and format with prettier:
+
+```bash
+pnpm lint:fix
+```
+
+
+
+### Build
+
+To build the site for production:
+
+```bash
+pnpm build
+```
+
+This command generates static content into the `build` directory and can be served using any static contents hosting service.

docs/docs-beta/babel.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  presets: [require.resolve('@docusaurus/core/lib/babel/preset')],
+};

docs/docs-beta/content-templates/concept.md

@@ -0,0 +1,96 @@
+---
+title: ''
+description: ''
+---
+
+# [TOPIC]
+
+<!-- This section is an intro that includes:
+
+- A brief description of what the topic is,
+- An example of how it could be used in the real-world
+- What it can do in the UI
+ -->
+
+---
+
+## Benefits
+
+<!-- This section lists the benefits of using the topic, whatever it is. The items listed here should be solutions to real-world problems that the user cares about, ex:
+
+Using schedules helps you:
+
+- Predictably process and deliver data to stakeholders and business-critical applications
+- Consistently run data pipelines without the need for manual intervention
+- Optimize resource usage by scheduling pipelines to run during off-peak hours
+-->
+
+Using [TOPIC] helps you:
+
+<!-- - A benefit of the thing
+- Another benefit
+- And one more -->
+
+---
+
+## Prerequisites
+
+<!-- This section lists the prerequisites users must complete before they should/can proceed. For concepts, we should list the other concepts they should be familiar with first. -->
+
+Before continuing, you should be familiar with:
+
+<!-- - Ex: To use asset checks, users should understand Asset definitions first
+- Another one
+- One more -->
+
+---
+
+## How it works
+
+<!-- This section provides a high-level overview of how the concept works without getting too into the technical details. Code can be shown here, but this section shouldn't focus on it. The goal is to help the user generally understand how the thing works and what they need to do to get it working without overwhelming them with details.
+
+For example, this is the How it works for Schedules:
+
+Schedules run jobs at fixed time intervals and have two main components:
+
+- A job, which targets a selection of assets or ops
+
+- A cron expression, which defines when the schedule runs. Simple and complex schedules are supported, allowing you to have fine-grained control over when runs are executed. With cron syntax, you can:
+
+   - Create custom schedules like Every hour from 9:00AM - 5:00PM with cron expressions (0 9-17 * * *)
+   - Quickly create basic schedules like Every day at midnight with predefined cron definitions (@daily, @midnight)
+
+   To make creating cron expressions easier, you can use an online tool like Crontab Guru. This tool allows you to create and describe cron expressions in a human-readable format and test the execution dates produced by the expression. Note: While this tool is useful for general cron expression testing, always remember to test your schedules in Dagster to ensure the results are as expected.
+
+For a schedule to run, it must be turned on and an active dagster-daemon process must be running. If you used dagster dev to start the Dagster UI/webserver, the daemon process will be automatically launched alongside the webserver.
+
+After these criteria are met, the schedule will run at the interval specified in the cron expression. Schedules will execute in UTC by default, but you can specify a custom timezone.
+
+-->
+
+---
+
+## Getting started
+
+<!-- This section is a list of guides / links to pages to help the user get started using the topic. -->
+
+Check out these guides to get started with [CONCEPT]:
+
+From here, you can:
+
+<!-- A list of things the user can do once they've got the basics down. These could be links to additional guides, ex:
+
+- Construct schedules to run partitioned jobs
+- Execute jobs in specific timezones
+- Learn to test your schedules
+- Identify and resolve common issues with our troubleshooting guide -->
+
+### Limitations [and notes]
+
+<!-- This section should describe any known limitations that could impact the user, ex: "Schedules will execute in UTC unless a timezone is specified" -->
+
+---
+
+## Related
+
+<!-- A list of related links and resources -->