[beta-docs] vercel build and sync api docs (#24362)

## Summary & Motivation

- Build API docs in Vercel
- Failure on broken anchor links was set to warn, as api docs have many
broken links still, will enable when prod ready
- Introduced new `yarn` directive to build and sync files
- Introduced new tox environment `sphinx-mdx` for building mdx files
separate from json
- Updated build in Vercel to use new directive

## Random screenshots for context

<img width="745" alt="image"
src="https://github.com/user-attachments/assets/3c073828-8521-482f-80af-c5adbafa7f8a">

## How I Tested These Changes

## Changelog

NOCHANGELOG

docs/Makefile

@@ -27,3 +27,10 @@ mdx-format:  ## Format mdx files
 	cd next; yarn mdx-format
 
 mdx-full-format: docs_ruff mdx-format
+
+
+mdx:
+	tox -e sphinx-mdx
+
+mdx_copy:
+	cp -rf sphinx/_build/mdx/sections/api/apidocs/* docs-beta/docs/api/

docs/docs-beta/README.md

@@ -117,3 +117,17 @@ yarn build
 ```
 
 This command generates static content into the `build` directory and can be served using any static contents hosting service. This also checks for any broken links in the documentation.
+
+## Deployment
+
+This site is built and deployed using Vercel.
+
+### API Documentation
+
+API documentation is built in Vercel by overriding the _Build Command_ to the following:
+
+```sh
+yarn sync-api-docs && yarn build
+```
+
+This runs the `scripts/vercel-sync-api-docs.sh` script which builds the MDX files using the custom `sphinx-mdx-builder`, and copies the resulting MDX files to `docs/api`.

docs/docs-beta/docusaurus.config.ts

@@ -12,7 +12,8 @@ const config: Config = {
   baseUrl: '/',
   onBrokenLinks: 'throw',
   onBrokenMarkdownLinks: 'throw',
-  onBrokenAnchors: 'throw',
+  // TODO - reintroduce when API doc anchors are fixed
+  onBrokenAnchors: 'warn',
   organizationName: 'dagster',
   projectName: 'dagster',
   markdown: {

docs/docs-beta/package.json

@@ -15,7 +15,8 @@
     "ts": "tsc -p . --noEmit",
     "vale": "vale ./docs --ext=.md,.mdx",
     "lint": "eslint . --ext=.tsx,.ts,.js,.md,.mdx --fix",
-    "lint-and-vale": "yarn run lint && yarn run vale"
+    "lint-and-vale": "yarn run lint && yarn run vale",
+    "sync-api-docs": "/bin/sh scripts/vercel-sync-api-docs.sh"
   },
   "dependencies": {
     "@docusaurus/core": "3.5.2",

docs/docs-beta/scripts/vercel-sync-api-docs.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# **NOTE: this script is intended to by used from Vercel only!**
+#
+# Description: Builds and synchronizes MDX API docs from Sphinx
+# Usage: yarn sync-api-docs
+#
+
+set -e
+
+cd ..
+
+curl -LsSf https://astral.sh/uv/install.sh | sh
+source ~/.cargo/env
+
+uv venv
+source .venv/bin/activate
+
+# Required as is locale is not set by default in Vercel runner
+export LC_ALL=C.UTF-8
+
+uv pip install tox
+make mdx
+make mdx_copy

docs/sphinx/Makefile

@@ -16,7 +16,7 @@ install:
 	uv pip install -e _ext/sphinx-mdx-builder
 
 mdx_copy:
-	rm -r ../docs-beta/docs/api/libraries
+	rm -rf ../docs-beta/docs/api/libraries
 	find ../docs-beta/docs/api/*.mdx ! -name index.mdx -exec rm {} +
 	cp -rf _build/mdx/sections/api/apidocs/* ../docs-beta/docs/api/
 

docs/sphinx/requirements.txt

@@ -1,5 +1,3 @@
   sphinx>=8,<9
   sphinx-click
   sphinx_toolbox
-  sphinxcontrib-serializinghtml
-  sphinxcontrib-mdxbuilder

docs/tox.ini

@@ -9,6 +9,7 @@ passenv =
     AWS_SECRET_ACCESS_KEY
     AWS_ACCESS_KEY_ID
     BUILDKITE*
+    LC_ALL
 allowlist_externals =
   make
   uv
@@ -18,6 +19,7 @@ install_command = uv pip install {opts} {packages}
 deps =
   -r sphinx/requirements.txt
   -e sphinx/_ext/dagster-sphinx
+  -e sphinx/_ext/sphinx-mdx-builder
 
   # Can't stub deps because processed by sphinx-click
   -e ../python_modules/dagster
@@ -47,6 +49,40 @@ commands =
   make --directory=sphinx clean
   make --directory=sphinx json SPHINXOPTS="-W --keep-going"
 
+[testenv:sphinx-mdx]
+deps =
+  -r sphinx/requirements.txt
+  -e sphinx/_ext/dagster-sphinx
+  -e sphinx/_ext/sphinx-mdx-builder
+
+  # Can't stub deps because processed by sphinx-click
+  -e ../python_modules/dagster
+  -e ../python_modules/dagster-pipes
+  -e ../python_modules/dagster-graphql
+  -e ../python_modules/dagster-webserver
+  -e ../python_modules/libraries/dagster-celery
+
+  # Can't stub deps due to import-time use of at least one dep
+  -e ../python_modules/libraries/dagstermill
+  -e ../python_modules/libraries/dagster-aws
+  -e ../python_modules/libraries/dagster-datahub
+  -e ../python_modules/libraries/dagster-gcp
+  -e ../python_modules/libraries/dagster-pyspark
+  -e ../python_modules/libraries/dagster-ssh
+  -e ../python_modules/libraries/dagster-duckdb
+  -e ../python_modules/libraries/dagster-dbt
+  -e ../python_modules/libraries/dagster-wandb
+  -e ../python_modules/libraries/dagster-embedded-elt
+  -e ../python_modules/libraries/dagster-deltalake
+  -e ../python_modules/libraries/dagster-deltalake-pandas
+  -e ../python_modules/libraries/dagster-deltalake-polars
+  -e ../python_modules/libraries/dagster-openai
+  -e ../python_modules/libraries/dagster-looker
+
+commands =
+  make --directory=sphinx clean
+  make --directory=sphinx mdx
+
 [testenv:audit-screenshots]
 deps =
   -e ./dagster-ui-screenshot