diff --git a/docs/Makefile b/docs/Makefile index df4e362ade6bb..d1b90a2520ca8 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -27,3 +27,10 @@ mdx-format: ## Format mdx files cd next; yarn mdx-format mdx-full-format: docs_ruff mdx-format + + +mdx: + tox -e sphinx-mdx + +mdx_copy: + cp -rf sphinx/_build/mdx/sections/api/apidocs/* docs-beta/docs/api/ diff --git a/docs/docs-beta/README.md b/docs/docs-beta/README.md index d12c9cf4d2a91..ed06fe5a341de 100644 --- a/docs/docs-beta/README.md +++ b/docs/docs-beta/README.md @@ -117,3 +117,17 @@ yarn build ``` This command generates static content into the `build` directory and can be served using any static contents hosting service. This also checks for any broken links in the documentation. + +## Deployment + +This site is built and deployed using Vercel. + +### API Documentation + +API documentation is built in Vercel by overriding the _Build Command_ to the following: + +```sh +yarn sync-api-docs && yarn build +``` + +This runs the `scripts/vercel-sync-api-docs.sh` script which builds the MDX files using the custom `sphinx-mdx-builder`, and copies the resulting MDX files to `docs/api`. diff --git a/docs/docs-beta/docusaurus.config.ts b/docs/docs-beta/docusaurus.config.ts index 9eaeb0839895d..dc00cb16a2961 100644 --- a/docs/docs-beta/docusaurus.config.ts +++ b/docs/docs-beta/docusaurus.config.ts @@ -12,7 +12,8 @@ const config: Config = { baseUrl: '/', onBrokenLinks: 'throw', onBrokenMarkdownLinks: 'throw', - onBrokenAnchors: 'throw', + // TODO - reintroduce when API doc anchors are fixed + onBrokenAnchors: 'warn', organizationName: 'dagster', projectName: 'dagster', markdown: { diff --git a/docs/docs-beta/package.json b/docs/docs-beta/package.json index 9677bdc17b235..bb6b5db9cd7e1 100644 --- a/docs/docs-beta/package.json +++ b/docs/docs-beta/package.json @@ -15,7 +15,8 @@ "ts": "tsc -p . --noEmit", "vale": "vale ./docs --ext=.md,.mdx", "lint": "eslint . --ext=.tsx,.ts,.js,.md,.mdx --fix", - "lint-and-vale": "yarn run lint && yarn run vale" + "lint-and-vale": "yarn run lint && yarn run vale", + "sync-api-docs": "/bin/sh scripts/vercel-sync-api-docs.sh" }, "dependencies": { "@docusaurus/core": "3.5.2", diff --git a/docs/docs-beta/scripts/vercel-sync-api-docs.sh b/docs/docs-beta/scripts/vercel-sync-api-docs.sh new file mode 100644 index 0000000000000..bab8fd8652ca0 --- /dev/null +++ b/docs/docs-beta/scripts/vercel-sync-api-docs.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# **NOTE: this script is intended to by used from Vercel only!** +# +# Description: Builds and synchronizes MDX API docs from Sphinx +# Usage: yarn sync-api-docs +# + +set -e + +cd .. + +curl -LsSf https://astral.sh/uv/install.sh | sh +source ~/.cargo/env + +uv venv +source .venv/bin/activate + +# Required as is locale is not set by default in Vercel runner +export LC_ALL=C.UTF-8 + +uv pip install tox +make mdx +make mdx_copy diff --git a/docs/sphinx/Makefile b/docs/sphinx/Makefile index f0b30f4b53105..5ecbc50834d48 100644 --- a/docs/sphinx/Makefile +++ b/docs/sphinx/Makefile @@ -16,7 +16,7 @@ install: uv pip install -e _ext/sphinx-mdx-builder mdx_copy: - rm -r ../docs-beta/docs/api/libraries + rm -rf ../docs-beta/docs/api/libraries find ../docs-beta/docs/api/*.mdx ! -name index.mdx -exec rm {} + cp -rf _build/mdx/sections/api/apidocs/* ../docs-beta/docs/api/ diff --git a/docs/sphinx/requirements.txt b/docs/sphinx/requirements.txt index 081e1bcf65cfb..0f89e4ab4548f 100644 --- a/docs/sphinx/requirements.txt +++ b/docs/sphinx/requirements.txt @@ -1,5 +1,3 @@ sphinx>=8,<9 sphinx-click sphinx_toolbox - sphinxcontrib-serializinghtml - sphinxcontrib-mdxbuilder \ No newline at end of file diff --git a/docs/tox.ini b/docs/tox.ini index 689cf89450406..03421c2508226 100644 --- a/docs/tox.ini +++ b/docs/tox.ini @@ -9,6 +9,7 @@ passenv = AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID BUILDKITE* + LC_ALL allowlist_externals = make uv @@ -18,6 +19,7 @@ install_command = uv pip install {opts} {packages} deps = -r sphinx/requirements.txt -e sphinx/_ext/dagster-sphinx + -e sphinx/_ext/sphinx-mdx-builder # Can't stub deps because processed by sphinx-click -e ../python_modules/dagster @@ -47,6 +49,40 @@ commands = make --directory=sphinx clean make --directory=sphinx json SPHINXOPTS="-W --keep-going" +[testenv:sphinx-mdx] +deps = + -r sphinx/requirements.txt + -e sphinx/_ext/dagster-sphinx + -e sphinx/_ext/sphinx-mdx-builder + + # Can't stub deps because processed by sphinx-click + -e ../python_modules/dagster + -e ../python_modules/dagster-pipes + -e ../python_modules/dagster-graphql + -e ../python_modules/dagster-webserver + -e ../python_modules/libraries/dagster-celery + + # Can't stub deps due to import-time use of at least one dep + -e ../python_modules/libraries/dagstermill + -e ../python_modules/libraries/dagster-aws + -e ../python_modules/libraries/dagster-datahub + -e ../python_modules/libraries/dagster-gcp + -e ../python_modules/libraries/dagster-pyspark + -e ../python_modules/libraries/dagster-ssh + -e ../python_modules/libraries/dagster-duckdb + -e ../python_modules/libraries/dagster-dbt + -e ../python_modules/libraries/dagster-wandb + -e ../python_modules/libraries/dagster-embedded-elt + -e ../python_modules/libraries/dagster-deltalake + -e ../python_modules/libraries/dagster-deltalake-pandas + -e ../python_modules/libraries/dagster-deltalake-polars + -e ../python_modules/libraries/dagster-openai + -e ../python_modules/libraries/dagster-looker + +commands = + make --directory=sphinx clean + make --directory=sphinx mdx + [testenv:audit-screenshots] deps = -e ./dagster-ui-screenshot