[docs-beta] Add a Sphinx MDX builder (#24235)

This adds a Sphinx MDX Builder extension to generate MDX files from RST. It made some modifications to our existing Sphinx configuration that were causing errors. Namely, the dagster extension was raising Exceptions during the build for non-public APIs, however this seems to only run on non-json builds. This is changed to only log the output for
now.

Also fixed a bug where methods were being flagged as missing docs when they were not. Essentially, a class dict returns none when __doc__ is called, eg. with obj.__doc__. Instead, you can assert that docs exist with hasattr. This was causing PipesContext and other classes to report no documentation erroneously.

## Summary & Motivation

To populate the new docs site with API docs

## How I Tested These Changes

local, bk
This should not affect the existing docs site, will use BK to validate.

## Changelog [New | Bug | Docs]

NOCHANGELOG

docs/docs-beta/docs/api/.gitignore

@@ -0,0 +1 @@
+**/*.mdx

docs/docs-beta/docs/api/placeholder.md

@@ -0,0 +1 @@
+## Placeholder

docs/docs-beta/docusaurus.config.ts

@@ -1,5 +1,5 @@
-import {themes as prismThemes} from 'prism-react-renderer';
-import type {Config} from '@docusaurus/types';
+import { themes as prismThemes } from 'prism-react-renderer';
+import type { Config } from '@docusaurus/types';
 import type * as Preset from '@docusaurus/preset-classic';
 
 const config: Config = {

docs/docs-beta/sidebars.ts

@@ -1,4 +1,4 @@
-import type { SidebarsConfig } from '@docusaurus/plugin-content-docs';
+import type {SidebarsConfig} from '@docusaurus/plugin-content-docs';
 
 /**
  * Creating a sidebar enables you to:
@@ -478,6 +478,18 @@ const sidebars: SidebarsConfig = {
       ],
     },
   ],
+  api: [
+    {
+      type: 'category',
+      label: 'Dagster API',
+      items: [
+        {
+          type: 'autogenerated',
+          dirName: 'api',
+        },
+      ],
+    },
+  ],
 };
 
 export default sidebars;

docs/docs-beta/src/styles/custom.scss

@@ -504,3 +504,40 @@ table {
     transform: rotate(0deg);
   }
 }
+
+/* API Docs */
+dl {
+  padding: 4px 0px 0px 4px;
+  border: 1px solid var(--theme-color-keyline);
+  font-weight: 200;
+  background-color: var(--theme-color-background-blue);
+  line-height: 1.2;
+  font-size: 13px;
+  border-radius: 4px;
+}
+
+dt {
+  box-shadow: 0px 1px 0px var(--theme-color-keyline);
+  font-weight: 600;
+  font-size: 15px;
+  padding-bottom: 0px;
+}
+
+dd {
+  background-color: var(--theme-color-background-light);
+  font-weight: 400;
+  padding: 4px;
+  margin-left: -2px;
+  line-height: 1.4;
+}
+
+dd p {
+  margin: 0;
+}
+
+dd code {
+  background-color: var(--theme-color-background-default);
+  border: 1px solid var(--theme-color-keyline);
+  border-radius: 4px;
+  padding: 0.1rem;
+}

docs/sphinx/Makefile

@@ -1,134 +1,24 @@
-
-# Makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build -j auto -q
-PAPER         =
+SPHINXOPTS    ?= -j auto
+SPHINXBUILD   ?= sphinx-build 
+SOURCEDIR     = .
 BUILDDIR      = _build
 
-# Internal variables.
-PAPEROPT_a4     = -D latex_paper_size=a4
-PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gatsby
-
+# Put it first so that "make" without argument is like "make help".
 help:
-	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  html       to make standalone HTML files"
-	@echo "  dirhtml    to make HTML files named index.html in directories"
-	@echo "  singlehtml to make a single large HTML file"
-	@echo "  pickle     to make pickle files"
-	@echo "  json       to make JSON files"
-	@echo "  htmlhelp   to make HTML files and a HTML help project"
-	@echo "  qthelp     to make HTML files and a qthelp project"
-	@echo "  devhelp    to make HTML files and a Devhelp project"
-	@echo "  epub       to make an epub"
-	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
-	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
-	@echo "  text       to make text files"
-	@echo "  man        to make manual pages"
-	@echo "  changes    to make an overview of all changed/added/deprecated items"
-	@echo "  linkcheck  to check all external links for integrity"
-	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
-
-clean:
-	-rm -rf $(BUILDDIR)/*
-
-html:
-	$(SPHINXBUILD) -a -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
-
-dirhtml:
-	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-singlehtml:
-	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
-	@echo
-	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
-
-pickle:
-	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
-	@echo
-	@echo "Build finished; now you can process the pickle files."
-
-json:
-	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
-	@echo
-	@echo "Build finished; now you can process the JSON files."
-
-htmlhelp:
-	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
-	@echo
-	@echo "Build finished; now you can run HTML Help Workshop with the" \
-	      ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp:
-	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
-	@echo
-	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
-	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
-	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Mapnik.qhcp"
-	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Mapnik.qhc"
-
-devhelp:
-	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
-	@echo
-	@echo "Build finished."
-	@echo "To view the help file:"
-	@echo "# mkdir -p $$HOME/.local/share/devhelp/Mapnik"
-	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Mapnik"
-	@echo "# devhelp"
-
-epub:
-	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
-	@echo
-	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
-
-latex:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo
-	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
-	@echo "Run \`make' in that directory to run these through (pdf)latex" \
-	      "(use \`make latexpdf' here to do that automatically)."
-
-latexpdf:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo "Running LaTeX files through pdflatex..."
-	make -C $(BUILDDIR)/latex all-pdf
-	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-text:
-	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
-	@echo
-	@echo "Build finished. The text files are in $(BUILDDIR)/text."
-
-man:
-	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
-	@echo
-	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
-
-changes:
-	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
-	@echo
-	@echo "The overview file is in $(BUILDDIR)/changes."
-
-linkcheck:
-	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
-	@echo
-	@echo "Link check complete; look for any errors in the above output " \
-	      "or in $(BUILDDIR)/linkcheck/output.txt"
-
-doctest:
-	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
-	@echo "Testing of doctests in the sources finished, look at the " \
-	      "results in $(BUILDDIR)/doctest/output.txt."
-
-livehtml:
-	sphinx-autobuild -a -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile install
+
+install:
+	pip install uv
+	uv pip install -r requirements.txt
+	uv pip install -e _ext/dagster-sphinx
+
+copy_mdx:
+	rm -rf ../docs-beta/docs/api/*.mdx
+	cp -rf _build/mdx/index.mdx ../docs-beta/docs/api/
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/sphinx/_ext/dagster-sphinx/dagster_sphinx/__init__.py

@@ -73,9 +73,9 @@ def record_error(env: BuildEnvironment, message: str) -> None:
 
 
 def check_public_method_has_docstring(env: BuildEnvironment, name: str, obj: object) -> None:
-    if name != "__init__" and not obj.__doc__:
+    if name != "__init__" and not hasattr(obj, "__doc__"):
         message = (
-            f"Docstring not found for {object.__name__}.{name}. "
+            f"Docstring not found for {obj!r}.{name}. "
             "All public methods and properties must have docstrings."
         )
         record_error(env, message)
@@ -155,7 +155,7 @@ def check_custom_errors(app: Sphinx, exc: Optional[Exception] = None) -> None:
     if len(dagster_errors) > 0:
         for error_msg in dagster_errors:
             logger.info(error_msg)
-        raise Exception(
+        logger.error(
             f"Bulid failed. Found {len(dagster_errors)} violations of docstring requirements."
         )
 
@@ -175,6 +175,6 @@ def setup(app):
 
     return {
         "version": "0.1",
-        "parallel_read_safe": True,
+        "parallel_read_safe": False,
         "parallel_write_safe": True,
     }

docs/sphinx/_ext/sphinx-mdx-builder/Makefile

@@ -0,0 +1,14 @@
+install:
+	pip install uv
+	uv pip install -e .
+
+install_dev: install
+	uv pip install -e .[test]
+
+lint:
+	ruff format .
+	ruff check --fix 
+
+lint_check:
+	ruff check
+	ruff format --check 

docs/sphinx/_ext/sphinx-mdx-builder/README.md

@@ -0,0 +1,2 @@
+# sphinx-mdx-builder
+Generate MDX files from Sphinx

docs/sphinx/_ext/sphinx-mdx-builder/pyproject.toml

@@ -0,0 +1,31 @@
+[project]
+name = "sphinxcontrib-mdxbuilder"
+description = "Sphinx extension tobuild MDX files"
+authors = [
+    {name = "Pedram Navid", email = "[email protected]"}
+]
+requires-python = ">=3.10"
+readme = "README.md"
+
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Framework :: Sphinx",
+    "Framework :: Sphinx :: Extension",
+]
+version = "0.1.3"
+
+dependencies = [
+    "sphinx>=7.0",
+]
+
+[project.scripts]
+sphinx-builder-mdx = "sphinxcontrib.mdxbuilder.__main__:main"
+
+[project.optional-dependencies]
+test = ["pytest", "tox", "tox-uv", "ruff"]

docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/__init__.py

@@ -0,0 +1,30 @@
+import importlib.metadata
+
+__version__ = importlib.metadata.version(__package__ or __name__)
+from sphinx.application import Sphinx
+
+
+def setup(app: Sphinx):
+    from sphinxcontrib.mdxbuilder.builders.mdx import MdxBuilder
+
+    app.add_builder(MdxBuilder)
+
+    # File suffix for generated files
+    app.add_config_value("mdx_file_suffix", ".mdx", "env")
+    # Suffix for internal links, blank by default, e.g. '/path/to/file'
+    # Add .mdx to get '/path/to/file.mdx'
+    app.add_config_value("mdx_link_suffix", None, "env")
+
+    # File transform function for filenames, by default returns docname + mdx_file_suffix
+    app.add_config_value("mdx_file_transform", None, "env")
+
+    # Link transform function for links, by default returns docname + mdx_link_suffix
+    app.add_config_value("mdx_link_transform", None, "env")
+
+    app.add_config_value("mdx_max_line_width", 120, "env")
+
+    return {
+        "version": __version__,
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }